Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(ingest): cleanup importlib.import_module calls #5490

Merged
merged 1 commit into from
Jul 26, 2022
Merged

refactor(ingest): cleanup importlib.import_module calls #5490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions metadata-ingestion/src/datahub/ingestion/source/kafka.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import logging
import types
from dataclasses import dataclass, field
from importlib import import_module
from typing import Dict, Iterable, List, Optional, Type, cast

import confluent_kafka
Expand All @@ -26,6 +24,7 @@
platform_name,
support_status,
)
from datahub.ingestion.api.registry import import_path
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase
from datahub.ingestion.source.state.checkpoint import Checkpoint
Expand Down Expand Up @@ -120,11 +119,7 @@ def create_schema_registry(
cls, config: KafkaSourceConfig, report: KafkaSourceReport
) -> KafkaSchemaRegistryBase:
try:
module_path: str
class_name: str
module_path, class_name = config.schema_registry_class.rsplit(".", 1)
module: types.ModuleType = import_module(module_path)
schema_registry_class: Type = getattr(module, class_name)
schema_registry_class: Type = import_path(config.schema_registry_class)
return schema_registry_class.create(config, report)
except (ImportError, AttributeError):
raise ImportError(config.schema_registry_class)
Expand Down
8 changes: 2 additions & 6 deletions metadata-ingestion/src/datahub/ingestion/source/lookml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import glob
import importlib
import itertools
import logging
import pathlib
Expand All @@ -25,6 +24,7 @@
platform_name,
support_status,
)
from datahub.ingestion.api.registry import import_path
from datahub.ingestion.source.looker_common import (
LookerCommonConfig,
LookerExplore,
Expand Down Expand Up @@ -515,14 +515,10 @@ class LookerView:
@classmethod
def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
assert "." in sql_parser_path, "sql_parser-path must contain a ."
module_name, cls_name = sql_parser_path.rsplit(".", 1)
import sys
parser_cls = import_path(sql_parser_path)

logger.debug(sys.path)
parser_cls = getattr(importlib.import_module(module_name), cls_name)
if not issubclass(parser_cls, SQLParser):
raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")

return parser_cls

@classmethod
Expand Down
7 changes: 3 additions & 4 deletions metadata-ingestion/src/datahub/ingestion/source/redash.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import importlib
import logging
import math
import sys
Expand All @@ -22,6 +21,7 @@
platform_name,
support_status,
)
from datahub.ingestion.api.registry import import_path
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.metadata.com.linkedin.pegasus2avro.common import (
Expand Down Expand Up @@ -378,11 +378,10 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
@classmethod
def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
assert "." in sql_parser_path, "sql_parser-path must contain a ."
module_name, cls_name = sql_parser_path.rsplit(".", 1)
parser_cls = getattr(importlib.import_module(module_name), cls_name)
parser_cls = import_path(sql_parser_path)

if not issubclass(parser_cls, SQLParser):
raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")

return parser_cls

@classmethod
Expand Down