refactor(ingest): cleanup importlib.import_module calls (#5490)

datahub-project · Jul 26, 2022 · c294e89 · c294e89
1 parent 97d508f
commit c294e89
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 17 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py
@@ -1,7 +1,5 @@
 import logging
-import types
 from dataclasses import dataclass, field
-from importlib import import_module
 from typing import Dict, Iterable, List, Optional, Type, cast
 
 import confluent_kafka
@@ -26,6 +24,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase
 from datahub.ingestion.source.state.checkpoint import Checkpoint
@@ -120,11 +119,7 @@ def create_schema_registry(
         cls, config: KafkaSourceConfig, report: KafkaSourceReport
     ) -> KafkaSchemaRegistryBase:
         try:
-            module_path: str
-            class_name: str
-            module_path, class_name = config.schema_registry_class.rsplit(".", 1)
-            module: types.ModuleType = import_module(module_path)
-            schema_registry_class: Type = getattr(module, class_name)
+            schema_registry_class: Type = import_path(config.schema_registry_class)
             return schema_registry_class.create(config, report)
         except (ImportError, AttributeError):
             raise ImportError(config.schema_registry_class)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/lookml.py b/metadata-ingestion/src/datahub/ingestion/source/lookml.py
@@ -1,5 +1,4 @@
 import glob
-import importlib
 import itertools
 import logging
 import pathlib
@@ -25,6 +24,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.source.looker_common import (
     LookerCommonConfig,
     LookerExplore,
@@ -515,14 +515,10 @@ class LookerView:
     @classmethod
     def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
         assert "." in sql_parser_path, "sql_parser-path must contain a ."
-        module_name, cls_name = sql_parser_path.rsplit(".", 1)
-        import sys
+        parser_cls = import_path(sql_parser_path)
 
-        logger.debug(sys.path)
-        parser_cls = getattr(importlib.import_module(module_name), cls_name)
         if not issubclass(parser_cls, SQLParser):
             raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")
-
         return parser_cls
 
     @classmethod

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py
@@ -1,4 +1,3 @@
-import importlib
 import logging
 import math
 import sys
@@ -22,6 +21,7 @@
     platform_name,
     support_status,
 )
+from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.api.source import Source, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.com.linkedin.pegasus2avro.common import (
@@ -378,11 +378,10 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source:
     @classmethod
     def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
         assert "." in sql_parser_path, "sql_parser-path must contain a ."
-        module_name, cls_name = sql_parser_path.rsplit(".", 1)
-        parser_cls = getattr(importlib.import_module(module_name), cls_name)
+        parser_cls = import_path(sql_parser_path)
+
         if not issubclass(parser_cls, SQLParser):
             raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")
-
         return parser_cls
 
     @classmethod