fix(ci): smoke test lint failures (#11044)

datahub-project · Jul 30, 2024 · a734d69 · a734d69
1 parent 7d4b645
commit a734d69
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 16 deletions.
diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py
@@ -1,18 +1,33 @@
 import setuptools
+import os
 
 folders = ["./smoke-test/tests"]
 
 for folder in folders:
     print(f"Checking folder {folder}")
-    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
-    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+    packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    namespace_packages = [
+        i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i
+    ]
 
-    in_a_not_b = set(a) - set(b)
-    in_b_not_a = set(b) - set(a)
+    print("Packages found:", packages)
+    print("Namespace packages found:", namespace_packages)
+
+    in_packages_not_namespace = set(packages) - set(namespace_packages)
+    in_namespace_not_packages = set(namespace_packages) - set(packages)
+
+    if in_packages_not_namespace:
+        print(f"Packages not in namespace packages: {in_packages_not_namespace}")
+    if in_namespace_not_packages:
+        print(f"Namespace packages not in packages: {in_namespace_not_packages}")
+        for pkg in in_namespace_not_packages:
+            pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep))
+            print(f"Contents of {pkg_path}:")
+            print(os.listdir(pkg_path))
 
     assert (
-        len(in_a_not_b) == 0
-    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+        len(in_packages_not_namespace) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}"
     assert (
-        len(in_b_not_a) == 0
-    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"
+        len(in_namespace_not_packages) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}"
diff --git a/smoke-test/tests/schema_fields/queries/__init__.py b/smoke-test/tests/schema_fields/queries/__init__.py
diff --git a/smoke-test/tests/schema_fields/test_schemafields.py b/smoke-test/tests/schema_fields/test_schemafields.py
@@ -6,10 +6,7 @@
 
 import datahub.metadata.schema_classes as models
 import pytest
-from datahub.emitter.mce_builder import (
-    make_dataset_urn,
-    make_schema_field_urn,
-)
+from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
 from datahub.ingestion.api.sink import NoopWriteCallback

diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py
@@ -119,9 +119,11 @@ def create_property_definition(
         qualifiedName=f"{namespace}.{property_name}",
         valueType=Urn.make_data_type_urn(value_type),
         description="The retention policy for the dataset",
-        entityTypes=[Urn.make_entity_type_urn(e) for e in entity_types]
-        if entity_types
-        else [Urn.make_entity_type_urn("dataset")],
+        entityTypes=(
+            [Urn.make_entity_type_urn(e) for e in entity_types]
+            if entity_types
+            else [Urn.make_entity_type_urn("dataset")]
+        ),
         cardinality=cardinality,
         allowedValues=allowed_values,
     )
@@ -137,7 +139,7 @@ def create_property_definition(
 def attach_property_to_entity(
     urn: str,
     property_name: str,
-    property_value: Union[str, float, List[str | float]],
+    property_value: Union[str, float, List[Union[str, float]]],
     graph: DataHubGraph,
     namespace: str = default_namespace,
 ):