feat(ingest/s3): support path_specs of different S3 buckets in the same recipe (datahub-project#7514)

harsha-mandadi-4026 · shirshanka · commit abd92aad6d74 · 2023-03-14T23:11:07.000-07:00
diff --git a/gradle.properties b/gradle.properties
@@ -12,4 +12,3 @@ org.gradle.internal.repository.initial.backoff=1000
 
 # Needed to publish to Nexus from a sub-module
 gnsp.disableApplyOnlyOnRootProjectEnforcement=true
-
diff --git a/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml b/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml
@@ -13,4 +13,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/hana_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/hana_to_datahub.dhub.yaml
@@ -18,4 +18,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/mode_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/mode_to_datahub.dhub.yaml
@@ -17,4 +17,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/okta_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/okta_to_datahub.dhub.yaml
@@ -6,4 +6,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/s3_to_file.dhub.yaml b/metadata-ingestion/examples/recipes/s3_to_file.dhub.yaml
@@ -16,4 +16,3 @@ sink:
   type: "file"
   config:
     filename: "./s3_data_lake_mces.json"
-
diff --git a/metadata-ingestion/examples/recipes/snowflake_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/snowflake_to_datahub.dhub.yaml
@@ -18,4 +18,4 @@ source:
     email_domain: mycompany.com
 
     classification:
-      enabled: True
+      enabled: True
diff --git a/metadata-ingestion/examples/recipes/tableau_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/tableau_to_datahub.dhub.yaml
@@ -17,4 +17,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh
@@ -117,4 +117,3 @@ fi
 
 
 printf "\n\e[38;2;0;255;0m&#9989; Preflight was successful\e[38;2;255;255;255m\n"
-
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -12,7 +12,6 @@
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.aws.path_spec import PathSpec
-from datahub.ingestion.source.aws.s3_util import get_bucket_name
 from datahub.ingestion.source.s3.profiling import DataLakeProfilerConfig
 
 # hide annoying debug errors from py4j
@@ -92,16 +91,6 @@ def check_path_specs_and_infer_platform(
             )
         guessed_platform = guessed_platforms.pop()
 
-        # If platform is s3, check that they're all the same bucket.
-        if guessed_platform == "s3":
-            bucket_names = set(
-                get_bucket_name(path_spec.include) for path_spec in path_specs
-            )
-            if len(bucket_names) > 1:
-                raise ValueError(
-                    f"All path_specs should reference the same s3 bucket. Got {bucket_names}"
-                )
-
         # Ensure s3 configs aren't used for file sources.
         if guessed_platform != "s3" and (
             values.get("use_s3_object_tags") or values.get("use_s3_bucket_tags")
diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json
diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json
diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/sources/s3/multiple_specs_of_different_buckets.json
diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py
diff --git a/repositories.gradle b/repositories.gradle

Original file line number	Diff line number	Diff line change
`@@ -12,4 +12,3 @@ org.gradle.internal.repository.initial.backoff=1000`
`12`	`12`
`13`	`13`	`# Needed to publish to Nexus from a sub-module`
`14`	`14`	`gnsp.disableApplyOnlyOnRootProjectEnforcement=true`
`15`		`-`
Original file line number	Diff line number	Diff line change
`@@ -117,4 +117,3 @@ fi`
`117`	`117`
`118`	`118`
`119`	`119`	`printf "\n\e[38;2;0;255;0m✅ Preflight was successful\e[38;2;255;255;255m\n"`
`120`		`-`