feat(ingest/s3): support path_specs of different S3 buckets in the sa…

…me recipe (#7514)
datahub-project · Mar 15, 2023 · bf36c93 · bf36c93
1 parent 756a4f3
commit bf36c93
Show file tree

Hide file tree

Showing 15 changed files with 2,547 additions and 43 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -12,4 +12,3 @@ org.gradle.internal.repository.initial.backoff=1000
 
 # Needed to publish to Nexus from a sub-module
 gnsp.disableApplyOnlyOnRootProjectEnforcement=true
-
diff --git a/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml b/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml
@@ -13,4 +13,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/hana_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/hana_to_datahub.dhub.yaml
@@ -18,4 +18,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/mode_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/mode_to_datahub.dhub.yaml
@@ -17,4 +17,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/okta_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/okta_to_datahub.dhub.yaml
@@ -6,4 +6,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/examples/recipes/s3_to_file.dhub.yaml b/metadata-ingestion/examples/recipes/s3_to_file.dhub.yaml
@@ -16,4 +16,3 @@ sink:
   type: "file"
   config:
     filename: "./s3_data_lake_mces.json"
-
diff --git a/metadata-ingestion/examples/recipes/snowflake_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/snowflake_to_datahub.dhub.yaml
@@ -18,4 +18,4 @@ source:
     email_domain: mycompany.com
 
     classification:
-      enabled: True
+      enabled: True
diff --git a/metadata-ingestion/examples/recipes/tableau_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/tableau_to_datahub.dhub.yaml
@@ -17,4 +17,4 @@ source:
 sink:
   type: "datahub-rest"
   config:
-    server: "http://localhost:8080"
+    server: "http://localhost:8080"
diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh
@@ -117,4 +117,3 @@ fi
 
 
 printf "\n\e[38;2;0;255;0m✅ Preflight was successful\e[38;2;255;255;255m\n"
-
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -12,7 +12,6 @@
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.aws.path_spec import PathSpec
-from datahub.ingestion.source.aws.s3_util import get_bucket_name
 from datahub.ingestion.source.s3.profiling import DataLakeProfilerConfig
 
 # hide annoying debug errors from py4j
@@ -92,16 +91,6 @@ def check_path_specs_and_infer_platform(
             )
         guessed_platform = guessed_platforms.pop()
 
-        # If platform is s3, check that they're all the same bucket.
-        if guessed_platform == "s3":
-            bucket_names = set(
-                get_bucket_name(path_spec.include) for path_spec in path_specs
-            )
-            if len(bucket_names) > 1:
-                raise ValueError(
-                    f"All path_specs should reference the same s3 bucket. Got {bucket_names}"
-                )
-
         # Ensure s3 configs aren't used for file sources.
         if guessed_platform != "s3" and (
             values.get("use_s3_object_tags") or values.get("use_s3_bucket_tags")
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,3 @@ org.gradle.internal.repository.initial.backoff=1000

		# Needed to publish to Nexus from a sub-module
		gnsp.disableApplyOnlyOnRootProjectEnforcement=true
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,4 +16,3 @@ sink:
		type: "file"
		config:
		filename: "./s3_data_lake_mces.json"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -117,4 +117,3 @@ fi


		printf "\n\e[38;2;0;255;0m✅ Preflight was successful\e[38;2;255;255;255m\n"