Adding retry option for glue and sagemaker as well

datahub-project · Aug 29, 2023 · 534f3ad · 534f3ad
1 parent 08be9e6
commit 534f3ad
Showing 1 changed file with 20 additions and 2 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
@@ -216,10 +216,28 @@ def get_s3_resource(
         return resource
 
     def get_glue_client(self) -> "GlueClient":
-        return self.get_session().client("glue")
+        return self.get_session().client(
+            "glue",
+            config=Config(
+                proxies=self.aws_proxy,
+                retries={
+                    "max_attempts": self.aws_retry_num,
+                    "mode": self.aws_retry_mode,
+                },
+            ),
+        )
 
     def get_sagemaker_client(self) -> "SageMakerClient":
-        return self.get_session().client("sagemaker")
+        return self.get_session().client(
+            "sagemaker",
+            config=Config(
+                proxies=self.aws_proxy,
+                retries={
+                    "max_attempts": self.aws_retry_num,
+                    "mode": self.aws_retry_mode,
+                },
+            ),
+        )
 
 
 class AwsSourceConfig(EnvConfigMixin, AwsConnectionConfig):