fix: Fix hostname resolution for spark tests (feast-dev#5610)

ntkathole · web-flow · commit 8f0e22d83986 · 2025-09-12T22:31:55.000-06:00
diff --git a/sdk/python/feast/infra/compute_engines/spark/utils.py b/sdk/python/feast/infra/compute_engines/spark/utils.py
@@ -21,6 +21,9 @@ def get_or_create_new_spark_session(
                 conf=SparkConf().setAll([(k, v) for k, v in spark_config.items()])
             )
 
+        spark_builder = spark_builder.config("spark.driver.host", "127.0.0.1")
+        spark_builder = spark_builder.config("spark.driver.bindAddress", "127.0.0.1")
+
         spark_session = spark_builder.getOrCreate()
     spark_session.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
     return spark_session
diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py
@@ -35,6 +35,8 @@ def __init__(self, project_name: str, *args, **kwargs):
             "spark.eventLog.enabled": "false",
             "spark.sql.parser.quotedRegexColumnNames": "true",
             "spark.sql.session.timeZone": "UTC",
+            "spark.driver.host": "127.0.0.1",
+            "spark.driver.bindAddress": "127.0.0.1",
         }
         if not self.spark_offline_store_config:
             self.create_offline_store_config()
diff --git a/sdk/python/tests/unit/infra/compute_engines/spark/test_nodes.py b/sdk/python/tests/unit/infra/compute_engines/spark/test_nodes.py
@@ -25,6 +25,8 @@ def spark_session():
         SparkSession.builder.appName("FeastSparkTests")
         .master("local[*]")
         .config("spark.sql.shuffle.partitions", "1")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
         .getOrCreate()
     )
 
diff --git a/sdk/python/tests/unit/transformation/test_spark_transformation.py b/sdk/python/tests/unit/transformation/test_spark_transformation.py
@@ -52,7 +52,12 @@ def remove_extra_spaces_sql(df, column_name):
 
 @pytest.fixture
 def spark_fixture():
-    spark = SparkSession.builder.appName("Testing PySpark Example").getOrCreate()
+    spark = (
+        SparkSession.builder.appName("Testing PySpark Example")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
     try:
         yield spark
     finally:
@@ -61,7 +66,12 @@ def spark_fixture():
 
 @patch("feast.infra.compute_engines.spark.utils.get_or_create_new_spark_session")
 def test_spark_transformation(spark_fixture):
-    spark = SparkSession.builder.appName("Testing PySpark Example").getOrCreate()
+    spark = (
+        SparkSession.builder.appName("Testing PySpark Example")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
     df = get_sample_df(spark)
 
     spark_transformation = Transformation(
@@ -77,7 +87,12 @@ def test_spark_transformation(spark_fixture):
 
 @patch("feast.infra.compute_engines.spark.utils.get_or_create_new_spark_session")
 def test_spark_transformation_init_transformation(spark_fixture):
-    spark = SparkSession.builder.appName("Testing PySpark Example").getOrCreate()
+    spark = (
+        SparkSession.builder.appName("Testing PySpark Example")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
     df = get_sample_df(spark)
 
     spark_transformation = SparkTransformation(
@@ -93,7 +108,12 @@ def test_spark_transformation_init_transformation(spark_fixture):
 
 @patch("feast.infra.compute_engines.spark.utils.get_or_create_new_spark_session")
 def test_spark_transformation_sql(spark_fixture):
-    spark = SparkSession.builder.appName("Testing PySpark Example").getOrCreate()
+    spark = (
+        SparkSession.builder.appName("Testing PySpark Example")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
     df = get_sample_df(spark)
 
     spark_transformation = SparkTransformation(

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,9 @@ def get_or_create_new_spark_session(`
`21`	`21`	`conf=SparkConf().setAll([(k, v) for k, v in spark_config.items()])`
`22`	`22`	`)`
`23`	`23`
	`24`	`+ spark_builder = spark_builder.config("spark.driver.host", "127.0.0.1")`
	`25`	`+ spark_builder = spark_builder.config("spark.driver.bindAddress", "127.0.0.1")`
	`26`	`+`
`24`	`27`	`spark_session = spark_builder.getOrCreate()`
`25`	`28`	`spark_session.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")`
`26`	`29`	`return spark_session`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,8 @@ def __init__(self, project_name: str, args, *kwargs):`
`35`	`35`	`"spark.eventLog.enabled": "false",`
`36`	`36`	`"spark.sql.parser.quotedRegexColumnNames": "true",`
`37`	`37`	`"spark.sql.session.timeZone": "UTC",`
	`38`	`+ "spark.driver.host": "127.0.0.1",`
	`39`	`+ "spark.driver.bindAddress": "127.0.0.1",`
`38`	`40`	`}`
`39`	`41`	`if not self.spark_offline_store_config:`
`40`	`42`	`self.create_offline_store_config()`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,8 @@ def spark_session():`
`25`	`25`	`SparkSession.builder.appName("FeastSparkTests")`
`26`	`26`	`.master("local[*]")`
`27`	`27`	`.config("spark.sql.shuffle.partitions", "1")`
	`28`	`+ .config("spark.driver.host", "127.0.0.1")`
	`29`	`+ .config("spark.driver.bindAddress", "127.0.0.1")`
`28`	`30`	`.getOrCreate()`
`29`	`31`	`)`
`30`	`32`