fix incorrect column mapping for delta constant columns

duckdb · Aug 1, 2024 · d320bb2 · d320bb2
1 parent 18fd5dc
commit d320bb2
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 6 deletions.
diff --git a/src/functions/delta_scan.cpp b/src/functions/delta_scan.cpp
@@ -563,7 +563,7 @@ void DeltaMultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_optio
             auto col_partition_entry = file_metadata->partition_map.find(global_names[col_id]);
             if (col_partition_entry != file_metadata->partition_map.end()) {
                 // Todo: use https://github.com/delta-io/delta/blob/master/PROTOCOL.md#partition-value-serialization
-                auto maybe_value = Value(col_partition_entry->second).DefaultCastAs(global_types[i]);
+                auto maybe_value = Value(col_partition_entry->second).DefaultCastAs(global_types[col_id]);
                 reader_data.constant_map.emplace_back(i, maybe_value);
             }
         }

diff --git a/test/sql/dat/all.test b/test/sql/dat/all.test
@@ -12,15 +12,26 @@ require-env DAT_PATH
 
 # all_primitive_types
 query I rowsort all_primitive_types
-SELECT *
+SELECT * EXCLUDE(timestamp)
 FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta')
 ----
 
+# TODO: exclude is necessary due to issue with the golden tables, this should be fixed upstream
 query I rowsort all_primitive_types
-SELECT *
+SELECT * EXCLUDE(timestamp)
 FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/expected/latest/**/*.parquet')
 ----
 
+query I
+SELECT timestamp
+FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta')
+----
+1970-01-01 00:00:00+00
+1970-01-01 01:00:00+00
+1970-01-01 02:00:00+00
+1970-01-01 03:00:00+00
+1970-01-01 04:00:00+00
+
 # nested_types
 query I rowsort nested_types
 SELECT *
@@ -120,15 +131,25 @@ require notwindows
 
 # multi_partitioned_2
 query I rowsort multi_partitioned_2
-SELECT *
+SELECT * EXCLUDE(time) 
 FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta')
 ----
 
+# TODO: exclude is necessary due to issue with the golden tables, this should be fixed upstream
 query I rowsort multi_partitioned_2
-SELECT *
+SELECT * EXCLUDE(time)
 FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/expected/latest/**/*.parquet')
 ----
 
+query I
+SELECT time
+FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta')
+order by time
+----
+1970-01-01 00:00:00+00
+1970-01-01 12:30:00+00
+1970-01-02 08:45:00+00
+
 # no_replay
 query I rowsort no_replay
 SELECT *

diff --git a/test/sql/delta_kernel_rs/timestamp_ntz.test b/test/sql/delta_kernel_rs/timestamp_ntz.test
@@ -8,7 +8,7 @@ require delta
 
 require-env DELTA_KERNEL_TESTS_PATH
 
-query III
+query IIIIII
 DESCRIBE FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/data-reader-timestamp_ntz') order by id
 ----
 id	INTEGER	YES	NULL	NULL	NULL