Merge pull request #22 from samansmink/add-missing-types

add missing nested types
duckdb · May 30, 2024 · 0106ae1 · 0106ae1
2 parents b0acfc0 + 510fece
commit 0106ae1
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 5 deletions.
diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp
@@ -20,6 +20,8 @@ class SchemaVisitor {
         visitor.data = &state;
         visitor.make_field_list = (uintptr_t (*)(void*, uintptr_t)) &MakeFieldList;
         visitor.visit_struct = (void (*)(void*, uintptr_t, ffi::KernelStringSlice, uintptr_t)) &VisitStruct;
+        visitor.visit_array = (void (*)(void*, uintptr_t, ffi::KernelStringSlice, bool, uintptr_t)) &VisitArray;
+        visitor.visit_map = (void (*)(void*, uintptr_t, ffi::KernelStringSlice, bool, uintptr_t)) &VisitMap;
         visitor.visit_decimal = (void (*)(void*, uintptr_t, ffi::KernelStringSlice, uint8_t , uint8_t)) &VisitDecimal;
         visitor.visit_string = VisitSimpleType<LogicalType::VARCHAR>();
         visitor.visit_long = VisitSimpleType<LogicalType::BIGINT>();
@@ -29,10 +31,10 @@ class SchemaVisitor {
         visitor.visit_float = VisitSimpleType<LogicalType::FLOAT>();
         visitor.visit_double = VisitSimpleType<LogicalType::DOUBLE>();
         visitor.visit_boolean = VisitSimpleType<LogicalType::BOOLEAN>();
-        visitor.visit_binary = VisitSimpleType<LogicalType::VARCHAR>(); // TODO: check
-        visitor.visit_date = VisitSimpleType<LogicalType::DATE>(); // TODO: check
-        visitor.visit_timestamp = VisitSimpleType<LogicalType::TIMESTAMP>(); // TODO: check
-        visitor.visit_timestamp_ntz = VisitSimpleType<LogicalType::TIMESTAMP_TZ>(); // TODO: check
+        visitor.visit_binary = VisitSimpleType<LogicalType::VARCHAR>();
+        visitor.visit_date = VisitSimpleType<LogicalType::DATE>();
+        visitor.visit_timestamp = VisitSimpleType<LogicalType::TIMESTAMP>();
+        visitor.visit_timestamp_ntz = VisitSimpleType<LogicalType::TIMESTAMP_TZ>();
 
         uintptr_t result = visit_schema(snapshot, &visitor);
         return state.TakeFieldList(result);
@@ -66,6 +68,20 @@ class SchemaVisitor {
         state->AppendToList(sibling_list_id, name, LogicalType::STRUCT(std::move(*children)));
     }
 
+    static void VisitArray(SchemaVisitor* state, uintptr_t sibling_list_id, ffi::KernelStringSlice name, bool contains_null, uintptr_t child_list_id) {
+        auto children = state->TakeFieldList(child_list_id);
+
+        D_ASSERT(children->size() == 1);
+        state->AppendToList(sibling_list_id, name, LogicalType::LIST(children->front().second));
+    }
+
+    static void VisitMap(SchemaVisitor* state, uintptr_t sibling_list_id, ffi::KernelStringSlice name, bool contains_null, uintptr_t child_list_id) {
+        auto children = state->TakeFieldList(child_list_id);
+
+        D_ASSERT(children->size() == 2);
+        state->AppendToList(sibling_list_id, name, LogicalType::MAP(LogicalType::STRUCT(std::move(*children))));
+    }
+
     uintptr_t MakeFieldListImpl(uintptr_t capacity_hint) {
         uintptr_t id = next_id++;
         auto list = make_uniq<FieldList>();
@@ -80,6 +96,7 @@ class SchemaVisitor {
         auto it = inflight_lists.find(id);
         if (it == inflight_lists.end()) {
             // TODO... some error...
+            throw InternalException("WEIRD SHIT");
         } else {
             it->second->emplace_back(std::make_pair(string(name.ptr, name.len), std::move(child)));
         }
@@ -89,7 +106,7 @@ class SchemaVisitor {
         auto it = inflight_lists.find(id);
         if (it == inflight_lists.end()) {
             // TODO: Raise some kind of error.
-            return {}; // not present
+            throw InternalException("WEIRD SHIT 2");
         }
         auto rval = std::move(it->second);
         inflight_lists.erase(it);

diff --git a/test/sql/dat/nested_types.test b/test/sql/dat/nested_types.test
@@ -0,0 +1,29 @@
+# name: test/sql/dat_primitive_types.test
+# description: DAT test suite: primitive types 
+# group: [delta]
+
+require parquet
+
+require delta
+
+require-env DAT_PATH
+
+query IIII
+SELECT *
+FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/nested_types/delta')
+----
+0	{'float64': 0.0, 'bool': true}	[0]	{}
+1	{'float64': 1.0, 'bool': false}	[0, 1]	{0=0}
+2	{'float64': 2.0, 'bool': true}	[0, 1, 2]	{0=0, 1=1}
+3	{'float64': 3.0, 'bool': false}	[0, 1, 2, 3]	{0=0, 1=1, 2=2}
+4	{'float64': 4.0, 'bool': true}	[0, 1, 2, 3, 4]	{0=0, 1=1, 2=2, 3=3}
+
+query IIII
+SELECT *
+FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/nested_types/expected/**/*.parquet')
+----
+0	{'float64': 0.0, 'bool': true}	[0]	{}
+1	{'float64': 1.0, 'bool': false}	[0, 1]	{0=0}
+2	{'float64': 2.0, 'bool': true}	[0, 1, 2]	{0=0, 1=1}
+3	{'float64': 3.0, 'bool': false}	[0, 1, 2, 3]	{0=0, 1=1, 2=2}
+4	{'float64': 4.0, 'bool': true}	[0, 1, 2, 3, 4]	{0=0, 1=1, 2=2, 3=3}
diff --git a/test/sql/dat/primitive_types.test b/test/sql/dat/primitive_types.test
@@ -17,3 +17,14 @@ FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delt
 2	2	2	2	2	2.0	2.0	true	\x00\x00	12.000	1970-01-03	1970-01-01 02:00:00
 3	3	3	3	3	3.0	3.0	false	\x00\x00\x00	13.000	1970-01-04	1970-01-01 03:00:00
 4	4	4	4	4	4.0	4.0	true	\x00\x00\x00\x00	14.000	1970-01-05	1970-01-01 04:00:00
+
+query IIIIIIIIIIII
+SELECT *
+FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/expected/**/*.parquet')
+----
+0	0	0	0	0	0.0	0.0	true	(empty)	10.000	1970-01-01	1970-01-01 00:00:00
+1	1	1	1	1	1.0	1.0	false	\x00	11.000	1970-01-02	1970-01-01 01:00:00
+2	2	2	2	2	2.0	2.0	true	\x00\x00	12.000	1970-01-03	1970-01-01 02:00:00
+3	3	3	3	3	3.0	3.0	false	\x00\x00\x00	13.000	1970-01-04	1970-01-01 03:00:00
+4	4	4	4	4	4.0	4.0	true	\x00\x00\x00\x00	14.000	1970-01-05	1970-01-01 04:00:00
+