Merge pull request #164 from samansmink/bump-kernel-to-0.7.0

bump kernel to 0.7.0

CMakeLists.txt

@@ -138,16 +138,18 @@ set(DELTA_KERNEL_FFI_HEADER_CXX
 # Add rust_example as a CMake target
 ExternalProject_Add(
   ${KERNEL_NAME}
-  GIT_REPOSITORY "https://github.com/delta-incubator/delta-kernel-rs"
+  GIT_REPOSITORY "https://github.com/delta-io/delta-kernel-rs"
   # WARNING: the FFI headers are currently pinned due to the C linkage issue of
   # the c++ headers. Currently, when bumping the kernel version, the produced
   # header in ./src/include/delta_kernel_ffi.hpp should be also bumped, applying
   # the fix
-  GIT_TAG v0.6.1
+  GIT_TAG v0.7.0
   # Prints the env variables passed to the cargo build to the terminal, useful
   # in debugging because passing them through CMake is an error-prone mess
   CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${RUST_UNSET_ENV_VARS}
                     ${RUST_ENV_VARS} env
+  # Temporary workaround to fix compile issue TODO: remove once fixed upstream
+  CONFIGURE_COMMAND cargo update -p chrono --precise 0.4.38
   UPDATE_COMMAND ""
   BUILD_IN_SOURCE 1
   # Build debug build

src/delta_utils.cpp

@@ -317,10 +317,10 @@ unique_ptr<SchemaVisitor::FieldList> SchemaVisitor::VisitSnapshotSchema(ffi::Sha
 
 	visitor.data = &state;
 	visitor.make_field_list = (uintptr_t(*)(void *, uintptr_t)) & MakeFieldList;
-	visitor.visit_struct = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, uintptr_t)) & VisitStruct;
-	visitor.visit_array = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, uintptr_t)) & VisitArray;
-	visitor.visit_map = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, uintptr_t)) & VisitMap;
-	visitor.visit_decimal = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, uint8_t, uint8_t)) & VisitDecimal;
+	visitor.visit_struct = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, const ffi::CStringMap *metadata, uintptr_t)) & VisitStruct;
+	visitor.visit_array = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, const ffi::CStringMap *metadata, uintptr_t)) & VisitArray;
+	visitor.visit_map = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, const ffi::CStringMap *metadata, uintptr_t)) & VisitMap;
+	visitor.visit_decimal = (void (*)(void *, uintptr_t, ffi::KernelStringSlice, bool, const ffi::CStringMap *metadata, uint8_t, uint8_t)) & VisitDecimal;
 	visitor.visit_string = VisitSimpleType<LogicalType::VARCHAR>();
 	visitor.visit_long = VisitSimpleType<LogicalType::BIGINT>();
 	visitor.visit_integer = VisitSimpleType<LogicalType::INTEGER>();
@@ -334,12 +334,15 @@ unique_ptr<SchemaVisitor::FieldList> SchemaVisitor::VisitSnapshotSchema(ffi::Sha
 	visitor.visit_timestamp = VisitSimpleType<LogicalType::TIMESTAMP_TZ>();
 	visitor.visit_timestamp_ntz = VisitSimpleType<LogicalType::TIMESTAMP>();
 
-	uintptr_t result = visit_schema(snapshot, &visitor);
+    auto schema = logical_schema(snapshot);
+	uintptr_t result = visit_schema(schema, &visitor);
+    free_schema(schema);
+
 	return state.TakeFieldList(result);
 }
 
 void SchemaVisitor::VisitDecimal(SchemaVisitor *state, uintptr_t sibling_list_id, ffi::KernelStringSlice name,
-                                 uint8_t precision, uint8_t scale) {
+                                 bool is_nullable, const ffi::CStringMap *metadata, uint8_t precision, uint8_t scale) {
 	state->AppendToList(sibling_list_id, name, LogicalType::DECIMAL(precision, scale));
 }
 
@@ -348,21 +351,21 @@ uintptr_t SchemaVisitor::MakeFieldList(SchemaVisitor *state, uintptr_t capacity_
 }
 
 void SchemaVisitor::VisitStruct(SchemaVisitor *state, uintptr_t sibling_list_id, ffi::KernelStringSlice name,
-                                uintptr_t child_list_id) {
+                                 bool is_nullable, const ffi::CStringMap *metadata, uintptr_t child_list_id) {
 	auto children = state->TakeFieldList(child_list_id);
 	state->AppendToList(sibling_list_id, name, LogicalType::STRUCT(std::move(*children)));
 }
 
 void SchemaVisitor::VisitArray(SchemaVisitor *state, uintptr_t sibling_list_id, ffi::KernelStringSlice name,
-                               bool contains_null, uintptr_t child_list_id) {
+                               bool is_nullable, const ffi::CStringMap *metadata, uintptr_t child_list_id) {
 	auto children = state->TakeFieldList(child_list_id);
 
 	D_ASSERT(children->size() == 1);
 	state->AppendToList(sibling_list_id, name, LogicalType::LIST(children->front().second));
 }
 
 void SchemaVisitor::VisitMap(SchemaVisitor *state, uintptr_t sibling_list_id, ffi::KernelStringSlice name,
-                             bool contains_null, uintptr_t child_list_id) {
+                             bool is_nullable, const ffi::CStringMap *metadata, uintptr_t child_list_id) {
 	auto children = state->TakeFieldList(child_list_id);
 
 	D_ASSERT(children->size() == 2);

src/functions/delta_scan/delta_multi_file_list.cpp

@@ -385,7 +385,7 @@ void ScanDataCallBack::VisitCallbackInternal(ffi::NullableCvoid engine_context,
 	case_insensitive_map_t<string> constant_map;
 	for (const auto &col : snapshot.names) {
 		auto key = KernelUtils::ToDeltaString(col);
-		auto *partition_val = (string *)ffi::get_from_map(partition_values, key, allocate_string);
+		auto *partition_val = (string *)ffi::get_from_string_map(partition_values, key, allocate_string);
 		if (partition_val) {
 			constant_map[col] = *partition_val;
 			delete partition_val;
@@ -395,7 +395,7 @@ void ScanDataCallBack::VisitCallbackInternal(ffi::NullableCvoid engine_context,
 }
 
 void ScanDataCallBack::VisitCallback(ffi::NullableCvoid engine_context, ffi::KernelStringSlice path, int64_t size,
-                                     const ffi::Stats *stats, const ffi::DvInfo *dv_info,
+                                     const ffi::Stats *stats, const ffi::DvInfo *dv_info, const ffi::Expression *transform,
                                      const ffi::CStringMap *partition_values) {
 	try {
 		return VisitCallbackInternal(engine_context, path, size, stats, dv_info, partition_values);
@@ -406,8 +406,8 @@ void ScanDataCallBack::VisitCallback(ffi::NullableCvoid engine_context, ffi::Ker
 }
 
 void ScanDataCallBack::VisitData(void *engine_context, ffi::ExclusiveEngineData *engine_data,
-                                 const struct ffi::KernelBoolSlice selection_vec) {
-	ffi::visit_scan_data(engine_data, selection_vec, engine_context, ScanDataCallBack::VisitCallback);
+                                 const struct ffi::KernelBoolSlice selection_vec, const ffi::CTransforms *transforms) {
+	ffi::visit_scan_data(engine_data, selection_vec, transforms, engine_context, ScanDataCallBack::VisitCallback);
 }
 
 DeltaMultiFileList::DeltaMultiFileList(ClientContext &context_p, const string &path)