diff --git a/include/vgi/types.h b/include/vgi/types.h index 6186fb0..34a0fc7 100644 --- a/include/vgi/types.h +++ b/include/vgi/types.h @@ -1,6 +1,7 @@ // © Copyright 2025, 2026 Query Farm LLC - https://query.farm #pragma once +#include #include #include #include @@ -176,6 +177,31 @@ inline constexpr const char* kNoOrderGuarantee = "NO_ORDER_GUARANTEE"; inline constexpr const char* kFixedOrder = "FIXED_ORDER"; } // namespace order_preservations +namespace filter_semantic_profiles { +inline constexpr const char* kDuckDBStandardV1 = "vgi.duckdb.standard.v1"; +} // namespace filter_semantic_profiles + +// A versioned extension function the worker can evaluate inside a Filter-v2 +// expression. The standard profile's built-in functions do not appear here. +struct FilterFunctionCapability { + std::string namespace_name; + std::string name; + uint64_t version = 0; +}; + +// A versioned runtime-filter artifact algorithm the worker can evaluate. +struct RuntimeFilterAlgorithmCapability { + std::string namespace_name; + std::string name; + uint64_t version = 0; +}; + +// An evaluation-context profile the worker can apply in an isolated session. +struct EvaluationContextCapability { + std::string profile; + std::optional provider_fingerprint; +}; + // Everything the engine shows a user about a function, plus the return type // when it is fixed. A function whose return type depends on its arguments // leaves `return_type` empty and answers during bind instead. @@ -226,6 +252,19 @@ struct FunctionMetadata { // On, the framework filters each emitted batch, which is what a fixture // that merely advertises the capability wants. bool auto_apply_filters = false; + // Filter Encoding v2 semantics implemented by this function. A function + // that enables filter_pushdown and leaves this empty advertises the C++ + // SDK's standard-v1 evaluator. + std::vector filter_semantic_profiles; + // Capability-gated extensions to the standard profile. The C++ SDK + // currently rejects non-empty lists until matching evaluators exist. + std::vector additional_filter_functions; + std::vector runtime_filter_algorithms; + std::vector filter_evaluation_contexts; + // True only when the worker applies every pushed predicate exactly and + // the engine may therefore remove its residual. Never inferred from + // auto_apply_filters. + bool filters_exactly_applied = false; // Whether the engine may rewrite a scan of this function into a // late-materialization plan: fetch the row ids first, then fetch only the // surviving rows' columns. @@ -269,6 +308,10 @@ struct FunctionMetadata { // engine forward their values; a setting not declared here never arrives, // however it was set. std::vector required_settings; + + // Resolve the wire advertisement, applying the SDK default and rejecting + // semantic profiles for which this SDK has no evaluator. + std::vector resolved_filter_semantic_profiles() const; }; } // namespace vgi diff --git a/src/catalog.cpp b/src/catalog.cpp index a16e23f..9522b86 100644 --- a/src/catalog.cpp +++ b/src/catalog.cpp @@ -220,6 +220,54 @@ const char* partition_kind_wire_value(const FunctionMetadata& metadata) { : metadata.partition_kind.c_str(); } +std::vector> filter_function_identities_of( + const FunctionMetadata& metadata) { + std::vector> identities; + identities.reserve(metadata.additional_filter_functions.size()); + for (const auto& capability : metadata.additional_filter_functions) { + identities.emplace_back(capability.namespace_name, capability.name, capability.version); + } + return identities; +} + +std::vector> runtime_filter_identities_of( + const FunctionMetadata& metadata) { + std::vector> identities; + identities.reserve(metadata.runtime_filter_algorithms.size()); + for (const auto& capability : metadata.runtime_filter_algorithms) { + identities.emplace_back(capability.namespace_name, capability.name, capability.version); + } + return identities; +} + +std::vector>> evaluation_contexts_of( + const FunctionMetadata& metadata) { + std::vector>> contexts; + contexts.reserve(metadata.filter_evaluation_contexts.size()); + for (const auto& capability : metadata.filter_evaluation_contexts) { + contexts.emplace_back(capability.profile, capability.provider_fingerprint); + } + return contexts; +} + +void validate_filter_capabilities(const FunctionMetadata& metadata) { + // Advertising a capability is a correctness claim: the producer may emit + // it in a required predicate and remove its local copy. Keep these lists + // empty until this SDK registers the corresponding evaluator. + if (!metadata.additional_filter_functions.empty()) { + throw std::invalid_argument( + "C++ SDK cannot advertise an extension filter function without an evaluator"); + } + if (!metadata.runtime_filter_algorithms.empty()) { + throw std::invalid_argument( + "C++ SDK has no registered runtime-filter artifact evaluator to advertise"); + } + if (!metadata.filter_evaluation_contexts.empty()) { + throw std::invalid_argument( + "C++ SDK has no isolated DuckDB session-context evaluator to advertise"); + } +} + // Wrap a payload batch in the `{result: binary}` envelope every non-void // method answers with. The engine unwraps it and validates the inner schema // against its own generated copy, so a drifted payload is caught there rather @@ -1070,6 +1118,8 @@ wire::ResultBuilder Dispatcher::common_function_info( const std::string& name, const SchemaPath& schema_path, const char* function_type, const std::vector& specs, const std::shared_ptr& output_schema, const FunctionMetadata& metadata) { + validate_filter_capabilities(metadata); + const auto filter_semantic_profiles = metadata.resolved_filter_semantic_profiles(); auto builder = wire::ResultBuilder(gen::FunctionInfoSchema()) .set_string("name", name) @@ -1089,6 +1139,13 @@ wire::ResultBuilder Dispatcher::common_function_info( .set_secret_lookups("required_secrets", secret_entries(metadata)) .set_bool("projection_pushdown", metadata.projection_pushdown) .set_bool("filter_pushdown", metadata.filter_pushdown) + .set_string_list("filter_semantic_profiles", filter_semantic_profiles) + .set_filter_identities("additional_filter_functions", + filter_function_identities_of(metadata)) + .set_filter_identities("runtime_filter_algorithms", + runtime_filter_identities_of(metadata)) + .set_evaluation_contexts("filter_evaluation_contexts", evaluation_contexts_of(metadata)) + .set_bool("filters_exactly_applied", metadata.filters_exactly_applied) .set_bool("sampling_pushdown", metadata.sampling_pushdown) .set_bool("input_from_args", metadata.input_from_args) .set_enum("partition_kind", partition_kind_wire_value(metadata)) diff --git a/src/function.cpp b/src/function.cpp index 132d990..b07217e 100644 --- a/src/function.cpp +++ b/src/function.cpp @@ -7,6 +7,21 @@ namespace vgi { +std::vector FunctionMetadata::resolved_filter_semantic_profiles() const { + auto profiles = filter_semantic_profiles; + if (filter_pushdown && profiles.empty()) { + profiles.emplace_back(filter_semantic_profiles::kDuckDBStandardV1); + } + for (const auto& profile : profiles) { + if (profile != filter_semantic_profiles::kDuckDBStandardV1) { + throw std::invalid_argument("C++ SDK supports only " + + std::string(filter_semantic_profiles::kDuckDBStandardV1) + + " filter semantics"); + } + } + return profiles; +} + ArgSpec ArgSpec::column(std::string name, int index, std::string type, std::string description) { ArgSpec s; s.name = std::move(name); diff --git a/src/wire.cpp b/src/wire.cpp index 8a3e576..9628f0c 100644 --- a/src/wire.cpp +++ b/src/wire.cpp @@ -544,6 +544,15 @@ arrow::StringBuilder* string_child(arrow::StructBuilder& entry, const char* name return builder; } +arrow::UInt64Builder* uint64_child(arrow::StructBuilder& entry, const char* name) { + const auto type = entry.type(); + const int index = static_cast(*type).GetFieldIndex(name); + if (index < 0) fail(std::string("struct has no '") + name + "' field"); + auto* builder = dynamic_cast(entry.field_builder(index)); + if (!builder) fail(std::string("struct field '") + name + "' is not uint64"); + return builder; +} + ResultBuilder& ResultBuilder::set_secret_lookups( const std::string& field, const std::vector>& lookups) { @@ -607,6 +616,62 @@ ResultBuilder& ResultBuilder::set_examples(const std::string& field, return *this; } +ResultBuilder& ResultBuilder::set_filter_identities( + const std::string& field, + const std::vector>& identities) { + const int index = field_index(field); + std::unique_ptr raw; + check_ok(arrow::MakeBuilder(arrow::default_memory_pool(), schema_->field(index)->type(), &raw), + "building filter capability field '" + field + "'"); + auto* list = dynamic_cast(raw.get()); + if (!list) fail("result field '" + field + "' is not a list"); + auto* entry = dynamic_cast(list->value_builder()); + if (!entry) fail("result field '" + field + "' is not a list of structs"); + + auto* namespace_builder = string_child(*entry, "namespace"); + auto* name_builder = string_child(*entry, "name"); + auto* version_builder = uint64_child(*entry, "version"); + + check_ok(list->Append(), "opening filter capability field '" + field + "'"); + for (const auto& [namespace_name, name, version] : identities) { + check_ok(entry->Append(), "opening a filter capability"); + check_ok(namespace_builder->Append(namespace_name), "appending filter namespace"); + check_ok(name_builder->Append(name), "appending filter name"); + check_ok(version_builder->Append(version), "appending filter version"); + } + arrays_[static_cast(index)] = + unwrap(list->Finish(), "finishing filter capability field '" + field + "'"); + return *this; +} + +ResultBuilder& ResultBuilder::set_evaluation_contexts( + const std::string& field, + const std::vector>>& contexts) { + const int index = field_index(field); + std::unique_ptr raw; + check_ok(arrow::MakeBuilder(arrow::default_memory_pool(), schema_->field(index)->type(), &raw), + "building evaluation-context field '" + field + "'"); + auto* list = dynamic_cast(raw.get()); + if (!list) fail("result field '" + field + "' is not a list"); + auto* entry = dynamic_cast(list->value_builder()); + if (!entry) fail("result field '" + field + "' is not a list of structs"); + + auto* profile_builder = string_child(*entry, "profile"); + auto* fingerprint_builder = string_child(*entry, "provider_fingerprint"); + + check_ok(list->Append(), "opening evaluation-context field '" + field + "'"); + for (const auto& [profile, fingerprint] : contexts) { + check_ok(entry->Append(), "opening an evaluation context"); + check_ok(profile_builder->Append(profile), "appending evaluation-context profile"); + check_ok(fingerprint ? fingerprint_builder->Append(*fingerprint) + : fingerprint_builder->AppendNull(), + "appending evaluation-context fingerprint"); + } + arrays_[static_cast(index)] = + unwrap(list->Finish(), "finishing evaluation-context field '" + field + "'"); + return *this; +} + ResultBuilder& ResultBuilder::set_int64_map( const std::string& field, const std::vector>& entries) { auto key_builder = std::make_shared(); diff --git a/src/wire.h b/src/wire.h index d1ef391..c6772f1 100644 --- a/src/wire.h +++ b/src/wire.h @@ -202,6 +202,14 @@ class ResultBuilder { // The `examples` column: a list of {sql, description, expected_output}. ResultBuilder& set_examples(const std::string& field, const std::vector& examples); + // A list<{namespace: utf8, name: utf8, version: uint64}> capability field. + ResultBuilder& set_filter_identities( + const std::string& field, + const std::vector>& identities); + // A list<{profile: utf8, provider_fingerprint: utf8?}> capability field. + ResultBuilder& set_evaluation_contexts( + const std::string& field, + const std::vector>>& contexts); // A map column. Arrow spells map entries key/value (not // keys/values), which is what the canonical Python protocol emits. ResultBuilder& set_int64_map(const std::string& field, diff --git a/tests/function_test.cpp b/tests/function_test.cpp index ee603b9..bee347f 100644 --- a/tests/function_test.cpp +++ b/tests/function_test.cpp @@ -93,11 +93,73 @@ TEST_CASE("protocol v2 named schemas use path lists", "[protocol]") { const auto function_info = vgi::generated::FunctionInfoSchema(); REQUIRE(function_info->GetFieldByName("parameter_default_values")->nullable()); + REQUIRE(function_info->GetFieldByName("filter_semantic_profiles")->type()->id() == + arrow::Type::LIST); + REQUIRE(function_info->GetFieldByName("additional_filter_functions")->type()->id() == + arrow::Type::LIST); + REQUIRE(function_info->GetFieldByName("runtime_filter_algorithms")->type()->id() == + arrow::Type::LIST); + REQUIRE(function_info->GetFieldByName("filter_evaluation_contexts")->type()->id() == + arrow::Type::LIST); + REQUIRE(function_info->GetFieldByName("filters_exactly_applied")->type()->id() == + arrow::Type::BOOL); REQUIRE(vgi::generated::BindRequestSchema()->GetFieldByName("argument_names")->nullable()); REQUIRE( vgi::generated::AggregateBindRequestSchema()->GetFieldByName("argument_names")->nullable()); } +TEST_CASE("filter pushdown advertises only the implemented semantic profile", "[protocol]") { + vgi::FunctionMetadata plain; + REQUIRE(plain.resolved_filter_semantic_profiles().empty()); + + vgi::FunctionMetadata filtering; + filtering.filter_pushdown = true; + REQUIRE(filtering.resolved_filter_semantic_profiles() == + std::vector{vgi::filter_semantic_profiles::kDuckDBStandardV1}); + filtering.auto_apply_filters = true; + REQUIRE_FALSE(filtering.filters_exactly_applied); + + filtering.filter_semantic_profiles = {"vgi.duckdb.standard.v2"}; + REQUIRE_THROWS(filtering.resolved_filter_semantic_profiles()); +} + +TEST_CASE("filter capability structs preserve generated FunctionInfo shape", "[protocol]") { + const auto batch = vgi::wire::ResultBuilder(vgi::generated::FunctionInfoSchema()) + .set_string_list("filter_semantic_profiles", {"vgi.duckdb.standard.v1"}) + .set_filter_identities("additional_filter_functions", + {{"duckdb.spatial", "intersects_extent", 1}}) + .set_filter_identities("runtime_filter_algorithms", + {{"duckdb.runtime_filter", "bloom", 2}}) + .set_evaluation_contexts("filter_evaluation_contexts", + {{"vgi.duckdb.session.v1", "duckdb-icu:test"}, + {"vgi.duckdb.session.v1", std::nullopt}}) + .fill_defaults() + .finish(); + + const auto identities = std::static_pointer_cast( + batch->GetColumnByName("additional_filter_functions")); + REQUIRE(identities->value_length(0) == 1); + const auto identity = std::static_pointer_cast(identities->values()); + const auto namespaces = + std::static_pointer_cast(identity->GetFieldByName("namespace")); + const auto names = + std::static_pointer_cast(identity->GetFieldByName("name")); + const auto versions = + std::static_pointer_cast(identity->GetFieldByName("version")); + REQUIRE(namespaces->GetString(0) == "duckdb.spatial"); + REQUIRE(names->GetString(0) == "intersects_extent"); + REQUIRE(versions->Value(0) == 1); + + const auto contexts = std::static_pointer_cast( + batch->GetColumnByName("filter_evaluation_contexts")); + REQUIRE(contexts->value_length(0) == 2); + const auto context = std::static_pointer_cast(contexts->values()); + const auto fingerprints = std::static_pointer_cast( + context->GetFieldByName("provider_fingerprint")); + REQUIRE(fingerprints->GetString(0) == "duckdb-icu:test"); + REQUIRE(fingerprints->IsNull(1)); +} + TEST_CASE("wire schema paths round trip every component", "[wire]") { const auto schema = arrow::schema( {arrow::field("schema_path", arrow::list(arrow::utf8()), /*nullable=*/false)});