duckdb: report single-column filters as not pushed

vortex-duckdb/cpp/include/table_function.h

@@ -25,8 +25,29 @@ void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, cons
 // Input data passed into the init_global and init_local callbacks.
 typedef struct {
     const void *bind_data;
+
+    /**
+     * Projected columns that are requested to be read. These are not
+     * all columns, only the ones DuckDB optimizer thinks we should read.
+     */
     idx_t *column_ids;
     size_t column_ids_count;
+
+    /**
+     * Post filter projected columns. Our table function implements filter
+     * pushdown so this list is a subset of columns referenced in column_ids
+     * after filter pushdown and filter pruning. May be empty, in which case
+     * column_ids should be used.
+     * Indices in this list reference values from column_ids. I.e. if
+     * column_ids=[1,5,6], projection_ids=[1], output column should be
+     * column_ids[1] = 5
+     *
+     * Example usage:
+     * https://github.com/duckdb/duckdb/blob/dc11eadd8f0a7c600f0034810706605ebe10d5b9/src/include/duckdb/function/table_function.hpp#L147
+     */
+    const idx_t *projection_ids;
+    size_t projection_ids_count;
+
     duckdb_vx_table_filter_set filters;
     duckdb_client_context client_context;
 } duckdb_vx_tfunc_init_input;

vortex-duckdb/cpp/multi_file_reader.cpp

@@ -7,6 +7,8 @@
 #include "vortex_duckdb.h"
 #include "vortex.h"
 
+#include "duckdb/execution/operator/scan/physical_table_scan.hpp"
+
 unique_ptr<FunctionData> VortexBindData::Copy() const {
     auto result = make_uniq<VortexBindData>();
     if (ffi_bind_data) {
@@ -150,11 +152,25 @@ VortexReaderInterface::InitializeGlobalState(ClientContext &context,
         column_ids[i] = storage_index;
     }
 
+    // MultiFileGlobalState projection_ids are filled only when this call
+    // returns. Take these from a physical operator.
+    const idx_t *projection_ids = nullptr;
+    size_t projection_ids_count = 0;
+    if (input.op && input.op->type == PhysicalOperatorType::TABLE_SCAN) {
+        const PhysicalTableScan &scan = input.op->Cast<PhysicalTableScan>();
+        if (!scan.projection_ids.empty() && scan.projection_ids.size() != column_ids.size()) {
+            projection_ids = scan.projection_ids.data();
+            projection_ids_count = scan.projection_ids.size();
+        }
+    }
+
     void *const ffi_bind = bind.ffi_bind_data->DataPtr();
     duckdb_vx_tfunc_init_input ffi_input = {
         .bind_data = ffi_bind,
         .column_ids = column_ids.data(),
         .column_ids_count = column_ids.size(),
+        .projection_ids = projection_ids,
+        .projection_ids_count = projection_ids_count,
         .filters = reinterpret_cast<duckdb_vx_table_filter_set>(input.filters.get()),
         .client_context = reinterpret_cast<duckdb_client_context>(&context),
     };

vortex-duckdb/src/convert/expr.rs

@@ -394,6 +394,9 @@ fn can_push_cast(cast: &duckdb::BoundCast<'_>, target: &duckdb::LogicalTypeRef)
 // If we return true here, and expression is in the list for
 // pushdown_complex_filter, we must handle it, or query engine will break.
 //
+// We also don't have access to scan schema at this point, so we're overly
+// restrictive.
+//
 // Example: we don't support substr() expression so we tell Duckdb we can't
 // push it.
 // Example: we support CAST but not TRY_CAST.

vortex-duckdb/src/duckdb/table_init_input.rs

@@ -17,6 +17,7 @@ impl Debug for TableInitInput<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> Result {
         f.debug_struct("TableInitInput")
             .field("column_ids", &self.column_ids())
+            .field("projection_ids", &self.projection_ids())
             .field("table_filter_set", &self.table_filter_set())
             .finish()
     }
@@ -31,6 +32,17 @@ impl<'a> TableInitInput<'a> {
         unsafe { std::slice::from_raw_parts(self.input.column_ids, self.input.column_ids_count) }
     }
 
+    pub fn projection_ids(&self) -> &[u64] {
+        if self.input.projection_ids_count == 0 {
+            // from_raw_parts requires a non-null pointer. C++'s empty vector
+            // may have a null pointer.
+            return &[];
+        }
+        unsafe {
+            std::slice::from_raw_parts(self.input.projection_ids, self.input.projection_ids_count)
+        }
+    }
+
     /// Returns the table filter set for the table function.
     pub fn table_filter_set(&self) -> Option<&TableFilterSetRef> {
         let ptr = self.input.filters;