6627

myrrc · myrrc · commit 11ada49d2283 · 2026-02-23T11:24:50.000Z
diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml
@@ -3,9 +3,9 @@ description: "Toolchain setup and Initial compilation"
 
 inputs:
   repo-token:
-    description: "Deprecated: no longer used. Protoc is now downloaded directly from GitHub releases CDN."
+    description: "GitHub token for accessing the repository (typically secrets.GITHUB_TOKEN)"
     required: false
-    default: ""
+    default: "${{ github.token }}"
   toolchain:
     description: "optional override for the toolchain version (e.g. nightly)"
     required: false
@@ -50,9 +50,23 @@ runs:
     - name: Rust Compile Cache
       if: inputs.enable-sccache == 'true'
       uses: mozilla-actions/sccache-action@v0.0.9
-      with:
-        version: "v0.14.0"
 
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
-      uses: ./.github/actions/setup-protoc
+      uses: arduino/setup-protoc@v3
+      with:
+        version: "29.3"
+        repo-token: ${{ inputs.repo-token }}
+
+    - name: Install Ninja (for DuckDB build system)
+      uses: seanmiddleditch/gha-setup-ninja@master
+
+    - name: Install Sweep
+      shell: bash
+      if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
+      run: cargo install cargo-sweep
+
+    - name: Timestamp Cache
+      shell: bash
+      if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
+      run: cargo sweep --stamp
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+ninja-build/
 develop-eggs/
 dist/
 downloads/
diff --git a/benchmarks/duckdb-bench/src/lib.rs b/benchmarks/duckdb-bench/src/lib.rs
@@ -96,6 +96,14 @@ impl DuckClient {
         // parquet_metadata_cache" when running DuckDB in debug mode.
         connection.query("SET parquet_metadata_cache = true")?;
 
+        // Try to install/load httpfs for remote data access, but don't fail if unavailable
+        // (e.g., when using local files or when extension download fails)
+        //if let Err(e) = connection.query("INSTALL httpfs; LOAD httpfs;") {
+        //    tracing::info!(
+        //        "Failed to load httpfs extension (may not be needed for local files): {e}"
+        //    );
+        //}
+
         Ok((db, connection))
     }
 
diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs
@@ -465,6 +465,7 @@ fn main() {
         .file("cpp/file_system.cpp")
         .file("cpp/logical_type.cpp")
         .file("cpp/object_cache.cpp")
+        .file("cpp/reusable_dict.cpp")
         .file("cpp/replacement_scan.cpp")
         .file("cpp/scalar_function.cpp")
         .file("cpp/table_filter.cpp")
diff --git a/vortex-duckdb/cpp/config.cpp b/vortex-duckdb/cpp/config.cpp
@@ -40,9 +40,9 @@ duckdb_state duckdb_vx_get_config_value(duckdb_config config, const char *key, d
 
         std::string key_str(key);
 
-        // First check set_variables (the primary location for config values)
-        auto set_it = db_config->options.set_variables.find(key_str);
-        if (set_it != db_config->options.set_variables.end()) {
+        // First check set_variable_defaults (the primary location for config values)
+        auto set_it = db_config->options.set_variable_defaults.find(key_str);
+        if (set_it != db_config->options.set_variable_defaults.end()) {
             *out_value = reinterpret_cast<duckdb_value>(new Value(set_it->second));
             return DuckDBSuccess;
         }
@@ -75,8 +75,9 @@ int duckdb_vx_config_has_key(duckdb_config config, const char *key) {
 
         std::string key_str(key);
 
-        // Check if the key exists in set_variables (primary location)
-        if (db_config->options.set_variables.find(key_str) != db_config->options.set_variables.end()) {
+        // Check if the key exists in set_variable_defaults (primary location)
+        if (db_config->options.set_variable_defaults.find(key_str) !=
+            db_config->options.set_variable_defaults.end()) {
             return 1;
         }
 
diff --git a/vortex-duckdb/cpp/include/duckdb_vx.h b/vortex-duckdb/cpp/include/duckdb_vx.h
@@ -10,11 +10,12 @@
 #include "duckdb_vx/data_chunk.h"
 #include "duckdb_vx/error.h"
 #include "duckdb_vx/expr.h"
-#include "duckdb_vx/file_system.h"
 #include "duckdb_vx/logical_type.h"
 #include "duckdb_vx/object_cache.h"
+#include "duckdb_vx/reusable_dict.h"
 #include "duckdb_vx/replacement_scan.h"
 #include "duckdb_vx/scalar_function.h"
+#include "duckdb_vx/file_system.h"
 #include "duckdb_vx/table_filter.h"
 #include "duckdb_vx/table_function.h"
 #include "duckdb_vx/value.h"
diff --git a/vortex-duckdb/cpp/include/duckdb_vx/object_cache.h b/vortex-duckdb/cpp/include/duckdb_vx/object_cache.h
@@ -20,6 +20,7 @@ typedef void (*duckdb_vx_deleter_fn)(void *ptr);
 void duckdb_vx_object_cache_put(duckdb_vx_object_cache object_cache,
                                 const char *key,
                                 void *value,
+                                uint64_t estimated_size,
                                 duckdb_vx_deleter_fn deleter);
 
 // Fetches the key from the object cache, returning nullptr if the key is not present.
diff --git a/vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h b/vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#pragma once
+
+#include "duckdb.h"
+#include "duckdb_vx/error.h"
+
+#ifdef __cplusplus /* If compiled as C++, use C ABI */
+extern "C" {
+#endif
+
+typedef struct duckdb_vx_reusable_dict_ *duckdb_vx_reusable_dict;
+
+/// Creates a new reusable dictionary from a logical type and size.
+/// The returned dictionary can be used with duckdb_vx_vector_dictionary_reusable.
+duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type logical_type, idx_t size);
+
+/// Destroys the reusable dictionary.
+void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict);
+
+/// Clones the reusable dictionary.
+duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict);
+
+/// Get the internal vector of the reusable dictionary.
+void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, duckdb_vector *out_vector);
+
+/// Creates a dictionary vector using a reusable dictionary and a selection vector.
+void duckdb_vx_vector_dictionary_reusable(duckdb_vector vector,
+                                          duckdb_vx_reusable_dict reusable,
+                                          duckdb_selection_vector sel_vec);
+
+#ifdef __cplusplus /* End C ABI */
+}
+#endif
diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_filter.h b/vortex-duckdb/cpp/include/duckdb_vx/table_filter.h
@@ -20,19 +20,16 @@ typedef enum DUCKDB_VX_TABLE_FILTER_TYPE {
     DUCKDB_VX_TABLE_FILTER_TYPE_OPTIONAL_FILTER = 6, // executing filter is not required for query correctness
     DUCKDB_VX_TABLE_FILTER_TYPE_IN_FILTER = 7,       // col IN (C1, C2, C3, ...)
     DUCKDB_VX_TABLE_FILTER_TYPE_DYNAMIC_FILTER = 8,  // dynamic filters can be updated at run-time
-    DUCKDB_VX_TABLE_FILTER_TYPE_EXPRESSION_FILTER = 9 // an arbitrary expression
+    DUCKDB_VX_TABLE_FILTER_TYPE_EXPRESSION_FILTER = 9, // an arbitrary expression
+    DUCKDB_VX_TABLE_FILTER_TYPE_BLOOM_FILTER =
+        10 // a probabilistic filter that can test whether a value is in a set of other value
 } duckdb_vx_table_filter_type;
 
 typedef struct duckdb_vx_table_filter_set_ *duckdb_vx_table_filter_set;
 
 typedef struct duckdb_vx_table_filter_ *duckdb_vx_table_filter;
 
-/// If a table filter with position idx exists, return its column index,
-/// assign the filter to table_filter_out.
-/// If such filter doesn't exist, set table_filter_out to nullptr and return 0
-/// idx is a position in a filter array sorted by increasing column indices.
-///
-/// TODO(myrrc) is idx really filter's position or it's a column's idx?
+/// Returns the column index associated with idx, if `table_filter_out` is null then the idx is out of bounds.
 idx_t duckdb_vx_table_filter_set_get(duckdb_vx_table_filter_set filter_set,
                                      size_t idx,
                                      duckdb_vx_table_filter *table_filter_out);
diff --git a/vortex-duckdb/cpp/object_cache.cpp b/vortex-duckdb/cpp/object_cache.cpp
@@ -12,11 +12,17 @@ namespace vortex {
 class OpaqueWrapper : public duckdb::ObjectCacheEntry {
 public:
     duckdb::unique_ptr<void, duckdb_vx_deleter_fn> ptr;
+    duckdb::optional_idx estimated_size;
 
-    explicit OpaqueWrapper(void *p, duckdb_vx_deleter_fn del) : ptr(p, del) {
+    explicit OpaqueWrapper(void *p, duckdb::optional_idx estimated_size, duckdb_vx_deleter_fn del)
+        : ptr(p, del), estimated_size(estimated_size) {
     }
     ~OpaqueWrapper() override = default;
 
+    duckdb::optional_idx GetEstimatedCacheMemory() const override {
+        return estimated_size;
+    }
+
     duckdb::string GetObjectType() override {
         return "vortex_opaque_wrapper";
     }
@@ -32,9 +38,11 @@ class OpaqueWrapper : public duckdb::ObjectCacheEntry {
 extern "C" void duckdb_vx_object_cache_put(duckdb_vx_object_cache cache,
                                            const char *key,
                                            void *value,
+                                           uint64_t estimated_size,
                                            duckdb_vx_deleter_fn deleter) {
     auto object_cache = reinterpret_cast<duckdb::ObjectCache *>(cache);
-    auto wrapper = duckdb::make_shared_ptr<vortex::OpaqueWrapper>(value, deleter);
+    auto wrapper =
+        duckdb::make_shared_ptr<vortex::OpaqueWrapper>(value, duckdb::optional_idx(estimated_size), deleter);
     object_cache->Put(std::string(key), wrapper);
 }
 
diff --git a/vortex-duckdb/cpp/reusable_dict.cpp b/vortex-duckdb/cpp/reusable_dict.cpp
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#include "duckdb/common/types/vector.hpp"
+
+#include "duckdb_vx.h"
+
+using namespace duckdb;
+
+namespace vortex {
+// Wrapper struct to hold the buffer_ptr<VectorChildBuffer>
+struct ReusableDict {
+    buffer_ptr<VectorChildBuffer> buffer;
+
+    explicit ReusableDict(buffer_ptr<VectorChildBuffer> buf) : buffer(std::move(buf)) {
+    }
+};
+} // namespace vortex
+
+extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type ffi_type, idx_t size) {
+    auto type = reinterpret_cast<duckdb::LogicalType *>(ffi_type);
+    auto buffer = DictionaryVector::CreateReusableDictionary(*type, size);
+    auto *wrapper = new vortex::ReusableDict(std::move(buffer));
+    return reinterpret_cast<duckdb_vx_reusable_dict>(wrapper);
+}
+
+extern "C" void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict) {
+    if (dict != nullptr && *dict != nullptr) {
+        auto *wrapper = reinterpret_cast<vortex::ReusableDict *>(*dict);
+        delete wrapper;
+        *dict = nullptr;
+    }
+}
+
+extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict) {
+    auto *wrapper = reinterpret_cast<vortex::ReusableDict *>(dict);
+    auto *cloned = new vortex::ReusableDict(wrapper->buffer);
+    return reinterpret_cast<duckdb_vx_reusable_dict>(cloned);
+}
+
+extern "C" void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable,
+                                                   duckdb_vector *out_vector) {
+    auto *wrapper = reinterpret_cast<vortex::ReusableDict *>(reusable);
+    *out_vector = reinterpret_cast<duckdb_vector>(&wrapper->buffer->data);
+}
+
+extern "C" void duckdb_vx_vector_dictionary_reusable(duckdb_vector ffi_vector,
+                                                     duckdb_vx_reusable_dict reusable,
+                                                     duckdb_selection_vector ffi_sel_vec) {
+    auto vector = reinterpret_cast<Vector *>(ffi_vector);
+    auto *wrapper = reinterpret_cast<vortex::ReusableDict *>(reusable);
+    auto sel_vec = reinterpret_cast<SelectionVector *>(ffi_sel_vec);
+    vector->Dictionary(wrapper->buffer, *sel_vec);
+}
diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp
@@ -30,8 +30,8 @@ extern "C" void duckdb_vx_vector_dictionary(duckdb_vector ffi_vector,
 }
 
 extern "C" void duckdb_vx_set_dictionary_vector_id(duckdb_vector dict, const char *id, unsigned int id_len) {
-    auto ddict = reinterpret_cast<duckdb::Vector *>(dict);
-    DictionaryVector::SetDictionaryId(*ddict, std::string(id, id_len));
+    // auto ddict = reinterpret_cast<duckdb::Vector *>(dict);
+    // DictionaryVector::SetDictionaryId(*ddict, std::string(id, id_len));
 }
 
 extern "C" void duckdb_vx_set_dictionary_vector_length(duckdb_vector dict, unsigned int len) {
diff --git a/vortex-duckdb/include/vortex.h b/vortex-duckdb/include/vortex.h
@@ -14,8 +14,6 @@ extern "C" {
 #include "duckdb.h"
 
 
-#define DUCKDB_STANDARD_VECTOR_SIZE 2048
-
 /**
  * Global symbol visibility in the Vortex extension:
  * - Rust functions use C ABI with "_rust" suffix (e.g., vortex_init_rust)
diff --git a/vortex-duckdb/src/convert/table_filter.rs b/vortex-duckdb/src/convert/table_filter.rs
@@ -120,5 +120,8 @@ pub fn try_from_table_filter(
             // TODO(ngates): figure out which column ID DuckDB is using for the expression.
             vortex_bail!("expression table filter is not supported: {}", expr);
         }
+        TableFilterClass::Bloom => {
+            vortex_bail!("bloom filter table filter is not supported")
+        }
     }))
 }
diff --git a/vortex-duckdb/src/duckdb/mod.rs b/vortex-duckdb/src/duckdb/mod.rs
@@ -16,6 +16,7 @@ mod logical_type;
 mod macro_;
 mod object_cache;
 mod query_result;
+mod reusable_dict;
 mod scalar_function;
 mod selection_vector;
 mod table_filter;
@@ -40,6 +41,7 @@ pub use file_system::*;
 pub use logical_type::*;
 pub use object_cache::*;
 pub use query_result::*;
+pub use reusable_dict::*;
 pub use scalar_function::*;
 pub use selection_vector::*;
 pub use table_filter::*;
diff --git a/vortex-duckdb/src/duckdb/object_cache.rs b/vortex-duckdb/src/duckdb/object_cache.rs
@@ -45,11 +45,16 @@ impl ObjectCacheRef {
             .vortex_expect("object cache key should be valid C string");
         let opaque_ptr = Box::into_raw(Box::new(entry));
 
+        // Pass 0 to allow eviction by DuckDB, u64::MAX to prevent eviction, otherwise provide an
+        // estimate of the size of the object in bytes.
+        let estimated_size: u64 = 0;
+
         unsafe {
             cpp::duckdb_vx_object_cache_put(
                 self.as_ptr(),
                 key_cstr.as_ptr(),
-                opaque_ptr.cast(),
+                opaque_ptr as *mut c_void,
+                estimated_size,
                 Some(rust_box_deleter::<T>),
             );
         }
diff --git a/vortex-duckdb/src/duckdb/reusable_dict.rs b/vortex-duckdb/src/duckdb/reusable_dict.rs
diff --git a/vortex-duckdb/src/duckdb/table_filter.rs b/vortex-duckdb/src/duckdb/table_filter.rs
diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs
diff --git a/vortex-duckdb/src/exporter/cache.rs b/vortex-duckdb/src/exporter/cache.rs
diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs
diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs
diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs
