Duckdb 1.5-variegata

.github/actions/setup-rust/action.yml

@@ -3,9 +3,9 @@ description: "Toolchain setup and Initial compilation"
 
 inputs:
   repo-token:
-    description: "Deprecated: no longer used. Protoc is now downloaded directly from GitHub releases CDN."
+    description: "GitHub token for accessing the repository (typically secrets.GITHUB_TOKEN)"
     required: false
-    default: ""
+    default: "${{ github.token }}"
   toolchain:
     description: "optional override for the toolchain version (e.g. nightly)"
     required: false
@@ -50,9 +50,23 @@ runs:
     - name: Rust Compile Cache
       if: inputs.enable-sccache == 'true'
       uses: mozilla-actions/sccache-action@v0.0.9
-      with:
-        version: "v0.14.0"
 
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
-      uses: ./.github/actions/setup-protoc
+      uses: arduino/setup-protoc@v3
+      with:
+        version: "29.3"
+        repo-token: ${{ inputs.repo-token }}
+
+    - name: Install Ninja (for DuckDB build system)
+      uses: seanmiddleditch/gha-setup-ninja@master
+
+    - name: Install Sweep
+      shell: bash
+      if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
+      run: cargo install cargo-sweep
+
+    - name: Timestamp Cache
+      shell: bash
+      if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
+      run: cargo sweep --stamp

.github/workflows/bench-pr.yml

@@ -54,6 +54,17 @@ jobs:
       - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/bench.yml

@@ -56,6 +56,17 @@ jobs:
         with:
           sccache: s3
       - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -188,6 +188,20 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      # TODO(myrrc): Ideally I'd use a awalsh128/cache-apt-pkgs-action@v1.5.2,
+      # however, it stopped working for some reason. At least we may want to gate
+      # changes under build.rs file changes
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - name: Docs
         run: |
           RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
@@ -287,6 +301,17 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install nightly for fmt
         run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - name: Rust Lint - Format
         run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check
       - name: Rustc check
@@ -412,6 +437,17 @@ jobs:
         uses: taiki-e/install-action@v2
         with:
           tool: nextest
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - name: Rust Tests
         if: ${{ matrix.suite == 'tests' }}
         run: |
@@ -476,10 +512,10 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           toolchain: ${{ env.NIGHTLY_TOOLCHAIN }}
           components: "rust-src, rustfmt, clippy, llvm-tools-preview"
-      - name: Install build dependencies
+      - name: install build dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y ninja-build cmake
+            sudo apt update
+            sudo apt install -y ninja-build cmake libcurl4-gnutls-dev
       - name: Install nextest
         uses: taiki-e/install-action@v2
         with:
@@ -713,6 +749,17 @@ jobs:
         uses: taiki-e/install-action@v2
         with:
           tool: nextest
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: matrix.os != 'windows-x64' && steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - name: Rust Tests (Windows)
         if: matrix.os == 'windows-x64'
         run: |
@@ -871,11 +918,21 @@ jobs:
         uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
       - name: Run sqllogictest tests
         run: |
           cargo test -p vortex-sqllogictest --test sqllogictests
 
-
   wasm-integration:
     name: "wasm-integration"
     runs-on: ubuntu-latest

.github/workflows/sql-benchmarks.yml

@@ -115,6 +115,19 @@ jobs:
 
       - uses: actions/checkout@v6
         if: inputs.mode != 'pr'
+
+      - uses: dorny/paths-filter@v3
+        id: duckdb-buildfile-changed
+        with:
+          filters: |
+            changed:
+              - 'vortex-duckdb/build.rs'
+      - name: install curl dev headers for duckdb source build
+        if: steps.duckdb-buildfile-changed.outputs.changed == 'true'
+        run: |
+            sudo apt update
+            sudo apt install -y libcurl4-gnutls-dev
+
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -12,6 +12,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+ninja-build/
 develop-eggs/
 dist/
 downloads/

vortex-duckdb/build.rs

@@ -29,7 +29,7 @@ static DUCKDB_VERSION: Lazy<DuckDBVersion> = Lazy::new(|| {
         parse_version(&version)
     } else {
         // The default DuckDB version to use when DUCKDB_VERSION env var is not set.
-        DuckDBVersion::Release("1.4.2".to_owned())
+        DuckDBVersion::Commit("7f992e5a96662d66134577cf9d8a5bad9053847e".to_owned())
     }
 });
 
@@ -252,7 +252,15 @@ fn extract_duckdb_source(source_dir: &Path) -> Result<PathBuf, Box<dyn std::erro
 }
 
 /// Build DuckDB from source. Used for commit hashes or when VX_DUCKDB_DEBUG is set.
-fn build_duckdb(duckdb_source_dir: &Path) -> Result<PathBuf, Box<dyn std::error::Error>> {
+fn build_duckdb(
+    duckdb_source_dir: &Path,
+    version: &DuckDBVersion,
+    debug: bool,
+) -> Result<PathBuf, Box<dyn std::error::Error>> {
+    let build_type = match debug {
+        true => "debug",
+        false => "release",
+    };
     // Check for ninja
     if Command::new("ninja").arg("--version").output().is_err() {
         return Err(
@@ -262,10 +270,12 @@ fn build_duckdb(duckdb_source_dir: &Path) -> Result<PathBuf, Box<dyn std::error:
 
     let inner_dir_name = DUCKDB_VERSION.archive_inner_dir_name();
     let duckdb_repo_dir = duckdb_source_dir.join(&inner_dir_name);
-    let build_dir = duckdb_repo_dir.join("build").join("debug");
+    let build_dir = duckdb_repo_dir.join("build").join(build_type);
 
-    // Check if already built
     let lib_dir = build_dir.join("src");
+    let lib_dir_str = lib_dir.display();
+    println!("cargo:info=Checking if DuckDB is already built in {lib_dir_str}",);
+
     let already_built = lib_dir.join("libduckdb.dylib").exists()
         || lib_dir.join("libduckdb.so").exists()
         || lib_dir
@@ -286,12 +296,26 @@ fn build_duckdb(duckdb_source_dir: &Path) -> Result<PathBuf, Box<dyn std::error:
                 ("1", "0")
             };
 
+        let mut envs = vec![
+            ("GEN", "ninja"),
+            ("DISABLE_SANITIZER", asan_option),
+            ("THREADSAN", tsan_option),
+            ("BUILD_SHELL", "false"),
+            ("BUILD_UNITTESTS", "false"),
+            ("ENABLE_UNITTEST_CPP_TESTS", "false"),
+        ];
+
+        // If we're building from a commit (likely a pre-release), we need to
+        // build extensions statically. Otherwise DuckDB tries to load them
+        // from an http endpoint with version 0.0.1 (all non-tagged builds)
+        // which doesn't exists. httpfs also requires CURL dev headers
+        if matches!(version, DuckDBVersion::Commit(_)) {
+            envs.push(("BUILD_EXTENSIONS", "httpfs;parquet;tpch;tpcds;jemalloc"));
+        };
+
         let output = Command::new("make")
             .current_dir(&duckdb_repo_dir)
-            .env("GEN", "ninja")
-            .env("DISABLE_SANITIZER", asan_option)
-            .env("THREADSAN", tsan_option)
-            .arg("debug")
+            .envs(envs)
             .output()?;
 
         if !output.status.success() {
@@ -375,15 +399,21 @@ fn main() {
     drop(fs::remove_dir_all(&duckdb_symlink));
     std::os::unix::fs::symlink(&extracted_source_path, &duckdb_symlink).unwrap();
 
-    // Determine whether to build from source or use prebuilt libraries
     let use_debug_build =
         env::var("VX_DUCKDB_DEBUG").is_ok_and(|v| matches!(v.as_str(), "1" | "true"));
+    println!("cargo:info=DuckDB debug build: {use_debug_build}");
 
     let library_path = if use_debug_build || !DUCKDB_VERSION.is_release() {
         // Build from source for:
         // - Commit hashes (no prebuilt available)
         // - When VX_DUCKDB_DEBUG=1 (user wants debug build)
-        build_duckdb(&extracted_source_path).unwrap()
+        match build_duckdb(&extracted_source_path, &DUCKDB_VERSION, use_debug_build) {
+            Ok(path) => path,
+            Err(err) => {
+                println!("cargo:error={err}");
+                panic!("duckdb build failed");
+            }
+        }
     } else {
         // Download prebuilt libraries for release versions
         let archive_path = download_duckdb_lib_archive().unwrap();
@@ -470,6 +500,7 @@ fn main() {
         .file("cpp/file_system.cpp")
         .file("cpp/logical_type.cpp")
         .file("cpp/object_cache.cpp")
+        .file("cpp/reusable_dict.cpp")
         .file("cpp/replacement_scan.cpp")
         .file("cpp/scalar_function.cpp")
         .file("cpp/table_filter.cpp")

vortex-duckdb/cpp/config.cpp

@@ -40,9 +40,9 @@ duckdb_state duckdb_vx_get_config_value(duckdb_config config, const char *key, d
 
         std::string key_str(key);
 
-        // First check set_variables (the primary location for config values)
-        auto set_it = db_config->options.set_variables.find(key_str);
-        if (set_it != db_config->options.set_variables.end()) {
+        // First check set_variable_defaults (the primary location for config values)
+        auto set_it = db_config->options.set_variable_defaults.find(key_str);
+        if (set_it != db_config->options.set_variable_defaults.end()) {
             *out_value = reinterpret_cast<duckdb_value>(new Value(set_it->second));
             return DuckDBSuccess;
         }
@@ -75,8 +75,9 @@ int duckdb_vx_config_has_key(duckdb_config config, const char *key) {
 
         std::string key_str(key);
 
-        // Check if the key exists in set_variables (primary location)
-        if (db_config->options.set_variables.find(key_str) != db_config->options.set_variables.end()) {
+        // Check if the key exists in set_variable_defaults (primary location)
+        if (db_config->options.set_variable_defaults.find(key_str) !=
+            db_config->options.set_variable_defaults.end()) {
             return 1;
         }
 

vortex-duckdb/cpp/include/duckdb_vx.h

@@ -13,6 +13,7 @@
 #include "duckdb_vx/file_system.h"
 #include "duckdb_vx/logical_type.h"
 #include "duckdb_vx/object_cache.h"
+#include "duckdb_vx/reusable_dict.h"
 #include "duckdb_vx/replacement_scan.h"
 #include "duckdb_vx/scalar_function.h"
 #include "duckdb_vx/table_filter.h"

vortex-duckdb/cpp/include/duckdb_vx/object_cache.h

@@ -20,6 +20,7 @@ typedef void (*duckdb_vx_deleter_fn)(void *ptr);
 void duckdb_vx_object_cache_put(duckdb_vx_object_cache object_cache,
                                 const char *key,
                                 void *value,
+                                uint64_t estimated_size,
                                 duckdb_vx_deleter_fn deleter);
 
 // Fetches the key from the object cache, returning nullptr if the key is not present.

vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#pragma once
+
+#include "duckdb.h"
+#include "duckdb_vx/error.h"
+
+#ifdef __cplusplus /* If compiled as C++, use C ABI */
+extern "C" {
+#endif
+
+typedef struct duckdb_vx_reusable_dict_ *duckdb_vx_reusable_dict;
+
+/// Creates a new reusable dictionary from a logical type and size.
+/// The returned dictionary can be used with duckdb_vx_vector_dictionary_reusable.
+duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type logical_type, idx_t size);
+
+/// Destroys the reusable dictionary.
+void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict);
+
+/// Clones the reusable dictionary.
+duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict);
+
+/// Get the internal vector of the reusable dictionary.
+void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, duckdb_vector *out_vector);
+
+/// Creates a dictionary vector using a reusable dictionary and a selection vector.
+void duckdb_vx_vector_dictionary_reusable(duckdb_vector vector,
+                                          duckdb_vx_reusable_dict reusable,
+                                          duckdb_selection_vector sel_vec);
+
+#ifdef __cplusplus /* End C ABI */
+}
+#endif