diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml
index ee50a440..437d9608 100644
--- a/.cargo/mutants.toml
+++ b/.cargo/mutants.toml
@@ -44,7 +44,10 @@ features = ["duckdb-1-5-4"]
# Thin re-export modules and test-only code produce unproductive mutants.
exclude_globs = [
"src/prelude.rs", # pure re-exports
- "src/callback.rs", # macro-generated FFI wrappers (untestable without DuckDB runtime)
+ # src/callback.rs is not excluded: the macros in it are invisible to
+ # cargo-mutants, but catch_ffi_panic, panic_message, panic_c_message and
+ # message_to_c_string are plain functions. `cargo mutants --no-config
+ # --file src/callback.rs --cargo-arg=--lib`: 5 mutants, 4 caught, 1 unviable.
"src/chunk_writer.rs", # Drop calls FFI; row tracking tested via unit tests
"src/vector/struct_writer.rs", # thin delegation to VectorWriter (requires DuckDB runtime)
"src/vector/struct_reader.rs", # thin delegation to VectorReader (requires DuckDB runtime)
@@ -225,6 +228,15 @@ exclude_re = [
# with the one asked for, which needs duckdb_create_logical_type; with the
# comparison inverted, every type creation fails end to end.
"src/types/logical_type/construct.rs:.*replace != with == in LogicalType::try_new",
+ # construct.rs: LogicalType::refuse_where_the_engine_lacks asks the engine for
+ # its version (abi::engine_version reads the dispatch table), so a --lib run
+ # with no engine cannot reach the TIMESTAMPTZ_NS refusal. Both its mutants
+ # (the body replaced with Ok(()), and `==` to `!=`) were run by hand against
+ # libduckdb 1.5.6: tests/ffi_roundtrip/timestamp_tz_ns.rs
+ # `the_type_id_is_reported_for_a_column_and_creatable_only_where_it_exists`
+ # passes unmutated and fails with each. The pure comparison it uses,
+ # predates_timestamp_tz_ns, stays in the gate.
+ "src/types/logical_type/construct.rs:.*LogicalType::refuse_where_the_engine_lacks",
# error_data.rs: a non-null ErrorData only comes from DuckDB, and
# is_null / has_error / message / Drop / check_valid_utf8 are only told
# apart from their mutants with one (duckdb_error_data_* /
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 00000000..90120276
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# https://editorconfig.org -- the whitespace rules the repository already follows
+# (`.gitattributes` fixes LF; no tracked .rs/.toml/.md/.yml/.py/.sh file lacks a
+# final newline or has trailing whitespace in Rust), written down so an editor
+# without rustfmt on save does not drift.
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.sh]
+indent_size = 2
+
+# Markdown keeps trailing double-space line breaks and mixed list indents.
+[*.md]
+trim_trailing_whitespace = false
+indent_size = unset
+
+# Makefiles (and the Makefile text generated by the scaffold) need tabs.
+[{Makefile,*.mk}]
+indent_style = tab
+
+[*.{png,bin}]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+indent_size = unset
+
+# Fuzz seeds must reach the fuzzer byte for byte (.gitattributes: binary).
+[fuzz/seeds/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+indent_size = unset
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index e7c2c343..63c37cd0 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -36,6 +36,14 @@ updates:
- "minor"
# ── Cargo dependencies for the hello-ext example ─────────────────────────────
+ # No duckdb / libduckdb-sys ignore here, on purpose. The root lockfile is held
+ # on the DuckDB 1.5.6 bindings for the 1.86 MSRV (above); the example is not
+ # bound by that and tracks the newest bindings (DuckDB 2.0's, 1.20000.x, as
+ # of 0.20.0). It is the one place CI builds them into a real cdylib and loads
+ # it into every pinned DuckDB (example-check, symbol-check, extension-load),
+ # so a bindings PR here is the first signal of a new DuckDB release, tested
+ # by those jobs on the PR itself. Its committed lockfile needs Rust 1.88
+ # (examples/hello-ext/README.md says so).
- package-ecosystem: cargo
directory: "/examples/hello-ext"
schedule:
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 6fd9c04d..022162b7 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -18,6 +18,7 @@
### Code quality
- [ ] `cargo test --all-targets` passes, and `cargo test --doc --features duckdb-1-5-4` (doctests are not in `--all-targets`)
- [ ] The end-to-end tests pass against a real libduckdb (CONTRIBUTING, gate 2c) when the change touches `unsafe`, vectors, or function registration
+- [ ] `cargo test --doc --manifest-path book/doctest/Cargo.toml` passes when the change touches `book/src/` or `README.md` (every Rust block there is compiled; gate 2c)
- [ ] `cargo clippy --all-targets -- -D warnings` passes
- [ ] `cargo fmt` applied (no diff)
- [ ] `cargo doc --no-deps` builds without warnings
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index e2bca71e..f7e2be6e 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -10,15 +10,15 @@ This directory contains all GitHub Actions workflows for the quack-rs project.
| Workflow | File | Trigger | Purpose |
|----------|------|---------|---------|
| **CI** | `ci.yml` | Push to `main`/`claude/**`, PRs to `main`, weekly schedule, manual, called by `release.yml` | Every quality gate (table below), summarised by the `ci-ok` job |
-| **Release** | `release.yml` | Semver tags (`vX.Y.Z`, and `vX.Y.Z-pre` prereleases) | Full release pipeline: validate the tag, run all of `ci.yml`, package with SLSA attestation, publish to crates.io (approval required), check crates.io serves the attested `.crate`, GitHub release, versioned-book deploy |
+| **Release** | `release.yml` | Semver tags (`vX.Y.Z`, and `vX.Y.Z-pre` prereleases) | Full release pipeline: validate the tag and the release metadata, run all of `ci.yml`, package with SLSA attestation, publish to crates.io (approval required; the `.crate` is re-packaged and compared with the attested hash before the upload), check crates.io serves the attested `.crate`, GitHub release, versioned-book deploy |
| **Documentation** | `docs.yml` | Push to `main`, manual, dispatched by `release.yml` after a release | Build the versioned book (latest release at `/`, each release line at `/vX.Y/`, `main` at `/dev/`) and deploy to GitHub Pages |
| **Coverage** | `coverage.yml` | Push to `main`, PRs to `main` | Generate LCOV coverage report and upload to Codecov |
| **Mutation Testing** | `mutants.yml` | Manual dispatch, PRs to `main` | Verify tests detect code changes via cargo-mutants |
-| **Benchmarks** | `benchmarks.yml` | Push to `main` (bench/src changes), manual | Run criterion benchmarks and archive reports |
+| **Benchmarks** | `benchmarks.yml` | Push to `main` (changes under `benches/` or `src/`, or to `Cargo.toml` / `Cargo.lock`), manual | Run criterion benchmarks and archive reports |
## Quality gates (enforced by CI)
-Every job in `ci.yml` must be green before merging a PR, except the three marked
+Every job in `ci.yml` must be green before merging a PR, except the four marked
informational. This table is generated from `ci.yml`; the hand-written list it
replaced had drifted to fewer than half the jobs.
@@ -35,6 +35,7 @@ replaced had drifted to fewer than half the jobs.
| `wasm` | WASM (wasm32-unknown-emscripten) | blocking |
| `clippy` | Clippy | blocking |
| `clippy-beta` | Clippy (beta, informational) | informational |
+| `latest-stable` | Latest stable Rust (informational) | informational |
| `fmt` | Format | blocking |
| `doc` | Documentation | blocking |
| `msrv` | MSRV (1.86.0) | blocking |
@@ -56,12 +57,14 @@ replaced had drifted to fewer than half the jobs.
| `publish-dry-run` | Publish dry-run | blocking |
| `security` | Security (cargo-deny) | blocking |
| `osv-scan` | Security (OSV / GHSA) | blocking |
+| `hygiene` | Hygiene (typos, taplo, machete) | blocking |
| `nightly` | Nightly (informational) | informational |
| `miri` | Miri (undefined behaviour) | blocking |
| `leak-check` | LeakSanitizer (RAII wrappers vs a real DuckDB) | blocking |
| `asan` | AddressSanitizer | blocking |
| `semver` | Public API (semver-checks) | blocking |
| `fuzz` | Fuzz (smoke) | blocking |
+| `workflow-lint` | Workflow lint (actionlint) | blocking |
| `ci-ok` | CI passed | blocking (aggregate) |
**Branch protection should require exactly one check: `CI passed`** (the
@@ -69,7 +72,10 @@ replaced had drifted to fewer than half the jobs.
it fails when any of them fails or is cancelled.
`scripts/check-ci-aggregate.py` (run by the `doc` job) fails when a blocking
job is missing from its `needs`, so a new job cannot drop out of the required
-set unnoticed. `release.yml` runs this whole workflow as its CI gate
+set unnoticed; it also fails when a job turns informational without being
+named in its `INFORMATIONAL` set, when a blocking job (or `ci-ok` itself) has a
+step-level `continue-on-error`, and when the table above disagrees with
+`ci.yml`. `release.yml` runs this whole workflow as its CI gate
(`workflow_call`), so a release passes the same checks.
Regenerate after adding or renaming a job:
@@ -108,14 +114,41 @@ Neither `yaml.safe_load` nor a JSON-Schema validator catches it, because both
treat the `run:` block as an opaque string. `scripts/check-workflow-expressions.py`
runs in the `doc` job and does.
-`actionlint` also catches it and catches much more besides. It is not wired in
-here only because it ships as a Go binary from GitHub releases and cannot be
-checksum-pinned the way `osv-scanner` is in `ci.yml`. If a pinned, verifiable
-install becomes available, adding it would be a strict improvement.
+`actionlint` also catches it and catches much more besides; the `workflow-lint`
+job runs it from the release tarball, pinned by version and sha256 the way
+`osv-scanner` is (the release publishes `actionlint__checksums.txt`).
+
+## Pinning, and the checks that hold it
+
+Everything CI downloads is pinned, and a script in the `doc` job keeps each
+kind of pin consistent:
+
+| What | Pinned by | Checked by |
+|---|---|---|
+| Actions | full commit SHA, version in the trailing comment (`# v2.9.3`, or `# master` for `dtolnay/rust-toolchain`, which has no release tags) | `scripts/check-action-pins.py`: one commit per action across all workflows, and the same commits in the workflows the scaffold generates (`src/scaffold/templates_ci.rs`), which Dependabot does not update |
+| Runner images | `ubuntu-24.04`, `macos-26`, `windows-2025`, never `*-latest` (GitHub moves those labels; each pinned label was the `-latest` image on 2026-10-30) | `scripts/check-action-pins.py`: a floating label in any workflow fails |
+| Rust | an exact release in `rust-toolchain.toml`, repeated in every `toolchain:` input | `scripts/check-toolchain-pin.py`: `stable` / `beta` only in informational jobs; MSRV and nightly where they are the point |
+| Release binaries (osv-scanner, actionlint, typos, taplo, cargo-machete) | version and SHA-256 in the job | the job itself (`sha256sum -c`) |
+| Cargo-installed tools (cargo-deny, cargo-semver-checks, cargo-fuzz, mdBook, ...) | `cargo install --locked --version` | — |
+
+The five jobs that compare a checked-in table with upstream (`abi-table`, which
+has two such steps, `platform-table`, `spdx-list`, `autoload-entries`,
+`msrv-vs-duckdb-ci`) run each script through `scripts/upstream-guard.sh`:
+exit 1 (drift) fails; exit 2
+("could not check": upstream unreachable, an HTTP 5xx or 429) is a warning on a
+pull request or a push and a failure on a release tag and on the weekly
+schedule, where nothing may be skipped; a missing script fails. The scripts
+return 1, not 2, for an HTTP 4xx and for a file that no longer has the
+expected shape: upstream moved something, and a job that degraded to "check
+skipped" for good would never say so. `upstream-guard.sh --self-test` runs in
+the `doc` job.
## Adding a new workflow
1. Create a new `.yml` file in this directory.
-2. Pin all third-party actions to their full commit SHA (not a tag).
-3. Add SPDX license header.
-4. Update this README.
+2. Pin all third-party actions to their full commit SHA (not a tag), at the
+ commit the other workflows use (`scripts/check-action-pins.py`).
+3. Install Rust with `toolchain:` set to the release `rust-toolchain.toml`
+ pins (`scripts/check-toolchain-pin.py`).
+4. Add SPDX license header.
+5. Update this README.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 13830e7d..aadda49a 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -32,17 +32,17 @@ permissions:
jobs:
bench:
name: Run Benchmarks
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Build benchmarks
run: cargo bench --no-run
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 453d28e4..1b2aea8d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,21 +5,37 @@
#
# Jobs: check, test, test-bundled, test-bundled-prebuilt, test-older-engines,
# test-duckdb-2-0, test-bundled-duckdb-2-0, test-duckdb-1-5,
-# wasm, clippy, clippy-beta, fmt, doc, msrv, dependency-floor,
-# bench-compile, example-check, scaffold-compile, symbol-check,
-# abi-table, platform-table, spdx-list, autoload-entries, book,
-# msrv-vs-duckdb-ci, extension-load, scaffold-e2e, abi-guard,
-# abi-guard-layout, publish-dry-run, security, osv-scan, nightly, miri,
-# leak-check, asan, semver, fuzz, ci-ok (the aggregate required check).
+# wasm, clippy, clippy-beta, latest-stable, fmt, doc, msrv,
+# dependency-floor, bench-compile, example-check, scaffold-compile,
+# symbol-check, abi-table, platform-table, spdx-list, autoload-entries,
+# book, msrv-vs-duckdb-ci, extension-load, scaffold-e2e, abi-guard,
+# abi-guard-layout, publish-dry-run, security, osv-scan, hygiene, nightly,
+# miri, leak-check, asan, semver, fuzz, workflow-lint, ci-ok (the
+# aggregate required check).
#
-# `dtolnay/rust-toolchain` is pinned, in every workflow, to e2a55d2 -- a commit
-# on the action's `master` branch (2026-10-08; it moved action inputs into
-# environment variables, upstream #187). The action's per-toolchain
-# branches (`stable`, `nightly`, ...) are regenerated, so a commit taken from
-# one of them stops being reachable from any ref and can be garbage-collected;
-# the previous pin, 631a55b, was such a commit. On `master` the `toolchain:`
-# input is required, so every use states it. The pin fixes the action's code,
-# not the Rust version: `toolchain: stable` still installs the current stable.
+# `dtolnay/rust-toolchain` is pinned, in every workflow, to one commit on the
+# action's `master` branch (commented `# master`: the branch publishes no
+# release tags, and Dependabot moves the pin to master's newest commit; a date
+# in the comment went stale with the first such bump). Every pin since e2a55d2
+# (2026-10-08) passes the action's inputs through environment variables
+# (upstream #187). The action's per-toolchain branches (`stable`, `nightly`,
+# ...) are regenerated, so a commit taken from one of them stops being
+# reachable from any ref and can be garbage-collected; the pin before e2a55d2,
+# 631a55b, was such a commit. On `master` the `toolchain:` input is required,
+# so every use states it. The pin fixes the action's code, not the Rust version.
+# scripts/check-action-pins.py (the `doc` job) keeps every workflow, and the
+# workflows the scaffold generates, on the same commit of each action.
+#
+# The Rust version is pinned separately: rust-toolchain.toml names an exact
+# stable release, and every job that is not about another toolchain installs
+# that one (`toolchain: 1.99.1`). With `-D warnings` and clippy's pedantic and
+# nursery groups, a floating `stable` let each Rust release (every six weeks)
+# change rustc, rustfmt and clippy under every job at once, with no change
+# here. scripts/check-toolchain-pin.py (the `doc` job) fails when a workflow
+# names another toolchain outside the informational jobs. Those see the next
+# releases first: `latest-stable` (the current stable), `clippy-beta`,
+# `nightly`. The pin is moved by a pull request (CONTRIBUTING.md, "The Rust
+# toolchain").
name: CI
@@ -60,6 +76,10 @@ concurrency:
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
+ # On a tag (a release, which reaches this file through workflow_call) the
+ # upstream-comparison guards (scripts/upstream-guard.sh) fail instead of
+ # skipping when upstream cannot be reached.
+ QUACK_STRICT_UPSTREAM: ${{ github.ref_type == 'tag' && '1' || '' }}
RUSTFLAGS: "-D warnings"
RUSTDOCFLAGS: "-D warnings"
# Harden against transient crates.io/CDN download flakes — notably
@@ -72,16 +92,16 @@ env:
jobs:
check:
name: Check
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- run: cargo check --all-targets
test:
@@ -91,16 +111,29 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
+ os: [ubuntu-24.04, macos-26, windows-2025]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
- with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ # The scaffold's tests cross-check the files it generates with PyYAML (the
+ # parser the community build reads description.yml with), make and git,
+ # and skip when one is missing; `cargo test` hides the note, so a runner
+ # without PyYAML kept them green while checking nothing. On Linux the tools
+ # are installed and a missing one fails the test (QUACK_REQUIRE_TOOLS=1);
+ # macOS and Windows runners keep the skip.
+ - name: Install the tools the scaffold's tests cross-check with
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y --no-install-recommends python3-yaml make git
- run: cargo test --all-targets
+ env:
+ QUACK_REQUIRE_TOOLS: ${{ runner.os == 'Linux' && '1' || '' }}
# Every other job builds tests in debug or with the repository's release
# profile (one codegen unit, fat LTO). A user's crate usually has Cargo's
# default release profile: 16 codegen units, no fat LTO, where rustc gives
@@ -127,15 +160,15 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
+ os: [ubuntu-24.04, macos-26, windows-2025]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# --no-fail-fast: cargo otherwise stops at the first failing test
# binary, hiding failures in the ones after it.
- run: cargo test --all-targets --features bundled-test --no-fail-fast
@@ -163,16 +196,16 @@ jobs:
# Linux only — the pre-built release archive is published per-platform.
test-bundled-prebuilt:
name: Test bundled-test-prebuilt (prebuilt libduckdb)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Download pre-built libduckdb
# The version is DERIVED from Cargo.lock, not written here. It used to
# be hard-coded with a comment asking the next person to keep it in
@@ -233,7 +266,7 @@ jobs:
# until the suite ran against the oldest release of each range, which
# this job now does on every run.
name: Test against DuckDB ${{ matrix.engine }} (${{ matrix.features }})
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
fail-fast: false
@@ -264,10 +297,10 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Pin the bindings to the engine and download its library
# The test shim refuses a library whose duckdb_ext_api_v1 size differs
# from the bindings', so both must be the same release.
@@ -304,16 +337,16 @@ jobs:
# bindings and nothing else covers on the 2.0 ones: clippy, rustdoc and the
# doctests. (`test-bundled-duckdb-2-0` covers the source-build path.)
name: Test against DuckDB v2.0.0
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Pin the DuckDB 2.0 bindings and download libduckdb v2.0.0
run: |
set -euo pipefail
@@ -347,17 +380,25 @@ jobs:
# quack-rs.
test-bundled-duckdb-2-0:
name: Test bundled-test on the DuckDB 2.0 bindings (informational)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
continue-on-error: true
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
- with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ with:
+ # The 1.20000 bindings unpack DuckDB's sources into target/duckdb-sources/,
+ # outside the directories the cache keeps for a build script; a restored
+ # target/ then holds the build record without the sources, and the build
+ # script (run 705, run 711) stops with "neither duckdb_extension.h nor
+ # duckdb.hpp is there". Only the registry is cached, so this builds DuckDB
+ # from scratch each time.
+ cache-targets: "false"
- name: Pin the DuckDB 2.0 bindings
run: |
set -euo pipefail
@@ -369,16 +410,16 @@ jobs:
test-duckdb-1-5:
name: Test duckdb-1-5 / duckdb-1-5-3 / duckdb-1-5-4 features
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- run: cargo check --all-targets --features duckdb-1-5
- run: cargo test --all-targets --features duckdb-1-5
- run: cargo clippy --all-targets --features duckdb-1-5 -- -D warnings
@@ -391,17 +432,17 @@ jobs:
wasm:
name: WASM (wasm32-unknown-emscripten)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
targets: wasm32-unknown-emscripten
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# DuckDB-WASM target. quack-rs's own lib must compile for wasm32; in
# loadable-extension mode libduckdb-sys is headers-only, so `cargo check`
# needs no emscripten toolchain. `--all-targets` is intentionally avoided —
@@ -427,7 +468,10 @@ jobs:
- name: Unit tests on wasm32 (node)
env:
CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER: node
- RUSTFLAGS: "-D warnings -C link-arg=-sALLOW_MEMORY_GROWTH=1 -C link-arg=-sMAXIMUM_MEMORY=4GB"
+ # STACK_SIZE: emscripten's default stack is 64 KB, and the description.yml
+ # reader's depth test (64 nested levels, debug build) ran out of it with
+ # "memory access out of bounds".
+ RUSTFLAGS: "-D warnings -C link-arg=-sALLOW_MEMORY_GROWTH=1 -C link-arg=-sMAXIMUM_MEMORY=4GB -C link-arg=-sSTACK_SIZE=8MB"
run: |
set -euo pipefail
source "$RUNNER_TEMP/emsdk/emsdk_env.sh"
@@ -436,17 +480,17 @@ jobs:
clippy:
name: Clippy
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
components: clippy
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- run: cargo clippy --all-targets -- -D warnings
# Informational early-warning for clippy lints graduating to stable.
@@ -454,18 +498,18 @@ jobs:
# before the lint blocks merges. continue-on-error keeps it non-blocking.
clippy-beta:
name: Clippy (beta, informational)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
continue-on-error: true
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: beta
components: clippy
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# `+beta` for the same reason as the MSRV job above: without it this ran
# stable clippy and duplicated the `clippy` job.
- run: cargo +beta clippy --all-targets --features duckdb-1-5-4 -- -D warnings
@@ -486,32 +530,64 @@ jobs:
LD_LIBRARY_PATH: ${{ runner.temp }}/libduckdb
run: cargo +beta clippy --all-targets --features bundled-test-prebuilt,duckdb-1-5-4 -- -D warnings
+ # Informational: the current stable Rust, whatever rust-toolchain.toml pins.
+ # Every blocking job builds with the pinned release, so a new stable cannot
+ # turn them red overnight; this job is where its new lints and warnings show
+ # up first, on every pull request and on the weekly schedule. When stable is
+ # newer than the pin and this job is green, move the pin (CONTRIBUTING.md,
+ # "The Rust toolchain"); when it is red, fix the findings in the same PR.
+ latest-stable:
+ name: Latest stable Rust (informational)
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ continue-on-error: true
+ env:
+ # Over rust-toolchain.toml, for every rustup proxy this job runs,
+ # rust-cache's own `rustc -vV` (part of its cache key) included.
+ RUSTUP_TOOLCHAIN: stable
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: stable
+ components: clippy, rustfmt
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ - name: The stable release this run checks
+ run: rustc +stable --version
+ - run: cargo +stable fmt -- --check
+ - run: cargo +stable clippy --all-targets -- -D warnings
+ - run: cargo +stable clippy --all-targets --features duckdb-1-5-4 -- -D warnings
+ - run: cargo +stable doc --no-deps --features duckdb-1-5-4
+ - run: cargo +stable test --lib --features duckdb-1-5-4
+
fmt:
name: Format
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
components: rustfmt
- run: cargo fmt -- --check
doc:
name: Documentation
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# --features duckdb-1-5-4 (implies duckdb-1-5-3, which implies
# duckdb-1-5) rather than --all-features, which would compile DuckDB
# from source here; `test-bundled` documents --all-features on Linux,
@@ -545,24 +621,48 @@ jobs:
# `ci-ok` is the one required check; it is only as good as its `needs`.
- name: The ci-ok job waits on every blocking job
run: scripts/check-ci-aggregate.py
+ # The six upstream-comparison steps run through this wrapper; a mapping
+ # that read a missing script, or exit 2 on a release, as a pass would turn
+ # each of them into a check of nothing.
+ - name: The upstream-guard wrapper maps exit codes as documented
+ run: scripts/upstream-guard.sh --self-test
+ # Cargo.toml's version, every committed Cargo.lock, the CHANGELOG,
+ # CITATION.cff, SECURITY.md and every `quack-rs = "0.X"` snippet must name
+ # the same release.
+ - name: Release metadata agrees with Cargo.toml
+ run: python3 scripts/check-release-metadata.py
+ # Every job builds with the toolchain rust-toolchain.toml pins, except the
+ # informational ones and those about another toolchain (MSRV, nightly).
+ - name: Workflows use the pinned Rust toolchain
+ run: python3 scripts/check-toolchain-pin.py
+ # One commit per action across the workflows, and the same commits in the
+ # workflows the scaffold generates (src/scaffold/templates_ci.rs), which
+ # Dependabot does not update.
+ - name: Action pins agree, here and in the scaffold's templates
+ run: python3 scripts/check-action-pins.py
+ # LESSONS.md (long form) and the book's pitfall page (condensed form) are two
+ # hand-kept copies; they must list the same pitfalls with the same titles.
+ - name: LESSONS.md and the book describe the same pitfalls
+ run: python3 scripts/check-pitfalls-parity.py
msrv:
name: MSRV (1.86.0)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: "1.86.0"
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# `+1.86.0` is not optional, for the same reason the `nightly` and `miri`
- # jobs below use `+nightly`: `rust-toolchain.toml` pins `channel =
- # "stable"`, and a rustup toolchain file overrides the default this action
- # sets. A bare `cargo check` here runs stable and tests nothing -- the
- # MSRV gate was inert. This matches `cargo +1.86.0 check` in CONTRIBUTING.
+ # jobs below use `+nightly`: `rust-toolchain.toml` pins the toolchain
+ # (`channel`), and a rustup toolchain file overrides the default this
+ # action sets. A bare `cargo check` here runs the pinned release and tests
+ # nothing -- the MSRV gate was inert. This matches `cargo +1.86.0 check`
+ # in CONTRIBUTING.
#
# Against what a 1.86.0 user's Cargo resolves, not the committed
# Cargo.lock: that carries the newest compatible releases, and
@@ -581,7 +681,7 @@ jobs:
- run: cargo +1.86.0 check --features duckdb-1-5-4
dependency-floor:
- # Cargo.toml declares `duckdb` / `libduckdb-sys` `>=1.4.4, <2`, but every
+ # Cargo.toml declares `duckdb` / `libduckdb-sys` `>=1.4.4, <1.20100`, but every
# other job builds the newest release Cargo.lock pins. This one resolves the
# declared floor and runs the library tests there, so the lower bound is a
# tested claim rather than a hope.
@@ -593,16 +693,16 @@ jobs:
# job above covers 1.86.0 with the locked dependencies; this covers the
# dependency floor with a current compiler.
name: Dependency floor (duckdb / libduckdb-sys 1.4.4)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Resolve the declared floor
run: |
set -euo pipefail
@@ -613,16 +713,16 @@ jobs:
bench-compile:
name: Benchmark (compile check)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- run: cargo bench --no-run
# The engine-level benchmark extension (benches/engine) is a separate crate;
# keep it compiling, and its driver parsing.
@@ -636,15 +736,21 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
+ os: [ubuntu-24.04, macos-26, windows-2025]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ with:
+ # hello-ext is its own Cargo workspace (examples/hello-ext/target); the
+ # action caches only the root one unless told otherwise.
+ workspaces: |
+ . -> target
+ examples/hello-ext -> target
- run: cargo check --manifest-path examples/hello-ext/Cargo.toml --all-targets
- name: Clippy (Linux only)
if: runner.os == 'Linux'
@@ -653,18 +759,26 @@ jobs:
scaffold-compile:
name: Scaffold (compile check)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
- with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ # `--test integration_test` builds the one test binary that holds the test
+ # (the unfiltered form compiled nine, and eight of them ran nothing).
+ # `--exact` plus the grep: `cargo test ` exits 0 when the filter
+ # matches no test, so renaming the test would have left this job green,
+ # running nothing. The summary line must say exactly one test passed.
- name: Verify scaffold output compiles
- run: cargo test scaffold_generated_code_compiles -- --nocapture
+ run: |
+ set -euo pipefail
+ cargo test --test integration_test scaffold_generated_code_compiles -- --exact --nocapture | tee /tmp/scaffold-compile.log
+ grep -qE '^test result: ok\. 1 passed;' /tmp/scaffold-compile.log
symbol-check:
name: Symbol check (hello-ext · ${{ matrix.os }})
@@ -673,15 +787,21 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-latest, macos-latest]
+ os: [ubuntu-24.04, macos-26]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ with:
+ # hello-ext is its own Cargo workspace (examples/hello-ext/target); the
+ # action caches only the root one unless told otherwise.
+ workspaces: |
+ . -> target
+ examples/hello-ext -> target
- name: Build hello-ext as cdylib
run: cargo build --manifest-path examples/hello-ext/Cargo.toml --release
- name: Verify extension entry point symbol (Linux)
@@ -709,7 +829,7 @@ jobs:
abi-table:
name: ABI layout table (vs upstream DuckDB headers), and every C API slot wrapped or listed
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -723,28 +843,20 @@ jobs:
# These guards fetch from upstream. Exit 2 is their documented
# "could not check" code -- a DuckDB release header was unreachable -- and must not
# be read as drift. Only exit 1 means the checked-in data is wrong.
- run: |
- python3 scripts/check-abi-table.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
- # Every slot of the C extension API is either used under src/ or listed,
- # with a reason, in scripts/api-coverage-allowlist.txt, so a slot a DuckDB
- # release adds is noticed instead of left for someone to read the header.
+ run: scripts/upstream-guard.sh scripts/check-abi-table.py
+ # Every slot of the C extension API is either used by library code under
+ # src/ or listed, with a reason, in scripts/api-coverage-allowlist.txt, so
+ # a slot a DuckDB release adds is noticed instead of left for someone to
+ # read the header. "Used" excludes comments, string literals and
+ # test-only code; the self-test holds the scanner to that.
+ - name: The C API surface scanner, on fixed snippets
+ run: python3 scripts/check-api-coverage.py --self-test
- name: Classify the C API surface
- run: |
- python3 scripts/check-api-coverage.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
+ run: scripts/upstream-guard.sh scripts/check-api-coverage.py
platform-table:
name: Platform list (vs upstream distribution matrix)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -759,17 +871,11 @@ jobs:
# These guards fetch from upstream. Exit 2 is their documented
# "could not check" code -- the distribution matrix was unreachable -- and must not
# be read as drift. Only exit 1 means the checked-in data is wrong.
- run: |
- python3 scripts/check-platform-table.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
+ run: scripts/upstream-guard.sh scripts/check-platform-table.py
spdx-list:
name: SPDX shortlist (vs official registry)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -783,17 +889,11 @@ jobs:
# These guards fetch from upstream. Exit 2 is their documented
# "could not check" code -- the SPDX registry was unreachable -- and must not
# be read as drift. Only exit 1 means the checked-in data is wrong.
- run: |
- python3 scripts/check-spdx-list.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
+ run: scripts/upstream-guard.sh scripts/check-spdx-list.py
autoload-entries:
name: Catalog autoload names (vs every supported DuckDB)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -807,17 +907,11 @@ jobs:
- name: Verify src/catalog.rs covers every supported release
# Exit 2 is the documented "could not check" code (upstream
# unreachable) and must not be read as drift.
- run: |
- python3 scripts/check-autoload-entries.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
+ run: scripts/upstream-guard.sh scripts/check-autoload-entries.py
book:
name: Book builds (mdBook)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -826,9 +920,9 @@ jobs:
# The versioned-site step below builds each release line from its tag.
fetch-depth: 0
fetch-tags: true
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
# docs.yml builds the book only when main is pushed, so a broken page
# was found after merge. The version is the one book/mdbook-version pins,
# which docs.yml's build script also reads.
@@ -856,7 +950,7 @@ jobs:
msrv-vs-duckdb-ci:
name: MSRV vs DuckDB's extension CI
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -871,17 +965,11 @@ jobs:
# These guards fetch from upstream. Exit 2 is their documented
# "could not check" code -- the pinned-toolchain config was unreachable -- and must not
# be read as drift. Only exit 1 means the checked-in data is wrong.
- run: |
- python3 scripts/check-msrv-vs-duckdb-ci.py || status=$?
- if [ "${status:-0}" -eq 2 ]; then
- echo "::warning::upstream unreachable; check skipped, not failed"
- exit 0
- fi
- exit "${status:-0}"
+ run: scripts/upstream-guard.sh scripts/check-msrv-vs-duckdb-ci.py
extension-load:
name: Extension load test (DuckDB ${{ matrix.duckdb }})
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 45
strategy:
fail-fast: false
@@ -903,10 +991,16 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ with:
+ # hello-ext is its own Cargo workspace (examples/hello-ext/target); the
+ # action caches only the root one unless told otherwise.
+ workspaces: |
+ . -> target
+ examples/hello-ext -> target
- name: Build hello-ext as cdylib
run: cargo build --manifest-path examples/hello-ext/Cargo.toml --release
- name: Build append_metadata
@@ -969,7 +1063,7 @@ jobs:
}
echo "Extension loaded and produced correct results"
- name: Every documented hello-ext check
- # examples/hello-ext/README.md documents 31 statements and their
+ # examples/hello-ext/README.md documents 32 statements and their
# results; this used to run two of them. The script runs all of
# examples/hello-ext/sql_checks.txt and also fails if the README's
# statements drift from that fixture.
@@ -977,16 +1071,16 @@ jobs:
scaffold-e2e:
name: Scaffold end-to-end (build, stamp, load, query)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Install DuckDB CLI
run: |
set -euo pipefail
@@ -1142,8 +1236,12 @@ jobs:
import pathlib, re, sys
d = pathlib.Path("/tmp/scaffolded_capi")
p = d / "Cargo.toml"
- s, n = re.subn(r'quack-rs = \{ version = "[^"]+" \}',
- 'quack-rs = { path = "%s", features = ["duckdb-1-5"] }' % sys.argv[1], p.read_text())
+ s = p.read_text()
+ # A project stamped C API v1.5.6 enables the duckdb-1-5* feature itself
+ # (the old step added `duckdb-1-5` by hand); only the version becomes a path.
+ assert 'features = ["duckdb-1-5-4"]' in s, s
+ s, n = re.subn(r'quack-rs = \{ version = "[^"]+"',
+ 'quack-rs = { path = "%s"' % sys.argv[1], s)
assert n == 1, n
p.write_text(s)
p = d / "src/lib.rs"
@@ -1176,16 +1274,16 @@ jobs:
# whatever DuckDB loads it. It never reaches the layout comparison itself --
# `abi-guard-layout` below is the job that exercises LayoutMismatch.
name: ABI guard refuses a build whose declared DuckDB version contradicts its bindings
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Install DuckDB CLI
run: |
set -euo pipefail
@@ -1310,16 +1408,16 @@ jobs:
# into the DuckDB its bindings came from it must work. The engines are
# pinned releases, not `latest`, so the expected verdict never shifts.
name: ABI guard refuses a real layout mismatch (and accepts the matching DuckDB)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Build and stamp a duckdb-1-5 demo against the pinned bindings
run: |
set -euo pipefail
@@ -1448,28 +1546,38 @@ jobs:
publish-dry-run:
name: Publish dry-run
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
- with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ # `--locked`, as release.yml's `package` and `publish` jobs run it: without
+ # it a Cargo.toml version bump that left Cargo.lock behind passed here
+ # (cargo quietly rewrote the lock file) and failed only at release time.
+ # No `--allow-dirty`: a clean checkout is what is being packaged.
- name: cargo publish --dry-run
- run: cargo publish --dry-run --allow-dirty
+ run: cargo publish --dry-run --locked
security:
name: Security (cargo-deny)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ # Installed explicitly, like every other job, rather than left to
+ # rustup's on-demand install of what rust-toolchain.toml names (on by
+ # default since rustup 1.28.1, but an image can turn it off).
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
with:
cache-targets: false
- name: Install cargo-deny
@@ -1481,6 +1589,20 @@ jobs:
run: cargo install cargo-deny --locked --version "0.20.2"
- name: Run cargo-deny check
run: cargo deny check
+ # The other committed graphs (hello-ext, the fuzz crate, the book's
+ # doctest crate and benches/engine) had their advisories read by the OSV
+ # job only. deny.toml cannot be applied
+ # to them (it is strict about unused licenses, and libfuzzer-sys is
+ # "(MIT OR Apache-2.0) AND NCSA"), so they have their own config.
+ - name: Run cargo-deny check on the CI crates
+ run: |
+ set -euo pipefail
+ crates=(examples/hello-ext fuzz book/doctest benches/engine)
+ for m in "${crates[@]}"; do
+ echo "::group::cargo deny: $m"
+ cargo deny --manifest-path "$m/Cargo.toml" --config deny.ci-crates.toml check
+ echo "::endgroup::"
+ done
# OSV / GHSA advisory scan. The cargo-deny job above consults only the RustSec
# advisory database; OSV.dev aggregates GHSA *and* RustSec, so GHSA-only
@@ -1489,15 +1611,15 @@ jobs:
# via the `libduckdb-sys` build-dependency. The binary is version- and checksum-pinned.
osv-scan:
name: Security (OSV / GHSA)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
- name: Install osv-scanner (pinned, checksum-verified)
run: |
VER=2.7.0
@@ -1509,47 +1631,100 @@ jobs:
chmod +x osv-scanner
- name: Scan every committed lockfile against OSV
run: |
- # All four committed lockfiles. fuzz/Cargo.lock and then
+ set -euo pipefail
+ # Every committed lockfile. fuzz/Cargo.lock and then
# book/doctest/Cargo.lock were missing: their dependencies are
# CI-only, but an unscanned lockfile is an unscanned lockfile.
- ./osv-scanner --config=osv-scanner.toml \
- --lockfile=Cargo.lock \
- --lockfile=examples/hello-ext/Cargo.lock \
- --lockfile=fuzz/Cargo.lock \
- --lockfile=book/doctest/Cargo.lock
+ # benches/engine/Cargo.lock was gitignored until 0.20.0
+ # (bench-compile resolved that crate afresh each run).
+ lockfiles=(Cargo.lock examples/hello-ext/Cargo.lock fuzz/Cargo.lock book/doctest/Cargo.lock benches/engine/Cargo.lock)
+ args=()
+ for f in "${lockfiles[@]}"; do args+=("--lockfile=$f"); done
+ ./osv-scanner --config=osv-scanner.toml "${args[@]}"
+
+ # Build-free hygiene checks: spelling (typos.toml), TOML style (.taplo.toml)
+ # and unused dependencies (cargo-machete; the manifests list under
+ # [package.metadata.cargo-machete] what it cannot see: build.rs, a
+ # [[bench]], generated doctests). Each tool is a release binary pinned by
+ # version and SHA-256, as in osv-scan. Only cargo-machete's release publishes
+ # a checksum file (it matches); typos and taplo publish none, so their
+ # hashes are of the release assets as GitHub served them when pinned.
+ hygiene:
+ name: Hygiene (typos, taplo, machete)
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ # cargo-machete --with-metadata runs `cargo metadata`.
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ toolchain: 1.99.1
+ - name: Install typos, taplo and cargo-machete (pinned, checksum-verified)
+ run: |
+ set -euo pipefail
+ mkdir -p "$RUNNER_TEMP/bin"
+ fetch() { #
+ curl -fsSL --retry 5 -o "$3" "$1"
+ echo "$2 $3" | sha256sum -c -
+ }
+ cd "$RUNNER_TEMP"
+ fetch https://github.com/crate-ci/typos/releases/download/v1.46.0/typos-v1.46.0-x86_64-unknown-linux-musl.tar.gz \
+ c62a626df385c96574743fd4974aeb271a41a67d080a0398e36e1a42782f614a typos.tgz
+ tar -xzf typos.tgz -C bin ./typos
+ fetch https://github.com/tamasfe/taplo/releases/download/0.10.0/taplo-linux-x86_64.gz \
+ 8fe196b894ccf9072f98d4e1013a180306e17d244830b03986ee5e8eabeb6156 taplo.gz
+ gunzip -c taplo.gz > bin/taplo
+ chmod +x bin/taplo
+ fetch https://github.com/bnjbvr/cargo-machete/releases/download/v0.9.2/cargo-machete-v0.9.2-x86_64-unknown-linux-musl.tar.gz \
+ 48200087f54c55aabcd4db4af1e25742b49846c02a1b1bfa134711945b35b2e9 machete.tgz
+ tar -xzf machete.tgz --strip-components=1 -C bin \
+ cargo-machete-v0.9.2-x86_64-unknown-linux-musl/cargo-machete
+ echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
+ - run: typos
+ - run: taplo fmt --check
+ - run: taplo lint
+ - run: cargo machete --with-metadata
nightly:
name: Nightly (informational)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
continue-on-error: true
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: nightly
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
- # `+nightly` is not optional: `rust-toolchain.toml` pins `channel =
- # "stable"`, and a rustup toolchain file overrides the default this action
- # sets, so a bare `cargo` here runs stable and this job tests nothing new.
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ # `+nightly` is not optional: `rust-toolchain.toml` pins a stable release
+ # (`channel`), and a rustup toolchain file overrides the default this
+ # action sets, so a bare `cargo` here runs that release and this job
+ # tests nothing new.
- name: Test (nightly, informational)
run: cargo +nightly test --all-targets
miri:
name: Miri (undefined behaviour)
- runs-on: ubuntu-latest
- timeout-minutes: 60
+ runs-on: ubuntu-24.04
+ # Two sequential runs of the library tests (native, then s390x). On the 0.20.0
+ # pull request the native run alone took 41.6 minutes, 18 of them one parser
+ # test on 100,000-level inputs and 3 more an SPDX test (both shrink under
+ # Miri now), and the s390x run was cancelled at the 60-minute limit with the
+ # aggregate check red. The limit is for the two runs together.
+ timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: nightly
components: miri, rust-src
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# This crate is ~40% unsafe by surface area, and the parts that are *not*
# FFI -- the duckdb_string_t decoder, the validity bitmap arithmetic, the
# UUID bit flip, the interval maths, the secret zeroizer -- are exactly
@@ -1575,9 +1750,9 @@ jobs:
# what this job is for. `cargo test` still runs the full 256.
PROPTEST_CASES: "8"
# `+nightly` is not optional -- see the note in the `nightly` job:
- # `rust-toolchain.toml` pins stable and overrides this action's default,
- # so a bare `cargo miri` resolves to stable, where the `miri` component
- # does not exist.
+ # `rust-toolchain.toml` pins a stable release and overrides this action's
+ # default, so a bare `cargo miri` resolves to that release, where the
+ # `miri` component does not exist.
# `--features duckdb-1-5-4` matters: without it every `duckdb-1-5*`
# module is cfg'd out, including `src/arrow.rs` -- the largest block of
# pure-Rust `unsafe` in the crate, and exactly what Miri is here for.
@@ -1601,17 +1776,17 @@ jobs:
leak-check:
name: LeakSanitizer (RAII wrappers vs a real DuckDB)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: nightly
components: rust-src
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# Half of quack-rs's value proposition is RAII over DuckDB handles:
# LogicalType, Value, QueryResult, OwnedDataChunk, FileHandle,
# PreparedStatement, ErrorData, Appender. A missing destructor is not a
@@ -1619,7 +1794,10 @@ jobs:
# way to catch one is to look at the heap afterwards.
#
# LeakSanitizer intercepts malloc globally, so it sees DuckDB's C++
- # allocations even though libduckdb itself is not instrumented.
+ # allocations even though libduckdb itself is not instrumented -- but
+ # not the vector data buffers and LIST entry arrays the release
+ # libduckdb takes from its own jemalloc (CONTRIBUTING.md, "What the
+ # sanitizer and valgrind runs can and cannot see").
- name: Fetch a prebuilt libduckdb
# Version derived from Cargo.lock -- see the note in
# `test-bundled-prebuilt`. Linking a libduckdb that does not match the
@@ -1638,8 +1816,12 @@ jobs:
RUSTFLAGS: "-Zsanitizer=leak"
# DuckDB's own one-time allocations (extension registry, the default
# allocator's arenas, static initialisers) are still reachable at
- # exit and are not quack-rs's business. Suppress by library, so a leak
- # from a quack-rs frame is still reported.
+ # exit, which LeakSanitizer does not report, so nothing needs
+ # suppressing. lsan-suppressions.txt holds no rules on purpose: a
+ # rule for libduckdb.so is tested against every frame of an
+ # allocation, and every handle DuckDB allocates has such a frame, so
+ # it silenced exactly the leaks this job is for (the step below
+ # leaks a handle on purpose and requires the job to notice).
# A two-frame stack is useless for attributing a leak to a missing
# destructor; 30 is enough to name the quack-rs frame.
LSAN_OPTIONS: "suppressions=lsan-suppressions.txt:print_suppressions=0:malloc_context_size=30"
@@ -1647,13 +1829,48 @@ jobs:
# `+nightly` is not optional -- see the note in the `nightly` job.
cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \
--features bundled-test-prebuilt,duckdb-1-5-4 --test ffi_roundtrip
+ # A leak check is only as good as its ability to fail. tests/lsan_probe.rs
+ # leaks, on request, a Rust allocation (the control), a LogicalType and a
+ # Value, and the run must fail with a leak report each time; with nothing
+ # leaked it must pass. The first version of this job passed over the
+ # LogicalType and the Value because of a suppression.
+ - name: A handle leaked on purpose must fail LeakSanitizer
+ env:
+ RUSTFLAGS: "-Zsanitizer=leak"
+ LSAN_OPTIONS: "suppressions=lsan-suppressions.txt:print_suppressions=0:malloc_context_size=30"
+ run: |
+ set -u
+ probe() {
+ PROBE="$1" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \
+ --features bundled-test-prebuilt,duckdb-1-5-4 --test lsan_probe 2>&1 \
+ | tee "$RUNNER_TEMP/probe-$1.log"
+ return "${PIPESTATUS[0]}"
+ }
+ if ! probe nothing; then
+ echo "::error::the probe failed although it leaked nothing"
+ exit 1
+ fi
+ for what in rust logical_type value; do
+ if probe "$what"; then
+ echo "::error::PROBE=$what leaked and LeakSanitizer did not fail the run"
+ exit 1
+ fi
+ if ! grep -q 'LeakSanitizer: detected memory leaks' "$RUNNER_TEMP/probe-$what.log"; then
+ echo "::error::PROBE=$what failed, but not with a LeakSanitizer report"
+ exit 1
+ fi
+ done
# AddressSanitizer. LeakSanitizer above answers "did we forget a destructor";
# ASAN answers "did we write outside an allocation, or use one after it was
# freed" -- the class that produced the two heap-corruption defects fixed in
# v0.16.0, and the one a crate doing raw pointer arithmetic into DuckDB's
# memory is most exposed to. Miri cannot reach it: these paths call foreign
- # functions.
+ # functions. Its blind spot: this links the release libduckdb, which takes
+ # vector data buffers and LIST entry arrays from its own jemalloc, and ASAN
+ # does not track those blocks, so an out-of-bounds or stale write into them
+ # is not reported here; Rust-heap and validity-mask errors are
+ # (CONTRIBUTING.md, "What the sanitizer and valgrind runs can and cannot see").
#
# Blocking since September 2026. It was `continue-on-error` until it had a
# green run on main, because libduckdb itself is not instrumented and the
@@ -1665,17 +1882,17 @@ jobs:
# ASAN subsumes LSAN, so once this is blocking, `leak-check` can fold into it.
asan:
name: AddressSanitizer
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: nightly
components: rust-src
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Fetch a prebuilt libduckdb
# Version derived from Cargo.lock -- see `test-bundled-prebuilt`.
run: |
@@ -1699,17 +1916,17 @@ jobs:
semver:
name: Public API (semver-checks)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 0
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# quack-rs is an SDK: its public API *is* the product, and a downstream
# extension that stops compiling on a patch release is a worse outcome
# than a missing feature. This compares the working tree against the
@@ -1769,7 +1986,7 @@ jobs:
fuzz:
name: Fuzz (smoke)
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
# 60 s per target on a push or pull request; 10 minutes per target on the
# weekly schedule, which is long enough to grow the corpus meaningfully.
timeout-minutes: 90
@@ -1779,10 +1996,10 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
toolchain: nightly
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
# The description.yml parser is hand-rolled and reads a file from an
# arbitrary GitHub repository; the duckdb_string_t decoder reads 16 bytes
# whose meaning depends on a length field inside them. Both are exactly
@@ -1867,6 +2084,38 @@ jobs:
path: fuzz/corpus
key: fuzz-corpus-${{ github.run_id }}
+ # actionlint: workflow syntax and types, `needs` and `uses` references, the
+ # expression contexts, and shellcheck on every `run:` block. The
+ # binary is version- and checksum-pinned, like osv-scanner below.
+ # scripts/check-workflow-expressions.py predates this job and covers one class
+ # of the same errors without a download.
+ workflow-lint:
+ name: Workflow lint (actionlint)
+ runs-on: ubuntu-24.04
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Install actionlint (pinned, checksum-verified)
+ run: |
+ set -euo pipefail
+ VER=1.7.12
+ # From the release's actionlint_${VER}_checksums.txt (linux_amd64).
+ SHA=8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
+ curl -fsSL --retry 5 -o actionlint.tar.gz \
+ "https://github.com/rhysd/actionlint/releases/download/v${VER}/actionlint_${VER}_linux_amd64.tar.gz"
+ echo "${SHA} actionlint.tar.gz" | sha256sum -c -
+ tar -xzf actionlint.tar.gz actionlint
+ - name: actionlint
+ run: |
+ set -euo pipefail
+ # Without shellcheck on PATH actionlint skips every `run:` block
+ # silently; a lint that checks less than it says is worse than none.
+ command -v shellcheck >/dev/null || { echo "::error::shellcheck is not installed"; exit 1; }
+ # SC2129 ("group these redirects") is style, not a defect.
+ ./actionlint -color -ignore 'SC2129'
+
# The one check branch protection should require. Every blocking job is in
# `needs` (scripts/check-ci-aggregate.py, run by the `doc` job, fails when one
# is missing), so adding or renaming a job never silently drops it from the
@@ -1907,12 +2156,14 @@ jobs:
- publish-dry-run
- security
- osv-scan
+ - hygiene
- miri
- leak-check
- asan
- semver
- fuzz
- runs-on: ubuntu-latest
+ - workflow-lint
+ runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Every blocking job succeeded
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index fa080255..2a4828a4 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -28,7 +28,7 @@ env:
jobs:
coverage:
name: Test coverage
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
# `--all-features` compiles DuckDB from C++; the default limit is 360 minutes.
timeout-minutes: 90
steps:
@@ -36,12 +36,12 @@ jobs:
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
components: llvm-tools-preview
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Install cargo-llvm-cov
env:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 87c9c6c5..05616402 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -35,7 +35,7 @@ concurrency:
jobs:
build:
name: Build versioned site
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- name: Checkout repository
@@ -47,9 +47,9 @@ jobs:
fetch-tags: true
- name: Install Rust toolchain
- uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
# main's API reference for /dev/api/, with the docs.rs feature set
# (Cargo.toml [package.metadata.docs.rs]). Released versions link to
@@ -94,7 +94,7 @@ jobs:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
needs: build
steps:
- name: Deploy to GitHub Pages
diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml
index 5077b76f..473ef2ce 100644
--- a/.github/workflows/mutants.yml
+++ b/.github/workflows/mutants.yml
@@ -31,16 +31,16 @@ jobs:
mutants-full:
name: "Mutants (full sweep)"
if: github.event_name == 'workflow_dispatch'
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Install cargo-mutants
env:
@@ -94,21 +94,33 @@ jobs:
retention-days: 30
# ── Incremental mutation testing (PRs only) ──────────────────────────────
+ #
+ # Sharded: each of the eight jobs below tests every eighth mutant
+ # (`--shard k/8 --sharding round-robin`), so a pull request that touches a
+ # hundred source files finishes in about an eighth of the time. One job did all
+ # of a 102-file, 1,639-mutant pull request (0.20.0) and was stopped by its
+ # runner after 33 and 55 minutes, long before it was done. `mutants-incremental-gate`
+ # below is the single check to require: it keeps the name this check always had.
mutants-incremental:
- name: "Mutation Testing (incremental)"
+ name: "Mutation Testing (incremental) shard ${{ matrix.shard }}"
if: github.event_name == 'pull_request'
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 60
+ strategy:
+ # A survivor in one shard must not hide the others' results.
+ fail-fast: false
+ matrix:
+ shard: [0, 1, 2, 3, 4, 5, 6, 7]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 0
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
- - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+ toolchain: 1.99.1
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Install cargo-mutants
env:
@@ -152,6 +164,8 @@ jobs:
if: steps.changed.outputs.skip == 'false'
env:
CHANGED_FILES: ${{ steps.changed.outputs.files }}
+ SHARD: ${{ matrix.shard }}
+ SHARDS: ${{ strategy.job-total }}
run: |
echo "::group::cargo-mutants execution log"
@@ -203,12 +217,20 @@ jobs:
# in `mutants.out/mutants.out/`, where the report step below never
# looked -- so every count read 0, the score printed 100%, and the
# job passed green no matter how many mutants survived.
+ # `--shard k/n` with `--sharding round-robin`: mutant i goes to shard
+ # i % n, which spreads the slow files over all shards. (The default,
+ # `slice`, hands each shard a consecutive run of mutants in file
+ # order.) A shard with no mutants -- more shards than mutants, on a
+ # small pull request -- finds nothing to test and exits 0, which the
+ # report below accepts for an incremental run.
set +e
cargo mutants "${ARGS[@]}" \
--features duckdb-1-5-4 \
--cargo-arg=--lib \
--exclude-re 'ErrorData' \
--output . \
+ --shard "${SHARD}/${SHARDS}" \
+ --sharding round-robin \
--timeout 120 \
--jobs 4 2>&1 | tee /tmp/mutants-run.log
# `${PIPESTATUS[0]}`, not `$?`: these steps run under the default
@@ -239,8 +261,30 @@ jobs:
if: always() && steps.changed.outputs.skip == 'false'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
- name: mutation-report-incremental
+ name: mutation-report-incremental-${{ matrix.shard }}
path: |
mutants.out/
/tmp/mutants-run.log
retention-days: 14
+
+ # The one check that stands for the shards. It keeps the name of the single
+ # job it replaces, "Mutation Testing (incremental)", so a branch-protection
+ # rule that requires that name still means the same thing. `always()` because
+ # a failed shard must turn this red rather than skip it, and a skipped
+ # required check counts as passed.
+ mutants-incremental-gate:
+ name: "Mutation Testing (incremental)"
+ if: always() && github.event_name == 'pull_request'
+ needs: mutants-incremental
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ steps:
+ - name: Every shard passed
+ env:
+ RESULT: ${{ needs.mutants-incremental.result }}
+ run: |
+ if [ "$RESULT" != success ]; then
+ echo "::error::the mutation shards finished as '${RESULT}', not 'success': see the shard jobs"
+ exit 1
+ fi
+ echo "All mutation shards passed."
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2e71f078..02d430af 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -51,7 +51,7 @@ env:
jobs:
validate:
name: Validate release tag
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
version: ${{ steps.meta.outputs.version }}
@@ -76,7 +76,7 @@ jobs:
shell: bash
run: |
if [[ "$(git cat-file -t "refs/tags/$TAG")" != "tag" ]]; then
- echo "::error::Tag '$TAG' is a lightweight tag. Release tags are annotated and signed: git tag -s '$TAG' -m 'Release $TAG' (RELEASING.md Step 5)."
+ echo "::error::Tag '$TAG' is a lightweight tag. Release tags must be annotated (RELEASING.md Step 5 also signs them; this job does not verify the signature): git tag -s '$TAG' -m 'Release $TAG'."
exit 1
fi
git fetch --no-tags --quiet origin '+refs/heads/main:refs/remotes/origin/main'
@@ -155,6 +155,13 @@ jobs:
fi
echo "::notice::CHANGELOG.md entry found for v$VERSION"
+ # Cargo.lock still naming the previous version fails `cargo package
+ # --locked` in the `package` job, after all of ci.yml has run; stale
+ # version snippets ship in the .crate. Say so in the first minute.
+ - name: Release metadata (Cargo.lock, CITATION.cff, SECURITY.md, doc snippets)
+ shell: bash
+ run: python3 scripts/check-release-metadata.py
+
# ── Job 2: Every CI gate, by running ci.yml itself ──────────────────────────
ci:
name: CI
@@ -167,7 +174,7 @@ jobs:
package:
name: Package and attest
needs: [ci]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
contents: read
@@ -175,13 +182,14 @@ jobs:
attestations: write # write attestation to the repository
outputs:
crate_file: ${{ steps.pkg.outputs.crate_file }}
+ rustc: ${{ steps.pkg.outputs.rustc }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
with:
- toolchain: stable
+ toolchain: 1.99.1
# No cache in this job or in `publish`: a tag-triggered run can restore
# caches the default branch saved, and this job holds the attestation
# token. Release speed does not matter.
@@ -201,6 +209,9 @@ jobs:
CRATE_FILE=$(ls target/package/*.crate)
CRATE_SIZE=$(wc -c < "$CRATE_FILE")
echo "crate_file=$CRATE_FILE" >> "$GITHUB_OUTPUT"
+ # The `publish` job installs this rustc and refuses to re-package with
+ # any other: it runs after a manual approval that can take days.
+ echo "rustc=$(rustc --version | awk '{ print $2 }')" >> "$GITHUB_OUTPUT"
printf '::notice::Packaged %s (%s bytes)\n' "$(basename "$CRATE_FILE")" "$CRATE_SIZE"
- name: Generate SHA-256 checksums
@@ -226,12 +237,14 @@ jobs:
target/package/SHA256SUMS
retention-days: 90
if-no-files-found: error
+ # A re-run of the whole workflow uploads under the same name again.
+ overwrite: true
# ── Job 5: Create GitHub release with artifacts & release notes ───────────────
github-release:
name: Create GitHub release
needs: [validate, package, publish]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
contents: write # create release, upload assets
@@ -300,7 +313,7 @@ jobs:
run: |
TAG="v${VERSION}"
- PRE_FLAG=""
+ PRE_FLAG="--prerelease=false"
[[ "$IS_PRE" == "true" ]] && PRE_FLAG="--prerelease"
RELEASE_URL="https://github.com/${REPO}/releases/tag/${TAG}"
@@ -311,7 +324,11 @@ jobs:
if gh release view "$TAG" &>/dev/null; then
echo "::warning::Release ${TAG} already exists — updating it."
echo "::group::Updating GitHub release ${TAG}"
+ # --draft=false: a release left as a draft (created by hand, or by an
+ # earlier failed run) would otherwise stay invisible. $PRE_FLAG sets
+ # the prerelease state both ways, not only to true.
gh release edit "$TAG" \
+ --draft=false \
$PRE_FLAG \
--title "${CRATE} ${TAG}" \
--notes-file release_notes.md
@@ -360,7 +377,7 @@ jobs:
publish:
name: Publish to crates.io
needs: [validate, package]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 30
# The 'crates-io' environment requires a manual approval step.
# Configure it at: Settings → Environments → crates-io → Required reviewers
@@ -374,9 +391,49 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+ with:
+ # The rustc the `package` job attested with: the .crate's bytes depend
+ # on cargo, and this job runs after an approval that can take days.
+ # (rust-toolchain.toml, which overrides the default this action sets,
+ # pins the same release; the next step checks that they agree.)
+ toolchain: ${{ needs.package.outputs.rustc }}
+
+ # Outside the checkout: `cargo package` would otherwise count the
+ # downloaded files as uncommitted changes, and include them in the .crate.
+ - name: Download the attested artifacts
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
- toolchain: stable
+ name: release-artifacts
+ path: ${{ runner.temp }}/release-artifacts
+
+ # crates.io versions cannot be replaced, and the post-publish check below
+ # can only report a mismatch. Package here exactly as `cargo publish` will
+ # (no build script runs: --no-verify) and compare with the attested hash
+ # first, so a .crate that differs fails before anything is uploaded, with
+ # no credential in this job yet.
+ - name: The .crate about to be published is the attested one
+ env:
+ VERSION: ${{ needs.validate.outputs.version }}
+ CRATE: ${{ needs.validate.outputs.crate_name }}
+ ATTESTED_RUSTC: ${{ needs.package.outputs.rustc }}
+ shell: bash
+ run: |
+ FILE="${CRATE}-${VERSION}.crate"
+ rustc_here=$(rustc --version | awk '{ print $2 }')
+ if [[ -z "$ATTESTED_RUSTC" || "$rustc_here" != "$ATTESTED_RUSTC" ]]; then
+ echo "::error::this job's rustc is '$rustc_here', the package job's was '$ATTESTED_RUSTC'; nothing was published."
+ exit 1
+ fi
+ cargo package --no-verify --locked
+ want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' "$RUNNER_TEMP/release-artifacts/SHA256SUMS")
+ got=$(sha256sum "${CARGO_TARGET_DIR:-target}/package/$FILE" | awk '{ print $1 }')
+ echo "attested: $want"
+ echo "packaged: $got"
+ if [[ -z "$want" || "$want" != "$got" ]]; then
+ echo "::error::this job packages a different $FILE than the attested one; nothing was published."
+ exit 1
+ fi
# Preferred credential: crates.io Trusted Publishing, which exchanges
# this job's OIDC token for a short-lived crates.io token (revoked when
@@ -388,7 +445,7 @@ jobs:
- name: Obtain a crates.io token via Trusted Publishing
id: trusted
continue-on-error: true
- uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5
+ uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6
- name: Publish to crates.io
env:
@@ -444,12 +501,6 @@ jobs:
echo "> docs.rs may take a few minutes to build and index the documentation."
} >> "$GITHUB_STEP_SUMMARY"
- - name: Download the attested artifacts
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- with:
- name: release-artifacts
- path: release-artifacts/
-
# The attestation and SHA256SUMS describe the `.crate` the `package` job
# built; crates.io serves the one `cargo publish` just built. `cargo
# package` is reproducible (crates.io's 0.18.0 is byte-identical to a
@@ -471,7 +522,7 @@ jobs:
sleep 20
done
test -s "/tmp/$FILE" || { echo "::error::could not download $FILE from crates.io"; exit 1; }
- want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' release-artifacts/SHA256SUMS)
+ want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' "$RUNNER_TEMP/release-artifacts/SHA256SUMS")
got=$(sha256sum "/tmp/$FILE" | awk '{ print $1 }')
echo "attested: $want"
echo "crates.io: $got"
@@ -488,7 +539,7 @@ jobs:
# A prerelease gets no book of its own (scripts/build-versioned-site.py
# skips prerelease tags); main's /dev/ already shows it.
if: needs.validate.outputs.is_prerelease == 'false'
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
actions: write
diff --git a/.gitignore b/.gitignore
index 2b70b876..d8c5bed7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
/target/
examples/hello-ext/target/
benches/engine/target/
-benches/engine/Cargo.lock
book/doctest/target/
book/book/
mutants.out/
diff --git a/.taplo.toml b/.taplo.toml
new file mode 100644
index 00000000..178a34e3
--- /dev/null
+++ b/.taplo.toml
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: MIT
+# Formatter/linter configuration for taplo (https://taplo.tamasfe.dev).
+# taplo fmt --check # as CI would run it
+# taplo lint
+# Written to match the TOML already in the repository (four-space indent,
+# trailing comments left where the author put them), so adopting it reformats
+# nothing.
+include = ["*.toml", "*/Cargo.toml", "*/*/Cargo.toml", ".cargo/*.toml"]
+# `.cargo/mutants.toml` is a long annotated glob list whose trailing comments are
+# aligned by hand; taplo would re-flow all 250 lines of it.
+exclude = ["**/target/**", ".cargo/mutants.toml"]
+
+[formatting]
+indent_string = " "
+align_comments = false
+align_entries = false
+array_auto_expand = false
+array_auto_collapse = false
+reorder_keys = false
+column_width = 100
diff --git a/AUDIT.md b/AUDIT.md
index 9ef094c0..801d836b 100644
--- a/AUDIT.md
+++ b/AUDIT.md
@@ -2,11 +2,12 @@
# quack-rs — audit log, August – October 2026
-Sections 1–6 are the first audit, below. Sections 7–13 are the later ones, newest
+Sections 1–6 are the first audit, below. Sections 7–14 are the later ones, newest
last: 7 (second audit, 0.17.0 → 0.18.0), 8–10 (third to fifth, before 0.18.0),
-11 (sixth review, after 0.18.0), 12 (the DuckDB 2.0.0 review) and 13 (the seventh
-review, after 0.19.0). Each ends with how it was verified, and later sections
-correct earlier ones where they were wrong (13.4 lists the corrections).
+11 (sixth review, after 0.18.0), 12 (the DuckDB 2.0.0 review), 13 (the seventh
+review, after 0.19.0) and 14 (the eighth review, released as 0.20.0). Each ends with
+how it was verified, and later sections correct earlier ones where they were wrong
+(13.4 and 14.9 list the corrections).
The first audit (sections 1–6) reviewed `v0.16.0`, against **DuckDB 1.5.4**
(prebuilt `libduckdb`, x86-64 Linux) with `libduckdb-sys 1.10504.0`.
@@ -2669,6 +2670,8 @@ one thread, and not on 2.0.0.
### 13.7 Performance
+Superseded where the cells overlap by 14.7, which re-measured them on the 0.20.0 tree.
+
Measured with `benches/engine/compare.py` (committed; see its README), the same
benchmark extension built against 0.19.0 and against this tree, and its hand-written
twin, interleaved across sessions. DuckDB CLI 1.5.6 and 2.0.0, `SET threads=1`, 20
@@ -2881,3 +2884,611 @@ Linux 1.5.6 build was run after the push, above), `cargo-deny`, `cargo-mutants`
the changed files, a Wasm build, and the changed workflows in GitHub Actions before
the push: they were checked by the three scripts above and by reading, and the
first run in Actions is the one that found the failures listed above.
+
+## 14. Eighth review, October 2026 (after 0.19.0, released as 0.20.0)
+
+Commissioned as the most adversarial review of the whole repository that could be made,
+with the instruction to fix and implement what it finds and not to call the result
+finished until a public release could be cut from the merged pull request. Reviewed at
+`3c8ee2d` (a frozen snapshot of the branch that follows 0.19.0, `9c2dc76`), against
+DuckDB 1.4.4, 1.5.6 and 2.0.0.
+
+Labels, as in section 13. **VALIDATED**: I ran it against the real binaries, or pinned it
+with a test and showed the test fails on the code without the fix. **REPORTED**: a
+reviewer's or work package's finding with its evidence, which I did not re-run; where I
+fixed one, the fix rests on the code or upstream source it names. **READ**: from
+DuckDB's source, not run. **CONJECTURED**: reasoned, not tested.
+
+### 14.1 Method
+
+- Twelve reviewers, each in its own scratch copy of the snapshot, each told to disprove
+ rather than confirm: `unsound` (soundness of the data plane), `lastdiff` (every line
+ that changed since 0.19.0), `apigaps` (the C API against both headers), `conform`
+ (a scaffolded project built and loaded into three engines, against the community
+ build's scripts), `docs`, `ci` (workflows and scripts, mutated one by one), `harden`
+ (hostile input, resource limits, the release process), `ffi` (every callback kind on
+ the control plane), `author-tables` and `author-types` (an extension written against
+ the SDK by someone who has not read its source: table functions, COPY, replacement
+ scans; scalar functions, aggregates, types), `deps` (dependency ranges, packaging,
+ tooling) and `tests` (coverage, mutation, fuzzing, sanitizers, flakiness).
+- The findings ledger had 130 rows when the implementation phase began (a row can cover
+ a range, e.g. DOCS-10..23): 2 High, 46 Medium, 56 Low (51 Low, 3 Low/Nit, 2
+ Low-Medium), 9 Nit, 14 informational and 3 recommendations. Two more High findings
+ came later: SQLMACRO-1 (14.2), found while implementing, and the `tests` reviewer's
+ 17 items, whose own count is 0 Critical, 2 High, 8 Medium, 5 Low and 2 Nit or
+ informational.
+- Six work packages fixed what the ledger assigned them, each in its own worktree and
+ each told to show that its test fails on the pre-fix commit and passes after: `infra`
+ (workflows, scripts, release process, repository configuration), `scaffold`
+ (`src/scaffold`, `src/validate`, `src/bin`, the build script), `corea` (vectors, the
+ chunk writer, Arrow, scalar and aggregate builders, types), `coreb` (the entry point,
+ callbacks, the ABI guard), `api` (values, queries, appenders, new wrappers) and `docs`.
+ I merged them with merge commits (`b833f5a` infra, `201a5c7` api, `327939d` coreb,
+ `90d397f` corea, `f2186d0` docs, `6664103` scaffold) and resolved by hand the places
+ where two packages had edited the same lines.
+- What I fixed myself, outside the packages, is in 14.2 and 14.3.
+- Reviewers' patches were adopted line by line. Two were rejected with evidence (14.4).
+
+### 14.2 High-severity findings
+
+1. **`ListBuilder` cached the parent's entry array; a nested list's reserve freed it
+ (LASTDIFF-1).** A regression against 0.19.0 inside the unreleased performance work,
+ never published. With an inner `ListBuilder` over a `LIST(LIST(..))` whose reserve grew
+ the parent, the next `set_entry` wrote through a dangling pointer. VALIDATED: the
+ regression test `a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves`
+ segfaults (signal 11) on `3c8ee2d` and passes with the fix, against 1.5.6 and 2.0.0. `set_entry` fetches the array on every call (`734bb15`).
+ The first version of the test used the arrow lambda `j -> ...`, which DuckDB 2.0.0
+ rejects; I committed it without running it on 2.0.0 and CI's 2.0.0 job failed
+ (`691c160` fixes the SQL, `lambda j:`).
+2. **`SqlMacro::register` ran statements ahead of a parse error on DuckDB 2.0.0
+ (SQLMACRO-1).** Found by the `corea` package, which ran `harden`'s hostile-input test
+ on 2.0.0. VALIDATED on 2.0.0: body `1); DROP TABLE canary; --` makes SQL that does not
+ parse as a whole, `duckdb_extract_statements` reports 0 statements, the old code
+ refused only a count above 1 and executed it, and 2.0.0's `duckdb_query` runs the
+ statements it has parsed before it reports the error (1.4.4 and 1.5.6 parse first).
+ The test `hostile_input::a_macro_body_cannot_turn_one_statement_into_several` fails
+ on the old code with "Table with name canary does not exist". Fixed in `32e8d64`:
+ exactly one statement, and a count of zero is an error carrying
+ `duckdb_extract_statements_error`. Severity: a macro body is the author's code, not
+ SQL data, and the documentation says never to build one from untrusted text, so
+ exploiting it needed an author who did; the guard the documentation described as a
+ defence failed open on 2.0.0. See 14.9 for the earlier claim this corrects.
+3. **The release process could publish a stale `Cargo.lock` (DOCS-1, CI-1).**
+ `RELEASING.md` told the maintainer not to touch `Cargo.lock`, `release.yml` packages
+ with `--locked`, and the failure arrives only after the whole of `ci.yml` has run.
+ REPORTED by `docs` and `ci` (the latter simulated it). Now
+ `scripts/check-release-metadata.py` checks every committed lockfile, the CHANGELOG
+ heading and links, `CITATION.cff`, `SECURITY.md`, the README's latest line and every
+ `quack-rs = "0.X"` snippet, in the `doc` job and in the release `validate` job, and
+ `RELEASING.md` lists the five lockfiles. VALIDATED here: with Cargo.toml at 0.20.0 and
+ the rest at 0.19.0 it listed 12 stale places, and it passes on the final tree.
+4. **The `leak-check` job could not fail for a missing `duckdb_destroy_*` (TESTS-1).**
+ VALIDATED with a canary (`tests/lsan_probe.rs`, nightly, `-Zsanitizer=leak`, prebuilt
+ libduckdb 1.5.6): with `leak:libduckdb.so` in `lsan-suppressions.txt` a leaked
+ `LogicalType` and a leaked `Value` exit 0 (stderr: "Suppressions used"); with the rule
+ removed they exit 23 with a leak report; a leaked Rust `Box` exits 23 either way. The
+ rule matched every frame of an allocation, so it covered every handle DuckDB
+ allocates. It was not needed: all 460 tests of `ffi_roundtrip` exit 0 under
+ LeakSanitizer with no suppression. The job now runs the canary and requires the
+ report. A first attempt at this measurement ran the probe through an empty variable
+ and "passed" vacuously; I caught it from the empty path in the output and redid it
+ with the executable cargo printed.
+5. **The mutation gate cannot see `unsafe fn` bodies (TESTS-2).** cargo-mutants 27.1.0
+ generates no mutants inside them, never mutates integer literals and cannot look into
+ macros; nothing said so, and the "0 missed" results in earlier sections never covered
+ the pointer arithmetic. VALIDATED on this tree: 2,493 mutants are listed with the
+ repository configuration (3,748 with `--no-config`); a heuristic count (comments and
+ strings blanked, braces matched) puts 506 `unsafe fn`s at 5,127 of 22,898 non-test
+ lines (22.4%); on `src/vector/reader.rs` and `writer.rs` the list grows from 6 to 192
+ with the `unsafe` keyword removed from signatures and to 344 with the literals
+ wrapped. `scripts/mutants-expose-unsafe.py` does that on a copy; CONTRIBUTING.md
+ says what the gate cannot see. The `tests` reviewer ran it over the vector layer and
+ found real holes (14.3).
+
+### 14.3 Other findings, by work package
+
+Each package's tests were required to fail on the pre-fix commit. I did not re-run those
+proofs unless the entry says VALIDATED; the final tree passes every suite (14.8).
+
+**`corea`** (REPORTED unless noted).
+- UNSOUND-1: `ChunkWriter` counts rows *claimed*; a claimed row never written was
+ published with stale contents. `release_row` and corrected documentation. The test
+ fails against `3c8ee2d`'s `chunk_writer.rs` (`[1, 2, 7]`, left 2051, right 2050). The
+ conjectured out-of-bounds variant for hand-written `LIST`/`MAP` columns was not built
+ (CONJECTURED).
+- FFI-1: `data_chunk_from_arrow` read past the child for list offsets that are negative,
+ decreasing or past the child, list-view offsets and sizes, struct and union children
+ shorter than their parent, and dictionary indices outside the dictionary
+ (`src/arrow/import_layout/bounds.rs`). The live test died with SIGSEGV on the old
+ code. Not covered: string and binary offsets, whose data buffer length is not in the
+ C Data Interface.
+- HARDEN-6/7/8: logical-type walks bounded at 256 levels (DuckDB's own
+ `LogicalType::ToString` overflows a 2 MiB thread at 6,000 levels, so the cap protects
+ quack-rs's walks only); `set_cardinality` clamped; `WarningCollector` capped at 10,000
+ (a behaviour change, listed).
+- AUTHT2-1: eight type pairs the duplicate-overload guard let through are ambiguous on
+ 1.5.6; they are refused at registration now and five distinct pairs still register.
+ `LIST(T)` beside `ARRAY(T, n)` and a `UNION` whose members are a subset of another's
+ were not detected (READ in `cast_rules.cpp`, not run).
+- AUTHT2-2 `overload_typed`, FFI-6 and FFI-12 (bind data), FFI-8 (the aggregate
+ name-collision diagnosis), LASTDIFF-9 (three live tests that kill mutants).
+- Live `ffi_roundtrip` as the package reported it: 1.5.6 413/413, 1.4.4 277/277, 2.0.0
+ 412/413 (the failure was SQLMACRO-1, outside its files).
+
+**`coreb`** (REPORTED; the pre/post evidence is in the commits).
+- FFI-2/FFI-3: registration is transactional (`BEGIN` before the closure, `COMMIT` on
+ `Ok`, `ROLLBACK` on `Err` or a panic), with `Registration::Autocommit` to opt out. A
+ probe extension that fails after step N leaves 0 catalog entries on 1.4.4, 1.5.6 and
+ 2.0.0 and the retry succeeds (before: 6 entries stay and the retry is refused); a `LOAD`
+ inside the host's own transaction does not wait; one `BEGIN`/`COMMIT` pair costs
+ 0.27–0.38 ms and the `hello-ext` `LOAD` medians went 37.6 → 36.5 ms (1.5.6),
+ 76.8 → 61.0 ms (2.0.0), 40.9 → 37.1 ms (1.4.4) over 30 interleaved runs each. Not
+ rolled back, because DuckDB keeps them in the database configuration: casts,
+ replacement scans, configuration options. The copy-format probe moved to its own
+ connection because 2.0 aborts a transaction on any failed statement.
+- FFI-5: `panic = "abort"` fails to compile in the extension's crate (scratch cdylib:
+ abort fails with the message, unwind builds, `RUSTFLAGS=-C panic=abort` fails;
+ `cargo check --target wasm32-unknown-emscripten` passes; no emscripten link was run).
+- DEPS-12: `#[unsafe(no_mangle)]` in the macros. The `734bb15` macros give four "unsafe
+ attribute used without unsafe" under edition 2024; the library itself compiles under
+ edition 2024. Remaining blockers are in tests (nine E0716 temporaries, an
+ `extern "C"` block without `unsafe`).
+- DEPS-13: four enums are `#[non_exhaustive]`. FFI-7: the catalog scan cannot be
+ narrowed (`duckdb_functions()` has no pushdown). FFI-4, FFI-10: documented.
+- DuckDB 2.0.x layout (decided in section 13.3): not extrapolated. `AbiPolicy::Strict`
+ keeps refusing a DuckDB release a `duckdb-1-5*` build was not verified against;
+ 2.0.1 is due 2026-11-18 and a quack-rs release listing it should follow (14.4).
+
+**`api`** (REPORTED).
+- APIGAPS-2/-3: `Value::try_cast`, `array_items`, `union_tag`, `bit`, `as_bit`, `bignum`,
+ `as_bignum`. The C getters' aborts were reproduced first with a raw-call probe on
+ 1.4.4, 1.5.6 and 2.0.0 (exit 134 on NULL and `'abc'`, 139 on a null handle); every such
+ input is `None` now.
+- Statement metadata (`StatementType`, `parameter_type`, `columns`, `table_names`),
+ `Appender::for_query`, `from_raw`/`into_raw` on four owner types, typed extra info on
+ four builders, typed `COPY TO` state, `FileHandle` and `ClientContext` `Send`
+ (justified from `file_system-c.cpp`, byte-identical on 1.5.6 and 2.0.0; not `Sync`).
+- `duckdb_prepared_statement_type` aborts on a failed prepare (1.5.6): guarded.
+- Mutation gate on the new code with the PR job's flags: 0 missed.
+- APIGAPS-5 (a typed query error) declined: it needs a central error-type change.
+
+**`scaffold`.**
+- CONFORM-1, HARDEN-1/2: `parse_description_yml` follows PyYAML 6 `safe_load`.
+ VALIDATED here: on the 395 descriptors of community-extensions `845779e` quack-rs and
+ PyYAML agree on all 395, and on 3 x 39,500 random mutants (characters from a
+ YAML-significant alphabet deleted, inserted or replaced; lines duplicated, deleted,
+ swapped or re-indented; truncation) quack-rs accepted none that PyYAML refuses and
+ never panicked. In the other direction quack-rs refuses more than PyYAML (it accepted
+ 18,398, 18,080 and 18,426 of the mutants against PyYAML's 24,846, 24,501 and 24,820);
+ what separates them was not itemised. The package reports 8 deliberate strictnesses and
+ hostile inputs that took 3.8 to 21.5 s now taking at most 330 ms (REPORTED).
+- CONFORM-3: 44 reserved names (10 aliases and 34 core extensions) checked against
+ DuckDB and the community list by `scripts/check-extension-names.py`, which passes here
+ and reports the package's 8 of 8 mutants caught (REPORTED).
+- CONFORM-4/5/6/12/13/14/15, HARDEN-5, DEPS-1/3, DOCS-8: the scaffold writes a native
+ example function, a SQLLogicTest with the upstream header, a `make test` that fails when
+ the runner skips a file, `README.md`, the license text, `deny.toml`,
+ `docs/function_descriptions.csv`, a hardened `extension-ci.yml`, and refuses names and
+ platforms the community build ignores. VALIDATED here: a freshly generated project
+ (`quack_scaffold`, 15 files) builds, is stamped by `append_metadata` and its LOAD works
+ on 1.4.4, 1.5.6 and 2.0.0 including `-readonly` against an on-disk database; its unit
+ test and `clippy -D warnings` pass; `cargo-deny` 0.20.2 passes on it (advisories,
+ bans, licenses, sources); the chapter of the book matches the generated tree, the v1.5.6
+ bindings pin (`~1.10506.0`), the refusal of an unstable 2.x target, the dual-license
+ file names and the `LICENSE.TODO` fallback.
+- HARDEN-3/4, LASTDIFF-6, FFI-11 (`append_metadata`: exclusive random temporary file,
+ mode kept, symlink output written through, non-UTF-8 argument, FIFO input) and HARDEN-12
+ (the build script forwards only an all-digit `vX.Y.Z`): REPORTED to fail against
+ `734bb15`. Windows behaviour of the rename and permission copy was not tested, and a
+ non-writable output directory (the rename needs a writable directory) is CONJECTURED.
+- `hello-ext`: `gen_series_v2` refuses a value past BIGINT and uses the guard macros
+ (FFI-9); its checks are 32 now.
+
+**`infra`** (REPORTED; the scripts and the YAML I re-ran, see 14.8).
+- CI-1/DOCS-1 (above), CI-2 (a missing guard script no longer reads as "upstream
+ unreachable"; on a tag an unreachable upstream fails), CI-3 (the Rust toolchain is
+ pinned to 1.99.1; an informational `latest-stable` job tracks the next one), CI-6/
+ HARDEN-11 (the `.crate` hash is compared before the upload), CI-7 (the aggregate check
+ no longer passes while a gate is removed; 18 mutation probes), CI-8, CI-9, CI-11
+ (actionlint), DEPS-4/6 (cargo-deny over the CI-only graphs, typos, taplo, machete,
+ `.editorconfig`), DEPS-15 (the scaffold's action pins are checked against the
+ workflows'), CI-13, and Dependabot #145 folded in.
+- Every action SHA and tool SHA-256 re-checked against upstream; shellcheck over 158
+ extracted `run:` blocks found the same 7 pre-existing notes as the baseline; the
+ release `validate` job simulated for v0.20.0 and for the documented hotfix flow.
+- Runner images are pinned (`ubuntu-24.04`, `macos-26`, `windows-2025`) and
+ `check-action-pins.py` fails on a floating label (CI-12). I first declined this as
+ unverifiable and reversed it when `infra` showed each label is a real image: the
+ decision was wrong and is recorded in 14.10.
+
+**`docs`** (REPORTED, and spot-checked by me where stated).
+- The tutorial now teaches guarded callbacks and its output was produced by building the
+ page's code and loading it into 1.4.4, 1.5.6 and 2.0.0. The window shapes that crash a
+ C API aggregate are listed as measured (one process per shape, 5,000 and 300,000
+ rows). `std::fs` bypassing `enable_external_access` is documented (AUTHT-1, new pitfall
+ P17). Four new pitfalls (38 in all), and `scripts/check-pitfalls-parity.py` keeps
+ `LESSONS.md` and the book on the same list. `known-limitations.md` is regrouped,
+ the upgrade guide "Upgrading from 0.18 to 0.20" is new, README and publishing pages
+ follow the community build (reserved names, the four revision keys, the 32-byte footer
+ field, reduced CI on pull requests, not running `strip`).
+- Five DuckDB 2.0 behaviours missing from section 12.4 are in `known-limitations.md`:
+ `duckdb_appender_close` returns an earlier flush error, `duckdb_data_chunk_set_size`
+ ignores a rejected size in release builds, every C API scalar is `SetFallible()`, the
+ constant-vector marking of an all-constant scalar call is gone, and `COPY` options with
+ several values of different types arrive as `TUPLE` (READ in the 1.5.6 and 2.0.0
+ sources, not run). Measured and also written down: `VerifyNullHandling` is compiled in
+ 2.0.0 and gated by `debug_verification_mode`; a table function's bind runs on every
+ `EXECUTE`; `INTERVAL '1 month' - INTERVAL '1 day'` equals `'29 days'`; a bare identifier
+ table-function argument warns.
+- Written from the packages' WIP: I re-checked the scaffold chapter against generated
+ projects (above), and removed the review names that had leaked into published pages.
+- Not verified: Windows and macOS rendering; the Go `extbuild` behaviours the `conform`
+ reviewer read; `duckdb_get_bignum` (same code as `duckdb_get_bit`, not run); static
+ registration (read in 2.0.0's `extension_load.cpp`).
+
+**The `tests` reviewer's findings, and what I did with them** (merge phase).
+- TESTS-1, -2: 14.2.
+- TESTS-3 to -7: 35 new tests (`wide_rows`, `vector_capacity_guard`, twelve Arrow
+ layout-walker unit tests, `abi_policy_enforced`, `lifecycle_gaps`,
+ `connection_registrar`, `oracle_temporal`, `property_inputs`, `lsan_probe`), each
+ written against a surviving mutant. The reviewer proved the kills on `3c8ee2d` with the
+ bodies of `unsafe fn`s exposed; I ran them on the merged tree against 1.5.6 and 2.0.0.
+ Three adaptations: the Arrow shapes now carry `indices` (a field `corea` added), one
+ test was dropped (a list starting 2^40 elements into a three-element child is refused
+ by the new bounds check before the arithmetic it targeted, and
+ `list_offsets_outside_the_child_are_refused` covers the refusal), and the timestamp
+ oracle skips the values DuckDB 2.0.0 cannot answer (below).
+- **A DuckDB 2.0.0 defect found by that oracle (VALIDATED, not reported upstream):**
+ `CAST(make_timestamp(m) AS VARCHAR)` for `m` below -9223372022400000000 raises
+ `INTERNAL Error: Scalar function "__cast" threw an execution error, but the function is
+ not marked as fallible ... Date out of range in timestamp conversion`, which `TRY()`
+ does not catch; 1.x answers NULL. quack-rs's render guard refuses such values before
+ they reach the engine.
+- TESTS-9: `parse_version` refused a signed part; fixed, with a proptest whose saved
+ failing case (`0.+0.0`) is checked in as `tests/property_inputs.proptest-regressions`.
+- TESTS-10: the upstream-comparison scripts returned 2 ("could not check") for an HTTP
+ 404 and for a reshaped file as well as for an unreachable network (VALIDATED for the
+ platform list: `--ref this-ref-does-not-exist` exited 2). Now 4xx and shape surprises
+ return 1 and the scheduled run fails a 2.
+- TESTS-13/14: four fuzz targets with seeds (the fuzz job copies `fuzz/seeds//`
+ and would have failed without them); `QUACK_REQUIRE_TOOLS=1` turns the scaffold tests'
+ silent skip into a failure on the Linux test leg.
+- TESTS-11 (partly): `src/callback.rs` is mutated now (5 mutants, 4 caught, 1 unviable).
+- TESTS-12, -15: two comments corrected (Miri cannot run the Arrow module; `validity.rs`
+ pointed at the wrong test file).
+
+**Manifest, packaging and dependency changes** (mine).
+- DEPS-2: `libduckdb-sys` and `duckdb` are `>=1.4.4, <1.20100` everywhere (was `<2`).
+ VALIDATED: `cargo check --all-targets` and `cargo test --lib` pass on the 1.4.4,
+ 1.10506.1 and 1.20000.1 bindings (867 tests each before the packages merged); deleting
+ the 1.20000.1 bindings' root re-export (marked "TODO: Remove in the future" upstream)
+ gave 214 compile errors, which is why the ceiling stops below 1.20100. A test fails if a
+ manifest states another range.
+- DEPS-7: an `include` allow-list; `cargo package --list --locked` lists 291 files,
+ 191 under `src/` and 87 under `tests/`, and `cargo publish --dry-run --locked` builds
+ the packaged crate.
+- `build.rs` fails in seconds, naming the directory, when `libduckdb-sys` reports an
+ include directory without `duckdb.hpp` (CI run 705's informational job polled for ten
+ minutes on a partial cache). Not exercised against a source build here.
+- Lockfiles: see the CHANGELOG. Dependabot #143, #148 and #149 are folded in by hand
+ and should close as superseded once the pull request merges.
+- Rustdoc the engines contradict was corrected (`44f67d7`); `check-api-coverage.py`
+ now reports 457 of 546 slots wrapped and 89 not used (8 candidates, 45 deprecated, 23
+ host-side, 13 superseded), after I removed `duckdb_extract_statements_error` from the
+ allowlist (a stale line that would have failed the `abi-table` job).
+
+### 14.4 Not done, declined or rejected
+
+- **The pure-Rust replacement of the C++ test shim** (`deps` patch 0004, a prototype that
+ generates the extern declarations in `build.rs`) is not applied. It passed the live
+ suite on three bindings and on Rust 1.86.0 in the reviewer's runs, but it touches the
+ `bundled-test` build, which I cannot run here, and Windows and macOS, which nobody has
+ run it on. It only affects the test harness (the shipped extension does not use it).
+ Worth a release of its own with CI as the judge.
+- **`tests` patch 04 (strip a leading `_` from every exported symbol) is rejected.** The
+ tree already tells a Mach-O symbol table from an ELF one by its magic number (the
+ `lastdiff` fix, `734bb15`); the reviewer's repro input produces no warning now, and an
+ ELF symbol `_my_ext_init_c_api` really is a different symbol DuckDB cannot load, which
+ the blanket strip would have silenced.
+- APIGAPS-5 (a typed query error), AUTHT2-8..14 (list builders for `LIST(STRUCT)` and
+ `LIST(LIST)`, further helpers), DEPS-10 (the exact `tempfile` pin; `criterion` for one
+ bench), DEPS-14 (splitting `append_metadata` out of the runtime crate) and the wrappers
+ for the 8 candidate slots (among them `duckdb_geometry_type_get_crs`) are left for a
+ later release. `src/copy_function/info.rs` is 735 lines against the 500-line guideline
+ (debt, marked here).
+- TESTS-11: no scheduled full mutation sweep was added (the sweep is
+ `workflow_dispatch` only, and the pull-request job runs `--lib` over changed files), and
+ the files whose mutants only a live engine can kill remain un-mutated in CI. TESTS-12
+ and -16, as nits: `tests/secret_zeroize.rs` cannot run under Miri (its allocator reads
+ uninitialised memory), `INSTALL icu` writes `~/.duckdb`, the interrupt test cancels
+ after a fixed 250 ms, and four test binaries repeat a ten-line fixture.
+- UNSOUND-2 (the 64-bit `FfiState` tag) is declined: the documented mitigation stands.
+- DEPS-9: only `proptest` and `cc` were updated in the root lock; the other compatible
+ updates (`arrow` 58 → 60, for one) are not applied, because `libduckdb-sys` and
+ `duckdb` must stay on the 1.5.6 bindings the lock is for.
+- A DuckDB 2.0.1 layout is not extrapolated (14.3, `coreb`).
+
+### 14.5 For the owner (outward-facing; nothing here was done)
+
+- Yank 0.3.0–0.15.0 and file RustSec advisories for the heap-corruption defects fixed in
+ 0.16.0 (HARDEN-10). The advisory drafts are in the review's scratch directory, not in
+ the repository.
+- Move the crates.io token into the `crates-io` environment and configure Trusted
+ Publishing (HARDEN-9; `RELEASING.md` has the order).
+- Report `duckdb_register_log_storage` freeing its extra data twice when the name is taken
+ (APIGAPS-1; present from DuckDB 1.5.0 to `main`; quack-rs does not wrap it).
+- Disable Cloudflare's email obfuscation for the book, or keep the reworded page (DOCS-6).
+- Branch protection, the `crates-io` environment and the tag ruleset could not be read
+ (CI-UNREADABLE); confirm `CI passed` is the required check.
+- CI-4 (the release gate depends on DuckDB `latest`, extension-ci-tools branches and
+ PyPI) and CI-10 (duplicate push and pull-request runs on `claude/**`) are policy
+ decisions.
+- `SECURITY.md` now supports 0.20.x and 0.19.x and marks 0.18.x end of life.
+
+### 14.6 Not verified
+
+Windows, macOS and WebAssembly (no local runner; the `test` matrix, the Wasm job and
+the `scaffold-e2e` job are the check); the `bundled-test` source build on any
+platform; live engines other than 1.4.4, 1.5.6 and 2.0.0 (CI's older-engine matrix
+covers 1.4.5, 1.5.0 and 1.5.3); AddressSanitizer, Miri and valgrind over the new code
+(CI runs `asan`, `leak-check` and `miri`; `leak-check`'s command ran here without
+`-Zbuild-std`); every changed workflow as GitHub runs it, including the new canary step,
+the `apt-get` step, the scheduled strictness and the release workflow; the fuzz targets
+beyond 60 to 120 seconds each; the `ListBuilder` change under valgrind (the release
+libduckdb's own allocator hides vector buffers from valgrind and AddressSanitizer: 14.9).
+
+### 14.7 Performance
+
+Measured with `benches/engine/compare.py` on a quiet shared 4-vCPU virtual machine
+("Intel(R) Xeon(R) Processor @ 2.10GHz", release build, fat LTO, one thread, base =
+`git archive v0.19.0`, new = this tree; both stamped by `append_metadata`; results
+checked equal before timing; load average at start 1.5 on 1.5.6 and 1.1 on 2.0.0).
+
+**DuckDB 1.5.6, 6 sessions x 3 timed runs (n = 18).** Seconds per query (median, min–max) and median ns per row; the hand-written twin is the floor, not a goal.
+
+| Query | 0.19.0 | now | hand-written |
+|---|---|---|---|
+| nullable BIGINT, map1 | 0.156 (0.140–0.197) s, 7.80 ns | 0.117 (0.113–0.204) s, 5.88 ns | 0.123 (0.110–0.157) s, 6.17 ns |
+| nullable BIGINT, map2 | 0.203 (0.182–0.243) s, 10.15 ns | 0.132 (0.118–0.186) s, 6.60 ns | 0.123 (0.112–0.161) s, 6.15 ns |
+| non-NULL BIGINT, map1 | 0.041 (0.034–0.062) s, 2.05 ns | 0.041 (0.034–0.073) s, 2.07 ns | 0.034 (0.031–0.056) s, 1.70 ns |
+| VARCHAR -> BIGINT, short strings | 0.671 (0.582–0.854) s, 33.55 ns | 0.488 (0.446–0.705) s, 24.40 ns | 0.434 (0.377–0.550) s, 21.73 ns |
+| VARCHAR -> BIGINT, long strings | 0.448 (0.415–0.595) s, 44.80 ns | 0.442 (0.407–0.561) s, 44.15 ns | 0.417 (0.369–0.585) s, 41.75 ns |
+| VARCHAR -> VARCHAR trim, short | 1.225 (1.111–1.423) s, 61.25 ns | 1.077 (1.020–1.211) s, 53.83 ns | 0.794 (0.750–0.962) s, 39.68 ns |
+| BIGINT -> [x, x+1, x+2] | 0.226 (0.213–0.295) s, 11.28 ns | 0.138 (0.125–0.189) s, 6.88 ns | 0.101 (0.092–0.126) s, 5.03 ns |
+| BIGINT -> [x, x+1, x+2, NULL] | 1.206 (1.086–1.423) s, 60.30 ns | 0.214 (0.197–0.249) s, 10.72 ns | 0.112 (0.091–0.156) s, 5.58 ns |
+
+**DuckDB 2.0.0, 4 sessions x 3 timed runs (n = 12).** Seconds per query (median, min–max) and median ns per row; the hand-written twin is the floor, not a goal.
+
+| Query | 0.19.0 | now | hand-written |
+|---|---|---|---|
+| nullable BIGINT, map1 | 0.142 (0.135–0.191) s, 7.08 ns | 0.124 (0.105–0.136) s, 6.20 ns | 0.121 (0.107–0.139) s, 6.08 ns |
+| nullable BIGINT, map2 | 0.217 (0.194–0.245) s, 10.88 ns | 0.123 (0.115–0.137) s, 6.17 ns | 0.121 (0.115–0.147) s, 6.08 ns |
+| non-NULL BIGINT, map1 | 0.044 (0.041–0.059) s, 2.20 ns | 0.045 (0.040–0.061) s, 2.27 ns | 0.042 (0.038–0.063) s, 2.08 ns |
+| VARCHAR -> BIGINT, short strings | 0.768 (0.609–0.928) s, 38.38 ns | 0.522 (0.449–0.719) s, 26.08 ns | 0.428 (0.395–0.614) s, 21.40 ns |
+| VARCHAR -> BIGINT, long strings | 0.619 (0.500–0.829) s, 61.90 ns | 0.549 (0.464–0.714) s, 54.90 ns | 0.487 (0.451–0.623) s, 48.75 ns |
+| VARCHAR -> VARCHAR trim, short | 1.324 (1.197–1.492) s, 66.20 ns | 1.220 (1.045–1.365) s, 61.00 ns | 0.952 (0.823–1.089) s, 47.62 ns |
+| BIGINT -> [x, x+1, x+2] | 0.307 (0.285–0.386) s, 15.38 ns | 0.163 (0.152–0.220) s, 8.15 ns | 0.104 (0.098–0.118) s, 5.22 ns |
+| BIGINT -> [x, x+1, x+2, NULL] | 1.241 (1.154–1.416) s, 62.05 ns | 0.257 (0.221–0.319) s, 12.85 ns | 0.105 (0.100–0.141) s, 5.28 ns |
+
+
+Reading: a difference is claimed only where the old build's fastest run is slower than
+the new median. That holds for `map1` and `map2` on a nullable column, short strings and
+both `ListBuilder` shapes on both engines, and for `trim` on 1.5.6 only; it does not
+hold for the control or the long strings on either engine, or for `trim` on 2.0.0. The
+`ListBuilder` NULL-element row is 10.72 ns on 1.5.6 here, against 8.60 ns in the table
+the CHANGELOG carried before I re-measured, in the same code path that `734bb15` changed
+to fetch the entry array on every call; I did not isolate the cost of that fix from
+run-to-run noise.
+
+### 14.8 How this pass was verified
+
+On the final tree (commands as CI runs them, with `RUSTFLAGS="-D warnings"`; a gate that
+printed 0 is listed with the exit code):
+
+- Formatting, `clippy --all-targets -D warnings` with default features, `duckdb-1-5`,
+ `duckdb-1-5-3`, `duckdb-1-5-4`, and the four live configurations
+ (`bundled-test-prebuilt` alone and with `duckdb-1-5`, `duckdb-1-5-3`, `duckdb-1-5-4`); `cargo doc --no-deps`
+ with and without `duckdb-1-5-4` under `RUSTDOCFLAGS="-D warnings"`: all exit 0.
+- `cargo test --all-targets`: 1,109 tests pass in 20 binaries (default features); 1,249
+ with `duckdb-1-5-4`; doctests 235 pass, 3 ignored.
+- Live suites (`bundled-test-prebuilt`, prebuilt libduckdb of the engine's own version),
+ `cargo test --all-targets --no-fail-fast`: **1.5.6** (bindings 1.10506.1, `duckdb-1-5-4`)
+ 1,789 pass, 0 fail; **2.0.0** (bindings 1.20000.1, `duckdb-1-5-4`) 1,789 pass, 0 fail
+ (four tests failed once with "No space left on device" when the disk filled, and passed
+ when re-run alone: the total is the 1,785 of the first run plus those four); **1.4.4**
+ (bindings 1.4.4, default features) 1,479 pass, 0 fail. All 21 test binaries ran each time.
+- The book and README code blocks (`cargo test --doc --manifest-path book/doctest/Cargo.toml`,
+ libduckdb 1.5.6, bindings 1.10506.1): 255 pass, 5 ignored, 0 fail.
+- LeakSanitizer (nightly, `-Zsanitizer=leak`, no `-Zbuild-std`) over `ffi_roundtrip`
+ with the empty suppression file: 460 tests, exit 0. The four new fuzz targets ran
+ 60 s each (`description_roundtrip` 120 s more after its fix, 787,689 inputs).
+- MSRV: `cargo +1.86.0 check` with and without `duckdb-1-5-4`, after the
+ fallback resolver picked `libduckdb-sys` 1.10506.0: exit 0.
+- `hello-ext` on the 1.20000.1 bindings: 43 unit tests, clippy, and its 32 live checks
+ pass on 1.4.4, 1.5.6 and 2.0.0 (32/32 each).
+- Scripts: every check of the `doc` job (`sync-book-changelog`, `generate-sitemap`,
+ `check-workflow-expressions`, `check-ci-aggregate` with 37 blocking jobs,
+ `upstream-guard --self-test` with 14 cases, `check-release-metadata`,
+ `check-toolchain-pin`, `check-action-pins`, `check-pitfalls-parity` with 38 pitfalls),
+ `check-api-coverage` and its self-test (23 expectations), `check-extension-names`,
+ the four upstream guards against live upstream (the ABI table over every release
+ header), `actionlint` with CI's `-ignore SC2129`, `typos`, `taplo`, `cargo-machete`,
+ `cargo-deny` over the root and the four CI-only crates, `cargo package --list --locked`
+ and `cargo publish --dry-run --locked`: exit 0. `zizmor --offline` still exits 12 on
+ informational and low-confidence findings (the `infra` package counted 50 before its
+ changes and 53 after, the three new ones being the pedantic `superfluous-actions`
+ note); it is not a CI gate.
+- `mdbook build` (0.4.40), the per-page SEO check (52 pages), the book and README link
+ check (327 relative and 52 docs.rs links) and the versioned-site build pass.
+- `cargo semver-checks` against v0.19.0 (`--release-type patch`): the four
+ `#[non_exhaustive]` enums and `ScalarBindInfo` no longer being `RefUnwindSafe`, nothing
+ else; all five are in the CHANGELOG.
+- **The pull request's own CI found eleven more problems after every gate above had passed
+ locally.** Items 1 to 10 needed no change to the shipped library code (item 7 also records
+ a limitation of it); item 11 changed the description.yml reader (MUT-1).
+ 1. Windows: `-D warnings` turned unused items in Unix-only test modules into errors
+ (helpers in `scaffold/tests_make.rs`, `temp_files` and a `Path` import in
+ `append_metadata/tests_output.rs`); they are `cfg(unix)` now.
+ 2. macOS: APFS refuses to create a file whose name is not UTF-8 (`EILSEQ`), so
+ `a_file_name_that_is_not_utf8_is_used_not_a_panic` cannot run there; it is gated off
+ macOS.
+ 3. `scaffold-e2e`: the regular expression that rewrites the scaffolded dependency line
+ dropped its `features`, and the assertion had not followed a change in the scaffold.
+ 4. WebAssembly: the test run links with `-C link-arg=-sSTACK_SIZE=8MB` (emscripten's
+ default stack is 64 KB); the job passed on run 724.
+ 5. A nightly Cargo lays build scripts out as `build///out/`;
+ `tests/build_script_env.rs` only knew the old layout and finds both now.
+ 6. The DuckDB 2.0 source-build job (informational): `rust-cache` restored `target/`
+ without the `duckdb-sources/` directory the 1.20000 bindings unpack there, so the
+ bindings' build script stopped. The job caches the registry only; it passed on run 724.
+ 7. `hostile_input::a_deeply_nested_type_is_refused_not_recursed_into` aborted with a
+ stack overflow on the macOS and Windows source builds and once on Linux with the 1.4.5
+ library, and never locally. Cause (a gdb backtrace on a 512 KiB stack, 1.5.6): the
+ test itself registered a scalar set whose parameter was 3,500 levels deep, and the
+ next `register_table` ran its `duckdb_functions()` name check, in which DuckDB's own
+ `LogicalType::ToString` recurses once per level; how much stack that takes depends on
+ how DuckDB was built. quack-rs lets a parameter type nested past its 256-level cap
+ register (a return type past it is refused), so an extension that registers such a
+ type makes every later `duckdb_functions()` scan, its own and other extensions',
+ overflow. That is a limitation, not fixed here: an extension must bound the depth of
+ types it builds from untrusted input. The test is replaced by one that stops at the
+ cap and asserts what the cap changes (a return type nested 256 levels registers, 257
+ is refused; overloads that differ only 300 levels down are refused as duplicates);
+ changing the cap to 255, 257 or 100000 fails it. A type past the cap is refused with
+ the ANY/INVALID message, which does not mention the depth.
+ 8. Coverage (`cargo llvm-cov`): all tests passed and the report failed with "invalid
+ instrumentation profile data". The instrumented child of
+ `a_failed_write_leaves_nothing_behind_and_names_the_output` runs under `ulimit -f 8`,
+ which cut its 42,888-byte profile off at 4 KiB. Reproduced locally (91 profile files, one
+ bad, 4,096 bytes) and fixed by sending that child's profile to `/dev/null`.
+ 9. Windows (`ffi_roundtrip`): `duckdb_extensions()` could not open a half-written
+ `parquet` file another test thread was installing. `Fixture` installs `icu` and
+ `parquet` once per process now. Not reproducible here; the windows-2025 job decides.
+ 10. Miri: the job's two runs (native, then s390x) share a 60-minute limit and the native one
+ alone took 41.6 minutes; by the timestamps in its log, 1,086 s went to one parser test on
+ 100,000-level inputs and 160 s to an SPDX test, both of which use smaller inputs under
+ Miri now. The limit is 90 minutes. The next run took 18.1 and 17.5 minutes (MUT-2).
+ 11. Mutation testing: one job for all of a 102-file pull request (1,639 mutants) was stopped
+ by its runner after 33 and 55 minutes, having reported 6 and 12 survivors. It is eight
+ shards now (`--shard k/8 --sharding round-robin`) and a gate job with the old check name;
+ all eight finished in 22 minutes and tested 1,617 mutants: 85 missed and 17 timed out,
+ 101 of them in the YAML reader added in this release and one in `arrow/import_layout`.
+ Four engineers (subagents, each in its own worktree) worked the YAML ones from PyYAML's
+ verdicts, not the reader's; I merged and re-checked their results.
+ - 79 of the 85 missed were killed by 33 new tests in four modules (`tests_mut_*.rs`) and
+ three regression tests (below); every row's expectation is PyYAML 6.0.3's, except rows
+ that pin the reader's own documented limits (nesting of 64, no anchors in a field it
+ reads, no explicit `? ` keys, error wording), each commented as such.
+ - 14 of the 17 timeouts were loops that spin when an index update is mutated. Each loop
+ now carries a `debug_assert!` that a round made progress (flow `skip`, `seq`, `map`
+ and `plain`; block `parse_mapping` and `parse_seq`; `split_lines`), so a mutant fails
+ instead of hanging. Release builds are unchanged. The other 3 were mutants of
+ `value_len`'s memo, which made a run of k blank lines cost k² steps: it is a linear
+ scan now.
+ - 4 of the missed were equivalent, and the code that made them so is gone: a parameter
+ of `value_len` that every caller passed as `false`, a `check_code` call on text that is
+ only spaces, the text of a comment that no caller read (`split_comment` returns whether
+ there is one), and a `<=` in the memo.
+ - 2 of the missed, and one more found while probing, were PyYAML disagreements in the
+ reader (CHANGELOG, Fixed). Random mutation of the 395 published descriptors (118,500
+ files) had found none of them, so the claim that the reader never accepts what PyYAML
+ refuses was wrong at the edges. After the fixes, 16,195 mutated descriptors and
+ 14,874 targeted documents show no regression against PyYAML; the reader still refuses
+ a tab-only line inside a block scalar and a tag alone on a line, which PyYAML accepts
+ (not fixed).
+ - A fourth disagreement came from checking an engineer's remark that PyYAML refuses a
+ mapping key continued on a second line inside `[ … ]`. The reader refused that file
+ only because `extension:` takes no nested flow collections; in a section it merely
+ checks (`docs:`) it accepted it, and `{ … }` in both. PyYAML keeps a flow key only if
+ its `:` is on the key's line and within 1,024 characters of the key's start. 30,120
+ random flow collections: 109 disagreements before the fix, all false accepts; none
+ after, none new, no panic. `cargo mutants` on the functions touched: 38 mutants, 38
+ caught. The 118,500-file random run of the published descriptors reported no false
+ accept and so did not find this one; the flow-collection generator and the question
+ about a split key did.
+ A local `cargo mutants` run of 195 mutants in the functions that changed or had
+ survived: 184 caught, 11 unviable, 0 missed, 0 timed out. The sharded run on the next
+ head is the check of the other mutants.
+- What 14.6 lists as unverified on Windows, macOS, WebAssembly and the source builds is
+ answered by the pull request's final CI run, which the pull request page shows.
+
+### 14.9 Corrections to earlier sections
+
+- **Section 13 and `harden`'s "no SQL injection" result** (35 hostile names through ten
+ constructors on live 1.5.6, canary untouched) held on 1.5.6 only. On DuckDB 2.0.0
+ `SqlMacro::register` ran statements ahead of a parse error (14.2, item 2). A hostile-input
+ sweep has to run on every supported engine major.
+- **Valgrind and AddressSanitizer do not see DuckDB's vector data and list-entry
+ buffers.** The release libduckdb allocates them with its own jemalloc (769 `duckdb_je_*`
+ symbols); earlier sections that read a clean valgrind or ASan run as covering them were
+ wrong, as the `ListBuilder` regression (14.2, item 1) showed. CONTRIBUTING.md says what
+ the sanitizer runs can and cannot see.
+- **"0 missed" mutation results never covered `unsafe fn` bodies** (14.2, item 5).
+- **The `leak-check` job did not check what its comment said** until 14.2, item 4.
+- Section 13.5 item 1 lists `check_string_len`, `try_write_varchar`/`try_write_blob`,
+ `ArrowArray::take_from`, `ArrowSchema::take_from`, `ErrorData::has_error`,
+ `WarningCollector::drain`, `set_max_threads`, `set_cardinality` and query progress as
+ unreached; the `tests` reviewer's line coverage of the live, unit and integration run
+ executes all nine (REPORTED). Section 13.2 item 14 put the unexecuted public functions
+ at 144; on `3c8ee2d` the reviewer counted 170 production functions never executed (29
+ containing `unsafe`), none in `struct_reader`, `struct_writer`, `validity` or
+ `MapVector` (REPORTED).
+- The 13.7 timings are superseded by 14.7. The CHANGELOG's coverage figure "445 of 546
+ slots wrapped" was a count at one moment; the script reports 457 now.
+- The "28 fuzz files" figure for the fuzz crate counted seeds and a lockfile; it had four
+ targets, one of them live, and has eight now.
+- Section 12.4 lacks five DuckDB 2.0 behaviours (14.3, `docs`).
+- Section 13.3 recorded that a core-extension name list for the scaffold was not
+ shipped, because `duckdb_extensions()` mixes core and autoloadable names. The list is
+ derived from DuckDB's own sources now (`scripts/check-extension-names.py` compares
+ it with `main` and `v2.0.0` and with the community list), and the scaffold and
+ `parse_description_yml` refuse its 10 aliases and 34 core names, because
+ `INSTALL s3 FROM community` fails on 2.0.0 (the alias is applied before the
+ repository is asked).
+
+### 14.10 Process lessons (mine)
+
+- A mutation proof must name the pre-fix commit. My first proof of the `ListBuilder`
+ fix reverted a file with `git show HEAD:...` and then I committed; `HEAD` already held
+ the fix and the "without the fix" run passed vacuously. I redid it against `3c8ee2d` and
+ grepped the reverted file for `self.entries` to confirm it held the old code.
+- Run a new live test on every engine major before committing it (the lambda syntax).
+- A check that prints nothing proves nothing: an empty variable made the first LeakSanitizer
+ probe run no probe, and `cmd | grep` let a failing `clippy` run reach a commit; both were
+ caught before they were pushed, one by the empty path in the output, one by re-running
+ with an explicit status check.
+- A full disk invalidates results quietly: four tests failed with ENOSPC and the other 1,785
+ passed; I re-ran only after reading the failure text.
+- Local gates miss what only a script checks: the stale API-coverage allowlist line, the
+ typos findings and two stale "31 checks" counts were found by running every script of
+ the `doc` and `hygiene` jobs, not by the test suites. And a doc comment I had edited in
+ `tests/ffi_roundtrip.rs`, which only the live configurations compile, failed
+ `clippy::doc_markdown` there and in no other gate; I found it by running clippy in
+ those configurations last.
+- A reviewer's snapshot is older than the fixes: `tests` patch 04 would have regressed
+ ELF handling, and two of its Arrow tests needed an `indices` field that did not exist
+ at its snapshot.
+- Random mutation of valid files is weak exactly where a hand-written parser has its edge
+ cases. 118,500 mutants of the published descriptors found no disagreement with PyYAML; the
+ mutation job, asked which boundary comparisons no test pins, found three in a day, and a
+ grid built around each (anchor lines, comment lines in block scalars, key lengths of
+ 1,020 to 1,025 with spaces) showed their extent. Use both.
+- Delegating a mutation sweep works when each worker has its own worktree, its own target
+ directory and a written rule that PyYAML, not the code under test, supplies the expected
+ value: three of four reports found real disagreements instead of encoding them. All four
+ worktrees started from `main`, not the pushed head, and the workers noticed because line
+ numbers did not match.
+- A test whose only assertion is that the process survived says little, and this one
+ created its own hazard. The deep-type test left a 3,500-level type in the catalog and the
+ next registration paid for it; how much stack DuckDB's recursion takes depends on the
+ build, so a run against the prebuilt library on my machine could not show it. Shrinking
+ the thread's stack until the local run failed, then reading the backtrace in gdb, found
+ the cause in minutes. Assert on the outcome, and do not leave a hostile object registered.
+- Coverage instrumentation changes what child processes do: a test that limits a child's
+ file size truncates its profile. `rustup component add llvm-tools-preview` and
+ `cargo llvm-cov` reproduce the Coverage workflow locally; I had not tried.
+- A check's `head_sha` has to be compared with the branch head before reading its failure.
+ Runs cancelled by a newer push report `CI passed` as failed (runs 721 and 722), and
+ each push restarts every job, so fixes should be batched.
+- A decision made for lack of verification can be wrong: I declined the runner pins
+ because I could not confirm the labels offline, and `infra` confirmed them.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12b133ee..a19a6331 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+## [0.20.0] - 2026-10-30
+
+This release comes out of an eighth review (`AUDIT.md` section 14): twelve parallel
+reviewers (soundness, the diff since 0.19.0, C API gaps, conformance with DuckDB's
+community-extension process, documentation, CI and supply chain, hardening, the FFI
+boundary, two authors' passes over tables and types, dependencies, and the test suite
+itself), then six work packages that fixed what they found, each with the tests that fail
+without the fix. It also carries the performance work that review measured against 0.19.0.
+
+**It contains breaking changes**, marked **Breaking:** under Changed: the entry-point
+macros refuse `panic = "abort"`, four public enums are `#[non_exhaustive]`, registration
+is transactional, `generate_scaffold` and `parse_description_yml` refuse inputs they used to
+accept, and a zero-argument aggregate no longer registers. "Upgrading from 0.18 to 0.20" in
+the book lists each with what to do. Two High-severity defects were found and fixed before
+release: a segmentation fault in an unpublished `ListBuilder` optimisation, and
+`SqlMacro::register` executing statements ahead of a parse error on DuckDB 2.0.0 (see
+Security).
+
### Added
- **`TIMESTAMPTZ_NS`, the type DuckDB 2.0.0 added** (`DUCKDB_TYPE_TIMESTAMP_TZ_NS`,
@@ -47,10 +65,144 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
a change to a vector, validity or list path.
- `scripts/check-api-coverage.py` and `scripts/api-coverage-allowlist.txt`: every
slot of `duckdb_ext_api_v1` is either used under `src/` or listed with a class and
- a reason (546 slots at v1.5.6: 447 used, 99 listed: 45 deprecated, 22 host-side,
- 11 superseded, 21 candidates to wrap). The `abi-table` CI job runs it, so a slot a
+ a reason (546 slots at v1.5.6: 457 wrapped, 89 listed: 45 deprecated, 23 host-side,
+ 13 superseded, 8 candidates to wrap). The `abi-table` CI job runs it, so a slot a
DuckDB release adds is noticed instead of left for someone to read the header.
+- **`quack_scaffold` binary** (`cargo install quack-rs --bin quack_scaffold`): writes a new
+ extension project to disk from the command line. Every `ScaffoldConfig` field is a flag
+ (`--description`, `--maintainer`, `--github-repo`, `--version`, `--license`,
+ `--excluded-platforms`, `--target-duckdb-version`, `--unstable-c-api`, `--git-ref`, `--dir`).
+ It refuses to overwrite an existing file unless `--force` is given, and writes nothing if it
+ refuses. It never writes through a symbolic link inside the project. When it is done it
+ prints the next steps: the `git submodule add` command for extension-ci-tools, the build and
+ test commands, and the exact `append_metadata` invocation for stamping without `make`.
+- **`scaffold::generate_scaffold_with_warnings`** and **`scaffold::Scaffold`**: the generated
+ files plus warnings about values that were accepted but deserve a second look. These are: a
+ license id outside quack-rs's shortlist, an excluded platform that is opt-in, excluding every
+ platform, and an unstable-ABI project the community build cannot ship as generated.
+ `generate_scaffold` is unchanged and drops the warnings.
+- **Scaffold: files a community submission needs** (CONFORM-14/15, DEPS-1):
+ - `README.md`: build, test and try instructions, the no-`strip`-after-stamping rule,
+ reduced CI on pull requests, and what to submit.
+ - `docs/function_descriptions.csv`: the duckdb.org extension page takes function
+ descriptions and examples only from this file. It has a row for the example function.
+ - The license text: `LICENSE` for MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD,
+ MIT-0 and Unlicense. An expression gets one `LICENSE-` per license. Anything else gets
+ a `LICENSE.TODO` naming the text to add.
+ - `deny.toml`: ignores RUSTSEC-2026-0367/-0368/-0369 (zip < 9), each with a reason. That
+ zip is only a build-script dependency of `libduckdb-sys` ≤ 1.10506.0, which is what Cargo
+ resolves for `rust-version = 1.86`. The file also allows the dependency graph's licenses
+ and the project's own. `cargo deny check` passes on a freshly generated, locked project;
+ without the ignores it fails on exactly those three advisories.
+- **`validate::validate_community_extension_name`**: rejects a name `DuckDB` itself uses, both
+ its aliases (`s3`, `md`, `postgres`, ...) and its core extensions (`json`, `parquet`, ...),
+ as well as a badly formed one. The parser and the scaffold use it. `validate_extension_name`
+ stays a syntax check only. (CONFORM-3)
+- **`description.yml`: `repo.variegata` / `repo.andium`** (the refs built for DuckDB v1.5.6 and
+ v1.4.5) are read into `ExtensionDescription::git_ref_variegata` / `git_ref_andium`. An
+ unknown `repo` key is now a warning, and so is a `repo` that is not `owner/repo`.
+ (CONFORM-10)
+
+- `ChunkWriter::release_row(row)` gives back the row the last `next_row` claimed, so the
+ drop does not publish it; the next `next_row` returns the same index. Giving back any
+ other row panics with the row that could be given back. For the
+ `while let Some(row) = cw.next_row()` loop whose source runs out (UNSOUND-1).
+- `ScalarFunctionSetBuilder::overload_typed(TypedScalarFunctionBuilder)`: a closure-built
+ function (`map1`, `map2`, `try_map1`, ...) can now be one overload of a set, keeping the
+ signature its closure fixed. This is the only way to overload closure-built functions on
+ DuckDB 1.4.x, where a second registration under a taken name is refused. An overload built
+ under another name than the set's is refused by `register` (AUTHT2-2).
+- `ScalarBindData::try_get`: the bind data, or the panic message of a `Clone` that failed
+ when DuckDB copied the bound call (FFI-6).
+- `WarningCollector::with_limit(n)`, `WarningCollector::dropped()` and
+ `warning::DEFAULT_MAX_WARNINGS` (HARDEN-8, see Changed).
+
+- `entry_point::Registration` (`Transactional`, the default, or `Autocommit`) and
+ `entry_point::EntryPointOptions` (an `AbiPolicy` plus a `Registration`, built with
+ `EntryPointOptions::new().abi_policy(..).registration(..)`; `From`). The entry
+ point macros take them as `entry_point!(name, options = EntryPointOptions::new()..., |con| ..)`
+ (and `entry_point_v2!`); hand-written entry points use the new
+ `init_extension_with_options` / `init_extension_v2_with_options`. The existing
+ `AbiPolicy` arm and `*_with_policy` functions are unchanged (FFI-2).
+
+- **`Value::try_cast(&LogicalType)`** — SQL `TRY_CAST` of a value to any
+ logical type, nested ones included, with no path for a C++ exception to reach
+ Rust (the cast runs inside `duckdb_create_list_value`'s `try`). Payloads
+ `DuckDB` cannot convert safely (out-of-range timestamps, over-wide `DECIMAL`s)
+ are refused first, inside `ARRAY`s too. All engines (1.4.4+), no feature.
+ (APIGAPS-2)
+- **`Value::array_items()`** — the elements of an `ARRAY` (or `LIST`) value; an
+ `ARRAY` read through `list_items` looks empty because the C API reads `LIST`
+ only. A fixed-size-list argument of a table function, `COPY` option or
+ scalar bind is now readable. All engines. (APIGAPS-2)
+- **`Value::union_tag()`** — the active member of a `UNION` value (its member
+ value reads as text through `try_cast` to `VARCHAR`). Feature `duckdb-1-5`.
+ (APIGAPS-2)
+- **`Value::bit(&[bool])`, `Value::as_bit()`, `Value::bignum(&[u8], bool)`,
+ `Value::as_bignum()`** — `BIT` and `BIGNUM` values. The getters never hand
+ `DuckDB` a value its `duckdb_get_bit` / `duckdb_get_bignum` would abort the
+ process on (SQL `NULL`, `'abc'`, a `BLOB`, `inf`; a null handle segfaulted
+ `duckdb_get_bit`); the constructors build only well-formed, canonical values.
+ All engines. (APIGAPS-3)
+- **`query::StatementType`**, **`PreparedStatement::statement_type()`**,
+ **`QueryResult::statement_type()`**, **`PreparedStatement::parameter_type(i)`**
+ — what a statement is and its parameter types before it runs. All engines
+ (stable slots); `COPY DATABASE` / `UPDATE EXTENSIONS` / `MERGE INTO` are
+ reported as `Invalid` by DuckDB before 1.5.5. (apigaps P1)
+- **`PreparedStatement::columns()`** — result column names and types before
+ execution (`None` while they depend on unbound parameters). Feature
+ `duckdb-1-5`. (apigaps P1)
+- **`query::table_names(con, sql, qualified)`**, **`OwnedConnection::table_names`**,
+ **`Connection::table_names`** — the tables a statement touches, without running
+ it (one statement; `JOIN … USING` is refused by DuckDB's names-only bind).
+ Feature `duckdb-1-5`. (apigaps P1)
+- **`Appender::for_query(con, sql, types, table, columns)`** — an appender whose
+ rows feed a query (`INSERT … SELECT`, `INSERT OR REPLACE`, `MERGE INTO`, …)
+ instead of a table. Feature `duckdb-1-5`. (apigaps P1, slot 409)
+- **`from_raw` / `into_raw`** on `Appender`, `PreparedStatement`, `QueryResult`
+ and `OwnedConnection`, so a handle from an unwrapped C call can be adopted
+ into RAII. All engines. (APIGAPS-9)
+- **Typed, tag-checked extra info**: `extra_info_typed(value)` on
+ `AggregateFunctionBuilder`, `AggregateOverloadBuilder`, `CastFunctionBuilder`
+ and `CopyFunctionBuilder`, read with `extra_info_ref::()` on
+ `AggregateFunctionInfo`, `CastFunctionInfo` and the four `Copy*Info` types; a
+ value of another type reads as `None` instead of reinterpreting memory, and
+ the destructor is generated and panic-safe. All engines (copy: `duckdb-1-5`).
+ (APIGAPS-8)
+- **Typed `COPY TO` state**: `CopyBindInfo::set_bind_data_typed`,
+ `CopyGlobalInitInfo::set_global_state_typed`, and `bind_data_ref` /
+ `global_state_ref` on the later phases — no hand-written `Box::into_raw` /
+ `extern "C"` destructors. Feature `duckdb-1-5`. (AUTHT-3)
+- **`FileHandle` and `ClientContext` are now `Send`** (not `Sync`), and
+ `FileHandle::into_raw` exists: a file opened in a `COPY` `global_init` can be
+ kept in the statement's global state (in a `Mutex`) without an
+ `unsafe impl Send` wrapper; the pattern is documented on `FileHandle`.
+ Feature `duckdb-1-5`. (AUTHT-2)
+- **`From for ErrorData`**, so `?` on a `Value` or builder error
+ works inside `Appender::row` with `duckdb-1-5`. (AUTHT-13)
+- **35 new tests from a review of the test suite itself** (`TESTS-3` to `-7`), each
+ written against a mutant that the existing suite let live, found by exposing the
+ bodies of `unsafe fn`s that `cargo-mutants` skips: every fixed-width and string type
+ echoed over 5,000 distinct rows (`wide_rows`, which kills the `u128` stride mutants);
+ the `OwnedVector` capacity guard for `MAP` and `UNION`; twelve Arrow layout-walker unit
+ tests; `AbiPolicy` enforced in-process against a v1.4.4 layout table
+ (`tests/abi_policy_enforced.rs`); `ScalarBindData::get_from_init`, two `extra_info`
+ destructors and `abi::check`; the seven `Registrar` methods of `Connection` that no test
+ executed; randomised cross-checks of `datetime` and `interval` against DuckDB's own SQL
+ (4,000 values per check, fixed seed); and property tests for `parse_version`, SPDX,
+ `description.yml` and the scaffold's escaping. They pass on DuckDB 1.5.6 and 2.0.0.
+- Four fuzz targets, each with a seed directory: `abi_version`, `append_metadata`,
+ `description_roundtrip` and `scaffold_text` (60 s each on this tree; the first run of
+ `description_roundtrip` found its own escaping wrong, because PyYAML refuses U+007F even
+ inside quotes). The fuzz job runs every target `cargo fuzz list` prints.
+- `scripts/mutants-expose-unsafe.py`: prepares a copy of the tree so `cargo-mutants` can
+ mutate what it skips (`unsafe fn` bodies, integer operands of pointer arithmetic). An
+ audit aid, not run in CI; see CONTRIBUTING.md, "Mutation testing". [TESTS-2]
+- `tests/lsan_probe.rs`: the `leak-check` job's canary. It leaks a Rust allocation, a
+ `LogicalType` and a `Value` on request, and the job now requires LeakSanitizer to fail
+ on each. [TESTS-1]
+
### Fixed
- **A `VARIANT` in a function signature crashed DuckDB.** `TypeId::Variant`
@@ -82,6 +234,128 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `Cargo.toml`: `cc = "1"` could not be resolved by
`cargo update -Z direct-minimal-versions`; the lower bound is now `1.0.69`.
+- CI: `scripts/check-ci-aggregate.py` no longer passes while a gate is removed
+ (a job demoted to informational and dropped from `needs`, a step-level
+ `continue-on-error` in a blocking job or in `ci-ok` itself, a job id with a
+ comment or anchor, a safety pattern kept only in a comment); it also checks
+ the workflows README's gate table. [CI-7]
+- CI: `scaffold-compile` can no longer pass by running zero tests (`--exact`,
+ one test binary, and the libtest summary must say `1 passed`). [CI-8]
+- CI: `gh release edit` now clears a leftover draft flag and sets the
+ prerelease flag both ways. [CI-9]
+- CI: `scripts/check-api-coverage.py` counts a C API slot as wrapped only when
+ library code uses it (not comments, string literals, `#[cfg(test)]` code or
+ test-only modules): `duckdb_open` / `duckdb_close` are test-only, so they are
+ listed as host-side. [APIGAPS-4, CI-14]
+- CI: `check-msrv-vs-duckdb-ci.py` reads `# v1.86.0` / `# rust 1.86.0` pin
+ comments and `toolchain:` inputs as the constraints they are;
+ `check-autoload-entries.py` refuses a reshaped upstream array instead of
+ passing on zero parsed names. [CI-14]
+
+- **Scaffold (HARDEN-5):** a license with a line break or other whitespace (`MIT\nOR
+ Apache-2.0`) was written raw into `Cargo.toml`, which then was not TOML. A version
+ component above `u64::MAX` was accepted, and Cargo then refused to read the project.
+- **Scaffold:** an `excluded_platforms` entry that excludes nothing is now refused. The
+ community build compares exact names, so this covers a platform group such as `wasm` (the
+ error lists its platforms), a retired platform, and a name it does not build. Before, any
+ of these was written into `description.yml` and the workflow as if it worked.
+- **`append_metadata` (HARDEN-3):** the temporary file was `
-**Write DuckDB loadable extensions in pure Rust — no C, no C++, no glue code.**
+**Write DuckDB loadable extensions in Rust, with no C or C++ to write.**
+Pre-1.0: the API changes between minor releases, and audits keep finding
+defects. Read [Status](#status) before you depend on it.
## What it is
`quack-rs` wraps DuckDB's [C Extension API](https://duckdb.org/community_extensions/development)
-so that you can write a loadable extension entirely in Rust.
+so that you can write a loadable extension entirely in Rust. Your repository
+holds Rust only: no C or C++ source, no CMake file, no glue layer of your own
+between your functions and the C API. The build still runs other people's C and
+tools: dependencies of the DuckDB bindings may compile C, the `bundled-test*`
+features compile C++ or C, and the community-extension flow needs Python and git
+([Installation](https://quack-rs.com/getting-started/installation.html#build-requirements)
+has the list).
- **Every function kind the C API can register:** scalar, aggregate, table, cast,
copy and replacement scan functions, plus SQL macros.
- **Scalar functions as safe closures.** `ScalarFunctionBuilder::map1` and its
siblings take a plain Rust closure, derive the SQL types from its signature,
return NULL for NULL input, and turn a panic into a SQL error.
-- **Guards against documented FFI pitfalls.** Most of the 34 pitfalls in
+- **Guards against documented FFI pitfalls.** Most of the 38 pitfalls in
[`LESSONS.md`](./LESSONS.md) are prevented by a wrapper, such as NULL output
that is silently lost, a boolean read that is undefined behaviour, or a double
free in an aggregate destructor. The rest are documented.
@@ -52,20 +66,26 @@ nothing can be caught, and a panic kills the user's DuckDB process.
name = "my_extension"
version = "0.1.0"
edition = "2021"
-rust-version = "1.86" # with resolver 3, Cargo picks dependency versions that build on it
+# With resolver 3, Cargo picks versions that build on this.
+rust-version = "1.86"
resolver = "3"
[lib]
-name = "my_extension" # must match the extension name (Pitfall P1)
+# Must match the extension name (Pitfall P1).
+name = "my_extension"
crate-type = ["cdylib", "rlib"]
[dependencies]
-quack-rs = "0.19"
-libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] }
+quack-rs = "0.20"
+
+[dependencies.libduckdb-sys]
+version = ">=1.4.4, <1.20100"
+default-features = false
+features = ["loadable-extension"]
[profile.release]
-panic = "unwind" # required
-lto = true # the rest are recommended
+panic = "unwind" # required
+lto = true # the rest are recommended
opt-level = 3
codegen-units = 1
strip = true
@@ -79,17 +99,25 @@ use quack_rs::entry_point;
use quack_rs::error::ExtensionError;
use quack_rs::scalar::ScalarFunctionBuilder;
-fn register(con: duckdb_connection) -> Result<(), ExtensionError> {
- // Parameter and return types come from the closure signatures.
- // A NULL argument gives a NULL result without calling the closure.
- // `try_map1` lets the closure fail: a plain `x * 2` wraps on overflow in a
- // release build, and `Err` becomes a SQL error instead.
- let double_it = ScalarFunctionBuilder::try_map1("double_it", |x: i64| {
- x.checked_mul(2).ok_or("double_it: BIGINT overflow")
- })?;
- let shout = ScalarFunctionBuilder::map1_str("shout", |s: &str| s.to_uppercase())?;
-
- // SAFETY: `con` is the live connection DuckDB passed to the entry point.
+fn register(
+ con: duckdb_connection,
+) -> Result<(), ExtensionError> {
+ // Types come from the closure signatures. A NULL
+ // argument gives a NULL result without calling the
+ // closure. `try_map1` lets the closure fail: a plain
+ // `x * 2` wraps on overflow in a release build, and
+ // `Err` becomes a SQL error instead.
+ let double_it = ScalarFunctionBuilder::try_map1(
+ "double_it",
+ |x: i64| x.checked_mul(2).ok_or("double_it: overflow"),
+ )?;
+ let shout = ScalarFunctionBuilder::map1_str(
+ "shout",
+ |s: &str| s.to_uppercase(),
+ )?;
+
+ // SAFETY: `con` is the live connection DuckDB passed to
+ // the entry point.
unsafe {
double_it.register(con)?;
shout.register(con)?;
@@ -111,11 +139,13 @@ project pins an older toolchain. Your extension itself still builds on 1.86.
```shell
cargo build --release
cargo install quack-rs --bin append_metadata
-append_metadata target/release/libmy_extension.so my_extension.duckdb_extension \
+append_metadata target/release/libmy_extension.so \
+ my_extension.duckdb_extension \
--abi-type C_STRUCT --extension-version v0.1.0 \
--duckdb-version v1.2.0 --platform linux_amd64
-duckdb -unsigned -c "LOAD './my_extension.duckdb_extension'; SELECT double_it(21), shout('hello');"
+duckdb -unsigned -c "LOAD './my_extension.duckdb_extension';
+SELECT double_it(21), shout('hello');"
# ┌───────────────┬────────────────┐
# │ double_it(21) │ shout('hello') │
# │ int64 │ varchar │
@@ -141,12 +171,12 @@ line. Checked against the published sources of `duckdb` and
`duckdb-loadable-macros` 1.20000.1, `duckdb-neo` 0.20000.1 and the
`extension-template-rs` repository's `main` on 2026-10-29:
-| | quack-rs | `duckdb` + `duckdb-loadable-macros` | `extension-template-rs` + `duckdb-neo` |
+| | quack-rs | `duckdb` + macros | template + `duckdb-neo` |
|---|---|---|---|
-| C API | v1, stable | v1 by default (`_init_c_api`); v2 behind the macros' `neo` feature | v2, unstable (`_init_c_api_v2`) |
-| Binary loads in | every DuckDB from 1.4.4 through 2.0.x, with the default features | set by the C API v1 stamp, as for quack-rs | only the DuckDB release it was built for ("Binaries load only in the exact TARGET_DUCKDB_VERSION", the template's `Makefile`) |
-| Wraps | scalar, aggregate, table, cast, copy and replacement-scan functions; SQL macros | scalar (`vscalar`) and table (`vtab`) functions | modules for scalar, aggregate, table, cast, copy, macro and pragma functions, replacement scans, custom types and options, and log sinks |
-| Status | pre-1.0, audited (below) | the `duckdb` crate | `duckdb-neo` describes itself as experimental |
+| C API | v1, stable | v1 by default; v2 behind the macros' `neo` feature | v2, unstable |
+| Binary loads in | every DuckDB 1.4.4 to 2.0.x (default features; tested to 2.0.0) | as quack-rs: set by the C API v1 stamp | the one DuckDB release it was built for (the template's `Makefile` says so) |
+| Wraps | scalar, aggregate, table, cast, copy, replacement scan; macros | scalar and table functions | scalar, aggregate, table, cast, copy, macro and pragma functions; scans, types, options, log sinks |
+| Status | pre-1.0 (see Status) | the `duckdb` crate | `duckdb-neo` calls itself experimental |
Use `duckdb-neo` if you target DuckDB 2.0 and later only and want upstream's
wrapper, or need what only the v2 API offers (pragma functions, for one).
@@ -157,19 +187,23 @@ declares part of that API stable.
## Status
-`quack-rs` is pre-1.0 (0.19.0). Judge it against your own requirements; the
-record so far:
+`quack-rs` is pre-1.0. Judge it against your own requirements; the record so
+far:
- **The API still changes.** A minor release before 1.0 can break it; each lists
its breaking changes and migration notes in [`CHANGELOG.md`](./CHANGELOG.md).
-- **Audits keep finding real defects.** The review released as 0.16.0 fixed
- defects in earlier releases, including two heap-corruption paths. The 0.18.0
- audits fixed further soundness holes in the safe API, process aborts and wrong
- answers, and found a serious defect in unreleased code before publication:
- aggregate functions returned wrong results in release builds with Cargo's
- default profile. [`AUDIT.md`](./AUDIT.md) records each audit, what it found,
- and whether each fix was reproduced against a real DuckDB or derived from
- DuckDB's source.
+- **Audits keep finding real defects.** The 0.16.0 release, which came out of a
+ review, fixed defects in earlier releases, including two heap-corruption
+ paths. The 0.18.0 audits fixed further soundness holes in the safe API,
+ process aborts and wrong answers, and found a serious defect in unreleased
+ code before publication: aggregate functions returned wrong results in release
+ builds with Cargo's default profile. [`AUDIT.md`](./AUDIT.md) records each
+ audit, what it found, and whether each fix was reproduced against a real
+ DuckDB or derived from DuckDB's source. **Use 0.18.0 or later.** Earlier
+ releases still have the soundness holes in the safe API, and 0.7.0 through
+ 0.15.0 can corrupt the heap when the `duckdb-1-5` features are used.
+- **Maintenance.** One maintainer ([`GOVERNANCE.md`](./GOVERNANCE.md)).
+ [`SECURITY.md`](./SECURITY.md) lists the release lines that receive fixes.
- **Some limits are DuckDB's.** Defects in DuckDB's C API that quack-rs can only
document or work around are listed under
[Known Limitations](https://quack-rs.com/reference/known-limitations.html).
@@ -185,11 +219,18 @@ release you support.
## Supported versions
-- **DuckDB:** 1.4.4 through 2.0.0. The `libduckdb-sys = ">=1.4.4, <2"` range
- covers the bindings for all of them, 1.4.4 through 1.20000.0 (DuckDB 2.0,
- published 2026-10-22). 1.20000.0 needs Rust 1.88; on 1.88 or newer
+- **DuckDB:** with the default features, 1.4.4 through 2.0.x (the tests ran
+ against 2.0.0, the newest release when 0.20.0 was published). With a
+ `duckdb-1-5*` feature, only the releases whose C API layout quack-rs has
+ verified: 1.5.6 and 2.0.0. Any other release, a future 2.0.1 included, is
+ refused at `LOAD` (see below).
+- **`libduckdb-sys`:** `>=1.4.4, <1.20100`. That covers the bindings for every
+ DuckDB release above, 1.4.4 through 1.20000.x (the DuckDB 2.0 bindings, first
+ published 2026-10-22). 1.20000.x needs Rust 1.88; on 1.88 or newer
`cargo update` selects it, and an older toolchain with MSRV-aware resolution
- stays on 1.10506.x (DuckDB 1.5.6).
+ stays on 1.10506.x (DuckDB 1.5.6). The upper bound keeps out the DuckDB 2.1
+ bindings (1.20100.x) until a quack-rs release has been tested against them
+ ([why](https://quack-rs.com/getting-started/installation.html#adding-quack-rs-to-an-existing-extension)).
- **Rust (MSRV):** 1.86.0.
DuckDB 2.0.0 (2026-10-21) is a long-term support release, supported until at
@@ -197,12 +238,21 @@ least October 2027; DuckDB supports 1.4 until late November 2026 and 1.5 until
March 2027.
With the default features, one extension binary loads into every DuckDB release
-from 1.4.4 through 2.0.0, because it is stamped with C API version `v1.2.0`.
+from 1.4.4 through 2.0.x, because it is stamped with C API version `v1.2.0`.
DuckDB 1.4.4 to 1.5.5 declare that version; 1.5.6 and 2.0.0 declare `v1.5.6`
and still load `v1.2.0` extensions. The `duckdb-1-5*` features use the part of
-DuckDB's C API whose layout differed between releases before 1.5.6: build such
-an extension against the 1.5.6 bindings and stamp it `C_STRUCT` with C API
-`v1.5.6`, and it loads into DuckDB 1.5.6 and 2.0.0. See
+DuckDB's C API whose layout changed between releases up to 1.5.5 (DuckDB
+labels C API v1.5.6 stable, with the caveat below). Build such an extension against the 1.5.6
+bindings and stamp it `C_STRUCT` with C API `v1.5.6`; it then loads into DuckDB
+1.5.6 and 2.0.0. quack-rs has verified that layout for DuckDB 1.5.2 to 1.5.6 and
+2.0.0 only, and DuckDB's 2.0.0 header says the API is guaranteed for that exact
+release (2.0.0 added two slots to the range that 1.5.6 declared stable). So under
+the default `Strict` policy such an extension refuses to load into a release
+quack-rs has not listed, with an error that says how to fix it. When one appears
+(DuckDB 2.0.1, for example), either rebuild with
+`QUACK_RS_TARGET_DUCKDB_VERSION` set to that release, once you have checked it,
+or move to a quack-rs release that lists it; or stay on the default features,
+which are not affected. See
[ABI Compatibility](https://quack-rs.com/concepts/abi.html) before you enable one.
## Cargo features
@@ -232,8 +282,12 @@ struct layouts first in 1.10504.0, although the DuckDB functions exist from
- **API reference**: [docs.rs/quack-rs](https://docs.rs/quack-rs). The crate
root lists every module.
- [Pitfall Catalog](https://quack-rs.com/reference/pitfalls.html) and
- [`LESSONS.md`](./LESSONS.md): all 34 known pitfalls (L1–L19, P1–P15), with
+ [`LESSONS.md`](./LESSONS.md): all 38 known pitfalls (L1–L21, P1–P17), with
symptom, cause and fix.
+- [`examples/hello-ext`](./examples/hello-ext): one extension with scalar,
+ aggregate, table, cast, macro and replacement-scan functions, checked in CI.
+ It uses raw callbacks, which are not panic-guarded: read it for the C API
+ calls, do not copy it as a template.
- [Community Extensions](https://quack-rs.com/publishing.html): `description.yml`,
naming, versioning, platforms and submission.
- [Testing Guide](https://quack-rs.com/testing.html): mocks, `AggregateTestHarness`
@@ -241,12 +295,22 @@ struct layouts first in 1.10504.0, although the DuckDB functions exist from
- [`docs/architecture.md`](./docs/architecture.md): module layout, design
principles and the architecture decision records.
- [`CHANGELOG.md`](./CHANGELOG.md): every release, with breaking changes and
- migration notes. The latest is 0.19.0 (2026-10-22).
+ migration notes. The latest is 0.20.0 (2026-10-30).
- [`CONTRIBUTING.md`](./CONTRIBUTING.md): development setup, quality gates and
the test strategy.
- [`SECURITY.md`](./SECURITY.md): supported versions and how to report a
vulnerability.
+## Getting help
+
+- A bug, or something the book does not answer:
+ [GitHub issues](https://github.com/tomtom215/quack-rs/issues). The bug report
+ template asks for the DuckDB version that fails and the footer stamp.
+- `LOAD` fails before your code runs: the [FAQ](https://quack-rs.com/faq.html)
+ lists the messages DuckDB gives and what causes each.
+- A vulnerability: report it privately, as [`SECURITY.md`](./SECURITY.md)
+ describes.
+
## License
MIT — see [`LICENSE`](./LICENSE).
diff --git a/RELEASING.md b/RELEASING.md
index 68b12885..81bf0212 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -13,8 +13,8 @@ Before you can cut a release, ensure the following are in place:
|-------------|-------------------|
| Push access to `main` and ability to push tags | GitHub repository settings |
| crates.io **Trusted Publishing** for `quack-rs`: repository `tomtom215/quack-rs`, workflow `release.yml`, environment `crates-io` | crates.io → the crate's Settings → Trusted Publishing |
-| `CARGO_REGISTRY_TOKEN` secret, **only until Trusted Publishing is configured**; then delete it | Settings → Secrets and variables → Actions |
-| `crates-io` environment with yourself as required reviewer | Settings → Environments → crates-io |
+| `CARGO_REGISTRY_TOKEN`, **only until Trusted Publishing is configured**; then delete it. Store it as a secret **of the `crates-io` environment**, never as a repository secret | Settings → Environments → crates-io → Environment secrets |
+| `crates-io` environment with yourself as required reviewer, and **Deployment branches and tags** limited to the tag pattern `v*` | Settings → Environments → crates-io |
| A tag ruleset restricting who may create `v*` tags | Settings → Rules → Rulesets |
| Branch protection on `main` requiring the single check **CI passed** | Settings → Rules (or Branches) |
| GPG key configured for signed tags | `git config --global user.signingkey ` |
@@ -27,6 +27,32 @@ fails. `release.yml` runs from the tagged tree, so the tag ruleset is what stops
a tag on an edited workflow; the workflow itself refuses a lightweight tag and a
tag on a commit that is not on `main`.
+Where the fallback token lives matters. A repository secret is readable by
+every workflow in the repository: anyone who can push a branch can add a
+workflow that prints it, and the `crates-io` environment's required reviewer is
+never consulted. An environment secret is released only to a job that names
+the environment, after the reviewer approves, and only for the refs the
+environment allows.
+
+### Setting up Trusted Publishing (once, in this order)
+
+1. crates.io → `quack-rs` → Settings → Trusted Publishing → add a GitHub
+ publisher: owner `tomtom215`, repository `quack-rs`, workflow `release.yml`,
+ environment `crates-io`.
+2. GitHub → Settings → Environments → `crates-io`: required reviewers, and
+ deployment branches and tags limited to `v*`. If a `CARGO_REGISTRY_TOKEN`
+ repository secret exists, move it into this environment's secrets and
+ delete the repository secret, as a stop-gap until step 4.
+3. Cut the next release. The `publish` job must log `Publishing with a Trusted
+ Publishing token.` (not the `::warning::` about the fallback), and the
+ crates.io version page must show the publisher.
+4. Only then: delete `CARGO_REGISTRY_TOKEN` (environment and repository), revoke
+ the token on crates.io, and remove the `CARGO_REGISTRY_TOKEN` fallback and
+ the `continue-on-error` of the `trusted` step from `release.yml`, so a failed
+ exchange fails the release instead of reaching for a long-lived credential.
+5. If the crate's Trusted Publishing settings offer to disallow publishing with
+ API tokens, turn that on.
+
---
## Semantic versioning policy
@@ -77,8 +103,12 @@ Pre-release versions use the suffix convention `vX.Y.Z-alpha.N`,
# 1. Prepare
vim CHANGELOG.md # Move [Unreleased] → [X.Y.Z] - YYYY-MM-DD
vim Cargo.toml # version = "X.Y.Z"
+# Every committed lockfile records quack-rs's own version; release.yml packages
+# and publishes with --locked, which refuses a stale one (Step 3).
+for d in . examples/hello-ext fuzz book/doctest benches/engine; do (cd "$d" && cargo update --workspace); done
scripts/sync-book-changelog.py # regenerate the book's changelog page
# ...and bump every "0.X" version snippet in README.md / book / CITATION.cff (Step 3b)
+python3 scripts/check-release-metadata.py # lists every place still on the old version
git add -A
git commit -m "chore: release vX.Y.Z"
git push origin main
@@ -112,11 +142,16 @@ literally could tag a release with `Miri`, `LeakSanitizer`, `osv-scan`,
```bash
gh run list --branch main --workflow ci.yml --limit 1
-gh run view # every job must be green; none may be skipped
+gh run view # every blocking job must be green; none may be skipped
```
Do not enumerate the jobs here again. Branch protection on `main` is the
-mechanism that should enforce this; this step is the human double-check.
+mechanism that should enforce this; this step is the human double-check. The
+jobs `.github/workflows/README.md` marks *informational* (`clippy-beta`,
+`latest-stable`, `nightly`, `test-bundled-duckdb-2-0`) may be red without
+blocking, but read why before tagging: a red `latest-stable`, for example,
+means the newest Rust reports something the pinned one does not, which the
+next toolchain bump will have to fix (CONTRIBUTING.md, "The Rust toolchain").
### Step 2 — Update CHANGELOG.md
@@ -148,14 +183,17 @@ if the two diverge). Do not edit it by hand.
### Step 2b — Update SECURITY.md
If this is a **minor or major** release, update the supported versions table in
-`SECURITY.md` to list the new version as supported and mark old versions
-as end-of-life if appropriate.
+`SECURITY.md`: add a `| X.Y.x | Yes` row for the new line, and end the support
+of the line before the previous one (the table's policy is the two newest minor
+lines).
The versioned book (quack-rs.com) publishes a copy for every release line from
`MIN_RELEASE` in `scripts/build-versioned-site.py` upwards. When a line goes
end-of-life here, raise `MIN_RELEASE` to the oldest line still supported, in the
same commit, so the site stops serving documentation for releases nobody
-should start with.
+should start with. `scripts/check-release-metadata.py` fails while the table
+and `MIN_RELEASE` disagree, or while the table has no row for the version in
+`Cargo.toml`.
### Step 3 — Bump the version in `Cargo.toml`
@@ -164,27 +202,63 @@ should start with.
version = "0.3.0" # ← update this
```
-Do **not** update `Cargo.lock` manually — it will update itself on next build.
+Then bring every committed lockfile along, **in the same commit**. Each one
+records quack-rs's own version: the root `Cargo.lock` as the package itself,
+the others (`examples/hello-ext`, `fuzz`, `book/doctest`, `benches/engine`) as
+a path dependency:
+
+```bash
+for d in . examples/hello-ext fuzz book/doctest benches/engine; do (cd "$d" && cargo update --workspace); done
+git diff --stat -- '*Cargo.lock' # one line changed in each
+```
+
+`--workspace` updates only the workspace's own packages, so no dependency
+moves. This is not optional: `release.yml` runs `cargo package --locked` and
+`cargo publish --locked`, which refuse a `Cargo.lock` that still names the
+previous version ("cannot update the lock file ... because --locked was
+passed"), and nothing in CI commits a lock file it rewrote. A `Cargo.toml` bump
+without its lockfiles used to pass every pull-request job and fail the
+`package` job after the whole of `ci.yml` had run. `ci.yml`'s publish dry-run
+now uses `--locked` too, and `scripts/check-release-metadata.py` (the `doc`
+job, and `validate` in `release.yml`) names any lockfile left behind.
### Step 3b — Bump the version in the documentation snippets, in the same PR
Every `quack-rs = "0.X"` / `version = "0.X"` snippet a reader copies — in
-`README.md`, the book, the feature documentation in `Cargo.toml`, and
-`CITATION.cff` — must name the new version **in the release PR itself**. The
-commit that gets tagged is that PR's merge commit, so a snippet bumped in a
-follow-up PR is wrong in the tagged source, in the published `.crate` (whose
-README is what crates.io renders) and in the docs.rs build of the release.
-Find the stale ones by grepping for the previous version, for example (replace
-`` with the previous minor, dots escaped — `0\.17` when releasing 0.18.0):
+`README.md`, the book, the feature documentation in `Cargo.toml`, the `src/`
+rustdoc, and `CITATION.cff` — must name the new version **in the release PR
+itself**, as must `SECURITY.md` (Step 2b) and the README's "The latest is ..."
+line. The commit that gets tagged is that PR's merge commit, so a snippet
+bumped in a follow-up PR is wrong in the tagged source, in the published
+`.crate` (whose README is what crates.io renders) and in the docs.rs build of
+the release. Find the stale ones by grepping for the previous version, for
+example (replace `` with the previous minor, dots escaped — `0\.17` when
+releasing 0.18.0):
```bash
git grep -n '' -- README.md book/src Cargo.toml CITATION.cff src
```
+Two places hold a date, not a version, so the grep cannot find them:
+`CITATION.cff`'s `date-released` and the README's "The latest is X.Y.Z
+(YYYY-MM-DD)". Give both the date of the CHANGELOG heading (the day you push
+the tag). The `publish` job waits for approval and can finish on the next day:
+0.19.0 is dated 2026-10-22 everywhere, and crates.io's `created_at` for it is
+2026-10-23T06:07Z.
+
+`python3 scripts/check-release-metadata.py` does all of this mechanically: the
+lockfiles, the CHANGELOG heading and its two link definitions, `CITATION.cff`
+(version and date), the `SECURITY.md` row and `MIN_RELEASE`, the README line,
+and every snippet. CI runs it in the `doc` job, so a release PR cannot merge
+with a place missed. For a pre-release version only the lockfile and CHANGELOG
+checks apply: the docs keep naming the latest stable release.
+
### Step 4 — Commit the version bump
```bash
-git add CHANGELOG.md Cargo.toml # plus the snippets from Step 3b
+# ...plus SECURITY.md and the snippets from Steps 2b and 3b
+git add CHANGELOG.md Cargo.toml Cargo.lock examples/hello-ext/Cargo.lock \
+ fuzz/Cargo.lock book/doctest/Cargo.lock benches/engine/Cargo.lock
git commit -m "chore: release v0.3.0"
git push origin main
```
@@ -219,7 +293,9 @@ git push origin "v0.3.0"
```
A lightweight tag (`git tag v0.3.0`) is refused by the `validate` job, as is a
-tag on a commit that is not on `main`.
+tag on a commit that is not on `main`. `validate` checks that the tag is
+annotated, not that it is signed: `git verify-tag` (above, and Step 9) is the
+only signature check.
### Step 6 — Monitor the release workflow
@@ -227,38 +303,48 @@ Navigate to **Actions → Release** in the GitHub UI. The workflow runs these jo
in order:
```
-validate Tag is annotated and on main; tag format, Cargo.toml version, dated CHANGELOG entry
- │
+validate Tag is annotated and on main; tag format, Cargo.toml version, dated CHANGELOG
+ │ entry, and scripts/check-release-metadata.py (lockfiles, CITATION.cff, SECURITY.md, snippets)
ci All of ci.yml (workflow_call): every gate a pull request runs,
│ including semver, Miri, the sanitizers, cargo-deny, OSV, the
│ publish dry-run and extension-load on every pinned DuckDB
-package Build .crate, generate SHA256SUMS, attest SLSA provenance
+package Build .crate, generate SHA256SUMS, attest SLSA provenance; record rustc
│
publish ← awaits manual approval in the crates-io environment;
- │ cargo publish, then checks crates.io serves the attested .crate
+ │ re-packages with the recorded rustc and stops unless the .crate's SHA-256
+ │ equals SHA256SUMS, then cargo publish, then checks crates.io serves the attested .crate
github-release Create GitHub release with notes, .crate, SHA256SUMS
│
docs Dispatch docs.yml on main: the book for this release at / and /vX.Y/
```
-Each job writes a structured summary visible in the Actions UI.
+`publish` and `github-release` write a summary to the Actions UI; the other
+jobs report in their logs.
### Step 7 — Approve the crates.io deployment
The `publish` job is gated behind the `crates-io` environment, which requires a
manual approval from an authorised reviewer. Before approving, read the
-`package` job's summary (the `.crate` contents and size).
+`package` job's log (the `.crate` file list in the "Package contents" group,
+and its size in the `::notice::` line).
1. In the Actions run for the release workflow, click **Review deployments**
2. Select the `crates-io` environment
3. Click **Approve and deploy**
-The job publishes with a Trusted Publishing token (or, until that is set up,
-the `CARGO_REGISTRY_TOKEN` secret), using `cargo publish --locked --no-verify`:
-ci.yml's publish dry-run already built the packaged sources, and skipping the
-second build keeps dependency build scripts away from the token. It then
-downloads the crate from crates.io and fails if its SHA-256 differs from the
-attested artifact's (`cargo package` is reproducible, so they must match).
+`publish` first installs the rustc the `package` job recorded (the release
+`rust-toolchain.toml` pins; it refuses to continue with any other), packages
+the tagged commit again with `cargo package --no-verify --locked`, and stops
+unless that `.crate`'s SHA-256 equals the attested `SHA256SUMS`: a crates.io
+version can be yanked but never replaced, so a difference must stop the
+release before the upload, and it does so before any credential is requested.
+It then publishes with a Trusted Publishing token (or, until that is set up,
+the `CARGO_REGISTRY_TOKEN` environment secret), using `cargo publish --locked
+--no-verify`: ci.yml's publish dry-run already built the packaged sources, and
+skipping the second build keeps dependency build scripts away from the token.
+Finally it downloads the crate from crates.io and fails if its SHA-256 differs
+from the attested artifact's (`cargo package` is reproducible: crates.io's
+0.19.0 is byte-identical to a fresh `cargo package` of the v0.19.0 commit).
### Step 8 — Review the GitHub release
@@ -305,7 +391,8 @@ curl -fsSL "https://docs.rs/crate/quack-rs/$V/status.json"
curl -fsSL https://quack-rs.com/versions.json
# "latest" must be "v${V%.*}" and a version entry must read "$V". release.yml's
# `docs` job dispatches docs.yml on main after publishing; if it did not run,
-# run `gh workflow run docs.yml --ref main`.
+# run `gh workflow run docs.yml --ref main`. For a pre-release, skip this check:
+# the `docs` job does not run and versions.json does not change.
```
If any of them fails, the release is incomplete: finish it (push the tag,
@@ -344,26 +431,45 @@ sha256sum quack-rs-0.3.0.crate
## Hotfix releases
-For a critical bug in an already-published version:
+For a critical bug in an already-published version. The `validate` job refuses
+a tag whose commit is not an ancestor of `main`, so the hotfix branch has to
+reach `main` *before* it is tagged, and by a merge commit: a squash merge or a
+cherry-pick creates a different commit, and the hotfix tip is then not on
+`main` (tagging the branch and cherry-picking the fix onto `main` afterwards,
+as this section used to say, is refused).
```bash
# Branch from the release tag
git checkout -b hotfix/0.2.1 v0.2.0
-# Apply the fix, update CHANGELOG and Cargo.toml
+# Apply the fix; update CHANGELOG.md, Cargo.toml and the lockfiles (Steps 2-4)
# ... make changes ...
git add -p
git commit -m "fix: "
+git push origin hotfix/0.2.1
+
+# Open a pull request from hotfix/0.2.1 into main and merge it with
+# "Create a merge commit". main may carry newer unreleased changes: that is
+# fine, the tag below points at the hotfix tip, not at the merge.
+# Then tag the hotfix tip (follow steps 5–9 above):
+git fetch origin
+git tag -s "v0.2.1" -m "Hotfix v0.2.1" origin/hotfix/0.2.1
+git push origin "v0.2.1"
+```
-# Tag and push (follow steps 5–9 above)
-git tag -s "v0.2.1" -m "Hotfix v0.2.1"
-git push origin hotfix/0.2.1 "v0.2.1"
+After the merge `main` must still agree with itself (`scripts/check-release-metadata.py`
+runs on it). Where only the hotfix changed the version, the lockfiles, the
+CHANGELOG, `CITATION.cff` or the README's "latest" line, `main` simply takes
+them, which is right: they describe the newest release. Where `main` changed
+them too (it is already on a newer version), keep `main`'s and add the hotfix's
+CHANGELOG section below `main`'s own.
-# After release: cherry-pick the fix onto main
-git checkout main
-git cherry-pick
-git push origin main
-```
+The release workflow that runs is the one in the tagged commit, i.e. the
+hotfix branch's: a hotfix of a line released before this runbook gained the
+release-metadata check runs that line's older `validate`. Copy
+`scripts/check-release-metadata.py` from `main` into the hotfix checkout and
+run it there before tagging (it checks the tree it sits in); do not commit it
+on the hotfix branch unless the fix needs it.
---
@@ -407,17 +513,36 @@ git push origin --delete "v0.3.0"
Add a `## [0.3.0] - YYYY-MM-DD` section to `CHANGELOG.md`, commit, push, then
delete and recreate the tag.
-### `security` fails: advisory found
+### `validate` fails: release metadata
+
+`scripts/check-release-metadata.py` lists each place that still names another
+version (a lockfile, a CHANGELOG link, `CITATION.cff`, `SECURITY.md`, a
+snippet). Run it locally, fix what it lists on `main`, then delete and recreate
+the tag on the new commit (Steps 3-5).
+
+### `security` or `osv-scan` fails: advisory found
A dependency has a known vulnerability. Either:
- Update the affected dependency, or
-- Add the advisory ID to `deny.toml`'s `[advisories] ignore` list with a
- documented justification comment
+- Suppress it with a documented justification: in `deny.toml`'s `[advisories]
+ ignore` for the root graph, in `deny.ci-crates.toml` for the CI-only graphs
+ (hello-ext, fuzz, book/doctest, benches/engine), and in `osv-scanner.toml`
+ for the `osv-scan` job, which reads every committed lockfile and has its own
+ rules for a suppression (the file's header).
+
+### `publish` stops before uploading: the `.crate` differs
+
+"this job packages a different quack-rs-X.Y.Z.crate than the attested one;
+nothing was published" (or "this job's rustc is ..."): crates.io has nothing,
+so nothing needs undoing. The `package` job's `.crate` and the one `publish`
+just built from the same commit differ, which `cargo package` reproducibility
+says they must not; compare the two file lists (`tar -tzf`) before re-running.
### `publish` fails: token invalid
Regenerate the crates.io API token and update the `CARGO_REGISTRY_TOKEN`
-repository secret.
+secret of the `crates-io` environment, or finish the Trusted Publishing set-up
+(Prerequisites), which needs no token.
### `publish` fails: crate already published
@@ -441,9 +566,10 @@ Every release produces and attaches:
| GitHub release notes | Extracted from `CHANGELOG.md` |
The attached `.crate` is the one the `package` job built and attested. The
-`publish` job packages again from the same tagged commit, so what crates.io
-serves should hash the same, and the `publish` job's "crates.io serves the
-attested .crate" step fails the release if it does not. To check by hand,
-compare `sha256sum` of
+`publish` job packages again from the same tagged commit with the same rustc,
+and refuses to upload unless that `.crate` hashes the same as `SHA256SUMS`
+("The .crate about to be published is the attested one"); after the upload,
+"crates.io serves the attested .crate" checks crates.io's copy against the same
+file. To check by hand, compare `sha256sum` of
`https://static.crates.io/crates/quack-rs/quack-rs-X.Y.Z.crate` against
`SHA256SUMS`, or check the attached file with `gh attestation verify`.
diff --git a/SECURITY.md b/SECURITY.md
index 98ab5fb8..f98a20b9 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -5,30 +5,42 @@
## Supported Versions
-| Version | Supported |
-|---------|--------------------|
-| 0.19.x | Yes |
-| 0.18.x | Yes |
-| 0.16.x | No (end-of-life) |
-| 0.15.x | No (end-of-life) |
-| 0.14.x | No (end-of-life) |
-| 0.13.x | No (end-of-life) |
-| 0.12.x | No (end-of-life) |
-| 0.11.x | No (end-of-life) |
-| 0.10.x | No (end-of-life) |
-| 0.9.x | No (end-of-life) |
-| 0.8.x | No (end-of-life) |
-| 0.7.x | No (end-of-life) |
-| 0.6.x | No (end-of-life) |
-| 0.5.x | No (end-of-life) |
-| 0.4.x | No (end-of-life) |
-| 0.3.x | No (end-of-life) |
-| 0.2.x | No (end-of-life) |
-| 0.1.x | No (end-of-life) |
-
-0.17.x does not appear in this table because it was never published: 0.17.0
-was prepared but not released, and its changes ship in 0.18.0 (see the 0.18.0
-entry in `CHANGELOG.md`). On crates.io, 0.18.0 follows 0.16.0.
+| Version | Supported |
+|----------------|-----------------------------------------|
+| 0.20.x | Yes |
+| 0.19.x | Yes |
+| 0.18.x | No (end-of-life) |
+| 0.16.x | No (end-of-life) |
+| 0.3.x – 0.15.x | No (end-of-life); **do not use** (below) |
+
+The two newest minor lines are supported. quack-rs is pre-1.0, so Cargo
+treats each minor release as a new major line: a project that depends on
+`quack-rs = "0.19"` never moves to 0.20 by itself. The previous line therefore
+keeps receiving security fixes, as patch releases of that line, until the next
+minor release ends its support. 0.18.x reached end-of-life with 0.20.0.
+
+**Releases before 0.16.0 must not be used.** They carry the memory-safety
+defects fixed in 0.16.0 (the "Security" part of its entry in `CHANGELOG.md`).
+Checked against the published crates:
+
+- `DuckStringView::from_bytes` (0.3.0 – 0.15.0) was a safe function that
+ dereferenced a pointer read from its argument's bytes, so safe code could
+ read arbitrary memory.
+- `CopyGlobalInitInfo::get_file_path` (0.8.0 – 0.15.0) freed a string DuckDB
+ still owned, corrupting the heap on every `COPY ... TO` through a quack-rs
+ copy function.
+- With the `duckdb-1-5` feature (0.4.0 – 0.15.0), nothing checked that the
+ running DuckDB's C API function table had the layout the extension was
+ compiled against; loaded into another DuckDB release, such a build could
+ call the wrong functions (`double free or corruption`).
+
+None of these versions has been yanked from crates.io; upgrade to a supported
+line.
+
+crates.io has no 0.1.x or 0.2.x: the first published version is 0.3.0. 0.17.x
+does not appear either because it was never published: 0.17.0 was prepared
+but not released, and its changes ship in 0.18.0 (see the 0.18.0 entry in
+`CHANGELOG.md`). On crates.io, 0.18.0 follows 0.16.0.
## Reporting a Vulnerability
diff --git a/benches/engine/Cargo.lock b/benches/engine/Cargo.lock
new file mode 100644
index 00000000..8c640a40
--- /dev/null
+++ b/benches/engine/Cargo.lock
@@ -0,0 +1,722 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "cc"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
+
+[[package]]
+name = "crc32fast"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
+
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "filetime"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
+dependencies = [
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
+
+[[package]]
+name = "flate2"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "r-efi",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "indexmap"
+version = "2.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "js-sys"
+version = "0.3.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "libduckdb-sys"
+version = "1.10506.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "113d1c26c9f518405a17b5729bb6ab6abf029fdc46626406922cf9a8dc9b4609"
+dependencies = [
+ "flate2",
+ "prettyplease",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn 2.0.119",
+ "tar",
+ "ureq",
+ "zip",
+]
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
+[[package]]
+name = "log"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "nix"
+version = "0.31.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
+[[package]]
+name = "prettyplease"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quack-rs"
+version = "0.20.0"
+dependencies = [
+ "cc",
+ "libduckdb-sys",
+]
+
+[[package]]
+name = "quack-rs-engine-bench"
+version = "0.0.0"
+dependencies = [
+ "libduckdb-sys",
+ "quack-rs",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustix"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
+dependencies = [
+ "log",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tar"
+version = "0.4.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
+[[package]]
+name = "typed-path"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "ureq"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b"
+dependencies = [
+ "base64",
+ "log",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922"
+dependencies = [
+ "base64",
+ "http",
+ "httparse",
+ "log",
+]
+
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
+dependencies = [
+ "cfg-if",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "xattr"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08"
+dependencies = [
+ "libc",
+ "rustix",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
+
+[[package]]
+name = "zip"
+version = "9.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "getrandom 0.4.3",
+ "indexmap",
+ "memchr",
+ "nix",
+ "typed-path",
+ "windows-sys 0.61.2",
+ "zopfli",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zopfli"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "log",
+ "simd-adler32",
+]
diff --git a/benches/engine/Cargo.toml b/benches/engine/Cargo.toml
index 34856f9c..b10b7126 100644
--- a/benches/engine/Cargo.toml
+++ b/benches/engine/Cargo.toml
@@ -18,7 +18,7 @@ crate-type = ["cdylib"]
[dependencies]
quack_rs = { path = "../..", package = "quack-rs" }
-libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] }
+libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] }
[profile.release]
panic = "unwind"
diff --git a/book/doctest/Cargo.lock b/book/doctest/Cargo.lock
index a7c65de1..28c21596 100644
--- a/book/doctest/Cargo.lock
+++ b/book/doctest/Cargo.lock
@@ -299,9 +299,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.8.0"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
+checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -493,9 +493,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
-version = "0.1.14"
+version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
[[package]]
name = "flate2"
@@ -966,7 +966,7 @@ dependencies = [
[[package]]
name = "quack-rs"
-version = "0.19.0"
+version = "0.20.0"
dependencies = [
"cc",
"duckdb",
diff --git a/book/doctest/Cargo.toml b/book/doctest/Cargo.toml
index bcaeb059..4c1ac1e4 100644
--- a/book/doctest/Cargo.toml
+++ b/book/doctest/Cargo.toml
@@ -23,9 +23,14 @@ test = false
[dependencies]
quack-rs = { path = "../..", features = ["duckdb-1-5-4", "bundled-test-prebuilt"] }
-libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] }
+libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] }
[dev-dependencies]
proptest = "1"
[workspace]
+
+[package.metadata.cargo-machete]
+# Used by the doctests that build.rs generates from the book and README, which
+# cargo-machete does not read.
+ignored = ["libduckdb-sys", "proptest", "quack-rs"]
diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
index ce03a113..7cacfa7c 100644
--- a/book/src/SUMMARY.md
+++ b/book/src/SUMMARY.md
@@ -9,6 +9,7 @@
- [Quick Start](getting-started/quick-start.md)
- [Installation](getting-started/installation.md)
- [Upgrading from 0.16 to 0.18](getting-started/upgrading-0-18.md)
+ - [Upgrading from 0.18 to 0.20](getting-started/upgrading-0-20.md)
- [Your First Extension](getting-started/first-extension.md)
- [Project Scaffold](getting-started/scaffold.md)
diff --git a/book/src/concepts/abi.md b/book/src/concepts/abi.md
index 6d0cb1e1..e40ab1f2 100644
--- a/book/src/concepts/abi.md
+++ b/book/src/concepts/abi.md
@@ -180,12 +180,38 @@ For an engine older than v1.5.0 the advice changes: such an engine cannot run a
`duckdb-1-5` build at all, so the message says to build a variant without those
features or to upgrade DuckDB.
+## The bindings read the whole struct before the guard runs
+
+`libduckdb-sys` generates its `duckdb_ext_api_v1` with every slot of the release it was
+generated from (546 for the 1.5.x bindings, 548 for 2.0), whatever quack-rs features are on.
+Its `duckdb_rs_extension_api_init` asks DuckDB for the struct and reads **every one of those
+slots** through the returned pointer before `abi::check` can run, because the check itself
+reaches `duckdb_library_version` through that table. DuckDB's struct has only the engine's own
+number of slots, so loading a build against the 546-slot bindings into DuckDB 1.4.x (459
+slots) reads 87 pointers (696 bytes on a 64-bit target) past the end of it, and into
+1.5.0 or 1.5.1 (545 slots) one pointer. This happens on every such `LOAD`, also for a
+default-feature build that the check then lets load as `StableOnly`. The memory is mapped
+and no crash has been observed or is expected; the values read are stored, and are never
+called by code that stays in the stable prefix or when the check refuses the load. They are
+what an unstable-region wrapper would call after `Warn` or `Trust` let a mismatched layout
+load, one more reason those policies are undefined behaviour on a mismatch. This was read
+in `libduckdb-sys` and in DuckDB's `extension_load.cpp`, not measured; quack-rs could avoid
+it only by calling `get_api` and reading slot 7 itself before handing the struct to the
+bindings.
+
## Unknown DuckDB versions
`Strict` also refuses a DuckDB release quack-rs has no verified layout for — a
newer release (such as v2.0.1 or v2.1.0), or a `-dev` build. That is deliberate:
DuckDB changed the unstable region in every minor release from 1.3.0 to 2.0.0, and
-in the 1.5.2 patch release, so "unknown" is not evidence of "compatible". A build
+in the 1.5.2 patch release, so "unknown" is not evidence of "compatible". DuckDB's
+own header says the same: DuckDB v2.0.0's `duckdb_extension.h` opens with "this API
+is not yet stable, meaning it is only guaranteed to work for this specific DuckDB
+version", and v2.0.0 itself added two slots to the band that v1.5.6 had declared
+stable. So a `duckdb-1-5*` extension that loads into 1.5.6 and 2.0.0 is refused by
+2.0.1 until a quack-rs release lists it or you rebuild with
+`QUACK_RS_TARGET_DUCKDB_VERSION=v2.0.1`; an extension built with the default
+features reports `StableOnly` and loads there. A build
against the v1.5.2 – v1.5.6 bindings is accepted on v2.0.0, whose layout only
appends to it; a build against 1.5.0/1.5.1 (545 slots) is still refused there.
The refusal lists the fixes, best first:
diff --git a/book/src/concepts/anatomy.md b/book/src/concepts/anatomy.md
index 2a0766f2..560bb45c 100644
--- a/book/src/concepts/anatomy.md
+++ b/book/src/concepts/anatomy.md
@@ -25,15 +25,17 @@ When DuckDB loads your extension, it:
`quack_rs::entry_point::init_extension` performs this sequence. It also checks the C API
layout before registration (see [ABI Compatibility](abi.md)) and converts a panic in your
-registration code into a load error. The `entry_point!` macro generates the required
-`#[no_mangle] extern "C"` symbol:
+registration code into a load error, and runs the registration in one transaction that a
+failed `LOAD` rolls back (see [The Entry Point](entry-point.md#registration-runs-in-one-transaction)).
+The `entry_point!` macro generates the required
+`#[unsafe(no_mangle)] extern "C"` symbol:
```rust
# use quack_rs::entry_point;
# use quack_rs::error::ExtensionError;
# fn register(_con: libduckdb_sys::duckdb_connection) -> Result<(), ExtensionError> { Ok(()) }
entry_point!(my_extension_init_c_api, |con| register(con));
-// emits: #[no_mangle] pub unsafe extern "C" fn my_extension_init_c_api(...)
+// emits: #[unsafe(no_mangle)] pub unsafe extern "C" fn my_extension_init_c_api(...)
```
---
@@ -86,7 +88,7 @@ when `InMemoryDb::open()` runs (see [Testing Guide](../testing.md)).
graph TD
EXT["your-extension"]
QR["quack-rs"]
- LDS["libduckdb-sys >=1.4.4, <2 {loadable-extension} (headers only — no linked library)"]
+ LDS["libduckdb-sys >=1.4.4, <1.20100 {loadable-extension} (headers only — no linked library)"]
EXT --> QR
EXT --> LDS
@@ -101,14 +103,18 @@ inside the host DuckDB process and uses that process's DuckDB instance, memory a
## Version support
-`libduckdb-sys = ">=1.4.4, <2"` — the bounded range is intentional.
+`libduckdb-sys = ">=1.4.4, <1.20100"` — the bounded range is intentional.
-quack-rs supports DuckDB 1.4.4 through 2.0.0. Every one of them loads extensions
-built for **C API version `v1.2.0`**
+With the default features, quack-rs supports DuckDB 1.4.4 through 2.0.x. Every one of
+them loads extensions built for **C API version `v1.2.0`**
(the version string passed to `duckdb_rs_extension_api_init`, `quack_rs::DUCKDB_API_VERSION`).
-DuckDB 1.5.6 and 2.0.0 declare C API version `v1.5.6` and still accept `v1.2.0` extensions. CI runs
+DuckDB 1.5.6 and 2.0.x declare C API version `v1.5.6` and still accept `v1.2.0` extensions. CI runs
the test suite against DuckDB 1.4.4, 1.4.5, 1.5.0, 1.5.3, 1.5.4, 1.5.5, 1.5.6 and 2.0.0, and loads the example
-extension into 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release.
+extension into 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release. An extension that uses a
+`duckdb-1-5*` feature is narrower: it loads into DuckDB 1.5.6 and 2.0.0, the releases whose layout
+quack-rs has verified, and under the default `Strict` policy it refuses any other release (such as a
+future 2.0.1) until quack-rs lists it or you build with `QUACK_RS_TARGET_DUCKDB_VERSION` set to it
+([ABI Compatibility](abi.md#unknown-duckdb-versions)).
Using a range rather than an exact pin means:
- Extension authors can choose which `libduckdb-sys` release to build against (for example
@@ -116,16 +122,24 @@ Using a range rather than an exact pin means:
numbers `1.10505.x`) and still resolve against `quack-rs`
- `quack-rs` itself doesn't force a DuckDB downgrade on users
-The `<2` upper bound limits the range to `libduckdb-sys` 1.x, but it does **not** exclude
-the DuckDB 2.0 bindings: `libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as
-`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0`, which the range
-admits. That release was published on 2026-10-22; it declares `rust-version` 1.88, so a
-Rust 1.86 toolchain cannot build it, and MSRV-aware resolution (resolver 3, or
-`CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`) keeps a 1.86 build on `1.10506.x`.
+`libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as
+`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0` and the range
+admits the whole DuckDB 2.0 bindings, `1.20000.x`. That release was published on
+2026-10-22; it declares `rust-version` 1.88, so a Rust 1.86 toolchain cannot build it,
+and MSRV-aware resolution (resolver 3, or `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`)
+keeps a 1.86 build on `1.10506.x`.
+
+**Why the range stops at `1.20100`.** That is the DuckDB 2.1 bindings. quack-rs names the
+C API through the `libduckdb_sys` crate root, which the 1.20000 bindings keep only through
+a re-export of their `v1` module that is marked `// TODO: Remove in the future` upstream;
+quack-rs's exported macros hard-code those root paths too, so an extension that uses them
+would break with it. quack-rs widens the bound for each DuckDB minor after testing it. If you
+want the newer bindings before that, there is no override: wait for the quack-rs release
+(see [Installation](../getting-started/installation.md#adding-quack-rs-to-an-existing-extension)).
> **For your own extension's `Cargo.toml`:** an extension that uses only the stable C API
-> (the default quack-rs features) can keep the same `">=1.4.4, <2"` range; this is what
-> `scaffold::generate_scaffold` writes. An extension that enables the `duckdb-1-5*` features must pin
+> (the default quack-rs features) can keep the same `">=1.4.4, <1.20100"` range; this is
+> what `scaffold::generate_scaffold` writes. An extension that enables the `duckdb-1-5*` features must pin
> `libduckdb-sys` to the bindings it is stamped for: `~1.10506.0` (the v1.5.2 – v1.5.6 layout)
> for C API `v1.5.6`, which is what the scaffold writes for `target_duckdb_version = "v1.5.6"`;
> or, for a `C_STRUCT_UNSTABLE` build pinned to one 1.x release, that release's bindings
diff --git a/book/src/concepts/entry-point.md b/book/src/concepts/entry-point.md
index 7a7606be..b09326ba 100644
--- a/book/src/concepts/entry-point.md
+++ b/book/src/concepts/entry-point.md
@@ -138,7 +138,7 @@ flowchart TD
L["2. ABI layout check Applies the AbiPolicy (Strict by default)"]
B["3. access.get_database(info) Returns the duckdb_database handle"]
C["4. duckdb_connect(db, &mut con) Opens a connection for function registration"]
- D["5. register(con) ← your closure A panic becomes an error"]
+ D["5. register(con) ← your closure Runs in one transaction; a panic becomes an error"]
E["6. duckdb_disconnect(&mut con) Always runs, even if registration failed"]
F{Error?}
G["return true"]
@@ -160,9 +160,41 @@ fails the `LOAD` with that message. (When DuckDB itself detects the failure, as
message.) The layout check in step 2 only runs when a `duckdb-1-5*`
feature is enabled; see [ABI Compatibility](abi.md).
-Registration is not transactional: functions registered before a failure stay registered
-for the life of the database. Do fallible setup work (reading configuration, building
-lookup tables) before the first `register` call.
+### Registration runs in one transaction
+
+From 0.20.0 the entry point runs your closure between `BEGIN TRANSACTION` and `COMMIT`
+on the registration connection, and rolls back when the closure returns `Err` or panics.
+Every registration the C API makes through a connection joins that connection's
+transaction, so a `LOAD` that fails part-way leaves **no catalog entry behind** (no
+scalar, aggregate or table function, macro, type, copy function or table the closure
+created), and loading again in the same database works. Before 0.20.0 the functions
+registered before the failure stayed, and every retry failed at the first name it
+registered again.
+
+Three things are **not** rolled back, because DuckDB keeps them in the database
+configuration, not the catalog: casts, replacement scans and configuration options.
+Register those last, after everything that can fail. A retry registers a cast again (the
+new one replaces the old), appends a second replacement scan, and quack-rs refuses a
+configuration option whose name is taken.
+
+What the transaction means for the closure:
+
+- A statement or registration that fails can abort the transaction (DuckDB 2.0 aborts it
+ on every failed statement; 1.x on most failures that are not binder, catalog or parser
+ errors). Every later statement in the closure then fails with "Current transaction is
+ aborted", so return the first error instead of carrying on.
+- Do not run `BEGIN`, `COMMIT` or `ROLLBACK` on the registration connection.
+- What the closure creates is visible to other connections only once the `LOAD`
+ commits, including to a connection the closure opened itself.
+- A `LOAD` issued inside the host's own transaction does not wait for it. The registration
+ connection has a transaction of its own and commits on its own, so the host rolling back
+ does not undo the registrations.
+
+To get the earlier behaviour, where every statement commits on its own, pass
+`options = EntryPointOptions::new().registration(Registration::Autocommit)` to
+`entry_point!` or `entry_point_v2!` (the types are in `quack_rs::entry_point`). Do fallible
+setup work (reading configuration, building lookup tables) before the first `register`
+call either way, so that the transaction holds registrations only.
---
diff --git a/book/src/concepts/types.md b/book/src/concepts/types.md
index 645e369c..147e0bf9 100644
--- a/book/src/concepts/types.md
+++ b/book/src/concepts/types.md
@@ -170,10 +170,9 @@ and the C API initialized.
### Custom types: `set_alias` and `register`
-An extension can add a named type to SQL, as `CREATE TYPE` does. Build the
-underlying `LogicalType`, give it a name with `set_alias`, then call
-`register(con)` from your entry point. After that, users can write the name
-anywhere a type name goes:
+An extension can add a named type to SQL. Build the underlying `LogicalType`, give
+it a name with `set_alias`, then call `register(con)` from your entry point. After
+that, users can write the name anywhere a type name goes:
```sql
SELECT 'happy'::mood;
@@ -182,6 +181,7 @@ CREATE TABLE diary(day DATE, m mood);
```rust
use quack_rs::types::LogicalType;
+# use quack_rs::types::TypeId;
# use quack_rs::query::OwnedConnection;
# std::mem::forget(quack_rs::testing::InMemoryDb::open().unwrap());
# let mut db = std::ptr::null_mut();
@@ -203,8 +203,29 @@ assert!(unsafe { mood.register(con.as_raw()) }.is_err());
# let chunk = result.next_chunk().unwrap().unwrap();
# let reader = unsafe { chunk.reader(0) };
# assert_eq!(unsafe { reader.read_str(0) }, "happy");
+# // An alias over BIGINT is opaque to arithmetic and `sum`, unlike a native CREATE TYPE.
+# let money = LogicalType::new(TypeId::BigInt);
+# unsafe { money.set_alias("money"); money.register(con.as_raw()).unwrap(); }
+# con.execute("CREATE TABLE acct(m money)").unwrap();
+# assert!(con.query("SELECT sum(m) FROM acct").err().unwrap().as_str().contains("No function matches"));
+# con.execute("CREATE TYPE native_money AS BIGINT").unwrap();
+# con.execute("CREATE TABLE native_acct(m native_money)").unwrap();
+# con.query("SELECT m + 1, abs(m), sum(m) FROM native_acct GROUP BY m").unwrap();
```
+**A registered type is a new type, not a synonym.** A native `CREATE TYPE money AS
+BIGINT` is transparent: `+`, `sum` and `abs` work on it as on `BIGINT`. A type
+registered through the C API keeps its alias and does not inherit the functions of the type
+it is built on. Comparison (`=`, `<`), `GROUP BY` and `max` bind on it; arithmetic, `sum`
+and `abs` do not:
+
+```text
+SELECT sum(m) FROM acct;
+Binder Error: No function matches the given name and argument types 'sum(money)'.
+```
+
+Register casts and the functions your type needs. Checked on DuckDB 1.5.6 and 2.0.0.
+
- `set_alias(name)` panics if `name` contains a NUL byte; `try_set_alias(name)`
returns an error instead. `get_alias()` reads the alias back.
- `register(con)` returns an error if the type has no alias, if the name already
diff --git a/book/src/contributing.md b/book/src/contributing.md
index 63f94860..f58fcad0 100644
--- a/book/src/contributing.md
+++ b/book/src/contributing.md
@@ -138,6 +138,8 @@ quack-rs/
│ ├── data_chunk.rs # Ergonomic wrapper around `DuckDB` data chunks
│ ├── debug_repr.rs # Internal helpers for the crate's `Debug` implementations
│ ├── entry_point.rs # Extension entry point helper
+│ ├── entry_point/
+│ │ └── registration.rs # Runs the registration closure in one transaction, or with autocommit
│ ├── error.rs # Error types for `DuckDB` extension FFI error propagation
│ ├── error_data.rs # Structured error data (`DuckDB` 1.5.0+)
│ ├── expression.rs # Bound expressions (`DuckDB` 1.5.0+)
@@ -161,6 +163,7 @@ quack-rs/
│ │ ├── mod.rs # Builders for registering `DuckDB` aggregate functions
│ │ ├── state.rs # Generic `FfiState` wrapper for safe aggregate state management
│ │ └── builder/
+│ │ ├── collision.rs # Which catalog entry holds the name a failed aggregate registration wanted
│ │ ├── mod.rs # Builder types for registering `DuckDB` aggregate functions
│ │ ├── overload.rs # One overload within an [`AggregateFunctionSetBuilder`]
│ │ ├── set.rs # Builder for registering a `DuckDB` aggregate function set (multiple overloads)
@@ -181,18 +184,26 @@ quack-rs/
│ │ ├── export_check.rs # Values `DuckDB` would export as different values, found before the export
│ │ ├── import_check.rs # Structural checks on an imported array, and flattening what the readers cannot index
│ │ ├── import_layout/
+│ │ │ ├── bounds.rs # Offsets, lengths and dictionary indices `DuckDB` would read past, refused
│ │ │ └── tests.rs # Unit tests
│ │ ├── import_layout.rs # Arrow layouts `DuckDB` imports wrongly, found by walking the array with its schema
│ │ ├── options.rs # `ArrowOptions` — the Arrow production settings of a connection or a result
│ │ ├── schema.rs # `ArrowSchema` — an owned Arrow C Data Interface schema
│ │ └── tests.rs # Unit tests
│ ├── bin/
-│ │ └── append_metadata/
-│ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap)
-│ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file
-│ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it
-│ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file,
-│ │ └── tests.rs # Unit tests
+│ │ ├── append_metadata/
+│ │ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap)
+│ │ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file
+│ │ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it
+│ │ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file,
+│ │ │ ├── output.rs # Reading the library and writing the stamped file
+│ │ │ ├── tests.rs # Unit tests
+│ │ │ └── tests_output.rs # Unit tests for reading the input and writing the output
+│ │ └── quack_scaffold/
+│ │ ├── cli.rs # Command-line parsing (std-only, no clap)
+│ │ ├── main.rs # Write a new `DuckDB` extension project to disk, from `generate_scaffold_with_warnings`
+│ │ ├── tests.rs # Unit tests
+│ │ └── write.rs # Writing the generated files: everything is checked before anything is written
│ ├── callback/
│ │ └── payload.rs # Disposing of a caught panic payload without re-entering the unwinder
│ ├── cast/
@@ -211,18 +222,25 @@ quack-rs/
│ │ ├── connection.rs # [`OwnedConnection`] and its cross-thread [`InterruptHandle`]
│ │ ├── cstr.rs # The two C-string conversions the `query` module runs everything through
│ │ ├── live_tests.rs # Tests that need a live `DuckDB`
+│ │ ├── metadata.rs # What a statement is before it runs: `StatementType`, parameter and column types, `table_names`
│ │ ├── prepared.rs # Inspecting and executing a [`PreparedStatement`]; `bind.rs` binds its parameters
│ │ └── result.rs # Reading a [`QueryResult`]: its columns, its chunks and what kind of outcome it is
│ ├── replacement_scan/
│ │ └── mod.rs # Builder for registering `DuckDB` replacement scans
│ ├── scaffold/
+│ │ ├── checks.rs # What `generate_scaffold` checks before it writes anything, and the warnings it gives
│ │ ├── escape.rs # Quoting configured free text for YAML and Rust doc comments
+│ │ ├── licenses.rs # The license text of a generated project
│ │ ├── mod.rs # Project scaffolding for `DuckDB` Rust extensions
│ │ ├── templates.rs # Template generators for scaffold file content
│ │ ├── templates_ci.rs # The generated project's CI and distribution workflows
+│ │ ├── templates_docs.rs # The generated project's `README.md`, `docs/function_descriptions.csv` and `deny.toml`
+│ │ ├── templates_make.rs # The generated `Makefile`: extension-ci-tools' variables and targets, and the skipped-test check
│ │ ├── tests.rs # Unit tests
+│ │ ├── tests_content.rs # What the scaffold accepts, warns about and refuses, and the files a community submission needs
│ │ ├── tests_escaping.rs # Free text reaches the generated files intact
│ │ ├── tests_generated.rs # Unit tests
+│ │ ├── tests_make.rs # Runs the generated `Makefile`'s own logic under `make`
│ │ └── tests_workflows.rs # The generated workflows match the community build
│ ├── scalar/
│ │ ├── info.rs # Ergonomic wrapper around `duckdb_function_info` for scalar function callbacks
@@ -287,14 +305,24 @@ quack-rs/
│ │ ├── parser.rs # Parses and validates a `description.yml` string
│ │ ├── tests.rs # Unit tests
│ │ ├── tests_corpus.rs # Unit tests
+│ │ ├── tests_mut_block.rs # Boundaries of the block-structure reader, found by mutation testing
+│ │ ├── tests_mut_flow_a.rs # Boundaries of the flow-collection reader (scanning, anchors), found by mutation testing
+│ │ ├── tests_mut_flow_b.rs # Boundaries of the flow-collection reader (nesting, plain items), found by mutation testing
+│ │ ├── tests_mut_scalar.rs # Boundaries of the scalar, section and timestamp readers, found by mutation testing
+│ │ ├── tests_pyyaml.rs # Accepted exactly when PyYAML accepts it, in linear time
│ │ ├── tests_yaml.rs # Unit tests
│ │ ├── validator.rs # Validates a `description.yml` string and returns `Ok(())` if it passes all checks
│ │ ├── yaml.rs # A reader for the subset of YAML that `description.yml` files use
│ │ ├── yaml11.rs # Which plain scalars PyYAML (YAML 1.1) reads as booleans, numbers, dates or null
│ │ └── yaml/
-│ │ └── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments
+│ │ ├── block.rs # Block mappings and sequences, and their keys
+│ │ ├── flow.rs # Flow collections, nested and spread over lines
+│ │ ├── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments
+│ │ └── value.rs # The value after a mapping key or a sequence marker
│ ├── value/
+│ │ ├── bits.rs # `BIT` and `BIGNUM` values: validated constructors and getters that check type and `NULL` first
│ │ ├── blob.rs # `Value::as_blob` — `BLOB` extraction
+│ │ ├── cast.rs # `Value::try_cast` (exception-free, any type), `array_items` and `union_tag`
│ │ ├── checks.rs # Pure-Rust preconditions checked before a `Value` call reaches `DuckDB`
│ │ ├── composite.rs # Composite constructors: `STRUCT`, `LIST`, `ARRAY`, `ENUM`, `MAP`, `UNION`
│ │ ├── defaults.rs # The defaulting accessors — `Value::as_*_or`
@@ -319,19 +347,27 @@ quack-rs/
│ ├── validity.rs # Validity bitmap helpers for `DuckDB` NULL tracking
│ └── writer.rs # Safe typed writing to `DuckDB` result vectors
├── tests/
+│ ├── abi_policy_enforced.rs # `AbiPolicy` enforced in-process against an engine whose layout differs
│ ├── aggregate_leaks.rs # Aggregate states `DuckDB` never destroys leak no Rust heap
│ ├── append_metadata_cli.rs # The `append_metadata` binary run end to end: exit status, output and the file it writes
+│ ├── build_script_env.rs # The build script run with a crafted `QUACK_RS_TARGET_DUCKDB_VERSION`
+│ ├── entry_point_expansion.rs # The entry-point macros compiled as edition 2024 and under `panic = "abort"`
│ ├── entry_point_lint.rs # The entry-point macros accept an `unsafe` registration closure warning-free
│ ├── ffi_roundtrip.rs # End-to-end FFI round-trips against a real `DuckDB`
│ ├── file_handle_close.rs # `FileHandle`'s `Drop` when the close fails (stubbed C API)
│ ├── handle_leaks.rs # Every RAII handle frees what `DuckDB` allocated for it (glibc)
│ ├── integration_test.rs # Integration tests for `quack-rs`
+│ ├── lsan_probe.rs # The `leak-check` job's canary: a handle leaked on request must fail LeakSanitizer
+│ ├── property_inputs.rs # Property tests for `parse_version`, SPDX, `description.yml` and the scaffold's escaping
+│ ├── quack_scaffold_cli.rs # The `quack_scaffold` binary run as a user runs it: exit status, output and the files it writes
+│ ├── secret_memory_scan.rs # `SecretEntry`'s zeroize, checked by scanning the process's own memory (Linux)
│ ├── secret_zeroize.rs # `SecretEntry` never frees a buffer that still holds a secret
│ └── ffi_roundtrip/
│ ├── agg_states.rs # Every aggregate state is dropped, including the ones `DuckDB` moves
│ ├── agg_window.rs # Aggregates in the running-window and sorted-aggregate paths
│ ├── aggregate_arity.rs # An aggregate with no parameter is refused at registration (DuckDB 2.0 passes `update` no row count)
│ ├── appender_api.rs # `Appender` methods a no-op replacement survived: schemas, column types, `clear_columns`, defaults
+│ ├── appender_query.rs # `Appender::for_query` and the appender's `from_raw` / `into_raw`
│ ├── appender_rows.rs # What happens to buffered rows when an append fails mid-row
│ ├── arrow_export.rs # `arrow::data_chunk_to_arrow` refuses values `DuckDB` would export wrongly
│ ├── arrow_import.rs # `arrow::data_chunk_from_arrow` checks against a live `DuckDB`
@@ -341,14 +377,19 @@ quack-rs/
│ ├── chunk_writer.rs # `ChunkWriter` against a chunk `DuckDB` allocated
│ ├── chunk_writer_cols.rs # `ChunkWriter::{vector, writer, struct_writer}` and `DataChunk::into_chunk_writer` filling a real chunk
│ ├── collision.rs # The scalar signature-collision check, held to `DuckDB`'s own binder
+│ ├── connection_registrar.rs # Every `Registrar` method of `Connection` reaches `DuckDB`
│ ├── copy_from_columns.rs # A typed `COPY … FROM` reader that declares a column is refused
│ ├── copy_info_errors.rs # `set_error` from all four COPY phases, and the state passed between them (`duckdb-1-5`)
+│ ├── copy_typed_state.rs # Typed `COPY TO` state, a `FileHandle` kept across callbacks, `Send` handles (`duckdb-1-5`)
│ ├── data_chunk_helpers.rs # `DataChunk::any_null`, `struct_writer`, `into_chunk_writer` and `ChunkWriter::is_full`
│ ├── entry_point_inprocess.rs # `init_extension*` and the entry-point macros run in process against a fake extension access
+│ ├── entry_point_txn.rs # A failed `LOAD` rolls back and can be retried; a `LOAD` inside the host's transaction
│ ├── file_errors.rs # `FileHandle` reports write, sync and seek failures, and refuses a seek past `i64::MAX`
│ ├── handles_api.rs # `StructWriter` child handles and `InMemoryDb::execute`'s row count
+│ ├── hostile_input.rs # Hostile names, SQL bodies, type nesting and cardinality hints against a real engine
│ ├── inline_paths.rs # The paths that skip a C call: `is_valid` / `set_null` bit operations and the `ListBuilder` child-writer cache, held to the C API
│ ├── lifecycle.rs # Aggregate NULL rows, name collisions, overload builders, bind-data sharing
+│ ├── lifecycle_gaps.rs # Lifecycle paths no other test reached, found by mutation testing
│ ├── list_api.rs # The raw `ListVector` workflow, `get_size`, `get_entry`, `child_writer`, `max_child_capacity` on live LIST and MAP vectors
│ ├── list_limits.rs # `ListBuilder` stops at `DuckDB`'s byte ceiling, not an element count
│ ├── logical_type_accessors.rs # Every `LogicalType` accessor, built by the constructors and read back, against the engine's own `typeof`
@@ -356,10 +397,13 @@ quack-rs/
│ ├── mock_parity.rs # `MockRegistrar` refuses a bad type with the live registration's message
│ ├── nested_reserve.rs # which nested buffers a `LIST` reserve moves (the writer contracts)
│ ├── nested_validity.rs # `VectorWriter::set_valid` on nested rows, against a live `DuckDB`
+│ ├── oracle_temporal.rs # Randomised cross-checks of `datetime` and `interval` against `DuckDB`'s own SQL
+│ ├── overload_keys.rs # Overloads `DuckDB` resolves by type id alone are refused; closure-built overloads; taken aggregate names
│ ├── owned_chunk.rs # `OwnedDataChunk::new`, `capacity` and `reset` against a live `DuckDB`
│ ├── panic_guards.rs # The panic-guard macros and `set_error` methods, against a live `DuckDB`
│ ├── query_docs.rs # Pins the documented behaviour of `query`, `PreparedStatement`, `DbConfig`
│ ├── query_extras.rs # `bind_str`, `bind_null`, `clear_bindings` and an idle `progress`
+│ ├── query_metadata.rs # Statement type, parameter and column metadata, `table_names`, raw-handle round trips
│ ├── query_stream.rs # A streaming result that stops early must not look like a finished one
│ ├── replacement_scan_api.rs # Replacement-scan `add_*_parameter`, `register_with_data` and `set_error`
│ ├── scalar_agg.rs # Scalar and aggregate builder regressions
@@ -373,17 +417,22 @@ quack-rs/
│ ├── temporal_binds.rs # Temporal and over-4-GiB values refused by `PreparedStatement` binds and the `Appender`
│ ├── timestamp_tz_ns.rs # `TIMESTAMPTZ_NS` (DuckDB 2.0): vectors, `Value`, and every type the engine can produce
│ ├── tooling.rs # Checks of quack-rs's tooling tables against the linked `DuckDB`
+│ ├── typed_extra_info.rs # `extra_info_typed` / `extra_info_ref` on casts and aggregates; a wrong type reads as `None`
│ ├── typed_scalar_opt.rs # Typed scalar closures returning `Option`, the `try_*` constructors and `map3`
│ ├── validity_api.rs # `ValidityBitmap` against the engine's masks, at the word and vector edges
+│ ├── value_bits.rs # `BIT` / `BIGNUM` values; every input the C getters abort on reads as `None`
+│ ├── value_cast.rs # `try_cast`, `array_items`, `union_tag`: reading `ARRAY` and `UNION` values
│ ├── value_defaults.rs # `Value::as_*_or` when a value is present
│ ├── value_getters.rs # Each typed `Value` getter at its own type; out-of-range TIMETZ / TIME_NS are not cast
│ ├── value_nested.rs # Nested `Value` construction and inspection against a live `DuckDB`
│ ├── value_query.rs # `Value` getters, DECIMAL binding, `Expression::fold`
│ ├── value_render.rs # `Value` rendering of values SQL builds and `DuckDB` cannot render
│ ├── value_temporal.rs # Every `Value` getter against every temporal source type, at every edge
+│ ├── vector_capacity_guard.rs # `OwnedVector::new` refuses a capacity whose nested `ARRAY` child would overflow
│ ├── vector_dt.rs # NULLs in nested output vectors; selection vectors
│ ├── vector_ops.rs # `OwnedVector::into_raw`, `copy_selected`, `slice`, `reference_value`, `reference_vector` (`duckdb-1-5`)
│ ├── vector_support.rs # Helpers shared by the vector-module tests
+│ ├── wide_rows.rs # Every fixed-width and string type echoed over thousands of distinct rows
│ └── writer_api.rs # `write_timestamp_tz`, `write_str`, and the Ok path of `try_write_varchar` / `try_write_blob`
├── benches/
│ ├── interval_bench.rs # Criterion benchmarks
@@ -399,7 +448,7 @@ quack-rs/
├── .github/workflows/ci.yml # CI pipeline
├── .github/workflows/docs.yml # GitHub Pages deployment (the versioned site)
├── CONTRIBUTING.md
-├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L19, P1–P15)
+├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L21, P1–P17)
├── CHANGELOG.md
└── README.md
```
@@ -408,19 +457,24 @@ quack-rs/
## Releasing
-quack-rs uses `libduckdb-sys = ">=1.4.4, <2"` — a bounded range covering the bindings of
-DuckDB 1.4.4 and later 1.x releases. Every DuckDB from 1.4.4 through 2.0.0 loads C API
-`v1.2.0` extensions. The `<2` upper bound does not exclude the DuckDB 2.0 bindings:
-`libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`,
-so DuckDB 2.0.0 is `1.20000.0`, which the range admits. That release was published on
-2026-10-22 and declares `rust-version` 1.88; MSRV-aware resolution keeps a Rust 1.86 build
-on `1.10506.x`. Before broadening the range to a new major band:
+quack-rs uses `libduckdb-sys = ">=1.4.4, <1.20100"` — a bounded range covering the bindings of
+DuckDB 1.4.4 through 2.0.x. Every DuckDB from 1.4.4 through 2.0.x loads C API
+`v1.2.0` extensions. `libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as
+`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0` and DuckDB 2.1.0 is
+`1.20100.0`, the first release the range excludes. The upper bound exists because the 1.20000
+bindings keep the C API names at the crate root only through a re-export that upstream marks
+for removal (see [Installation](getting-started/installation.md#adding-quack-rs-to-an-existing-extension)).
+The 1.20000 bindings were published on 2026-10-22 and declare `rust-version` 1.88;
+MSRV-aware resolution keeps a Rust 1.86 build on `1.10506.x`. Before widening the upper bound
+to the next DuckDB minor:Before broadening the range to a new major band:
1. Read the DuckDB changelog for C API changes
2. Check the new C API version string (used in `duckdb_rs_extension_api_init`)
3. Update `DUCKDB_API_VERSION` in `src/lib.rs` if the C API version changed
4. Audit all callback signatures against the new `libduckdb-sys` bindings
-5. Update the `libduckdb-sys` and `duckdb` version requirements in `Cargo.toml`
+5. Update the `libduckdb-sys` and `duckdb` version requirements in `Cargo.toml`, in the
+ manifests of `examples/hello-ext`, `fuzz`, `book/doctest` and `benches/engine`, and in
+ every snippet the documentation shows
Versions follow [Semantic Versioning](https://semver.org/) as Cargo applies it.
While the crate is pre-1.0, a breaking change to the public API bumps the
diff --git a/book/src/data/intervals.md b/book/src/data/intervals.md
index a5928cba..2e5b0ce4 100644
--- a/book/src/data/intervals.md
+++ b/book/src/data/intervals.md
@@ -82,8 +82,8 @@ intervals and in `epoch_us(interval)`: **1 month = 30 days**.
This is not date arithmetic: DuckDB adds an interval to a date by calendar
months (`DATE '2024-01-31' + INTERVAL 1 MONTH` is `2024-02-29`). It also matches
SQL comparison only when the three fields share a sign: `INTERVAL '1 month' -
-INTERVAL '1 day'` converts to the same total as `INTERVAL '29 days'` but compares
-greater in SQL.
+INTERVAL '1 day'` converts to the same total as `INTERVAL '29 days'`, but DuckDB 1.4.4
+and 1.5.6 compare it as greater (measured), while DuckDB 2.0.0 compares the two as equal.
### Checked conversion (returns `Option`)
diff --git a/book/src/data/running-sql.md b/book/src/data/running-sql.md
index 643a0378..e947b378 100644
--- a/book/src/data/running-sql.md
+++ b/book/src/data/running-sql.md
@@ -107,6 +107,11 @@ fails the call, after the ones before it have run (and, outside an explicit
transaction, committed). An empty string, or just `;`, succeeds with an empty
result. `prepare` takes exactly one statement.
+Text that does not parse as a whole is not safe to pass through: DuckDB 1.4.4 and 1.5.6 run
+nothing, but DuckDB 2.0.0 runs the statements before the error, and the call may still report
+success ([Known
+Limitations](../reference/known-limitations.md#duckdb_query-runs-the-statements-before-a-parse-error-on-duckdb-20)).
+
## Bind values, do not interpolate them
Anything that did not come from your own source text — a table name from a
@@ -242,6 +247,14 @@ fn register(con: &Connection) -> Result<(), ExtensionError> {
connection between threads, not using one concurrently. Open one connection per
thread, or guard it with a mutex.
+**Do not change `threads` from a callback.** `SET threads = N` run on a connection you
+kept, from inside a table function's scan, resizes DuckDB's worker pool from one of its
+own workers. The outcome varied from run to run: on DuckDB 2.0.0 the process terminated
+with `terminate called after throwing an instance of 'std::system_error'` in three of four
+runs; on DuckDB 1.5.6 the statement failed with "Invalid argument" in five of six runs and
+succeeded once (the termination was also seen on 1.5.6). It is DuckDB
+behaviour that quack-rs cannot intercept. Set `threads` before the query starts.
+
## Cancelling a query and reading its progress
`OwnedConnection::interrupt_handle` returns an `InterruptHandle`, which is
@@ -250,7 +263,11 @@ thread can call its `cancel()` to stop the running query, which then fails with
an interrupt error at DuckDB's next check, or its `progress()` to read a
`QueryProgress` (`percentage`, `rows_processed`, `total_rows_to_process`).
`percentage` is `-1.0` when DuckDB cannot report progress, for example when
-the progress bar is disabled (`SET enable_progress_bar = true`).
+the progress bar is disabled (`SET enable_progress_bar = true`). On DuckDB 1.x it also
+stays `-1.0` for as long as any pipeline of the running query cannot estimate its
+progress (`-1.0` was read throughout one query on 1.5.6 where 2.0.0 reported
+1.18 %, 1.34 % and 1.47 %; observed in a review, not re-run for this page). Treat it as
+"unknown", not as an error.
`OwnedConnection::interrupt` and `progress` do the same on the calling thread.
```rust
diff --git a/book/src/data/values-and-parameters.md b/book/src/data/values-and-parameters.md
index b30bbd6c..d7b38816 100644
--- a/book/src/data/values-and-parameters.md
+++ b/book/src/data/values-and-parameters.md
@@ -220,6 +220,26 @@ options.struct_child(idx)?.as_str().ok()
# }
```
+### `ARRAY`, `UNION`, `BIT` and `BIGNUM` values, and casting a value
+
+`list_items` reads a `LIST` only. On a fixed-size `ARRAY` value (an `INTEGER[3]`
+argument of a table function, a `COPY` option or a scalar bind argument) the C API reports
+no elements, so the call returns an empty `Vec`. From 0.20.0:
+
+- `Value::array_items()` returns the elements of an `ARRAY` or a `LIST`, and `None` for a
+ SQL `NULL` or any other type.
+- `Value::union_tag()` (`duckdb-1-5`) is the index of the active member of a `UNION` value.
+ The member's value reads as text through `try_cast` to `VARCHAR`.
+- `Value::try_cast(&LogicalType)` is SQL `TRY_CAST` of a value to any logical type, nested
+ ones included. It returns `None` where DuckDB cannot convert, and refuses first the
+ payloads DuckDB cannot convert safely (an out-of-range timestamp, a `DECIMAL` that is too
+ wide, also inside an `ARRAY`); no C++ exception can reach Rust.
+- `Value::bit(&[bool])` and `Value::bignum(magnitude, negative)` build `BIT` and `BIGNUM`
+ values (an error for bytes DuckDB cannot use), and `Value::as_bit()` and
+ `Value::as_bignum()` read them. The getters return `None` for a SQL `NULL` and for a
+ value of another type, where the C getters abort the process
+ ([Pitfall P16](../reference/pitfalls.md#p16)).
+
---
`DataChunk`, which wraps the chunk a table function's scan callback writes its
diff --git a/book/src/data/vectors.md b/book/src/data/vectors.md
index 2adfd7b6..c01527eb 100644
--- a/book/src/data/vectors.md
+++ b/book/src/data/vectors.md
@@ -277,8 +277,13 @@ for row in 0..chunk.size() {
`ChunkWriter` wraps an output `duckdb_data_chunk` and counts the rows handed out
by `next_row`. It calls `set_size` with that count on drop, so the row count
-cannot be forgotten or set wrongly. `next_row` returns `None` once the chunk
-holds `duckdb_vector_size()` rows:
+cannot be forgotten. `next_row` returns `None` once the chunk
+holds `duckdb_vector_size()` rows. The count is of rows you **claimed**: write every row
+you claim. A claimed row that is never written is published with whatever the reused
+output buffer held last ([Pitfall L21](../reference/pitfalls.md#l21-a-chunk-row-you-claim-is-published-written-or-not)),
+so check that the source has another value before you claim the row, or give the last
+claimed row back with `release_row(row)` before the writer is dropped (it panics for any
+other row, so the mistake is loud):
```rust
# use libduckdb_sys::{duckdb_data_chunk, duckdb_function_info, duckdb_vector};
diff --git a/book/src/duckdb-1-5/arrow.md b/book/src/duckdb-1-5/arrow.md
index 83921a83..73d0f5b8 100644
--- a/book/src/duckdb-1-5/arrow.md
+++ b/book/src/duckdb-1-5/arrow.md
@@ -31,12 +31,11 @@ engine.
## The types
-| Type | Wraps | Freed by |
-|---|---|---|
-| `ArrowOptions` | `duckdb_arrow_options` | `duckdb_destroy_arrow_options` |
-| `ArrowSchema` | the `ArrowSchema` ABI record | `release(schema)` |
-| `ArrowArray` | the `ArrowArray` ABI record | `release(array)` |
-| `ArrowConvertedSchema` | `duckdb_arrow_converted_schema` | `duckdb_destroy_arrow_converted_schema` |
+- `ArrowOptions` wraps `duckdb_arrow_options` and is freed by `duckdb_destroy_arrow_options`.
+- `ArrowSchema` wraps the `ArrowSchema` ABI record and is freed by `release(schema)`.
+- `ArrowArray` wraps the `ArrowArray` ABI record and is freed by `release(array)`.
+- `ArrowConvertedSchema` wraps `duckdb_arrow_converted_schema` and is freed by
+ `duckdb_destroy_arrow_converted_schema`.
`RawArrowSchema` and `RawArrowArray` are the ABI records themselves, re-exported
for code that already speaks the raw interface.
@@ -162,6 +161,15 @@ therefore makes the caller's job:
It also refuses valid Arrow layouts that DuckDB imports wrongly: it walks the
array alongside its schema and returns `InvalidInput`, naming the node, for
+- a **record batch with a nonzero `offset`** (the top-level struct array): DuckDB ignores
+ a top-level offset and would import the wrong rows. The error says to export the batch
+ without an offset and slice its children instead; an arrow-rs `RecordBatch::slice` is
+ exported that way;
+- a list, large list or map whose offsets are negative, decrease, or end past the child
+ (DuckDB reads the child through them without a bounds check: a segmentation fault on 1.5.6
+ and 2.0.0), a list view with a negative offset or size, a child (struct field, fixed-size-list
+ child, union member) shorter than the rows its parent reads, and a dictionary index outside the
+ dictionary;
- an offset below the top level that DuckDB applies to the wrong rows: a
struct inside an offset struct or a list, a union's members, a run-end-encoded
array's value validity (`docs/upstream-duckdb-reports.md`, item 24);
diff --git a/book/src/duckdb-1-5/catalog.md b/book/src/duckdb-1-5/catalog.md
index 137997bd..7971cfb6 100644
--- a/book/src/duckdb-1-5/catalog.md
+++ b/book/src/duckdb-1-5/catalog.md
@@ -88,8 +88,9 @@ database's catalog is named `memory`; a database file's is the file's stem
(`analytics.duckdb` is `analytics`). An empty name returns `None`.
`catalog` also returns `None` when no transaction is active. Inside a function
-callback a transaction is always active. On an idle connection outside a query,
-there is none:
+callback a transaction is always active, and so it is in the registration closure of an
+entry point, which runs in one transaction from 0.20.0 (with `Registration::Autocommit` it
+does not). On an idle connection outside a query, there is none:
```rust
# use quack_rs::catalog::CatalogEntryType;
diff --git a/book/src/duckdb-1-5/error-data.md b/book/src/duckdb-1-5/error-data.md
index eeecddf8..0023c4e2 100644
--- a/book/src/duckdb-1-5/error-data.md
+++ b/book/src/duckdb-1-5/error-data.md
@@ -30,7 +30,8 @@ if err.has_error() {
## Constructing an error
-Build a structured error to hand back to DuckDB (for example from a callback):
+Build a structured error value, for example in a test or to pass between your own
+functions:
```rust
# // ErrorData is a DuckDB object: fill the dispatch table first.
@@ -42,6 +43,13 @@ assert!(err.has_error());
assert_eq!(err.error_type(), DuckDbErrorType::InvalidInput);
```
+**No callback API accepts an `ErrorData`.** A scalar, aggregate, table, cast or `COPY`
+callback reports a failure with its `set_error(&str)`, and DuckDB chooses the error
+category: a scalar function's execution error surfaces as an `Invalid Input Error`, a
+table function's bind error as a `Binder Error`, whatever `DuckDbErrorType` you hold.
+To branch on a kind of failure inside your extension, keep your own error enum and
+format it into the message.
+
## Propagating with `?`
`into_extension_error` converts an `ErrorData` into the SDK's [`ExtensionError`],
diff --git a/book/src/duckdb-1-5/expression.md b/book/src/duckdb-1-5/expression.md
index bf5e315a..2a10f2f0 100644
--- a/book/src/duckdb-1-5/expression.md
+++ b/book/src/duckdb-1-5/expression.md
@@ -49,6 +49,10 @@ unsafe extern "C" fn my_bind(info: RawScalarBindInfo) {
| `fold(&client_context)` | `Result<`[`Value`]`, `[`ErrorData`]`>` — evaluate a constant expression |
| `from_raw(raw)` (unsafe) / `as_raw()` / `is_null()` | Handle inspection / escape hatches |
+A prepared-statement parameter (`?`) is not foldable at bind time: a bind callback that
+requires a constant makes `PREPARE p AS SELECT f(x, ?)` fail (measured on DuckDB 1.5.6 and
+2.0.0). A literal and a constant expression such as `1 + 1` are foldable.
+
`fold` only succeeds when [`is_foldable`][is_foldable] returns `true`; otherwise it
returns an [`ErrorData`] of type `InvalidInput`. A successful `fold` can still
yield a SQL `NULL` (for example `NULL::INTEGER`), whose typed getters return
diff --git a/book/src/duckdb-1-5/file-system.md b/book/src/duckdb-1-5/file-system.md
index a9e7e77e..4e260522 100644
--- a/book/src/duckdb-1-5/file-system.md
+++ b/book/src/duckdb-1-5/file-system.md
@@ -9,6 +9,12 @@ then resolve through every registered file system — `httpfs` (`s3://`,
`http://`) when it is loaded, in-memory files, and so on — where `std::fs` only
ever sees local disk.
+> **Security: `std::fs` ignores DuckDB's permission settings.** A deployment can lock
+> DuckDB down with `SET enable_external_access = false` or `SET allowed_directories`. An
+> extension that opens a user-supplied path with `std::fs` reads it anyway. Open such
+> paths with `FileSystem` instead: see
+> [the section below](#security-only-filesystem-honours-duckdbs-permission-settings).
+
## Obtaining a `FileSystem`
A `FileSystem` comes from a [`ClientContext`], which most function callbacks can
@@ -104,6 +110,36 @@ silently treat as an empty file.
`FileSystem` exposes `open(path, options)` and `error_data()`. Both `FileSystem`
and `FileHandle` are RAII: they are destroyed (and the handle closed) on drop.
+## Security: only `FileSystem` honours DuckDB's permission settings
+
+DuckDB can be locked down so that SQL cannot reach the file system:
+`SET enable_external_access = false`, or `SET allowed_directories = ['/data']` together
+with it. Everything DuckDB opens itself goes through its file system layer and obeys those
+settings, and so does a file you open with `FileSystem`. **`std::fs` does not.** It talks
+to the operating system directly, so a table function, replacement scan or `COPY` reader
+that passes a path from SQL to `std::fs::read` or `File::open` is a way around the
+restriction the person running DuckDB set. Measured on DuckDB 1.5.6 and 2.0.0 with a table
+function that returns a file's size, once through each API:
+
+```sql
+SET enable_external_access = false;
+SELECT count(*) FROM read_csv('data.csv'); -- Permission Error: Cannot access file ...
+SELECT * FROM size_via_filesystem('data.csv'); -- Permission error from FileSystem::open
+SELECT * FROM size_via_std_fs('data.csv'); -- returns the size: no restriction applied
+```
+
+`allowed_directories` behaves the same way: `read_csv` and `FileSystem` refuse a path
+outside the list, `std::fs` reads it. The same holds for anything else your extension does
+on its own, such as a network connection opened with a Rust crate: no DuckDB setting
+reaches it.
+
+The rule for an extension that reads or writes a path it did not hard-code: open it with
+`FileSystem`, from the `ClientContext` of the bind callback (or the copy callbacks). That
+needs the `duckdb-1-5` feature and DuckDB 1.5.0 or later. Without it, do not open
+user-supplied paths at all; take the data as a `VARCHAR` or `BLOB` argument and let SQL
+do the reading. `FileSystem` also resolves `s3://` and `http://` paths when `httpfs` is
+loaded, which `std::fs` never does.
+
## Lifetimes
A `FileSystem<'ctx>` borrows the `ClientContext` it came from, and a
diff --git a/book/src/faq.md b/book/src/faq.md
index 220b824d..b08f5a9f 100644
--- a/book/src/faq.md
+++ b/book/src/faq.md
@@ -19,25 +19,32 @@ utilities for publishing community extensions.
Building a DuckDB extension in Rust means solving a set of undocumented FFI
problems that each developer otherwise discovers independently. quack-rs
-documents all 34 known pitfalls, and its API prevents most of them. See the
+documents all 38 known pitfalls, and its API prevents most of them. See the
[Pitfall Catalog](reference/pitfalls.md).
### What DuckDB version does quack-rs target?
-quack-rs requires `libduckdb-sys = ">=1.4.4, <2"` and supports DuckDB 1.4.4
-through 2.0.0. Releases before 1.4.4 are not supported. CI runs the test
+quack-rs requires `libduckdb-sys = ">=1.4.4, <1.20100"` and, with the default features,
+supports DuckDB 1.4.4 through 2.0.x. An extension that uses a `duckdb-1-5*` feature loads into
+1.5.6 and 2.0.0 only, and refuses a release quack-rs has not listed (see
+[ABI Compatibility](concepts/abi.md#unknown-duckdb-versions)). Releases before 1.4.4 are not
+supported. The upper bound keeps out the
+DuckDB 2.1 bindings until a quack-rs release has been tested against them; see
+[Installation](getting-started/installation.md#adding-quack-rs-to-an-existing-extension).
+CI runs the test
suite against DuckDB v1.4.4, v1.4.5, v1.5.0, v1.5.3, v1.5.4, v1.5.5, v1.5.6 and
-v2.0.0 (the last with the DuckDB 2.0 bindings, `libduckdb-sys` 1.20000.0), and loads a built
+v2.0.0 (the last with the DuckDB 2.0 bindings, `libduckdb-sys` 1.20000.x), and loads a built
extension into v1.4.4, v1.5.0, v1.5.5, v1.5.6, v2.0.0 and the latest release.
DuckDB supports 2.0 (a long-term support release) until at least October 2027,
1.5 until March 2027 and 1.4 until late November 2026.
The C API version passed to the dispatch-table initializer is `"v1.2.0"`,
available as `quack_rs::DUCKDB_API_VERSION`. Every supported DuckDB release
-(1.4.4 and later) loads extensions built for it (1.5.6 and 2.0.0 declare C API
+(1.4.4 and later) loads extensions built for it (1.5.6 and 2.0.x declare C API
`v1.5.6`, but still accept `v1.2.0`). An extension that uses the `duckdb-1-5*`
features is stamped C API `v1.5.6` instead
-(`quack_rs::DUCKDB_API_VERSION_1_5_6`) and loads into DuckDB 1.5.6 and 2.0.0.
+(`quack_rs::DUCKDB_API_VERSION_1_5_6`) and loads into DuckDB 1.5.6 and 2.0.0; the
+runtime guard refuses a release quack-rs has not listed (2.0.1, for now).
The C API version is a separate identifier from the DuckDB release and from the
`libduckdb-sys` crate version, which encodes DuckDB `vMAJOR.MINOR.PATCH` as
`1.(10000·MAJOR + 100·MINOR + PATCH).N` (e.g. `1.10505.0` for DuckDB 1.5.5,
@@ -50,24 +57,26 @@ Rust **1.86.0** or later. This is enforced in `Cargo.toml` with
### Is quack-rs production-ready?
-It is pre-1.0, and you should judge it against your own requirements rather than
-take a yes. What the record shows:
+It is pre-1.0, and you should judge it against your own requirements; this page cannot
+give you a yes or a no. What the record shows:
- **The API still changes.** Minor releases before 1.0 can break it; each one lists
its breaking changes, with migration notes, in the [changelog](reference/changelog.md).
-- **Audits keep finding real defects.** The review released as 0.16.0 fixed 24
- defects in earlier releases, including two heap-corruption paths. The 0.18.0 audits fixed further soundness holes,
- process aborts and wrong answers, and found one serious defect in unreleased
- code before it was published: aggregate functions returned wrong results in
- release builds with Cargo's default profile. `AUDIT.md` in the repository
- records each audit, what it found, and whether each fix was reproduced against
- a real DuckDB or derived from DuckDB's source.
+- **Audits keep finding real defects.** The 0.16.0 release, which came out of a review,
+ fixed defects in earlier releases, including two heap-corruption paths. The 0.18.0
+ audits fixed further soundness holes, process aborts and wrong answers, and found one
+ serious defect in unreleased code before it was published: aggregate functions returned
+ wrong results in release builds with Cargo's default profile. `AUDIT.md` in the
+ repository records each audit, what it found, and whether each fix was reproduced
+ against a real DuckDB or derived from DuckDB's source. Use 0.18.0 or later: earlier
+ releases still have the soundness holes in the safe API, and 0.7.0 through 0.15.0 can
+ corrupt the heap when the `duckdb-1-5` features are used.
- **Some limits are DuckDB's.** The C API has defects quack-rs can only document or
work around; see [Known Limitations](reference/known-limitations.md).
It was extracted from
[duckdb-behavioral](https://github.com/tomtom215/duckdb-behavioral), a DuckDB
-community extension, where the first 16 of the pitfalls it now documents were
+community extension, where the first of the pitfalls it now documents were
discovered. If you ship an extension built on it, run end-to-end tests that load
the extension into each DuckDB release you support (see the
[Testing Guide](testing.md)).
@@ -103,6 +112,15 @@ bound when it is created, so the
`events` table must already exist or `register` returns an error.
See [SQL Macros](functions/sql-macros.md).
+### Can my extension read a file with `std::fs`?
+
+It can, and it should not when the path comes from SQL. `std::fs` ignores DuckDB's
+`enable_external_access` and `allowed_directories` settings, so a function that reads a
+user-supplied path with it works around a restriction the person running DuckDB set
+(`read_csv` on the same path fails with a permission error). Open such paths with the
+`FileSystem` wrapper (feature `duckdb-1-5`); see
+[Virtual File System](duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings).
+
### Can I register multiple overloads of the same function?
Yes, using `AggregateFunctionSetBuilder` (for aggregates) or
@@ -344,7 +362,8 @@ populated by DuckDB at extension load time.
### Why not use `duckdb-loadable-macros`?
Use it when it fits. As of `duckdb` and `duckdb-loadable-macros` 1.20000.1 the
-macro opens a `Connection` from the raw database (`Connection::open_from_raw`)
+macro opens a `duckdb::Connection` from the raw database
+(`duckdb::Connection::open_from_raw`)
and reports errors through the API's `set_error`; an earlier release of the
macro depended on `extract_raw_connection` and the `duckdb` crate's internal
layout, which crashed when the layout changed, and that history is why
diff --git a/book/src/functions/aggregate-sets.md b/book/src/functions/aggregate-sets.md
index 443b6885..288571ea 100644
--- a/book/src/functions/aggregate-sets.md
+++ b/book/src/functions/aggregate-sets.md
@@ -6,11 +6,11 @@ function in Rust with `AggregateFunctionSetBuilder`, including variadic aggregat
such as `retention(c1, c2, ..., c32)` and overloads with different return types.
> **Known DuckDB limitation.** C-API aggregates — including every overload in a
-> set — read out of bounds under `agg(x) OVER ()` (whole-partition window frames)
-> and `agg(x ORDER BY y)`. This is a DuckDB C API defect; see
+> set — read out of bounds under `agg(x) OVER ()` and other whole-partition window frames,
+> and under `agg(x ORDER BY y)`. This is a DuckDB C API defect; see
> [Aggregate Functions](aggregate.md#known-duckdb-limitation) for the details and
-> DuckDB source lines. On DuckDB 1.4.x and 1.5.x, do not use C-API aggregates
-> in those two query shapes.
+> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) for the exact shapes. On DuckDB 1.4.x and 1.5.x, do not use
+> C-API aggregates in those query shapes.
>
> Reported upstream as [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109)
> and fixed in DuckDB 2.0.0.
@@ -77,7 +77,7 @@ works:
```text
arg_max(ANY, ANY) -> ANY
-arg_max(ANY, ANY, ANY) -> ANY[]
+arg_max(ANY, ANY, BIGINT) -> ANY[]
```
Set the return type on the overload with `AggregateOverloadBuilder::returns` (or
diff --git a/book/src/functions/aggregate-state.md b/book/src/functions/aggregate-state.md
index 9cba9252..285b519e 100644
--- a/book/src/functions/aggregate-state.md
+++ b/book/src/functions/aggregate-state.md
@@ -6,11 +6,11 @@ the `AggregateState` trait and `FfiState`, which manages each state's lifecyc
raw-pointer code for it.
> **Known DuckDB limitation.** Every C-API aggregate, and so every aggregate that
-> uses `FfiState`, reads out of bounds under `agg(x) OVER ()` (whole-partition
-> window frames) and `agg(x ORDER BY y)`. This is a DuckDB C API defect; see
+> uses `FfiState`, reads out of bounds under `agg(x) OVER ()` and other whole-partition
+> window frames, and under `agg(x ORDER BY y)`. This is a DuckDB C API defect; see
> [Aggregate Functions](aggregate.md#known-duckdb-limitation) for the details and
-> DuckDB source lines. On DuckDB 1.4.x and 1.5.x, do not use C-API aggregates
-> in those two query shapes.
+> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) for the exact shapes. On DuckDB 1.4.x and 1.5.x, do not use
+> C-API aggregates in those query shapes.
>
> Reported upstream as [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109)
> and fixed in DuckDB 2.0.0.
diff --git a/book/src/functions/aggregate.md b/book/src/functions/aggregate.md
index 5aa5d173..62c1c602 100644
--- a/book/src/functions/aggregate.md
+++ b/book/src/functions/aggregate.md
@@ -8,9 +8,11 @@ also has a `combine` step that merges partial results from parallel workers.
## Known DuckDB limitation
-> **Before DuckDB 2.0.0, do not use a C API aggregate as `agg(x) OVER ()` or
-> `agg(x ORDER BY y)`.** In those two query shapes DuckDB 1.4.x and 1.5.x read
-> past the end of the state array, which usually crashes the process. This is a
+> **Before DuckDB 2.0.0, do not use a C API aggregate as `agg(x) OVER ()`, over an
+> explicit `UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, or as
+> `agg(x ORDER BY y)`.** In those query shapes DuckDB 1.4.x and 1.5.x read
+> past the end of the state array, which usually crashes the process (and on 1.4.4,
+> `OVER (PARTITION BY p)` without an `ORDER BY` too). This is a
> DuckDB defect, fixed in 2.0.0, that an extension cannot detect or prevent; see
> [Pitfall L11](../reference/pitfalls.md#l11-c-api-aggregates-crash-under-aggx-over--and-aggx-order-by-y)
> and [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11).
diff --git a/book/src/functions/copy-functions.md b/book/src/functions/copy-functions.md
index 42d3583b..9669a89b 100644
--- a/book/src/functions/copy-functions.md
+++ b/book/src/functions/copy-functions.md
@@ -6,6 +6,12 @@ This page shows how to register one from a Rust extension with quack-rs'
> **Requires the `duckdb-1-5` feature flag** (DuckDB 1.5.0+).
+> **Open the file through DuckDB.** The path of a `COPY` is user input. `std::fs` ignores
+> `SET enable_external_access = false` and `allowed_directories`, and cannot reach
+> `s3://` or `http://` paths; the
+> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings)
+> wrapper honours the settings and resolves every registered file system.
+
A format can support writing, reading, or both:
| Direction | You supply | DuckDB calls |
@@ -149,6 +155,16 @@ for i in 0..bind.result_column_count() {
# }
```
+## Choosing the format from the file name
+
+`COPY t TO 'out.my_format'` with no `FORMAT` clause picks your format when the file
+extension equals the copy function's name, and so does `COPY t FROM 'in.my_format'` for the
+reader. Measured on DuckDB 1.5.6 and 2.0.0: a copy function named `pr_fmt` was used for
+`out.pr_fmt` and was not used for `out.dat`. A function cannot declare a different
+extension (the C API v1 has no way to: see [Known
+Limitations](../reference/known-limitations.md#custom-copy-formats-cannot-declare-a-file-extension)),
+so name the format after the extension you want recognised.
+
## Registering a format name twice
`duckdb_register_copy_function` drops a copy function whose name already exists —
@@ -175,6 +191,13 @@ persisted, so reloading into a database file does not trip the check.
- Several values (`LST (1, 2)`) arrive as a `LIST`, or an unnamed `STRUCT` when
their types differ.
- An explicit `NULL` value is rejected by the binder before your callback runs.
+- **DuckDB does not validate the options of a `COPY … TO` format.** A copy function cannot
+ declare which options it takes, so `(FORMAT my_format, bogus 1)`, an option without a
+ value, a list where you expect a scalar, and a value of the wrong type all reach your
+ bind callback and are accepted unless you reject them. (A built-in format rejects the
+ same typo: `Unrecognized option "bogus" for csv`.) Check every name and type yourself and
+ call `set_error`; otherwise a misspelt option is silently ignored. A `COPY … FROM` reader
+ is different: its options are the table function's named parameters, which DuckDB checks.
- Field order follows DuckDB's internal hash map, not the statement — look fields
up by name:
@@ -204,6 +227,26 @@ The data pointers are untyped, so the compiler cannot check this for you:
partition with `PARTITION_BY` (reachable from more than one thread). Guard any
mutation with a `Mutex` unless you know neither option is in use.
+### Typed state, without raw pointers
+
+From 0.20.0 the three data pointers have typed forms that box the value behind a type tag
+and generate a destructor that contains a panic in `T::drop`, so no `Box::into_raw` and no
+`extern "C"` destructor is written by hand (all need `duckdb-1-5`):
+
+| Set | Read |
+|-----|------|
+| `CopyFunctionBuilder::extra_info_typed(value)` | `extra_info_ref::()` on each of the four info types |
+| `CopyBindInfo::set_bind_data_typed(value)` | `bind_data_ref::()` in global init, sink and finalize |
+| `CopyGlobalInitInfo::set_global_state_typed(value)` | `global_state_ref::()` in sink and finalize |
+
+`T` is `Send + Sync + 'static`, because the sink can run on several threads at once; state
+that changes needs a `Mutex` or atomics inside `T`. A read returns `None` when nothing was
+set or when the value has another type, instead of reinterpreting memory. The readers are
+`unsafe` only because they cannot tell a typed value from one set through the raw
+`set_bind_data` or `set_global_state`: use one form per slot. `FileHandle` is `Send` (not
+`Sync`) from 0.20.0, so a file opened in global init can be kept in the global state inside
+a `Mutex`.
+
## Callback signatures
| Phase | Signature | Guarded macro |
@@ -227,7 +270,7 @@ the handle at the top of your callback to access helper methods:
| Method | Description |
|--------|-------------|
| `column_count()` | Number of output columns |
-| `column_type(index)` | `LogicalType` of the column at `index`, or `None` if out of range |
+| `column_type(index)` | `LogicalType` of the column at `index`, or `None` if out of range. There is no column *name*: the C API does not give a `COPY … TO` bind callback the names, so a writer that wants a header needs a `names` option from the user |
| `options()` | The `COPY … TO` options, as one `STRUCT` `Value` (see [Options](#options)) |
| `get_extra_info()` | Extra-info pointer set on the copy function |
| `set_bind_data(data, destroy)` | Store bind data and its destructor |
diff --git a/book/src/functions/null-handling.md b/book/src/functions/null-handling.md
index 94108822..44e0bbd0 100644
--- a/book/src/functions/null-handling.md
+++ b/book/src/functions/null-handling.md
@@ -54,8 +54,11 @@ static void VerifyNullHandling(const BoundFunctionExpression &expr, DataChunk &a
}
```
-Every DuckDB a user installs is a release build, so that assertion is compiled
-out.
+Through DuckDB 1.5.6 that check exists only in debug builds, so every DuckDB a user
+installs skips it. DuckDB 2.0.0 compiles it into release builds but runs it only after
+`SET debug_verification_mode = 'VERIFY_FUNCTIONS'` (the default is `NONE`); with that
+setting a callback that ignores validity fails with `INTERNAL Error: VerifyNullHandling
+failed for scalar function ...`, and with the default it is believed, as before.
### Why the obvious test passes anyway
diff --git a/book/src/functions/panic-safe-callbacks.md b/book/src/functions/panic-safe-callbacks.md
index 5911db98..602d9ca6 100644
--- a/book/src/functions/panic-safe-callbacks.md
+++ b/book/src/functions/panic-safe-callbacks.md
@@ -165,8 +165,9 @@ output does not compile in a scalar function's.
`catch_unwind` cannot catch anything when the crate is built with
`panic = "abort"`: the process ends at the panic site, before any guard runs.
-Keep your extension on `panic = "unwind"`. The scaffold generates it, and
-`validate_release_profile` rejects `"abort"`. See
+Keep your extension on `panic = "unwind"`. The scaffold generates it,
+`validate_release_profile` rejects `"abort"`, and the `entry_point!` macros refuse to
+compile under it. See
[Installation](../getting-started/installation.md#why-panic--unwind-not-abort).
## Prefer the closure-based builders where they fit
diff --git a/book/src/functions/replacement-scan.md b/book/src/functions/replacement-scan.md
index 7919a994..cdd710d0 100644
--- a/book/src/functions/replacement-scan.md
+++ b/book/src/functions/replacement-scan.md
@@ -15,6 +15,12 @@ register one from a Rust extension with quack-rs.
`quack-rs` provides `ReplacementScanBuilder` (a static registration helper) and
`ReplacementScanInfo` (an ergonomic wrapper for callbacks).
+> **The table function a replacement scan names must open files through DuckDB.** The
+> path comes straight from the user's SQL. `std::fs` ignores
+> `SET enable_external_access = false` and `allowed_directories`; the
+> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings)
+> wrapper (feature `duckdb-1-5`) honours them.
+
## Registration API
Unlike the other builders in quack-rs, `ReplacementScanBuilder` uses a single
diff --git a/book/src/functions/scalar.md b/book/src/functions/scalar.md
index 892f35fd..0e5e9199 100644
--- a/book/src/functions/scalar.md
+++ b/book/src/functions/scalar.md
@@ -104,11 +104,16 @@ with `register(con)`, or through a `Registrar` with `register_typed_scalar`.
# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
# unsafe fn demo(con: duckdb_connection) -> Result<(), ExtensionError> {
-ScalarFunctionBuilder::map1("double_it", |x: i64| x * 2)?.register(con)?;
+ScalarFunctionBuilder::map1("is_even", |x: i64| x % 2 == 0)?.register(con)?;
# Ok(())
# }
```
+Arithmetic inside the closure is yours to check: a plain `x * 2` wraps on overflow in a
+release build and panics (so fails the query) in a debug build. Use `try_map1` and
+`checked_mul` to turn overflow into an error in both, as the
+[Quick Start](../getting-started/quick-start.md) does.
+
A closure can also return `Option`, where `None` is SQL `NULL`, and the
`try_map1` / `try_map2` / `try_map3` / `try_map1_str` / `try_map2_str`
constructors take a closure returning `Result`. An `Err` fails the query
@@ -147,6 +152,12 @@ ScalarFunctionBuilder::map3("lerp", |a: f64, b: f64, t: f64| a + (b - a) * t)?.r
## Complete example: `double_it(BIGINT) → BIGINT`
+For a function this small the closure builder above is the better choice
+(`try_map1` with `checked_mul`, as in the [Quick Start](../getting-started/quick-start.md)).
+Written as a raw callback, with `scalar_callback!` for the panic guard, it looks like this.
+`value * 2` would wrap silently in a release build, so the callback reports overflow
+through `ScalarFunctionInfo::set_error`:
+
```rust
# use quack_rs::prelude::*;
# fn live_connection() -> libduckdb_sys::duckdb_connection {
@@ -165,20 +176,15 @@ ScalarFunctionBuilder::map3("lerp", |a: f64, b: f64, t: f64| a + (b - a) * t)?.r
# let reader = unsafe { chunk.reader(0) };
# unsafe { reader.is_valid(0).then(|| reader.read_i64(0)) }
# }
+use quack_rs::scalar::ScalarFunctionInfo;
use quack_rs::vector::{VectorReader, VectorWriter};
-use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk, duckdb_vector};
-unsafe extern "C" fn double_it(
- _info: duckdb_function_info,
- input: duckdb_data_chunk,
- output: duckdb_vector,
-) {
+quack_rs::scalar_callback!(double_it, |info, input, output| {
// SAFETY: DuckDB provides valid chunk and vector pointers.
let reader = unsafe { VectorReader::new(input, 0) }; // column 0
let mut writer = unsafe { VectorWriter::new(output) };
- let row_count = reader.row_count();
- for row in 0..row_count {
+ for row in 0..reader.row_count() {
if unsafe { !reader.is_valid(row) } {
// NULL input → NULL output
// SAFETY: row < row_count, writer is valid.
@@ -186,9 +192,13 @@ unsafe extern "C" fn double_it(
continue;
}
let value = unsafe { reader.read_i64(row) };
- unsafe { writer.write_i64(row, value * 2) };
+ let Some(doubled) = value.checked_mul(2) else {
+ unsafe { ScalarFunctionInfo::new(info) }.set_error("double_it: BIGINT overflow");
+ return;
+ };
+ unsafe { writer.write_i64(row, doubled) };
}
-}
+});
# let con = live_connection();
# unsafe {
# ScalarFunctionBuilder::new("double_it").param(TypeId::BigInt).returns(TypeId::BigInt)
@@ -197,6 +207,8 @@ unsafe extern "C" fn double_it(
# assert_eq!(query_i64(con, "SELECT double_it(21)"), Some(42));
# // From a column, not a literal: `double_it(NULL::BIGINT)` is constant-folded.
# assert_eq!(query_i64(con, "SELECT double_it(i) FROM (VALUES (NULL::BIGINT)) t(i)"), None);
+# let err = unsafe { quack_rs::query::query(con, "SELECT double_it(9223372036854775807)") }.err().unwrap();
+# assert!(err.as_str().contains("double_it: BIGINT overflow"), "{err}");
```
---
@@ -204,14 +216,10 @@ unsafe extern "C" fn double_it(
## Multi-parameter example: `add(BIGINT, BIGINT) → BIGINT`
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
-unsafe extern "C" fn add(
- _info: duckdb_function_info,
- input: duckdb_data_chunk,
- output: duckdb_vector,
-) {
+use quack_rs::scalar::ScalarFunctionInfo;
+
+quack_rs::scalar_callback!(add, |info, input, output| {
let col0 = unsafe { VectorReader::new(input, 0) }; // first param
let col1 = unsafe { VectorReader::new(input, 1) }; // second param
let mut writer = unsafe { VectorWriter::new(output) };
@@ -223,9 +231,13 @@ unsafe extern "C" fn add(
}
let a = unsafe { col0.read_i64(row) };
let b = unsafe { col1.read_i64(row) };
- unsafe { writer.write_i64(row, a + b) };
+ let Some(sum) = a.checked_add(b) else {
+ unsafe { ScalarFunctionInfo::new(info) }.set_error("add: BIGINT overflow");
+ return;
+ };
+ unsafe { writer.write_i64(row, sum) };
}
-}
+});
```
---
@@ -233,14 +245,8 @@ unsafe extern "C" fn add(
## VARCHAR example: `shout(VARCHAR) → VARCHAR`
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
-unsafe extern "C" fn shout(
- _info: duckdb_function_info,
- input: duckdb_data_chunk,
- output: duckdb_vector,
-) {
+quack_rs::scalar_callback!(shout, |_info, input, output| {
let reader = unsafe { VectorReader::new(input, 0) };
let mut writer = unsafe { VectorWriter::new(output) };
@@ -253,7 +259,7 @@ unsafe extern "C" fn shout(
let upper = s.to_uppercase();
unsafe { writer.write_varchar(row, &upper) };
}
-}
+});
```
---
@@ -293,6 +299,12 @@ unsafe fn register(con: duckdb_connection) -> Result<(), ExtensionError> {
}
```
+A function built from a closure (`map1`, `try_map1`, ...) is not a `ScalarOverloadBuilder`: its
+signature is fixed by the closure. Add it to a set with
+`ScalarFunctionSetBuilder::overload_typed(builder)`. That matters on DuckDB 1.4.x, where a
+second registration under a taken name is refused, so closures for several argument types need
+one set; from 1.5.0 DuckDB also accepts them registered one by one.
+
Like `AggregateFunctionSetBuilder`, this builder calls `duckdb_scalar_function_set_name`
on every individual function before adding it to the set
([Pitfall L6](../reference/pitfalls.md#l6-function-set-name-must-be-set-on-each-member)).
@@ -368,6 +380,24 @@ ScalarFunctionBuilder::new("flatten_list")
# }
```
+### `DECIMAL` parameters are not generic
+
+A `DECIMAL(w, s)` parameter takes exactly that width and scale. DuckDB casts every argument
+to the declared type before your callback runs, and the cast **rounds**: a function
+declared over `DECIMAL(4,1)` that doubles its argument, called with
+`0.123::DECIMAL(3,3)`, sees `0.1` and returns `0.2`, not `0.246` (measured on 1.5.6 and
+2.0.0). Declaring several widths as overloads does not work either: DuckDB's overload
+resolution does not tell `DECIMAL(4,1)` from `DECIMAL(9,3)` apart, so every call to such a
+set fails with "Could not choose a best candidate function" (measured on 1.5.6 and 2.0.0;
+the same holds for `MAP` and `ENUM` parameters of different parameterisations, and for those
+nested inside a `LIST`, `ARRAY` or `STRUCT`). quack-rs 0.20 refuses such a set when you
+register it; 0.19 and earlier registered it, and the first call failed. To accept any `DECIMAL`, declare the
+parameter `TypeId::Any`, read the argument vector's actual type at run time
+(`quack_rs::vector::vector_get_column_type`, then `LogicalType::decimal_width`,
+`decimal_scale` and `decimal_internal_type`) and return a type that does not depend on it
+(`DOUBLE`, `VARCHAR` or `HUGEINT`); or register one function per width under different
+names.
+
These methods are also available on `ScalarOverloadBuilder` for function sets:
```rust
@@ -481,9 +511,19 @@ feature is enabled:
### `bind(bind_fn)`
-Sets a custom bind callback that runs at plan time. Use this to inspect argument
-types and set the return type dynamically. Maps to
-`duckdb_scalar_function_set_bind`.
+Sets a custom bind callback that runs at plan time. Use it to inspect the argument
+types and constant arguments, to store bind data for the execute callback, and to reject
+a call with `set_error`. Maps to `duckdb_scalar_function_set_bind`.
+
+**The bind callback cannot set the return type.** The C API v1 has no such function: it
+offers `duckdb_scalar_function_bind_get_argument`, `_get_argument_count`,
+`_get_extra_info` and `_set_error`, and `ScalarBindInfo` wraps exactly those, plus
+`set_bind_data`, `set_bind_data_copy` and `get_client_context`. The return type is the one
+you declared with `returns`. When the result type must depend on the argument types,
+register one overload per result type with `ScalarFunctionSetBuilder` (for example
+`f(INTEGER) → INTEGER[]` and `f(VARCHAR) → STRUCT(v VARCHAR)`). Choosing the return type
+at bind time exists only in the C API v2
+([Known Limitations](../reference/known-limitations.md#duckdb-20s-c-api-v2-offers-what-v1-cannot)).
The callback takes a `RawScalarBindInfo` (wrap it with `ScalarBindInfo::new`),
not a bare `duckdb_bind_info`: DuckDB passes a table function's bind callback
@@ -502,7 +542,7 @@ do not type-check here.
# unsafe fn demo(con: duckdb_connection) -> Result<(), ExtensionError> {
ScalarFunctionBuilder::new("dynamic_return")
.varargs(TypeId::Varchar)
- .returns(TypeId::Varchar) // default; overridden in bind
+ .returns(TypeId::Varchar) // fixed: bind cannot change it
.bind(my_bind_fn)
.function(dynamic_return_fn)
.register(con)?;
@@ -598,10 +638,19 @@ ScalarFunctionBuilder::new("scaled")
optimizer duplicates it (filter pushdown through a projection does). Without a
copy callback the copy has **no** bind data at all, so `set` registers one that
clones `T`. Wrap data that is expensive or impossible to clone in an `Arc`.
+- **A bind-time constant cannot come from a prepared-statement parameter.** If `bind`
+ rejects an argument that is not foldable (see [Bound Expressions](../duckdb-1-5/expression.md)),
+ a `?` is not foldable: `PREPARE p AS SELECT f(x, ?)` fails at the `PREPARE`, with your
+ message, on DuckDB 1.5.6 and 2.0.0, while a literal or a constant expression such as `1 + 1`
+ folds. Say so in the error text, and tell users to pass such an argument as a literal.
- **Local state must be `Send`**: it is per thread, but may be freed on another.
-- **Call `set` at most once per callback.** DuckDB overwrites the stored pointer
- on a second call without freeing the first value, so that value is leaked
- (never dropped).
+- **A panicking `Clone` fails the query.** DuckDB gives a copied expression no bind data when
+ the copy callback fails; quack-rs records the panic instead, and the first `get` reports it
+ as the call's error (with the panic message), while `try_get` hands it back as an `Err`.
+- **Call `set` once per callback.** A second `ScalarBindData::set` in the same bind frees the
+ value it replaces. DuckDB's own call overwrites the stored pointer without freeing the first
+ value, so a value replaced through the raw `set_bind_data`, or through another wrapper of
+ the same bind call, is leaked (never dropped).
- **Bind data must depend only on the call's arguments** (their values when
constant, their types) and `extra_info`. DuckDB's
`CScalarFunctionBindData::Equals` ignores bind data, so two calls with the same
diff --git a/book/src/functions/sql-macros.md b/book/src/functions/sql-macros.md
index d0e8eff1..0d61c966 100644
--- a/book/src/functions/sql-macros.md
+++ b/book/src/functions/sql-macros.md
@@ -123,7 +123,7 @@ unsafe {
# unsafe { quack_rs::query::execute(con, "CREATE TABLE orders AS SELECT current_date - 3 AS order_date UNION ALL SELECT current_date - 30") }.unwrap();
# register(con).unwrap();
# assert_eq!(query_i64(con, "SELECT count(*) FROM recent_orders(7)"), 1);
-# assert_eq!(query_i64(con, "SELECT count(*) FROM active_users(users)"), 2);
+# assert_eq!(query_i64(con, "SELECT count(*) FROM active_users('users')"), 2);
```
A table macro's body is bound when the macro is created, so every table it names
@@ -134,7 +134,7 @@ fails with a catalog error. To take a table as a parameter, read it through
Use in DuckDB:
```sql
-SELECT * FROM active_users(users);
+SELECT * FROM active_users('users');
SELECT count(*) FROM recent_orders(7);
```
@@ -210,8 +210,12 @@ The **body** (expression or query) is your own extension code — it is included
**Never build macro bodies from untrusted user input.** `register` does refuse a body
that turns the statement into several (`1); DROP TABLE t; SELECT (1`) — it counts
statements with DuckDB's own parser (`duckdb_extract_statements`) and executes
-nothing if there is more than one — but a body can still change the meaning of the
-single statement it is part of.
+nothing unless the text is exactly one statement — but a body can still change the
+meaning of the single statement it is part of. Text that does not parse at all is
+refused too, with DuckDB's parse message: before 0.20.0 it was executed, which is
+harmless on DuckDB 1.4.4 and 1.5.6 (they parse the whole string first) but not on 2.0.0,
+which runs the statements that precede the parse error (see [Known
+Limitations](../reference/known-limitations.md#duckdb_query-runs-the-statements-before-a-parse-error-on-duckdb-20)).
A scalar body containing `--` gets a newline before the closing parenthesis, so a
trailing line comment (`"x + 1 -- plus one"`) does not comment it out.
@@ -240,8 +244,8 @@ schema — the **user's** database:
## How it works under the hood
`SqlMacro::register` counts the statements in `to_sql()` with
-`duckdb_extract_statements`, refuses more than one, and executes the
-`CREATE OR REPLACE MACRO` statement via `duckdb_query`.
+`duckdb_extract_statements`, refuses a text that is not exactly one statement, and
+executes the `CREATE OR REPLACE MACRO` statement via `duckdb_query`.
The query result is zero-initialized before `duckdb_query`, any error message is read
with `duckdb_result_error`, and `duckdb_destroy_result` runs on success and failure
diff --git a/book/src/functions/table-functions.md b/book/src/functions/table-functions.md
index c9f3f8f1..09f594e8 100644
--- a/book/src/functions/table-functions.md
+++ b/book/src/functions/table-functions.md
@@ -23,18 +23,32 @@ Both builders are backed by the helper types `BindInfo`, `InitInfo`, `FunctionIn
| Phase | Callback | Called when | Typical work |
|-------|----------|-------------|--------------|
-| **bind** | `bind_fn` | Query is planned (once per plan) | Extract parameters; declare output columns; store configuration in bind data |
+| **bind** | `bind_fn` | Query is planned (once per plan on DuckDB 1.x; also on every `EXECUTE` of a prepared statement on 2.0.0) | Extract parameters; declare output columns; store configuration in bind data |
| **init** | `init_fn` | Each execution of the plan starts | Allocate per-scan state (cursor, row index, etc.) |
| **scan** | `scan_fn` | Each output batch | Fill the output chunk with rows; set its size with `DataChunk::set_size` |
DuckDB calls the scan callback repeatedly until it sets the chunk size to 0, which
signals the end of the results.
-> **Bind once, init many times.** DuckDB keeps the bind data for as long as the
-> bound plan lives and runs `init` against it on **every** execution: each
-> `EXECUTE` of a prepared statement, each iteration of a recursive CTE that
-> references the function. Treat bind data as immutable after bind and build
+> **Bind data outlives the plan; bind itself may run again.** DuckDB keeps the bind
+> data for as long as the bound plan lives and runs `init` against it on **every**
+> execution: each `EXECUTE` of a prepared statement, each iteration of a recursive CTE
+> that references the function. Treat bind data as immutable after bind and build
> anything a scan consumes (cursors, open files) in `init`.
+>
+> How often `bind` runs depends on the DuckDB release. With a prepared statement that is
+> executed three times, counters in the callbacks read 1 bind and 3 inits on DuckDB 1.4.4
+> and 1.5.6, and **4 binds** and 3 inits on 2.0.0 (one for the `PREPARE`, one for each
+> `EXECUTE`). So a table function that reads a setting in `bind`, where a client
+> context is available, keeps the value from the first bind on 1.x and picks up a changed
+> setting on 2.0. Do not rely on either behaviour: only `bind` is given a client
+> context (`InitInfo` and `FunctionInfo` have none), so a table function cannot read a
+> setting at `init` or scan time.
+
+> **A table function that reads a file must open it through DuckDB.** `std::fs` ignores
+> `SET enable_external_access = false` and `allowed_directories`; the
+> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings)
+> wrapper (feature `duckdb-1-5`) honours them.
## Closure-based typed state (`with_state`)
@@ -138,17 +152,21 @@ fn register(reg: &impl Registrar) -> ExtResult<()> {
- `S` must be `Send + 'static` (plus `Clone` for `with_state`). `Sync` is **not**
required, so `TypedTableFunctionBuilder` forces scans to run on a single worker by
- calling `InitInfo::set_max_threads(1)` internally.
+ calling `InitInfo::set_max_threads(1)` internally. Measured: at `SET threads = 8` a
+ typed function's scan callbacks all ran on one thread, on DuckDB 1.5.6 and 2.0.0. A
+ scan that is the bottleneck of a query, over a large input, needs the raw builder
+ with `set_max_threads` above 1.
- The typed builder does **not** offer projection pushdown: with pushdown on, the
scan's chunk holds only the projected columns and a closure written against the
declared schema would write the wrong column. `build()` returns an error if
`projection_pushdown(true)` was set on the raw builder before `with_state` /
`with_bind_init`. Use the raw builder for pushdown.
- The bind closure must declare at least one column. With `duckdb-1-5`, a bind
- that declares none is reported as an ordinary bind error; DuckDB itself raises an
- `INTERNAL Error` with a C++ stack trace for it (and does so for a raw bind
- callback, which quack-rs cannot check after it returns — call `set_error`
- yourself).
+ that declares none is reported as an ordinary bind error. Without `duckdb-1-5` the
+ typed builder cannot check, and DuckDB itself raises an `INTERNAL Error` with a C++
+ stack trace (measured on 1.4.4, 1.5.6 and 2.0.0; the database stays usable). A raw bind
+ callback gets the same `INTERNAL Error` on every feature set, because quack-rs cannot
+ check it after it returns: call `set_error` yourself.
- Extensions that need multi-worker parallelism (`set_max_threads` above 1, with
`local_init` + thread-local buffers) should use the raw
[`TableFunctionBuilder`](#builder-api) directly.
@@ -274,7 +292,9 @@ quack_rs::table_scan_callback!(gs_scan, |info, output| {
};
let remaining = bind.total - state.pos;
- let batch = remaining.min(2048).max(0) as usize;
+ // Never hard-code the chunk size: use the vector size of the running DuckDB.
+ let capacity = quack_rs::vector::vector_size() as i64;
+ let batch = remaining.min(capacity).max(0) as usize;
let mut writer = unsafe { chunk.writer(0) };
for i in 0..batch {
diff --git a/book/src/getting-started/first-extension.md b/book/src/getting-started/first-extension.md
index efd8dc67..dad19297 100644
--- a/book/src/getting-started/first-extension.md
+++ b/book/src/getting-started/first-extension.md
@@ -1,57 +1,66 @@
# Your First Extension
-This page builds a DuckDB extension in Rust step by step, using `hello-ext`, the example
-extension bundled with quack-rs. The table lists four of the functions it registers, one of
-each major kind (the full list is in its
-[README](https://github.com/tomtom215/quack-rs/tree/main/examples/hello-ext)); this page walks
-through the aggregate and the scalar. The table function and the cast are covered in
-[Table Functions](../functions/table-functions.md) and
-[Cast Functions](../functions/cast-functions.md).
+This page builds a small DuckDB extension step by step: an aggregate function,
+`word_count`, and a scalar function, `first_word`. Every callback is guarded, so a
+panic in it fails the query with an error instead of aborting the DuckDB process. If
+you have not built an extension before, the [Quick Start](quick-start.md) is shorter
+and a good place to begin.
| SQL | Kind | Signature |
|-----|------|-----------|
| `word_count(text)` | Aggregate | `VARCHAR → BIGINT` |
| `first_word(text)` | Scalar | `VARCHAR → VARCHAR` |
-| `generate_series_ext(n)` | Table | `BIGINT → TABLE(value BIGINT)` |
-| `CAST(VARCHAR AS INTEGER)` | Cast | `VARCHAR → INTEGER` |
-Full source: [`examples/hello-ext/src/lib.rs`](https://github.com/tomtom215/quack-rs/blob/main/examples/hello-ext/src/lib.rs)
+> **Window functions and DuckDB before 2.0.** Every aggregate registered through the
+> C API, `word_count` included, crashes the DuckDB process on 1.4.x and 1.5.x when a
+> user runs it as `word_count(x) OVER ()`, over an explicit
+> `ROWS` or `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, or as
+> `word_count(x ORDER BY y)`. On 1.4.4, `OVER (PARTITION BY p)` without an `ORDER BY`
+> crashes too. This is a DuckDB defect that no extension can prevent, fixed in 2.0.0; see
+> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11).
+> Tell the users of your own aggregates.
+
+The repository's [`examples/hello-ext`](https://github.com/tomtom215/quack-rs/tree/main/examples/hello-ext)
+registers these two functions among many others. It writes them as hand-rolled
+`unsafe extern "C" fn`s, which are installed as written and run with no
+`catch_unwind`; its README says it shows what the C API calls look like, not how to
+write new code. The code below is the guarded form.
---
-## Build and try it
-
-```bash
-cargo build --release --manifest-path examples/hello-ext/Cargo.toml
-
-# DuckDB only loads files ending in `.duckdb_extension` that carry its
-# 512-byte metadata footer; a bare `.so` is refused. Append the footer:
-cargo run --bin append_metadata -- \
- examples/hello-ext/target/release/libhello_ext.so \
- hello_ext.duckdb_extension \
- --abi-type C_STRUCT --extension-version v0.1.0 \
- --duckdb-version v1.2.0 --platform linux_amd64
-
-# An unsigned local build needs -unsigned (it must be a startup flag).
-duckdb -unsigned
-```
-
-Then in the DuckDB CLI:
-
-```sql
-LOAD './hello_ext.duckdb_extension';
-
--- Aggregate: total words across all rows
-SELECT word_count(sentence) FROM (
- VALUES ('hello world'), ('one two three'), (NULL)
-) t(sentence);
--- → 5 (2 + 3; NULL contributes 0)
-
--- Scalar: first word of each row
-SELECT first_word(sentence) FROM (
- VALUES ('hello world'), (' padded '), (''), (NULL)
-) t(sentence);
--- → 'hello', 'padded', '', NULL
+## Set up the project
+
+Create a library crate and give it the `Cargo.toml` from the
+[Quick Start](quick-start.md), with the name `word_tools`:
+
+```toml
+[package]
+name = "word_tools"
+version = "0.1.0"
+edition = "2021"
+# With resolver 3, Cargo picks dependency versions that build on this.
+rust-version = "1.86"
+resolver = "3"
+
+[lib]
+# Must match the extension name: see Pitfall P1.
+name = "word_tools"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+quack-rs = "0.20"
+
+[dependencies.libduckdb-sys]
+version = ">=1.4.4, <1.20100"
+default-features = false
+features = ["loadable-extension"]
+
+[profile.release]
+panic = "unwind"
+lto = true
+opt-level = 3
+codegen-units = 1
+strip = true
```
---
@@ -60,18 +69,30 @@ SELECT first_word(sentence) FROM (
An extension has four parts:
-1. **State struct** — holds data accumulated during aggregation (aggregate only)
-2. **Callbacks** — `update`, `combine`, `finalize` (aggregate; `ffi_state::()` supplies `state_size`, `state_init` and `state_destroy`) or a single function callback (scalar)
-3. **Registration** — wire callbacks to DuckDB via `AggregateFunctionBuilder` / `ScalarFunctionBuilder`
-4. **Entry point** — DuckDB's initialization hook, generated by `entry_point!`
+1. **State struct**: holds data accumulated during aggregation (aggregate only)
+2. **Callbacks**: `update`, `combine` and `finalize` for the aggregate, written with
+ the `aggregate_*_callback!` macros (`ffi_state::()` supplies `state_size`,
+ `state_init` and `state_destroy`); for the scalar, a closure
+3. **Registration**: wire the callbacks to DuckDB with `AggregateFunctionBuilder` and
+ `ScalarFunctionBuilder`
+4. **Entry point**: DuckDB's initialization hook, generated by `entry_point!`
---
-## Part 1 — Aggregate function: `word_count`
+## Part 1: aggregate function `word_count`
An aggregate function accumulates state across many rows and emits one result per group.
-### 1a. The state struct
+### 1a. The logic and the state struct
+
+The counting is ordinary Rust, so it is easy to unit-test (see
+[Unit tests](#unit-tests-no-duckdb-process-needed)):
+
+```rust
+fn count_words(s: &str) -> i64 {
+ s.split_whitespace().count() as i64
+}
+```
```rust
# use quack_rs::prelude::*;
@@ -92,188 +113,199 @@ impl AggregateState for WordCountState {}
### 1b. `state_size`, `state_init` and `state_destroy`
These three callbacks are always identical boilerplate, so you do not write them:
-`.ffi_state::()` on the builder (see [Part 3](#part-3--registration))
+`.ffi_state::()` on the builder (see [Part 3](#part-3-registration))
installs `FfiState::::size_callback`, `init_callback` and
`destroy_callback` together. Installing all three for one type in a single call means
DuckDB never allocates space sized for one state and has it initialised as another.
-`size_callback` returns `FfiState::::size()` — one `usize` tag word followed
+`size_callback` returns `FfiState::::size()`: one `usize` tag word followed
by the state. A `T` aligned no more strictly than `usize` and at most 256 bytes (like
`WordCountState`) is stored inline in the bytes DuckDB allocates per group; a larger or more
strictly aligned `T` is boxed, and the slot holds the `Box` pointer. `init_callback` writes
-`WordCountState::default()` into the slot (or boxes it), then sets the tag — a value salted per
+`WordCountState::default()` into the slot (or boxes it), then sets the tag, a value salted per
state type, so a destructor only drops states that were initialised for its own `T`.
`destroy_callback` drops the `T` in each state (and its box, if the state is too
large to be stored inline), clearing the state's tag first so a second call is a
no-op.
-### 1c. `update` — accumulate one batch
+### 1c. `update`: accumulate one batch
+
+`aggregate_update_callback!` generates the `unsafe extern "C" fn` the builder expects and
+runs your body under `catch_unwind`: a panic in it, such as an arithmetic overflow in a
+debug build, fails the query with the panic message and leaves the process running. See
+[Panic-Safe Callbacks](../functions/panic-safe-callbacks.md).
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
# #[derive(Default, Debug)] struct WordCountState { count: i64 }
# impl AggregateState for WordCountState {}
# fn count_words(s: &str) -> i64 { s.split_whitespace().count() as i64 }
-unsafe extern "C" fn wc_update(
- _info: duckdb_function_info,
- input: duckdb_data_chunk,
- states: *mut duckdb_aggregate_state,
-) {
+quack_rs::aggregate_update_callback!(wc_update, |_info, input, states| {
let reader = unsafe { VectorReader::new(input, 0) };
- let row_count = reader.row_count();
-
- for row in 0..row_count {
+ for row in 0..reader.row_count() {
if !unsafe { reader.is_valid(row) } {
- continue; // NULL input → skip (contributes 0 words)
+ continue; // NULL input contributes no words
}
- let s = unsafe { reader.read_str(row) };
- let words = count_words(s);
+ let words = count_words(unsafe { reader.read_str(row) });
let state_ptr = unsafe { *states.add(row) };
if let Some(st) = unsafe { FfiState::::with_state_mut(state_ptr) } {
st.count += words;
}
}
-}
+});
```
Key points:
-- Check `is_valid(row)` before reading — never dereference an invalid (NULL) row
-- `VectorReader::new(chunk, col)` gives column `col` from the chunk
-- `count_words` is pure Rust — no unsafe, easy to unit-test separately
+- Check `is_valid(row)` before reading. Never read an invalid (NULL) row.
+- `VectorReader::new(chunk, col)` gives column `col` from the chunk, and `states[row]` is the
+ state of that row's group: rows in the same group share one state.
+- Each `unsafe { … }` block is its own scope: the macro body is not an `unsafe` context.
-### 1d. `combine` — merge parallel results
+### 1d. `combine`: merge parallel results
> **Pitfall L1**: DuckDB creates fresh target states before calling `combine`, set up by
> `state_init` (here `WordCountState::default()`), not copies of the source.
-> You must copy **all** fields — not just the result field. In an aggregate with config
+> You must copy **all** fields, not just the result field. In an aggregate with config
> fields (e.g., a histogram with a `bin_width`) you must also copy those, or results will
> be silently corrupted.
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
# #[derive(Default, Debug)] struct WordCountState { count: i64 }
# impl AggregateState for WordCountState {}
-unsafe extern "C" fn wc_combine(
- _info: duckdb_function_info,
- source: *mut duckdb_aggregate_state,
- target: *mut duckdb_aggregate_state,
- count: idx_t,
-) {
+quack_rs::aggregate_combine_callback!(wc_combine, |_info, source, target, count| {
for i in 0..count as usize {
- let src_ptr = unsafe { *source.add(i) };
- let tgt_ptr = unsafe { *target.add(i) };
- let src = unsafe { FfiState::::with_state(src_ptr) };
- let tgt = unsafe { FfiState::::with_state_mut(tgt_ptr) };
+ let src = unsafe { FfiState::::with_state(*source.add(i)) };
+ let tgt = unsafe { FfiState::::with_state_mut(*target.add(i)) };
if let (Some(s), Some(t)) = (src, tgt) {
t.count += s.count;
// If you add fields to WordCountState, combine them here too.
}
}
-}
+});
```
-### 1e. `finalize` — write output
+`combine` reads `source` and must leave it unchanged ([Pitfall L15](../reference/pitfalls.md#l15-combine-must-leave-its-source-states-unchanged)):
+the same source state can be combined into several targets.
+
+### 1e. `finalize`: write output
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
# #[derive(Default, Debug)] struct WordCountState { count: i64 }
# impl AggregateState for WordCountState {}
-unsafe extern "C" fn wc_finalize(
- _info: duckdb_function_info,
- source: *mut duckdb_aggregate_state,
- result: duckdb_vector,
- count: idx_t,
- offset: idx_t,
-) {
+quack_rs::aggregate_finalize_callback!(wc_finalize, |_info, source, result, count, offset| {
let mut writer = unsafe { VectorWriter::new(result) };
-
for i in 0..count as usize {
let state_ptr = unsafe { *source.add(i) };
match unsafe { FfiState::::with_state(state_ptr) } {
Some(st) => unsafe { writer.write_i64(offset as usize + i, st.count) },
- None => unsafe { writer.set_null(offset as usize + i) },
+ None => unsafe { writer.set_null(offset as usize + i) },
}
}
-}
+});
```
-`offset` is DuckDB's output row offset — always use `offset as usize + i`, not just `i`.
+`offset` is DuckDB's output row offset: always use `offset as usize + i`, not just `i`.
---
-## Part 2 — Scalar function: `first_word`
+## Part 2: scalar function `first_word`
-A scalar function processes one data chunk and returns one output value per row.
-The callback receives the full chunk and an output vector (not per-row state pointers).
+A scalar function processes one data chunk and returns one output value per row. For a
+function of one to three primitive or string arguments, the closure builders are the
+whole job: you write the per-row logic, and the builder derives the SQL types from the
+closure's signature, skips NULL rows and writes NULL for them, and runs the closure under
+`catch_unwind`.
-### Key rule: always propagate NULL
+```rust
+# use quack_rs::prelude::*;
+# fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") }
+# fn demo() -> Result<(), ExtensionError> {
+let first_word_builder = ScalarFunctionBuilder::map1_str("first_word", |s: &str| {
+ first_word(s).to_owned()
+})?;
+# let _ = first_word_builder;
+# Ok(())
+# }
+```
+
+where `first_word` is the pure logic:
+
+```rust
+fn first_word(s: &str) -> &str {
+ s.split_whitespace().next().unwrap_or("")
+}
+```
-If the input row is NULL, write NULL to output — never read from an invalid row.
+### When the closure builders do not fit
+
+A function over `STRUCT` or `LIST` arguments, with variable arity, or with bind-time
+state needs the raw chunk. Write the callback with `scalar_callback!`, which gives it the
+same guard. It is then your job to propagate NULL: if the input row is NULL, write NULL to
+the output and never read from an invalid row.
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
# use quack_rs::prelude::*;
# fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") }
-unsafe extern "C" fn first_word_scalar(
- _info: duckdb_function_info,
- input: duckdb_data_chunk,
- output: duckdb_vector,
-) {
+quack_rs::scalar_callback!(first_word_raw, |_info, input, output| {
let reader = unsafe { VectorReader::new(input, 0) };
let mut writer = unsafe { VectorWriter::new(output) };
- let row_count = reader.row_count();
-
- for row in 0..row_count {
+ for row in 0..reader.row_count() {
if !unsafe { reader.is_valid(row) } {
- unsafe { writer.set_null(row) }; // NULL in → NULL out
+ unsafe { writer.set_null(row) }; // NULL in, NULL out
continue;
}
let s = unsafe { reader.read_str(row) };
unsafe { writer.write_varchar(row, first_word(s)) };
}
-}
+});
```
-The pure logic:
+The raw builder takes the callback together with the signature you declare for it:
```rust
-pub fn first_word(s: &str) -> &str {
- s.split_whitespace().next().unwrap_or("")
+# use libduckdb_sys::duckdb_connection;
+# use quack_rs::prelude::*;
+# quack_rs::scalar_callback!(first_word_raw, |_info, _input, _output| {});
+# fn register(con: duckdb_connection) -> Result<(), ExtensionError> {
+// SAFETY: `con` is the live connection DuckDB passed to the entry point.
+unsafe {
+ ScalarFunctionBuilder::new("first_word_raw")
+ .param(TypeId::Varchar)
+ .returns(TypeId::Varchar)
+ .function(first_word_raw)
+ .register(con)?;
}
+# Ok(())
+# }
```
-Note: `set_null` internally calls `duckdb_vector_ensure_validity_writable` before writing
-the null flag — this is required by DuckDB and handled for you by `VectorWriter`.
+`set_null` calls `duckdb_vector_ensure_validity_writable` before writing the NULL flag,
+which DuckDB requires; `VectorWriter` does it for you.
+
+Keep arithmetic in either style checked: in a release build `x * 2` wraps on overflow
+without any error, which is why [Scalar Functions](../functions/scalar.md) and the Quick
+Start use `checked_mul`.
---
-## Part 3 — Registration
+## Part 3: registration
-The snippets below register through the raw builders and `entry_point!`, which take a
-`duckdb_connection`. `hello-ext` itself uses the equivalent `entry_point_v2!`, whose closure
-receives a `&Connection` and registers each builder through the `Registrar` trait
-(`con.register_aggregate(...)`, `con.register_scalar(...)`); see
-[The Entry Point](../concepts/entry-point.md).
+The aggregate builder takes the three callbacks and `ffi_state`; the closure builder
+already holds its callback. `register` is `unsafe` because it calls the C API on a
+connection that must be live, and the entry point is where DuckDB hands you one:
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
+# use libduckdb_sys::duckdb_connection;
# use quack_rs::prelude::*;
# #[derive(Default, Debug)] struct WordCountState { count: i64 }
# impl AggregateState for WordCountState {}
# fn count_words(s: &str) -> i64 { s.split_whitespace().count() as i64 }
# fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") }
-# unsafe extern "C" fn wc_update(_info: duckdb_function_info, input: duckdb_data_chunk,
-# states: *mut duckdb_aggregate_state) {
+# quack_rs::aggregate_update_callback!(wc_update, |_info, input, states| {
# let reader = unsafe { VectorReader::new(input, 0) };
# for row in 0..reader.row_count() {
# if !unsafe { reader.is_valid(row) } { continue; }
@@ -282,17 +314,15 @@ receives a `&Connection` and registers each builder through the `Registrar` trai
# st.count += words;
# }
# }
-# }
-# unsafe extern "C" fn wc_combine(_info: duckdb_function_info, source: *mut duckdb_aggregate_state,
-# target: *mut duckdb_aggregate_state, count: idx_t) {
+# });
+# quack_rs::aggregate_combine_callback!(wc_combine, |_info, source, target, count| {
# for i in 0..count as usize {
# let src = unsafe { FfiState::::with_state(*source.add(i)) };
# let tgt = unsafe { FfiState::::with_state_mut(*target.add(i)) };
# if let (Some(s), Some(t)) = (src, tgt) { t.count += s.count; }
# }
-# }
-# unsafe extern "C" fn wc_finalize(_info: duckdb_function_info, source: *mut duckdb_aggregate_state,
-# result: duckdb_vector, count: idx_t, offset: idx_t) {
+# });
+# quack_rs::aggregate_finalize_callback!(wc_finalize, |_info, source, result, count, offset| {
# let mut writer = unsafe { VectorWriter::new(result) };
# for i in 0..count as usize {
# match unsafe { FfiState::::with_state(*source.add(i)) } {
@@ -300,16 +330,7 @@ receives a `&Connection` and registers each builder through the `Registrar` trai
# None => unsafe { writer.set_null(offset as usize + i) },
# }
# }
-# }
-# unsafe extern "C" fn first_word_scalar(_info: duckdb_function_info, input: duckdb_data_chunk,
-# output: duckdb_vector) {
-# let reader = unsafe { VectorReader::new(input, 0) };
-# let mut writer = unsafe { VectorWriter::new(output) };
-# for row in 0..reader.row_count() {
-# if !unsafe { reader.is_valid(row) } { unsafe { writer.set_null(row) }; continue; }
-# unsafe { writer.write_varchar(row, first_word(reader.read_str(row))) };
-# }
-# }
+# });
# fn live_connection() -> libduckdb_sys::duckdb_connection {
# std::mem::forget(quack_rs::testing::InMemoryDb::open().unwrap());
# let (mut db, mut con) = (std::ptr::null_mut(), std::ptr::null_mut());
@@ -326,7 +347,12 @@ receives a `&Connection` and registers each builder through the `Registrar` trai
# let reader = unsafe { chunk.reader(0) };
# unsafe { reader.is_valid(0).then(|| reader.read_i64(0)) }
# }
-unsafe fn register(con: libduckdb_sys::duckdb_connection) -> Result<(), ExtensionError> {
+fn register(con: duckdb_connection) -> Result<(), ExtensionError> {
+ let first_word_builder = ScalarFunctionBuilder::map1_str("first_word", |s: &str| {
+ first_word(s).to_owned()
+ })?;
+
+ // SAFETY: `con` is the live connection DuckDB passed to the entry point.
unsafe {
AggregateFunctionBuilder::new("word_count")
.param(TypeId::Varchar)
@@ -337,36 +363,39 @@ unsafe fn register(con: libduckdb_sys::duckdb_connection) -> Result<(), Extensio
.finalize(wc_finalize)
.register(con)?;
- ScalarFunctionBuilder::new("first_word")
- .param(TypeId::Varchar)
- .returns(TypeId::Varchar)
- .function(first_word_scalar)
- .register(con)?;
+ first_word_builder.register(con)?;
}
Ok(())
}
# let con = live_connection();
-# unsafe { register(con) }.unwrap();
+# register(con).unwrap();
# assert_eq!(query_i64(con, "SELECT word_count(s) FROM (VALUES ('hello world'), (NULL), ('one two three')) t(s)"), Some(5));
# assert_eq!(query_i64(con, "SELECT length(first_word(' quack rs'))::BIGINT"), Some(5));
-# assert_eq!(query_i64(con, "SELECT count(*) FROM (SELECT first_word(NULL) AS w) WHERE w IS NULL"), Some(1));
+# assert_eq!(query_i64(con, "SELECT count(*) FROM (SELECT first_word(x) AS w FROM (VALUES (NULL::VARCHAR)) t(x)) WHERE w IS NULL"), Some(1));
# // Many groups, so DuckDB runs combine on partial states.
# assert_eq!(query_i64(con, "SELECT sum(c)::BIGINT FROM (SELECT word_count('a b') AS c FROM range(100000) GROUP BY range % 997)"), Some(200000));
+# // A running window frame is not one of the shapes that crash before DuckDB 2.0.
+# assert_eq!(query_i64(con, "SELECT max(w) FROM (SELECT word_count(s) OVER (ORDER BY i) AS w FROM (SELECT 'a b c' AS s, i FROM range(5000) t(i)))"), Some(15000));
```
-Both builders call the DuckDB C API internally. `register` returns `Err` if DuckDB reports
-a failure — this propagates to the entry point and is surfaced to the user.
+`register` returns `Err` if DuckDB reports a failure, and the entry point turns that into
+a `LOAD` error the user sees.
+
+When you test NULL handling, read the NULL from a column, as
+`SELECT first_word(x) FROM (VALUES (NULL::VARCHAR)) t(x)` does, not from a literal:
+DuckDB folds a literal `NULL` argument into the plan before it ever calls the function,
+so `SELECT first_word(NULL)` passes even for a callback that gets NULL wrong
+([Pitfall L8](../reference/pitfalls.md#l8-default_null_handling-does-not-propagate-nulls-for-scalar-functions)).
---
-## Part 4 — Entry point
+## Part 4: entry point
```rust
-# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection,
-# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t};
+# use libduckdb_sys::duckdb_connection;
# use quack_rs::prelude::*;
-# unsafe fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) }
-quack_rs::entry_point!(hello_ext_init_c_api, |con| unsafe { register(con) });
+# fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) }
+quack_rs::entry_point!(word_tools_init_c_api, |con| register(con));
```
This one line expands to the equivalent of:
@@ -374,9 +403,9 @@ This one line expands to the equivalent of:
```rust
# use libduckdb_sys::{duckdb_connection, duckdb_extension_access, duckdb_extension_info};
# use quack_rs::error::ExtensionError;
-# unsafe fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) }
+# fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) }
#[unsafe(no_mangle)]
-pub unsafe extern "C" fn hello_ext_init_c_api(
+pub unsafe extern "C" fn word_tools_init_c_api(
info: duckdb_extension_info,
access: *const duckdb_extension_access,
) -> bool {
@@ -384,15 +413,73 @@ pub unsafe extern "C" fn hello_ext_init_c_api(
quack_rs::entry_point::init_extension_with_policy(
info, access, quack_rs::DUCKDB_API_VERSION,
quack_rs::abi::AbiPolicy::Strict,
- |con| unsafe { register(con) },
+ |con| register(con),
)
}
}
```
-Pass the **full symbol name** — `hello_ext_init_c_api` here. DuckDB looks up this exact
+Pass the **full symbol name**, `word_tools_init_c_api` here: DuckDB looks up this exact
symbol when loading the extension. See [The Entry Point](../concepts/entry-point.md) for
-the full initialization sequence.
+the full initialization sequence. Put the pieces from Parts 1 to 4 in `src/lib.rs`, after
+`use libduckdb_sys::duckdb_connection;` and `use quack_rs::prelude::*;`.
+
+---
+
+## Build, stamp and load
+
+```bash
+cargo build --release
+
+# DuckDB only loads files ending in `.duckdb_extension` that carry its
+# 512-byte metadata footer; a bare `.so` is refused. Append the footer:
+cargo install quack-rs --bin append_metadata # needs Rust 1.88 or newer
+append_metadata target/release/libword_tools.so word_tools.duckdb_extension \
+ --abi-type C_STRUCT --extension-version v0.1.0 \
+ --duckdb-version v1.2.0 --platform linux_amd64
+
+# An unsigned local build needs -unsigned (it must be a startup flag).
+duckdb -unsigned
+```
+
+Then in the DuckDB CLI:
+
+```sql
+LOAD './word_tools.duckdb_extension';
+
+-- Aggregate: total words across all rows
+SELECT word_count(sentence) AS words FROM (
+ VALUES ('hello world'), ('one two three'), (NULL)
+) t(sentence);
+
+-- Scalar: first word of each row
+SELECT first_word(sentence) AS first FROM (
+ VALUES ('hello world'), (' padded '), (''), (NULL)
+) t(sentence);
+```
+
+```text
+┌───────┐
+│ words │
+│ int64 │
+├───────┤
+│ 5 │
+└───────┘
+┌─────────┐
+│ first │
+│ varchar │
+├─────────┤
+│ hello │
+│ padded │
+│ │
+│ NULL │
+└─────────┘
+```
+
+The 5 is 2 + 3 words; the NULL row contributes none. The empty string has no first word,
+so its result is the empty string (the blank row), and the NULL row gives NULL. Use the
+library file name and platform string for your system as the
+[Quick Start](quick-start.md) lists them.
---
@@ -449,10 +536,7 @@ fn word_count_combine() {
}
```
-Run all tests with:
-
-```bash
-cargo test --manifest-path examples/hello-ext/Cargo.toml
-```
-
-See the [Testing Guide](../testing.md) for the full test strategy.
+Run all tests with `cargo test`. The harness cannot exercise the DuckDB side of the
+callbacks (NULL handling, the window shapes above, the state layout); for that, load the
+built extension into a real DuckDB, or use the `bundled-test` features. See the
+[Testing Guide](../testing.md) for the full test strategy.
diff --git a/book/src/getting-started/installation.md b/book/src/getting-started/installation.md
index 317fcf90..36b13b15 100644
--- a/book/src/getting-started/installation.md
+++ b/book/src/getting-started/installation.md
@@ -10,36 +10,83 @@ Add the following to your extension's `Cargo.toml`:
```toml
[dependencies]
-quack-rs = "0.19"
-libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] }
+quack-rs = "0.20"
+
+[dependencies.libduckdb-sys]
+version = ">=1.4.4, <1.20100"
+default-features = false
+features = ["loadable-extension"]
```
-> **Why `>=1.4.4, <2`?**
-> DuckDB 1.4.4 through 2.0.0 load extensions built for C API version `v1.2.0` (1.5.6 and
-> 2.0.0 declare `v1.5.6` and accept every earlier one), so `quack-rs` supports them all with
-> a single bounded range. Releases before 1.4.4 are not supported. The range covers the
-> bindings of every supported release, 1.4.4 through 1.20000.x. The `<2` upper bound does not exclude the DuckDB
-> 2.0 bindings: `libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as
-> `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0`, which the range
-> admits. The 1.20000.x releases need Rust 1.88. On Rust 1.86 or 1.87, Cargo's default
-> (edition 2021, resolver 2) ignores `rust-version` and selects them anyway, so the build
-> fails: set `rust-version = "1.86"` and `resolver = "3"` in your `[package]` (as the
+Copy this block rather than running `cargo add libduckdb-sys`: `cargo add` writes the
+newest version, with the default features on and without `loadable-extension`.
+
+> **Why `>=1.4.4, <1.20100`?**
+> DuckDB 1.4.4 through 2.0.x load extensions built for C API version `v1.2.0` (1.5.6 and
+> 2.0.x declare `v1.5.6` and accept every earlier one), so `quack-rs` supports them all with
+> a single bounded range (an extension that uses a `duckdb-1-5*` feature is narrower: see
+> [ABI Compatibility](../concepts/abi.md#unknown-duckdb-versions)). Releases before 1.4.4 are
+> not supported. `libduckdb-sys` numbers
+> DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so the range runs
+> from the DuckDB 1.4.4 bindings (`1.10404.x`) through the DuckDB 2.0 bindings
+> (`1.20000.x`) and stops before DuckDB 2.1 (`1.20100.0`).
+>
+> **The upper bound.** quack-rs, and every extension that uses its macros, names the C API
+> through the crate root: `libduckdb_sys::duckdb_connection`,
+> `libduckdb_sys::duckdb_register_scalar_function`, and so on. In the 1.20000 bindings
+> those root names exist only as a compatibility re-export of the crate's `v1` module,
+> marked `// TODO: Remove in the future` upstream. If a later release removes it, quack-rs
+> and your extension stop compiling. So quack-rs widens the range one DuckDB minor at a time,
+> after running its tests against the new bindings and the new engine.
+>
+> **When DuckDB 2.1 bindings (`1.20100.x`) are published**, nothing breaks: Cargo keeps
+> resolving `1.20000.x`. An extension built that way keeps the default stamp (`C_STRUCT`,
+> `v1.2.0`), which DuckDB's development branch, the 2.1 line, still loads as 2.0 does (its
+> C API v1 loader was unchanged from v2.0.0 to `main` when this was checked; a release of 2.1
+> could still change that). If you ask for a newer `libduckdb-sys` yourself, Cargo
+> reports a conflict with this range. There is nothing to configure on your side (a
+> `[patch]` entry does not change a version requirement): wait for the quack-rs release that
+> widens the range; its changelog entry says so.
+>
+> **Rust 1.88.** The 1.20000.x releases need Rust 1.88. On Rust 1.86 or 1.87, Cargo's
+> default (edition 2021, resolver 2) ignores `rust-version` and selects them anyway, so the
+> build fails: set `rust-version = "1.86"` and `resolver = "3"` in your `[package]` (as the
> [Quick Start](quick-start.md) does), or resolve with
> `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`. Either keeps the build on
> `1.10506.0`, the newest release that builds there.
> See [Extension Anatomy](../concepts/anatomy.md#version-support).
+>
+> **`default-features = false`** is optional. With the 1.20000 bindings it leaves out the
+> HTTP, TLS and download tooling their default features compile: `cargo tree` lists 65 crates
+> for a bare `libduckdb-sys` dependency with the defaults and 30 without them. With the
+> 1.10506 bindings the difference is two crates, because that tooling is not optional
+> there. Write it in your own manifest: a `features = [...]` list elsewhere in the graph
+> turns the defaults back on. `cargo test` with `bundled-test` or `bundled-test-prebuilt` does
+> so too, because the `duckdb` crate asks for them; a plain `cargo build --release` of the
+> extension does not.
+
+### Build requirements
+
+| What you build | What it needs |
+|----------------|---------------|
+| The extension (`cargo build --release`) | Rust 1.86 or newer. A C compiler may be needed: the 1.10506 bindings, and the 1.20000 bindings with their default features, have a build dependency (`ring`, through the download tooling) that compiles C. `default-features = false` with the 1.20000 bindings removes it. |
+| `append_metadata` (`cargo install quack-rs --bin append_metadata`) | Rust 1.88 or newer, because `cargo install` resolves the newest `libduckdb-sys`. |
+| Tests with `bundled-test` | A C++ toolchain; DuckDB compiles from source, about 5 to 10 minutes cold. |
+| Tests with `bundled-test-prebuilt` | A prebuilt libduckdb (`DUCKDB_DOWNLOAD_LIB=1` or `DUCKDB_LIB_DIR`) and a C compiler for the small shim that fills the dispatch table (a C++ compiler when the library comes from a DuckDB 1.4 release zip). |
+| The community-extension flow (`make configure release test`, [scaffold](scaffold.md)) | Python 3 with `venv`, `pip` and network access to PyPI and GitHub, `git` (with a user identity set, for the first commit), and `make`. |
Upgrading an extension from quack-rs 0.16? 0.18 has breaking changes; see
-[Upgrading from 0.16 to 0.18](upgrading-0-18.md) for what to change.
+[Upgrading from 0.16 to 0.18](upgrading-0-18.md) for what to change. From 0.18 or 0.19,
+see [Upgrading from 0.18 to 0.20](upgrading-0-20.md).
---
## Cargo features
The default build uses only the C API's stable prefix and works with every DuckDB
-release in the `>=1.4.4, <2` range. Each feature below unlocks more API and raises the
-`libduckdb-sys` version Cargo must resolve. The floors are the ones the comments in
-quack-rs's `Cargo.toml` state.
+release in the `>=1.4.4, <1.20100` range. Each feature below unlocks more API and raises
+the `libduckdb-sys` version Cargo must resolve. The floors are the ones in the feature
+documentation in quack-rs's `Cargo.toml`.
| Feature | Requires | Enables |
|---------|----------|---------|
@@ -103,8 +150,13 @@ as a SQL error, or wrap its body in `quack_rs::callback::catch_ffi_panic` and re
behaviour on Rust ≥ 1.81 — the runtime aborts the process — but that is exactly the
outcome the guards exist to prevent.)
-`validate_release_profile` rejects `panic = "abort"`, and the scaffold generator emits
-`panic = "unwind"`.
+`validate_release_profile` rejects `panic = "abort"`, the scaffold generator emits
+`panic = "unwind"`, and `entry_point!` and `entry_point_v2!` stop the build with an error
+that names the fix when the extension is compiled under `panic = "abort"` (a profile
+setting, or `-C panic=abort` in `RUSTFLAGS`). An entry point you write by hand around
+`init_extension` has no such check. Under `unwind` a few failures still abort the process
+whatever quack-rs does: a panic while another panic unwinds, a panic hook that itself
+panics, and running out of memory.
---
@@ -112,11 +164,11 @@ outcome the guards exist to prevent.)
quack-rs requires **Rust ≥ 1.86.0**.
-1.86.0 is a ceiling as much as a floor while DuckDB 1.x is built. The
+While DuckDB 1.x is built, 1.86.0 is a ceiling as much as a floor. The
community-extension build workflow (`_extension_distribution.yml` in
`duckdb/extension-ci-tools`) pins Rust **1.86.0** for its WebAssembly jobs on the
-`v1.4-andium` and `v1.5-variegata` branches, so an extension — and therefore
-quack-rs — must build on it; CI's `msrv-vs-duckdb-ci` job re-derives that pin
+`v1.4-andium` and `v1.5-variegata` branches, so an extension, and therefore
+quack-rs, must build on it; CI's `msrv-vs-duckdb-ci` job re-derives that pin
and fails if the MSRV rises above it. (The 2.0 branch, `v2.0-cyanoptera`, pins
1.97.1 for Windows and WebAssembly.) The `msrv` job checks the crate with
`cargo +1.86.0 check`, and the benchmark dev-dependency (`criterion`) needs
@@ -145,9 +197,9 @@ features as a dev-dependency:
```toml
[dev-dependencies]
# Compiles DuckDB from C++ source: no setup, slow cold build.
-quack-rs = { version = "0.19", features = ["bundled-test"] }
+quack-rs = { version = "0.20", features = ["bundled-test"] }
# ...or link a prebuilt libduckdb instead (set DUCKDB_DOWNLOAD_LIB=1 or DUCKDB_LIB_DIR):
-# quack-rs = { version = "0.19", features = ["bundled-test-prebuilt"] }
+# quack-rs = { version = "0.20", features = ["bundled-test-prebuilt"] }
```
Either one initialises the `loadable-extension` dispatch table from the linked DuckDB, so
diff --git a/book/src/getting-started/quick-start.md b/book/src/getting-started/quick-start.md
index 6696137d..66f6e723 100644
--- a/book/src/getting-started/quick-start.md
+++ b/book/src/getting-started/quick-start.md
@@ -21,25 +21,35 @@ In your extension's `Cargo.toml`:
name = "my_extension"
version = "0.1.0"
edition = "2021"
-rust-version = "1.86" # with resolver 3, Cargo picks dependency versions that build on it
+# With resolver 3, Cargo picks dependency versions that build on this.
+rust-version = "1.86"
resolver = "3"
[dependencies]
-quack-rs = "0.19"
-libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] }
+quack-rs = "0.20"
+
+[dependencies.libduckdb-sys]
+version = ">=1.4.4, <1.20100"
+default-features = false
+features = ["loadable-extension"]
[lib]
-name = "my_extension" # must match your extension name — see Pitfall P1
+# Must match your extension name: see Pitfall P1.
+name = "my_extension"
crate-type = ["cdylib", "rlib"]
[profile.release]
-panic = "unwind" # required — quack-rs's panic guards need unwinding (see Installation)
+# Required: quack-rs's panic guards need unwinding (see Installation).
+panic = "unwind"
lto = true
opt-level = 3
codegen-units = 1
strip = true
```
+The `libduckdb-sys` range and why it has an upper bound are explained on the
+[Installation](installation.md#adding-quack-rs-to-an-existing-extension) page.
+
> **Starting from scratch?** The [scaffold generator](scaffold.md) generates a complete
> community extension project, including this `Cargo.toml`.
@@ -88,8 +98,9 @@ cargo build --release
# DuckDB refuses to LOAD a bare .so: append the metadata footer first.
# quack-rs ships the `append_metadata` binary for this step. Installing it needs
-# Rust 1.88 or newer (it resolves the newest libduckdb-sys); use
-# `cargo +stable install` if this project pins an older toolchain.
+# Rust 1.88 or newer, because `cargo install` resolves the newest libduckdb-sys,
+# which requires it; use `cargo +stable install` if this project pins an older
+# toolchain.
cargo install quack-rs --bin append_metadata
append_metadata target/release/libmy_extension.so my_extension.duckdb_extension \
--abi-type C_STRUCT --extension-version v0.1.0 \
diff --git a/book/src/getting-started/scaffold.md b/book/src/getting-started/scaffold.md
index 3fed89d6..86a5f4d6 100644
--- a/book/src/getting-started/scaffold.md
+++ b/book/src/getting-started/scaffold.md
@@ -1,8 +1,10 @@
# Project Scaffold
`quack_rs::scaffold::generate_scaffold` generates every file of a new DuckDB community
-extension project in Rust — `Cargo.toml`, `Makefile`, CI workflow, `description.yml`, an
-example function and its tests — from a single function call.
+extension project in Rust — `Cargo.toml`, `Makefile`, CI workflows, `description.yml`, an
+example function and its tests, a README, the license text and a `cargo-deny` configuration
+— from a single function call, and the `quack_scaffold` command does the same from the
+command line.
---
@@ -12,10 +14,15 @@ example function and its tests — from a single function call.
my_extension/
├── Cargo.toml # cdylib + rlib crate, MSRV, dependencies, release profile
├── Makefile # delegates to cargo + extension-ci-tools
+├── README.md # build, test and try instructions, and what to submit
+├── LICENSE # the license text (see "The generated project")
+├── deny.toml # cargo-deny configuration
├── src/
-│ ├── lib.rs # entry point, example function, unit test
+│ ├── lib.rs # entry point, an example scalar function, unit test
│ └── wasm_lib.rs # WASM staticlib shim
├── description.yml # community extension metadata
+├── docs/
+│ └── function_descriptions.csv # the function descriptions the community site shows
├── test/
│ └── sql/
│ └── my_extension.test # SQLLogicTest for the example function
@@ -72,19 +79,53 @@ fn main() {
}
```
-The repository's `scaffold_to_dir` example does the same from the command
-line:
+`generate_scaffold_with_warnings` returns a `Scaffold`: the same files, plus warnings about
+values that were accepted but deserve a second look (a license outside quack-rs's shortlist,
+an excluded platform that is opt-in, every platform excluded, an unstable-ABI project the
+community build cannot ship as generated). `generate_scaffold` is the same call without the
+warnings.
+
+### The `quack_scaffold` command
```bash
-cargo run --example scaffold_to_dir -- my_extension # writes ./my_extension
+cargo install quack-rs --bin quack_scaffold # once (0.20.0 or later)
+quack_scaffold my_extension \
+ --description "What the extension does" \
+ --maintainer "Your Name" \
+ --github-repo yourorg/duckdb-my-extension # writes ./my_extension
+
# An extension using the duckdb-1-5* features: C_STRUCT, C API v1.5.6
-cargo run --example scaffold_to_dir -- my_extension ../my_extension --c-api v1.5.6
+quack_scaffold my_extension --description "..." --maintainer "..." \
+ --github-repo yourorg/duckdb-my-extension --target-duckdb-version v1.5.6
```
-`--c-api v1.5.6` sets `target_duckdb_version` to `v1.5.6`: the binary loads into
-DuckDB 1.5.6 and 2.0.0. `--unstable ` (`use_unstable_c_api`) pins the
-binary to one DuckDB 1.x release instead, such as `--unstable v1.5.5`; DuckDB 2.0
-cannot load such a binary, and the scaffold refuses a 2.x release there.
+Every `ScaffoldConfig` field is a flag; `--description`, `--maintainer` and `--github-repo`
+are required, and each value can also be written `--flag=VALUE`:
+
+| Flag | Field | Default |
+|------|-------|---------|
+| `--version` | `version` | `0.1.0` |
+| `--license` | `license` | `MIT` |
+| `--excluded-platforms` | `excluded_platforms`, separated by `;` or `,` (an empty value excludes none) | Wasm and MinGW |
+| `--target-duckdb-version` | `target_duckdb_version`: the C API version (`v1.2.0`, or `v1.5.6` for the `duckdb-1-5*` features), or with `--unstable-c-api` the exact DuckDB 1.x release | `v1.2.0` |
+| `--unstable-c-api` | `use_unstable_c_api` | off |
+| `--git-ref` | `git_ref` | the placeholder |
+| `--dir` | where to write the project | `./` |
+
+The command refuses to overwrite a file it generates unless `--force` is given, and writes
+nothing when it refuses; files it does not generate are left alone, and it never writes
+through a symbolic link inside the project. Warnings go to stderr. When it is done it prints
+the next steps: the `git submodule add` command for extension-ci-tools, the build and test
+commands, and the exact `append_metadata` invocation for stamping a build without `make`.
+(Older releases ship no such command: use the program above in a project that already
+depends on `quack-rs`.)
+
+`--target-duckdb-version v1.5.6` sets `target_duckdb_version` to the C API version `v1.5.6`:
+the binary loads into DuckDB 1.5.6 and 2.0.0, and the generated `Cargo.toml` pins the
+bindings to `~1.10506.0` and enables the `duckdb-1-5*` feature it describes in a comment.
+`--unstable-c-api` with `--target-duckdb-version v1.5.5` pins the binary to one DuckDB 1.x
+release instead; DuckDB 2.0 cannot load such a binary, and the scaffold refuses a 2.x release
+there.
---
@@ -94,11 +135,11 @@ cannot load such a binary, and the scaffold refuses a 2.x release there.
|-------|------|-------------|
| `name` | `String` | Extension name — must match `[lib] name` in Cargo.toml and `description.yml` |
| `description` | `String` | Description for `description.yml` and the `//!` docs of `src/lib.rs`. Quoted and escaped, so `:`, `#` and quotes are fine; must not be empty, padded with whitespace, or contain control characters other than newline and tab |
-| `version` | `String` | The extension's version, also the Cargo package version — so it must be semver without a leading `v` (`validate_semver`): `0.1.0`, not `v0.1.0`, `1.0` or a date stamp |
-| `license` | `String` | SPDX license identifier or expression (e.g., `"MIT"`, `"MIT OR Apache-2.0"`) |
+| `version` | `String` | The extension's version, also the Cargo package version — so it must be semver without a leading `v` (`validate_semver`): `0.1.0`, not `v0.1.0`, `1.0` or a date stamp. A component above `u64::MAX` is refused |
+| `license` | `String` | SPDX license identifier or expression (e.g., `"MIT"`, `"MIT OR Apache-2.0"`). An identifier outside quack-rs's shortlist (`BUSL-1.1`, `CC0-1.0`) is accepted with a warning; the wrong case of a listed one (`mit`), or a character SPDX does not allow, is an error. Line breaks and runs of whitespace are normalised to single spaces |
| `maintainer` | `String` | Your name or org, listed in `description.yml` — one non-empty line |
| `github_repo` | `String` | `"owner/repo"`, in the characters GitHub allows |
-| `excluded_platforms` | `Vec` | Platforms the community build skips, written to `description.yml` and the distribution workflow alike. Defaults to `DEFAULT_EXCLUDED_PLATFORMS` (Wasm and MinGW) |
+| `excluded_platforms` | `Vec` | Platforms the community build skips, written to `description.yml` and the distribution workflow alike. Defaults to `DEFAULT_EXCLUDED_PLATFORMS` (Wasm and MinGW). An entry that excludes nothing is refused: the community build compares exact names, so a platform group such as `wasm` (the error lists its platforms), a retired platform or a name it does not build is an error |
| `git_ref` | `String` | `repo.ref` — **a commit hash** (or tag), not a branch. Defaults to `REF_PLACEHOLDER` so it cannot be submitted unset |
| `target_duckdb_version` | `String` | Written as `TARGET_DUCKDB_VERSION` in the Makefile. Without `use_unstable_c_api`, the C API version: `DUCKDB_API_VERSION` (`v1.2.0`, the default) or `DUCKDB_API_VERSION_1_5_6` (`v1.5.6`) for an extension that enables `duckdb-1-5` / `duckdb-1-5-3` / `duckdb-1-5-4`; `v1.5.6` pins `libduckdb-sys` to `~1.10506.0` and loads into DuckDB 1.5.6 and 2.0.0. With `use_unstable_c_api`, an exact DuckDB 1.x release such as `v1.5.5` |
| `use_unstable_c_api` | `bool` | Stamps `C_STRUCT_UNSTABLE`, pinning the binary to the one DuckDB 1.x release in `target_duckdb_version`. DuckDB 2.0 cannot load such a binary, and 2.x releases are refused. Defaults to `false` |
@@ -114,13 +155,21 @@ Extension names must satisfy all of:
Use vendor-prefixed names to avoid collisions: `myorg_analytics`, not `analytics`.
-The scaffold generator checks the first two rules before generating any files and returns
-an error if the name breaks one. Uniqueness is yours to check.
+The scaffold generator checks the first two rules before generating any files, and
+refuses the names DuckDB itself reserves (aliases such as `s3` and `md`, and the core
+extensions: see [Names DuckDB reserves](../publishing.md#names-duckdb-reserves)). It
+returns an error if the name breaks one. Uniqueness among community extensions is yours
+to check.
---
## After scaffolding
+You need `git` with a user name and email configured (the first commit fails without),
+`make`, Python 3 with the `venv` module, and network access to PyPI and GitHub:
+`make configure` creates a virtual environment, installs the `duckdb` package from PyPI,
+and installs the SQLLogicTest runner from a pinned commit on GitHub.
+
```bash
cd my_extension
git init
@@ -139,6 +188,11 @@ commit for every DuckDB release; without a committed lock file, each rebuild
resolves its dependencies afresh. (`make release` would write the file too, but
after `git add -A`, so it would stay untracked.)
+**Leave `CARGO_TARGET_DIR` unset for `make`.** extension-ci-tools' `rust.Makefile` copies
+the library from `./target/release/`; with `CARGO_TARGET_DIR` set, Cargo builds elsewhere
+and `make release` stops with `FileNotFoundError: './target/release/lib.so'`
+(seen with a scaffolded project).
+
`git submodule add` is not optional in a new project: the scaffold writes
`.gitmodules`, but `git submodule update --init` does nothing until the
submodule has been added once (Pitfall P4). The generated `Makefile` stops with
@@ -151,15 +205,16 @@ community build uses the branch for the DuckDB release it targets, so keep the
submodule, `.gitmodules` and the workflow on the same one.
**Commit before `make configure`.** `make configure` writes the extension
-version into `configure/extension_version.txt` from git — the tag on `HEAD`,
-otherwise the short commit hash. With no commit, extension-ci-tools records
-git's error message instead, which overflows the 32-byte version field of the
-metadata footer, and DuckDB then refuses the file with
-`Unknown ABI type for extension`. The generated `Makefile` guards against this
-by using the Cargo package version (`v0.1.0`) while there is no commit, and
-`make EXTENSION_VERSION=v0.1.0 configure` sets one explicitly. If you configured
-before committing, run `make clean_all configure` after the first commit so the
-version comes from git.
+version into `configure/extension_version.txt`, which the metadata footer
+holds in a 32-byte field. extension-ci-tools writes every tag on `HEAD`, one
+per line, or the short commit hash, or, with no commit, git's error message;
+several tags, a tag over 32 bytes or that message overflow the field, and DuckDB
+then refuses the file with `Unknown ABI type for extension`. The generated
+`Makefile` chooses the value itself: the first tag on `HEAD` if it fits and holds
+only `A-Za-z0-9._+-`, otherwise the short commit hash, and before the first commit
+the Cargo package version (`v0.1.0`). `make EXTENSION_VERSION=v0.1.0 configure`
+sets one explicitly. The value is cached, so run `make clean_all configure` after
+committing or tagging.
The project declares `rust-version` (quack-rs's MSRV, 1.86.0; for DuckDB 2.0,
extension-ci-tools pins Rust 1.97.1 on Windows and WebAssembly and uses the current
@@ -174,6 +229,45 @@ places, and push to GitHub — CI runs automatically.
---
+## The generated project
+
+- **The example function** is a native scalar function, `my_extension_hello(name)`, written
+ with `ScalarFunctionBuilder::map1_str` as a one-line closure: a NULL argument gives NULL
+ without calling it, and a panic fails the query instead of aborting. (Before 0.20.0 it was
+ a `CREATE MACRO`; a macro is written into the user's database, so `LOAD` failed on a
+ read-only one.)
+- **The SQLLogicTest** carries the header lines the upstream tests have (`# name:`,
+ `# description:`, `# group:`), checks that the function is missing before `require`, and
+ reads its NULL from a column, because a literal `NULL` is constant-folded before the
+ function is called.
+- **`make test` fails when the runner skips a test file or runs none.** The SQLLogicTest
+ runner reports a file whose `require` is not met as `SKIPPED` and exits 0, which is what a
+ rename that leaves the extension name, the `[lib]` name and the test file out of step looks
+ like, so `make test` would pass having tested nothing.
+- **`Cargo.toml`** names `libduckdb-sys` as `>=1.4.4, <1.20100` with
+ `default-features = false`, and, for a project stamped C API `v1.5.6` (or an unstable
+ DuckDB 1.5 target), enables the `duckdb-1-5*` feature its comments describe on `quack-rs`.
+- **License files.** `LICENSE` holds the text for a single license among MIT, Apache-2.0,
+ BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, MIT-0 and Unlicense. An expression gets one
+ `LICENSE-` file per license (`MIT OR Apache-2.0` gives `LICENSE-MIT` and
+ `LICENSE-APACHE`); any other license gets a `LICENSE.TODO` that names the text to add.
+- **`deny.toml`** configures `cargo deny check`. It ignores RUSTSEC-2026-0367, -0368 and -0369,
+ each with a reason: they concern the `zip` crate before 9, which is only a build-script
+ dependency of `libduckdb-sys` up to 1.10506.0, the release Cargo resolves for
+ `rust-version = 1.86`. It also allows the licenses of the dependency graph and the
+ project's own.
+- **`extension-ci.yml`** sets `permissions: contents: read`, checks out with
+ `persist-credentials: false`, and pins `Swatinem/rust-cache` and `dtolnay/rust-toolchain`
+ to commit hashes.
+- **`README.md`** says how to build, test and try the extension, that a stamped extension must
+ not be `strip`ped, that pull requests get a reduced CI run, and what to submit.
+- **`docs/function_descriptions.csv`** has a row for the example function: the extension page on
+ duckdb.org takes function descriptions and examples only from this file, which belongs in
+ the community-extensions repository next to `description.yml` when you submit (see
+ [Community Extensions](../publishing.md)).
+
+---
+
## Excluded platforms
The community build compiles every extension for nine platforms with
diff --git a/book/src/getting-started/upgrading-0-20.md b/book/src/getting-started/upgrading-0-20.md
new file mode 100644
index 00000000..158421be
--- /dev/null
+++ b/book/src/getting-started/upgrading-0-20.md
@@ -0,0 +1,144 @@
+# Upgrading from 0.18 to 0.20
+
+This page covers the two steps after [0.16 to 0.18](upgrading-0-18.md): 0.18 to 0.19.0,
+and 0.19.0 to 0.20.0. The [changelog](../reference/changelog.md) is the full record; this
+page says what to look for. Neither 0.19.0 nor 0.20.0 removes a public item that a correct
+extension used. 0.20.0 has two changes that can stop a build (see "Changes that stop a
+build"); several calls that used to succeed now return an error, which can fail a `LOAD`
+that worked before, and a few results that were wrong are now right.
+
+To upgrade, set the version, build, run your SQL tests, and then load the built extension
+into every DuckDB release you support:
+
+```toml
+[dependencies]
+quack-rs = "0.20"
+
+[dependencies.libduckdb-sys]
+version = ">=1.4.4, <1.20100"
+default-features = false
+features = ["loadable-extension"]
+```
+
+## From 0.18 to 0.19.0
+
+You need 0.19.0 to support DuckDB 2.0.0. On 2.0.0, quack-rs 0.18.0 refuses to load a
+build that uses the `duckdb-1-5*` features, every casting `Value` getter (`as_i64`,
+`as_str`, …) returns `None`, and `bundled-test` no longer builds once Cargo resolves
+`libduckdb-sys` 1.20000.x (a fresh resolve on Rust 1.88 or newer).
+
+### An extension that uses the `duckdb-1-5*` features
+
+DuckDB 2.0 cannot load a `C_STRUCT_UNSTABLE` binary through the entry point quack-rs
+generates, so the stamp 0.18 advised no longer works there. Rebuild against the 1.5.6
+bindings and stamp the binary with the C API version instead:
+
+| In 0.18 | From 0.19 |
+|---------|-----------|
+| `append_metadata … --abi-type C_STRUCT_UNSTABLE --duckdb-version v1.5.5` (the exact release) | `append_metadata … --abi-type C_STRUCT --duckdb-version v1.5.6`, built against `libduckdb-sys = "~1.10506.0"`. DuckDB 1.5.6 and 2.0.0 load it; earlier releases refuse it, and the default `Strict` guard refuses a release quack-rs has not listed (2.0.1, for now). |
+| `TARGET_DUCKDB_VERSION=v1.5.5` and `USE_UNSTABLE_C_API=1` in the `Makefile` | `TARGET_DUCKDB_VERSION=v1.5.6` and no `USE_UNSTABLE_C_API`, with the `extension-ci-tools` submodule on the `v2.0-cyanoptera` branch |
+| `ScaffoldConfig { use_unstable_c_api: true, target_duckdb_version: "v1.5.5", … }` | `target_duckdb_version: quack_rs::DUCKDB_API_VERSION_1_5_6.to_string()` and `use_unstable_c_api: false` |
+
+An extension that uses only the default features needs no change: its `C_STRUCT`
+`v1.2.0` binary loads into every DuckDB from 1.4.4 through 2.0.x. See
+[ABI Compatibility](../concepts/abi.md).
+
+### Behaviour changes
+
+| Change | What to check |
+|--------|---------------|
+| `parse_description_yml` and `validate_description_yml_str`: a `repo.ref` that is not a commit hash is a warning; quoted values are no longer trimmed (`name: " my_ext"` is refused, as the community build refuses it); `validate_rust_extension` requires an exact `rust` token in `requires_toolchains` (`"python3; rust"` installs no Rust toolchain). | A `description.yml` that passed before can now fail. Run the validator on it. |
+| `generate_scaffold` requires `version` to be semver (`2025120401`, `1.0` and `v1.0.0` are refused), and `ScaffoldConfig::default()` excludes `wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw`. | Pass an explicit `excluded_platforms` if you build those platforms. A project generated earlier is not rewritten; diff a fresh one against yours. |
+| `validate::platform`: `osx_amd64` is an opt-in platform, as in `extension-ci-tools` for DuckDB 2.0 (DuckDB 2.1 drops it). | Listing it in `excluded_platforms` has no effect; list it in `opt_in_platforms` to have it built. |
+| On DuckDB 2.0.0, an appender error caused by an open stream reports `ResourceInUse` instead of `Invalid`. | Code that matched on the error category. |
+| Registering several table functions through the entry point's `Connection` lists the catalog once per `LOAD`, not once per function. | Nothing: faster. A table function registered on the raw connection after that listing is not seen by the check (see [Table Functions](../functions/table-functions.md#registering-a-name-twice)). |
+| `DuckDbSecretInfo`'s `Debug` prints the length of `secret_string`, not its text. | A test that compared the `Debug` output. |
+| The published crate no longer contains `scripts/`, `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md` or `GOVERNANCE.md`, and `mutants` is a dev-dependency only. | Nothing, unless a build script of yours read those files from the registry source. |
+
+## From 0.19.0 to 0.20.0
+
+### Dependency requirement
+
+0.20.0 requires `libduckdb-sys` `>=1.4.4, <1.20100`; 0.18.0 and 0.19.0 required
+`>=1.4.4, <2`, which also admits bindings for DuckDB releases that do not exist yet.
+The new bound stops before the DuckDB 2.1 bindings because `libduckdb-sys` keeps the C API
+names at its crate root only through a re-export that upstream marks for removal; the
+[Installation](installation.md#adding-quack-rs-to-an-existing-extension) page has the
+reasoning and what happens when 2.1 bindings are published.
+
+| Change | What to do |
+|--------|------------|
+| A project that asks for `libduckdb-sys` 1.20100 or newer no longer resolves with 0.20.0. | Remove that requirement. Cargo picks 1.20000.x, which is the DuckDB 2.0 bindings. |
+| You stay on 0.18.0 or 0.19.0 and want the same ceiling. | Add `libduckdb-sys = ">=1.4.4, <1.20100"` to your own `Cargo.toml`; those releases cannot be changed after publication. |
+| `default-features = false` on `libduckdb-sys` is now the documented form (the scaffold, the README and the book write it). | Optional. With the 1.20000 bindings it leaves out the HTTP and TLS download tooling their default features build. Set it in your own manifest too: a dependency elsewhere in the graph that leaves the defaults on turns them back on. |
+
+### Changes that stop a build
+
+| Change | What to do |
+|--------|------------|
+| `AbiCheck`, `AbiPolicy`, `ConfigOptionScope` and `TlsVersion` are `#[non_exhaustive]`, so DuckDB releases that add a variant do not break your `match`. An exhaustive `match` on one of them outside quack-rs no longer compiles (error E0004). Constructing the variants, comparing them and `TlsVersion`'s ordering are unchanged. | Add a `_` arm. |
+| `ScalarBindInfo` (`duckdb-1-5`) is no longer `RefUnwindSafe`: it remembers the bind data it stored so that a second `set` can free the first. It matters only if you hold a `&ScalarBindInfo` across your own `catch_unwind`. quack-rs's own guards are unaffected. | Wrap it in `AssertUnwindSafe`. |
+| `entry_point!` and `entry_point_v2!` fail to compile when the extension is built with `panic = "abort"` (a profile setting, or `-C panic=abort` in `RUSTFLAGS`), because every panic guard in quack-rs is `catch_unwind`, which does nothing there. The error names the fix. An entry point written by hand around `init_extension` has no such check. | Set `panic = "unwind"` under `[profile.release]` (the scaffold already does). |
+
+### Behaviour changes
+
+| Change | What to do |
+|--------|------------|
+| Registering an aggregate, or an overload in a set, **with no parameter** is an error. `SELECT my_count() FROM t` counted 100 rows on DuckDB 1.5.6 and 0 on 2.0.0. If your registration function propagates the error with `?`, `LOAD` fails. | Give the aggregate one parameter and put a macro in front for the zero-argument spelling: `SqlMacro::scalar("my_count", &[], "my_count_1(1::BIGINT)")`. See [Known Limitations](../reference/known-limitations.md#aggregates-that-take-no-argument-count-no-rows-on-duckdb-20). |
+| `TypeId::Variant` (feature `duckdb-1-5-3`) is refused in a function signature: `LogicalType::try_new`, and so every builder's `param`, `returns` and result-column slot, returns an error, and `LogicalType::new` panics. It registered before, and the first query that reached it crashed DuckDB 2.0.0 or invalidated the database on 1.5.x. | Declare the parameter as `TypeId::Any`, which accepts a `VARIANT`. |
+| `ListVector::get_entry` and `MapVector::get_entry` return stale bytes for a NULL row (lengths around 7·10¹³ were seen on 2.0.0). | Read with `get_entry_checked`, which returns `None` for a NULL row and for an entry outside the child, or test `is_valid(row)` first. See [Complex Types](../data/complex-types.md). |
+| `append_metadata` writes a sibling file, created exclusively under a random name (it was a guessable `