diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index ee50a440..437d9608 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -44,7 +44,10 @@ features = ["duckdb-1-5-4"] # Thin re-export modules and test-only code produce unproductive mutants. exclude_globs = [ "src/prelude.rs", # pure re-exports - "src/callback.rs", # macro-generated FFI wrappers (untestable without DuckDB runtime) + # src/callback.rs is not excluded: the macros in it are invisible to + # cargo-mutants, but catch_ffi_panic, panic_message, panic_c_message and + # message_to_c_string are plain functions. `cargo mutants --no-config + # --file src/callback.rs --cargo-arg=--lib`: 5 mutants, 4 caught, 1 unviable. "src/chunk_writer.rs", # Drop calls FFI; row tracking tested via unit tests "src/vector/struct_writer.rs", # thin delegation to VectorWriter (requires DuckDB runtime) "src/vector/struct_reader.rs", # thin delegation to VectorReader (requires DuckDB runtime) @@ -225,6 +228,15 @@ exclude_re = [ # with the one asked for, which needs duckdb_create_logical_type; with the # comparison inverted, every type creation fails end to end. "src/types/logical_type/construct.rs:.*replace != with == in LogicalType::try_new", + # construct.rs: LogicalType::refuse_where_the_engine_lacks asks the engine for + # its version (abi::engine_version reads the dispatch table), so a --lib run + # with no engine cannot reach the TIMESTAMPTZ_NS refusal. Both its mutants + # (the body replaced with Ok(()), and `==` to `!=`) were run by hand against + # libduckdb 1.5.6: tests/ffi_roundtrip/timestamp_tz_ns.rs + # `the_type_id_is_reported_for_a_column_and_creatable_only_where_it_exists` + # passes unmutated and fails with each. The pure comparison it uses, + # predates_timestamp_tz_ns, stays in the gate. + "src/types/logical_type/construct.rs:.*LogicalType::refuse_where_the_engine_lacks", # error_data.rs: a non-null ErrorData only comes from DuckDB, and # is_null / has_error / message / Drop / check_valid_utf8 are only told # apart from their mutants with one (duckdb_error_data_* / diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..90120276 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: MIT +# https://editorconfig.org -- the whitespace rules the repository already follows +# (`.gitattributes` fixes LF; no tracked .rs/.toml/.md/.yml/.py/.sh file lacks a +# final newline or has trailing whitespace in Rust), written down so an editor +# without rustfmt on save does not drift. +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{yml,yaml}] +indent_size = 2 + +[*.sh] +indent_size = 2 + +# Markdown keeps trailing double-space line breaks and mixed list indents. +[*.md] +trim_trailing_whitespace = false +indent_size = unset + +# Makefiles (and the Makefile text generated by the scaffold) need tabs. +[{Makefile,*.mk}] +indent_style = tab + +[*.{png,bin}] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset +indent_size = unset + +# Fuzz seeds must reach the fuzzer byte for byte (.gitattributes: binary). +[fuzz/seeds/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset +indent_size = unset diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e7c2c343..63c37cd0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -36,6 +36,14 @@ updates: - "minor" # ── Cargo dependencies for the hello-ext example ───────────────────────────── + # No duckdb / libduckdb-sys ignore here, on purpose. The root lockfile is held + # on the DuckDB 1.5.6 bindings for the 1.86 MSRV (above); the example is not + # bound by that and tracks the newest bindings (DuckDB 2.0's, 1.20000.x, as + # of 0.20.0). It is the one place CI builds them into a real cdylib and loads + # it into every pinned DuckDB (example-check, symbol-check, extension-load), + # so a bindings PR here is the first signal of a new DuckDB release, tested + # by those jobs on the PR itself. Its committed lockfile needs Rust 1.88 + # (examples/hello-ext/README.md says so). - package-ecosystem: cargo directory: "/examples/hello-ext" schedule: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6fd9c04d..022162b7 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -18,6 +18,7 @@ ### Code quality - [ ] `cargo test --all-targets` passes, and `cargo test --doc --features duckdb-1-5-4` (doctests are not in `--all-targets`) - [ ] The end-to-end tests pass against a real libduckdb (CONTRIBUTING, gate 2c) when the change touches `unsafe`, vectors, or function registration +- [ ] `cargo test --doc --manifest-path book/doctest/Cargo.toml` passes when the change touches `book/src/` or `README.md` (every Rust block there is compiled; gate 2c) - [ ] `cargo clippy --all-targets -- -D warnings` passes - [ ] `cargo fmt` applied (no diff) - [ ] `cargo doc --no-deps` builds without warnings diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e2bca71e..f7e2be6e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,15 +10,15 @@ This directory contains all GitHub Actions workflows for the quack-rs project. | Workflow | File | Trigger | Purpose | |----------|------|---------|---------| | **CI** | `ci.yml` | Push to `main`/`claude/**`, PRs to `main`, weekly schedule, manual, called by `release.yml` | Every quality gate (table below), summarised by the `ci-ok` job | -| **Release** | `release.yml` | Semver tags (`vX.Y.Z`, and `vX.Y.Z-pre` prereleases) | Full release pipeline: validate the tag, run all of `ci.yml`, package with SLSA attestation, publish to crates.io (approval required), check crates.io serves the attested `.crate`, GitHub release, versioned-book deploy | +| **Release** | `release.yml` | Semver tags (`vX.Y.Z`, and `vX.Y.Z-pre` prereleases) | Full release pipeline: validate the tag and the release metadata, run all of `ci.yml`, package with SLSA attestation, publish to crates.io (approval required; the `.crate` is re-packaged and compared with the attested hash before the upload), check crates.io serves the attested `.crate`, GitHub release, versioned-book deploy | | **Documentation** | `docs.yml` | Push to `main`, manual, dispatched by `release.yml` after a release | Build the versioned book (latest release at `/`, each release line at `/vX.Y/`, `main` at `/dev/`) and deploy to GitHub Pages | | **Coverage** | `coverage.yml` | Push to `main`, PRs to `main` | Generate LCOV coverage report and upload to Codecov | | **Mutation Testing** | `mutants.yml` | Manual dispatch, PRs to `main` | Verify tests detect code changes via cargo-mutants | -| **Benchmarks** | `benchmarks.yml` | Push to `main` (bench/src changes), manual | Run criterion benchmarks and archive reports | +| **Benchmarks** | `benchmarks.yml` | Push to `main` (changes under `benches/` or `src/`, or to `Cargo.toml` / `Cargo.lock`), manual | Run criterion benchmarks and archive reports | ## Quality gates (enforced by CI) -Every job in `ci.yml` must be green before merging a PR, except the three marked +Every job in `ci.yml` must be green before merging a PR, except the four marked informational. This table is generated from `ci.yml`; the hand-written list it replaced had drifted to fewer than half the jobs. @@ -35,6 +35,7 @@ replaced had drifted to fewer than half the jobs. | `wasm` | WASM (wasm32-unknown-emscripten) | blocking | | `clippy` | Clippy | blocking | | `clippy-beta` | Clippy (beta, informational) | informational | +| `latest-stable` | Latest stable Rust (informational) | informational | | `fmt` | Format | blocking | | `doc` | Documentation | blocking | | `msrv` | MSRV (1.86.0) | blocking | @@ -56,12 +57,14 @@ replaced had drifted to fewer than half the jobs. | `publish-dry-run` | Publish dry-run | blocking | | `security` | Security (cargo-deny) | blocking | | `osv-scan` | Security (OSV / GHSA) | blocking | +| `hygiene` | Hygiene (typos, taplo, machete) | blocking | | `nightly` | Nightly (informational) | informational | | `miri` | Miri (undefined behaviour) | blocking | | `leak-check` | LeakSanitizer (RAII wrappers vs a real DuckDB) | blocking | | `asan` | AddressSanitizer | blocking | | `semver` | Public API (semver-checks) | blocking | | `fuzz` | Fuzz (smoke) | blocking | +| `workflow-lint` | Workflow lint (actionlint) | blocking | | `ci-ok` | CI passed | blocking (aggregate) | **Branch protection should require exactly one check: `CI passed`** (the @@ -69,7 +72,10 @@ replaced had drifted to fewer than half the jobs. it fails when any of them fails or is cancelled. `scripts/check-ci-aggregate.py` (run by the `doc` job) fails when a blocking job is missing from its `needs`, so a new job cannot drop out of the required -set unnoticed. `release.yml` runs this whole workflow as its CI gate +set unnoticed; it also fails when a job turns informational without being +named in its `INFORMATIONAL` set, when a blocking job (or `ci-ok` itself) has a +step-level `continue-on-error`, and when the table above disagrees with +`ci.yml`. `release.yml` runs this whole workflow as its CI gate (`workflow_call`), so a release passes the same checks. Regenerate after adding or renaming a job: @@ -108,14 +114,41 @@ Neither `yaml.safe_load` nor a JSON-Schema validator catches it, because both treat the `run:` block as an opaque string. `scripts/check-workflow-expressions.py` runs in the `doc` job and does. -`actionlint` also catches it and catches much more besides. It is not wired in -here only because it ships as a Go binary from GitHub releases and cannot be -checksum-pinned the way `osv-scanner` is in `ci.yml`. If a pinned, verifiable -install becomes available, adding it would be a strict improvement. +`actionlint` also catches it and catches much more besides; the `workflow-lint` +job runs it from the release tarball, pinned by version and sha256 the way +`osv-scanner` is (the release publishes `actionlint__checksums.txt`). + +## Pinning, and the checks that hold it + +Everything CI downloads is pinned, and a script in the `doc` job keeps each +kind of pin consistent: + +| What | Pinned by | Checked by | +|---|---|---| +| Actions | full commit SHA, version in the trailing comment (`# v2.9.3`, or `# master` for `dtolnay/rust-toolchain`, which has no release tags) | `scripts/check-action-pins.py`: one commit per action across all workflows, and the same commits in the workflows the scaffold generates (`src/scaffold/templates_ci.rs`), which Dependabot does not update | +| Runner images | `ubuntu-24.04`, `macos-26`, `windows-2025`, never `*-latest` (GitHub moves those labels; each pinned label was the `-latest` image on 2026-10-30) | `scripts/check-action-pins.py`: a floating label in any workflow fails | +| Rust | an exact release in `rust-toolchain.toml`, repeated in every `toolchain:` input | `scripts/check-toolchain-pin.py`: `stable` / `beta` only in informational jobs; MSRV and nightly where they are the point | +| Release binaries (osv-scanner, actionlint, typos, taplo, cargo-machete) | version and SHA-256 in the job | the job itself (`sha256sum -c`) | +| Cargo-installed tools (cargo-deny, cargo-semver-checks, cargo-fuzz, mdBook, ...) | `cargo install --locked --version` | — | + +The five jobs that compare a checked-in table with upstream (`abi-table`, which +has two such steps, `platform-table`, `spdx-list`, `autoload-entries`, +`msrv-vs-duckdb-ci`) run each script through `scripts/upstream-guard.sh`: +exit 1 (drift) fails; exit 2 +("could not check": upstream unreachable, an HTTP 5xx or 429) is a warning on a +pull request or a push and a failure on a release tag and on the weekly +schedule, where nothing may be skipped; a missing script fails. The scripts +return 1, not 2, for an HTTP 4xx and for a file that no longer has the +expected shape: upstream moved something, and a job that degraded to "check +skipped" for good would never say so. `upstream-guard.sh --self-test` runs in +the `doc` job. ## Adding a new workflow 1. Create a new `.yml` file in this directory. -2. Pin all third-party actions to their full commit SHA (not a tag). -3. Add SPDX license header. -4. Update this README. +2. Pin all third-party actions to their full commit SHA (not a tag), at the + commit the other workflows use (`scripts/check-action-pins.py`). +3. Install Rust with `toolchain:` set to the release `rust-toolchain.toml` + pins (`scripts/check-toolchain-pin.py`). +4. Add SPDX license header. +5. Update this README. diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 13830e7d..aadda49a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -32,17 +32,17 @@ permissions: jobs: bench: name: Run Benchmarks - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Build benchmarks run: cargo bench --no-run diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 453d28e4..1b2aea8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,21 +5,37 @@ # # Jobs: check, test, test-bundled, test-bundled-prebuilt, test-older-engines, # test-duckdb-2-0, test-bundled-duckdb-2-0, test-duckdb-1-5, -# wasm, clippy, clippy-beta, fmt, doc, msrv, dependency-floor, -# bench-compile, example-check, scaffold-compile, symbol-check, -# abi-table, platform-table, spdx-list, autoload-entries, book, -# msrv-vs-duckdb-ci, extension-load, scaffold-e2e, abi-guard, -# abi-guard-layout, publish-dry-run, security, osv-scan, nightly, miri, -# leak-check, asan, semver, fuzz, ci-ok (the aggregate required check). +# wasm, clippy, clippy-beta, latest-stable, fmt, doc, msrv, +# dependency-floor, bench-compile, example-check, scaffold-compile, +# symbol-check, abi-table, platform-table, spdx-list, autoload-entries, +# book, msrv-vs-duckdb-ci, extension-load, scaffold-e2e, abi-guard, +# abi-guard-layout, publish-dry-run, security, osv-scan, hygiene, nightly, +# miri, leak-check, asan, semver, fuzz, workflow-lint, ci-ok (the +# aggregate required check). # -# `dtolnay/rust-toolchain` is pinned, in every workflow, to e2a55d2 -- a commit -# on the action's `master` branch (2026-10-08; it moved action inputs into -# environment variables, upstream #187). The action's per-toolchain -# branches (`stable`, `nightly`, ...) are regenerated, so a commit taken from -# one of them stops being reachable from any ref and can be garbage-collected; -# the previous pin, 631a55b, was such a commit. On `master` the `toolchain:` -# input is required, so every use states it. The pin fixes the action's code, -# not the Rust version: `toolchain: stable` still installs the current stable. +# `dtolnay/rust-toolchain` is pinned, in every workflow, to one commit on the +# action's `master` branch (commented `# master`: the branch publishes no +# release tags, and Dependabot moves the pin to master's newest commit; a date +# in the comment went stale with the first such bump). Every pin since e2a55d2 +# (2026-10-08) passes the action's inputs through environment variables +# (upstream #187). The action's per-toolchain branches (`stable`, `nightly`, +# ...) are regenerated, so a commit taken from one of them stops being +# reachable from any ref and can be garbage-collected; the pin before e2a55d2, +# 631a55b, was such a commit. On `master` the `toolchain:` input is required, +# so every use states it. The pin fixes the action's code, not the Rust version. +# scripts/check-action-pins.py (the `doc` job) keeps every workflow, and the +# workflows the scaffold generates, on the same commit of each action. +# +# The Rust version is pinned separately: rust-toolchain.toml names an exact +# stable release, and every job that is not about another toolchain installs +# that one (`toolchain: 1.99.1`). With `-D warnings` and clippy's pedantic and +# nursery groups, a floating `stable` let each Rust release (every six weeks) +# change rustc, rustfmt and clippy under every job at once, with no change +# here. scripts/check-toolchain-pin.py (the `doc` job) fails when a workflow +# names another toolchain outside the informational jobs. Those see the next +# releases first: `latest-stable` (the current stable), `clippy-beta`, +# `nightly`. The pin is moved by a pull request (CONTRIBUTING.md, "The Rust +# toolchain"). name: CI @@ -60,6 +76,10 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + # On a tag (a release, which reaches this file through workflow_call) the + # upstream-comparison guards (scripts/upstream-guard.sh) fail instead of + # skipping when upstream cannot be reached. + QUACK_STRICT_UPSTREAM: ${{ github.ref_type == 'tag' && '1' || '' }} RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" # Harden against transient crates.io/CDN download flakes — notably @@ -72,16 +92,16 @@ env: jobs: check: name: Check - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - run: cargo check --all-targets test: @@ -91,16 +111,29 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-24.04, macos-26, windows-2025] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + # The scaffold's tests cross-check the files it generates with PyYAML (the + # parser the community build reads description.yml with), make and git, + # and skip when one is missing; `cargo test` hides the note, so a runner + # without PyYAML kept them green while checking nothing. On Linux the tools + # are installed and a missing one fails the test (QUACK_REQUIRE_TOOLS=1); + # macOS and Windows runners keep the skip. + - name: Install the tools the scaffold's tests cross-check with + if: runner.os == 'Linux' + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends python3-yaml make git - run: cargo test --all-targets + env: + QUACK_REQUIRE_TOOLS: ${{ runner.os == 'Linux' && '1' || '' }} # Every other job builds tests in debug or with the repository's release # profile (one codegen unit, fat LTO). A user's crate usually has Cargo's # default release profile: 16 codegen units, no fat LTO, where rustc gives @@ -127,15 +160,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-24.04, macos-26, windows-2025] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # --no-fail-fast: cargo otherwise stops at the first failing test # binary, hiding failures in the ones after it. - run: cargo test --all-targets --features bundled-test --no-fail-fast @@ -163,16 +196,16 @@ jobs: # Linux only — the pre-built release archive is published per-platform. test-bundled-prebuilt: name: Test bundled-test-prebuilt (prebuilt libduckdb) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Download pre-built libduckdb # The version is DERIVED from Cargo.lock, not written here. It used to # be hard-coded with a comment asking the next person to keep it in @@ -233,7 +266,7 @@ jobs: # until the suite ran against the oldest release of each range, which # this job now does on every run. name: Test against DuckDB ${{ matrix.engine }} (${{ matrix.features }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: fail-fast: false @@ -264,10 +297,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Pin the bindings to the engine and download its library # The test shim refuses a library whose duckdb_ext_api_v1 size differs # from the bindings', so both must be the same release. @@ -304,16 +337,16 @@ jobs: # bindings and nothing else covers on the 2.0 ones: clippy, rustdoc and the # doctests. (`test-bundled-duckdb-2-0` covers the source-build path.) name: Test against DuckDB v2.0.0 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Pin the DuckDB 2.0 bindings and download libduckdb v2.0.0 run: | set -euo pipefail @@ -347,17 +380,25 @@ jobs: # quack-rs. test-bundled-duckdb-2-0: name: Test bundled-test on the DuckDB 2.0 bindings (informational) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 continue-on-error: true timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # The 1.20000 bindings unpack DuckDB's sources into target/duckdb-sources/, + # outside the directories the cache keeps for a build script; a restored + # target/ then holds the build record without the sources, and the build + # script (run 705, run 711) stops with "neither duckdb_extension.h nor + # duckdb.hpp is there". Only the registry is cached, so this builds DuckDB + # from scratch each time. + cache-targets: "false" - name: Pin the DuckDB 2.0 bindings run: | set -euo pipefail @@ -369,16 +410,16 @@ jobs: test-duckdb-1-5: name: Test duckdb-1-5 / duckdb-1-5-3 / duckdb-1-5-4 features - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - run: cargo check --all-targets --features duckdb-1-5 - run: cargo test --all-targets --features duckdb-1-5 - run: cargo clippy --all-targets --features duckdb-1-5 -- -D warnings @@ -391,17 +432,17 @@ jobs: wasm: name: WASM (wasm32-unknown-emscripten) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 targets: wasm32-unknown-emscripten - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # DuckDB-WASM target. quack-rs's own lib must compile for wasm32; in # loadable-extension mode libduckdb-sys is headers-only, so `cargo check` # needs no emscripten toolchain. `--all-targets` is intentionally avoided — @@ -427,7 +468,10 @@ jobs: - name: Unit tests on wasm32 (node) env: CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER: node - RUSTFLAGS: "-D warnings -C link-arg=-sALLOW_MEMORY_GROWTH=1 -C link-arg=-sMAXIMUM_MEMORY=4GB" + # STACK_SIZE: emscripten's default stack is 64 KB, and the description.yml + # reader's depth test (64 nested levels, debug build) ran out of it with + # "memory access out of bounds". + RUSTFLAGS: "-D warnings -C link-arg=-sALLOW_MEMORY_GROWTH=1 -C link-arg=-sMAXIMUM_MEMORY=4GB -C link-arg=-sSTACK_SIZE=8MB" run: | set -euo pipefail source "$RUNNER_TEMP/emsdk/emsdk_env.sh" @@ -436,17 +480,17 @@ jobs: clippy: name: Clippy - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 components: clippy - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - run: cargo clippy --all-targets -- -D warnings # Informational early-warning for clippy lints graduating to stable. @@ -454,18 +498,18 @@ jobs: # before the lint blocks merges. continue-on-error keeps it non-blocking. clippy-beta: name: Clippy (beta, informational) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 continue-on-error: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: beta components: clippy - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # `+beta` for the same reason as the MSRV job above: without it this ran # stable clippy and duplicated the `clippy` job. - run: cargo +beta clippy --all-targets --features duckdb-1-5-4 -- -D warnings @@ -486,32 +530,64 @@ jobs: LD_LIBRARY_PATH: ${{ runner.temp }}/libduckdb run: cargo +beta clippy --all-targets --features bundled-test-prebuilt,duckdb-1-5-4 -- -D warnings + # Informational: the current stable Rust, whatever rust-toolchain.toml pins. + # Every blocking job builds with the pinned release, so a new stable cannot + # turn them red overnight; this job is where its new lints and warnings show + # up first, on every pull request and on the weekly schedule. When stable is + # newer than the pin and this job is green, move the pin (CONTRIBUTING.md, + # "The Rust toolchain"); when it is red, fix the findings in the same PR. + latest-stable: + name: Latest stable Rust (informational) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + continue-on-error: true + env: + # Over rust-toolchain.toml, for every rustup proxy this job runs, + # rust-cache's own `rustc -vV` (part of its cache key) included. + RUSTUP_TOOLCHAIN: stable + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - name: The stable release this run checks + run: rustc +stable --version + - run: cargo +stable fmt -- --check + - run: cargo +stable clippy --all-targets -- -D warnings + - run: cargo +stable clippy --all-targets --features duckdb-1-5-4 -- -D warnings + - run: cargo +stable doc --no-deps --features duckdb-1-5-4 + - run: cargo +stable test --lib --features duckdb-1-5-4 + fmt: name: Format - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 components: rustfmt - run: cargo fmt -- --check doc: name: Documentation - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # --features duckdb-1-5-4 (implies duckdb-1-5-3, which implies # duckdb-1-5) rather than --all-features, which would compile DuckDB # from source here; `test-bundled` documents --all-features on Linux, @@ -545,24 +621,48 @@ jobs: # `ci-ok` is the one required check; it is only as good as its `needs`. - name: The ci-ok job waits on every blocking job run: scripts/check-ci-aggregate.py + # The six upstream-comparison steps run through this wrapper; a mapping + # that read a missing script, or exit 2 on a release, as a pass would turn + # each of them into a check of nothing. + - name: The upstream-guard wrapper maps exit codes as documented + run: scripts/upstream-guard.sh --self-test + # Cargo.toml's version, every committed Cargo.lock, the CHANGELOG, + # CITATION.cff, SECURITY.md and every `quack-rs = "0.X"` snippet must name + # the same release. + - name: Release metadata agrees with Cargo.toml + run: python3 scripts/check-release-metadata.py + # Every job builds with the toolchain rust-toolchain.toml pins, except the + # informational ones and those about another toolchain (MSRV, nightly). + - name: Workflows use the pinned Rust toolchain + run: python3 scripts/check-toolchain-pin.py + # One commit per action across the workflows, and the same commits in the + # workflows the scaffold generates (src/scaffold/templates_ci.rs), which + # Dependabot does not update. + - name: Action pins agree, here and in the scaffold's templates + run: python3 scripts/check-action-pins.py + # LESSONS.md (long form) and the book's pitfall page (condensed form) are two + # hand-kept copies; they must list the same pitfalls with the same titles. + - name: LESSONS.md and the book describe the same pitfalls + run: python3 scripts/check-pitfalls-parity.py msrv: name: MSRV (1.86.0) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.86.0" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # `+1.86.0` is not optional, for the same reason the `nightly` and `miri` - # jobs below use `+nightly`: `rust-toolchain.toml` pins `channel = - # "stable"`, and a rustup toolchain file overrides the default this action - # sets. A bare `cargo check` here runs stable and tests nothing -- the - # MSRV gate was inert. This matches `cargo +1.86.0 check` in CONTRIBUTING. + # jobs below use `+nightly`: `rust-toolchain.toml` pins the toolchain + # (`channel`), and a rustup toolchain file overrides the default this + # action sets. A bare `cargo check` here runs the pinned release and tests + # nothing -- the MSRV gate was inert. This matches `cargo +1.86.0 check` + # in CONTRIBUTING. # # Against what a 1.86.0 user's Cargo resolves, not the committed # Cargo.lock: that carries the newest compatible releases, and @@ -581,7 +681,7 @@ jobs: - run: cargo +1.86.0 check --features duckdb-1-5-4 dependency-floor: - # Cargo.toml declares `duckdb` / `libduckdb-sys` `>=1.4.4, <2`, but every + # Cargo.toml declares `duckdb` / `libduckdb-sys` `>=1.4.4, <1.20100`, but every # other job builds the newest release Cargo.lock pins. This one resolves the # declared floor and runs the library tests there, so the lower bound is a # tested claim rather than a hope. @@ -593,16 +693,16 @@ jobs: # job above covers 1.86.0 with the locked dependencies; this covers the # dependency floor with a current compiler. name: Dependency floor (duckdb / libduckdb-sys 1.4.4) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Resolve the declared floor run: | set -euo pipefail @@ -613,16 +713,16 @@ jobs: bench-compile: name: Benchmark (compile check) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - run: cargo bench --no-run # The engine-level benchmark extension (benches/engine) is a separate crate; # keep it compiling, and its driver parsing. @@ -636,15 +736,21 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-24.04, macos-26, windows-2025] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # hello-ext is its own Cargo workspace (examples/hello-ext/target); the + # action caches only the root one unless told otherwise. + workspaces: | + . -> target + examples/hello-ext -> target - run: cargo check --manifest-path examples/hello-ext/Cargo.toml --all-targets - name: Clippy (Linux only) if: runner.os == 'Linux' @@ -653,18 +759,26 @@ jobs: scaffold-compile: name: Scaffold (compile check) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + # `--test integration_test` builds the one test binary that holds the test + # (the unfiltered form compiled nine, and eight of them ran nothing). + # `--exact` plus the grep: `cargo test ` exits 0 when the filter + # matches no test, so renaming the test would have left this job green, + # running nothing. The summary line must say exactly one test passed. - name: Verify scaffold output compiles - run: cargo test scaffold_generated_code_compiles -- --nocapture + run: | + set -euo pipefail + cargo test --test integration_test scaffold_generated_code_compiles -- --exact --nocapture | tee /tmp/scaffold-compile.log + grep -qE '^test result: ok\. 1 passed;' /tmp/scaffold-compile.log symbol-check: name: Symbol check (hello-ext · ${{ matrix.os }}) @@ -673,15 +787,21 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-24.04, macos-26] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # hello-ext is its own Cargo workspace (examples/hello-ext/target); the + # action caches only the root one unless told otherwise. + workspaces: | + . -> target + examples/hello-ext -> target - name: Build hello-ext as cdylib run: cargo build --manifest-path examples/hello-ext/Cargo.toml --release - name: Verify extension entry point symbol (Linux) @@ -709,7 +829,7 @@ jobs: abi-table: name: ABI layout table (vs upstream DuckDB headers), and every C API slot wrapped or listed - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -723,28 +843,20 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- a DuckDB release header was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-abi-table.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" - # Every slot of the C extension API is either used under src/ or listed, - # with a reason, in scripts/api-coverage-allowlist.txt, so a slot a DuckDB - # release adds is noticed instead of left for someone to read the header. + run: scripts/upstream-guard.sh scripts/check-abi-table.py + # Every slot of the C extension API is either used by library code under + # src/ or listed, with a reason, in scripts/api-coverage-allowlist.txt, so + # a slot a DuckDB release adds is noticed instead of left for someone to + # read the header. "Used" excludes comments, string literals and + # test-only code; the self-test holds the scanner to that. + - name: The C API surface scanner, on fixed snippets + run: python3 scripts/check-api-coverage.py --self-test - name: Classify the C API surface - run: | - python3 scripts/check-api-coverage.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-api-coverage.py platform-table: name: Platform list (vs upstream distribution matrix) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -759,17 +871,11 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the distribution matrix was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-platform-table.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-platform-table.py spdx-list: name: SPDX shortlist (vs official registry) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -783,17 +889,11 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the SPDX registry was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-spdx-list.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-spdx-list.py autoload-entries: name: Catalog autoload names (vs every supported DuckDB) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -807,17 +907,11 @@ jobs: - name: Verify src/catalog.rs covers every supported release # Exit 2 is the documented "could not check" code (upstream # unreachable) and must not be read as drift. - run: | - python3 scripts/check-autoload-entries.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-autoload-entries.py book: name: Book builds (mdBook) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -826,9 +920,9 @@ jobs: # The versioned-site step below builds each release line from its tag. fetch-depth: 0 fetch-tags: true - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 # docs.yml builds the book only when main is pushed, so a broken page # was found after merge. The version is the one book/mdbook-version pins, # which docs.yml's build script also reads. @@ -856,7 +950,7 @@ jobs: msrv-vs-duckdb-ci: name: MSRV vs DuckDB's extension CI - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -871,17 +965,11 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the pinned-toolchain config was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-msrv-vs-duckdb-ci.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-msrv-vs-duckdb-ci.py extension-load: name: Extension load test (DuckDB ${{ matrix.duckdb }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: fail-fast: false @@ -903,10 +991,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # hello-ext is its own Cargo workspace (examples/hello-ext/target); the + # action caches only the root one unless told otherwise. + workspaces: | + . -> target + examples/hello-ext -> target - name: Build hello-ext as cdylib run: cargo build --manifest-path examples/hello-ext/Cargo.toml --release - name: Build append_metadata @@ -969,7 +1063,7 @@ jobs: } echo "Extension loaded and produced correct results" - name: Every documented hello-ext check - # examples/hello-ext/README.md documents 31 statements and their + # examples/hello-ext/README.md documents 32 statements and their # results; this used to run two of them. The script runs all of # examples/hello-ext/sql_checks.txt and also fails if the README's # statements drift from that fixture. @@ -977,16 +1071,16 @@ jobs: scaffold-e2e: name: Scaffold end-to-end (build, stamp, load, query) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Install DuckDB CLI run: | set -euo pipefail @@ -1142,8 +1236,12 @@ jobs: import pathlib, re, sys d = pathlib.Path("/tmp/scaffolded_capi") p = d / "Cargo.toml" - s, n = re.subn(r'quack-rs = \{ version = "[^"]+" \}', - 'quack-rs = { path = "%s", features = ["duckdb-1-5"] }' % sys.argv[1], p.read_text()) + s = p.read_text() + # A project stamped C API v1.5.6 enables the duckdb-1-5* feature itself + # (the old step added `duckdb-1-5` by hand); only the version becomes a path. + assert 'features = ["duckdb-1-5-4"]' in s, s + s, n = re.subn(r'quack-rs = \{ version = "[^"]+"', + 'quack-rs = { path = "%s"' % sys.argv[1], s) assert n == 1, n p.write_text(s) p = d / "src/lib.rs" @@ -1176,16 +1274,16 @@ jobs: # whatever DuckDB loads it. It never reaches the layout comparison itself -- # `abi-guard-layout` below is the job that exercises LayoutMismatch. name: ABI guard refuses a build whose declared DuckDB version contradicts its bindings - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Install DuckDB CLI run: | set -euo pipefail @@ -1310,16 +1408,16 @@ jobs: # into the DuckDB its bindings came from it must work. The engines are # pinned releases, not `latest`, so the expected verdict never shifts. name: ABI guard refuses a real layout mismatch (and accepts the matching DuckDB) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Build and stamp a duckdb-1-5 demo against the pinned bindings run: | set -euo pipefail @@ -1448,28 +1546,38 @@ jobs: publish-dry-run: name: Publish dry-run - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + # `--locked`, as release.yml's `package` and `publish` jobs run it: without + # it a Cargo.toml version bump that left Cargo.lock behind passed here + # (cargo quietly rewrote the lock file) and failed only at release time. + # No `--allow-dirty`: a clean checkout is what is being packaged. - name: cargo publish --dry-run - run: cargo publish --dry-run --allow-dirty + run: cargo publish --dry-run --locked security: name: Security (cargo-deny) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + # Installed explicitly, like every other job, rather than left to + # rustup's on-demand install of what rust-toolchain.toml names (on by + # default since rustup 1.28.1, but an image can turn it off). + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-targets: false - name: Install cargo-deny @@ -1481,6 +1589,20 @@ jobs: run: cargo install cargo-deny --locked --version "0.20.2" - name: Run cargo-deny check run: cargo deny check + # The other committed graphs (hello-ext, the fuzz crate, the book's + # doctest crate and benches/engine) had their advisories read by the OSV + # job only. deny.toml cannot be applied + # to them (it is strict about unused licenses, and libfuzzer-sys is + # "(MIT OR Apache-2.0) AND NCSA"), so they have their own config. + - name: Run cargo-deny check on the CI crates + run: | + set -euo pipefail + crates=(examples/hello-ext fuzz book/doctest benches/engine) + for m in "${crates[@]}"; do + echo "::group::cargo deny: $m" + cargo deny --manifest-path "$m/Cargo.toml" --config deny.ci-crates.toml check + echo "::endgroup::" + done # OSV / GHSA advisory scan. The cargo-deny job above consults only the RustSec # advisory database; OSV.dev aggregates GHSA *and* RustSec, so GHSA-only @@ -1489,15 +1611,15 @@ jobs: # via the `libduckdb-sys` build-dependency. The binary is version- and checksum-pinned. osv-scan: name: Security (OSV / GHSA) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 - name: Install osv-scanner (pinned, checksum-verified) run: | VER=2.7.0 @@ -1509,47 +1631,100 @@ jobs: chmod +x osv-scanner - name: Scan every committed lockfile against OSV run: | - # All four committed lockfiles. fuzz/Cargo.lock and then + set -euo pipefail + # Every committed lockfile. fuzz/Cargo.lock and then # book/doctest/Cargo.lock were missing: their dependencies are # CI-only, but an unscanned lockfile is an unscanned lockfile. - ./osv-scanner --config=osv-scanner.toml \ - --lockfile=Cargo.lock \ - --lockfile=examples/hello-ext/Cargo.lock \ - --lockfile=fuzz/Cargo.lock \ - --lockfile=book/doctest/Cargo.lock + # benches/engine/Cargo.lock was gitignored until 0.20.0 + # (bench-compile resolved that crate afresh each run). + lockfiles=(Cargo.lock examples/hello-ext/Cargo.lock fuzz/Cargo.lock book/doctest/Cargo.lock benches/engine/Cargo.lock) + args=() + for f in "${lockfiles[@]}"; do args+=("--lockfile=$f"); done + ./osv-scanner --config=osv-scanner.toml "${args[@]}" + + # Build-free hygiene checks: spelling (typos.toml), TOML style (.taplo.toml) + # and unused dependencies (cargo-machete; the manifests list under + # [package.metadata.cargo-machete] what it cannot see: build.rs, a + # [[bench]], generated doctests). Each tool is a release binary pinned by + # version and SHA-256, as in osv-scan. Only cargo-machete's release publishes + # a checksum file (it matches); typos and taplo publish none, so their + # hashes are of the release assets as GitHub served them when pinned. + hygiene: + name: Hygiene (typos, taplo, machete) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # cargo-machete --with-metadata runs `cargo metadata`. + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: 1.99.1 + - name: Install typos, taplo and cargo-machete (pinned, checksum-verified) + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/bin" + fetch() { # + curl -fsSL --retry 5 -o "$3" "$1" + echo "$2 $3" | sha256sum -c - + } + cd "$RUNNER_TEMP" + fetch https://github.com/crate-ci/typos/releases/download/v1.46.0/typos-v1.46.0-x86_64-unknown-linux-musl.tar.gz \ + c62a626df385c96574743fd4974aeb271a41a67d080a0398e36e1a42782f614a typos.tgz + tar -xzf typos.tgz -C bin ./typos + fetch https://github.com/tamasfe/taplo/releases/download/0.10.0/taplo-linux-x86_64.gz \ + 8fe196b894ccf9072f98d4e1013a180306e17d244830b03986ee5e8eabeb6156 taplo.gz + gunzip -c taplo.gz > bin/taplo + chmod +x bin/taplo + fetch https://github.com/bnjbvr/cargo-machete/releases/download/v0.9.2/cargo-machete-v0.9.2-x86_64-unknown-linux-musl.tar.gz \ + 48200087f54c55aabcd4db4af1e25742b49846c02a1b1bfa134711945b35b2e9 machete.tgz + tar -xzf machete.tgz --strip-components=1 -C bin \ + cargo-machete-v0.9.2-x86_64-unknown-linux-musl/cargo-machete + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - run: typos + - run: taplo fmt --check + - run: taplo lint + - run: cargo machete --with-metadata nightly: name: Nightly (informational) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 continue-on-error: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - # `+nightly` is not optional: `rust-toolchain.toml` pins `channel = - # "stable"`, and a rustup toolchain file overrides the default this action - # sets, so a bare `cargo` here runs stable and this job tests nothing new. + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + # `+nightly` is not optional: `rust-toolchain.toml` pins a stable release + # (`channel`), and a rustup toolchain file overrides the default this + # action sets, so a bare `cargo` here runs that release and this job + # tests nothing new. - name: Test (nightly, informational) run: cargo +nightly test --all-targets miri: name: Miri (undefined behaviour) - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: ubuntu-24.04 + # Two sequential runs of the library tests (native, then s390x). On the 0.20.0 + # pull request the native run alone took 41.6 minutes, 18 of them one parser + # test on 100,000-level inputs and 3 more an SPDX test (both shrink under + # Miri now), and the s390x run was cancelled at the 60-minute limit with the + # aggregate check red. The limit is for the two runs together. + timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly components: miri, rust-src - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # This crate is ~40% unsafe by surface area, and the parts that are *not* # FFI -- the duckdb_string_t decoder, the validity bitmap arithmetic, the # UUID bit flip, the interval maths, the secret zeroizer -- are exactly @@ -1575,9 +1750,9 @@ jobs: # what this job is for. `cargo test` still runs the full 256. PROPTEST_CASES: "8" # `+nightly` is not optional -- see the note in the `nightly` job: - # `rust-toolchain.toml` pins stable and overrides this action's default, - # so a bare `cargo miri` resolves to stable, where the `miri` component - # does not exist. + # `rust-toolchain.toml` pins a stable release and overrides this action's + # default, so a bare `cargo miri` resolves to that release, where the + # `miri` component does not exist. # `--features duckdb-1-5-4` matters: without it every `duckdb-1-5*` # module is cfg'd out, including `src/arrow.rs` -- the largest block of # pure-Rust `unsafe` in the crate, and exactly what Miri is here for. @@ -1601,17 +1776,17 @@ jobs: leak-check: name: LeakSanitizer (RAII wrappers vs a real DuckDB) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly components: rust-src - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # Half of quack-rs's value proposition is RAII over DuckDB handles: # LogicalType, Value, QueryResult, OwnedDataChunk, FileHandle, # PreparedStatement, ErrorData, Appender. A missing destructor is not a @@ -1619,7 +1794,10 @@ jobs: # way to catch one is to look at the heap afterwards. # # LeakSanitizer intercepts malloc globally, so it sees DuckDB's C++ - # allocations even though libduckdb itself is not instrumented. + # allocations even though libduckdb itself is not instrumented -- but + # not the vector data buffers and LIST entry arrays the release + # libduckdb takes from its own jemalloc (CONTRIBUTING.md, "What the + # sanitizer and valgrind runs can and cannot see"). - name: Fetch a prebuilt libduckdb # Version derived from Cargo.lock -- see the note in # `test-bundled-prebuilt`. Linking a libduckdb that does not match the @@ -1638,8 +1816,12 @@ jobs: RUSTFLAGS: "-Zsanitizer=leak" # DuckDB's own one-time allocations (extension registry, the default # allocator's arenas, static initialisers) are still reachable at - # exit and are not quack-rs's business. Suppress by library, so a leak - # from a quack-rs frame is still reported. + # exit, which LeakSanitizer does not report, so nothing needs + # suppressing. lsan-suppressions.txt holds no rules on purpose: a + # rule for libduckdb.so is tested against every frame of an + # allocation, and every handle DuckDB allocates has such a frame, so + # it silenced exactly the leaks this job is for (the step below + # leaks a handle on purpose and requires the job to notice). # A two-frame stack is useless for attributing a leak to a missing # destructor; 30 is enough to name the quack-rs frame. LSAN_OPTIONS: "suppressions=lsan-suppressions.txt:print_suppressions=0:malloc_context_size=30" @@ -1647,13 +1829,48 @@ jobs: # `+nightly` is not optional -- see the note in the `nightly` job. cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ --features bundled-test-prebuilt,duckdb-1-5-4 --test ffi_roundtrip + # A leak check is only as good as its ability to fail. tests/lsan_probe.rs + # leaks, on request, a Rust allocation (the control), a LogicalType and a + # Value, and the run must fail with a leak report each time; with nothing + # leaked it must pass. The first version of this job passed over the + # LogicalType and the Value because of a suppression. + - name: A handle leaked on purpose must fail LeakSanitizer + env: + RUSTFLAGS: "-Zsanitizer=leak" + LSAN_OPTIONS: "suppressions=lsan-suppressions.txt:print_suppressions=0:malloc_context_size=30" + run: | + set -u + probe() { + PROBE="$1" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + --features bundled-test-prebuilt,duckdb-1-5-4 --test lsan_probe 2>&1 \ + | tee "$RUNNER_TEMP/probe-$1.log" + return "${PIPESTATUS[0]}" + } + if ! probe nothing; then + echo "::error::the probe failed although it leaked nothing" + exit 1 + fi + for what in rust logical_type value; do + if probe "$what"; then + echo "::error::PROBE=$what leaked and LeakSanitizer did not fail the run" + exit 1 + fi + if ! grep -q 'LeakSanitizer: detected memory leaks' "$RUNNER_TEMP/probe-$what.log"; then + echo "::error::PROBE=$what failed, but not with a LeakSanitizer report" + exit 1 + fi + done # AddressSanitizer. LeakSanitizer above answers "did we forget a destructor"; # ASAN answers "did we write outside an allocation, or use one after it was # freed" -- the class that produced the two heap-corruption defects fixed in # v0.16.0, and the one a crate doing raw pointer arithmetic into DuckDB's # memory is most exposed to. Miri cannot reach it: these paths call foreign - # functions. + # functions. Its blind spot: this links the release libduckdb, which takes + # vector data buffers and LIST entry arrays from its own jemalloc, and ASAN + # does not track those blocks, so an out-of-bounds or stale write into them + # is not reported here; Rust-heap and validity-mask errors are + # (CONTRIBUTING.md, "What the sanitizer and valgrind runs can and cannot see"). # # Blocking since September 2026. It was `continue-on-error` until it had a # green run on main, because libduckdb itself is not instrumented and the @@ -1665,17 +1882,17 @@ jobs: # ASAN subsumes LSAN, so once this is blocking, `leak-check` can fold into it. asan: name: AddressSanitizer - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly components: rust-src - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Fetch a prebuilt libduckdb # Version derived from Cargo.lock -- see `test-bundled-prebuilt`. run: | @@ -1699,17 +1916,17 @@ jobs: semver: name: Public API (semver-checks) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # quack-rs is an SDK: its public API *is* the product, and a downstream # extension that stops compiling on a patch release is a worse outcome # than a missing feature. This compares the working tree against the @@ -1769,7 +1986,7 @@ jobs: fuzz: name: Fuzz (smoke) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # 60 s per target on a push or pull request; 10 minutes per target on the # weekly schedule, which is long enough to grow the corpus meaningfully. timeout-minutes: 90 @@ -1779,10 +1996,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 # The description.yml parser is hand-rolled and reads a file from an # arbitrary GitHub repository; the duckdb_string_t decoder reads 16 bytes # whose meaning depends on a length field inside them. Both are exactly @@ -1867,6 +2084,38 @@ jobs: path: fuzz/corpus key: fuzz-corpus-${{ github.run_id }} + # actionlint: workflow syntax and types, `needs` and `uses` references, the + # expression contexts, and shellcheck on every `run:` block. The + # binary is version- and checksum-pinned, like osv-scanner below. + # scripts/check-workflow-expressions.py predates this job and covers one class + # of the same errors without a download. + workflow-lint: + name: Workflow lint (actionlint) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install actionlint (pinned, checksum-verified) + run: | + set -euo pipefail + VER=1.7.12 + # From the release's actionlint_${VER}_checksums.txt (linux_amd64). + SHA=8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + curl -fsSL --retry 5 -o actionlint.tar.gz \ + "https://github.com/rhysd/actionlint/releases/download/v${VER}/actionlint_${VER}_linux_amd64.tar.gz" + echo "${SHA} actionlint.tar.gz" | sha256sum -c - + tar -xzf actionlint.tar.gz actionlint + - name: actionlint + run: | + set -euo pipefail + # Without shellcheck on PATH actionlint skips every `run:` block + # silently; a lint that checks less than it says is worse than none. + command -v shellcheck >/dev/null || { echo "::error::shellcheck is not installed"; exit 1; } + # SC2129 ("group these redirects") is style, not a defect. + ./actionlint -color -ignore 'SC2129' + # The one check branch protection should require. Every blocking job is in # `needs` (scripts/check-ci-aggregate.py, run by the `doc` job, fails when one # is missing), so adding or renaming a job never silently drops it from the @@ -1907,12 +2156,14 @@ jobs: - publish-dry-run - security - osv-scan + - hygiene - miri - leak-check - asan - semver - fuzz - runs-on: ubuntu-latest + - workflow-lint + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Every blocking job succeeded diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fa080255..2a4828a4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -28,7 +28,7 @@ env: jobs: coverage: name: Test coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # `--all-features` compiles DuckDB from C++; the default limit is 360 minutes. timeout-minutes: 90 steps: @@ -36,12 +36,12 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 components: llvm-tools-preview - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Install cargo-llvm-cov env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 87c9c6c5..05616402 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,7 +35,7 @@ concurrency: jobs: build: name: Build versioned site - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Checkout repository @@ -47,9 +47,9 @@ jobs: fetch-tags: true - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 # main's API reference for /dev/api/, with the docs.rs feature set # (Cargo.toml [package.metadata.docs.rs]). Released versions link to @@ -94,7 +94,7 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build steps: - name: Deploy to GitHub Pages diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 5077b76f..473ef2ce 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -31,16 +31,16 @@ jobs: mutants-full: name: "Mutants (full sweep)" if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 120 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Install cargo-mutants env: @@ -94,21 +94,33 @@ jobs: retention-days: 30 # ── Incremental mutation testing (PRs only) ────────────────────────────── + # + # Sharded: each of the eight jobs below tests every eighth mutant + # (`--shard k/8 --sharding round-robin`), so a pull request that touches a + # hundred source files finishes in about an eighth of the time. One job did all + # of a 102-file, 1,639-mutant pull request (0.20.0) and was stopped by its + # runner after 33 and 55 minutes, long before it was done. `mutants-incremental-gate` + # below is the single check to require: it keeps the name this check always had. mutants-incremental: - name: "Mutation Testing (incremental)" + name: "Mutation Testing (incremental) shard ${{ matrix.shard }}" if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 + strategy: + # A survivor in one shard must not hide the others' results. + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + toolchain: 1.99.1 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - name: Install cargo-mutants env: @@ -152,6 +164,8 @@ jobs: if: steps.changed.outputs.skip == 'false' env: CHANGED_FILES: ${{ steps.changed.outputs.files }} + SHARD: ${{ matrix.shard }} + SHARDS: ${{ strategy.job-total }} run: | echo "::group::cargo-mutants execution log" @@ -203,12 +217,20 @@ jobs: # in `mutants.out/mutants.out/`, where the report step below never # looked -- so every count read 0, the score printed 100%, and the # job passed green no matter how many mutants survived. + # `--shard k/n` with `--sharding round-robin`: mutant i goes to shard + # i % n, which spreads the slow files over all shards. (The default, + # `slice`, hands each shard a consecutive run of mutants in file + # order.) A shard with no mutants -- more shards than mutants, on a + # small pull request -- finds nothing to test and exits 0, which the + # report below accepts for an incremental run. set +e cargo mutants "${ARGS[@]}" \ --features duckdb-1-5-4 \ --cargo-arg=--lib \ --exclude-re 'ErrorData' \ --output . \ + --shard "${SHARD}/${SHARDS}" \ + --sharding round-robin \ --timeout 120 \ --jobs 4 2>&1 | tee /tmp/mutants-run.log # `${PIPESTATUS[0]}`, not `$?`: these steps run under the default @@ -239,8 +261,30 @@ jobs: if: always() && steps.changed.outputs.skip == 'false' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: mutation-report-incremental + name: mutation-report-incremental-${{ matrix.shard }} path: | mutants.out/ /tmp/mutants-run.log retention-days: 14 + + # The one check that stands for the shards. It keeps the name of the single + # job it replaces, "Mutation Testing (incremental)", so a branch-protection + # rule that requires that name still means the same thing. `always()` because + # a failed shard must turn this red rather than skip it, and a skipped + # required check counts as passed. + mutants-incremental-gate: + name: "Mutation Testing (incremental)" + if: always() && github.event_name == 'pull_request' + needs: mutants-incremental + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Every shard passed + env: + RESULT: ${{ needs.mutants-incremental.result }} + run: | + if [ "$RESULT" != success ]; then + echo "::error::the mutation shards finished as '${RESULT}', not 'success': see the shard jobs" + exit 1 + fi + echo "All mutation shards passed." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e71f078..02d430af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,7 @@ env: jobs: validate: name: Validate release tag - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: version: ${{ steps.meta.outputs.version }} @@ -76,7 +76,7 @@ jobs: shell: bash run: | if [[ "$(git cat-file -t "refs/tags/$TAG")" != "tag" ]]; then - echo "::error::Tag '$TAG' is a lightweight tag. Release tags are annotated and signed: git tag -s '$TAG' -m 'Release $TAG' (RELEASING.md Step 5)." + echo "::error::Tag '$TAG' is a lightweight tag. Release tags must be annotated (RELEASING.md Step 5 also signs them; this job does not verify the signature): git tag -s '$TAG' -m 'Release $TAG'." exit 1 fi git fetch --no-tags --quiet origin '+refs/heads/main:refs/remotes/origin/main' @@ -155,6 +155,13 @@ jobs: fi echo "::notice::CHANGELOG.md entry found for v$VERSION" + # Cargo.lock still naming the previous version fails `cargo package + # --locked` in the `package` job, after all of ci.yml has run; stale + # version snippets ship in the .crate. Say so in the first minute. + - name: Release metadata (Cargo.lock, CITATION.cff, SECURITY.md, doc snippets) + shell: bash + run: python3 scripts/check-release-metadata.py + # ── Job 2: Every CI gate, by running ci.yml itself ────────────────────────── ci: name: CI @@ -167,7 +174,7 @@ jobs: package: name: Package and attest needs: [ci] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: contents: read @@ -175,13 +182,14 @@ jobs: attestations: write # write attestation to the repository outputs: crate_file: ${{ steps.pkg.outputs.crate_file }} + rustc: ${{ steps.pkg.outputs.rustc }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: - toolchain: stable + toolchain: 1.99.1 # No cache in this job or in `publish`: a tag-triggered run can restore # caches the default branch saved, and this job holds the attestation # token. Release speed does not matter. @@ -201,6 +209,9 @@ jobs: CRATE_FILE=$(ls target/package/*.crate) CRATE_SIZE=$(wc -c < "$CRATE_FILE") echo "crate_file=$CRATE_FILE" >> "$GITHUB_OUTPUT" + # The `publish` job installs this rustc and refuses to re-package with + # any other: it runs after a manual approval that can take days. + echo "rustc=$(rustc --version | awk '{ print $2 }')" >> "$GITHUB_OUTPUT" printf '::notice::Packaged %s (%s bytes)\n' "$(basename "$CRATE_FILE")" "$CRATE_SIZE" - name: Generate SHA-256 checksums @@ -226,12 +237,14 @@ jobs: target/package/SHA256SUMS retention-days: 90 if-no-files-found: error + # A re-run of the whole workflow uploads under the same name again. + overwrite: true # ── Job 5: Create GitHub release with artifacts & release notes ─────────────── github-release: name: Create GitHub release needs: [validate, package, publish] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: contents: write # create release, upload assets @@ -300,7 +313,7 @@ jobs: run: | TAG="v${VERSION}" - PRE_FLAG="" + PRE_FLAG="--prerelease=false" [[ "$IS_PRE" == "true" ]] && PRE_FLAG="--prerelease" RELEASE_URL="https://github.com/${REPO}/releases/tag/${TAG}" @@ -311,7 +324,11 @@ jobs: if gh release view "$TAG" &>/dev/null; then echo "::warning::Release ${TAG} already exists — updating it." echo "::group::Updating GitHub release ${TAG}" + # --draft=false: a release left as a draft (created by hand, or by an + # earlier failed run) would otherwise stay invisible. $PRE_FLAG sets + # the prerelease state both ways, not only to true. gh release edit "$TAG" \ + --draft=false \ $PRE_FLAG \ --title "${CRATE} ${TAG}" \ --notes-file release_notes.md @@ -360,7 +377,7 @@ jobs: publish: name: Publish to crates.io needs: [validate, package] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 # The 'crates-io' environment requires a manual approval step. # Configure it at: Settings → Environments → crates-io → Required reviewers @@ -374,9 +391,49 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + # The rustc the `package` job attested with: the .crate's bytes depend + # on cargo, and this job runs after an approval that can take days. + # (rust-toolchain.toml, which overrides the default this action sets, + # pins the same release; the next step checks that they agree.) + toolchain: ${{ needs.package.outputs.rustc }} + + # Outside the checkout: `cargo package` would otherwise count the + # downloaded files as uncommitted changes, and include them in the .crate. + - name: Download the attested artifacts + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - toolchain: stable + name: release-artifacts + path: ${{ runner.temp }}/release-artifacts + + # crates.io versions cannot be replaced, and the post-publish check below + # can only report a mismatch. Package here exactly as `cargo publish` will + # (no build script runs: --no-verify) and compare with the attested hash + # first, so a .crate that differs fails before anything is uploaded, with + # no credential in this job yet. + - name: The .crate about to be published is the attested one + env: + VERSION: ${{ needs.validate.outputs.version }} + CRATE: ${{ needs.validate.outputs.crate_name }} + ATTESTED_RUSTC: ${{ needs.package.outputs.rustc }} + shell: bash + run: | + FILE="${CRATE}-${VERSION}.crate" + rustc_here=$(rustc --version | awk '{ print $2 }') + if [[ -z "$ATTESTED_RUSTC" || "$rustc_here" != "$ATTESTED_RUSTC" ]]; then + echo "::error::this job's rustc is '$rustc_here', the package job's was '$ATTESTED_RUSTC'; nothing was published." + exit 1 + fi + cargo package --no-verify --locked + want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' "$RUNNER_TEMP/release-artifacts/SHA256SUMS") + got=$(sha256sum "${CARGO_TARGET_DIR:-target}/package/$FILE" | awk '{ print $1 }') + echo "attested: $want" + echo "packaged: $got" + if [[ -z "$want" || "$want" != "$got" ]]; then + echo "::error::this job packages a different $FILE than the attested one; nothing was published." + exit 1 + fi # Preferred credential: crates.io Trusted Publishing, which exchanges # this job's OIDC token for a short-lived crates.io token (revoked when @@ -388,7 +445,7 @@ jobs: - name: Obtain a crates.io token via Trusted Publishing id: trusted continue-on-error: true - uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 + uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 - name: Publish to crates.io env: @@ -444,12 +501,6 @@ jobs: echo "> docs.rs may take a few minutes to build and index the documentation." } >> "$GITHUB_STEP_SUMMARY" - - name: Download the attested artifacts - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: release-artifacts - path: release-artifacts/ - # The attestation and SHA256SUMS describe the `.crate` the `package` job # built; crates.io serves the one `cargo publish` just built. `cargo # package` is reproducible (crates.io's 0.18.0 is byte-identical to a @@ -471,7 +522,7 @@ jobs: sleep 20 done test -s "/tmp/$FILE" || { echo "::error::could not download $FILE from crates.io"; exit 1; } - want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' release-artifacts/SHA256SUMS) + want=$(awk -v f="$FILE" '$2 == f || $2 == "./" f { print $1 }' "$RUNNER_TEMP/release-artifacts/SHA256SUMS") got=$(sha256sum "/tmp/$FILE" | awk '{ print $1 }') echo "attested: $want" echo "crates.io: $got" @@ -488,7 +539,7 @@ jobs: # A prerelease gets no book of its own (scripts/build-versioned-site.py # skips prerelease tags); main's /dev/ already shows it. if: needs.validate.outputs.is_prerelease == 'false' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: actions: write diff --git a/.gitignore b/.gitignore index 2b70b876..d8c5bed7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /target/ examples/hello-ext/target/ benches/engine/target/ -benches/engine/Cargo.lock book/doctest/target/ book/book/ mutants.out/ diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 00000000..178a34e3 --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: MIT +# Formatter/linter configuration for taplo (https://taplo.tamasfe.dev). +# taplo fmt --check # as CI would run it +# taplo lint +# Written to match the TOML already in the repository (four-space indent, +# trailing comments left where the author put them), so adopting it reformats +# nothing. +include = ["*.toml", "*/Cargo.toml", "*/*/Cargo.toml", ".cargo/*.toml"] +# `.cargo/mutants.toml` is a long annotated glob list whose trailing comments are +# aligned by hand; taplo would re-flow all 250 lines of it. +exclude = ["**/target/**", ".cargo/mutants.toml"] + +[formatting] +indent_string = " " +align_comments = false +align_entries = false +array_auto_expand = false +array_auto_collapse = false +reorder_keys = false +column_width = 100 diff --git a/AUDIT.md b/AUDIT.md index 9ef094c0..801d836b 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -2,11 +2,12 @@ # quack-rs — audit log, August – October 2026 -Sections 1–6 are the first audit, below. Sections 7–13 are the later ones, newest +Sections 1–6 are the first audit, below. Sections 7–14 are the later ones, newest last: 7 (second audit, 0.17.0 → 0.18.0), 8–10 (third to fifth, before 0.18.0), -11 (sixth review, after 0.18.0), 12 (the DuckDB 2.0.0 review) and 13 (the seventh -review, after 0.19.0). Each ends with how it was verified, and later sections -correct earlier ones where they were wrong (13.4 lists the corrections). +11 (sixth review, after 0.18.0), 12 (the DuckDB 2.0.0 review), 13 (the seventh +review, after 0.19.0) and 14 (the eighth review, released as 0.20.0). Each ends with +how it was verified, and later sections correct earlier ones where they were wrong +(13.4 and 14.9 list the corrections). The first audit (sections 1–6) reviewed `v0.16.0`, against **DuckDB 1.5.4** (prebuilt `libduckdb`, x86-64 Linux) with `libduckdb-sys 1.10504.0`. @@ -2669,6 +2670,8 @@ one thread, and not on 2.0.0. ### 13.7 Performance +Superseded where the cells overlap by 14.7, which re-measured them on the 0.20.0 tree. + Measured with `benches/engine/compare.py` (committed; see its README), the same benchmark extension built against 0.19.0 and against this tree, and its hand-written twin, interleaved across sessions. DuckDB CLI 1.5.6 and 2.0.0, `SET threads=1`, 20 @@ -2881,3 +2884,611 @@ Linux 1.5.6 build was run after the push, above), `cargo-deny`, `cargo-mutants` the changed files, a Wasm build, and the changed workflows in GitHub Actions before the push: they were checked by the three scripts above and by reading, and the first run in Actions is the one that found the failures listed above. + +## 14. Eighth review, October 2026 (after 0.19.0, released as 0.20.0) + +Commissioned as the most adversarial review of the whole repository that could be made, +with the instruction to fix and implement what it finds and not to call the result +finished until a public release could be cut from the merged pull request. Reviewed at +`3c8ee2d` (a frozen snapshot of the branch that follows 0.19.0, `9c2dc76`), against +DuckDB 1.4.4, 1.5.6 and 2.0.0. + +Labels, as in section 13. **VALIDATED**: I ran it against the real binaries, or pinned it +with a test and showed the test fails on the code without the fix. **REPORTED**: a +reviewer's or work package's finding with its evidence, which I did not re-run; where I +fixed one, the fix rests on the code or upstream source it names. **READ**: from +DuckDB's source, not run. **CONJECTURED**: reasoned, not tested. + +### 14.1 Method + +- Twelve reviewers, each in its own scratch copy of the snapshot, each told to disprove + rather than confirm: `unsound` (soundness of the data plane), `lastdiff` (every line + that changed since 0.19.0), `apigaps` (the C API against both headers), `conform` + (a scaffolded project built and loaded into three engines, against the community + build's scripts), `docs`, `ci` (workflows and scripts, mutated one by one), `harden` + (hostile input, resource limits, the release process), `ffi` (every callback kind on + the control plane), `author-tables` and `author-types` (an extension written against + the SDK by someone who has not read its source: table functions, COPY, replacement + scans; scalar functions, aggregates, types), `deps` (dependency ranges, packaging, + tooling) and `tests` (coverage, mutation, fuzzing, sanitizers, flakiness). +- The findings ledger had 130 rows when the implementation phase began (a row can cover + a range, e.g. DOCS-10..23): 2 High, 46 Medium, 56 Low (51 Low, 3 Low/Nit, 2 + Low-Medium), 9 Nit, 14 informational and 3 recommendations. Two more High findings + came later: SQLMACRO-1 (14.2), found while implementing, and the `tests` reviewer's + 17 items, whose own count is 0 Critical, 2 High, 8 Medium, 5 Low and 2 Nit or + informational. +- Six work packages fixed what the ledger assigned them, each in its own worktree and + each told to show that its test fails on the pre-fix commit and passes after: `infra` + (workflows, scripts, release process, repository configuration), `scaffold` + (`src/scaffold`, `src/validate`, `src/bin`, the build script), `corea` (vectors, the + chunk writer, Arrow, scalar and aggregate builders, types), `coreb` (the entry point, + callbacks, the ABI guard), `api` (values, queries, appenders, new wrappers) and `docs`. + I merged them with merge commits (`b833f5a` infra, `201a5c7` api, `327939d` coreb, + `90d397f` corea, `f2186d0` docs, `6664103` scaffold) and resolved by hand the places + where two packages had edited the same lines. +- What I fixed myself, outside the packages, is in 14.2 and 14.3. +- Reviewers' patches were adopted line by line. Two were rejected with evidence (14.4). + +### 14.2 High-severity findings + +1. **`ListBuilder` cached the parent's entry array; a nested list's reserve freed it + (LASTDIFF-1).** A regression against 0.19.0 inside the unreleased performance work, + never published. With an inner `ListBuilder` over a `LIST(LIST(..))` whose reserve grew + the parent, the next `set_entry` wrote through a dangling pointer. VALIDATED: the + regression test `a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves` + segfaults (signal 11) on `3c8ee2d` and passes with the fix, against 1.5.6 and 2.0.0. `set_entry` fetches the array on every call (`734bb15`). + The first version of the test used the arrow lambda `j -> ...`, which DuckDB 2.0.0 + rejects; I committed it without running it on 2.0.0 and CI's 2.0.0 job failed + (`691c160` fixes the SQL, `lambda j:`). +2. **`SqlMacro::register` ran statements ahead of a parse error on DuckDB 2.0.0 + (SQLMACRO-1).** Found by the `corea` package, which ran `harden`'s hostile-input test + on 2.0.0. VALIDATED on 2.0.0: body `1); DROP TABLE canary; --` makes SQL that does not + parse as a whole, `duckdb_extract_statements` reports 0 statements, the old code + refused only a count above 1 and executed it, and 2.0.0's `duckdb_query` runs the + statements it has parsed before it reports the error (1.4.4 and 1.5.6 parse first). + The test `hostile_input::a_macro_body_cannot_turn_one_statement_into_several` fails + on the old code with "Table with name canary does not exist". Fixed in `32e8d64`: + exactly one statement, and a count of zero is an error carrying + `duckdb_extract_statements_error`. Severity: a macro body is the author's code, not + SQL data, and the documentation says never to build one from untrusted text, so + exploiting it needed an author who did; the guard the documentation described as a + defence failed open on 2.0.0. See 14.9 for the earlier claim this corrects. +3. **The release process could publish a stale `Cargo.lock` (DOCS-1, CI-1).** + `RELEASING.md` told the maintainer not to touch `Cargo.lock`, `release.yml` packages + with `--locked`, and the failure arrives only after the whole of `ci.yml` has run. + REPORTED by `docs` and `ci` (the latter simulated it). Now + `scripts/check-release-metadata.py` checks every committed lockfile, the CHANGELOG + heading and links, `CITATION.cff`, `SECURITY.md`, the README's latest line and every + `quack-rs = "0.X"` snippet, in the `doc` job and in the release `validate` job, and + `RELEASING.md` lists the five lockfiles. VALIDATED here: with Cargo.toml at 0.20.0 and + the rest at 0.19.0 it listed 12 stale places, and it passes on the final tree. +4. **The `leak-check` job could not fail for a missing `duckdb_destroy_*` (TESTS-1).** + VALIDATED with a canary (`tests/lsan_probe.rs`, nightly, `-Zsanitizer=leak`, prebuilt + libduckdb 1.5.6): with `leak:libduckdb.so` in `lsan-suppressions.txt` a leaked + `LogicalType` and a leaked `Value` exit 0 (stderr: "Suppressions used"); with the rule + removed they exit 23 with a leak report; a leaked Rust `Box` exits 23 either way. The + rule matched every frame of an allocation, so it covered every handle DuckDB + allocates. It was not needed: all 460 tests of `ffi_roundtrip` exit 0 under + LeakSanitizer with no suppression. The job now runs the canary and requires the + report. A first attempt at this measurement ran the probe through an empty variable + and "passed" vacuously; I caught it from the empty path in the output and redid it + with the executable cargo printed. +5. **The mutation gate cannot see `unsafe fn` bodies (TESTS-2).** cargo-mutants 27.1.0 + generates no mutants inside them, never mutates integer literals and cannot look into + macros; nothing said so, and the "0 missed" results in earlier sections never covered + the pointer arithmetic. VALIDATED on this tree: 2,493 mutants are listed with the + repository configuration (3,748 with `--no-config`); a heuristic count (comments and + strings blanked, braces matched) puts 506 `unsafe fn`s at 5,127 of 22,898 non-test + lines (22.4%); on `src/vector/reader.rs` and `writer.rs` the list grows from 6 to 192 + with the `unsafe` keyword removed from signatures and to 344 with the literals + wrapped. `scripts/mutants-expose-unsafe.py` does that on a copy; CONTRIBUTING.md + says what the gate cannot see. The `tests` reviewer ran it over the vector layer and + found real holes (14.3). + +### 14.3 Other findings, by work package + +Each package's tests were required to fail on the pre-fix commit. I did not re-run those +proofs unless the entry says VALIDATED; the final tree passes every suite (14.8). + +**`corea`** (REPORTED unless noted). +- UNSOUND-1: `ChunkWriter` counts rows *claimed*; a claimed row never written was + published with stale contents. `release_row` and corrected documentation. The test + fails against `3c8ee2d`'s `chunk_writer.rs` (`[1, 2, 7]`, left 2051, right 2050). The + conjectured out-of-bounds variant for hand-written `LIST`/`MAP` columns was not built + (CONJECTURED). +- FFI-1: `data_chunk_from_arrow` read past the child for list offsets that are negative, + decreasing or past the child, list-view offsets and sizes, struct and union children + shorter than their parent, and dictionary indices outside the dictionary + (`src/arrow/import_layout/bounds.rs`). The live test died with SIGSEGV on the old + code. Not covered: string and binary offsets, whose data buffer length is not in the + C Data Interface. +- HARDEN-6/7/8: logical-type walks bounded at 256 levels (DuckDB's own + `LogicalType::ToString` overflows a 2 MiB thread at 6,000 levels, so the cap protects + quack-rs's walks only); `set_cardinality` clamped; `WarningCollector` capped at 10,000 + (a behaviour change, listed). +- AUTHT2-1: eight type pairs the duplicate-overload guard let through are ambiguous on + 1.5.6; they are refused at registration now and five distinct pairs still register. + `LIST(T)` beside `ARRAY(T, n)` and a `UNION` whose members are a subset of another's + were not detected (READ in `cast_rules.cpp`, not run). +- AUTHT2-2 `overload_typed`, FFI-6 and FFI-12 (bind data), FFI-8 (the aggregate + name-collision diagnosis), LASTDIFF-9 (three live tests that kill mutants). +- Live `ffi_roundtrip` as the package reported it: 1.5.6 413/413, 1.4.4 277/277, 2.0.0 + 412/413 (the failure was SQLMACRO-1, outside its files). + +**`coreb`** (REPORTED; the pre/post evidence is in the commits). +- FFI-2/FFI-3: registration is transactional (`BEGIN` before the closure, `COMMIT` on + `Ok`, `ROLLBACK` on `Err` or a panic), with `Registration::Autocommit` to opt out. A + probe extension that fails after step N leaves 0 catalog entries on 1.4.4, 1.5.6 and + 2.0.0 and the retry succeeds (before: 6 entries stay and the retry is refused); a `LOAD` + inside the host's own transaction does not wait; one `BEGIN`/`COMMIT` pair costs + 0.27–0.38 ms and the `hello-ext` `LOAD` medians went 37.6 → 36.5 ms (1.5.6), + 76.8 → 61.0 ms (2.0.0), 40.9 → 37.1 ms (1.4.4) over 30 interleaved runs each. Not + rolled back, because DuckDB keeps them in the database configuration: casts, + replacement scans, configuration options. The copy-format probe moved to its own + connection because 2.0 aborts a transaction on any failed statement. +- FFI-5: `panic = "abort"` fails to compile in the extension's crate (scratch cdylib: + abort fails with the message, unwind builds, `RUSTFLAGS=-C panic=abort` fails; + `cargo check --target wasm32-unknown-emscripten` passes; no emscripten link was run). +- DEPS-12: `#[unsafe(no_mangle)]` in the macros. The `734bb15` macros give four "unsafe + attribute used without unsafe" under edition 2024; the library itself compiles under + edition 2024. Remaining blockers are in tests (nine E0716 temporaries, an + `extern "C"` block without `unsafe`). +- DEPS-13: four enums are `#[non_exhaustive]`. FFI-7: the catalog scan cannot be + narrowed (`duckdb_functions()` has no pushdown). FFI-4, FFI-10: documented. +- DuckDB 2.0.x layout (decided in section 13.3): not extrapolated. `AbiPolicy::Strict` + keeps refusing a DuckDB release a `duckdb-1-5*` build was not verified against; + 2.0.1 is due 2026-11-18 and a quack-rs release listing it should follow (14.4). + +**`api`** (REPORTED). +- APIGAPS-2/-3: `Value::try_cast`, `array_items`, `union_tag`, `bit`, `as_bit`, `bignum`, + `as_bignum`. The C getters' aborts were reproduced first with a raw-call probe on + 1.4.4, 1.5.6 and 2.0.0 (exit 134 on NULL and `'abc'`, 139 on a null handle); every such + input is `None` now. +- Statement metadata (`StatementType`, `parameter_type`, `columns`, `table_names`), + `Appender::for_query`, `from_raw`/`into_raw` on four owner types, typed extra info on + four builders, typed `COPY TO` state, `FileHandle` and `ClientContext` `Send` + (justified from `file_system-c.cpp`, byte-identical on 1.5.6 and 2.0.0; not `Sync`). +- `duckdb_prepared_statement_type` aborts on a failed prepare (1.5.6): guarded. +- Mutation gate on the new code with the PR job's flags: 0 missed. +- APIGAPS-5 (a typed query error) declined: it needs a central error-type change. + +**`scaffold`.** +- CONFORM-1, HARDEN-1/2: `parse_description_yml` follows PyYAML 6 `safe_load`. + VALIDATED here: on the 395 descriptors of community-extensions `845779e` quack-rs and + PyYAML agree on all 395, and on 3 x 39,500 random mutants (characters from a + YAML-significant alphabet deleted, inserted or replaced; lines duplicated, deleted, + swapped or re-indented; truncation) quack-rs accepted none that PyYAML refuses and + never panicked. In the other direction quack-rs refuses more than PyYAML (it accepted + 18,398, 18,080 and 18,426 of the mutants against PyYAML's 24,846, 24,501 and 24,820); + what separates them was not itemised. The package reports 8 deliberate strictnesses and + hostile inputs that took 3.8 to 21.5 s now taking at most 330 ms (REPORTED). +- CONFORM-3: 44 reserved names (10 aliases and 34 core extensions) checked against + DuckDB and the community list by `scripts/check-extension-names.py`, which passes here + and reports the package's 8 of 8 mutants caught (REPORTED). +- CONFORM-4/5/6/12/13/14/15, HARDEN-5, DEPS-1/3, DOCS-8: the scaffold writes a native + example function, a SQLLogicTest with the upstream header, a `make test` that fails when + the runner skips a file, `README.md`, the license text, `deny.toml`, + `docs/function_descriptions.csv`, a hardened `extension-ci.yml`, and refuses names and + platforms the community build ignores. VALIDATED here: a freshly generated project + (`quack_scaffold`, 15 files) builds, is stamped by `append_metadata` and its LOAD works + on 1.4.4, 1.5.6 and 2.0.0 including `-readonly` against an on-disk database; its unit + test and `clippy -D warnings` pass; `cargo-deny` 0.20.2 passes on it (advisories, + bans, licenses, sources); the chapter of the book matches the generated tree, the v1.5.6 + bindings pin (`~1.10506.0`), the refusal of an unstable 2.x target, the dual-license + file names and the `LICENSE.TODO` fallback. +- HARDEN-3/4, LASTDIFF-6, FFI-11 (`append_metadata`: exclusive random temporary file, + mode kept, symlink output written through, non-UTF-8 argument, FIFO input) and HARDEN-12 + (the build script forwards only an all-digit `vX.Y.Z`): REPORTED to fail against + `734bb15`. Windows behaviour of the rename and permission copy was not tested, and a + non-writable output directory (the rename needs a writable directory) is CONJECTURED. +- `hello-ext`: `gen_series_v2` refuses a value past BIGINT and uses the guard macros + (FFI-9); its checks are 32 now. + +**`infra`** (REPORTED; the scripts and the YAML I re-ran, see 14.8). +- CI-1/DOCS-1 (above), CI-2 (a missing guard script no longer reads as "upstream + unreachable"; on a tag an unreachable upstream fails), CI-3 (the Rust toolchain is + pinned to 1.99.1; an informational `latest-stable` job tracks the next one), CI-6/ + HARDEN-11 (the `.crate` hash is compared before the upload), CI-7 (the aggregate check + no longer passes while a gate is removed; 18 mutation probes), CI-8, CI-9, CI-11 + (actionlint), DEPS-4/6 (cargo-deny over the CI-only graphs, typos, taplo, machete, + `.editorconfig`), DEPS-15 (the scaffold's action pins are checked against the + workflows'), CI-13, and Dependabot #145 folded in. +- Every action SHA and tool SHA-256 re-checked against upstream; shellcheck over 158 + extracted `run:` blocks found the same 7 pre-existing notes as the baseline; the + release `validate` job simulated for v0.20.0 and for the documented hotfix flow. +- Runner images are pinned (`ubuntu-24.04`, `macos-26`, `windows-2025`) and + `check-action-pins.py` fails on a floating label (CI-12). I first declined this as + unverifiable and reversed it when `infra` showed each label is a real image: the + decision was wrong and is recorded in 14.10. + +**`docs`** (REPORTED, and spot-checked by me where stated). +- The tutorial now teaches guarded callbacks and its output was produced by building the + page's code and loading it into 1.4.4, 1.5.6 and 2.0.0. The window shapes that crash a + C API aggregate are listed as measured (one process per shape, 5,000 and 300,000 + rows). `std::fs` bypassing `enable_external_access` is documented (AUTHT-1, new pitfall + P17). Four new pitfalls (38 in all), and `scripts/check-pitfalls-parity.py` keeps + `LESSONS.md` and the book on the same list. `known-limitations.md` is regrouped, + the upgrade guide "Upgrading from 0.18 to 0.20" is new, README and publishing pages + follow the community build (reserved names, the four revision keys, the 32-byte footer + field, reduced CI on pull requests, not running `strip`). +- Five DuckDB 2.0 behaviours missing from section 12.4 are in `known-limitations.md`: + `duckdb_appender_close` returns an earlier flush error, `duckdb_data_chunk_set_size` + ignores a rejected size in release builds, every C API scalar is `SetFallible()`, the + constant-vector marking of an all-constant scalar call is gone, and `COPY` options with + several values of different types arrive as `TUPLE` (READ in the 1.5.6 and 2.0.0 + sources, not run). Measured and also written down: `VerifyNullHandling` is compiled in + 2.0.0 and gated by `debug_verification_mode`; a table function's bind runs on every + `EXECUTE`; `INTERVAL '1 month' - INTERVAL '1 day'` equals `'29 days'`; a bare identifier + table-function argument warns. +- Written from the packages' WIP: I re-checked the scaffold chapter against generated + projects (above), and removed the review names that had leaked into published pages. +- Not verified: Windows and macOS rendering; the Go `extbuild` behaviours the `conform` + reviewer read; `duckdb_get_bignum` (same code as `duckdb_get_bit`, not run); static + registration (read in 2.0.0's `extension_load.cpp`). + +**The `tests` reviewer's findings, and what I did with them** (merge phase). +- TESTS-1, -2: 14.2. +- TESTS-3 to -7: 35 new tests (`wide_rows`, `vector_capacity_guard`, twelve Arrow + layout-walker unit tests, `abi_policy_enforced`, `lifecycle_gaps`, + `connection_registrar`, `oracle_temporal`, `property_inputs`, `lsan_probe`), each + written against a surviving mutant. The reviewer proved the kills on `3c8ee2d` with the + bodies of `unsafe fn`s exposed; I ran them on the merged tree against 1.5.6 and 2.0.0. + Three adaptations: the Arrow shapes now carry `indices` (a field `corea` added), one + test was dropped (a list starting 2^40 elements into a three-element child is refused + by the new bounds check before the arithmetic it targeted, and + `list_offsets_outside_the_child_are_refused` covers the refusal), and the timestamp + oracle skips the values DuckDB 2.0.0 cannot answer (below). +- **A DuckDB 2.0.0 defect found by that oracle (VALIDATED, not reported upstream):** + `CAST(make_timestamp(m) AS VARCHAR)` for `m` below -9223372022400000000 raises + `INTERNAL Error: Scalar function "__cast" threw an execution error, but the function is + not marked as fallible ... Date out of range in timestamp conversion`, which `TRY()` + does not catch; 1.x answers NULL. quack-rs's render guard refuses such values before + they reach the engine. +- TESTS-9: `parse_version` refused a signed part; fixed, with a proptest whose saved + failing case (`0.+0.0`) is checked in as `tests/property_inputs.proptest-regressions`. +- TESTS-10: the upstream-comparison scripts returned 2 ("could not check") for an HTTP + 404 and for a reshaped file as well as for an unreachable network (VALIDATED for the + platform list: `--ref this-ref-does-not-exist` exited 2). Now 4xx and shape surprises + return 1 and the scheduled run fails a 2. +- TESTS-13/14: four fuzz targets with seeds (the fuzz job copies `fuzz/seeds//` + and would have failed without them); `QUACK_REQUIRE_TOOLS=1` turns the scaffold tests' + silent skip into a failure on the Linux test leg. +- TESTS-11 (partly): `src/callback.rs` is mutated now (5 mutants, 4 caught, 1 unviable). +- TESTS-12, -15: two comments corrected (Miri cannot run the Arrow module; `validity.rs` + pointed at the wrong test file). + +**Manifest, packaging and dependency changes** (mine). +- DEPS-2: `libduckdb-sys` and `duckdb` are `>=1.4.4, <1.20100` everywhere (was `<2`). + VALIDATED: `cargo check --all-targets` and `cargo test --lib` pass on the 1.4.4, + 1.10506.1 and 1.20000.1 bindings (867 tests each before the packages merged); deleting + the 1.20000.1 bindings' root re-export (marked "TODO: Remove in the future" upstream) + gave 214 compile errors, which is why the ceiling stops below 1.20100. A test fails if a + manifest states another range. +- DEPS-7: an `include` allow-list; `cargo package --list --locked` lists 291 files, + 191 under `src/` and 87 under `tests/`, and `cargo publish --dry-run --locked` builds + the packaged crate. +- `build.rs` fails in seconds, naming the directory, when `libduckdb-sys` reports an + include directory without `duckdb.hpp` (CI run 705's informational job polled for ten + minutes on a partial cache). Not exercised against a source build here. +- Lockfiles: see the CHANGELOG. Dependabot #143, #148 and #149 are folded in by hand + and should close as superseded once the pull request merges. +- Rustdoc the engines contradict was corrected (`44f67d7`); `check-api-coverage.py` + now reports 457 of 546 slots wrapped and 89 not used (8 candidates, 45 deprecated, 23 + host-side, 13 superseded), after I removed `duckdb_extract_statements_error` from the + allowlist (a stale line that would have failed the `abi-table` job). + +### 14.4 Not done, declined or rejected + +- **The pure-Rust replacement of the C++ test shim** (`deps` patch 0004, a prototype that + generates the extern declarations in `build.rs`) is not applied. It passed the live + suite on three bindings and on Rust 1.86.0 in the reviewer's runs, but it touches the + `bundled-test` build, which I cannot run here, and Windows and macOS, which nobody has + run it on. It only affects the test harness (the shipped extension does not use it). + Worth a release of its own with CI as the judge. +- **`tests` patch 04 (strip a leading `_` from every exported symbol) is rejected.** The + tree already tells a Mach-O symbol table from an ELF one by its magic number (the + `lastdiff` fix, `734bb15`); the reviewer's repro input produces no warning now, and an + ELF symbol `_my_ext_init_c_api` really is a different symbol DuckDB cannot load, which + the blanket strip would have silenced. +- APIGAPS-5 (a typed query error), AUTHT2-8..14 (list builders for `LIST(STRUCT)` and + `LIST(LIST)`, further helpers), DEPS-10 (the exact `tempfile` pin; `criterion` for one + bench), DEPS-14 (splitting `append_metadata` out of the runtime crate) and the wrappers + for the 8 candidate slots (among them `duckdb_geometry_type_get_crs`) are left for a + later release. `src/copy_function/info.rs` is 735 lines against the 500-line guideline + (debt, marked here). +- TESTS-11: no scheduled full mutation sweep was added (the sweep is + `workflow_dispatch` only, and the pull-request job runs `--lib` over changed files), and + the files whose mutants only a live engine can kill remain un-mutated in CI. TESTS-12 + and -16, as nits: `tests/secret_zeroize.rs` cannot run under Miri (its allocator reads + uninitialised memory), `INSTALL icu` writes `~/.duckdb`, the interrupt test cancels + after a fixed 250 ms, and four test binaries repeat a ten-line fixture. +- UNSOUND-2 (the 64-bit `FfiState` tag) is declined: the documented mitigation stands. +- DEPS-9: only `proptest` and `cc` were updated in the root lock; the other compatible + updates (`arrow` 58 → 60, for one) are not applied, because `libduckdb-sys` and + `duckdb` must stay on the 1.5.6 bindings the lock is for. +- A DuckDB 2.0.1 layout is not extrapolated (14.3, `coreb`). + +### 14.5 For the owner (outward-facing; nothing here was done) + +- Yank 0.3.0–0.15.0 and file RustSec advisories for the heap-corruption defects fixed in + 0.16.0 (HARDEN-10). The advisory drafts are in the review's scratch directory, not in + the repository. +- Move the crates.io token into the `crates-io` environment and configure Trusted + Publishing (HARDEN-9; `RELEASING.md` has the order). +- Report `duckdb_register_log_storage` freeing its extra data twice when the name is taken + (APIGAPS-1; present from DuckDB 1.5.0 to `main`; quack-rs does not wrap it). +- Disable Cloudflare's email obfuscation for the book, or keep the reworded page (DOCS-6). +- Branch protection, the `crates-io` environment and the tag ruleset could not be read + (CI-UNREADABLE); confirm `CI passed` is the required check. +- CI-4 (the release gate depends on DuckDB `latest`, extension-ci-tools branches and + PyPI) and CI-10 (duplicate push and pull-request runs on `claude/**`) are policy + decisions. +- `SECURITY.md` now supports 0.20.x and 0.19.x and marks 0.18.x end of life. + +### 14.6 Not verified + +Windows, macOS and WebAssembly (no local runner; the `test` matrix, the Wasm job and +the `scaffold-e2e` job are the check); the `bundled-test` source build on any +platform; live engines other than 1.4.4, 1.5.6 and 2.0.0 (CI's older-engine matrix +covers 1.4.5, 1.5.0 and 1.5.3); AddressSanitizer, Miri and valgrind over the new code +(CI runs `asan`, `leak-check` and `miri`; `leak-check`'s command ran here without +`-Zbuild-std`); every changed workflow as GitHub runs it, including the new canary step, +the `apt-get` step, the scheduled strictness and the release workflow; the fuzz targets +beyond 60 to 120 seconds each; the `ListBuilder` change under valgrind (the release +libduckdb's own allocator hides vector buffers from valgrind and AddressSanitizer: 14.9). + +### 14.7 Performance + +Measured with `benches/engine/compare.py` on a quiet shared 4-vCPU virtual machine +("Intel(R) Xeon(R) Processor @ 2.10GHz", release build, fat LTO, one thread, base = +`git archive v0.19.0`, new = this tree; both stamped by `append_metadata`; results +checked equal before timing; load average at start 1.5 on 1.5.6 and 1.1 on 2.0.0). + +**DuckDB 1.5.6, 6 sessions x 3 timed runs (n = 18).** Seconds per query (median, min–max) and median ns per row; the hand-written twin is the floor, not a goal. + +| Query | 0.19.0 | now | hand-written | +|---|---|---|---| +| nullable BIGINT, map1 | 0.156 (0.140–0.197) s, 7.80 ns | 0.117 (0.113–0.204) s, 5.88 ns | 0.123 (0.110–0.157) s, 6.17 ns | +| nullable BIGINT, map2 | 0.203 (0.182–0.243) s, 10.15 ns | 0.132 (0.118–0.186) s, 6.60 ns | 0.123 (0.112–0.161) s, 6.15 ns | +| non-NULL BIGINT, map1 | 0.041 (0.034–0.062) s, 2.05 ns | 0.041 (0.034–0.073) s, 2.07 ns | 0.034 (0.031–0.056) s, 1.70 ns | +| VARCHAR -> BIGINT, short strings | 0.671 (0.582–0.854) s, 33.55 ns | 0.488 (0.446–0.705) s, 24.40 ns | 0.434 (0.377–0.550) s, 21.73 ns | +| VARCHAR -> BIGINT, long strings | 0.448 (0.415–0.595) s, 44.80 ns | 0.442 (0.407–0.561) s, 44.15 ns | 0.417 (0.369–0.585) s, 41.75 ns | +| VARCHAR -> VARCHAR trim, short | 1.225 (1.111–1.423) s, 61.25 ns | 1.077 (1.020–1.211) s, 53.83 ns | 0.794 (0.750–0.962) s, 39.68 ns | +| BIGINT -> [x, x+1, x+2] | 0.226 (0.213–0.295) s, 11.28 ns | 0.138 (0.125–0.189) s, 6.88 ns | 0.101 (0.092–0.126) s, 5.03 ns | +| BIGINT -> [x, x+1, x+2, NULL] | 1.206 (1.086–1.423) s, 60.30 ns | 0.214 (0.197–0.249) s, 10.72 ns | 0.112 (0.091–0.156) s, 5.58 ns | + +**DuckDB 2.0.0, 4 sessions x 3 timed runs (n = 12).** Seconds per query (median, min–max) and median ns per row; the hand-written twin is the floor, not a goal. + +| Query | 0.19.0 | now | hand-written | +|---|---|---|---| +| nullable BIGINT, map1 | 0.142 (0.135–0.191) s, 7.08 ns | 0.124 (0.105–0.136) s, 6.20 ns | 0.121 (0.107–0.139) s, 6.08 ns | +| nullable BIGINT, map2 | 0.217 (0.194–0.245) s, 10.88 ns | 0.123 (0.115–0.137) s, 6.17 ns | 0.121 (0.115–0.147) s, 6.08 ns | +| non-NULL BIGINT, map1 | 0.044 (0.041–0.059) s, 2.20 ns | 0.045 (0.040–0.061) s, 2.27 ns | 0.042 (0.038–0.063) s, 2.08 ns | +| VARCHAR -> BIGINT, short strings | 0.768 (0.609–0.928) s, 38.38 ns | 0.522 (0.449–0.719) s, 26.08 ns | 0.428 (0.395–0.614) s, 21.40 ns | +| VARCHAR -> BIGINT, long strings | 0.619 (0.500–0.829) s, 61.90 ns | 0.549 (0.464–0.714) s, 54.90 ns | 0.487 (0.451–0.623) s, 48.75 ns | +| VARCHAR -> VARCHAR trim, short | 1.324 (1.197–1.492) s, 66.20 ns | 1.220 (1.045–1.365) s, 61.00 ns | 0.952 (0.823–1.089) s, 47.62 ns | +| BIGINT -> [x, x+1, x+2] | 0.307 (0.285–0.386) s, 15.38 ns | 0.163 (0.152–0.220) s, 8.15 ns | 0.104 (0.098–0.118) s, 5.22 ns | +| BIGINT -> [x, x+1, x+2, NULL] | 1.241 (1.154–1.416) s, 62.05 ns | 0.257 (0.221–0.319) s, 12.85 ns | 0.105 (0.100–0.141) s, 5.28 ns | + + +Reading: a difference is claimed only where the old build's fastest run is slower than +the new median. That holds for `map1` and `map2` on a nullable column, short strings and +both `ListBuilder` shapes on both engines, and for `trim` on 1.5.6 only; it does not +hold for the control or the long strings on either engine, or for `trim` on 2.0.0. The +`ListBuilder` NULL-element row is 10.72 ns on 1.5.6 here, against 8.60 ns in the table +the CHANGELOG carried before I re-measured, in the same code path that `734bb15` changed +to fetch the entry array on every call; I did not isolate the cost of that fix from +run-to-run noise. + +### 14.8 How this pass was verified + +On the final tree (commands as CI runs them, with `RUSTFLAGS="-D warnings"`; a gate that +printed 0 is listed with the exit code): + +- Formatting, `clippy --all-targets -D warnings` with default features, `duckdb-1-5`, + `duckdb-1-5-3`, `duckdb-1-5-4`, and the four live configurations + (`bundled-test-prebuilt` alone and with `duckdb-1-5`, `duckdb-1-5-3`, `duckdb-1-5-4`); `cargo doc --no-deps` + with and without `duckdb-1-5-4` under `RUSTDOCFLAGS="-D warnings"`: all exit 0. +- `cargo test --all-targets`: 1,109 tests pass in 20 binaries (default features); 1,249 + with `duckdb-1-5-4`; doctests 235 pass, 3 ignored. +- Live suites (`bundled-test-prebuilt`, prebuilt libduckdb of the engine's own version), + `cargo test --all-targets --no-fail-fast`: **1.5.6** (bindings 1.10506.1, `duckdb-1-5-4`) + 1,789 pass, 0 fail; **2.0.0** (bindings 1.20000.1, `duckdb-1-5-4`) 1,789 pass, 0 fail + (four tests failed once with "No space left on device" when the disk filled, and passed + when re-run alone: the total is the 1,785 of the first run plus those four); **1.4.4** + (bindings 1.4.4, default features) 1,479 pass, 0 fail. All 21 test binaries ran each time. +- The book and README code blocks (`cargo test --doc --manifest-path book/doctest/Cargo.toml`, + libduckdb 1.5.6, bindings 1.10506.1): 255 pass, 5 ignored, 0 fail. +- LeakSanitizer (nightly, `-Zsanitizer=leak`, no `-Zbuild-std`) over `ffi_roundtrip` + with the empty suppression file: 460 tests, exit 0. The four new fuzz targets ran + 60 s each (`description_roundtrip` 120 s more after its fix, 787,689 inputs). +- MSRV: `cargo +1.86.0 check` with and without `duckdb-1-5-4`, after the + fallback resolver picked `libduckdb-sys` 1.10506.0: exit 0. +- `hello-ext` on the 1.20000.1 bindings: 43 unit tests, clippy, and its 32 live checks + pass on 1.4.4, 1.5.6 and 2.0.0 (32/32 each). +- Scripts: every check of the `doc` job (`sync-book-changelog`, `generate-sitemap`, + `check-workflow-expressions`, `check-ci-aggregate` with 37 blocking jobs, + `upstream-guard --self-test` with 14 cases, `check-release-metadata`, + `check-toolchain-pin`, `check-action-pins`, `check-pitfalls-parity` with 38 pitfalls), + `check-api-coverage` and its self-test (23 expectations), `check-extension-names`, + the four upstream guards against live upstream (the ABI table over every release + header), `actionlint` with CI's `-ignore SC2129`, `typos`, `taplo`, `cargo-machete`, + `cargo-deny` over the root and the four CI-only crates, `cargo package --list --locked` + and `cargo publish --dry-run --locked`: exit 0. `zizmor --offline` still exits 12 on + informational and low-confidence findings (the `infra` package counted 50 before its + changes and 53 after, the three new ones being the pedantic `superfluous-actions` + note); it is not a CI gate. +- `mdbook build` (0.4.40), the per-page SEO check (52 pages), the book and README link + check (327 relative and 52 docs.rs links) and the versioned-site build pass. +- `cargo semver-checks` against v0.19.0 (`--release-type patch`): the four + `#[non_exhaustive]` enums and `ScalarBindInfo` no longer being `RefUnwindSafe`, nothing + else; all five are in the CHANGELOG. +- **The pull request's own CI found eleven more problems after every gate above had passed + locally.** Items 1 to 10 needed no change to the shipped library code (item 7 also records + a limitation of it); item 11 changed the description.yml reader (MUT-1). + 1. Windows: `-D warnings` turned unused items in Unix-only test modules into errors + (helpers in `scaffold/tests_make.rs`, `temp_files` and a `Path` import in + `append_metadata/tests_output.rs`); they are `cfg(unix)` now. + 2. macOS: APFS refuses to create a file whose name is not UTF-8 (`EILSEQ`), so + `a_file_name_that_is_not_utf8_is_used_not_a_panic` cannot run there; it is gated off + macOS. + 3. `scaffold-e2e`: the regular expression that rewrites the scaffolded dependency line + dropped its `features`, and the assertion had not followed a change in the scaffold. + 4. WebAssembly: the test run links with `-C link-arg=-sSTACK_SIZE=8MB` (emscripten's + default stack is 64 KB); the job passed on run 724. + 5. A nightly Cargo lays build scripts out as `build///out/`; + `tests/build_script_env.rs` only knew the old layout and finds both now. + 6. The DuckDB 2.0 source-build job (informational): `rust-cache` restored `target/` + without the `duckdb-sources/` directory the 1.20000 bindings unpack there, so the + bindings' build script stopped. The job caches the registry only; it passed on run 724. + 7. `hostile_input::a_deeply_nested_type_is_refused_not_recursed_into` aborted with a + stack overflow on the macOS and Windows source builds and once on Linux with the 1.4.5 + library, and never locally. Cause (a gdb backtrace on a 512 KiB stack, 1.5.6): the + test itself registered a scalar set whose parameter was 3,500 levels deep, and the + next `register_table` ran its `duckdb_functions()` name check, in which DuckDB's own + `LogicalType::ToString` recurses once per level; how much stack that takes depends on + how DuckDB was built. quack-rs lets a parameter type nested past its 256-level cap + register (a return type past it is refused), so an extension that registers such a + type makes every later `duckdb_functions()` scan, its own and other extensions', + overflow. That is a limitation, not fixed here: an extension must bound the depth of + types it builds from untrusted input. The test is replaced by one that stops at the + cap and asserts what the cap changes (a return type nested 256 levels registers, 257 + is refused; overloads that differ only 300 levels down are refused as duplicates); + changing the cap to 255, 257 or 100000 fails it. A type past the cap is refused with + the ANY/INVALID message, which does not mention the depth. + 8. Coverage (`cargo llvm-cov`): all tests passed and the report failed with "invalid + instrumentation profile data". The instrumented child of + `a_failed_write_leaves_nothing_behind_and_names_the_output` runs under `ulimit -f 8`, + which cut its 42,888-byte profile off at 4 KiB. Reproduced locally (91 profile files, one + bad, 4,096 bytes) and fixed by sending that child's profile to `/dev/null`. + 9. Windows (`ffi_roundtrip`): `duckdb_extensions()` could not open a half-written + `parquet` file another test thread was installing. `Fixture` installs `icu` and + `parquet` once per process now. Not reproducible here; the windows-2025 job decides. + 10. Miri: the job's two runs (native, then s390x) share a 60-minute limit and the native one + alone took 41.6 minutes; by the timestamps in its log, 1,086 s went to one parser test on + 100,000-level inputs and 160 s to an SPDX test, both of which use smaller inputs under + Miri now. The limit is 90 minutes. The next run took 18.1 and 17.5 minutes (MUT-2). + 11. Mutation testing: one job for all of a 102-file pull request (1,639 mutants) was stopped + by its runner after 33 and 55 minutes, having reported 6 and 12 survivors. It is eight + shards now (`--shard k/8 --sharding round-robin`) and a gate job with the old check name; + all eight finished in 22 minutes and tested 1,617 mutants: 85 missed and 17 timed out, + 101 of them in the YAML reader added in this release and one in `arrow/import_layout`. + Four engineers (subagents, each in its own worktree) worked the YAML ones from PyYAML's + verdicts, not the reader's; I merged and re-checked their results. + - 79 of the 85 missed were killed by 33 new tests in four modules (`tests_mut_*.rs`) and + three regression tests (below); every row's expectation is PyYAML 6.0.3's, except rows + that pin the reader's own documented limits (nesting of 64, no anchors in a field it + reads, no explicit `? ` keys, error wording), each commented as such. + - 14 of the 17 timeouts were loops that spin when an index update is mutated. Each loop + now carries a `debug_assert!` that a round made progress (flow `skip`, `seq`, `map` + and `plain`; block `parse_mapping` and `parse_seq`; `split_lines`), so a mutant fails + instead of hanging. Release builds are unchanged. The other 3 were mutants of + `value_len`'s memo, which made a run of k blank lines cost k² steps: it is a linear + scan now. + - 4 of the missed were equivalent, and the code that made them so is gone: a parameter + of `value_len` that every caller passed as `false`, a `check_code` call on text that is + only spaces, the text of a comment that no caller read (`split_comment` returns whether + there is one), and a `<=` in the memo. + - 2 of the missed, and one more found while probing, were PyYAML disagreements in the + reader (CHANGELOG, Fixed). Random mutation of the 395 published descriptors (118,500 + files) had found none of them, so the claim that the reader never accepts what PyYAML + refuses was wrong at the edges. After the fixes, 16,195 mutated descriptors and + 14,874 targeted documents show no regression against PyYAML; the reader still refuses + a tab-only line inside a block scalar and a tag alone on a line, which PyYAML accepts + (not fixed). + - A fourth disagreement came from checking an engineer's remark that PyYAML refuses a + mapping key continued on a second line inside `[ … ]`. The reader refused that file + only because `extension:` takes no nested flow collections; in a section it merely + checks (`docs:`) it accepted it, and `{ … }` in both. PyYAML keeps a flow key only if + its `:` is on the key's line and within 1,024 characters of the key's start. 30,120 + random flow collections: 109 disagreements before the fix, all false accepts; none + after, none new, no panic. `cargo mutants` on the functions touched: 38 mutants, 38 + caught. The 118,500-file random run of the published descriptors reported no false + accept and so did not find this one; the flow-collection generator and the question + about a split key did. + A local `cargo mutants` run of 195 mutants in the functions that changed or had + survived: 184 caught, 11 unviable, 0 missed, 0 timed out. The sharded run on the next + head is the check of the other mutants. +- What 14.6 lists as unverified on Windows, macOS, WebAssembly and the source builds is + answered by the pull request's final CI run, which the pull request page shows. + +### 14.9 Corrections to earlier sections + +- **Section 13 and `harden`'s "no SQL injection" result** (35 hostile names through ten + constructors on live 1.5.6, canary untouched) held on 1.5.6 only. On DuckDB 2.0.0 + `SqlMacro::register` ran statements ahead of a parse error (14.2, item 2). A hostile-input + sweep has to run on every supported engine major. +- **Valgrind and AddressSanitizer do not see DuckDB's vector data and list-entry + buffers.** The release libduckdb allocates them with its own jemalloc (769 `duckdb_je_*` + symbols); earlier sections that read a clean valgrind or ASan run as covering them were + wrong, as the `ListBuilder` regression (14.2, item 1) showed. CONTRIBUTING.md says what + the sanitizer runs can and cannot see. +- **"0 missed" mutation results never covered `unsafe fn` bodies** (14.2, item 5). +- **The `leak-check` job did not check what its comment said** until 14.2, item 4. +- Section 13.5 item 1 lists `check_string_len`, `try_write_varchar`/`try_write_blob`, + `ArrowArray::take_from`, `ArrowSchema::take_from`, `ErrorData::has_error`, + `WarningCollector::drain`, `set_max_threads`, `set_cardinality` and query progress as + unreached; the `tests` reviewer's line coverage of the live, unit and integration run + executes all nine (REPORTED). Section 13.2 item 14 put the unexecuted public functions + at 144; on `3c8ee2d` the reviewer counted 170 production functions never executed (29 + containing `unsafe`), none in `struct_reader`, `struct_writer`, `validity` or + `MapVector` (REPORTED). +- The 13.7 timings are superseded by 14.7. The CHANGELOG's coverage figure "445 of 546 + slots wrapped" was a count at one moment; the script reports 457 now. +- The "28 fuzz files" figure for the fuzz crate counted seeds and a lockfile; it had four + targets, one of them live, and has eight now. +- Section 12.4 lacks five DuckDB 2.0 behaviours (14.3, `docs`). +- Section 13.3 recorded that a core-extension name list for the scaffold was not + shipped, because `duckdb_extensions()` mixes core and autoloadable names. The list is + derived from DuckDB's own sources now (`scripts/check-extension-names.py` compares + it with `main` and `v2.0.0` and with the community list), and the scaffold and + `parse_description_yml` refuse its 10 aliases and 34 core names, because + `INSTALL s3 FROM community` fails on 2.0.0 (the alias is applied before the + repository is asked). + +### 14.10 Process lessons (mine) + +- A mutation proof must name the pre-fix commit. My first proof of the `ListBuilder` + fix reverted a file with `git show HEAD:...` and then I committed; `HEAD` already held + the fix and the "without the fix" run passed vacuously. I redid it against `3c8ee2d` and + grepped the reverted file for `self.entries` to confirm it held the old code. +- Run a new live test on every engine major before committing it (the lambda syntax). +- A check that prints nothing proves nothing: an empty variable made the first LeakSanitizer + probe run no probe, and `cmd | grep` let a failing `clippy` run reach a commit; both were + caught before they were pushed, one by the empty path in the output, one by re-running + with an explicit status check. +- A full disk invalidates results quietly: four tests failed with ENOSPC and the other 1,785 + passed; I re-ran only after reading the failure text. +- Local gates miss what only a script checks: the stale API-coverage allowlist line, the + typos findings and two stale "31 checks" counts were found by running every script of + the `doc` and `hygiene` jobs, not by the test suites. And a doc comment I had edited in + `tests/ffi_roundtrip.rs`, which only the live configurations compile, failed + `clippy::doc_markdown` there and in no other gate; I found it by running clippy in + those configurations last. +- A reviewer's snapshot is older than the fixes: `tests` patch 04 would have regressed + ELF handling, and two of its Arrow tests needed an `indices` field that did not exist + at its snapshot. +- Random mutation of valid files is weak exactly where a hand-written parser has its edge + cases. 118,500 mutants of the published descriptors found no disagreement with PyYAML; the + mutation job, asked which boundary comparisons no test pins, found three in a day, and a + grid built around each (anchor lines, comment lines in block scalars, key lengths of + 1,020 to 1,025 with spaces) showed their extent. Use both. +- Delegating a mutation sweep works when each worker has its own worktree, its own target + directory and a written rule that PyYAML, not the code under test, supplies the expected + value: three of four reports found real disagreements instead of encoding them. All four + worktrees started from `main`, not the pushed head, and the workers noticed because line + numbers did not match. +- A test whose only assertion is that the process survived says little, and this one + created its own hazard. The deep-type test left a 3,500-level type in the catalog and the + next registration paid for it; how much stack DuckDB's recursion takes depends on the + build, so a run against the prebuilt library on my machine could not show it. Shrinking + the thread's stack until the local run failed, then reading the backtrace in gdb, found + the cause in minutes. Assert on the outcome, and do not leave a hostile object registered. +- Coverage instrumentation changes what child processes do: a test that limits a child's + file size truncates its profile. `rustup component add llvm-tools-preview` and + `cargo llvm-cov` reproduce the Coverage workflow locally; I had not tried. +- A check's `head_sha` has to be compared with the branch head before reading its failure. + Runs cancelled by a newer push report `CI passed` as failed (runs 721 and 722), and + each push restarts every job, so fixes should be batched. +- A decision made for lack of verification can be wrong: I declined the runner pins + because I could not confirm the labels offline, and `infra` confirmed them. diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b133ee..a19a6331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.20.0] - 2026-10-30 + +This release comes out of an eighth review (`AUDIT.md` section 14): twelve parallel +reviewers (soundness, the diff since 0.19.0, C API gaps, conformance with DuckDB's +community-extension process, documentation, CI and supply chain, hardening, the FFI +boundary, two authors' passes over tables and types, dependencies, and the test suite +itself), then six work packages that fixed what they found, each with the tests that fail +without the fix. It also carries the performance work that review measured against 0.19.0. + +**It contains breaking changes**, marked **Breaking:** under Changed: the entry-point +macros refuse `panic = "abort"`, four public enums are `#[non_exhaustive]`, registration +is transactional, `generate_scaffold` and `parse_description_yml` refuse inputs they used to +accept, and a zero-argument aggregate no longer registers. "Upgrading from 0.18 to 0.20" in +the book lists each with what to do. Two High-severity defects were found and fixed before +release: a segmentation fault in an unpublished `ListBuilder` optimisation, and +`SqlMacro::register` executing statements ahead of a parse error on DuckDB 2.0.0 (see +Security). + ### Added - **`TIMESTAMPTZ_NS`, the type DuckDB 2.0.0 added** (`DUCKDB_TYPE_TIMESTAMP_TZ_NS`, @@ -47,10 +65,144 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a change to a vector, validity or list path. - `scripts/check-api-coverage.py` and `scripts/api-coverage-allowlist.txt`: every slot of `duckdb_ext_api_v1` is either used under `src/` or listed with a class and - a reason (546 slots at v1.5.6: 447 used, 99 listed: 45 deprecated, 22 host-side, - 11 superseded, 21 candidates to wrap). The `abi-table` CI job runs it, so a slot a + a reason (546 slots at v1.5.6: 457 wrapped, 89 listed: 45 deprecated, 23 host-side, + 13 superseded, 8 candidates to wrap). The `abi-table` CI job runs it, so a slot a DuckDB release adds is noticed instead of left for someone to read the header. +- **`quack_scaffold` binary** (`cargo install quack-rs --bin quack_scaffold`): writes a new + extension project to disk from the command line. Every `ScaffoldConfig` field is a flag + (`--description`, `--maintainer`, `--github-repo`, `--version`, `--license`, + `--excluded-platforms`, `--target-duckdb-version`, `--unstable-c-api`, `--git-ref`, `--dir`). + It refuses to overwrite an existing file unless `--force` is given, and writes nothing if it + refuses. It never writes through a symbolic link inside the project. When it is done it + prints the next steps: the `git submodule add` command for extension-ci-tools, the build and + test commands, and the exact `append_metadata` invocation for stamping without `make`. +- **`scaffold::generate_scaffold_with_warnings`** and **`scaffold::Scaffold`**: the generated + files plus warnings about values that were accepted but deserve a second look. These are: a + license id outside quack-rs's shortlist, an excluded platform that is opt-in, excluding every + platform, and an unstable-ABI project the community build cannot ship as generated. + `generate_scaffold` is unchanged and drops the warnings. +- **Scaffold: files a community submission needs** (CONFORM-14/15, DEPS-1): + - `README.md`: build, test and try instructions, the no-`strip`-after-stamping rule, + reduced CI on pull requests, and what to submit. + - `docs/function_descriptions.csv`: the duckdb.org extension page takes function + descriptions and examples only from this file. It has a row for the example function. + - The license text: `LICENSE` for MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, + MIT-0 and Unlicense. An expression gets one `LICENSE-` per license. Anything else gets + a `LICENSE.TODO` naming the text to add. + - `deny.toml`: ignores RUSTSEC-2026-0367/-0368/-0369 (zip < 9), each with a reason. That + zip is only a build-script dependency of `libduckdb-sys` ≤ 1.10506.0, which is what Cargo + resolves for `rust-version = 1.86`. The file also allows the dependency graph's licenses + and the project's own. `cargo deny check` passes on a freshly generated, locked project; + without the ignores it fails on exactly those three advisories. +- **`validate::validate_community_extension_name`**: rejects a name `DuckDB` itself uses, both + its aliases (`s3`, `md`, `postgres`, ...) and its core extensions (`json`, `parquet`, ...), + as well as a badly formed one. The parser and the scaffold use it. `validate_extension_name` + stays a syntax check only. (CONFORM-3) +- **`description.yml`: `repo.variegata` / `repo.andium`** (the refs built for DuckDB v1.5.6 and + v1.4.5) are read into `ExtensionDescription::git_ref_variegata` / `git_ref_andium`. An + unknown `repo` key is now a warning, and so is a `repo` that is not `owner/repo`. + (CONFORM-10) + +- `ChunkWriter::release_row(row)` gives back the row the last `next_row` claimed, so the + drop does not publish it; the next `next_row` returns the same index. Giving back any + other row panics with the row that could be given back. For the + `while let Some(row) = cw.next_row()` loop whose source runs out (UNSOUND-1). +- `ScalarFunctionSetBuilder::overload_typed(TypedScalarFunctionBuilder)`: a closure-built + function (`map1`, `map2`, `try_map1`, ...) can now be one overload of a set, keeping the + signature its closure fixed. This is the only way to overload closure-built functions on + DuckDB 1.4.x, where a second registration under a taken name is refused. An overload built + under another name than the set's is refused by `register` (AUTHT2-2). +- `ScalarBindData::try_get`: the bind data, or the panic message of a `Clone` that failed + when DuckDB copied the bound call (FFI-6). +- `WarningCollector::with_limit(n)`, `WarningCollector::dropped()` and + `warning::DEFAULT_MAX_WARNINGS` (HARDEN-8, see Changed). + +- `entry_point::Registration` (`Transactional`, the default, or `Autocommit`) and + `entry_point::EntryPointOptions` (an `AbiPolicy` plus a `Registration`, built with + `EntryPointOptions::new().abi_policy(..).registration(..)`; `From`). The entry + point macros take them as `entry_point!(name, options = EntryPointOptions::new()..., |con| ..)` + (and `entry_point_v2!`); hand-written entry points use the new + `init_extension_with_options` / `init_extension_v2_with_options`. The existing + `AbiPolicy` arm and `*_with_policy` functions are unchanged (FFI-2). + +- **`Value::try_cast(&LogicalType)`** — SQL `TRY_CAST` of a value to any + logical type, nested ones included, with no path for a C++ exception to reach + Rust (the cast runs inside `duckdb_create_list_value`'s `try`). Payloads + `DuckDB` cannot convert safely (out-of-range timestamps, over-wide `DECIMAL`s) + are refused first, inside `ARRAY`s too. All engines (1.4.4+), no feature. + (APIGAPS-2) +- **`Value::array_items()`** — the elements of an `ARRAY` (or `LIST`) value; an + `ARRAY` read through `list_items` looks empty because the C API reads `LIST` + only. A fixed-size-list argument of a table function, `COPY` option or + scalar bind is now readable. All engines. (APIGAPS-2) +- **`Value::union_tag()`** — the active member of a `UNION` value (its member + value reads as text through `try_cast` to `VARCHAR`). Feature `duckdb-1-5`. + (APIGAPS-2) +- **`Value::bit(&[bool])`, `Value::as_bit()`, `Value::bignum(&[u8], bool)`, + `Value::as_bignum()`** — `BIT` and `BIGNUM` values. The getters never hand + `DuckDB` a value its `duckdb_get_bit` / `duckdb_get_bignum` would abort the + process on (SQL `NULL`, `'abc'`, a `BLOB`, `inf`; a null handle segfaulted + `duckdb_get_bit`); the constructors build only well-formed, canonical values. + All engines. (APIGAPS-3) +- **`query::StatementType`**, **`PreparedStatement::statement_type()`**, + **`QueryResult::statement_type()`**, **`PreparedStatement::parameter_type(i)`** + — what a statement is and its parameter types before it runs. All engines + (stable slots); `COPY DATABASE` / `UPDATE EXTENSIONS` / `MERGE INTO` are + reported as `Invalid` by DuckDB before 1.5.5. (apigaps P1) +- **`PreparedStatement::columns()`** — result column names and types before + execution (`None` while they depend on unbound parameters). Feature + `duckdb-1-5`. (apigaps P1) +- **`query::table_names(con, sql, qualified)`**, **`OwnedConnection::table_names`**, + **`Connection::table_names`** — the tables a statement touches, without running + it (one statement; `JOIN … USING` is refused by DuckDB's names-only bind). + Feature `duckdb-1-5`. (apigaps P1) +- **`Appender::for_query(con, sql, types, table, columns)`** — an appender whose + rows feed a query (`INSERT … SELECT`, `INSERT OR REPLACE`, `MERGE INTO`, …) + instead of a table. Feature `duckdb-1-5`. (apigaps P1, slot 409) +- **`from_raw` / `into_raw`** on `Appender`, `PreparedStatement`, `QueryResult` + and `OwnedConnection`, so a handle from an unwrapped C call can be adopted + into RAII. All engines. (APIGAPS-9) +- **Typed, tag-checked extra info**: `extra_info_typed(value)` on + `AggregateFunctionBuilder`, `AggregateOverloadBuilder`, `CastFunctionBuilder` + and `CopyFunctionBuilder`, read with `extra_info_ref::()` on + `AggregateFunctionInfo`, `CastFunctionInfo` and the four `Copy*Info` types; a + value of another type reads as `None` instead of reinterpreting memory, and + the destructor is generated and panic-safe. All engines (copy: `duckdb-1-5`). + (APIGAPS-8) +- **Typed `COPY TO` state**: `CopyBindInfo::set_bind_data_typed`, + `CopyGlobalInitInfo::set_global_state_typed`, and `bind_data_ref` / + `global_state_ref` on the later phases — no hand-written `Box::into_raw` / + `extern "C"` destructors. Feature `duckdb-1-5`. (AUTHT-3) +- **`FileHandle` and `ClientContext` are now `Send`** (not `Sync`), and + `FileHandle::into_raw` exists: a file opened in a `COPY` `global_init` can be + kept in the statement's global state (in a `Mutex`) without an + `unsafe impl Send` wrapper; the pattern is documented on `FileHandle`. + Feature `duckdb-1-5`. (AUTHT-2) +- **`From for ErrorData`**, so `?` on a `Value` or builder error + works inside `Appender::row` with `duckdb-1-5`. (AUTHT-13) +- **35 new tests from a review of the test suite itself** (`TESTS-3` to `-7`), each + written against a mutant that the existing suite let live, found by exposing the + bodies of `unsafe fn`s that `cargo-mutants` skips: every fixed-width and string type + echoed over 5,000 distinct rows (`wide_rows`, which kills the `u128` stride mutants); + the `OwnedVector` capacity guard for `MAP` and `UNION`; twelve Arrow layout-walker unit + tests; `AbiPolicy` enforced in-process against a v1.4.4 layout table + (`tests/abi_policy_enforced.rs`); `ScalarBindData::get_from_init`, two `extra_info` + destructors and `abi::check`; the seven `Registrar` methods of `Connection` that no test + executed; randomised cross-checks of `datetime` and `interval` against DuckDB's own SQL + (4,000 values per check, fixed seed); and property tests for `parse_version`, SPDX, + `description.yml` and the scaffold's escaping. They pass on DuckDB 1.5.6 and 2.0.0. +- Four fuzz targets, each with a seed directory: `abi_version`, `append_metadata`, + `description_roundtrip` and `scaffold_text` (60 s each on this tree; the first run of + `description_roundtrip` found its own escaping wrong, because PyYAML refuses U+007F even + inside quotes). The fuzz job runs every target `cargo fuzz list` prints. +- `scripts/mutants-expose-unsafe.py`: prepares a copy of the tree so `cargo-mutants` can + mutate what it skips (`unsafe fn` bodies, integer operands of pointer arithmetic). An + audit aid, not run in CI; see CONTRIBUTING.md, "Mutation testing". [TESTS-2] +- `tests/lsan_probe.rs`: the `leak-check` job's canary. It leaks a Rust allocation, a + `LogicalType` and a `Value` on request, and the job now requires LeakSanitizer to fail + on each. [TESTS-1] + ### Fixed - **A `VARIANT` in a function signature crashed DuckDB.** `TypeId::Variant` @@ -82,6 +234,128 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Cargo.toml`: `cc = "1"` could not be resolved by `cargo update -Z direct-minimal-versions`; the lower bound is now `1.0.69`. +- CI: `scripts/check-ci-aggregate.py` no longer passes while a gate is removed + (a job demoted to informational and dropped from `needs`, a step-level + `continue-on-error` in a blocking job or in `ci-ok` itself, a job id with a + comment or anchor, a safety pattern kept only in a comment); it also checks + the workflows README's gate table. [CI-7] +- CI: `scaffold-compile` can no longer pass by running zero tests (`--exact`, + one test binary, and the libtest summary must say `1 passed`). [CI-8] +- CI: `gh release edit` now clears a leftover draft flag and sets the + prerelease flag both ways. [CI-9] +- CI: `scripts/check-api-coverage.py` counts a C API slot as wrapped only when + library code uses it (not comments, string literals, `#[cfg(test)]` code or + test-only modules): `duckdb_open` / `duckdb_close` are test-only, so they are + listed as host-side. [APIGAPS-4, CI-14] +- CI: `check-msrv-vs-duckdb-ci.py` reads `# v1.86.0` / `# rust 1.86.0` pin + comments and `toolchain:` inputs as the constraints they are; + `check-autoload-entries.py` refuses a reshaped upstream array instead of + passing on zero parsed names. [CI-14] + +- **Scaffold (HARDEN-5):** a license with a line break or other whitespace (`MIT\nOR + Apache-2.0`) was written raw into `Cargo.toml`, which then was not TOML. A version + component above `u64::MAX` was accepted, and Cargo then refused to read the project. +- **Scaffold:** an `excluded_platforms` entry that excludes nothing is now refused. The + community build compares exact names, so this covers a platform group such as `wasm` (the + error lists its platforms), a retired platform, and a name it does not build. Before, any + of these was written into `description.yml` and the workflow as if it worked. +- **`append_metadata` (HARDEN-3):** the temporary file was `..tmp`, opened with + create-and-truncate. A symbolic link planted at that guessable name made the tool overwrite + the link's target. It is now created exclusively, under a random name. +- **`append_metadata` (LASTDIFF-6, FFI-11):** + - a failed write left the temporary file behind; it is now removed on every failure; + - a re-stamp reset the output's permissions; they are now kept; + - an output that is a symbolic link was replaced by a regular file; it is now written + through, as 0.19.0 did; + - errors named the temporary file, and a missing output directory surfaced as a + temporary-file error. Errors now name the output and say when the directory does not + exist. +- **`append_metadata` (HARDEN-4):** a file name that is not UTF-8 made it panic (exit 101). + Such names are now used as paths; a flag's value must still be UTF-8. +- **`append_metadata`:** the input is read only if it is a regular file, and only up to 4 GiB. + A FIFO blocked it forever, and `/dev/zero` grew its memory until it was killed. +- **Build script (HARDEN-12):** `QUACK_RS_TARGET_DUCKDB_VERSION` holding a newline could add + `cargo:` directives of its own. Only an all-digit `vX.Y.Z` is now forwarded; anything else + is a one-line, quoted warning. +- **`description.yml`:** the `repo.ref` warning now says why a tag is not enough either (a + branch or a tag can be moved). (CONFORM-15) + +- `ChunkWriter`'s documentation said it counts rows *written*; it counts rows *claimed*, + and a claimed row never written was published with stale contents from DuckDB's reused + output chunk (a wrong answer; for a hand-written `LIST`/`MAP` column a stale entry). + The docs now say so, and `release_row` gives such a row back (UNSOUND-1). +- A `Clone` of `ScalarBindData` that panicked while DuckDB copied the bound call (filter + pushdown does) left the copy with no bind data, so the query answered as if none had + been set and the panic message was lost. `ScalarBindData::get` now reports the message as + the call's error (the query fails with it), `get_from_init` likewise (FFI-6). +- The walks over a logical type (`type_string`, `describe`, `contains_type`) are bounded at + 256 levels: a type nested a few thousand levels deep overflowed the stack and aborted the + process (HARDEN-6). +- `data_chunk_from_arrow`'s list-offset arithmetic is checked (`[i64::MIN, 0]` overflowed + `last - first`) (FFI-1). +- **The `leak-check` CI job could not fail for a missing `duckdb_destroy_*`.** + `lsan-suppressions.txt` held `leak:libduckdb.so`, and LeakSanitizer matches a + suppression against every frame of an allocation, so it silenced every handle DuckDB + allocates. Measured with `tests/lsan_probe.rs`: a leaked `LogicalType` or `Value` exited + 0 with the rule and 23 with a leak report without it. The rule was not needed: the + 460 tests of `ffi_roundtrip` exit 0 under LeakSanitizer with no suppression, because + DuckDB's own one-time allocations are still reachable at exit. The file now holds no + rule, and the job runs the canary. [TESTS-1] +- `abi::parse_version` accepted a signed part (`"1.5.+4"`, because `u64::from_str` takes a + leading `+`); it takes ASCII digits only, like the build script's copy of the rule. + [TESTS-9] +- **`parse_description_yml` disagreed with PyYAML 6.0.3 in four places**, three found by the + first sharded run of the mutation job (101 of the 102 mutants that survived were in this + reader) and one by a differential run on flow collections afterwards. Each has a + regression test whose expectation is PyYAML's verdict: + - A block scalar (`description: |`) lost the lines after a blank line when the next line was + a `#` line indented as far as the text, so `l1`, a blank line and `# c` read as `l1` + where PyYAML reads `l1\n\n# c\n`. A comment at the key's own level still ends the scalar. + - An anchor on a line of its own, followed by a sequence written at the key's indentation + (`k:\n &a\n- x`), was refused; PyYAML reads `{k: [x]}`. + - A quoted key was measured without the spaces before its colon, so one PyYAML refuses at + 1,024 characters (`"…" : v`, "mapping values are not allowed here") was accepted. + - Inside a flow collection, a node followed by `:` was taken for a mapping key wherever the + `:` was. PyYAML takes it for one only if the `:` is on the key's own line and at most + 1,024 characters from where the key began, an anchor in front of it included; otherwise + the `:` has no key and the parser stops there ("expected ',' or '}'"). `k: {a` with + `b: c}` on the next line, `k: [a` with `b: c]` on the next, `{"a"` with `: c}` on the + next, and a key of 1,025 characters were accepted in a section the reader only checks + (and in `extension:`, for the mapping form). + The statement above, that the reader never accepts a file PyYAML refuses, held for 118,500 + random mutants of the published descriptors and missed the third and the fourth. On 30,120 + random flow collections (token soup over brackets, braces, quotes, anchors, aliases, + colons and line breaks, plus keys at the 1,024 boundary) the reader disagreed with PyYAML + on 109 documents before the fix, every one a file only PyYAML refuses, and on none after, + with no document newly refused and no panic. On 14,874 targeted + documents (block scalars with comment-like lines, plain, quoted and flow values with + comments below them) the fixed reader's `description` equals PyYAML's wherever both accept + (200 differed before), and it matches PyYAML on 474 more documents than before and on none + fewer. It still refuses two things PyYAML accepts, both unchanged: a tab-only content line + inside a block scalar (a `|` scalar with a line of six spaces and a tab) and a tag, or an + anchor and a tag, alone on a line whose value starts on the next one (`k:\n !!str`). + [MUT-1] + +### Security + +- **`SqlMacro::register` ran the statements that precede a parse error on DuckDB + 2.0.0 (SQLMACRO-1).** It counted the body's statements with + `duckdb_extract_statements` and refused only a count above one; a body that does + not parse counts as zero and was then executed. DuckDB 1.4.4 and 1.5.6 parse the + whole string before running anything, so nothing ran there; DuckDB 2.0.0 runs + the statements it had already parsed. With the body `1); DROP TABLE canary; --` + (the shape of the module documentation's own example) the table was dropped. + `register` now requires exactly one statement, and a count of zero is an error + that carries the parser's message. A macro body is the extension author's code, + not SQL data (the module documentation says never to build one from untrusted + text), so exploiting this needed an author who passed untrusted text; the guard + the documentation described as a defence failed open on 2.0.0 all the same. + Regression test `hostile_input::a_macro_body_cannot_turn_one_statement_into_several`: + on the 0.19.0 code it fails on 2.0.0 with "Table with name canary does not exist"; + it passes on 1.5.6 and 2.0.0 with the fix. Earlier hostile-input sweeps ran on + 1.5.6 only and could not have seen this (`AUDIT.md` section 14.9 lists the + corrections). + ### Performance - **Nullable columns, string reads and list building cost less in the callback @@ -90,31 +364,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `duckdb_vector_get_validity`) instead of making one C call per row; `DuckStringView::as_str` checks for ASCII before falling back to full UTF-8 validation; `ListBuilder` keeps its child writer between rows (dropped whenever - a reserve grows the child) and writes the parent entries through the entry array. + a reserve grows the child) and writes the parent entries through the entry array, + which it fetches on every call. An earlier version of this change cached the + array's address and crashed with a segmentation fault when a nested list's reserve + grew the parent (found in review before release, never published); the test + `a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves` fails on that + version and passes on this one. Measured with `benches/engine/compare.py`, DuckDB 1.5.6, one thread, release build with fat LTO, a shared 4-vCPU virtual machine ("Intel(R) Xeon(R) Processor @ 2.10GHz"), - 20 million rows, base and new interleaved, 18 timed runs per cell; median ns per - row, with the hand-written twin in brackets: + 20 million rows (10 million for the long strings), base and new interleaved, 18 + timed runs per cell, load average 1.5 at the start; median ns per row, with the + hand-written twin in brackets: | | 0.19.0 | now | |---|---|---| - | `map1` on a column with 10% NULLs | 7.45 (5.95) | 6.08 | - | `map2`, same column | 9.95 (6.20) | 6.75 | - | `map1` on a column with no NULLs (control) | 1.95 (1.80) | 2.00 | - | `map1_str` returning an `i64`, short strings | 33.15 (21.40) | 24.35 | - | the same, 46 to 50-byte strings | 47.30 (39.40) | 42.15 | - | `map1_str` returning a `String` (trim), short strings | 63.52 (41.70) | 59.02 | - | `ListBuilder`, `[x, x+1, x+2]` | 11.40 (5.45) | 7.45 | - | `ListBuilder`, a NULL fourth element | 57.70 (5.25) | 8.60 | - - The same run on DuckDB 2.0.0 (12 runs per cell) gave 7.47 → 5.75, 9.43 → 6.50, - 2.20 → 2.20, 30.90 → 22.85, 49.60 → 46.50, 65.67 → 57.85, 15.60 → 7.87 and - 66.27 → 9.30. Non-NULL data is unchanged. A difference is claimed only where - the old build's fastest run was slower than the new median; that excludes the - 46 to 50-byte string row on both engines and `trim` on 1.5.6, whose runs overlap. - The remaining gap in `trim` is the `String` that `map1_str` makes you return (one - allocation per row); no API change was made for it. `AUDIT.md` section 13 has the - spread of every cell. + | `map1` on a column with 10% NULLs | 7.80 (6.17) | 5.88 | + | `map2`, same column | 10.15 (6.15) | 6.60 | + | `map1` on a column with no NULLs (control) | 2.05 (1.70) | 2.07 | + | `map1_str` returning an `i64`, short strings | 33.55 (21.73) | 24.40 | + | the same, 46 to 50-byte strings | 44.80 (41.75) | 44.15 | + | `map1_str` returning a `String` (trim), short strings | 61.25 (39.68) | 53.83 | + | `ListBuilder`, `[x, x+1, x+2]` | 11.28 (5.03) | 6.88 | + | `ListBuilder`, a NULL fourth element | 60.30 (5.58) | 10.72 | + + The same run on DuckDB 2.0.0 (12 runs per cell, load average 1.1) gave 7.08 → 6.20, + 10.88 → 6.17, 2.20 → 2.27, 38.38 → 26.08, 61.90 → 54.90, 66.20 → 61.00, + 15.38 → 8.15 and 62.05 → 12.85. Non-NULL data is unchanged. A difference is claimed + only where the old build's fastest run was slower than the new median; that excludes + the control and the 46 to 50-byte string row on both engines, and `trim` on 2.0.0, + whose runs overlap. The remaining gap in `trim` is the `String` that `map1_str` makes + you return (one allocation per row); no API change was made for it. `AUDIT.md` + section 14.7 has the spread of every cell. ### Changed @@ -155,13 +435,245 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 crate types" (reproduced; extension-ci-tools' Rust makefile passes nothing for musl, and the platform is opt-in, so the failure only shows after opting in). +- CI: **the Rust toolchain is pinned to 1.99.1** (`rust-toolchain.toml` and + every job's `toolchain:`), so a new Rust release can no longer turn every + `-D warnings` / clippy-pedantic job red with no change in the repository; an + informational `latest-stable` job runs fmt, clippy, rustdoc and the library + tests on the current stable, and `scripts/check-toolchain-pin.py` keeps the + pins consistent. CONTRIBUTING.md describes how to move the pin. [CI-3] +- CI: the release workflow re-packages the `.crate` with the rustc the + `package` job recorded and **compares it with the attested SHA-256 before + uploading it** to crates.io (the existing comparison ran only after the + irreversible upload). [CI-6, HARDEN-11] +- CI: a release commit whose lockfiles still name the previous version now + fails in minutes: `scripts/check-release-metadata.py` (the `doc` job and the + release `validate` job) checks every committed lockfile, the CHANGELOG + heading and links, `CITATION.cff`, the `SECURITY.md` row, README's "latest" + line, `MIN_RELEASE` and every `quack-rs = "0.X"` snippet against + `Cargo.toml`, and the publish dry-run uses `--locked` like the release. + [CI-1, DOCS-1] +- CI: the five upstream-comparison jobs run through `scripts/upstream-guard.sh`: + a missing guard script fails instead of reading as "upstream unreachable", + and on a release tag an unreachable upstream fails instead of being skipped. + [CI-2] +- CI: new blocking jobs `workflow-lint` (actionlint, checksum-pinned) and + `hygiene` (typos, taplo, cargo-machete, checksum-pinned); cargo-deny now also + checks the example and CI-only crates (`deny.ci-crates.toml`), and the OSV + scan includes `benches/engine/Cargo.lock`, which is now committed. [CI-11, + DEPS-4, DEPS-6] +- CI: actions bumped (Dependabot #145, folded in by hand): `Swatinem/rust-cache` + v2.9.3, `rust-lang/crates-io-auth-action` v1.0.6, `dtolnay/rust-toolchain` + master `683b588`; `scripts/check-action-pins.py` fails when the scaffold's + generated workflows (`src/scaffold/templates_ci.rs`) pin a different commit + than this repository's workflows. [DEPS-15] + +- **`parse_description_yml` reads every section the way PyYAML 6 `safe_load` does**, which is + what the community build uses. On the 395 published descriptors (community-extensions + `845779e`) it agrees with PyYAML 6.0.3 on every one, and on 118,500 random mutants of them + (three seeds) it never accepts a file PyYAML refuses and never panics. Tabs, flow collections, + anchors and aliases, quoted and block scalars, document markers, line breaks (NEL, LS and + PS included) and nesting are handled as PyYAML handles them. The parse runs in linear time: + hostile inputs that took 3.8 to 21.5 s (and over 100 s for a comment run) now take at most + 330 ms. (CONFORM-1, HARDEN-1, HARDEN-2) +- **Scaffold: the example function is a native scalar function**, written with the typed + builder as a one-line closure (`ScalarFunctionBuilder::map1_str`), instead of a + `CREATE MACRO`. A macro is written into the user's database, so `LOAD` failed on a + read-only one. +- **Scaffold: the generated `.test` file** now: + - carries the upstream header lines (`# name:`, `# description:`, `# group:`); + - checks that the function is missing before `require`; + - reads its NULL from a column, since a literal NULL is constant-folded before the function + is called. +- **Scaffold: `make test` fails when the SQLLogicTest runner skips a test file or runs none.** + The runner reports an unmet `require` as `SKIPPED` and exits 0, so a project whose + extension name, `[lib]` name and test file had drifted apart passed having tested nothing. +- **Scaffold: the extension version written into the metadata footer** is now: + - the first tag on HEAD, if it fits the 32-byte field and holds only `A-Za-z0-9._+-`; + - otherwise the short commit hash; + - before the first commit, the `Cargo.toml` version. + + extension-ci-tools wrote every tag (one per line), or git's error message, and DuckDB 2.0 + refused the result. +- **Scaffold: generated `Cargo.toml`.** The `libduckdb-sys` requirement is + `>=1.4.4, <1.20100` with `default-features = false` (it was `<2`, which admits the untested + DuckDB 2.x bindings). With `--target-duckdb-version v1.5.6` (C API v1.5.6), or for an + unstable DuckDB 1.5 target, it enables the `duckdb-1-5*` feature its comments describe. + Before, it enabled none, and the author had to add it. (DEPS-2/3, CONFORM-14) +- **Scaffold: generated `extension-ci.yml`:** + - sets `permissions: contents: read`; + - checks out with `persist-credentials: false`; + - pins `Swatinem/rust-cache` v2.9.3 (`d1527c8f…`) and `dtolnay/rust-toolchain` master + 2026-10-15 (`683b588c…`). + `zizmor --offline` reports nothing for it. +- **Scaffold: a license may be any SPDX expression.** An identifier outside the shortlist + (`BUSL-1.1`, `CC0-1.0`) is a warning instead of an error. A misspelled case of a listed + one (`mit`) or a character SPDX does not allow is still an error. (CONFORM-14) +- **Generated files and `append_metadata` link to the book's pitfalls page** for P2, P4, P5 + and L11, instead of `LESSONS.md`, which the published crate does not contain. (DOCS-8) +- **hello-ext: `gen_series_v2`'s callbacks are written with + `table_bind_callback!` / `table_init_callback!` / `table_scan_callback!`.** A value past + BIGINT is now a query error instead of a wrapped number. New check T30. (FFI-9) + +- **Breaking:** `generate_scaffold` refuses configurations it used to accept: a name DuckDB + uses (an alias such as `s3` or a core extension such as `json`); a platform group, a + retired platform, or a name the community build does not build, in `excluded_platforms`; + a version component above `u64::MAX`; a license with characters SPDX does not allow, or a + listed id in the wrong case. `parse_description_yml` is stricter where PyYAML is (it + rejects files the community build cannot read) and rejects a reserved extension name. + (CONFORM-1, -3, HARDEN-5) +- **Behaviour change:** `WarningCollector` keeps at most 10,000 warnings between drains + (`DEFAULT_MAX_WARNINGS`; `with_limit` for another cap). Warnings past the cap are counted + in `dropped()` instead of stored, and `drain`/`clear` reset the count. Before, every + warning was stored, and a collector never drained grew without bound (2,000,000 + warnings were 381 MB) (HARDEN-8). +- **Behaviour change:** the duplicate-overload guard of `ScalarFunctionSetBuilder` and + `AggregateFunctionSetBuilder`, and the catalog collision check of scalar registrations, + compare types the way DuckDB's overload resolution does: a `DECIMAL`, `MAP` or `ENUM` + parameter by type alone (also inside `LIST`, `ARRAY`, `STRUCT`, `UNION`), and named + `STRUCT` fields / `UNION` members by name in any order and case. Sets such as + `f(DECIMAL(4,1))` + `f(DECIMAL(18,3))`, and a scalar `round(DECIMAL(18,3))` beside the + built-in `round(DECIMAL)`, which registered and then made every such call fail with + "Could not choose a best candidate function", are now refused at registration with the + reason. `LIST(INTEGER)`/`LIST(VARCHAR)`, `INTEGER[3]`/`INTEGER[4]` and other pairs DuckDB + resolves still register (AUTHT2-1). +- **Behaviour change:** `data_chunk_from_arrow` refuses, with `InvalidInput`, Arrow arrays + that break bounds DuckDB takes on trust and reads past later: a list, large-list or map + offset that is negative, decreases or lies past the child; a negative list-view offset + or size; a struct field, list or fixed-size-list child, or union member shorter than the + rows its parent reads; a dictionary index outside the dictionary (in any row the indices' + own validity does not mark NULL); dictionary indices of a non-integer type. Such arrays + violate the Arrow specification; before, they imported and reading the chunk was an + out-of-bounds read (SIGSEGV on 1.5.6 and 2.0.0 for a list offset past its child) (FFI-1). +- `BindInfo::set_cardinality` clamps the hint to `i64::MAX`: above it any join with the + function failed with "INTERNAL Error: Information loss on integer cast" (1.5.6) (HARDEN-7). +- A failed aggregate registration under a taken name now says which catalog entry holds + the name (scalar function, aggregate or macro) and what to do, instead of the generic + "the C API reports no reason" hint; the catalog is consulted only after the failure + (FFI-8). +- `ScalarBindData::set` called twice through the same `ScalarBindInfo` frees the value it + replaces; it used to be leaked (FFI-12). +- The DuckDB-before-1.5.0 "cannot add an overload to an existing name" error names + `overload_typed` beside `overload` (AUTHT2-2). + +- **Behaviour change:** the entry point runs the registration closure in one transaction on + the registration connection: `BEGIN TRANSACTION`, then `COMMIT` when the closure returns + `Ok`, `ROLLBACK` when it returns `Err` or panics. A `LOAD` that fails part-way now leaves + no catalog entry behind (scalar, aggregate and table functions, macros, types, copy + functions, tables the closure created) and can be retried in the same database; before, + everything registered up to the failure stayed and every retry failed at the first name + it registered again ("... already exists"). Validated with a real `LOAD` on DuckDB 1.4.4, + 1.5.6 and 2.0.0. **Not rolled back**, because DuckDB keeps them in the database + configuration: casts, replacement scans and configuration options — register those last. + Inside the closure: a failed statement can abort the transaction (DuckDB 2.0 on every + failed statement), after which later statements fail with "Current transaction is + aborted", so return the first error; do not run `BEGIN`/`COMMIT`/`ROLLBACK` on the + registration connection; what the closure creates is visible to other connections + (including one it opened itself) only once the `LOAD` commits. A `LOAD` issued inside the + host's own transaction does not wait on it and is not undone by the host's `ROLLBACK` + (as before). Cost: one `BEGIN`/`COMMIT` pair, about 0.3 ms; no measurable change in + `LOAD` time. `Registration::Autocommit` restores the old behaviour (FFI-2). +- `CopyFunctionBuilder::register` on the entry point's registration connection runs its + format-name probe (a statement that fails on purpose) on a short-lived connection of its + own and checks formats registered earlier in the same transaction from the entry point's + own record. On DuckDB 2.0, which aborts a transaction on any failed statement, the probe + otherwise made the registration fail with "duckdb_register_copy_function failed for + ''" and aborted the transaction. On a connection of yours inside a transaction you + opened, the probe still runs on that connection (documented) (FFI-3). +- The entry-point macros emit `#[unsafe(no_mangle)]` instead of `#[no_mangle]`. Identical + today on every edition (Rust 1.82+); a bare `#[no_mangle]` in an exported macro would + fail every extension that expands it once quack-rs moves to edition 2024, and + `cargo fix --edition` does not rewrite macro bodies (DEPS-12). +- docs.rs labels every feature-gated item "Available on crate feature `duckdb-1-5` only" + (`#![cfg_attr(docsrs, feature(doc_cfg))]`) (DOCS-5, DEPS-7). +- Docs: the C API struct over-read by `libduckdb-sys` before the ABI guard runs is stated + precisely (87 pointers past the engine's struct on 1.4.x for a 546-slot build) (FFI-4); + the panics no guard can catch (a panic during an unwind, a panicking panic hook, out of + memory) (FFI-10); what the catalog collision scans cost on a large catalog (~0.1 s per + scan with 20,000 user macros; narrowing the query does not help) (FFI-7); what a + `duckdb-1-5` build does on DuckDB v2.0.1 (refused under `AbiPolicy::Strict` until quack-rs + lists it or the build sets `QUACK_RS_TARGET_DUCKDB_VERSION`). + +- **Breaking:** `entry_point!` and `entry_point_v2!` refuse to compile an extension built + with `panic = "abort"` (`[profile.*] panic = "abort"` or `RUSTFLAGS=-C panic=abort`), + with an error that names the fix: under `panic = "abort"` none of quack-rs's + `catch_unwind` guards work and any panic aborts the DuckDB process. Build with + `panic = "unwind"` (what the scaffold and hello-ext already use). The check runs in the + extension's crate; a hand-written entry point around `init_extension_v2` is not + checked. `wasm32-unknown-emscripten` defaults to unwinding and is unaffected (FFI-5). +- **Breaking:** `abi::AbiCheck` is `#[non_exhaustive]`: a `match` on it outside quack-rs + needs a wildcard arm (use `is_compatible` / `error_message` to cover every outcome) + (DEPS-13). +- **Breaking:** `abi::AbiPolicy` is `#[non_exhaustive]`: a `match` on it outside quack-rs + needs a wildcard arm. Passing a policy is unchanged (DEPS-13). +- **Breaking:** `config_option::ConfigOptionScope` is `#[non_exhaustive]` (DEPS-13). +- **Breaking:** `tls::TlsVersion` is `#[non_exhaustive]`; comparisons and ordering are + unchanged (DEPS-13). +- **Breaking (found by `cargo semver-checks` against v0.19.0):** `ScalarBindInfo` + (`duckdb-1-5`) is no longer `RefUnwindSafe`, because it now remembers the bind data it + stored so a second `set` can free the first (FFI-12). Code that holds a `&ScalarBindInfo` + across its own `catch_unwind` needs `AssertUnwindSafe`; quack-rs's guards already do. The + same run reports no other breaking change, and lists the four `#[non_exhaustive]` enums + above. +- **Breaking (behaviour):** see the transactional-registration entry under Changed — an + extension whose closure relied on carrying on after a failed registration, or that runs + `BEGIN`/`COMMIT` on the registration connection itself, needs + `options = EntryPointOptions::new().registration(Registration::Autocommit)` (FFI-2). + +- A `BIT` value whose bytes are malformed (only the raw `duckdb_create_bit` + builds one) is now refused by `Value::as_str`, `display_string` and `Debug` + (`UNRENDERABLE`) instead of being rendered through an out-of-bounds write + inside `DuckDB`; the `UNRENDERABLE` text names it. (APIGAPS-3) +- `QueryProgress` / `InterruptHandle::progress` document why `DuckDB` 1.x reports + `-1.0` for whole queries (any pipeline without progress keeps it there). + (AUTHT-MEAS) +- `scripts/api-coverage-allowlist.txt`: 11 candidate slots are now wrapped; + `duckdb_param_type` / `duckdb_prepared_statement_column_type` are listed as + superseded; the log-storage rows carry the upstream double-free caveat and the + 1.5.2 slot move (APIGAPS-1, APIGAPS-11). +- **Dependency ceiling.** The `libduckdb-sys` and `duckdb` requirements are + `>=1.4.4, <1.20100` (they were `<2`) in the crate and in every manifest in the repository. + The 1.20000.x bindings are the DuckDB 2.0 bindings; the next ones (1.20100.x, DuckDB 2.1) + are not selected until a quack-rs release widens the range, because `<2` also admitted + every release up to 1.99999, whose C API layout nobody had checked. No action for + dependants already on 1.10xxx or 1.20000.x; one who needs a later release must wait for + quack-rs or pin. CONTRIBUTING.md has the procedure for widening the range, and a test fails + if a manifest states a different one. (DEPS-2) +- `cargo package` selects files with an allow-list (`include`) instead of an `exclude` + list, so a new top-level directory is not published by default. (DEPS-7) +- The build script of `bundled-test` / `bundled-test-prebuilt` fails in seconds, naming + the directory, when `libduckdb-sys` reports an include directory that has no + `duckdb.hpp` (a partial build cache); it used to poll for ten minutes before panicking. +- CI: runners are pinned to `ubuntu-24.04`, `macos-26` and `windows-2025` (the images + `-latest` named on 2026-10-30), and `scripts/check-action-pins.py` fails on a floating + label in any workflow. [CI-12] +- CI: the upstream-comparison scripts return 2 ("could not check", a warning) only for a + network failure, an HTTP 5xx or a 429; an HTTP 4xx, an entry without the expected key, + a header the parser cannot read, or every release answering 404 returns 1. Before, a moved + or reshaped upstream file turned each of these jobs into a skipped check for good + (reproduced for the platform list with a nonexistent ref). `scripts/upstream-guard.sh` also + fails a 2 on the weekly schedule, so a gate that has not run for a week is noticed. [TESTS-10] +- CI: the Linux leg of the `test` job installs `python3-yaml`, `make` and `git` and sets + `QUACK_REQUIRE_TOOLS=1`, which turns the scaffold tests' "tool missing, skipping" into a + failure (`cargo test` hides the note, so a runner without PyYAML kept them green while they + checked nothing). macOS and Windows keep the skip. [TESTS-14] +- `.cargo/mutants.toml` no longer excludes `src/callback.rs`: the macros in it are invisible + to `cargo-mutants`, but its four plain functions are mutated (5 mutants, 4 caught, 1 + unviable). CONTRIBUTING.md now says what the mutation gate cannot see: `unsafe fn` bodies + (506 functions, 22.4% of non-test lines by a heuristic count), integer literals and macro + bodies. [TESTS-2, TESTS-11] +- CI: the incremental mutation job runs as eight shards (`cargo mutants --shard k/8 + --sharding round-robin`) and a gate job named "Mutation Testing (incremental)". One job did + the 1,639 mutants of this release's 102 changed files and was stopped by its runner after 33 + and 55 minutes; the eight shards took 22 minutes together. The Miri job's limit is 90 + minutes (the library tests had grown to 41.6 minutes for the native run alone, 18 of them + one parser test on 100,000-level inputs, which now uses 1,000 under Miri). [MUT-2] + ### Documentation - `LESSONS.md` and the book's pitfalls page gain P13 (a `VARIANT` in a signature crashes DuckDB), P14 (an aggregate with no parameter counts no rows on DuckDB 2.0) and P15 (the entry of a NULL `LIST` or `MAP` row holds stale bytes): the three - engine behaviours this release guards against. The pitfall count is now 34 - (L1–L19, P1–P15) wherever it is stated. + engine behaviours this release guards against (four more follow below). - README: the Quick Start manifest now sets `rust-version` and `resolver = "3"` (edition 2021's default resolver ignores `rust-version`, so Rust 1.86 or 1.87 selected the 2.0 bindings, which need 1.88, and failed); the example uses @@ -213,6 +725,166 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stamp and the enabled features; the pull request template lists the doctest and changelog-sync steps. +- `SECURITY.md`: supported versions for 0.20.0 (0.20.x and 0.19.x; 0.18.x + end-of-life), the two-newest-lines policy, and what releases before 0.16.0 + carry, with version ranges checked against the published crates; no more + rows for 0.1.x / 0.2.x, which were never published. [DOCS-13, HARDEN-10] +- `RELEASING.md`: the release commit updates every lockfile (`cargo update + --workspace` in five directories) because the release packages with + `--locked`; the hotfix branch is merged into `main` with a merge commit and + its tip tagged (the old "tag, then cherry-pick" flow was refused by + `validate`); the crates.io fallback token belongs in the `crates-io` + environment, with a Trusted Publishing set-up order; several statements + corrected to what the workflow does. [DOCS-1, CI-1, CI-5, CI-15, HARDEN-9] +- `CONTRIBUTING.md`: what the sanitizer and valgrind runs cannot see (the + release libduckdb allocates vector data buffers and LIST entry arrays with + its own jemalloc, invisible to valgrind and AddressSanitizer), the toolchain + pin policy, and DuckDB 2.0's CLI "agent mode" (`AI_AGENT` / `CLAUDECODE`; + use `-csv`/`-json`, or `-no-agent` on 2.0 only). [LASTDIFF-3, CI-3] + +- **"Your First Extension" teaches guarded callbacks.** The page walked through + `hello-ext`'s unguarded `unsafe extern "C" fn`s (a panic in one aborts the DuckDB + process) and its `word_count` aggregate, which segfaults under `OVER ()` on DuckDB + 1.4.4 and 1.5.x, with no warning. It now builds `word_count` with the + `aggregate_*_callback!` macros and `first_word` with `ScalarFunctionBuilder::map1_str` + (and shows the `scalar_callback!` form), opens with a warning that lists the crashing + query shapes, and prints output that was produced by building the page's own code and + loading it into DuckDB 1.4.4, 1.5.6 and 2.0.0 (byte-identical). The raw scalar examples in + `scalar.md` use `scalar_callback!` and report overflow with `checked_mul` / `checked_add` + instead of wrapping (DOCS-4). +- **The window shapes that crash a C API aggregate are listed as measured.** On DuckDB 1.4.4 + and 1.5.6 an explicit `ROWS`/`RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` + frame (with or without `PARTITION BY`) crashes like `OVER ()` and `agg(x ORDER BY y)` do; + `OVER (PARTITION BY p)` without an `ORDER BY` crashes on 1.4.4 but not on 1.5.6; every + shape works on 2.0.0 (one process per shape, 5,000 and 300,000 rows). Pitfall L11, + `known-limitations.md`, the aggregate pages and the tutorial say so (AUTHOR-TYPES-4). +- **`std::fs` bypasses `SET enable_external_access = false` and `allowed_directories`** + (AUTHT-1, measured on DuckDB 1.5.6 and 2.0.0: `read_csv` and quack-rs's `FileSystem` get a + Permission Error, a table function using `std::fs::read` returns the file). Stated in the + Virtual File System chapter (new section), the table-function, replacement-scan and + `COPY` chapters, Known Limitations, the FAQ, the publishing checklist and a new + pitfall, P17. +- **Four new pitfalls, 38 in all** (L1-L21, P1-P17): L20 `duckdb_register_log_storage` + frees its extra data twice when the name is taken (DuckDB 1.5.0 to `main`; quack-rs does not + wrap log storage because of it), L21 a `ChunkWriter` row that is claimed but not written is + published with stale contents, P16 `duckdb_get_bit` / `duckdb_get_bignum` abort the process + on NULL and on values that do not cast, P17 `std::fs` (above). `LESSONS.md` L3 no longer + says a raw callback is "made impossible" to panic in (it is not guarded); the six + `LESSONS.md` titles that differed from the book's (P2, P3, P5, P7, P9, P12) match it, and + every place that states the pitfall count says 38 (DOCS-2). +- **`known-limitations.md` is grouped** (what your extension must get right / DuckDB defects / + what the C API does not offer / DuckDB 2.0 changes / resolved) instead of interleaving + resolved history with current limits (DOCS-12), with new entries: no table-in-out functions, + progress callbacks, filter pushdown, logging or secret access in the v1 C API; a quack-rs + extension cannot be linked statically into a DuckDB 2.0 host; five DuckDB 2.0 behaviour + changes that reach C API extensions; panics that cannot be caught (double panic, a panicking + hook, an allocation failure). The C API v2 list no longer cites a function that does not + exist (`duckdb_v2_table_function_set_filter_pushdown` is `..._callback`) and covers the v2 + capabilities an author of the wrapped function kinds would look for, each name grepped in + the 2.0.0 headers (APIGAPS-6, -7, -10). Table function arguments may be constant + expressions, not only literals (AUTHT-10). +- **Corrections found by running the book against DuckDB 1.4.4, 1.5.6 and 2.0.0.** The scalar + bind callback cannot set the return type (the v1 C API has no such function); a `DECIMAL` + parameter rounds its argument and DECIMAL/MAP/ENUM overloads cannot be told apart; a type + registered with `set_alias` is not a `CREATE TYPE` synonym (`sum`, `abs` and `+` do not bind + on it); no callback API accepts an `ErrorData`; a table function's `bind` runs again on every + `EXECUTE` on DuckDB 2.0.0 (4 binds against 1 for a prepared statement executed three times); + the typed table-function builder scans on one thread; without `duckdb-1-5` a zero-column typed + bind is an `INTERNAL Error`; the table-function example no longer hard-codes 2048; a + `COPY ... TO` format cannot declare options (unknown names are accepted silently), sees no + column names, and is chosen from the file extension only when that equals its name; Arrow + import refuses a record batch with a root `offset`; `SET threads` from a callback can abort + the process; a prepared-statement parameter is not foldable at bind time; a SQLLogicTest file + whose `require` fails is reported `SKIPPED` and exits 0. On DuckDB 2.0.0 an + `INTERVAL '1 month' - INTERVAL '1 day'` compares equal to `INTERVAL '29 days'`, a bare + identifier as a table macro argument warns (write `'users'`), `VerifyNullHandling` is compiled + into release builds and runs after `SET debug_verification_mode = 'VERIFY_FUNCTIONS'`, and + `arg_max`'s third parameter is `BIGINT` (DOCS-3, AUTHOR-TYPES-3, -5, -6, -7, -13, AUTHT-4 to -12). +- **Dependency requirement.** Every snippet and sentence shows `libduckdb-sys` as + `>=1.4.4, <1.20100` with `default-features = false`, `quack-rs = "0.20"`, and Installation + explains why the range has an upper bound (the 1.20000 bindings keep the C API names at their + crate root through a re-export that upstream marks for removal), what to do when the DuckDB 2.1 + bindings are published (nothing: Cargo stays on 1.20000.x until quack-rs widens the range), + what `default-features = false` saves (65 crates against 30 with the 1.20000 bindings, + measured with `cargo tree`; 74 against 72 with 1.10506), and the build requirements of each + part (a C compiler may be needed by the bindings' build dependencies, a C++ toolchain for + `bundled-test`, Python 3, git and network access for the community flow). The README no + longer says "no C, no C++, no glue code" without saying what that covers (DEPS-2, DEPS-11). +- **README front page.** Code lines are at most 64 columns (crates.io clips at about that + width), a link row, a maintenance bullet, a "Getting help" section, a pointer to + `examples/hello-ext` with the warning that it is not a template, a "use 0.18.0 or later" + notice, a shorter comparison table that is no longer cut off, and DuckDB support stated as + 1.4.4 through 2.0.x everywhere (DOCS-9, DOCS-22). +- **A `duckdb-1-5*` extension loads only into DuckDB releases whose layout quack-rs has verified** + (README "Supported versions", the ABI chapter, FAQ, Installation, Anatomy, a new Known Limitations + entry). "DuckDB 1.4.4 through 2.0.x" now describes the default features only; an extension that + enables `duckdb-1-5*` loads into 1.5.6 and 2.0.0, and under the default `AbiPolicy::Strict` refuses any + release quack-rs has not listed (2.0.1, for example) with an error that names the fixes: build with + `QUACK_RS_TARGET_DUCKDB_VERSION` set to the release (after checking its header), upgrade quack-rs, or stay + on the default features. The evidence is stated: DuckDB 2.0.0's `duckdb_extension.h` opens with "this API is + not yet stable ... only guaranteed to work for this specific DuckDB version", and 2.0.0 put + `duckdb_create_timestamp_tz_ns` and `duckdb_get_timestamp_tz_ns` into the range that 1.5.6 declared stable + (checked against the 2.0.0 header and the 1.5.6 header). The ABI chapter also says that `libduckdb-sys` + reads every slot of its 546- or 548-slot struct before the guard runs, which is 87 pointers past DuckDB + 1.4.x's smaller struct on every `LOAD` (read from the sources, not measured) (FFI-4). +- **Registration is transactional, `panic = "abort"` does not compile, four enums are `#[non_exhaustive]`** + (The Entry Point chapter, Anatomy, Installation, Panic-Safe Callbacks, Pitfall L3, Known Limitations, the + catalog chapter, and the upgrade guide). The sentence "Registration is not transactional: functions + registered before a failure stay registered" is replaced by what the entry point now does (FFI-2, FFI-5, + FFI-10, DEPS-13): one transaction, what is not rolled back (casts, replacement scans, configuration + options: register them last), what a failed statement inside the closure does, and how to get the old + behaviour (`Registration::Autocommit`). +- **Pitfall P16 is now "prevented in quack-rs"** (`Value::as_bit` and `as_bignum` return `None` instead of + reaching the aborting C getters), and the book covers the new `Value` readers (`try_cast`, `array_items`, + `union_tag`), typed `COPY TO` state and typed extra info, the `Send` `FileHandle`, the catalog-scan cost of a + large database (FFI-7), and the `-1.0` progress of DuckDB 1.x (APIGAPS-2, -3, -8, AUTHT-2, -3, -MEAS). +- **`duckdb_query` runs the statements before a parse error on DuckDB 2.0.0** (Known Limitations, Running SQL, + SQL Macros, the upgrade guide). Measured with a C program on 1.4.4, 1.5.6 and 2.0.0: for the text + `CREATE OR REPLACE MACRO "m"() AS (1); DROP TABLE canary; SELECT ()`, 1.4.4 and 1.5.6 report a parser error and + run nothing; 2.0.0 ran the `DROP TABLE` and returned success. An extension that passes text it did not write to + `duckdb_query` must check that it parses as the expected statement count first (`SqlMacro::register` now + requires exactly one). +- **The Project Scaffold chapter and the publishing page follow the new scaffold**: the `quack_scaffold` command (flags, + `--force`, the printed next steps), `generate_scaffold_with_warnings`, the new generated files (`README.md`, `LICENSE` + files, `deny.toml`, `docs/function_descriptions.csv`), the native example function and `.test` layout, the + `make test` skip check, the extension-version choice, the hardened `extension-ci.yml`, what `generate_scaffold` now + refuses, and the `Cargo.toml` it writes (`<1.20100`, `default-features = false`, the `duckdb-1-5*` feature for a + `v1.5.6` project). +- **Community-extension publishing** (`publishing.md`): the names DuckDB reserves (ten aliases and + the 34 core names; `INSTALL s3 FROM community` fails on 2.0.0; the scaffold and `parse_description_yml` now + refuse them), what the description.yml reader checks (PyYAML's rules, `variegata` and `andium`), + `function_descriptions.csv` + (lives in the community-extensions repository, replaces the generated text), the four revision + keys `ref`, `ref_next`, `variegata` and `andium` with `build.py`'s rule and what a project that + uses the unstable C API must do, which version DuckDB shows (the tag on `HEAD`, else the short + hash; `description.yml`'s `version` is not read; tags over 32 bytes or two tags on one commit + break the footer), the keys the build reads (anything else is ignored), reduced CI on pull + requests, and not running `strip` on a stamped extension. The page's `description.yml` and + `Cargo.toml` examples match the scaffold's output, and the statements that had drifted from + upstream (the `v1.5.6` branch is not the `v1.5-variegata` commit, the matrix script differs + per `extension-ci-tools` branch, the version is not from `git describe`) are corrected + (CONFORM-3, -7, -8, -9, -11, -14). +- **New page "Upgrading from 0.18 to 0.20"** (Installation > Upgrading): every behaviour change + of 0.19.0 and 0.20.0 that can fail a `LOAD` or change a result, with what to do. +- `scaffold.md` says the scaffold example is in the repository, not the published crate, lists + the prerequisites of `make configure`, and warns that `make` ignores `CARGO_TARGET_DIR`; the + Testing Guide explains the DuckDB 2.0 CLI's agent mode (`-no-agent`) and reads the NULL test + case from a column. `GOVERNANCE.md` says there is one maintainer. The publishing page no longer + prints `..._distribution.yml@v2.0-cyanoptera`, which Cloudflare's email obfuscation rewrote to + `[email protected]` on the live site (DOCS-6). `docs/E2E_DOGFOOD_REPORT.md` and + `docs/duckdb-v1.5.1-evaluation.md` moved to `docs/archive/`. + +### Dependencies + +- `Cargo.lock`: `proptest` 1.12.0 (a dev-dependency) and `cc` 1.10.0, the updates Dependabot + proposed in #148 (`cc` 1.9.0 there); `fuzz/Cargo.lock` and `book/doctest/Cargo.lock`: `cc` + 1.10.0 (#149). `examples/hello-ext/Cargo.lock`: `libduckdb-sys` 1.20000.1, so the example + builds against the DuckDB 2.0 bindings (#143 asked for 1.20000.0, whose bundled engine + reports itself as `v2.0.0-dev88614`, which the ABI check refuses). The main lockfile stays + on the 1.5.6 bindings (1.10506.1). `benches/engine/Cargo.lock` is committed, so the + benchmark crate's dependencies are scanned and reproducible. +- GitHub Actions bumps from Dependabot #145 are folded in (listed under Changed). + ## [0.19.0] - 2026-10-22 This section comes out of a sixth review (`AUDIT.md` section 11): seven @@ -3135,7 +3807,7 @@ is not grouped. `-D warnings` makes it an error. `testing::InMemoryDb` was a 59th, only visible once the lint ran with `bundled-test` on. -[C-DEBUG]: https://rust-lang.github.io/api-guidelines/debugging.html +[C-DEBUG]: https://rust-lang.github.io/api-guidelines/debuggability.html#c-debug ### Changed @@ -4215,6 +4887,9 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. ## [0.2.0] - 2026-03-07 +0.1.0 and 0.2.0 were never tagged or published: the oldest git tag and the oldest +crates.io release are 0.3.0, so these two headings have no compare link. + ### Added - **`validate::description_yml` module** — parse and validate a complete `description.yml` @@ -4314,7 +4989,8 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. - CI pipeline: check, test, clippy, fmt, doc, MSRV, bench-compile - `SECURITY.md` vulnerability disclosure policy -[Unreleased]: https://github.com/tomtom215/quack-rs/compare/v0.19.0...HEAD +[Unreleased]: https://github.com/tomtom215/quack-rs/compare/v0.20.0...HEAD +[0.20.0]: https://github.com/tomtom215/quack-rs/compare/v0.19.0...v0.20.0 [0.19.0]: https://github.com/tomtom215/quack-rs/compare/v0.18.0...v0.19.0 [0.18.0]: https://github.com/tomtom215/quack-rs/compare/v0.16.0...v0.18.0 [0.16.0]: https://github.com/tomtom215/quack-rs/compare/v0.15.0...v0.16.0 @@ -4333,6 +5009,4 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. [0.5.1]: https://github.com/tomtom215/quack-rs/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/tomtom215/quack-rs/compare/v0.4.0...v0.5.0 [0.4.0]: https://github.com/tomtom215/quack-rs/compare/v0.3.0...v0.4.0 -[0.3.0]: https://github.com/tomtom215/quack-rs/compare/v0.2.0...v0.3.0 -[0.2.0]: https://github.com/tomtom215/quack-rs/compare/v0.1.0...v0.2.0 -[0.1.0]: https://github.com/tomtom215/quack-rs/releases/tag/v0.1.0 +[0.3.0]: https://github.com/tomtom215/quack-rs/releases/tag/v0.3.0 diff --git a/CITATION.cff b/CITATION.cff index d577dfb4..2a676191 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,8 +9,8 @@ authors: website: "https://github.com/tomtom215" title: "quack-rs" abstract: "Rust SDK for building DuckDB loadable extensions on the DuckDB C Extension API." -version: 0.19.0 -date-released: 2026-10-22 +version: 0.20.0 +date-released: 2026-10-30 license: MIT repository-code: "https://github.com/tomtom215/quack-rs" url: "https://quack-rs.com" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb1bc2a3..e21fd594 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,19 +27,31 @@ Everyone taking part is expected to follow the | Tool | Version | Purpose | |------|---------|---------| -| Rust | stable (`rust-toolchain.toml`) | Compiler | +| Rust | the exact release `rust-toolchain.toml` pins (1.99.1); rustup installs it on first use | Compiler (see [The Rust toolchain](#the-rust-toolchain)) | | Rust 1.86.0 toolchain | `rustup toolchain install 1.86.0` | MSRV check (gate 6) | -| `rustfmt`, `clippy` | stable | Formatting and linting (gates 3, 4) | -| Python 3 | any current | `scripts/*.py`, including the live extension check (gates 5b, 7) | +| `rustfmt`, `clippy` | the pinned release (`rust-toolchain.toml` lists both components) | Formatting and linting (gates 3, 4) | +| Python 3 | 3.11 or later (two scripts use `tomllib`) | `scripts/*.py`, including the live extension check (gates 5b, 7) | | DuckDB CLI | 1.4.4 or later; CI also uses 2.0.0 and the latest (currently 2.0.0) | Live extension testing (gate 7, required) | | libduckdb shared library | the release `Cargo.lock` pins (currently 1.5.6) | End-to-end tests and book doctests (gate 2c) | | `cargo-deny` | 0.20.2 (the version CI installs) | License and advisory checks (gate 8) | | `cargo-mutants` | 27.1.0 (the version CI installs) | Mutation testing (gate 9) | | `mdbook` | the version in `book/mdbook-version` | Building the book (only for book changes) | +| `typos`, `taplo`, `cargo-machete`, `actionlint` | 1.46.0, 0.10.0, 0.9.2, 1.7.12 (the release binaries CI downloads) | Spelling, TOML style, unused dependencies, workflow lint (gate 8b; optional locally) | Install the Rust toolchain via [rustup](https://rustup.rs/). `cargo-msrv` is not needed: the MSRV is checked by compiling with the pinned 1.86.0 toolchain. +**DuckDB 2.0's CLI changes its output when it thinks an AI agent is driving +it.** With `AI_AGENT` or `CLAUDECODE` set in the environment and stdout not a +terminal, the 2.0.0 CLI prints `duckdb agent mode on: ...` and renders results +as a Markdown table with column types (`| answer:INTEGER |`) instead of its +default box layout. Anything that parses the CLI's output must choose a +format: `-csv` or `-json` (both work on every DuckDB, and override agent +mode), or `-no-agent`, which turns it off but exists only in the 2.0 CLI +(1.5.6 exits with "Unrecognized option '-no-agent'"). The scripts here that +read query results (`scripts/check-hello-ext.py`, the CI probes) pass `-csv`; +GitHub's runners set neither variable. + Install DuckDB with `scripts/fetch-duckdb.sh` (no system package manager needed; the script fetches the linux-amd64 build). CI's `extension-load` job exercises **v1.4.4, v1.5.0, v1.5.5, v1.5.6, v2.0.0 and `latest`** — the floor, the 1.5 floor, the newest 1.5 releases, the @@ -187,9 +199,9 @@ scripts/sync-book-changelog.py --check # 6. MSRV — must compile on Rust 1.86.0 [msrv]. Benches are excluded (criterion # needs a newer compiler). `+1.86.0` is required, not stylistic: -# `rust-toolchain.toml` pins `channel = "stable"` and a toolchain file -# overrides rustup's default, so a bare `cargo check` silently runs stable -# and checks nothing. +# `rust-toolchain.toml` pins a newer release (`channel`) and a toolchain +# file overrides rustup's default, so a bare `cargo check` silently runs +# the pinned release and checks nothing. # # Against what a 1.86.0 user's Cargo resolves, not the committed Cargo.lock: # the lock carries `libduckdb-sys` 1.10506.1, which declares rust-version @@ -202,7 +214,7 @@ cargo +1.86.0 check cargo +1.86.0 check --features duckdb-1-5-4 # 7. Live extension test — build hello-ext, append the metadata footer, then -# run every documented SQL check (examples/hello-ext/sql_checks.txt, 31 +# run every documented SQL check (examples/hello-ext/sql_checks.txt, 32 # statements) in the DuckDB CLI. [extension-load: DuckDB 1.4.4, 1.5.0, # 1.5.5, 1.5.6, 2.0.0 and the latest release] cargo build --release --manifest-path examples/hello-ext/Cargo.toml @@ -213,14 +225,27 @@ cargo run --bin append_metadata -- \ --duckdb-version v1.2.0 --platform linux_amd64 python3 scripts/check-hello-ext.py --duckdb /tmp/duckdb \ --extension /tmp/hello_ext.duckdb_extension -# → 31/31 checks pass with the extension loaded +# → 32/32 checks pass with the extension loaded # 7b. The example extension's own tests and lints [example-check] cargo test --manifest-path examples/hello-ext/Cargo.toml cargo clippy --manifest-path examples/hello-ext/Cargo.toml --all-targets -- -D warnings -# 8. Supply chain — license policy and advisories [security] +# 8. Supply chain — license policy and advisories [security]. The example and +# CI-only crates have their own policy file (deny.toml's license rules +# cannot hold for them; see its header). cargo deny check +for m in examples/hello-ext fuzz book/doctest; do + cargo deny --manifest-path "$m/Cargo.toml" --config deny.ci-crates.toml check +done + +# 8b. Hygiene and workflow lint [hygiene, workflow-lint]: spelling (typos.toml), +# TOML style (.taplo.toml), unused dependencies, and the workflow files +# (actionlint runs shellcheck on every `run:` block when it is installed). +typos +taplo fmt --check && taplo lint +cargo machete --with-metadata +actionlint ``` ```bash @@ -230,13 +255,47 @@ cargo mutants --file ``` CI sets `RUSTFLAGS="-D warnings"` for every job, so a compiler warning anywhere -fails the build. CI runs these checks on every push and pull request, together with jobs that +fails the build. CI runs these checks on every pull request to `main`, on pushes to +`main` and `claude/**`, weekly, and as the release's gate, together with jobs that have no single local command: the other test operating systems, older DuckDB engines (`test-older-engines`), WASM, Miri, sanitizers, semver checks, the scaffold and ABI-guard end-to-end tests, the book build and its link check, and several consistency scripts under `scripts/`. Coverage and mutation testing run in separate workflows. +### The Rust toolchain + +`rust-toolchain.toml` pins one exact stable release (`channel = "1.99.1"`). +rustup reads it, so a plain `cargo` here uses that release, and every CI job +that is not about another toolchain installs the same one; the release +workflow packages and publishes with it. `scripts/check-toolchain-pin.py` (in +the `doc` job) fails when a workflow names any other toolchain outside the +informational jobs. The reason: CI runs with `-D warnings` and clippy's +`pedantic`, `nursery` and `cargo` groups, so a floating `stable` let every Rust +release (one every six weeks) change rustc, rustfmt and clippy under every job +at once, with no change in this repository; 1.99.0's `clippy::assert_is_empty` +did exactly that (AUDIT.md 12.7). + +The exceptions are the MSRV job (`cargo +1.86.0`), the jobs that need nightly +(Miri, the sanitizers, the fuzzer), and three informational jobs that look +ahead without blocking: `latest-stable` (the current stable: `fmt`, clippy on +both feature sets, rustdoc, the library tests), `clippy-beta` and `nightly`. +They run on every pull request and on the weekly schedule. + +Move the pin by a pull request of its own, when `latest-stable` shows a newer +stable: + +1. Check that the release exists: `rustup toolchain install X.Y.Z` (or read + `https://static.rust-lang.org/dist/channel-rust-X.Y.Z.toml`). +2. Change `channel` in `rust-toolchain.toml` and every `toolchain: ` in + `.github/workflows/*.yml`; `python3 scripts/check-toolchain-pin.py` lists + any you missed. +3. Run the quality gates above (plain `cargo` now uses the new release) and fix + what new lints and warnings report, in the same pull request. + +The MSRV (`rust-version`, 1.86.0) is a separate decision with its own rules +(RELEASING.md, "Semantic versioning policy"). + --- ## Test Strategy @@ -269,7 +328,32 @@ function built with quack-rs, runs SQL and checks the answer. Requires AddressSanitizer jobs run exactly this target (`--test ffi_roundtrip`), so new end-to-end tests belong in it — as a submodule declared with `#[path = "ffi_roundtrip/.rs"] mod ;` when the root file would -otherwise grow further. +otherwise grow further. Know what those runs cannot see (next section). + +### What the sanitizer and valgrind runs can and cannot see + +CI's `leak-check` (LeakSanitizer) and `asan` (AddressSanitizer) jobs, like a +local valgrind run of the end-to-end suite, use the **release** libduckdb: the +same prebuilt library `test-bundled-prebuilt` links. That library carries its +own copy of jemalloc (it exports 769 `duckdb_je_*` symbols at v1.5.6 and 768 +at v2.0.0, and defines no `malloc` or `free` of its own), and DuckDB's +`Allocator` takes vector data buffers, LIST entry arrays and string heaps from +it. Neither valgrind nor AddressSanitizer tracks the blocks jemalloc hands out, +so **an out-of-bounds or use-after-free write into a vector's data buffer or a +LIST's entry array is invisible to all of these runs**: the stale writes into a +LIST entry array of the `ListBuilder` regression found before 0.20.0 drew no +report under valgrind. + +What they do catch: errors in memory the Rust side allocates (Rust's heap, +boxed aggregate states, the bind and init data quack-rs owns), and in what +DuckDB allocates through C++ `new` and `malloc`, which includes validity masks +(an out-of-bounds validity write *is* reported) and the objects behind its +handles, so LeakSanitizer still sees a handle that is never destroyed. A clean +sanitizer or valgrind run therefore says nothing about the pointer arithmetic +into vector buffers; the end-to-end tests' answers, and Miri for the pure-Rust +paths, are what cover it. A sanitizer run against a DuckDB built from source +without jemalloc would close the gap; none is set up (CI compiles DuckDB from +source in `test-bundled` but does not sanitize that build). ### Property-based tests @@ -284,7 +368,7 @@ tests for all pure-Rust logic (`count_words`, `first_word`, `parse_varchar_to_in aggregate state transitions). **Full end-to-end testing against a live DuckDB is required** — not left to consumers. This means building the `.so`, appending the extension metadata footer with `append_metadata`, and running every SQL check in -`examples/hello-ext/sql_checks.txt` (29 numbered checks, 31 statements) with +`examples/hello-ext/sql_checks.txt` (30 numbered checks, 32 statements) with `scripts/check-hello-ext.py`. The script also fails if the statements in `examples/hello-ext/README.md` drift from that file. CI's `extension-load` job runs it against DuckDB 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release. Gate 7 under @@ -314,6 +398,39 @@ cargo mutants --list Configuration is in `.cargo/mutants.toml` — the one place cargo-mutants reads it from without `--config`. +On a pull request, `mutants.yml` mutates the whole of every changed source file, +split over eight jobs (`--shard k/8 --sharding round-robin`); the job named +"Mutation Testing (incremental)" is the one that fails if any shard does. One +shard locally: `cargo mutants --file --shard 0/8 --sharding round-robin`. +A pull request that changes a hundred files generates over a thousand mutants, +about twenty seconds each: a shard of one-eighth fits the 60-minute limit, one +job for all of them did not. + +**What the gate cannot see.** cargo-mutants 27.1.0 generates no mutants inside +the body of an `unsafe fn` (free, associated, `const`, trait-default and +`unsafe extern "C"` alike; it does mutate a safe function that holds `unsafe` +blocks), never mutates integer literals (`base.add(8)`, `>> 6`, `& 63`, `* 16` +have no mutants even in a visible function), and cannot look into a macro body. +In this crate that is the pointer arithmetic: by a heuristic count +(`scripts/mutants-expose-unsafe.py` strips comments and strings and matches +braces), 506 `unsafe fn`s hold 5,127 of 22,898 non-test source lines (22.4%), +among them most of `VectorReader`, `VectorWriter`, `ListBuilder`, the Arrow +layout walker and `FfiState`'s callbacks. So a "0 missed" result covers the 2,493 +mutants cargo-mutants lists with this repository's configuration (3,748 with +`--no-config`), not those lines. The live suite and the sanitizer jobs +([above](#what-the-sanitizer-and-valgrind-runs-can-and-cannot-see)) are what +stand behind them. + +To mutate them in an audit, run `scripts/mutants-expose-unsafe.py` on a copy of +the tree: it drops the `unsafe` keyword from signatures (the crate denies +`unsafe_op_in_unsafe_fn`, so the copy still compiles) and, with `--literals`, +turns the integer operands of pointer arithmetic into expressions cargo-mutants +can flip. On `src/vector/reader.rs` and `writer.rs` the mutant list goes from 6 +to 192 (344 with literals). The script's header gives the commands. It is not +run in CI. Moving such arithmetic into small safe functions over integers and +slices (`stride_offset(idx, width)`, `validity_bit(words, idx)`) makes it both +mutable and checkable by Miri, and is the better fix where a function allows it. + ### Test naming convention Tests follow the pattern: `{component}_{scenario}_{expected_outcome}` @@ -397,6 +514,8 @@ quack-rs/ │ ├── data_chunk.rs # Ergonomic wrapper around `DuckDB` data chunks │ ├── debug_repr.rs # Internal helpers for the crate's `Debug` implementations │ ├── entry_point.rs # Extension entry point helper +│ ├── entry_point/ +│ │ └── registration.rs # Runs the registration closure in one transaction, or with autocommit │ ├── error.rs # Error types for `DuckDB` extension FFI error propagation │ ├── error_data.rs # Structured error data (`DuckDB` 1.5.0+) │ ├── expression.rs # Bound expressions (`DuckDB` 1.5.0+) @@ -420,6 +539,7 @@ quack-rs/ │ │ ├── mod.rs # Builders for registering `DuckDB` aggregate functions │ │ ├── state.rs # Generic `FfiState` wrapper for safe aggregate state management │ │ └── builder/ +│ │ ├── collision.rs # Which catalog entry holds the name a failed aggregate registration wanted │ │ ├── mod.rs # Builder types for registering `DuckDB` aggregate functions │ │ ├── overload.rs # One overload within an [`AggregateFunctionSetBuilder`] │ │ ├── set.rs # Builder for registering a `DuckDB` aggregate function set (multiple overloads) @@ -440,18 +560,26 @@ quack-rs/ │ │ ├── export_check.rs # Values `DuckDB` would export as different values, found before the export │ │ ├── import_check.rs # Structural checks on an imported array, and flattening what the readers cannot index │ │ ├── import_layout/ +│ │ │ ├── bounds.rs # Offsets, lengths and dictionary indices `DuckDB` would read past, refused │ │ │ └── tests.rs # Unit tests │ │ ├── import_layout.rs # Arrow layouts `DuckDB` imports wrongly, found by walking the array with its schema │ │ ├── options.rs # `ArrowOptions` — the Arrow production settings of a connection or a result │ │ ├── schema.rs # `ArrowSchema` — an owned Arrow C Data Interface schema │ │ └── tests.rs # Unit tests │ ├── bin/ -│ │ └── append_metadata/ -│ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap) -│ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file -│ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it -│ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file, -│ │ └── tests.rs # Unit tests +│ │ ├── append_metadata/ +│ │ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap) +│ │ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file +│ │ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it +│ │ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file, +│ │ │ ├── output.rs # Reading the library and writing the stamped file +│ │ │ ├── tests.rs # Unit tests +│ │ │ └── tests_output.rs # Unit tests for reading the input and writing the output +│ │ └── quack_scaffold/ +│ │ ├── cli.rs # Command-line parsing (std-only, no clap) +│ │ ├── main.rs # Write a new `DuckDB` extension project to disk, from `generate_scaffold_with_warnings` +│ │ ├── tests.rs # Unit tests +│ │ └── write.rs # Writing the generated files: everything is checked before anything is written │ ├── callback/ │ │ └── payload.rs # Disposing of a caught panic payload without re-entering the unwinder │ ├── cast/ @@ -470,18 +598,25 @@ quack-rs/ │ │ ├── connection.rs # [`OwnedConnection`] and its cross-thread [`InterruptHandle`] │ │ ├── cstr.rs # The two C-string conversions the `query` module runs everything through │ │ ├── live_tests.rs # Tests that need a live `DuckDB` +│ │ ├── metadata.rs # What a statement is before it runs: `StatementType`, parameter and column types, `table_names` │ │ ├── prepared.rs # Inspecting and executing a [`PreparedStatement`]; `bind.rs` binds its parameters │ │ └── result.rs # Reading a [`QueryResult`]: its columns, its chunks and what kind of outcome it is │ ├── replacement_scan/ │ │ └── mod.rs # Builder for registering `DuckDB` replacement scans │ ├── scaffold/ +│ │ ├── checks.rs # What `generate_scaffold` checks before it writes anything, and the warnings it gives │ │ ├── escape.rs # Quoting configured free text for YAML and Rust doc comments +│ │ ├── licenses.rs # The license text of a generated project │ │ ├── mod.rs # Project scaffolding for `DuckDB` Rust extensions │ │ ├── templates.rs # Template generators for scaffold file content │ │ ├── templates_ci.rs # The generated project's CI and distribution workflows +│ │ ├── templates_docs.rs # The generated project's `README.md`, `docs/function_descriptions.csv` and `deny.toml` +│ │ ├── templates_make.rs # The generated `Makefile`: extension-ci-tools' variables and targets, and the skipped-test check │ │ ├── tests.rs # Unit tests +│ │ ├── tests_content.rs # What the scaffold accepts, warns about and refuses, and the files a community submission needs │ │ ├── tests_escaping.rs # Free text reaches the generated files intact │ │ ├── tests_generated.rs # Unit tests +│ │ ├── tests_make.rs # Runs the generated `Makefile`'s own logic under `make` │ │ └── tests_workflows.rs # The generated workflows match the community build │ ├── scalar/ │ │ ├── info.rs # Ergonomic wrapper around `duckdb_function_info` for scalar function callbacks @@ -546,14 +681,24 @@ quack-rs/ │ │ ├── parser.rs # Parses and validates a `description.yml` string │ │ ├── tests.rs # Unit tests │ │ ├── tests_corpus.rs # Unit tests +│ │ ├── tests_mut_block.rs # Boundaries of the block-structure reader, found by mutation testing +│ │ ├── tests_mut_flow_a.rs # Boundaries of the flow-collection reader (scanning, anchors), found by mutation testing +│ │ ├── tests_mut_flow_b.rs # Boundaries of the flow-collection reader (nesting, plain items), found by mutation testing +│ │ ├── tests_mut_scalar.rs # Boundaries of the scalar, section and timestamp readers, found by mutation testing +│ │ ├── tests_pyyaml.rs # Accepted exactly when PyYAML accepts it, in linear time │ │ ├── tests_yaml.rs # Unit tests │ │ ├── validator.rs # Validates a `description.yml` string and returns `Ok(())` if it passes all checks │ │ ├── yaml.rs # A reader for the subset of YAML that `description.yml` files use │ │ ├── yaml11.rs # Which plain scalars PyYAML (YAML 1.1) reads as booleans, numbers, dates or null │ │ └── yaml/ -│ │ └── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments +│ │ ├── block.rs # Block mappings and sequences, and their keys +│ │ ├── flow.rs # Flow collections, nested and spread over lines +│ │ ├── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments +│ │ └── value.rs # The value after a mapping key or a sequence marker │ ├── value/ +│ │ ├── bits.rs # `BIT` and `BIGNUM` values: validated constructors and getters that check type and `NULL` first │ │ ├── blob.rs # `Value::as_blob` — `BLOB` extraction +│ │ ├── cast.rs # `Value::try_cast` (exception-free, any type), `array_items` and `union_tag` │ │ ├── checks.rs # Pure-Rust preconditions checked before a `Value` call reaches `DuckDB` │ │ ├── composite.rs # Composite constructors: `STRUCT`, `LIST`, `ARRAY`, `ENUM`, `MAP`, `UNION` │ │ ├── defaults.rs # The defaulting accessors — `Value::as_*_or` @@ -578,19 +723,27 @@ quack-rs/ │ ├── validity.rs # Validity bitmap helpers for `DuckDB` NULL tracking │ └── writer.rs # Safe typed writing to `DuckDB` result vectors ├── tests/ +│ ├── abi_policy_enforced.rs # `AbiPolicy` enforced in-process against an engine whose layout differs │ ├── aggregate_leaks.rs # Aggregate states `DuckDB` never destroys leak no Rust heap │ ├── append_metadata_cli.rs # The `append_metadata` binary run end to end: exit status, output and the file it writes +│ ├── build_script_env.rs # The build script run with a crafted `QUACK_RS_TARGET_DUCKDB_VERSION` +│ ├── entry_point_expansion.rs # The entry-point macros compiled as edition 2024 and under `panic = "abort"` │ ├── entry_point_lint.rs # The entry-point macros accept an `unsafe` registration closure warning-free │ ├── ffi_roundtrip.rs # End-to-end FFI round-trips against a real `DuckDB` │ ├── file_handle_close.rs # `FileHandle`'s `Drop` when the close fails (stubbed C API) │ ├── handle_leaks.rs # Every RAII handle frees what `DuckDB` allocated for it (glibc) │ ├── integration_test.rs # Integration tests for `quack-rs` +│ ├── lsan_probe.rs # The `leak-check` job's canary: a handle leaked on request must fail LeakSanitizer +│ ├── property_inputs.rs # Property tests for `parse_version`, SPDX, `description.yml` and the scaffold's escaping +│ ├── quack_scaffold_cli.rs # The `quack_scaffold` binary run as a user runs it: exit status, output and the files it writes +│ ├── secret_memory_scan.rs # `SecretEntry`'s zeroize, checked by scanning the process's own memory (Linux) │ ├── secret_zeroize.rs # `SecretEntry` never frees a buffer that still holds a secret │ └── ffi_roundtrip/ │ ├── agg_states.rs # Every aggregate state is dropped, including the ones `DuckDB` moves │ ├── agg_window.rs # Aggregates in the running-window and sorted-aggregate paths │ ├── aggregate_arity.rs # An aggregate with no parameter is refused at registration (DuckDB 2.0 passes `update` no row count) │ ├── appender_api.rs # `Appender` methods a no-op replacement survived: schemas, column types, `clear_columns`, defaults +│ ├── appender_query.rs # `Appender::for_query` and the appender's `from_raw` / `into_raw` │ ├── appender_rows.rs # What happens to buffered rows when an append fails mid-row │ ├── arrow_export.rs # `arrow::data_chunk_to_arrow` refuses values `DuckDB` would export wrongly │ ├── arrow_import.rs # `arrow::data_chunk_from_arrow` checks against a live `DuckDB` @@ -600,14 +753,19 @@ quack-rs/ │ ├── chunk_writer.rs # `ChunkWriter` against a chunk `DuckDB` allocated │ ├── chunk_writer_cols.rs # `ChunkWriter::{vector, writer, struct_writer}` and `DataChunk::into_chunk_writer` filling a real chunk │ ├── collision.rs # The scalar signature-collision check, held to `DuckDB`'s own binder +│ ├── connection_registrar.rs # Every `Registrar` method of `Connection` reaches `DuckDB` │ ├── copy_from_columns.rs # A typed `COPY … FROM` reader that declares a column is refused │ ├── copy_info_errors.rs # `set_error` from all four COPY phases, and the state passed between them (`duckdb-1-5`) +│ ├── copy_typed_state.rs # Typed `COPY TO` state, a `FileHandle` kept across callbacks, `Send` handles (`duckdb-1-5`) │ ├── data_chunk_helpers.rs # `DataChunk::any_null`, `struct_writer`, `into_chunk_writer` and `ChunkWriter::is_full` │ ├── entry_point_inprocess.rs # `init_extension*` and the entry-point macros run in process against a fake extension access +│ ├── entry_point_txn.rs # A failed `LOAD` rolls back and can be retried; a `LOAD` inside the host's transaction │ ├── file_errors.rs # `FileHandle` reports write, sync and seek failures, and refuses a seek past `i64::MAX` │ ├── handles_api.rs # `StructWriter` child handles and `InMemoryDb::execute`'s row count +│ ├── hostile_input.rs # Hostile names, SQL bodies, type nesting and cardinality hints against a real engine │ ├── inline_paths.rs # The paths that skip a C call: `is_valid` / `set_null` bit operations and the `ListBuilder` child-writer cache, held to the C API │ ├── lifecycle.rs # Aggregate NULL rows, name collisions, overload builders, bind-data sharing +│ ├── lifecycle_gaps.rs # Lifecycle paths no other test reached, found by mutation testing │ ├── list_api.rs # The raw `ListVector` workflow, `get_size`, `get_entry`, `child_writer`, `max_child_capacity` on live LIST and MAP vectors │ ├── list_limits.rs # `ListBuilder` stops at `DuckDB`'s byte ceiling, not an element count │ ├── logical_type_accessors.rs # Every `LogicalType` accessor, built by the constructors and read back, against the engine's own `typeof` @@ -615,10 +773,13 @@ quack-rs/ │ ├── mock_parity.rs # `MockRegistrar` refuses a bad type with the live registration's message │ ├── nested_reserve.rs # which nested buffers a `LIST` reserve moves (the writer contracts) │ ├── nested_validity.rs # `VectorWriter::set_valid` on nested rows, against a live `DuckDB` +│ ├── oracle_temporal.rs # Randomised cross-checks of `datetime` and `interval` against `DuckDB`'s own SQL +│ ├── overload_keys.rs # Overloads `DuckDB` resolves by type id alone are refused; closure-built overloads; taken aggregate names │ ├── owned_chunk.rs # `OwnedDataChunk::new`, `capacity` and `reset` against a live `DuckDB` │ ├── panic_guards.rs # The panic-guard macros and `set_error` methods, against a live `DuckDB` │ ├── query_docs.rs # Pins the documented behaviour of `query`, `PreparedStatement`, `DbConfig` │ ├── query_extras.rs # `bind_str`, `bind_null`, `clear_bindings` and an idle `progress` +│ ├── query_metadata.rs # Statement type, parameter and column metadata, `table_names`, raw-handle round trips │ ├── query_stream.rs # A streaming result that stops early must not look like a finished one │ ├── replacement_scan_api.rs # Replacement-scan `add_*_parameter`, `register_with_data` and `set_error` │ ├── scalar_agg.rs # Scalar and aggregate builder regressions @@ -632,17 +793,22 @@ quack-rs/ │ ├── temporal_binds.rs # Temporal and over-4-GiB values refused by `PreparedStatement` binds and the `Appender` │ ├── timestamp_tz_ns.rs # `TIMESTAMPTZ_NS` (DuckDB 2.0): vectors, `Value`, and every type the engine can produce │ ├── tooling.rs # Checks of quack-rs's tooling tables against the linked `DuckDB` +│ ├── typed_extra_info.rs # `extra_info_typed` / `extra_info_ref` on casts and aggregates; a wrong type reads as `None` │ ├── typed_scalar_opt.rs # Typed scalar closures returning `Option`, the `try_*` constructors and `map3` │ ├── validity_api.rs # `ValidityBitmap` against the engine's masks, at the word and vector edges +│ ├── value_bits.rs # `BIT` / `BIGNUM` values; every input the C getters abort on reads as `None` +│ ├── value_cast.rs # `try_cast`, `array_items`, `union_tag`: reading `ARRAY` and `UNION` values │ ├── value_defaults.rs # `Value::as_*_or` when a value is present │ ├── value_getters.rs # Each typed `Value` getter at its own type; out-of-range TIMETZ / TIME_NS are not cast │ ├── value_nested.rs # Nested `Value` construction and inspection against a live `DuckDB` │ ├── value_query.rs # `Value` getters, DECIMAL binding, `Expression::fold` │ ├── value_render.rs # `Value` rendering of values SQL builds and `DuckDB` cannot render │ ├── value_temporal.rs # Every `Value` getter against every temporal source type, at every edge +│ ├── vector_capacity_guard.rs # `OwnedVector::new` refuses a capacity whose nested `ARRAY` child would overflow │ ├── vector_dt.rs # NULLs in nested output vectors; selection vectors │ ├── vector_ops.rs # `OwnedVector::into_raw`, `copy_selected`, `slice`, `reference_value`, `reference_vector` (`duckdb-1-5`) │ ├── vector_support.rs # Helpers shared by the vector-module tests +│ ├── wide_rows.rs # Every fixed-width and string type echoed over thousands of distinct rows │ └── writer_api.rs # `write_timestamp_tz`, `write_str`, and the Ok path of `try_write_varchar` / `try_write_blob` ├── benches/ │ ├── interval_bench.rs # Criterion benchmarks for interval conversion @@ -665,7 +831,7 @@ quack-rs/ │ ├── benchmarks.yml # Criterion benchmark execution │ └── README.md # Workflow overview and quality gate summary ├── CONTRIBUTING.md # This file -├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L19, P1–P15), documented in full +├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L21, P1–P17), documented in full └── README.md # Quick start, SDK overview, badge table ``` @@ -691,21 +857,37 @@ quack-rs/ ## Releasing -This crate supports `libduckdb-sys = ">=1.4.4, <2"` (the bindings of DuckDB 1.4.4 and -later 1.x releases). Every DuckDB from 1.4.4 through 2.0.0 loads C API `v1.2.0` extensions. -The `<2` upper bound does not exclude the DuckDB 2.0 bindings: `libduckdb-sys` encodes DuckDB -`vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is -1.20000.0, which the range admits. The 1.20000.x releases (1.20000.0 on 2026-10-22, -1.20000.1 on 2026-10-28) declare `rust-version` 1.88, so a Rust 1.86 toolchain cannot build them; MSRV-aware resolution -(resolver 3, or `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`) keeps a 1.86 build on -1.10506.x. The bound would stop a future `libduckdb-sys` 2.x. -Before broadening the range to a new major band: - -1. Read the DuckDB changelog for C API changes. -2. Check the new C API version string (used in `duckdb_rs_extension_api_init`). -3. Update `DUCKDB_API_VERSION` in `src/lib.rs` if the C API version changed. -4. Audit all callback signatures against the new `libduckdb-sys` bindings. -5. Update the `libduckdb-sys` and `duckdb` version requirements in `Cargo.toml`. +This crate supports `libduckdb-sys = ">=1.4.4, <1.20100"` (the bindings of DuckDB 1.4.4 +through 2.0.x). Every DuckDB from 1.4.4 through 2.0.0 loads C API `v1.2.0` extensions. +`libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, +so DuckDB 2.0.0 is 1.20000.0 and DuckDB 2.1.0 will be 1.20100.0; the bound admits the first +and stops before the second. The 1.20000.x releases (1.20000.0 on 2026-10-22, 1.20000.1 on +2026-10-28) declare `rust-version` 1.88, so a Rust 1.86 toolchain cannot build them; +MSRV-aware resolution (resolver 3, or `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`) +keeps a 1.86 build on 1.10506.x. + +**Why there is an upper bound.** 1.20000.x keeps the root-level names quack-rs and its +exported macros use (`libduckdb_sys::duckdb_connection`, ...) behind the default feature +`capi-v1`, beside a `// TODO: Remove in the future`. With only that re-export deleted from a +copy of 1.20000.1, quack-rs failed with 214 compile errors (the eighth review reproduced +it). Without an upper bound, the day a later binding drops it, every new project resolves to +bindings this crate cannot build against. Published releases cannot be narrowed after the +fact, which is why the bound is on the newest release and is widened deliberately. + +**Widening the bound for a new DuckDB minor** (a patch release of quack-rs): + +1. Read the DuckDB release notes and the C API diff (`duckdb.h`, `duckdb_extension.h`): + `scripts/check-abi-table.py` and `scripts/check-api-coverage.py` report new slots. +2. Resolve the new bindings (`cargo update -p duckdb --precise ` and + `-p libduckdb-sys --precise `) in a copy and run the whole suite against the new + engine, as the `test-duckdb-2-0` job does for 2.0. Check specifically that the root-level + re-exports still exist (`cargo check` is enough to tell). +3. Add the release to `KNOWN_LAYOUTS` / `PREFIX_LAYOUTS` in `src/abi.rs` if its slot count + differs (the abi-table job derives them), and to the `extension-load` and older-engine + matrices. +4. Update the `libduckdb-sys` and `duckdb` requirements in `Cargo.toml` and in every + manifest and document that repeats them (`git grep '<1.20100'`; a test compares them). +5. Update `DUCKDB_API_VERSION` in `src/lib.rs` if the C API version changed. Versions follow [Semantic Versioning](https://semver.org/) as Cargo applies it. While the crate is pre-1.0, a breaking change to the public API bumps the diff --git a/Cargo.lock b/Cargo.lock index f4e91c05..bec405ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,18 +253,22 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bit-set" -version = "0.8.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.8.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] [[package]] name = "bitflags" @@ -272,6 +276,30 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -292,9 +320,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "jobserver", @@ -314,6 +342,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "chrono" version = "0.4.45" @@ -414,6 +453,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -547,7 +601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -580,9 +634,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -663,6 +717,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core", ] [[package]] @@ -1072,15 +1127,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -1091,6 +1137,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1102,16 +1157,17 @@ dependencies = [ [[package]] name = "proptest" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ "bit-set", "bit-vec", "bitflags", + "chacha20", + "core_detect", "num-traits", "rand", - "rand_chacha", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -1121,7 +1177,7 @@ dependencies = [ [[package]] name = "quack-rs" -version = "0.19.0" +version = "0.20.0" dependencies = [ "cc", "criterion", @@ -1161,38 +1217,25 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.5" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", + "getrandom 0.4.3", "rand_core", ] [[package]] name = "rand_core" -version = "0.9.5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_xorshift" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ "rand_core", ] @@ -1292,7 +1335,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1506,7 +1549,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.5", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1528,6 +1571,36 @@ dependencies = [ "serde_json", ] +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "typed-path" version = "0.12.4" @@ -1730,7 +1803,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1816,15 +1889,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -1889,6 +1953,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index 75496537..0b40efde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quack-rs" -version = "0.19.0" +version = "0.20.0" edition = "2021" rust-version = "1.86.0" authors = ["Tom F. "] @@ -13,31 +13,22 @@ readme = "README.md" keywords = ["duckdb", "extension", "ffi", "sql", "analytics"] categories = ["database", "api-bindings", "development-tools::ffi"] autobenches = false -exclude = [ - "docs/", - "examples/", - "benches/engine/", - "fuzz/", - ".github/", - "book/", - "book.toml", - "assets/", - "deny.toml", - "osv-scanner.toml", - "AUDIT.md", - "RELEASING.md", - "LESSONS.md", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "GOVERNANCE.md", - # Repository tooling (CI checks, the book and release scripts); nothing in - # the build uses it. - "scripts/", - "clippy.toml", - "codecov.yml", - ".cargo/", - "rust-toolchain.toml", - "lsan-suppressions.txt", +# An allow-list: a new top-level file or directory is NOT published unless it is +# named here (the previous `exclude` list shipped anything new by default). +# `examples/*.rs` ship because Cargo packages every auto-discovered example and +# warns ("ignoring example ...") when its file is missing; `examples/hello-ext` +# is a separate package and is not matched. +include = [ + "/src/**", + "/tests/**", + "/benches/interval_bench.rs", + "/examples/*.rs", + "/build.rs", + "/README.md", + "/LICENSE", + "/CHANGELOG.md", + "/SECURITY.md", + "/CITATION.cff", ] [lib] @@ -75,7 +66,7 @@ crate-type = ["rlib"] ## ## ```toml ## [dev-dependencies] -## quack-rs = { version = "0.19", features = ["bundled-test"] } +## quack-rs = { version = "0.20", features = ["bundled-test"] } ## ``` bundled-test = ["_duckdb-testing", "duckdb/bundled"] @@ -94,7 +85,7 @@ bundled-test = ["_duckdb-testing", "duckdb/bundled"] ## ## ```toml ## [dev-dependencies] -## quack-rs = { version = "0.19", features = ["bundled-test-prebuilt"] } +## quack-rs = { version = "0.20", features = ["bundled-test-prebuilt"] } ## ``` bundled-test-prebuilt = ["_duckdb-testing"] @@ -185,8 +176,16 @@ duckdb-1-5-4 = ["duckdb-1-5-3"] # so it does not keep out the DuckDB 2.0 bindings. [dependencies] -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } -duckdb = { version = ">=1.4.4, <2", optional = true } +# `libduckdb-sys` numbers DuckDB vX.Y.Z as 1.(10000*X + 100*Y + Z).N, so `<1.20100` +# admits every DuckDB 1.4.4 - 2.0.x binding and stops before DuckDB 2.1 (1.20100.0). +# The bound is deliberate: 1.20000.x keeps the root-level re-exports quack-rs and its +# exported macros name (`libduckdb_sys::duckdb_connection`, ...) behind the default +# feature `capi-v1`, next to a `// TODO: Remove in the future`. If a later binding +# drops them, an unbounded range would resolve to bindings this crate cannot compile +# against (214 errors, reproduced). Widen it in a patch release once a new DuckDB +# minor has been run through CI (AUDIT.md section 14). +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } +duckdb = { version = ">=1.4.4, <1.20100", optional = true } [build-dependencies] # Used only when `bundled-test` / `bundled-test-prebuilt` is active: compiles the @@ -265,3 +264,8 @@ undocumented_unsafe_blocks = "warn" # feature-gated item would be missing. `bundled-test` is # intentionally excluded (it compiles DuckDB from source). features = ["duckdb-1-5-4"] + +[package.metadata.cargo-machete] +# Used, but outside the places cargo-machete looks: `cc` by build.rs, `criterion` +# by benches/interval_bench.rs (a `[[bench]]` with `autobenches = false`). +ignored = ["cc", "criterion"] diff --git a/GOVERNANCE.md b/GOVERNANCE.md index a47e6fcd..6568340d 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -30,6 +30,12 @@ final decision-making authority. | ------ | ------------- | ------------------ | | Tom F. | @tomtom215 | Initial Maintainer | +**There is one maintainer, and no other committers.** The rules below that need a +second person (a second approver, a majority vote, a tie-break) describe what happens +once there is one; today the maintainer decides. What gates a merge to `main` is the +`CI passed` check that [RELEASING.md](./RELEASING.md) configures as the required status +check, not a second reviewer. + ## Decision-Making Process The project uses **lazy consensus**: @@ -65,7 +71,9 @@ how reports are handled. following the [Keep a Changelog](https://keepachangelog.com/) format. 3. **Release Steps:** - A maintainer opens a release PR updating version numbers and the changelog. - - The PR must pass CI and receive at least one maintainer approval. + - The PR must pass CI. When there is more than one maintainer, it also needs the + approval of a maintainer who did not write it; with one maintainer, the + maintainer merges once CI is green. - After the PR is merged and CI on `main` passes, the maintainer pushes a tag (`vX.Y.Z`, GPG-signed where possible; RELEASING.md Step 5) for the merged commit. The tag push starts the diff --git a/LESSONS.md b/LESSONS.md index 9ad8bf7a..386839d1 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -64,8 +64,10 @@ writing your own destructor. ## L3: No panic across FFI boundaries -**Status**: Made impossible by `init_extension` and the callback guards (which require -`panic = "unwind"` in the release profile). +**Status**: Prevented for the entry point and for every callback defined with a +`*_callback!` macro or a closure-based builder; these guards require +`panic = "unwind"`. A raw `unsafe extern "C" fn` that you pass to a builder +yourself is not guarded. **Symptom**: The whole DuckDB process aborts when an extension callback panics. @@ -76,6 +78,19 @@ uncaught `panic!()` or `.unwrap()` in a callback takes down the user's session. **Fix**: Use `Result` and `?` inside `init_extension`. Never use `unwrap()` in FFI callbacks. `FfiState::with_state_mut` returns `Option`, not `Result`, so callers use `if let`. +The entry-point macros, the `*_callback!` macros in `quack_rs::callback`, the +closure-based builders (`ScalarFunctionBuilder::map1` and its siblings, +`TypedTableFunctionBuilder`) and `FfiState`'s callbacks catch a panic and report +it as a SQL error. A raw `unsafe extern "C" fn` that you pass to a builder +yourself is installed as written, with no guard: define it with the matching +`*_callback!` macro, or wrap its body in `quack_rs::callback::catch_ffi_panic` +and report the `Err` it returns yourself. + +All of these guards require `panic = "unwind"` in the release profile, which is +what the scaffold generates and what `validate_release_profile` insists on: +under `panic = "abort"` nothing can be caught, so `entry_point!` and `entry_point_v2!` +refuse to compile there (an entry point written by hand has no such check). + --- ## L4: `ensure_validity_writable` is required before NULL output @@ -159,11 +174,15 @@ survives review and ships. and calls the extension's callback for the whole chunk, NULL rows included, then checks only the *error* flag. It never inspects the result's validity. - `src/execution/expression_executor/execute_function.cpp` — the only NULL check - is `VerifyNullHandling`, whose entire body is inside `#ifdef DEBUG`. It - *asserts* that the function already produced NULL wherever an input was NULL. + is `VerifyNullHandling`, whose entire body is inside `#ifdef DEBUG` through + DuckDB 1.5.6. It *asserts* that the function already produced NULL wherever an + input was NULL. -Every DuckDB a user installs is a release build, so that assertion is compiled -out and a callback that ignores validity is simply believed. +Every DuckDB a user installs is a release build, so through 1.5.6 that assertion is +compiled out. DuckDB 2.0.0 compiles it into release builds but runs it only after +`SET debug_verification_mode = 'VERIFY_FUNCTIONS'` (the default is `NONE`; measured: +with the setting on, `INTERNAL Error: VerifyNullHandling failed for scalar function`). +With the default, a callback that ignores validity is simply believed. A literal `NULL` argument is **constant-folded** during binding — DuckDB substitutes NULL without calling the function at all — which is why the obvious @@ -329,8 +348,12 @@ documented on `AggregateFunctionBuilder`, `AggregateFunctionSetBuilder` and **Symptom**: An aggregate that works under `SELECT agg(x) FROM t` and `GROUP BY` segfaults (or corrupts memory, or returns a wrong answer) when used as a window -over a whole-partition frame — `agg(x) OVER ()`, `OVER (PARTITION BY p)` — or as -an ordered aggregate, `agg(x ORDER BY y)`. +over a whole-partition frame — `agg(x) OVER ()`, or an explicit `ROWS` or `RANGE +BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, with or without a +`PARTITION BY` — or as an ordered aggregate, `agg(x ORDER BY y)`. On 1.4.4 `OVER +(PARTITION BY p)` without an `ORDER BY` crashes too; on 1.5.6 it did not in the shapes +measured (5,000 and 300,000 rows). Whether that is a different code path or luck was not +established, so do not rely on it. **Root cause**: `CAPIAggregateUpdate` (`src/main/capi/aggregate_function-c.cpp`) flattens the input vectors but not the state vector, then hands the callback @@ -347,9 +370,12 @@ callback, called from `CAPIAggregateUpdate`. `duckdb_aggregate_state *` and cannot tell a constant vector from a flat one, and reading `states[1]` to find out is itself the out-of-bounds read. Until you require DuckDB 2.0.0, document for your users that the aggregate must not be used -in those two query shapes. Frames that are not whole-partition (`ROWS BETWEEN 5 -PRECEDING AND CURRENT ROW`, segment-tree windows) and `DISTINCT` windows were -checked and work. +in those query shapes. Frames that are not whole-partition (the default frame of +`OVER (ORDER BY i)`, `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`, `ROWS BETWEEN 5 +PRECEDING AND CURRENT ROW`, `EXCLUDE`) and `DISTINCT` and `FILTER` were measured and work +on 1.4.4, 1.5.6 and 2.0.0 ([Known +Limitations](https://quack-rs.com/reference/known-limitations.html#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) +has the table). DuckDB 2.0.0 fixes it in `CAPIAggregateUpdate` (`src/main/capi/v1/aggregate_function-c.cpp`, lines 104-108): a state vector @@ -580,6 +606,74 @@ runs this build. --- +## L20: `duckdb_register_log_storage` frees the extra data twice when the name is taken + +**Status**: Documented. quack-rs does not wrap the log-storage functions because of it. +This is a DuckDB defect, present since 1.5.0 and still on `main` (2026-10-30), not a +quack-rs one. + +**Symptom**: registering a log storage under a name that is already taken returns +`DuckDBError`, and the extra-data destructor you gave DuckDB has already run. The +`duckdb_destroy_log_storage` you must still call runs it again on the same pointer. +Observed on DuckDB 1.5.6 and 2.0.0 with a probe extension that registers the same name +twice: the destructor ran twice on one address. With a destructor that frees its box this +is a heap double free (`free(): double free detected in tcache 2`, seen in the eighth +review). The same happens for a name that a built-in storage already holds. + +**Root cause** (`src/main/capi/v1/logging-c.cpp`, identical at 1.5.6, 2.0.0 and `main`): + +```cpp +auto shared_storage_ptr = make_shared_ptr( + wrapper->name, wrapper->write_log_entry, wrapper->extra_data, wrapper->delete_callback); +const auto success = RegisterLogStorage(wrapper->name, storage_ptr); +if (!success) { + return DuckDBError; // shared_storage_ptr dies here and calls delete_callback(extra_data) +} +wrapper->extra_data = nullptr; // only cleared on success +wrapper->delete_callback = nullptr; +``` + +The early failures (a null argument, an empty name, no write callback) return before the +`CallbackLogStorage` exists and do not consume the data, so a caller cannot treat every +`DuckDBError` alike. + +**Workaround**: never register a name that can already exist. After a failure that got +past argument validation, clear the extra data with +`duckdb_log_storage_set_extra_data(ls, NULL, NULL)` before destroying the wrapper (proposed, +not tested). + +**Rule**: a destructor you hand to DuckDB may already have run when the call that took +it reports failure; read the failure path before you free again. + +--- + +## L21: A chunk row you claim is published, written or not + +**Status**: Documented. Since 0.20.0 `ChunkWriter::release_row` gives a claimed row back; +before 0.20.0 every claimed row was published. The rule is about the chunk the +scan callback returns. + +**Symptom**: a table function whose scan claims a row from the writer and then finds +nothing to write (its source ended) still returns that row to DuckDB, holding whatever the +reused output buffer held last. Measured in the eighth review on DuckDB 1.5.6: a scan that +wrote 2 rows returned 3, and the third was the `7` an earlier chunk had left in that +position. For a fixed-width column that is a wrong answer, not undefined behaviour +(valgrind-clean). For a `LIST` or `MAP` column written by hand, a row never written keeps the +`{offset, length}` entry of an earlier chunk, which can point past the child data this chunk +filled (CONJECTURED, not reproduced). + +**Root cause**: the writer counts the rows you *claimed* and sets the chunk's size from that +count when it is dropped. DuckDB reuses output vectors across chunks, so a claimed row you +did not write is published with stale contents. + +**Fix**: claim a row only when you are going to write it: check that the source has another +value first, then claim the row and write it in the same step. A row you have already +claimed can be given back with `ChunkWriter::release_row(row)` before the writer is dropped. + +**Rule**: an output chunk holds the rows you wrote, not the rows you asked for. + +--- + ## P1: Library name must match extension name **Status**: Must be configured manually in `Cargo.toml`. @@ -599,7 +693,7 @@ crate-type = ["cdylib", "rlib"] --- -## P2: Extension metadata version is C API version, not DuckDB version +## P2: Metadata version is C API version, not DuckDB version **Status**: Must be handled manually when using `append_extension_metadata.py`. @@ -628,7 +722,7 @@ DuckDB 2.0 this was how quack-rs advised stamping a `duckdb-1-5` build; DuckDB 2 --- -## P3: E2E testing is mandatory — unit tests alone are insufficient +## P3: E2E testing is mandatory **Status**: Documented. See testing guide. @@ -688,7 +782,7 @@ from it and prints both commands if it is missing. --- -## P5: SQLLogicTest expected values must match actual DuckDB output +## P5: SQLLogicTest expected values must match exactly **Status**: Test-authoring care required. @@ -716,7 +810,7 @@ Additionally, use `duckdb_get_function` to verify registration in development. --- -## P7: `duckdb_string_t` format is undocumented in Rust bindings +## P7: `duckdb_string_t` format is undocumented **Status**: Handled by `VectorReader::read_str`, `VectorReader::read_blob`, `read_duck_string`, and `read_duck_blob`. @@ -754,7 +848,7 @@ Month conversion uses the approximation: **1 month = 30 days** (matching DuckDB' --- -## P9: `bundled-test` + `loadable-extension` dispatch-table conflict +## P9: `loadable-extension` dispatch table uninitialised in `cargo test` **Status**: Fixed in quack-rs v0.6.0. `InMemoryDb::open()` now initialises the dispatch table automatically. No user action required. @@ -1010,7 +1104,7 @@ implementation. `const char *` is a strong hint that it is borrowed, but `duckdb_parameter_name` proves it is only a hint. -## P12: `duckdb_client_context_get_config_option` aborts on a missing setting (debug builds) +## P12: `duckdb_client_context_get_config_option` aborts on a missing setting **Status**: Documented on `ClientContext::config_option`; the abort-free alternative is SQL. This is a DuckDB defect, not a quack-rs one. @@ -1111,6 +1205,58 @@ the entry. **Rule**: never read a list entry before the row's validity. +## P16: `duckdb_get_bit` and `duckdb_get_bignum` abort on NULL and on values that do not cast + +**Status**: Prevented in quack-rs's own wrappers, a DuckDB defect for everything else. +`Value::as_bit()` and `Value::as_bignum()` return `None` for a value they cannot read +instead of calling the C getter on it, and `Value::bit` and `Value::bignum` build only +well-formed values. Code that calls `duckdb_get_bit` or `duckdb_get_bignum` through the +raw bindings must make the checks below itself. + +**Symptom**: the process aborts with `fatal runtime error: Rust cannot catch foreign +exceptions` (`SIGABRT`, exit code 134) when `duckdb_get_bit` is called on a SQL `NULL` +value or on a value that does not cast to `BIT` (such as the string `'abc'`). Reproduced on +DuckDB 1.5.6 and 2.0.0. `duckdb_get_bignum` is built the same way (read in the 2.0.0 +source, not run). + +**Root cause**: both functions cast the value (`DefaultCastAs(BIT)` or `BIGNUM`) and read +the result with no `try`, so the cast's C++ exception leaves an `extern "C"` function. + +**Fix**: use `Value::as_bit()` / `Value::as_bignum()`. In raw code, before calling either +getter, check that the handle is not null (a null handle is dereferenced), that +`Value::type_id()` is the type you expect (convert with `Value::try_cast`, which cannot +throw, when it is not), and `Value::is_sql_null()`; these are plain accessors that cannot +throw. + +**Rule**: a C API getter that casts can throw; check the type and NULL-ness first, because +Rust cannot catch a C++ exception. + + +## P17: `std::fs` bypasses DuckDB's permission settings + +**Status**: Documented in the [Virtual File +System](https://quack-rs.com/duckdb-1-5/file-system.html#security-only-filesystem-honours-duckdbs-permission-settings) +chapter, in the table-function, replacement-scan and `COPY` chapters, in [Known +Limitations](https://quack-rs.com/reference/known-limitations.html) and in the FAQ. quack-rs's `FileSystem` honours the +settings; nothing can make `std::fs` do so. + +**Symptom**: with `SET enable_external_access = false` (or `allowed_directories` set), +`read_csv('data.csv')` fails with "Permission Error: Cannot access file … file system +operations are disabled by configuration", and so does a table function that opens the same +path through `FileSystem`. A table function that reads it with `std::fs::read` returns the +file. Measured on DuckDB 1.5.6 and 2.0.0. + +**Root cause**: DuckDB enforces the settings in its own file system layer. `std::fs` calls +the operating system directly and never consults it. + +**Fix**: open every path that comes from SQL (arguments, options, `COPY` targets, +replacement-scan names) with `FileSystem::from_client_context(...)` (feature `duckdb-1-5`). +Without that feature, take the data as a `VARCHAR` or `BLOB` argument. + +**Rule**: an extension runs with the process's full privileges; the only restrictions it +inherits are the ones it asks DuckDB to apply. + + ## Community Extension Submission ### Build System Requirements @@ -1227,9 +1373,10 @@ the ABI type stamped in their metadata footer: - A `C_STRUCT` extension declares a C extension API version (quack-rs stamps `v1.2.0`) and loads into every DuckDB release that provides that API — every - release from 1.4.4 through 2.0.0, for quack-rs's stable-API surface. One + release from 1.4.4 through 2.0.x, for quack-rs's stable-API surface. One binary serves them all. An extension using the `duckdb-1-5` features is stamped - C API `v1.5.6` and loads into 1.5.6 and 2.0.0. + C API `v1.5.6` and loads into 1.5.6 and 2.0.0 (the default `Strict` guard refuses a + release quack-rs has not listed, such as 2.0.1). - A `C_STRUCT_UNSTABLE` or `CPP` extension declares an exact DuckDB release and loads only into that release, so it is rebuilt for each one. DuckDB 2.0 loads a `C_STRUCT_UNSTABLE` binary only through the C API v2 entry point, which quack-rs diff --git a/README.md b/README.md index f8fb0c03..cf48ebf2 100644 --- a/README.md +++ b/README.md @@ -11,21 +11,35 @@ License: MIT MSRV: 1.86.0

+

+ Book  ·  + API reference  ·  + Changelog  ·  + Known limitations +

-**Write DuckDB loadable extensions in pure Rust — no C, no C++, no glue code.** +**Write DuckDB loadable extensions in Rust, with no C or C++ to write.** +Pre-1.0: the API changes between minor releases, and audits keep finding +defects. Read [Status](#status) before you depend on it. ## What it is `quack-rs` wraps DuckDB's [C Extension API](https://duckdb.org/community_extensions/development) -so that you can write a loadable extension entirely in Rust. +so that you can write a loadable extension entirely in Rust. Your repository +holds Rust only: no C or C++ source, no CMake file, no glue layer of your own +between your functions and the C API. The build still runs other people's C and +tools: dependencies of the DuckDB bindings may compile C, the `bundled-test*` +features compile C++ or C, and the community-extension flow needs Python and git +([Installation](https://quack-rs.com/getting-started/installation.html#build-requirements) +has the list). - **Every function kind the C API can register:** scalar, aggregate, table, cast, copy and replacement scan functions, plus SQL macros. - **Scalar functions as safe closures.** `ScalarFunctionBuilder::map1` and its siblings take a plain Rust closure, derive the SQL types from its signature, return NULL for NULL input, and turn a panic into a SQL error. -- **Guards against documented FFI pitfalls.** Most of the 34 pitfalls in +- **Guards against documented FFI pitfalls.** Most of the 38 pitfalls in [`LESSONS.md`](./LESSONS.md) are prevented by a wrapper, such as NULL output that is silently lost, a boolean read that is undefined behaviour, or a double free in an aggregate destructor. The rest are documented. @@ -52,20 +66,26 @@ nothing can be caught, and a panic kills the user's DuckDB process. name = "my_extension" version = "0.1.0" edition = "2021" -rust-version = "1.86" # with resolver 3, Cargo picks dependency versions that build on it +# With resolver 3, Cargo picks versions that build on this. +rust-version = "1.86" resolver = "3" [lib] -name = "my_extension" # must match the extension name (Pitfall P1) +# Must match the extension name (Pitfall P1). +name = "my_extension" crate-type = ["cdylib", "rlib"] [dependencies] -quack-rs = "0.19" -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] [profile.release] -panic = "unwind" # required -lto = true # the rest are recommended +panic = "unwind" # required +lto = true # the rest are recommended opt-level = 3 codegen-units = 1 strip = true @@ -79,17 +99,25 @@ use quack_rs::entry_point; use quack_rs::error::ExtensionError; use quack_rs::scalar::ScalarFunctionBuilder; -fn register(con: duckdb_connection) -> Result<(), ExtensionError> { - // Parameter and return types come from the closure signatures. - // A NULL argument gives a NULL result without calling the closure. - // `try_map1` lets the closure fail: a plain `x * 2` wraps on overflow in a - // release build, and `Err` becomes a SQL error instead. - let double_it = ScalarFunctionBuilder::try_map1("double_it", |x: i64| { - x.checked_mul(2).ok_or("double_it: BIGINT overflow") - })?; - let shout = ScalarFunctionBuilder::map1_str("shout", |s: &str| s.to_uppercase())?; - - // SAFETY: `con` is the live connection DuckDB passed to the entry point. +fn register( + con: duckdb_connection, +) -> Result<(), ExtensionError> { + // Types come from the closure signatures. A NULL + // argument gives a NULL result without calling the + // closure. `try_map1` lets the closure fail: a plain + // `x * 2` wraps on overflow in a release build, and + // `Err` becomes a SQL error instead. + let double_it = ScalarFunctionBuilder::try_map1( + "double_it", + |x: i64| x.checked_mul(2).ok_or("double_it: overflow"), + )?; + let shout = ScalarFunctionBuilder::map1_str( + "shout", + |s: &str| s.to_uppercase(), + )?; + + // SAFETY: `con` is the live connection DuckDB passed to + // the entry point. unsafe { double_it.register(con)?; shout.register(con)?; @@ -111,11 +139,13 @@ project pins an older toolchain. Your extension itself still builds on 1.86. ```shell cargo build --release cargo install quack-rs --bin append_metadata -append_metadata target/release/libmy_extension.so my_extension.duckdb_extension \ +append_metadata target/release/libmy_extension.so \ + my_extension.duckdb_extension \ --abi-type C_STRUCT --extension-version v0.1.0 \ --duckdb-version v1.2.0 --platform linux_amd64 -duckdb -unsigned -c "LOAD './my_extension.duckdb_extension'; SELECT double_it(21), shout('hello');" +duckdb -unsigned -c "LOAD './my_extension.duckdb_extension'; +SELECT double_it(21), shout('hello');" # ┌───────────────┬────────────────┐ # │ double_it(21) │ shout('hello') │ # │ int64 │ varchar │ @@ -141,12 +171,12 @@ line. Checked against the published sources of `duckdb` and `duckdb-loadable-macros` 1.20000.1, `duckdb-neo` 0.20000.1 and the `extension-template-rs` repository's `main` on 2026-10-29: -| | quack-rs | `duckdb` + `duckdb-loadable-macros` | `extension-template-rs` + `duckdb-neo` | +| | quack-rs | `duckdb` + macros | template + `duckdb-neo` | |---|---|---|---| -| C API | v1, stable | v1 by default (`_init_c_api`); v2 behind the macros' `neo` feature | v2, unstable (`_init_c_api_v2`) | -| Binary loads in | every DuckDB from 1.4.4 through 2.0.x, with the default features | set by the C API v1 stamp, as for quack-rs | only the DuckDB release it was built for ("Binaries load only in the exact TARGET_DUCKDB_VERSION", the template's `Makefile`) | -| Wraps | scalar, aggregate, table, cast, copy and replacement-scan functions; SQL macros | scalar (`vscalar`) and table (`vtab`) functions | modules for scalar, aggregate, table, cast, copy, macro and pragma functions, replacement scans, custom types and options, and log sinks | -| Status | pre-1.0, audited (below) | the `duckdb` crate | `duckdb-neo` describes itself as experimental | +| C API | v1, stable | v1 by default; v2 behind the macros' `neo` feature | v2, unstable | +| Binary loads in | every DuckDB 1.4.4 to 2.0.x (default features; tested to 2.0.0) | as quack-rs: set by the C API v1 stamp | the one DuckDB release it was built for (the template's `Makefile` says so) | +| Wraps | scalar, aggregate, table, cast, copy, replacement scan; macros | scalar and table functions | scalar, aggregate, table, cast, copy, macro and pragma functions; scans, types, options, log sinks | +| Status | pre-1.0 (see Status) | the `duckdb` crate | `duckdb-neo` calls itself experimental | Use `duckdb-neo` if you target DuckDB 2.0 and later only and want upstream's wrapper, or need what only the v2 API offers (pragma functions, for one). @@ -157,19 +187,23 @@ declares part of that API stable. ## Status -`quack-rs` is pre-1.0 (0.19.0). Judge it against your own requirements; the -record so far: +`quack-rs` is pre-1.0. Judge it against your own requirements; the record so +far: - **The API still changes.** A minor release before 1.0 can break it; each lists its breaking changes and migration notes in [`CHANGELOG.md`](./CHANGELOG.md). -- **Audits keep finding real defects.** The review released as 0.16.0 fixed - defects in earlier releases, including two heap-corruption paths. The 0.18.0 - audits fixed further soundness holes in the safe API, process aborts and wrong - answers, and found a serious defect in unreleased code before publication: - aggregate functions returned wrong results in release builds with Cargo's - default profile. [`AUDIT.md`](./AUDIT.md) records each audit, what it found, - and whether each fix was reproduced against a real DuckDB or derived from - DuckDB's source. +- **Audits keep finding real defects.** The 0.16.0 release, which came out of a + review, fixed defects in earlier releases, including two heap-corruption + paths. The 0.18.0 audits fixed further soundness holes in the safe API, + process aborts and wrong answers, and found a serious defect in unreleased + code before publication: aggregate functions returned wrong results in release + builds with Cargo's default profile. [`AUDIT.md`](./AUDIT.md) records each + audit, what it found, and whether each fix was reproduced against a real + DuckDB or derived from DuckDB's source. **Use 0.18.0 or later.** Earlier + releases still have the soundness holes in the safe API, and 0.7.0 through + 0.15.0 can corrupt the heap when the `duckdb-1-5` features are used. +- **Maintenance.** One maintainer ([`GOVERNANCE.md`](./GOVERNANCE.md)). + [`SECURITY.md`](./SECURITY.md) lists the release lines that receive fixes. - **Some limits are DuckDB's.** Defects in DuckDB's C API that quack-rs can only document or work around are listed under [Known Limitations](https://quack-rs.com/reference/known-limitations.html). @@ -185,11 +219,18 @@ release you support. ## Supported versions -- **DuckDB:** 1.4.4 through 2.0.0. The `libduckdb-sys = ">=1.4.4, <2"` range - covers the bindings for all of them, 1.4.4 through 1.20000.0 (DuckDB 2.0, - published 2026-10-22). 1.20000.0 needs Rust 1.88; on 1.88 or newer +- **DuckDB:** with the default features, 1.4.4 through 2.0.x (the tests ran + against 2.0.0, the newest release when 0.20.0 was published). With a + `duckdb-1-5*` feature, only the releases whose C API layout quack-rs has + verified: 1.5.6 and 2.0.0. Any other release, a future 2.0.1 included, is + refused at `LOAD` (see below). +- **`libduckdb-sys`:** `>=1.4.4, <1.20100`. That covers the bindings for every + DuckDB release above, 1.4.4 through 1.20000.x (the DuckDB 2.0 bindings, first + published 2026-10-22). 1.20000.x needs Rust 1.88; on 1.88 or newer `cargo update` selects it, and an older toolchain with MSRV-aware resolution - stays on 1.10506.x (DuckDB 1.5.6). + stays on 1.10506.x (DuckDB 1.5.6). The upper bound keeps out the DuckDB 2.1 + bindings (1.20100.x) until a quack-rs release has been tested against them + ([why](https://quack-rs.com/getting-started/installation.html#adding-quack-rs-to-an-existing-extension)). - **Rust (MSRV):** 1.86.0. DuckDB 2.0.0 (2026-10-21) is a long-term support release, supported until at @@ -197,12 +238,21 @@ least October 2027; DuckDB supports 1.4 until late November 2026 and 1.5 until March 2027. With the default features, one extension binary loads into every DuckDB release -from 1.4.4 through 2.0.0, because it is stamped with C API version `v1.2.0`. +from 1.4.4 through 2.0.x, because it is stamped with C API version `v1.2.0`. DuckDB 1.4.4 to 1.5.5 declare that version; 1.5.6 and 2.0.0 declare `v1.5.6` and still load `v1.2.0` extensions. The `duckdb-1-5*` features use the part of -DuckDB's C API whose layout differed between releases before 1.5.6: build such -an extension against the 1.5.6 bindings and stamp it `C_STRUCT` with C API -`v1.5.6`, and it loads into DuckDB 1.5.6 and 2.0.0. See +DuckDB's C API whose layout changed between releases up to 1.5.5 (DuckDB +labels C API v1.5.6 stable, with the caveat below). Build such an extension against the 1.5.6 +bindings and stamp it `C_STRUCT` with C API `v1.5.6`; it then loads into DuckDB +1.5.6 and 2.0.0. quack-rs has verified that layout for DuckDB 1.5.2 to 1.5.6 and +2.0.0 only, and DuckDB's 2.0.0 header says the API is guaranteed for that exact +release (2.0.0 added two slots to the range that 1.5.6 declared stable). So under +the default `Strict` policy such an extension refuses to load into a release +quack-rs has not listed, with an error that says how to fix it. When one appears +(DuckDB 2.0.1, for example), either rebuild with +`QUACK_RS_TARGET_DUCKDB_VERSION` set to that release, once you have checked it, +or move to a quack-rs release that lists it; or stay on the default features, +which are not affected. See [ABI Compatibility](https://quack-rs.com/concepts/abi.html) before you enable one. ## Cargo features @@ -232,8 +282,12 @@ struct layouts first in 1.10504.0, although the DuckDB functions exist from - **API reference**: [docs.rs/quack-rs](https://docs.rs/quack-rs). The crate root lists every module. - [Pitfall Catalog](https://quack-rs.com/reference/pitfalls.html) and - [`LESSONS.md`](./LESSONS.md): all 34 known pitfalls (L1–L19, P1–P15), with + [`LESSONS.md`](./LESSONS.md): all 38 known pitfalls (L1–L21, P1–P17), with symptom, cause and fix. +- [`examples/hello-ext`](./examples/hello-ext): one extension with scalar, + aggregate, table, cast, macro and replacement-scan functions, checked in CI. + It uses raw callbacks, which are not panic-guarded: read it for the C API + calls, do not copy it as a template. - [Community Extensions](https://quack-rs.com/publishing.html): `description.yml`, naming, versioning, platforms and submission. - [Testing Guide](https://quack-rs.com/testing.html): mocks, `AggregateTestHarness` @@ -241,12 +295,22 @@ struct layouts first in 1.10504.0, although the DuckDB functions exist from - [`docs/architecture.md`](./docs/architecture.md): module layout, design principles and the architecture decision records. - [`CHANGELOG.md`](./CHANGELOG.md): every release, with breaking changes and - migration notes. The latest is 0.19.0 (2026-10-22). + migration notes. The latest is 0.20.0 (2026-10-30). - [`CONTRIBUTING.md`](./CONTRIBUTING.md): development setup, quality gates and the test strategy. - [`SECURITY.md`](./SECURITY.md): supported versions and how to report a vulnerability. +## Getting help + +- A bug, or something the book does not answer: + [GitHub issues](https://github.com/tomtom215/quack-rs/issues). The bug report + template asks for the DuckDB version that fails and the footer stamp. +- `LOAD` fails before your code runs: the [FAQ](https://quack-rs.com/faq.html) + lists the messages DuckDB gives and what causes each. +- A vulnerability: report it privately, as [`SECURITY.md`](./SECURITY.md) + describes. + ## License MIT — see [`LICENSE`](./LICENSE). diff --git a/RELEASING.md b/RELEASING.md index 68b12885..81bf0212 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -13,8 +13,8 @@ Before you can cut a release, ensure the following are in place: |-------------|-------------------| | Push access to `main` and ability to push tags | GitHub repository settings | | crates.io **Trusted Publishing** for `quack-rs`: repository `tomtom215/quack-rs`, workflow `release.yml`, environment `crates-io` | crates.io → the crate's Settings → Trusted Publishing | -| `CARGO_REGISTRY_TOKEN` secret, **only until Trusted Publishing is configured**; then delete it | Settings → Secrets and variables → Actions | -| `crates-io` environment with yourself as required reviewer | Settings → Environments → crates-io | +| `CARGO_REGISTRY_TOKEN`, **only until Trusted Publishing is configured**; then delete it. Store it as a secret **of the `crates-io` environment**, never as a repository secret | Settings → Environments → crates-io → Environment secrets | +| `crates-io` environment with yourself as required reviewer, and **Deployment branches and tags** limited to the tag pattern `v*` | Settings → Environments → crates-io | | A tag ruleset restricting who may create `v*` tags | Settings → Rules → Rulesets | | Branch protection on `main` requiring the single check **CI passed** | Settings → Rules (or Branches) | | GPG key configured for signed tags | `git config --global user.signingkey ` | @@ -27,6 +27,32 @@ fails. `release.yml` runs from the tagged tree, so the tag ruleset is what stops a tag on an edited workflow; the workflow itself refuses a lightweight tag and a tag on a commit that is not on `main`. +Where the fallback token lives matters. A repository secret is readable by +every workflow in the repository: anyone who can push a branch can add a +workflow that prints it, and the `crates-io` environment's required reviewer is +never consulted. An environment secret is released only to a job that names +the environment, after the reviewer approves, and only for the refs the +environment allows. + +### Setting up Trusted Publishing (once, in this order) + +1. crates.io → `quack-rs` → Settings → Trusted Publishing → add a GitHub + publisher: owner `tomtom215`, repository `quack-rs`, workflow `release.yml`, + environment `crates-io`. +2. GitHub → Settings → Environments → `crates-io`: required reviewers, and + deployment branches and tags limited to `v*`. If a `CARGO_REGISTRY_TOKEN` + repository secret exists, move it into this environment's secrets and + delete the repository secret, as a stop-gap until step 4. +3. Cut the next release. The `publish` job must log `Publishing with a Trusted + Publishing token.` (not the `::warning::` about the fallback), and the + crates.io version page must show the publisher. +4. Only then: delete `CARGO_REGISTRY_TOKEN` (environment and repository), revoke + the token on crates.io, and remove the `CARGO_REGISTRY_TOKEN` fallback and + the `continue-on-error` of the `trusted` step from `release.yml`, so a failed + exchange fails the release instead of reaching for a long-lived credential. +5. If the crate's Trusted Publishing settings offer to disallow publishing with + API tokens, turn that on. + --- ## Semantic versioning policy @@ -77,8 +103,12 @@ Pre-release versions use the suffix convention `vX.Y.Z-alpha.N`, # 1. Prepare vim CHANGELOG.md # Move [Unreleased] → [X.Y.Z] - YYYY-MM-DD vim Cargo.toml # version = "X.Y.Z" +# Every committed lockfile records quack-rs's own version; release.yml packages +# and publishes with --locked, which refuses a stale one (Step 3). +for d in . examples/hello-ext fuzz book/doctest benches/engine; do (cd "$d" && cargo update --workspace); done scripts/sync-book-changelog.py # regenerate the book's changelog page # ...and bump every "0.X" version snippet in README.md / book / CITATION.cff (Step 3b) +python3 scripts/check-release-metadata.py # lists every place still on the old version git add -A git commit -m "chore: release vX.Y.Z" git push origin main @@ -112,11 +142,16 @@ literally could tag a release with `Miri`, `LeakSanitizer`, `osv-scan`, ```bash gh run list --branch main --workflow ci.yml --limit 1 -gh run view # every job must be green; none may be skipped +gh run view # every blocking job must be green; none may be skipped ``` Do not enumerate the jobs here again. Branch protection on `main` is the -mechanism that should enforce this; this step is the human double-check. +mechanism that should enforce this; this step is the human double-check. The +jobs `.github/workflows/README.md` marks *informational* (`clippy-beta`, +`latest-stable`, `nightly`, `test-bundled-duckdb-2-0`) may be red without +blocking, but read why before tagging: a red `latest-stable`, for example, +means the newest Rust reports something the pinned one does not, which the +next toolchain bump will have to fix (CONTRIBUTING.md, "The Rust toolchain"). ### Step 2 — Update CHANGELOG.md @@ -148,14 +183,17 @@ if the two diverge). Do not edit it by hand. ### Step 2b — Update SECURITY.md If this is a **minor or major** release, update the supported versions table in -`SECURITY.md` to list the new version as supported and mark old versions -as end-of-life if appropriate. +`SECURITY.md`: add a `| X.Y.x | Yes` row for the new line, and end the support +of the line before the previous one (the table's policy is the two newest minor +lines). The versioned book (quack-rs.com) publishes a copy for every release line from `MIN_RELEASE` in `scripts/build-versioned-site.py` upwards. When a line goes end-of-life here, raise `MIN_RELEASE` to the oldest line still supported, in the same commit, so the site stops serving documentation for releases nobody -should start with. +should start with. `scripts/check-release-metadata.py` fails while the table +and `MIN_RELEASE` disagree, or while the table has no row for the version in +`Cargo.toml`. ### Step 3 — Bump the version in `Cargo.toml` @@ -164,27 +202,63 @@ should start with. version = "0.3.0" # ← update this ``` -Do **not** update `Cargo.lock` manually — it will update itself on next build. +Then bring every committed lockfile along, **in the same commit**. Each one +records quack-rs's own version: the root `Cargo.lock` as the package itself, +the others (`examples/hello-ext`, `fuzz`, `book/doctest`, `benches/engine`) as +a path dependency: + +```bash +for d in . examples/hello-ext fuzz book/doctest benches/engine; do (cd "$d" && cargo update --workspace); done +git diff --stat -- '*Cargo.lock' # one line changed in each +``` + +`--workspace` updates only the workspace's own packages, so no dependency +moves. This is not optional: `release.yml` runs `cargo package --locked` and +`cargo publish --locked`, which refuse a `Cargo.lock` that still names the +previous version ("cannot update the lock file ... because --locked was +passed"), and nothing in CI commits a lock file it rewrote. A `Cargo.toml` bump +without its lockfiles used to pass every pull-request job and fail the +`package` job after the whole of `ci.yml` had run. `ci.yml`'s publish dry-run +now uses `--locked` too, and `scripts/check-release-metadata.py` (the `doc` +job, and `validate` in `release.yml`) names any lockfile left behind. ### Step 3b — Bump the version in the documentation snippets, in the same PR Every `quack-rs = "0.X"` / `version = "0.X"` snippet a reader copies — in -`README.md`, the book, the feature documentation in `Cargo.toml`, and -`CITATION.cff` — must name the new version **in the release PR itself**. The -commit that gets tagged is that PR's merge commit, so a snippet bumped in a -follow-up PR is wrong in the tagged source, in the published `.crate` (whose -README is what crates.io renders) and in the docs.rs build of the release. -Find the stale ones by grepping for the previous version, for example (replace -`` with the previous minor, dots escaped — `0\.17` when releasing 0.18.0): +`README.md`, the book, the feature documentation in `Cargo.toml`, the `src/` +rustdoc, and `CITATION.cff` — must name the new version **in the release PR +itself**, as must `SECURITY.md` (Step 2b) and the README's "The latest is ..." +line. The commit that gets tagged is that PR's merge commit, so a snippet +bumped in a follow-up PR is wrong in the tagged source, in the published +`.crate` (whose README is what crates.io renders) and in the docs.rs build of +the release. Find the stale ones by grepping for the previous version, for +example (replace `` with the previous minor, dots escaped — `0\.17` when +releasing 0.18.0): ```bash git grep -n '' -- README.md book/src Cargo.toml CITATION.cff src ``` +Two places hold a date, not a version, so the grep cannot find them: +`CITATION.cff`'s `date-released` and the README's "The latest is X.Y.Z +(YYYY-MM-DD)". Give both the date of the CHANGELOG heading (the day you push +the tag). The `publish` job waits for approval and can finish on the next day: +0.19.0 is dated 2026-10-22 everywhere, and crates.io's `created_at` for it is +2026-10-23T06:07Z. + +`python3 scripts/check-release-metadata.py` does all of this mechanically: the +lockfiles, the CHANGELOG heading and its two link definitions, `CITATION.cff` +(version and date), the `SECURITY.md` row and `MIN_RELEASE`, the README line, +and every snippet. CI runs it in the `doc` job, so a release PR cannot merge +with a place missed. For a pre-release version only the lockfile and CHANGELOG +checks apply: the docs keep naming the latest stable release. + ### Step 4 — Commit the version bump ```bash -git add CHANGELOG.md Cargo.toml # plus the snippets from Step 3b +# ...plus SECURITY.md and the snippets from Steps 2b and 3b +git add CHANGELOG.md Cargo.toml Cargo.lock examples/hello-ext/Cargo.lock \ + fuzz/Cargo.lock book/doctest/Cargo.lock benches/engine/Cargo.lock git commit -m "chore: release v0.3.0" git push origin main ``` @@ -219,7 +293,9 @@ git push origin "v0.3.0" ``` A lightweight tag (`git tag v0.3.0`) is refused by the `validate` job, as is a -tag on a commit that is not on `main`. +tag on a commit that is not on `main`. `validate` checks that the tag is +annotated, not that it is signed: `git verify-tag` (above, and Step 9) is the +only signature check. ### Step 6 — Monitor the release workflow @@ -227,38 +303,48 @@ Navigate to **Actions → Release** in the GitHub UI. The workflow runs these jo in order: ``` -validate Tag is annotated and on main; tag format, Cargo.toml version, dated CHANGELOG entry - │ +validate Tag is annotated and on main; tag format, Cargo.toml version, dated CHANGELOG + │ entry, and scripts/check-release-metadata.py (lockfiles, CITATION.cff, SECURITY.md, snippets) ci All of ci.yml (workflow_call): every gate a pull request runs, │ including semver, Miri, the sanitizers, cargo-deny, OSV, the │ publish dry-run and extension-load on every pinned DuckDB -package Build .crate, generate SHA256SUMS, attest SLSA provenance +package Build .crate, generate SHA256SUMS, attest SLSA provenance; record rustc │ publish ← awaits manual approval in the crates-io environment; - │ cargo publish, then checks crates.io serves the attested .crate + │ re-packages with the recorded rustc and stops unless the .crate's SHA-256 + │ equals SHA256SUMS, then cargo publish, then checks crates.io serves the attested .crate github-release Create GitHub release with notes, .crate, SHA256SUMS │ docs Dispatch docs.yml on main: the book for this release at / and /vX.Y/ ``` -Each job writes a structured summary visible in the Actions UI. +`publish` and `github-release` write a summary to the Actions UI; the other +jobs report in their logs. ### Step 7 — Approve the crates.io deployment The `publish` job is gated behind the `crates-io` environment, which requires a manual approval from an authorised reviewer. Before approving, read the -`package` job's summary (the `.crate` contents and size). +`package` job's log (the `.crate` file list in the "Package contents" group, +and its size in the `::notice::` line). 1. In the Actions run for the release workflow, click **Review deployments** 2. Select the `crates-io` environment 3. Click **Approve and deploy** -The job publishes with a Trusted Publishing token (or, until that is set up, -the `CARGO_REGISTRY_TOKEN` secret), using `cargo publish --locked --no-verify`: -ci.yml's publish dry-run already built the packaged sources, and skipping the -second build keeps dependency build scripts away from the token. It then -downloads the crate from crates.io and fails if its SHA-256 differs from the -attested artifact's (`cargo package` is reproducible, so they must match). +`publish` first installs the rustc the `package` job recorded (the release +`rust-toolchain.toml` pins; it refuses to continue with any other), packages +the tagged commit again with `cargo package --no-verify --locked`, and stops +unless that `.crate`'s SHA-256 equals the attested `SHA256SUMS`: a crates.io +version can be yanked but never replaced, so a difference must stop the +release before the upload, and it does so before any credential is requested. +It then publishes with a Trusted Publishing token (or, until that is set up, +the `CARGO_REGISTRY_TOKEN` environment secret), using `cargo publish --locked +--no-verify`: ci.yml's publish dry-run already built the packaged sources, and +skipping the second build keeps dependency build scripts away from the token. +Finally it downloads the crate from crates.io and fails if its SHA-256 differs +from the attested artifact's (`cargo package` is reproducible: crates.io's +0.19.0 is byte-identical to a fresh `cargo package` of the v0.19.0 commit). ### Step 8 — Review the GitHub release @@ -305,7 +391,8 @@ curl -fsSL "https://docs.rs/crate/quack-rs/$V/status.json" curl -fsSL https://quack-rs.com/versions.json # "latest" must be "v${V%.*}" and a version entry must read "$V". release.yml's # `docs` job dispatches docs.yml on main after publishing; if it did not run, -# run `gh workflow run docs.yml --ref main`. +# run `gh workflow run docs.yml --ref main`. For a pre-release, skip this check: +# the `docs` job does not run and versions.json does not change. ``` If any of them fails, the release is incomplete: finish it (push the tag, @@ -344,26 +431,45 @@ sha256sum quack-rs-0.3.0.crate ## Hotfix releases -For a critical bug in an already-published version: +For a critical bug in an already-published version. The `validate` job refuses +a tag whose commit is not an ancestor of `main`, so the hotfix branch has to +reach `main` *before* it is tagged, and by a merge commit: a squash merge or a +cherry-pick creates a different commit, and the hotfix tip is then not on +`main` (tagging the branch and cherry-picking the fix onto `main` afterwards, +as this section used to say, is refused). ```bash # Branch from the release tag git checkout -b hotfix/0.2.1 v0.2.0 -# Apply the fix, update CHANGELOG and Cargo.toml +# Apply the fix; update CHANGELOG.md, Cargo.toml and the lockfiles (Steps 2-4) # ... make changes ... git add -p git commit -m "fix: " +git push origin hotfix/0.2.1 + +# Open a pull request from hotfix/0.2.1 into main and merge it with +# "Create a merge commit". main may carry newer unreleased changes: that is +# fine, the tag below points at the hotfix tip, not at the merge. +# Then tag the hotfix tip (follow steps 5–9 above): +git fetch origin +git tag -s "v0.2.1" -m "Hotfix v0.2.1" origin/hotfix/0.2.1 +git push origin "v0.2.1" +``` -# Tag and push (follow steps 5–9 above) -git tag -s "v0.2.1" -m "Hotfix v0.2.1" -git push origin hotfix/0.2.1 "v0.2.1" +After the merge `main` must still agree with itself (`scripts/check-release-metadata.py` +runs on it). Where only the hotfix changed the version, the lockfiles, the +CHANGELOG, `CITATION.cff` or the README's "latest" line, `main` simply takes +them, which is right: they describe the newest release. Where `main` changed +them too (it is already on a newer version), keep `main`'s and add the hotfix's +CHANGELOG section below `main`'s own. -# After release: cherry-pick the fix onto main -git checkout main -git cherry-pick -git push origin main -``` +The release workflow that runs is the one in the tagged commit, i.e. the +hotfix branch's: a hotfix of a line released before this runbook gained the +release-metadata check runs that line's older `validate`. Copy +`scripts/check-release-metadata.py` from `main` into the hotfix checkout and +run it there before tagging (it checks the tree it sits in); do not commit it +on the hotfix branch unless the fix needs it. --- @@ -407,17 +513,36 @@ git push origin --delete "v0.3.0" Add a `## [0.3.0] - YYYY-MM-DD` section to `CHANGELOG.md`, commit, push, then delete and recreate the tag. -### `security` fails: advisory found +### `validate` fails: release metadata + +`scripts/check-release-metadata.py` lists each place that still names another +version (a lockfile, a CHANGELOG link, `CITATION.cff`, `SECURITY.md`, a +snippet). Run it locally, fix what it lists on `main`, then delete and recreate +the tag on the new commit (Steps 3-5). + +### `security` or `osv-scan` fails: advisory found A dependency has a known vulnerability. Either: - Update the affected dependency, or -- Add the advisory ID to `deny.toml`'s `[advisories] ignore` list with a - documented justification comment +- Suppress it with a documented justification: in `deny.toml`'s `[advisories] + ignore` for the root graph, in `deny.ci-crates.toml` for the CI-only graphs + (hello-ext, fuzz, book/doctest, benches/engine), and in `osv-scanner.toml` + for the `osv-scan` job, which reads every committed lockfile and has its own + rules for a suppression (the file's header). + +### `publish` stops before uploading: the `.crate` differs + +"this job packages a different quack-rs-X.Y.Z.crate than the attested one; +nothing was published" (or "this job's rustc is ..."): crates.io has nothing, +so nothing needs undoing. The `package` job's `.crate` and the one `publish` +just built from the same commit differ, which `cargo package` reproducibility +says they must not; compare the two file lists (`tar -tzf`) before re-running. ### `publish` fails: token invalid Regenerate the crates.io API token and update the `CARGO_REGISTRY_TOKEN` -repository secret. +secret of the `crates-io` environment, or finish the Trusted Publishing set-up +(Prerequisites), which needs no token. ### `publish` fails: crate already published @@ -441,9 +566,10 @@ Every release produces and attaches: | GitHub release notes | Extracted from `CHANGELOG.md` | The attached `.crate` is the one the `package` job built and attested. The -`publish` job packages again from the same tagged commit, so what crates.io -serves should hash the same, and the `publish` job's "crates.io serves the -attested .crate" step fails the release if it does not. To check by hand, -compare `sha256sum` of +`publish` job packages again from the same tagged commit with the same rustc, +and refuses to upload unless that `.crate` hashes the same as `SHA256SUMS` +("The .crate about to be published is the attested one"); after the upload, +"crates.io serves the attested .crate" checks crates.io's copy against the same +file. To check by hand, compare `sha256sum` of `https://static.crates.io/crates/quack-rs/quack-rs-X.Y.Z.crate` against `SHA256SUMS`, or check the attached file with `gh attestation verify`. diff --git a/SECURITY.md b/SECURITY.md index 98ab5fb8..f98a20b9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,30 +5,42 @@ ## Supported Versions -| Version | Supported | -|---------|--------------------| -| 0.19.x | Yes | -| 0.18.x | Yes | -| 0.16.x | No (end-of-life) | -| 0.15.x | No (end-of-life) | -| 0.14.x | No (end-of-life) | -| 0.13.x | No (end-of-life) | -| 0.12.x | No (end-of-life) | -| 0.11.x | No (end-of-life) | -| 0.10.x | No (end-of-life) | -| 0.9.x | No (end-of-life) | -| 0.8.x | No (end-of-life) | -| 0.7.x | No (end-of-life) | -| 0.6.x | No (end-of-life) | -| 0.5.x | No (end-of-life) | -| 0.4.x | No (end-of-life) | -| 0.3.x | No (end-of-life) | -| 0.2.x | No (end-of-life) | -| 0.1.x | No (end-of-life) | - -0.17.x does not appear in this table because it was never published: 0.17.0 -was prepared but not released, and its changes ship in 0.18.0 (see the 0.18.0 -entry in `CHANGELOG.md`). On crates.io, 0.18.0 follows 0.16.0. +| Version | Supported | +|----------------|-----------------------------------------| +| 0.20.x | Yes | +| 0.19.x | Yes | +| 0.18.x | No (end-of-life) | +| 0.16.x | No (end-of-life) | +| 0.3.x – 0.15.x | No (end-of-life); **do not use** (below) | + +The two newest minor lines are supported. quack-rs is pre-1.0, so Cargo +treats each minor release as a new major line: a project that depends on +`quack-rs = "0.19"` never moves to 0.20 by itself. The previous line therefore +keeps receiving security fixes, as patch releases of that line, until the next +minor release ends its support. 0.18.x reached end-of-life with 0.20.0. + +**Releases before 0.16.0 must not be used.** They carry the memory-safety +defects fixed in 0.16.0 (the "Security" part of its entry in `CHANGELOG.md`). +Checked against the published crates: + +- `DuckStringView::from_bytes` (0.3.0 – 0.15.0) was a safe function that + dereferenced a pointer read from its argument's bytes, so safe code could + read arbitrary memory. +- `CopyGlobalInitInfo::get_file_path` (0.8.0 – 0.15.0) freed a string DuckDB + still owned, corrupting the heap on every `COPY ... TO` through a quack-rs + copy function. +- With the `duckdb-1-5` feature (0.4.0 – 0.15.0), nothing checked that the + running DuckDB's C API function table had the layout the extension was + compiled against; loaded into another DuckDB release, such a build could + call the wrong functions (`double free or corruption`). + +None of these versions has been yanked from crates.io; upgrade to a supported +line. + +crates.io has no 0.1.x or 0.2.x: the first published version is 0.3.0. 0.17.x +does not appear either because it was never published: 0.17.0 was prepared +but not released, and its changes ship in 0.18.0 (see the 0.18.0 entry in +`CHANGELOG.md`). On crates.io, 0.18.0 follows 0.16.0. ## Reporting a Vulnerability diff --git a/benches/engine/Cargo.lock b/benches/engine/Cargo.lock new file mode 100644 index 00000000..8c640a40 --- /dev/null +++ b/benches/engine/Cargo.lock @@ -0,0 +1,722 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libduckdb-sys" +version = "1.10506.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "113d1c26c9f518405a17b5729bb6ab6abf029fdc46626406922cf9a8dc9b4609" +dependencies = [ + "flate2", + "prettyplease", + "quote", + "serde", + "serde_json", + "syn 2.0.119", + "tar", + "ureq", + "zip", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quack-rs" +version = "0.20.0" +dependencies = [ + "cc", + "libduckdb-sys", +] + +[[package]] +name = "quack-rs-engine-bench" +version = "0.0.0" +dependencies = [ + "libduckdb-sys", + "quack-rs", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "typed-path" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "xattr" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zip" +version = "9.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb" +dependencies = [ + "crc32fast", + "flate2", + "getrandom 0.4.3", + "indexmap", + "memchr", + "nix", + "typed-path", + "windows-sys 0.61.2", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/benches/engine/Cargo.toml b/benches/engine/Cargo.toml index 34856f9c..b10b7126 100644 --- a/benches/engine/Cargo.toml +++ b/benches/engine/Cargo.toml @@ -18,7 +18,7 @@ crate-type = ["cdylib"] [dependencies] quack_rs = { path = "../..", package = "quack-rs" } -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } [profile.release] panic = "unwind" diff --git a/book/doctest/Cargo.lock b/book/doctest/Cargo.lock index a7c65de1..28c21596 100644 --- a/book/doctest/Cargo.lock +++ b/book/doctest/Cargo.lock @@ -299,9 +299,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "shlex", @@ -493,9 +493,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -966,7 +966,7 @@ dependencies = [ [[package]] name = "quack-rs" -version = "0.19.0" +version = "0.20.0" dependencies = [ "cc", "duckdb", diff --git a/book/doctest/Cargo.toml b/book/doctest/Cargo.toml index bcaeb059..4c1ac1e4 100644 --- a/book/doctest/Cargo.toml +++ b/book/doctest/Cargo.toml @@ -23,9 +23,14 @@ test = false [dependencies] quack-rs = { path = "../..", features = ["duckdb-1-5-4", "bundled-test-prebuilt"] } -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } [dev-dependencies] proptest = "1" [workspace] + +[package.metadata.cargo-machete] +# Used by the doctests that build.rs generates from the book and README, which +# cargo-machete does not read. +ignored = ["libduckdb-sys", "proptest", "quack-rs"] diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index ce03a113..7cacfa7c 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -9,6 +9,7 @@ - [Quick Start](getting-started/quick-start.md) - [Installation](getting-started/installation.md) - [Upgrading from 0.16 to 0.18](getting-started/upgrading-0-18.md) + - [Upgrading from 0.18 to 0.20](getting-started/upgrading-0-20.md) - [Your First Extension](getting-started/first-extension.md) - [Project Scaffold](getting-started/scaffold.md) diff --git a/book/src/concepts/abi.md b/book/src/concepts/abi.md index 6d0cb1e1..e40ab1f2 100644 --- a/book/src/concepts/abi.md +++ b/book/src/concepts/abi.md @@ -180,12 +180,38 @@ For an engine older than v1.5.0 the advice changes: such an engine cannot run a `duckdb-1-5` build at all, so the message says to build a variant without those features or to upgrade DuckDB. +## The bindings read the whole struct before the guard runs + +`libduckdb-sys` generates its `duckdb_ext_api_v1` with every slot of the release it was +generated from (546 for the 1.5.x bindings, 548 for 2.0), whatever quack-rs features are on. +Its `duckdb_rs_extension_api_init` asks DuckDB for the struct and reads **every one of those +slots** through the returned pointer before `abi::check` can run, because the check itself +reaches `duckdb_library_version` through that table. DuckDB's struct has only the engine's own +number of slots, so loading a build against the 546-slot bindings into DuckDB 1.4.x (459 +slots) reads 87 pointers (696 bytes on a 64-bit target) past the end of it, and into +1.5.0 or 1.5.1 (545 slots) one pointer. This happens on every such `LOAD`, also for a +default-feature build that the check then lets load as `StableOnly`. The memory is mapped +and no crash has been observed or is expected; the values read are stored, and are never +called by code that stays in the stable prefix or when the check refuses the load. They are +what an unstable-region wrapper would call after `Warn` or `Trust` let a mismatched layout +load, one more reason those policies are undefined behaviour on a mismatch. This was read +in `libduckdb-sys` and in DuckDB's `extension_load.cpp`, not measured; quack-rs could avoid +it only by calling `get_api` and reading slot 7 itself before handing the struct to the +bindings. + ## Unknown DuckDB versions `Strict` also refuses a DuckDB release quack-rs has no verified layout for — a newer release (such as v2.0.1 or v2.1.0), or a `-dev` build. That is deliberate: DuckDB changed the unstable region in every minor release from 1.3.0 to 2.0.0, and -in the 1.5.2 patch release, so "unknown" is not evidence of "compatible". A build +in the 1.5.2 patch release, so "unknown" is not evidence of "compatible". DuckDB's +own header says the same: DuckDB v2.0.0's `duckdb_extension.h` opens with "this API +is not yet stable, meaning it is only guaranteed to work for this specific DuckDB +version", and v2.0.0 itself added two slots to the band that v1.5.6 had declared +stable. So a `duckdb-1-5*` extension that loads into 1.5.6 and 2.0.0 is refused by +2.0.1 until a quack-rs release lists it or you rebuild with +`QUACK_RS_TARGET_DUCKDB_VERSION=v2.0.1`; an extension built with the default +features reports `StableOnly` and loads there. A build against the v1.5.2 – v1.5.6 bindings is accepted on v2.0.0, whose layout only appends to it; a build against 1.5.0/1.5.1 (545 slots) is still refused there. The refusal lists the fixes, best first: diff --git a/book/src/concepts/anatomy.md b/book/src/concepts/anatomy.md index 2a0766f2..560bb45c 100644 --- a/book/src/concepts/anatomy.md +++ b/book/src/concepts/anatomy.md @@ -25,15 +25,17 @@ When DuckDB loads your extension, it: `quack_rs::entry_point::init_extension` performs this sequence. It also checks the C API layout before registration (see [ABI Compatibility](abi.md)) and converts a panic in your -registration code into a load error. The `entry_point!` macro generates the required -`#[no_mangle] extern "C"` symbol: +registration code into a load error, and runs the registration in one transaction that a +failed `LOAD` rolls back (see [The Entry Point](entry-point.md#registration-runs-in-one-transaction)). +The `entry_point!` macro generates the required +`#[unsafe(no_mangle)] extern "C"` symbol: ```rust # use quack_rs::entry_point; # use quack_rs::error::ExtensionError; # fn register(_con: libduckdb_sys::duckdb_connection) -> Result<(), ExtensionError> { Ok(()) } entry_point!(my_extension_init_c_api, |con| register(con)); -// emits: #[no_mangle] pub unsafe extern "C" fn my_extension_init_c_api(...) +// emits: #[unsafe(no_mangle)] pub unsafe extern "C" fn my_extension_init_c_api(...) ``` --- @@ -86,7 +88,7 @@ when `InMemoryDb::open()` runs (see [Testing Guide](../testing.md)). graph TD EXT["your-extension"] QR["quack-rs"] - LDS["libduckdb-sys >=1.4.4, <2
{loadable-extension}
(headers only — no linked library)"] + LDS["libduckdb-sys >=1.4.4, <1.20100
{loadable-extension}
(headers only — no linked library)"] EXT --> QR EXT --> LDS @@ -101,14 +103,18 @@ inside the host DuckDB process and uses that process's DuckDB instance, memory a ## Version support -`libduckdb-sys = ">=1.4.4, <2"` — the bounded range is intentional. +`libduckdb-sys = ">=1.4.4, <1.20100"` — the bounded range is intentional. -quack-rs supports DuckDB 1.4.4 through 2.0.0. Every one of them loads extensions -built for **C API version `v1.2.0`** +With the default features, quack-rs supports DuckDB 1.4.4 through 2.0.x. Every one of +them loads extensions built for **C API version `v1.2.0`** (the version string passed to `duckdb_rs_extension_api_init`, `quack_rs::DUCKDB_API_VERSION`). -DuckDB 1.5.6 and 2.0.0 declare C API version `v1.5.6` and still accept `v1.2.0` extensions. CI runs +DuckDB 1.5.6 and 2.0.x declare C API version `v1.5.6` and still accept `v1.2.0` extensions. CI runs the test suite against DuckDB 1.4.4, 1.4.5, 1.5.0, 1.5.3, 1.5.4, 1.5.5, 1.5.6 and 2.0.0, and loads the example -extension into 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release. +extension into 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release. An extension that uses a +`duckdb-1-5*` feature is narrower: it loads into DuckDB 1.5.6 and 2.0.0, the releases whose layout +quack-rs has verified, and under the default `Strict` policy it refuses any other release (such as a +future 2.0.1) until quack-rs lists it or you build with `QUACK_RS_TARGET_DUCKDB_VERSION` set to it +([ABI Compatibility](abi.md#unknown-duckdb-versions)). Using a range rather than an exact pin means: - Extension authors can choose which `libduckdb-sys` release to build against (for example @@ -116,16 +122,24 @@ Using a range rather than an exact pin means: numbers `1.10505.x`) and still resolve against `quack-rs` - `quack-rs` itself doesn't force a DuckDB downgrade on users -The `<2` upper bound limits the range to `libduckdb-sys` 1.x, but it does **not** exclude -the DuckDB 2.0 bindings: `libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as -`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0`, which the range -admits. That release was published on 2026-10-22; it declares `rust-version` 1.88, so a -Rust 1.86 toolchain cannot build it, and MSRV-aware resolution (resolver 3, or -`CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`) keeps a 1.86 build on `1.10506.x`. +`libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as +`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0` and the range +admits the whole DuckDB 2.0 bindings, `1.20000.x`. That release was published on +2026-10-22; it declares `rust-version` 1.88, so a Rust 1.86 toolchain cannot build it, +and MSRV-aware resolution (resolver 3, or `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`) +keeps a 1.86 build on `1.10506.x`. + +**Why the range stops at `1.20100`.** That is the DuckDB 2.1 bindings. quack-rs names the +C API through the `libduckdb_sys` crate root, which the 1.20000 bindings keep only through +a re-export of their `v1` module that is marked `// TODO: Remove in the future` upstream; +quack-rs's exported macros hard-code those root paths too, so an extension that uses them +would break with it. quack-rs widens the bound for each DuckDB minor after testing it. If you +want the newer bindings before that, there is no override: wait for the quack-rs release +(see [Installation](../getting-started/installation.md#adding-quack-rs-to-an-existing-extension)). > **For your own extension's `Cargo.toml`:** an extension that uses only the stable C API -> (the default quack-rs features) can keep the same `">=1.4.4, <2"` range; this is what -> `scaffold::generate_scaffold` writes. An extension that enables the `duckdb-1-5*` features must pin +> (the default quack-rs features) can keep the same `">=1.4.4, <1.20100"` range; this is +> what `scaffold::generate_scaffold` writes. An extension that enables the `duckdb-1-5*` features must pin > `libduckdb-sys` to the bindings it is stamped for: `~1.10506.0` (the v1.5.2 – v1.5.6 layout) > for C API `v1.5.6`, which is what the scaffold writes for `target_duckdb_version = "v1.5.6"`; > or, for a `C_STRUCT_UNSTABLE` build pinned to one 1.x release, that release's bindings diff --git a/book/src/concepts/entry-point.md b/book/src/concepts/entry-point.md index 7a7606be..b09326ba 100644 --- a/book/src/concepts/entry-point.md +++ b/book/src/concepts/entry-point.md @@ -138,7 +138,7 @@ flowchart TD L["2. ABI layout check
Applies the AbiPolicy (Strict by default)"] B["3. access.get_database(info)
Returns the duckdb_database handle"] C["4. duckdb_connect(db, &mut con)
Opens a connection for function registration"] - D["5. register(con) ← your closure
A panic becomes an error"] + D["5. register(con) ← your closure
Runs in one transaction; a panic becomes an error"] E["6. duckdb_disconnect(&mut con)
Always runs, even if registration failed"] F{Error?} G["return true"] @@ -160,9 +160,41 @@ fails the `LOAD` with that message. (When DuckDB itself detects the failure, as message.) The layout check in step 2 only runs when a `duckdb-1-5*` feature is enabled; see [ABI Compatibility](abi.md). -Registration is not transactional: functions registered before a failure stay registered -for the life of the database. Do fallible setup work (reading configuration, building -lookup tables) before the first `register` call. +### Registration runs in one transaction + +From 0.20.0 the entry point runs your closure between `BEGIN TRANSACTION` and `COMMIT` +on the registration connection, and rolls back when the closure returns `Err` or panics. +Every registration the C API makes through a connection joins that connection's +transaction, so a `LOAD` that fails part-way leaves **no catalog entry behind** (no +scalar, aggregate or table function, macro, type, copy function or table the closure +created), and loading again in the same database works. Before 0.20.0 the functions +registered before the failure stayed, and every retry failed at the first name it +registered again. + +Three things are **not** rolled back, because DuckDB keeps them in the database +configuration, not the catalog: casts, replacement scans and configuration options. +Register those last, after everything that can fail. A retry registers a cast again (the +new one replaces the old), appends a second replacement scan, and quack-rs refuses a +configuration option whose name is taken. + +What the transaction means for the closure: + +- A statement or registration that fails can abort the transaction (DuckDB 2.0 aborts it + on every failed statement; 1.x on most failures that are not binder, catalog or parser + errors). Every later statement in the closure then fails with "Current transaction is + aborted", so return the first error instead of carrying on. +- Do not run `BEGIN`, `COMMIT` or `ROLLBACK` on the registration connection. +- What the closure creates is visible to other connections only once the `LOAD` + commits, including to a connection the closure opened itself. +- A `LOAD` issued inside the host's own transaction does not wait for it. The registration + connection has a transaction of its own and commits on its own, so the host rolling back + does not undo the registrations. + +To get the earlier behaviour, where every statement commits on its own, pass +`options = EntryPointOptions::new().registration(Registration::Autocommit)` to +`entry_point!` or `entry_point_v2!` (the types are in `quack_rs::entry_point`). Do fallible +setup work (reading configuration, building lookup tables) before the first `register` +call either way, so that the transaction holds registrations only. --- diff --git a/book/src/concepts/types.md b/book/src/concepts/types.md index 645e369c..147e0bf9 100644 --- a/book/src/concepts/types.md +++ b/book/src/concepts/types.md @@ -170,10 +170,9 @@ and the C API initialized. ### Custom types: `set_alias` and `register` -An extension can add a named type to SQL, as `CREATE TYPE` does. Build the -underlying `LogicalType`, give it a name with `set_alias`, then call -`register(con)` from your entry point. After that, users can write the name -anywhere a type name goes: +An extension can add a named type to SQL. Build the underlying `LogicalType`, give +it a name with `set_alias`, then call `register(con)` from your entry point. After +that, users can write the name anywhere a type name goes: ```sql SELECT 'happy'::mood; @@ -182,6 +181,7 @@ CREATE TABLE diary(day DATE, m mood); ```rust use quack_rs::types::LogicalType; +# use quack_rs::types::TypeId; # use quack_rs::query::OwnedConnection; # std::mem::forget(quack_rs::testing::InMemoryDb::open().unwrap()); # let mut db = std::ptr::null_mut(); @@ -203,8 +203,29 @@ assert!(unsafe { mood.register(con.as_raw()) }.is_err()); # let chunk = result.next_chunk().unwrap().unwrap(); # let reader = unsafe { chunk.reader(0) }; # assert_eq!(unsafe { reader.read_str(0) }, "happy"); +# // An alias over BIGINT is opaque to arithmetic and `sum`, unlike a native CREATE TYPE. +# let money = LogicalType::new(TypeId::BigInt); +# unsafe { money.set_alias("money"); money.register(con.as_raw()).unwrap(); } +# con.execute("CREATE TABLE acct(m money)").unwrap(); +# assert!(con.query("SELECT sum(m) FROM acct").err().unwrap().as_str().contains("No function matches")); +# con.execute("CREATE TYPE native_money AS BIGINT").unwrap(); +# con.execute("CREATE TABLE native_acct(m native_money)").unwrap(); +# con.query("SELECT m + 1, abs(m), sum(m) FROM native_acct GROUP BY m").unwrap(); ``` +**A registered type is a new type, not a synonym.** A native `CREATE TYPE money AS +BIGINT` is transparent: `+`, `sum` and `abs` work on it as on `BIGINT`. A type +registered through the C API keeps its alias and does not inherit the functions of the type +it is built on. Comparison (`=`, `<`), `GROUP BY` and `max` bind on it; arithmetic, `sum` +and `abs` do not: + +```text +SELECT sum(m) FROM acct; +Binder Error: No function matches the given name and argument types 'sum(money)'. +``` + +Register casts and the functions your type needs. Checked on DuckDB 1.5.6 and 2.0.0. + - `set_alias(name)` panics if `name` contains a NUL byte; `try_set_alias(name)` returns an error instead. `get_alias()` reads the alias back. - `register(con)` returns an error if the type has no alias, if the name already diff --git a/book/src/contributing.md b/book/src/contributing.md index 63f94860..f58fcad0 100644 --- a/book/src/contributing.md +++ b/book/src/contributing.md @@ -138,6 +138,8 @@ quack-rs/ │ ├── data_chunk.rs # Ergonomic wrapper around `DuckDB` data chunks │ ├── debug_repr.rs # Internal helpers for the crate's `Debug` implementations │ ├── entry_point.rs # Extension entry point helper +│ ├── entry_point/ +│ │ └── registration.rs # Runs the registration closure in one transaction, or with autocommit │ ├── error.rs # Error types for `DuckDB` extension FFI error propagation │ ├── error_data.rs # Structured error data (`DuckDB` 1.5.0+) │ ├── expression.rs # Bound expressions (`DuckDB` 1.5.0+) @@ -161,6 +163,7 @@ quack-rs/ │ │ ├── mod.rs # Builders for registering `DuckDB` aggregate functions │ │ ├── state.rs # Generic `FfiState` wrapper for safe aggregate state management │ │ └── builder/ +│ │ ├── collision.rs # Which catalog entry holds the name a failed aggregate registration wanted │ │ ├── mod.rs # Builder types for registering `DuckDB` aggregate functions │ │ ├── overload.rs # One overload within an [`AggregateFunctionSetBuilder`] │ │ ├── set.rs # Builder for registering a `DuckDB` aggregate function set (multiple overloads) @@ -181,18 +184,26 @@ quack-rs/ │ │ ├── export_check.rs # Values `DuckDB` would export as different values, found before the export │ │ ├── import_check.rs # Structural checks on an imported array, and flattening what the readers cannot index │ │ ├── import_layout/ +│ │ │ ├── bounds.rs # Offsets, lengths and dictionary indices `DuckDB` would read past, refused │ │ │ └── tests.rs # Unit tests │ │ ├── import_layout.rs # Arrow layouts `DuckDB` imports wrongly, found by walking the array with its schema │ │ ├── options.rs # `ArrowOptions` — the Arrow production settings of a connection or a result │ │ ├── schema.rs # `ArrowSchema` — an owned Arrow C Data Interface schema │ │ └── tests.rs # Unit tests │ ├── bin/ -│ │ └── append_metadata/ -│ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap) -│ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file -│ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it -│ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file, -│ │ └── tests.rs # Unit tests +│ │ ├── append_metadata/ +│ │ │ ├── cli.rs # Command-line parsing and validation (std-only, no clap) +│ │ │ ├── entry.rs # The entry point DuckDB will look for, and the atomic write of the stamped file +│ │ │ ├── footer.rs # The 512-byte `DuckDB` extension footer, and the optional 22-byte WebAssembly custom-section header that precedes it +│ │ │ ├── main.rs # Append a DuckDB extension metadata block to a compiled .so / .dylib / .dll file, +│ │ │ ├── output.rs # Reading the library and writing the stamped file +│ │ │ ├── tests.rs # Unit tests +│ │ │ └── tests_output.rs # Unit tests for reading the input and writing the output +│ │ └── quack_scaffold/ +│ │ ├── cli.rs # Command-line parsing (std-only, no clap) +│ │ ├── main.rs # Write a new `DuckDB` extension project to disk, from `generate_scaffold_with_warnings` +│ │ ├── tests.rs # Unit tests +│ │ └── write.rs # Writing the generated files: everything is checked before anything is written │ ├── callback/ │ │ └── payload.rs # Disposing of a caught panic payload without re-entering the unwinder │ ├── cast/ @@ -211,18 +222,25 @@ quack-rs/ │ │ ├── connection.rs # [`OwnedConnection`] and its cross-thread [`InterruptHandle`] │ │ ├── cstr.rs # The two C-string conversions the `query` module runs everything through │ │ ├── live_tests.rs # Tests that need a live `DuckDB` +│ │ ├── metadata.rs # What a statement is before it runs: `StatementType`, parameter and column types, `table_names` │ │ ├── prepared.rs # Inspecting and executing a [`PreparedStatement`]; `bind.rs` binds its parameters │ │ └── result.rs # Reading a [`QueryResult`]: its columns, its chunks and what kind of outcome it is │ ├── replacement_scan/ │ │ └── mod.rs # Builder for registering `DuckDB` replacement scans │ ├── scaffold/ +│ │ ├── checks.rs # What `generate_scaffold` checks before it writes anything, and the warnings it gives │ │ ├── escape.rs # Quoting configured free text for YAML and Rust doc comments +│ │ ├── licenses.rs # The license text of a generated project │ │ ├── mod.rs # Project scaffolding for `DuckDB` Rust extensions │ │ ├── templates.rs # Template generators for scaffold file content │ │ ├── templates_ci.rs # The generated project's CI and distribution workflows +│ │ ├── templates_docs.rs # The generated project's `README.md`, `docs/function_descriptions.csv` and `deny.toml` +│ │ ├── templates_make.rs # The generated `Makefile`: extension-ci-tools' variables and targets, and the skipped-test check │ │ ├── tests.rs # Unit tests +│ │ ├── tests_content.rs # What the scaffold accepts, warns about and refuses, and the files a community submission needs │ │ ├── tests_escaping.rs # Free text reaches the generated files intact │ │ ├── tests_generated.rs # Unit tests +│ │ ├── tests_make.rs # Runs the generated `Makefile`'s own logic under `make` │ │ └── tests_workflows.rs # The generated workflows match the community build │ ├── scalar/ │ │ ├── info.rs # Ergonomic wrapper around `duckdb_function_info` for scalar function callbacks @@ -287,14 +305,24 @@ quack-rs/ │ │ ├── parser.rs # Parses and validates a `description.yml` string │ │ ├── tests.rs # Unit tests │ │ ├── tests_corpus.rs # Unit tests +│ │ ├── tests_mut_block.rs # Boundaries of the block-structure reader, found by mutation testing +│ │ ├── tests_mut_flow_a.rs # Boundaries of the flow-collection reader (scanning, anchors), found by mutation testing +│ │ ├── tests_mut_flow_b.rs # Boundaries of the flow-collection reader (nesting, plain items), found by mutation testing +│ │ ├── tests_mut_scalar.rs # Boundaries of the scalar, section and timestamp readers, found by mutation testing +│ │ ├── tests_pyyaml.rs # Accepted exactly when PyYAML accepts it, in linear time │ │ ├── tests_yaml.rs # Unit tests │ │ ├── validator.rs # Validates a `description.yml` string and returns `Ok(())` if it passes all checks │ │ ├── yaml.rs # A reader for the subset of YAML that `description.yml` files use │ │ ├── yaml11.rs # Which plain scalars PyYAML (YAML 1.1) reads as booleans, numbers, dates or null │ │ └── yaml/ -│ │ └── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments +│ │ ├── block.rs # Block mappings and sequences, and their keys +│ │ ├── flow.rs # Flow collections, nested and spread over lines +│ │ ├── scalar.rs # Scalar-level pieces of the `description.yml` YAML reader: decoding plain, quoted, block and flow values, and recognising keys and comments +│ │ └── value.rs # The value after a mapping key or a sequence marker │ ├── value/ +│ │ ├── bits.rs # `BIT` and `BIGNUM` values: validated constructors and getters that check type and `NULL` first │ │ ├── blob.rs # `Value::as_blob` — `BLOB` extraction +│ │ ├── cast.rs # `Value::try_cast` (exception-free, any type), `array_items` and `union_tag` │ │ ├── checks.rs # Pure-Rust preconditions checked before a `Value` call reaches `DuckDB` │ │ ├── composite.rs # Composite constructors: `STRUCT`, `LIST`, `ARRAY`, `ENUM`, `MAP`, `UNION` │ │ ├── defaults.rs # The defaulting accessors — `Value::as_*_or` @@ -319,19 +347,27 @@ quack-rs/ │ ├── validity.rs # Validity bitmap helpers for `DuckDB` NULL tracking │ └── writer.rs # Safe typed writing to `DuckDB` result vectors ├── tests/ +│ ├── abi_policy_enforced.rs # `AbiPolicy` enforced in-process against an engine whose layout differs │ ├── aggregate_leaks.rs # Aggregate states `DuckDB` never destroys leak no Rust heap │ ├── append_metadata_cli.rs # The `append_metadata` binary run end to end: exit status, output and the file it writes +│ ├── build_script_env.rs # The build script run with a crafted `QUACK_RS_TARGET_DUCKDB_VERSION` +│ ├── entry_point_expansion.rs # The entry-point macros compiled as edition 2024 and under `panic = "abort"` │ ├── entry_point_lint.rs # The entry-point macros accept an `unsafe` registration closure warning-free │ ├── ffi_roundtrip.rs # End-to-end FFI round-trips against a real `DuckDB` │ ├── file_handle_close.rs # `FileHandle`'s `Drop` when the close fails (stubbed C API) │ ├── handle_leaks.rs # Every RAII handle frees what `DuckDB` allocated for it (glibc) │ ├── integration_test.rs # Integration tests for `quack-rs` +│ ├── lsan_probe.rs # The `leak-check` job's canary: a handle leaked on request must fail LeakSanitizer +│ ├── property_inputs.rs # Property tests for `parse_version`, SPDX, `description.yml` and the scaffold's escaping +│ ├── quack_scaffold_cli.rs # The `quack_scaffold` binary run as a user runs it: exit status, output and the files it writes +│ ├── secret_memory_scan.rs # `SecretEntry`'s zeroize, checked by scanning the process's own memory (Linux) │ ├── secret_zeroize.rs # `SecretEntry` never frees a buffer that still holds a secret │ └── ffi_roundtrip/ │ ├── agg_states.rs # Every aggregate state is dropped, including the ones `DuckDB` moves │ ├── agg_window.rs # Aggregates in the running-window and sorted-aggregate paths │ ├── aggregate_arity.rs # An aggregate with no parameter is refused at registration (DuckDB 2.0 passes `update` no row count) │ ├── appender_api.rs # `Appender` methods a no-op replacement survived: schemas, column types, `clear_columns`, defaults +│ ├── appender_query.rs # `Appender::for_query` and the appender's `from_raw` / `into_raw` │ ├── appender_rows.rs # What happens to buffered rows when an append fails mid-row │ ├── arrow_export.rs # `arrow::data_chunk_to_arrow` refuses values `DuckDB` would export wrongly │ ├── arrow_import.rs # `arrow::data_chunk_from_arrow` checks against a live `DuckDB` @@ -341,14 +377,19 @@ quack-rs/ │ ├── chunk_writer.rs # `ChunkWriter` against a chunk `DuckDB` allocated │ ├── chunk_writer_cols.rs # `ChunkWriter::{vector, writer, struct_writer}` and `DataChunk::into_chunk_writer` filling a real chunk │ ├── collision.rs # The scalar signature-collision check, held to `DuckDB`'s own binder +│ ├── connection_registrar.rs # Every `Registrar` method of `Connection` reaches `DuckDB` │ ├── copy_from_columns.rs # A typed `COPY … FROM` reader that declares a column is refused │ ├── copy_info_errors.rs # `set_error` from all four COPY phases, and the state passed between them (`duckdb-1-5`) +│ ├── copy_typed_state.rs # Typed `COPY TO` state, a `FileHandle` kept across callbacks, `Send` handles (`duckdb-1-5`) │ ├── data_chunk_helpers.rs # `DataChunk::any_null`, `struct_writer`, `into_chunk_writer` and `ChunkWriter::is_full` │ ├── entry_point_inprocess.rs # `init_extension*` and the entry-point macros run in process against a fake extension access +│ ├── entry_point_txn.rs # A failed `LOAD` rolls back and can be retried; a `LOAD` inside the host's transaction │ ├── file_errors.rs # `FileHandle` reports write, sync and seek failures, and refuses a seek past `i64::MAX` │ ├── handles_api.rs # `StructWriter` child handles and `InMemoryDb::execute`'s row count +│ ├── hostile_input.rs # Hostile names, SQL bodies, type nesting and cardinality hints against a real engine │ ├── inline_paths.rs # The paths that skip a C call: `is_valid` / `set_null` bit operations and the `ListBuilder` child-writer cache, held to the C API │ ├── lifecycle.rs # Aggregate NULL rows, name collisions, overload builders, bind-data sharing +│ ├── lifecycle_gaps.rs # Lifecycle paths no other test reached, found by mutation testing │ ├── list_api.rs # The raw `ListVector` workflow, `get_size`, `get_entry`, `child_writer`, `max_child_capacity` on live LIST and MAP vectors │ ├── list_limits.rs # `ListBuilder` stops at `DuckDB`'s byte ceiling, not an element count │ ├── logical_type_accessors.rs # Every `LogicalType` accessor, built by the constructors and read back, against the engine's own `typeof` @@ -356,10 +397,13 @@ quack-rs/ │ ├── mock_parity.rs # `MockRegistrar` refuses a bad type with the live registration's message │ ├── nested_reserve.rs # which nested buffers a `LIST` reserve moves (the writer contracts) │ ├── nested_validity.rs # `VectorWriter::set_valid` on nested rows, against a live `DuckDB` +│ ├── oracle_temporal.rs # Randomised cross-checks of `datetime` and `interval` against `DuckDB`'s own SQL +│ ├── overload_keys.rs # Overloads `DuckDB` resolves by type id alone are refused; closure-built overloads; taken aggregate names │ ├── owned_chunk.rs # `OwnedDataChunk::new`, `capacity` and `reset` against a live `DuckDB` │ ├── panic_guards.rs # The panic-guard macros and `set_error` methods, against a live `DuckDB` │ ├── query_docs.rs # Pins the documented behaviour of `query`, `PreparedStatement`, `DbConfig` │ ├── query_extras.rs # `bind_str`, `bind_null`, `clear_bindings` and an idle `progress` +│ ├── query_metadata.rs # Statement type, parameter and column metadata, `table_names`, raw-handle round trips │ ├── query_stream.rs # A streaming result that stops early must not look like a finished one │ ├── replacement_scan_api.rs # Replacement-scan `add_*_parameter`, `register_with_data` and `set_error` │ ├── scalar_agg.rs # Scalar and aggregate builder regressions @@ -373,17 +417,22 @@ quack-rs/ │ ├── temporal_binds.rs # Temporal and over-4-GiB values refused by `PreparedStatement` binds and the `Appender` │ ├── timestamp_tz_ns.rs # `TIMESTAMPTZ_NS` (DuckDB 2.0): vectors, `Value`, and every type the engine can produce │ ├── tooling.rs # Checks of quack-rs's tooling tables against the linked `DuckDB` +│ ├── typed_extra_info.rs # `extra_info_typed` / `extra_info_ref` on casts and aggregates; a wrong type reads as `None` │ ├── typed_scalar_opt.rs # Typed scalar closures returning `Option`, the `try_*` constructors and `map3` │ ├── validity_api.rs # `ValidityBitmap` against the engine's masks, at the word and vector edges +│ ├── value_bits.rs # `BIT` / `BIGNUM` values; every input the C getters abort on reads as `None` +│ ├── value_cast.rs # `try_cast`, `array_items`, `union_tag`: reading `ARRAY` and `UNION` values │ ├── value_defaults.rs # `Value::as_*_or` when a value is present │ ├── value_getters.rs # Each typed `Value` getter at its own type; out-of-range TIMETZ / TIME_NS are not cast │ ├── value_nested.rs # Nested `Value` construction and inspection against a live `DuckDB` │ ├── value_query.rs # `Value` getters, DECIMAL binding, `Expression::fold` │ ├── value_render.rs # `Value` rendering of values SQL builds and `DuckDB` cannot render │ ├── value_temporal.rs # Every `Value` getter against every temporal source type, at every edge +│ ├── vector_capacity_guard.rs # `OwnedVector::new` refuses a capacity whose nested `ARRAY` child would overflow │ ├── vector_dt.rs # NULLs in nested output vectors; selection vectors │ ├── vector_ops.rs # `OwnedVector::into_raw`, `copy_selected`, `slice`, `reference_value`, `reference_vector` (`duckdb-1-5`) │ ├── vector_support.rs # Helpers shared by the vector-module tests +│ ├── wide_rows.rs # Every fixed-width and string type echoed over thousands of distinct rows │ └── writer_api.rs # `write_timestamp_tz`, `write_str`, and the Ok path of `try_write_varchar` / `try_write_blob` ├── benches/ │ ├── interval_bench.rs # Criterion benchmarks @@ -399,7 +448,7 @@ quack-rs/ ├── .github/workflows/ci.yml # CI pipeline ├── .github/workflows/docs.yml # GitHub Pages deployment (the versioned site) ├── CONTRIBUTING.md -├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L19, P1–P15) +├── LESSONS.md # The DuckDB Rust FFI pitfalls (L1–L21, P1–P17) ├── CHANGELOG.md └── README.md ``` @@ -408,19 +457,24 @@ quack-rs/ ## Releasing -quack-rs uses `libduckdb-sys = ">=1.4.4, <2"` — a bounded range covering the bindings of -DuckDB 1.4.4 and later 1.x releases. Every DuckDB from 1.4.4 through 2.0.0 loads C API -`v1.2.0` extensions. The `<2` upper bound does not exclude the DuckDB 2.0 bindings: -`libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, -so DuckDB 2.0.0 is `1.20000.0`, which the range admits. That release was published on -2026-10-22 and declares `rust-version` 1.88; MSRV-aware resolution keeps a Rust 1.86 build -on `1.10506.x`. Before broadening the range to a new major band: +quack-rs uses `libduckdb-sys = ">=1.4.4, <1.20100"` — a bounded range covering the bindings of +DuckDB 1.4.4 through 2.0.x. Every DuckDB from 1.4.4 through 2.0.x loads C API +`v1.2.0` extensions. `libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as +`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0` and DuckDB 2.1.0 is +`1.20100.0`, the first release the range excludes. The upper bound exists because the 1.20000 +bindings keep the C API names at the crate root only through a re-export that upstream marks +for removal (see [Installation](getting-started/installation.md#adding-quack-rs-to-an-existing-extension)). +The 1.20000 bindings were published on 2026-10-22 and declare `rust-version` 1.88; +MSRV-aware resolution keeps a Rust 1.86 build on `1.10506.x`. Before widening the upper bound +to the next DuckDB minor:Before broadening the range to a new major band: 1. Read the DuckDB changelog for C API changes 2. Check the new C API version string (used in `duckdb_rs_extension_api_init`) 3. Update `DUCKDB_API_VERSION` in `src/lib.rs` if the C API version changed 4. Audit all callback signatures against the new `libduckdb-sys` bindings -5. Update the `libduckdb-sys` and `duckdb` version requirements in `Cargo.toml` +5. Update the `libduckdb-sys` and `duckdb` version requirements in `Cargo.toml`, in the + manifests of `examples/hello-ext`, `fuzz`, `book/doctest` and `benches/engine`, and in + every snippet the documentation shows Versions follow [Semantic Versioning](https://semver.org/) as Cargo applies it. While the crate is pre-1.0, a breaking change to the public API bumps the diff --git a/book/src/data/intervals.md b/book/src/data/intervals.md index a5928cba..2e5b0ce4 100644 --- a/book/src/data/intervals.md +++ b/book/src/data/intervals.md @@ -82,8 +82,8 @@ intervals and in `epoch_us(interval)`: **1 month = 30 days**. This is not date arithmetic: DuckDB adds an interval to a date by calendar months (`DATE '2024-01-31' + INTERVAL 1 MONTH` is `2024-02-29`). It also matches SQL comparison only when the three fields share a sign: `INTERVAL '1 month' - -INTERVAL '1 day'` converts to the same total as `INTERVAL '29 days'` but compares -greater in SQL. +INTERVAL '1 day'` converts to the same total as `INTERVAL '29 days'`, but DuckDB 1.4.4 +and 1.5.6 compare it as greater (measured), while DuckDB 2.0.0 compares the two as equal. ### Checked conversion (returns `Option`) diff --git a/book/src/data/running-sql.md b/book/src/data/running-sql.md index 643a0378..e947b378 100644 --- a/book/src/data/running-sql.md +++ b/book/src/data/running-sql.md @@ -107,6 +107,11 @@ fails the call, after the ones before it have run (and, outside an explicit transaction, committed). An empty string, or just `;`, succeeds with an empty result. `prepare` takes exactly one statement. +Text that does not parse as a whole is not safe to pass through: DuckDB 1.4.4 and 1.5.6 run +nothing, but DuckDB 2.0.0 runs the statements before the error, and the call may still report +success ([Known +Limitations](../reference/known-limitations.md#duckdb_query-runs-the-statements-before-a-parse-error-on-duckdb-20)). + ## Bind values, do not interpolate them Anything that did not come from your own source text — a table name from a @@ -242,6 +247,14 @@ fn register(con: &Connection) -> Result<(), ExtensionError> { connection between threads, not using one concurrently. Open one connection per thread, or guard it with a mutex. +**Do not change `threads` from a callback.** `SET threads = N` run on a connection you +kept, from inside a table function's scan, resizes DuckDB's worker pool from one of its +own workers. The outcome varied from run to run: on DuckDB 2.0.0 the process terminated +with `terminate called after throwing an instance of 'std::system_error'` in three of four +runs; on DuckDB 1.5.6 the statement failed with "Invalid argument" in five of six runs and +succeeded once (the termination was also seen on 1.5.6). It is DuckDB +behaviour that quack-rs cannot intercept. Set `threads` before the query starts. + ## Cancelling a query and reading its progress `OwnedConnection::interrupt_handle` returns an `InterruptHandle`, which is @@ -250,7 +263,11 @@ thread can call its `cancel()` to stop the running query, which then fails with an interrupt error at DuckDB's next check, or its `progress()` to read a `QueryProgress` (`percentage`, `rows_processed`, `total_rows_to_process`). `percentage` is `-1.0` when DuckDB cannot report progress, for example when -the progress bar is disabled (`SET enable_progress_bar = true`). +the progress bar is disabled (`SET enable_progress_bar = true`). On DuckDB 1.x it also +stays `-1.0` for as long as any pipeline of the running query cannot estimate its +progress (`-1.0` was read throughout one query on 1.5.6 where 2.0.0 reported +1.18 %, 1.34 % and 1.47 %; observed in a review, not re-run for this page). Treat it as +"unknown", not as an error. `OwnedConnection::interrupt` and `progress` do the same on the calling thread. ```rust diff --git a/book/src/data/values-and-parameters.md b/book/src/data/values-and-parameters.md index b30bbd6c..d7b38816 100644 --- a/book/src/data/values-and-parameters.md +++ b/book/src/data/values-and-parameters.md @@ -220,6 +220,26 @@ options.struct_child(idx)?.as_str().ok() # } ``` +### `ARRAY`, `UNION`, `BIT` and `BIGNUM` values, and casting a value + +`list_items` reads a `LIST` only. On a fixed-size `ARRAY` value (an `INTEGER[3]` +argument of a table function, a `COPY` option or a scalar bind argument) the C API reports +no elements, so the call returns an empty `Vec`. From 0.20.0: + +- `Value::array_items()` returns the elements of an `ARRAY` or a `LIST`, and `None` for a + SQL `NULL` or any other type. +- `Value::union_tag()` (`duckdb-1-5`) is the index of the active member of a `UNION` value. + The member's value reads as text through `try_cast` to `VARCHAR`. +- `Value::try_cast(&LogicalType)` is SQL `TRY_CAST` of a value to any logical type, nested + ones included. It returns `None` where DuckDB cannot convert, and refuses first the + payloads DuckDB cannot convert safely (an out-of-range timestamp, a `DECIMAL` that is too + wide, also inside an `ARRAY`); no C++ exception can reach Rust. +- `Value::bit(&[bool])` and `Value::bignum(magnitude, negative)` build `BIT` and `BIGNUM` + values (an error for bytes DuckDB cannot use), and `Value::as_bit()` and + `Value::as_bignum()` read them. The getters return `None` for a SQL `NULL` and for a + value of another type, where the C getters abort the process + ([Pitfall P16](../reference/pitfalls.md#p16)). + --- `DataChunk`, which wraps the chunk a table function's scan callback writes its diff --git a/book/src/data/vectors.md b/book/src/data/vectors.md index 2adfd7b6..c01527eb 100644 --- a/book/src/data/vectors.md +++ b/book/src/data/vectors.md @@ -277,8 +277,13 @@ for row in 0..chunk.size() { `ChunkWriter` wraps an output `duckdb_data_chunk` and counts the rows handed out by `next_row`. It calls `set_size` with that count on drop, so the row count -cannot be forgotten or set wrongly. `next_row` returns `None` once the chunk -holds `duckdb_vector_size()` rows: +cannot be forgotten. `next_row` returns `None` once the chunk +holds `duckdb_vector_size()` rows. The count is of rows you **claimed**: write every row +you claim. A claimed row that is never written is published with whatever the reused +output buffer held last ([Pitfall L21](../reference/pitfalls.md#l21-a-chunk-row-you-claim-is-published-written-or-not)), +so check that the source has another value before you claim the row, or give the last +claimed row back with `release_row(row)` before the writer is dropped (it panics for any +other row, so the mistake is loud): ```rust # use libduckdb_sys::{duckdb_data_chunk, duckdb_function_info, duckdb_vector}; diff --git a/book/src/duckdb-1-5/arrow.md b/book/src/duckdb-1-5/arrow.md index 83921a83..73d0f5b8 100644 --- a/book/src/duckdb-1-5/arrow.md +++ b/book/src/duckdb-1-5/arrow.md @@ -31,12 +31,11 @@ engine. ## The types -| Type | Wraps | Freed by | -|---|---|---| -| `ArrowOptions` | `duckdb_arrow_options` | `duckdb_destroy_arrow_options` | -| `ArrowSchema` | the `ArrowSchema` ABI record | `release(schema)` | -| `ArrowArray` | the `ArrowArray` ABI record | `release(array)` | -| `ArrowConvertedSchema` | `duckdb_arrow_converted_schema` | `duckdb_destroy_arrow_converted_schema` | +- `ArrowOptions` wraps `duckdb_arrow_options` and is freed by `duckdb_destroy_arrow_options`. +- `ArrowSchema` wraps the `ArrowSchema` ABI record and is freed by `release(schema)`. +- `ArrowArray` wraps the `ArrowArray` ABI record and is freed by `release(array)`. +- `ArrowConvertedSchema` wraps `duckdb_arrow_converted_schema` and is freed by + `duckdb_destroy_arrow_converted_schema`. `RawArrowSchema` and `RawArrowArray` are the ABI records themselves, re-exported for code that already speaks the raw interface. @@ -162,6 +161,15 @@ therefore makes the caller's job: It also refuses valid Arrow layouts that DuckDB imports wrongly: it walks the array alongside its schema and returns `InvalidInput`, naming the node, for +- a **record batch with a nonzero `offset`** (the top-level struct array): DuckDB ignores + a top-level offset and would import the wrong rows. The error says to export the batch + without an offset and slice its children instead; an arrow-rs `RecordBatch::slice` is + exported that way; +- a list, large list or map whose offsets are negative, decrease, or end past the child + (DuckDB reads the child through them without a bounds check: a segmentation fault on 1.5.6 + and 2.0.0), a list view with a negative offset or size, a child (struct field, fixed-size-list + child, union member) shorter than the rows its parent reads, and a dictionary index outside the + dictionary; - an offset below the top level that DuckDB applies to the wrong rows: a struct inside an offset struct or a list, a union's members, a run-end-encoded array's value validity (`docs/upstream-duckdb-reports.md`, item 24); diff --git a/book/src/duckdb-1-5/catalog.md b/book/src/duckdb-1-5/catalog.md index 137997bd..7971cfb6 100644 --- a/book/src/duckdb-1-5/catalog.md +++ b/book/src/duckdb-1-5/catalog.md @@ -88,8 +88,9 @@ database's catalog is named `memory`; a database file's is the file's stem (`analytics.duckdb` is `analytics`). An empty name returns `None`. `catalog` also returns `None` when no transaction is active. Inside a function -callback a transaction is always active. On an idle connection outside a query, -there is none: +callback a transaction is always active, and so it is in the registration closure of an +entry point, which runs in one transaction from 0.20.0 (with `Registration::Autocommit` it +does not). On an idle connection outside a query, there is none: ```rust # use quack_rs::catalog::CatalogEntryType; diff --git a/book/src/duckdb-1-5/error-data.md b/book/src/duckdb-1-5/error-data.md index eeecddf8..0023c4e2 100644 --- a/book/src/duckdb-1-5/error-data.md +++ b/book/src/duckdb-1-5/error-data.md @@ -30,7 +30,8 @@ if err.has_error() { ## Constructing an error -Build a structured error to hand back to DuckDB (for example from a callback): +Build a structured error value, for example in a test or to pass between your own +functions: ```rust # // ErrorData is a DuckDB object: fill the dispatch table first. @@ -42,6 +43,13 @@ assert!(err.has_error()); assert_eq!(err.error_type(), DuckDbErrorType::InvalidInput); ``` +**No callback API accepts an `ErrorData`.** A scalar, aggregate, table, cast or `COPY` +callback reports a failure with its `set_error(&str)`, and DuckDB chooses the error +category: a scalar function's execution error surfaces as an `Invalid Input Error`, a +table function's bind error as a `Binder Error`, whatever `DuckDbErrorType` you hold. +To branch on a kind of failure inside your extension, keep your own error enum and +format it into the message. + ## Propagating with `?` `into_extension_error` converts an `ErrorData` into the SDK's [`ExtensionError`], diff --git a/book/src/duckdb-1-5/expression.md b/book/src/duckdb-1-5/expression.md index bf5e315a..2a10f2f0 100644 --- a/book/src/duckdb-1-5/expression.md +++ b/book/src/duckdb-1-5/expression.md @@ -49,6 +49,10 @@ unsafe extern "C" fn my_bind(info: RawScalarBindInfo) { | `fold(&client_context)` | `Result<`[`Value`]`, `[`ErrorData`]`>` — evaluate a constant expression | | `from_raw(raw)` (unsafe) / `as_raw()` / `is_null()` | Handle inspection / escape hatches | +A prepared-statement parameter (`?`) is not foldable at bind time: a bind callback that +requires a constant makes `PREPARE p AS SELECT f(x, ?)` fail (measured on DuckDB 1.5.6 and +2.0.0). A literal and a constant expression such as `1 + 1` are foldable. + `fold` only succeeds when [`is_foldable`][is_foldable] returns `true`; otherwise it returns an [`ErrorData`] of type `InvalidInput`. A successful `fold` can still yield a SQL `NULL` (for example `NULL::INTEGER`), whose typed getters return diff --git a/book/src/duckdb-1-5/file-system.md b/book/src/duckdb-1-5/file-system.md index a9e7e77e..4e260522 100644 --- a/book/src/duckdb-1-5/file-system.md +++ b/book/src/duckdb-1-5/file-system.md @@ -9,6 +9,12 @@ then resolve through every registered file system — `httpfs` (`s3://`, `http://`) when it is loaded, in-memory files, and so on — where `std::fs` only ever sees local disk. +> **Security: `std::fs` ignores DuckDB's permission settings.** A deployment can lock +> DuckDB down with `SET enable_external_access = false` or `SET allowed_directories`. An +> extension that opens a user-supplied path with `std::fs` reads it anyway. Open such +> paths with `FileSystem` instead: see +> [the section below](#security-only-filesystem-honours-duckdbs-permission-settings). + ## Obtaining a `FileSystem` A `FileSystem` comes from a [`ClientContext`], which most function callbacks can @@ -104,6 +110,36 @@ silently treat as an empty file. `FileSystem` exposes `open(path, options)` and `error_data()`. Both `FileSystem` and `FileHandle` are RAII: they are destroyed (and the handle closed) on drop. +## Security: only `FileSystem` honours DuckDB's permission settings + +DuckDB can be locked down so that SQL cannot reach the file system: +`SET enable_external_access = false`, or `SET allowed_directories = ['/data']` together +with it. Everything DuckDB opens itself goes through its file system layer and obeys those +settings, and so does a file you open with `FileSystem`. **`std::fs` does not.** It talks +to the operating system directly, so a table function, replacement scan or `COPY` reader +that passes a path from SQL to `std::fs::read` or `File::open` is a way around the +restriction the person running DuckDB set. Measured on DuckDB 1.5.6 and 2.0.0 with a table +function that returns a file's size, once through each API: + +```sql +SET enable_external_access = false; +SELECT count(*) FROM read_csv('data.csv'); -- Permission Error: Cannot access file ... +SELECT * FROM size_via_filesystem('data.csv'); -- Permission error from FileSystem::open +SELECT * FROM size_via_std_fs('data.csv'); -- returns the size: no restriction applied +``` + +`allowed_directories` behaves the same way: `read_csv` and `FileSystem` refuse a path +outside the list, `std::fs` reads it. The same holds for anything else your extension does +on its own, such as a network connection opened with a Rust crate: no DuckDB setting +reaches it. + +The rule for an extension that reads or writes a path it did not hard-code: open it with +`FileSystem`, from the `ClientContext` of the bind callback (or the copy callbacks). That +needs the `duckdb-1-5` feature and DuckDB 1.5.0 or later. Without it, do not open +user-supplied paths at all; take the data as a `VARCHAR` or `BLOB` argument and let SQL +do the reading. `FileSystem` also resolves `s3://` and `http://` paths when `httpfs` is +loaded, which `std::fs` never does. + ## Lifetimes A `FileSystem<'ctx>` borrows the `ClientContext` it came from, and a diff --git a/book/src/faq.md b/book/src/faq.md index 220b824d..b08f5a9f 100644 --- a/book/src/faq.md +++ b/book/src/faq.md @@ -19,25 +19,32 @@ utilities for publishing community extensions. Building a DuckDB extension in Rust means solving a set of undocumented FFI problems that each developer otherwise discovers independently. quack-rs -documents all 34 known pitfalls, and its API prevents most of them. See the +documents all 38 known pitfalls, and its API prevents most of them. See the [Pitfall Catalog](reference/pitfalls.md). ### What DuckDB version does quack-rs target? -quack-rs requires `libduckdb-sys = ">=1.4.4, <2"` and supports DuckDB 1.4.4 -through 2.0.0. Releases before 1.4.4 are not supported. CI runs the test +quack-rs requires `libduckdb-sys = ">=1.4.4, <1.20100"` and, with the default features, +supports DuckDB 1.4.4 through 2.0.x. An extension that uses a `duckdb-1-5*` feature loads into +1.5.6 and 2.0.0 only, and refuses a release quack-rs has not listed (see +[ABI Compatibility](concepts/abi.md#unknown-duckdb-versions)). Releases before 1.4.4 are not +supported. The upper bound keeps out the +DuckDB 2.1 bindings until a quack-rs release has been tested against them; see +[Installation](getting-started/installation.md#adding-quack-rs-to-an-existing-extension). +CI runs the test suite against DuckDB v1.4.4, v1.4.5, v1.5.0, v1.5.3, v1.5.4, v1.5.5, v1.5.6 and -v2.0.0 (the last with the DuckDB 2.0 bindings, `libduckdb-sys` 1.20000.0), and loads a built +v2.0.0 (the last with the DuckDB 2.0 bindings, `libduckdb-sys` 1.20000.x), and loads a built extension into v1.4.4, v1.5.0, v1.5.5, v1.5.6, v2.0.0 and the latest release. DuckDB supports 2.0 (a long-term support release) until at least October 2027, 1.5 until March 2027 and 1.4 until late November 2026. The C API version passed to the dispatch-table initializer is `"v1.2.0"`, available as `quack_rs::DUCKDB_API_VERSION`. Every supported DuckDB release -(1.4.4 and later) loads extensions built for it (1.5.6 and 2.0.0 declare C API +(1.4.4 and later) loads extensions built for it (1.5.6 and 2.0.x declare C API `v1.5.6`, but still accept `v1.2.0`). An extension that uses the `duckdb-1-5*` features is stamped C API `v1.5.6` instead -(`quack_rs::DUCKDB_API_VERSION_1_5_6`) and loads into DuckDB 1.5.6 and 2.0.0. +(`quack_rs::DUCKDB_API_VERSION_1_5_6`) and loads into DuckDB 1.5.6 and 2.0.0; the +runtime guard refuses a release quack-rs has not listed (2.0.1, for now). The C API version is a separate identifier from the DuckDB release and from the `libduckdb-sys` crate version, which encodes DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N` (e.g. `1.10505.0` for DuckDB 1.5.5, @@ -50,24 +57,26 @@ Rust **1.86.0** or later. This is enforced in `Cargo.toml` with ### Is quack-rs production-ready? -It is pre-1.0, and you should judge it against your own requirements rather than -take a yes. What the record shows: +It is pre-1.0, and you should judge it against your own requirements; this page cannot +give you a yes or a no. What the record shows: - **The API still changes.** Minor releases before 1.0 can break it; each one lists its breaking changes, with migration notes, in the [changelog](reference/changelog.md). -- **Audits keep finding real defects.** The review released as 0.16.0 fixed 24 - defects in earlier releases, including two heap-corruption paths. The 0.18.0 audits fixed further soundness holes, - process aborts and wrong answers, and found one serious defect in unreleased - code before it was published: aggregate functions returned wrong results in - release builds with Cargo's default profile. `AUDIT.md` in the repository - records each audit, what it found, and whether each fix was reproduced against - a real DuckDB or derived from DuckDB's source. +- **Audits keep finding real defects.** The 0.16.0 release, which came out of a review, + fixed defects in earlier releases, including two heap-corruption paths. The 0.18.0 + audits fixed further soundness holes, process aborts and wrong answers, and found one + serious defect in unreleased code before it was published: aggregate functions returned + wrong results in release builds with Cargo's default profile. `AUDIT.md` in the + repository records each audit, what it found, and whether each fix was reproduced + against a real DuckDB or derived from DuckDB's source. Use 0.18.0 or later: earlier + releases still have the soundness holes in the safe API, and 0.7.0 through 0.15.0 can + corrupt the heap when the `duckdb-1-5` features are used. - **Some limits are DuckDB's.** The C API has defects quack-rs can only document or work around; see [Known Limitations](reference/known-limitations.md). It was extracted from [duckdb-behavioral](https://github.com/tomtom215/duckdb-behavioral), a DuckDB -community extension, where the first 16 of the pitfalls it now documents were +community extension, where the first of the pitfalls it now documents were discovered. If you ship an extension built on it, run end-to-end tests that load the extension into each DuckDB release you support (see the [Testing Guide](testing.md)). @@ -103,6 +112,15 @@ bound when it is created, so the `events` table must already exist or `register` returns an error. See [SQL Macros](functions/sql-macros.md). +### Can my extension read a file with `std::fs`? + +It can, and it should not when the path comes from SQL. `std::fs` ignores DuckDB's +`enable_external_access` and `allowed_directories` settings, so a function that reads a +user-supplied path with it works around a restriction the person running DuckDB set +(`read_csv` on the same path fails with a permission error). Open such paths with the +`FileSystem` wrapper (feature `duckdb-1-5`); see +[Virtual File System](duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings). + ### Can I register multiple overloads of the same function? Yes, using `AggregateFunctionSetBuilder` (for aggregates) or @@ -344,7 +362,8 @@ populated by DuckDB at extension load time. ### Why not use `duckdb-loadable-macros`? Use it when it fits. As of `duckdb` and `duckdb-loadable-macros` 1.20000.1 the -macro opens a `Connection` from the raw database (`Connection::open_from_raw`) +macro opens a `duckdb::Connection` from the raw database +(`duckdb::Connection::open_from_raw`) and reports errors through the API's `set_error`; an earlier release of the macro depended on `extract_raw_connection` and the `duckdb` crate's internal layout, which crashed when the layout changed, and that history is why diff --git a/book/src/functions/aggregate-sets.md b/book/src/functions/aggregate-sets.md index 443b6885..288571ea 100644 --- a/book/src/functions/aggregate-sets.md +++ b/book/src/functions/aggregate-sets.md @@ -6,11 +6,11 @@ function in Rust with `AggregateFunctionSetBuilder`, including variadic aggregat such as `retention(c1, c2, ..., c32)` and overloads with different return types. > **Known DuckDB limitation.** C-API aggregates — including every overload in a -> set — read out of bounds under `agg(x) OVER ()` (whole-partition window frames) -> and `agg(x ORDER BY y)`. This is a DuckDB C API defect; see +> set — read out of bounds under `agg(x) OVER ()` and other whole-partition window frames, +> and under `agg(x ORDER BY y)`. This is a DuckDB C API defect; see > [Aggregate Functions](aggregate.md#known-duckdb-limitation) for the details and -> DuckDB source lines. On DuckDB 1.4.x and 1.5.x, do not use C-API aggregates -> in those two query shapes. +> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) for the exact shapes. On DuckDB 1.4.x and 1.5.x, do not use +> C-API aggregates in those query shapes. > > Reported upstream as [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109) > and fixed in DuckDB 2.0.0. @@ -77,7 +77,7 @@ works: ```text arg_max(ANY, ANY) -> ANY -arg_max(ANY, ANY, ANY) -> ANY[] +arg_max(ANY, ANY, BIGINT) -> ANY[] ``` Set the return type on the overload with `AggregateOverloadBuilder::returns` (or diff --git a/book/src/functions/aggregate-state.md b/book/src/functions/aggregate-state.md index 9cba9252..285b519e 100644 --- a/book/src/functions/aggregate-state.md +++ b/book/src/functions/aggregate-state.md @@ -6,11 +6,11 @@ the `AggregateState` trait and `FfiState`, which manages each state's lifecyc raw-pointer code for it. > **Known DuckDB limitation.** Every C-API aggregate, and so every aggregate that -> uses `FfiState`, reads out of bounds under `agg(x) OVER ()` (whole-partition -> window frames) and `agg(x ORDER BY y)`. This is a DuckDB C API defect; see +> uses `FfiState`, reads out of bounds under `agg(x) OVER ()` and other whole-partition +> window frames, and under `agg(x ORDER BY y)`. This is a DuckDB C API defect; see > [Aggregate Functions](aggregate.md#known-duckdb-limitation) for the details and -> DuckDB source lines. On DuckDB 1.4.x and 1.5.x, do not use C-API aggregates -> in those two query shapes. +> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) for the exact shapes. On DuckDB 1.4.x and 1.5.x, do not use +> C-API aggregates in those query shapes. > > Reported upstream as [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109) > and fixed in DuckDB 2.0.0. diff --git a/book/src/functions/aggregate.md b/book/src/functions/aggregate.md index 5aa5d173..62c1c602 100644 --- a/book/src/functions/aggregate.md +++ b/book/src/functions/aggregate.md @@ -8,9 +8,11 @@ also has a `combine` step that merges partial results from parallel workers. ## Known DuckDB limitation -> **Before DuckDB 2.0.0, do not use a C API aggregate as `agg(x) OVER ()` or -> `agg(x ORDER BY y)`.** In those two query shapes DuckDB 1.4.x and 1.5.x read -> past the end of the state array, which usually crashes the process. This is a +> **Before DuckDB 2.0.0, do not use a C API aggregate as `agg(x) OVER ()`, over an +> explicit `UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, or as +> `agg(x ORDER BY y)`.** In those query shapes DuckDB 1.4.x and 1.5.x read +> past the end of the state array, which usually crashes the process (and on 1.4.4, +> `OVER (PARTITION BY p)` without an `ORDER BY` too). This is a > DuckDB defect, fixed in 2.0.0, that an extension cannot detect or prevent; see > [Pitfall L11](../reference/pitfalls.md#l11-c-api-aggregates-crash-under-aggx-over--and-aggx-order-by-y) > and [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11). diff --git a/book/src/functions/copy-functions.md b/book/src/functions/copy-functions.md index 42d3583b..9669a89b 100644 --- a/book/src/functions/copy-functions.md +++ b/book/src/functions/copy-functions.md @@ -6,6 +6,12 @@ This page shows how to register one from a Rust extension with quack-rs' > **Requires the `duckdb-1-5` feature flag** (DuckDB 1.5.0+). +> **Open the file through DuckDB.** The path of a `COPY` is user input. `std::fs` ignores +> `SET enable_external_access = false` and `allowed_directories`, and cannot reach +> `s3://` or `http://` paths; the +> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings) +> wrapper honours the settings and resolves every registered file system. + A format can support writing, reading, or both: | Direction | You supply | DuckDB calls | @@ -149,6 +155,16 @@ for i in 0..bind.result_column_count() { # } ``` +## Choosing the format from the file name + +`COPY t TO 'out.my_format'` with no `FORMAT` clause picks your format when the file +extension equals the copy function's name, and so does `COPY t FROM 'in.my_format'` for the +reader. Measured on DuckDB 1.5.6 and 2.0.0: a copy function named `pr_fmt` was used for +`out.pr_fmt` and was not used for `out.dat`. A function cannot declare a different +extension (the C API v1 has no way to: see [Known +Limitations](../reference/known-limitations.md#custom-copy-formats-cannot-declare-a-file-extension)), +so name the format after the extension you want recognised. + ## Registering a format name twice `duckdb_register_copy_function` drops a copy function whose name already exists — @@ -175,6 +191,13 @@ persisted, so reloading into a database file does not trip the check. - Several values (`LST (1, 2)`) arrive as a `LIST`, or an unnamed `STRUCT` when their types differ. - An explicit `NULL` value is rejected by the binder before your callback runs. +- **DuckDB does not validate the options of a `COPY … TO` format.** A copy function cannot + declare which options it takes, so `(FORMAT my_format, bogus 1)`, an option without a + value, a list where you expect a scalar, and a value of the wrong type all reach your + bind callback and are accepted unless you reject them. (A built-in format rejects the + same typo: `Unrecognized option "bogus" for csv`.) Check every name and type yourself and + call `set_error`; otherwise a misspelt option is silently ignored. A `COPY … FROM` reader + is different: its options are the table function's named parameters, which DuckDB checks. - Field order follows DuckDB's internal hash map, not the statement — look fields up by name: @@ -204,6 +227,26 @@ The data pointers are untyped, so the compiler cannot check this for you: partition with `PARTITION_BY` (reachable from more than one thread). Guard any mutation with a `Mutex` unless you know neither option is in use. +### Typed state, without raw pointers + +From 0.20.0 the three data pointers have typed forms that box the value behind a type tag +and generate a destructor that contains a panic in `T::drop`, so no `Box::into_raw` and no +`extern "C"` destructor is written by hand (all need `duckdb-1-5`): + +| Set | Read | +|-----|------| +| `CopyFunctionBuilder::extra_info_typed(value)` | `extra_info_ref::()` on each of the four info types | +| `CopyBindInfo::set_bind_data_typed(value)` | `bind_data_ref::()` in global init, sink and finalize | +| `CopyGlobalInitInfo::set_global_state_typed(value)` | `global_state_ref::()` in sink and finalize | + +`T` is `Send + Sync + 'static`, because the sink can run on several threads at once; state +that changes needs a `Mutex` or atomics inside `T`. A read returns `None` when nothing was +set or when the value has another type, instead of reinterpreting memory. The readers are +`unsafe` only because they cannot tell a typed value from one set through the raw +`set_bind_data` or `set_global_state`: use one form per slot. `FileHandle` is `Send` (not +`Sync`) from 0.20.0, so a file opened in global init can be kept in the global state inside +a `Mutex`. + ## Callback signatures | Phase | Signature | Guarded macro | @@ -227,7 +270,7 @@ the handle at the top of your callback to access helper methods: | Method | Description | |--------|-------------| | `column_count()` | Number of output columns | -| `column_type(index)` | `LogicalType` of the column at `index`, or `None` if out of range | +| `column_type(index)` | `LogicalType` of the column at `index`, or `None` if out of range. There is no column *name*: the C API does not give a `COPY … TO` bind callback the names, so a writer that wants a header needs a `names` option from the user | | `options()` | The `COPY … TO` options, as one `STRUCT` `Value` (see [Options](#options)) | | `get_extra_info()` | Extra-info pointer set on the copy function | | `set_bind_data(data, destroy)` | Store bind data and its destructor | diff --git a/book/src/functions/null-handling.md b/book/src/functions/null-handling.md index 94108822..44e0bbd0 100644 --- a/book/src/functions/null-handling.md +++ b/book/src/functions/null-handling.md @@ -54,8 +54,11 @@ static void VerifyNullHandling(const BoundFunctionExpression &expr, DataChunk &a } ``` -Every DuckDB a user installs is a release build, so that assertion is compiled -out. +Through DuckDB 1.5.6 that check exists only in debug builds, so every DuckDB a user +installs skips it. DuckDB 2.0.0 compiles it into release builds but runs it only after +`SET debug_verification_mode = 'VERIFY_FUNCTIONS'` (the default is `NONE`); with that +setting a callback that ignores validity fails with `INTERNAL Error: VerifyNullHandling +failed for scalar function ...`, and with the default it is believed, as before. ### Why the obvious test passes anyway diff --git a/book/src/functions/panic-safe-callbacks.md b/book/src/functions/panic-safe-callbacks.md index 5911db98..602d9ca6 100644 --- a/book/src/functions/panic-safe-callbacks.md +++ b/book/src/functions/panic-safe-callbacks.md @@ -165,8 +165,9 @@ output does not compile in a scalar function's. `catch_unwind` cannot catch anything when the crate is built with `panic = "abort"`: the process ends at the panic site, before any guard runs. -Keep your extension on `panic = "unwind"`. The scaffold generates it, and -`validate_release_profile` rejects `"abort"`. See +Keep your extension on `panic = "unwind"`. The scaffold generates it, +`validate_release_profile` rejects `"abort"`, and the `entry_point!` macros refuse to +compile under it. See [Installation](../getting-started/installation.md#why-panic--unwind-not-abort). ## Prefer the closure-based builders where they fit diff --git a/book/src/functions/replacement-scan.md b/book/src/functions/replacement-scan.md index 7919a994..cdd710d0 100644 --- a/book/src/functions/replacement-scan.md +++ b/book/src/functions/replacement-scan.md @@ -15,6 +15,12 @@ register one from a Rust extension with quack-rs. `quack-rs` provides `ReplacementScanBuilder` (a static registration helper) and `ReplacementScanInfo` (an ergonomic wrapper for callbacks). +> **The table function a replacement scan names must open files through DuckDB.** The +> path comes straight from the user's SQL. `std::fs` ignores +> `SET enable_external_access = false` and `allowed_directories`; the +> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings) +> wrapper (feature `duckdb-1-5`) honours them. + ## Registration API Unlike the other builders in quack-rs, `ReplacementScanBuilder` uses a single diff --git a/book/src/functions/scalar.md b/book/src/functions/scalar.md index 892f35fd..0e5e9199 100644 --- a/book/src/functions/scalar.md +++ b/book/src/functions/scalar.md @@ -104,11 +104,16 @@ with `register(con)`, or through a `Registrar` with `register_typed_scalar`. # duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; # unsafe fn demo(con: duckdb_connection) -> Result<(), ExtensionError> { -ScalarFunctionBuilder::map1("double_it", |x: i64| x * 2)?.register(con)?; +ScalarFunctionBuilder::map1("is_even", |x: i64| x % 2 == 0)?.register(con)?; # Ok(()) # } ``` +Arithmetic inside the closure is yours to check: a plain `x * 2` wraps on overflow in a +release build and panics (so fails the query) in a debug build. Use `try_map1` and +`checked_mul` to turn overflow into an error in both, as the +[Quick Start](../getting-started/quick-start.md) does. + A closure can also return `Option`, where `None` is SQL `NULL`, and the `try_map1` / `try_map2` / `try_map3` / `try_map1_str` / `try_map2_str` constructors take a closure returning `Result`. An `Err` fails the query @@ -147,6 +152,12 @@ ScalarFunctionBuilder::map3("lerp", |a: f64, b: f64, t: f64| a + (b - a) * t)?.r ## Complete example: `double_it(BIGINT) → BIGINT` +For a function this small the closure builder above is the better choice +(`try_map1` with `checked_mul`, as in the [Quick Start](../getting-started/quick-start.md)). +Written as a raw callback, with `scalar_callback!` for the panic guard, it looks like this. +`value * 2` would wrap silently in a release build, so the callback reports overflow +through `ScalarFunctionInfo::set_error`: + ```rust # use quack_rs::prelude::*; # fn live_connection() -> libduckdb_sys::duckdb_connection { @@ -165,20 +176,15 @@ ScalarFunctionBuilder::map3("lerp", |a: f64, b: f64, t: f64| a + (b - a) * t)?.r # let reader = unsafe { chunk.reader(0) }; # unsafe { reader.is_valid(0).then(|| reader.read_i64(0)) } # } +use quack_rs::scalar::ScalarFunctionInfo; use quack_rs::vector::{VectorReader, VectorWriter}; -use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk, duckdb_vector}; -unsafe extern "C" fn double_it( - _info: duckdb_function_info, - input: duckdb_data_chunk, - output: duckdb_vector, -) { +quack_rs::scalar_callback!(double_it, |info, input, output| { // SAFETY: DuckDB provides valid chunk and vector pointers. let reader = unsafe { VectorReader::new(input, 0) }; // column 0 let mut writer = unsafe { VectorWriter::new(output) }; - let row_count = reader.row_count(); - for row in 0..row_count { + for row in 0..reader.row_count() { if unsafe { !reader.is_valid(row) } { // NULL input → NULL output // SAFETY: row < row_count, writer is valid. @@ -186,9 +192,13 @@ unsafe extern "C" fn double_it( continue; } let value = unsafe { reader.read_i64(row) }; - unsafe { writer.write_i64(row, value * 2) }; + let Some(doubled) = value.checked_mul(2) else { + unsafe { ScalarFunctionInfo::new(info) }.set_error("double_it: BIGINT overflow"); + return; + }; + unsafe { writer.write_i64(row, doubled) }; } -} +}); # let con = live_connection(); # unsafe { # ScalarFunctionBuilder::new("double_it").param(TypeId::BigInt).returns(TypeId::BigInt) @@ -197,6 +207,8 @@ unsafe extern "C" fn double_it( # assert_eq!(query_i64(con, "SELECT double_it(21)"), Some(42)); # // From a column, not a literal: `double_it(NULL::BIGINT)` is constant-folded. # assert_eq!(query_i64(con, "SELECT double_it(i) FROM (VALUES (NULL::BIGINT)) t(i)"), None); +# let err = unsafe { quack_rs::query::query(con, "SELECT double_it(9223372036854775807)") }.err().unwrap(); +# assert!(err.as_str().contains("double_it: BIGINT overflow"), "{err}"); ``` --- @@ -204,14 +216,10 @@ unsafe extern "C" fn double_it( ## Multi-parameter example: `add(BIGINT, BIGINT) → BIGINT` ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; -unsafe extern "C" fn add( - _info: duckdb_function_info, - input: duckdb_data_chunk, - output: duckdb_vector, -) { +use quack_rs::scalar::ScalarFunctionInfo; + +quack_rs::scalar_callback!(add, |info, input, output| { let col0 = unsafe { VectorReader::new(input, 0) }; // first param let col1 = unsafe { VectorReader::new(input, 1) }; // second param let mut writer = unsafe { VectorWriter::new(output) }; @@ -223,9 +231,13 @@ unsafe extern "C" fn add( } let a = unsafe { col0.read_i64(row) }; let b = unsafe { col1.read_i64(row) }; - unsafe { writer.write_i64(row, a + b) }; + let Some(sum) = a.checked_add(b) else { + unsafe { ScalarFunctionInfo::new(info) }.set_error("add: BIGINT overflow"); + return; + }; + unsafe { writer.write_i64(row, sum) }; } -} +}); ``` --- @@ -233,14 +245,8 @@ unsafe extern "C" fn add( ## VARCHAR example: `shout(VARCHAR) → VARCHAR` ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; -unsafe extern "C" fn shout( - _info: duckdb_function_info, - input: duckdb_data_chunk, - output: duckdb_vector, -) { +quack_rs::scalar_callback!(shout, |_info, input, output| { let reader = unsafe { VectorReader::new(input, 0) }; let mut writer = unsafe { VectorWriter::new(output) }; @@ -253,7 +259,7 @@ unsafe extern "C" fn shout( let upper = s.to_uppercase(); unsafe { writer.write_varchar(row, &upper) }; } -} +}); ``` --- @@ -293,6 +299,12 @@ unsafe fn register(con: duckdb_connection) -> Result<(), ExtensionError> { } ``` +A function built from a closure (`map1`, `try_map1`, ...) is not a `ScalarOverloadBuilder`: its +signature is fixed by the closure. Add it to a set with +`ScalarFunctionSetBuilder::overload_typed(builder)`. That matters on DuckDB 1.4.x, where a +second registration under a taken name is refused, so closures for several argument types need +one set; from 1.5.0 DuckDB also accepts them registered one by one. + Like `AggregateFunctionSetBuilder`, this builder calls `duckdb_scalar_function_set_name` on every individual function before adding it to the set ([Pitfall L6](../reference/pitfalls.md#l6-function-set-name-must-be-set-on-each-member)). @@ -368,6 +380,24 @@ ScalarFunctionBuilder::new("flatten_list") # } ``` +### `DECIMAL` parameters are not generic + +A `DECIMAL(w, s)` parameter takes exactly that width and scale. DuckDB casts every argument +to the declared type before your callback runs, and the cast **rounds**: a function +declared over `DECIMAL(4,1)` that doubles its argument, called with +`0.123::DECIMAL(3,3)`, sees `0.1` and returns `0.2`, not `0.246` (measured on 1.5.6 and +2.0.0). Declaring several widths as overloads does not work either: DuckDB's overload +resolution does not tell `DECIMAL(4,1)` from `DECIMAL(9,3)` apart, so every call to such a +set fails with "Could not choose a best candidate function" (measured on 1.5.6 and 2.0.0; +the same holds for `MAP` and `ENUM` parameters of different parameterisations, and for those +nested inside a `LIST`, `ARRAY` or `STRUCT`). quack-rs 0.20 refuses such a set when you +register it; 0.19 and earlier registered it, and the first call failed. To accept any `DECIMAL`, declare the +parameter `TypeId::Any`, read the argument vector's actual type at run time +(`quack_rs::vector::vector_get_column_type`, then `LogicalType::decimal_width`, +`decimal_scale` and `decimal_internal_type`) and return a type that does not depend on it +(`DOUBLE`, `VARCHAR` or `HUGEINT`); or register one function per width under different +names. + These methods are also available on `ScalarOverloadBuilder` for function sets: ```rust @@ -481,9 +511,19 @@ feature is enabled: ### `bind(bind_fn)` -Sets a custom bind callback that runs at plan time. Use this to inspect argument -types and set the return type dynamically. Maps to -`duckdb_scalar_function_set_bind`. +Sets a custom bind callback that runs at plan time. Use it to inspect the argument +types and constant arguments, to store bind data for the execute callback, and to reject +a call with `set_error`. Maps to `duckdb_scalar_function_set_bind`. + +**The bind callback cannot set the return type.** The C API v1 has no such function: it +offers `duckdb_scalar_function_bind_get_argument`, `_get_argument_count`, +`_get_extra_info` and `_set_error`, and `ScalarBindInfo` wraps exactly those, plus +`set_bind_data`, `set_bind_data_copy` and `get_client_context`. The return type is the one +you declared with `returns`. When the result type must depend on the argument types, +register one overload per result type with `ScalarFunctionSetBuilder` (for example +`f(INTEGER) → INTEGER[]` and `f(VARCHAR) → STRUCT(v VARCHAR)`). Choosing the return type +at bind time exists only in the C API v2 +([Known Limitations](../reference/known-limitations.md#duckdb-20s-c-api-v2-offers-what-v1-cannot)). The callback takes a `RawScalarBindInfo` (wrap it with `ScalarBindInfo::new`), not a bare `duckdb_bind_info`: DuckDB passes a table function's bind callback @@ -502,7 +542,7 @@ do not type-check here. # unsafe fn demo(con: duckdb_connection) -> Result<(), ExtensionError> { ScalarFunctionBuilder::new("dynamic_return") .varargs(TypeId::Varchar) - .returns(TypeId::Varchar) // default; overridden in bind + .returns(TypeId::Varchar) // fixed: bind cannot change it .bind(my_bind_fn) .function(dynamic_return_fn) .register(con)?; @@ -598,10 +638,19 @@ ScalarFunctionBuilder::new("scaled") optimizer duplicates it (filter pushdown through a projection does). Without a copy callback the copy has **no** bind data at all, so `set` registers one that clones `T`. Wrap data that is expensive or impossible to clone in an `Arc`. +- **A bind-time constant cannot come from a prepared-statement parameter.** If `bind` + rejects an argument that is not foldable (see [Bound Expressions](../duckdb-1-5/expression.md)), + a `?` is not foldable: `PREPARE p AS SELECT f(x, ?)` fails at the `PREPARE`, with your + message, on DuckDB 1.5.6 and 2.0.0, while a literal or a constant expression such as `1 + 1` + folds. Say so in the error text, and tell users to pass such an argument as a literal. - **Local state must be `Send`**: it is per thread, but may be freed on another. -- **Call `set` at most once per callback.** DuckDB overwrites the stored pointer - on a second call without freeing the first value, so that value is leaked - (never dropped). +- **A panicking `Clone` fails the query.** DuckDB gives a copied expression no bind data when + the copy callback fails; quack-rs records the panic instead, and the first `get` reports it + as the call's error (with the panic message), while `try_get` hands it back as an `Err`. +- **Call `set` once per callback.** A second `ScalarBindData::set` in the same bind frees the + value it replaces. DuckDB's own call overwrites the stored pointer without freeing the first + value, so a value replaced through the raw `set_bind_data`, or through another wrapper of + the same bind call, is leaked (never dropped). - **Bind data must depend only on the call's arguments** (their values when constant, their types) and `extra_info`. DuckDB's `CScalarFunctionBindData::Equals` ignores bind data, so two calls with the same diff --git a/book/src/functions/sql-macros.md b/book/src/functions/sql-macros.md index d0e8eff1..0d61c966 100644 --- a/book/src/functions/sql-macros.md +++ b/book/src/functions/sql-macros.md @@ -123,7 +123,7 @@ unsafe { # unsafe { quack_rs::query::execute(con, "CREATE TABLE orders AS SELECT current_date - 3 AS order_date UNION ALL SELECT current_date - 30") }.unwrap(); # register(con).unwrap(); # assert_eq!(query_i64(con, "SELECT count(*) FROM recent_orders(7)"), 1); -# assert_eq!(query_i64(con, "SELECT count(*) FROM active_users(users)"), 2); +# assert_eq!(query_i64(con, "SELECT count(*) FROM active_users('users')"), 2); ``` A table macro's body is bound when the macro is created, so every table it names @@ -134,7 +134,7 @@ fails with a catalog error. To take a table as a parameter, read it through Use in DuckDB: ```sql -SELECT * FROM active_users(users); +SELECT * FROM active_users('users'); SELECT count(*) FROM recent_orders(7); ``` @@ -210,8 +210,12 @@ The **body** (expression or query) is your own extension code — it is included **Never build macro bodies from untrusted user input.** `register` does refuse a body that turns the statement into several (`1); DROP TABLE t; SELECT (1`) — it counts statements with DuckDB's own parser (`duckdb_extract_statements`) and executes -nothing if there is more than one — but a body can still change the meaning of the -single statement it is part of. +nothing unless the text is exactly one statement — but a body can still change the +meaning of the single statement it is part of. Text that does not parse at all is +refused too, with DuckDB's parse message: before 0.20.0 it was executed, which is +harmless on DuckDB 1.4.4 and 1.5.6 (they parse the whole string first) but not on 2.0.0, +which runs the statements that precede the parse error (see [Known +Limitations](../reference/known-limitations.md#duckdb_query-runs-the-statements-before-a-parse-error-on-duckdb-20)). A scalar body containing `--` gets a newline before the closing parenthesis, so a trailing line comment (`"x + 1 -- plus one"`) does not comment it out. @@ -240,8 +244,8 @@ schema — the **user's** database: ## How it works under the hood `SqlMacro::register` counts the statements in `to_sql()` with -`duckdb_extract_statements`, refuses more than one, and executes the -`CREATE OR REPLACE MACRO` statement via `duckdb_query`. +`duckdb_extract_statements`, refuses a text that is not exactly one statement, and +executes the `CREATE OR REPLACE MACRO` statement via `duckdb_query`. The query result is zero-initialized before `duckdb_query`, any error message is read with `duckdb_result_error`, and `duckdb_destroy_result` runs on success and failure diff --git a/book/src/functions/table-functions.md b/book/src/functions/table-functions.md index c9f3f8f1..09f594e8 100644 --- a/book/src/functions/table-functions.md +++ b/book/src/functions/table-functions.md @@ -23,18 +23,32 @@ Both builders are backed by the helper types `BindInfo`, `InitInfo`, `FunctionIn | Phase | Callback | Called when | Typical work | |-------|----------|-------------|--------------| -| **bind** | `bind_fn` | Query is planned (once per plan) | Extract parameters; declare output columns; store configuration in bind data | +| **bind** | `bind_fn` | Query is planned (once per plan on DuckDB 1.x; also on every `EXECUTE` of a prepared statement on 2.0.0) | Extract parameters; declare output columns; store configuration in bind data | | **init** | `init_fn` | Each execution of the plan starts | Allocate per-scan state (cursor, row index, etc.) | | **scan** | `scan_fn` | Each output batch | Fill the output chunk with rows; set its size with `DataChunk::set_size` | DuckDB calls the scan callback repeatedly until it sets the chunk size to 0, which signals the end of the results. -> **Bind once, init many times.** DuckDB keeps the bind data for as long as the -> bound plan lives and runs `init` against it on **every** execution: each -> `EXECUTE` of a prepared statement, each iteration of a recursive CTE that -> references the function. Treat bind data as immutable after bind and build +> **Bind data outlives the plan; bind itself may run again.** DuckDB keeps the bind +> data for as long as the bound plan lives and runs `init` against it on **every** +> execution: each `EXECUTE` of a prepared statement, each iteration of a recursive CTE +> that references the function. Treat bind data as immutable after bind and build > anything a scan consumes (cursors, open files) in `init`. +> +> How often `bind` runs depends on the DuckDB release. With a prepared statement that is +> executed three times, counters in the callbacks read 1 bind and 3 inits on DuckDB 1.4.4 +> and 1.5.6, and **4 binds** and 3 inits on 2.0.0 (one for the `PREPARE`, one for each +> `EXECUTE`). So a table function that reads a setting in `bind`, where a client +> context is available, keeps the value from the first bind on 1.x and picks up a changed +> setting on 2.0. Do not rely on either behaviour: only `bind` is given a client +> context (`InitInfo` and `FunctionInfo` have none), so a table function cannot read a +> setting at `init` or scan time. + +> **A table function that reads a file must open it through DuckDB.** `std::fs` ignores +> `SET enable_external_access = false` and `allowed_directories`; the +> [`FileSystem`](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings) +> wrapper (feature `duckdb-1-5`) honours them. ## Closure-based typed state (`with_state`) @@ -138,17 +152,21 @@ fn register(reg: &impl Registrar) -> ExtResult<()> { - `S` must be `Send + 'static` (plus `Clone` for `with_state`). `Sync` is **not** required, so `TypedTableFunctionBuilder` forces scans to run on a single worker by - calling `InitInfo::set_max_threads(1)` internally. + calling `InitInfo::set_max_threads(1)` internally. Measured: at `SET threads = 8` a + typed function's scan callbacks all ran on one thread, on DuckDB 1.5.6 and 2.0.0. A + scan that is the bottleneck of a query, over a large input, needs the raw builder + with `set_max_threads` above 1. - The typed builder does **not** offer projection pushdown: with pushdown on, the scan's chunk holds only the projected columns and a closure written against the declared schema would write the wrong column. `build()` returns an error if `projection_pushdown(true)` was set on the raw builder before `with_state` / `with_bind_init`. Use the raw builder for pushdown. - The bind closure must declare at least one column. With `duckdb-1-5`, a bind - that declares none is reported as an ordinary bind error; DuckDB itself raises an - `INTERNAL Error` with a C++ stack trace for it (and does so for a raw bind - callback, which quack-rs cannot check after it returns — call `set_error` - yourself). + that declares none is reported as an ordinary bind error. Without `duckdb-1-5` the + typed builder cannot check, and DuckDB itself raises an `INTERNAL Error` with a C++ + stack trace (measured on 1.4.4, 1.5.6 and 2.0.0; the database stays usable). A raw bind + callback gets the same `INTERNAL Error` on every feature set, because quack-rs cannot + check it after it returns: call `set_error` yourself. - Extensions that need multi-worker parallelism (`set_max_threads` above 1, with `local_init` + thread-local buffers) should use the raw [`TableFunctionBuilder`](#builder-api) directly. @@ -274,7 +292,9 @@ quack_rs::table_scan_callback!(gs_scan, |info, output| { }; let remaining = bind.total - state.pos; - let batch = remaining.min(2048).max(0) as usize; + // Never hard-code the chunk size: use the vector size of the running DuckDB. + let capacity = quack_rs::vector::vector_size() as i64; + let batch = remaining.min(capacity).max(0) as usize; let mut writer = unsafe { chunk.writer(0) }; for i in 0..batch { diff --git a/book/src/getting-started/first-extension.md b/book/src/getting-started/first-extension.md index efd8dc67..dad19297 100644 --- a/book/src/getting-started/first-extension.md +++ b/book/src/getting-started/first-extension.md @@ -1,57 +1,66 @@ # Your First Extension -This page builds a DuckDB extension in Rust step by step, using `hello-ext`, the example -extension bundled with quack-rs. The table lists four of the functions it registers, one of -each major kind (the full list is in its -[README](https://github.com/tomtom215/quack-rs/tree/main/examples/hello-ext)); this page walks -through the aggregate and the scalar. The table function and the cast are covered in -[Table Functions](../functions/table-functions.md) and -[Cast Functions](../functions/cast-functions.md). +This page builds a small DuckDB extension step by step: an aggregate function, +`word_count`, and a scalar function, `first_word`. Every callback is guarded, so a +panic in it fails the query with an error instead of aborting the DuckDB process. If +you have not built an extension before, the [Quick Start](quick-start.md) is shorter +and a good place to begin. | SQL | Kind | Signature | |-----|------|-----------| | `word_count(text)` | Aggregate | `VARCHAR → BIGINT` | | `first_word(text)` | Scalar | `VARCHAR → VARCHAR` | -| `generate_series_ext(n)` | Table | `BIGINT → TABLE(value BIGINT)` | -| `CAST(VARCHAR AS INTEGER)` | Cast | `VARCHAR → INTEGER` | -Full source: [`examples/hello-ext/src/lib.rs`](https://github.com/tomtom215/quack-rs/blob/main/examples/hello-ext/src/lib.rs) +> **Window functions and DuckDB before 2.0.** Every aggregate registered through the +> C API, `word_count` included, crashes the DuckDB process on 1.4.x and 1.5.x when a +> user runs it as `word_count(x) OVER ()`, over an explicit +> `ROWS` or `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, or as +> `word_count(x ORDER BY y)`. On 1.4.4, `OVER (PARTITION BY p)` without an `ORDER BY` +> crashes too. This is a DuckDB defect that no extension can prevent, fixed in 2.0.0; see +> [Known Limitations](../reference/known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11). +> Tell the users of your own aggregates. + +The repository's [`examples/hello-ext`](https://github.com/tomtom215/quack-rs/tree/main/examples/hello-ext) +registers these two functions among many others. It writes them as hand-rolled +`unsafe extern "C" fn`s, which are installed as written and run with no +`catch_unwind`; its README says it shows what the C API calls look like, not how to +write new code. The code below is the guarded form. --- -## Build and try it - -```bash -cargo build --release --manifest-path examples/hello-ext/Cargo.toml - -# DuckDB only loads files ending in `.duckdb_extension` that carry its -# 512-byte metadata footer; a bare `.so` is refused. Append the footer: -cargo run --bin append_metadata -- \ - examples/hello-ext/target/release/libhello_ext.so \ - hello_ext.duckdb_extension \ - --abi-type C_STRUCT --extension-version v0.1.0 \ - --duckdb-version v1.2.0 --platform linux_amd64 - -# An unsigned local build needs -unsigned (it must be a startup flag). -duckdb -unsigned -``` - -Then in the DuckDB CLI: - -```sql -LOAD './hello_ext.duckdb_extension'; - --- Aggregate: total words across all rows -SELECT word_count(sentence) FROM ( - VALUES ('hello world'), ('one two three'), (NULL) -) t(sentence); --- → 5 (2 + 3; NULL contributes 0) - --- Scalar: first word of each row -SELECT first_word(sentence) FROM ( - VALUES ('hello world'), (' padded '), (''), (NULL) -) t(sentence); --- → 'hello', 'padded', '', NULL +## Set up the project + +Create a library crate and give it the `Cargo.toml` from the +[Quick Start](quick-start.md), with the name `word_tools`: + +```toml +[package] +name = "word_tools" +version = "0.1.0" +edition = "2021" +# With resolver 3, Cargo picks dependency versions that build on this. +rust-version = "1.86" +resolver = "3" + +[lib] +# Must match the extension name: see Pitfall P1. +name = "word_tools" +crate-type = ["cdylib", "rlib"] + +[dependencies] +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] + +[profile.release] +panic = "unwind" +lto = true +opt-level = 3 +codegen-units = 1 +strip = true ``` --- @@ -60,18 +69,30 @@ SELECT first_word(sentence) FROM ( An extension has four parts: -1. **State struct** — holds data accumulated during aggregation (aggregate only) -2. **Callbacks** — `update`, `combine`, `finalize` (aggregate; `ffi_state::()` supplies `state_size`, `state_init` and `state_destroy`) or a single function callback (scalar) -3. **Registration** — wire callbacks to DuckDB via `AggregateFunctionBuilder` / `ScalarFunctionBuilder` -4. **Entry point** — DuckDB's initialization hook, generated by `entry_point!` +1. **State struct**: holds data accumulated during aggregation (aggregate only) +2. **Callbacks**: `update`, `combine` and `finalize` for the aggregate, written with + the `aggregate_*_callback!` macros (`ffi_state::()` supplies `state_size`, + `state_init` and `state_destroy`); for the scalar, a closure +3. **Registration**: wire the callbacks to DuckDB with `AggregateFunctionBuilder` and + `ScalarFunctionBuilder` +4. **Entry point**: DuckDB's initialization hook, generated by `entry_point!` --- -## Part 1 — Aggregate function: `word_count` +## Part 1: aggregate function `word_count` An aggregate function accumulates state across many rows and emits one result per group. -### 1a. The state struct +### 1a. The logic and the state struct + +The counting is ordinary Rust, so it is easy to unit-test (see +[Unit tests](#unit-tests-no-duckdb-process-needed)): + +```rust +fn count_words(s: &str) -> i64 { + s.split_whitespace().count() as i64 +} +``` ```rust # use quack_rs::prelude::*; @@ -92,188 +113,199 @@ impl AggregateState for WordCountState {} ### 1b. `state_size`, `state_init` and `state_destroy` These three callbacks are always identical boilerplate, so you do not write them: -`.ffi_state::()` on the builder (see [Part 3](#part-3--registration)) +`.ffi_state::()` on the builder (see [Part 3](#part-3-registration)) installs `FfiState::::size_callback`, `init_callback` and `destroy_callback` together. Installing all three for one type in a single call means DuckDB never allocates space sized for one state and has it initialised as another. -`size_callback` returns `FfiState::::size()` — one `usize` tag word followed +`size_callback` returns `FfiState::::size()`: one `usize` tag word followed by the state. A `T` aligned no more strictly than `usize` and at most 256 bytes (like `WordCountState`) is stored inline in the bytes DuckDB allocates per group; a larger or more strictly aligned `T` is boxed, and the slot holds the `Box` pointer. `init_callback` writes -`WordCountState::default()` into the slot (or boxes it), then sets the tag — a value salted per +`WordCountState::default()` into the slot (or boxes it), then sets the tag, a value salted per state type, so a destructor only drops states that were initialised for its own `T`. `destroy_callback` drops the `T` in each state (and its box, if the state is too large to be stored inline), clearing the state's tag first so a second call is a no-op. -### 1c. `update` — accumulate one batch +### 1c. `update`: accumulate one batch + +`aggregate_update_callback!` generates the `unsafe extern "C" fn` the builder expects and +runs your body under `catch_unwind`: a panic in it, such as an arithmetic overflow in a +debug build, fails the query with the panic message and leaves the process running. See +[Panic-Safe Callbacks](../functions/panic-safe-callbacks.md). ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; # #[derive(Default, Debug)] struct WordCountState { count: i64 } # impl AggregateState for WordCountState {} # fn count_words(s: &str) -> i64 { s.split_whitespace().count() as i64 } -unsafe extern "C" fn wc_update( - _info: duckdb_function_info, - input: duckdb_data_chunk, - states: *mut duckdb_aggregate_state, -) { +quack_rs::aggregate_update_callback!(wc_update, |_info, input, states| { let reader = unsafe { VectorReader::new(input, 0) }; - let row_count = reader.row_count(); - - for row in 0..row_count { + for row in 0..reader.row_count() { if !unsafe { reader.is_valid(row) } { - continue; // NULL input → skip (contributes 0 words) + continue; // NULL input contributes no words } - let s = unsafe { reader.read_str(row) }; - let words = count_words(s); + let words = count_words(unsafe { reader.read_str(row) }); let state_ptr = unsafe { *states.add(row) }; if let Some(st) = unsafe { FfiState::::with_state_mut(state_ptr) } { st.count += words; } } -} +}); ``` Key points: -- Check `is_valid(row)` before reading — never dereference an invalid (NULL) row -- `VectorReader::new(chunk, col)` gives column `col` from the chunk -- `count_words` is pure Rust — no unsafe, easy to unit-test separately +- Check `is_valid(row)` before reading. Never read an invalid (NULL) row. +- `VectorReader::new(chunk, col)` gives column `col` from the chunk, and `states[row]` is the + state of that row's group: rows in the same group share one state. +- Each `unsafe { … }` block is its own scope: the macro body is not an `unsafe` context. -### 1d. `combine` — merge parallel results +### 1d. `combine`: merge parallel results > **Pitfall L1**: DuckDB creates fresh target states before calling `combine`, set up by > `state_init` (here `WordCountState::default()`), not copies of the source. -> You must copy **all** fields — not just the result field. In an aggregate with config +> You must copy **all** fields, not just the result field. In an aggregate with config > fields (e.g., a histogram with a `bin_width`) you must also copy those, or results will > be silently corrupted. ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; # #[derive(Default, Debug)] struct WordCountState { count: i64 } # impl AggregateState for WordCountState {} -unsafe extern "C" fn wc_combine( - _info: duckdb_function_info, - source: *mut duckdb_aggregate_state, - target: *mut duckdb_aggregate_state, - count: idx_t, -) { +quack_rs::aggregate_combine_callback!(wc_combine, |_info, source, target, count| { for i in 0..count as usize { - let src_ptr = unsafe { *source.add(i) }; - let tgt_ptr = unsafe { *target.add(i) }; - let src = unsafe { FfiState::::with_state(src_ptr) }; - let tgt = unsafe { FfiState::::with_state_mut(tgt_ptr) }; + let src = unsafe { FfiState::::with_state(*source.add(i)) }; + let tgt = unsafe { FfiState::::with_state_mut(*target.add(i)) }; if let (Some(s), Some(t)) = (src, tgt) { t.count += s.count; // If you add fields to WordCountState, combine them here too. } } -} +}); ``` -### 1e. `finalize` — write output +`combine` reads `source` and must leave it unchanged ([Pitfall L15](../reference/pitfalls.md#l15-combine-must-leave-its-source-states-unchanged)): +the same source state can be combined into several targets. + +### 1e. `finalize`: write output ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; # #[derive(Default, Debug)] struct WordCountState { count: i64 } # impl AggregateState for WordCountState {} -unsafe extern "C" fn wc_finalize( - _info: duckdb_function_info, - source: *mut duckdb_aggregate_state, - result: duckdb_vector, - count: idx_t, - offset: idx_t, -) { +quack_rs::aggregate_finalize_callback!(wc_finalize, |_info, source, result, count, offset| { let mut writer = unsafe { VectorWriter::new(result) }; - for i in 0..count as usize { let state_ptr = unsafe { *source.add(i) }; match unsafe { FfiState::::with_state(state_ptr) } { Some(st) => unsafe { writer.write_i64(offset as usize + i, st.count) }, - None => unsafe { writer.set_null(offset as usize + i) }, + None => unsafe { writer.set_null(offset as usize + i) }, } } -} +}); ``` -`offset` is DuckDB's output row offset — always use `offset as usize + i`, not just `i`. +`offset` is DuckDB's output row offset: always use `offset as usize + i`, not just `i`. --- -## Part 2 — Scalar function: `first_word` +## Part 2: scalar function `first_word` -A scalar function processes one data chunk and returns one output value per row. -The callback receives the full chunk and an output vector (not per-row state pointers). +A scalar function processes one data chunk and returns one output value per row. For a +function of one to three primitive or string arguments, the closure builders are the +whole job: you write the per-row logic, and the builder derives the SQL types from the +closure's signature, skips NULL rows and writes NULL for them, and runs the closure under +`catch_unwind`. -### Key rule: always propagate NULL +```rust +# use quack_rs::prelude::*; +# fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") } +# fn demo() -> Result<(), ExtensionError> { +let first_word_builder = ScalarFunctionBuilder::map1_str("first_word", |s: &str| { + first_word(s).to_owned() +})?; +# let _ = first_word_builder; +# Ok(()) +# } +``` + +where `first_word` is the pure logic: + +```rust +fn first_word(s: &str) -> &str { + s.split_whitespace().next().unwrap_or("") +} +``` -If the input row is NULL, write NULL to output — never read from an invalid row. +### When the closure builders do not fit + +A function over `STRUCT` or `LIST` arguments, with variable arity, or with bind-time +state needs the raw chunk. Write the callback with `scalar_callback!`, which gives it the +same guard. It is then your job to propagate NULL: if the input row is NULL, write NULL to +the output and never read from an invalid row. ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; # use quack_rs::prelude::*; # fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") } -unsafe extern "C" fn first_word_scalar( - _info: duckdb_function_info, - input: duckdb_data_chunk, - output: duckdb_vector, -) { +quack_rs::scalar_callback!(first_word_raw, |_info, input, output| { let reader = unsafe { VectorReader::new(input, 0) }; let mut writer = unsafe { VectorWriter::new(output) }; - let row_count = reader.row_count(); - - for row in 0..row_count { + for row in 0..reader.row_count() { if !unsafe { reader.is_valid(row) } { - unsafe { writer.set_null(row) }; // NULL in → NULL out + unsafe { writer.set_null(row) }; // NULL in, NULL out continue; } let s = unsafe { reader.read_str(row) }; unsafe { writer.write_varchar(row, first_word(s)) }; } -} +}); ``` -The pure logic: +The raw builder takes the callback together with the signature you declare for it: ```rust -pub fn first_word(s: &str) -> &str { - s.split_whitespace().next().unwrap_or("") +# use libduckdb_sys::duckdb_connection; +# use quack_rs::prelude::*; +# quack_rs::scalar_callback!(first_word_raw, |_info, _input, _output| {}); +# fn register(con: duckdb_connection) -> Result<(), ExtensionError> { +// SAFETY: `con` is the live connection DuckDB passed to the entry point. +unsafe { + ScalarFunctionBuilder::new("first_word_raw") + .param(TypeId::Varchar) + .returns(TypeId::Varchar) + .function(first_word_raw) + .register(con)?; } +# Ok(()) +# } ``` -Note: `set_null` internally calls `duckdb_vector_ensure_validity_writable` before writing -the null flag — this is required by DuckDB and handled for you by `VectorWriter`. +`set_null` calls `duckdb_vector_ensure_validity_writable` before writing the NULL flag, +which DuckDB requires; `VectorWriter` does it for you. + +Keep arithmetic in either style checked: in a release build `x * 2` wraps on overflow +without any error, which is why [Scalar Functions](../functions/scalar.md) and the Quick +Start use `checked_mul`. --- -## Part 3 — Registration +## Part 3: registration -The snippets below register through the raw builders and `entry_point!`, which take a -`duckdb_connection`. `hello-ext` itself uses the equivalent `entry_point_v2!`, whose closure -receives a `&Connection` and registers each builder through the `Registrar` trait -(`con.register_aggregate(...)`, `con.register_scalar(...)`); see -[The Entry Point](../concepts/entry-point.md). +The aggregate builder takes the three callbacks and `ffi_state`; the closure builder +already holds its callback. `register` is `unsafe` because it calls the C API on a +connection that must be live, and the entry point is where DuckDB hands you one: ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; +# use libduckdb_sys::duckdb_connection; # use quack_rs::prelude::*; # #[derive(Default, Debug)] struct WordCountState { count: i64 } # impl AggregateState for WordCountState {} # fn count_words(s: &str) -> i64 { s.split_whitespace().count() as i64 } # fn first_word(s: &str) -> &str { s.split_whitespace().next().unwrap_or("") } -# unsafe extern "C" fn wc_update(_info: duckdb_function_info, input: duckdb_data_chunk, -# states: *mut duckdb_aggregate_state) { +# quack_rs::aggregate_update_callback!(wc_update, |_info, input, states| { # let reader = unsafe { VectorReader::new(input, 0) }; # for row in 0..reader.row_count() { # if !unsafe { reader.is_valid(row) } { continue; } @@ -282,17 +314,15 @@ receives a `&Connection` and registers each builder through the `Registrar` trai # st.count += words; # } # } -# } -# unsafe extern "C" fn wc_combine(_info: duckdb_function_info, source: *mut duckdb_aggregate_state, -# target: *mut duckdb_aggregate_state, count: idx_t) { +# }); +# quack_rs::aggregate_combine_callback!(wc_combine, |_info, source, target, count| { # for i in 0..count as usize { # let src = unsafe { FfiState::::with_state(*source.add(i)) }; # let tgt = unsafe { FfiState::::with_state_mut(*target.add(i)) }; # if let (Some(s), Some(t)) = (src, tgt) { t.count += s.count; } # } -# } -# unsafe extern "C" fn wc_finalize(_info: duckdb_function_info, source: *mut duckdb_aggregate_state, -# result: duckdb_vector, count: idx_t, offset: idx_t) { +# }); +# quack_rs::aggregate_finalize_callback!(wc_finalize, |_info, source, result, count, offset| { # let mut writer = unsafe { VectorWriter::new(result) }; # for i in 0..count as usize { # match unsafe { FfiState::::with_state(*source.add(i)) } { @@ -300,16 +330,7 @@ receives a `&Connection` and registers each builder through the `Registrar` trai # None => unsafe { writer.set_null(offset as usize + i) }, # } # } -# } -# unsafe extern "C" fn first_word_scalar(_info: duckdb_function_info, input: duckdb_data_chunk, -# output: duckdb_vector) { -# let reader = unsafe { VectorReader::new(input, 0) }; -# let mut writer = unsafe { VectorWriter::new(output) }; -# for row in 0..reader.row_count() { -# if !unsafe { reader.is_valid(row) } { unsafe { writer.set_null(row) }; continue; } -# unsafe { writer.write_varchar(row, first_word(reader.read_str(row))) }; -# } -# } +# }); # fn live_connection() -> libduckdb_sys::duckdb_connection { # std::mem::forget(quack_rs::testing::InMemoryDb::open().unwrap()); # let (mut db, mut con) = (std::ptr::null_mut(), std::ptr::null_mut()); @@ -326,7 +347,12 @@ receives a `&Connection` and registers each builder through the `Registrar` trai # let reader = unsafe { chunk.reader(0) }; # unsafe { reader.is_valid(0).then(|| reader.read_i64(0)) } # } -unsafe fn register(con: libduckdb_sys::duckdb_connection) -> Result<(), ExtensionError> { +fn register(con: duckdb_connection) -> Result<(), ExtensionError> { + let first_word_builder = ScalarFunctionBuilder::map1_str("first_word", |s: &str| { + first_word(s).to_owned() + })?; + + // SAFETY: `con` is the live connection DuckDB passed to the entry point. unsafe { AggregateFunctionBuilder::new("word_count") .param(TypeId::Varchar) @@ -337,36 +363,39 @@ unsafe fn register(con: libduckdb_sys::duckdb_connection) -> Result<(), Extensio .finalize(wc_finalize) .register(con)?; - ScalarFunctionBuilder::new("first_word") - .param(TypeId::Varchar) - .returns(TypeId::Varchar) - .function(first_word_scalar) - .register(con)?; + first_word_builder.register(con)?; } Ok(()) } # let con = live_connection(); -# unsafe { register(con) }.unwrap(); +# register(con).unwrap(); # assert_eq!(query_i64(con, "SELECT word_count(s) FROM (VALUES ('hello world'), (NULL), ('one two three')) t(s)"), Some(5)); # assert_eq!(query_i64(con, "SELECT length(first_word(' quack rs'))::BIGINT"), Some(5)); -# assert_eq!(query_i64(con, "SELECT count(*) FROM (SELECT first_word(NULL) AS w) WHERE w IS NULL"), Some(1)); +# assert_eq!(query_i64(con, "SELECT count(*) FROM (SELECT first_word(x) AS w FROM (VALUES (NULL::VARCHAR)) t(x)) WHERE w IS NULL"), Some(1)); # // Many groups, so DuckDB runs combine on partial states. # assert_eq!(query_i64(con, "SELECT sum(c)::BIGINT FROM (SELECT word_count('a b') AS c FROM range(100000) GROUP BY range % 997)"), Some(200000)); +# // A running window frame is not one of the shapes that crash before DuckDB 2.0. +# assert_eq!(query_i64(con, "SELECT max(w) FROM (SELECT word_count(s) OVER (ORDER BY i) AS w FROM (SELECT 'a b c' AS s, i FROM range(5000) t(i)))"), Some(15000)); ``` -Both builders call the DuckDB C API internally. `register` returns `Err` if DuckDB reports -a failure — this propagates to the entry point and is surfaced to the user. +`register` returns `Err` if DuckDB reports a failure, and the entry point turns that into +a `LOAD` error the user sees. + +When you test NULL handling, read the NULL from a column, as +`SELECT first_word(x) FROM (VALUES (NULL::VARCHAR)) t(x)` does, not from a literal: +DuckDB folds a literal `NULL` argument into the plan before it ever calls the function, +so `SELECT first_word(NULL)` passes even for a callback that gets NULL wrong +([Pitfall L8](../reference/pitfalls.md#l8-default_null_handling-does-not-propagate-nulls-for-scalar-functions)). --- -## Part 4 — Entry point +## Part 4: entry point ```rust -# use libduckdb_sys::{duckdb_aggregate_state, duckdb_bind_info, duckdb_connection, -# duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_vector, idx_t}; +# use libduckdb_sys::duckdb_connection; # use quack_rs::prelude::*; -# unsafe fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) } -quack_rs::entry_point!(hello_ext_init_c_api, |con| unsafe { register(con) }); +# fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) } +quack_rs::entry_point!(word_tools_init_c_api, |con| register(con)); ``` This one line expands to the equivalent of: @@ -374,9 +403,9 @@ This one line expands to the equivalent of: ```rust # use libduckdb_sys::{duckdb_connection, duckdb_extension_access, duckdb_extension_info}; # use quack_rs::error::ExtensionError; -# unsafe fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) } +# fn register(_: duckdb_connection) -> Result<(), ExtensionError> { Ok(()) } #[unsafe(no_mangle)] -pub unsafe extern "C" fn hello_ext_init_c_api( +pub unsafe extern "C" fn word_tools_init_c_api( info: duckdb_extension_info, access: *const duckdb_extension_access, ) -> bool { @@ -384,15 +413,73 @@ pub unsafe extern "C" fn hello_ext_init_c_api( quack_rs::entry_point::init_extension_with_policy( info, access, quack_rs::DUCKDB_API_VERSION, quack_rs::abi::AbiPolicy::Strict, - |con| unsafe { register(con) }, + |con| register(con), ) } } ``` -Pass the **full symbol name** — `hello_ext_init_c_api` here. DuckDB looks up this exact +Pass the **full symbol name**, `word_tools_init_c_api` here: DuckDB looks up this exact symbol when loading the extension. See [The Entry Point](../concepts/entry-point.md) for -the full initialization sequence. +the full initialization sequence. Put the pieces from Parts 1 to 4 in `src/lib.rs`, after +`use libduckdb_sys::duckdb_connection;` and `use quack_rs::prelude::*;`. + +--- + +## Build, stamp and load + +```bash +cargo build --release + +# DuckDB only loads files ending in `.duckdb_extension` that carry its +# 512-byte metadata footer; a bare `.so` is refused. Append the footer: +cargo install quack-rs --bin append_metadata # needs Rust 1.88 or newer +append_metadata target/release/libword_tools.so word_tools.duckdb_extension \ + --abi-type C_STRUCT --extension-version v0.1.0 \ + --duckdb-version v1.2.0 --platform linux_amd64 + +# An unsigned local build needs -unsigned (it must be a startup flag). +duckdb -unsigned +``` + +Then in the DuckDB CLI: + +```sql +LOAD './word_tools.duckdb_extension'; + +-- Aggregate: total words across all rows +SELECT word_count(sentence) AS words FROM ( + VALUES ('hello world'), ('one two three'), (NULL) +) t(sentence); + +-- Scalar: first word of each row +SELECT first_word(sentence) AS first FROM ( + VALUES ('hello world'), (' padded '), (''), (NULL) +) t(sentence); +``` + +```text +┌───────┐ +│ words │ +│ int64 │ +├───────┤ +│ 5 │ +└───────┘ +┌─────────┐ +│ first │ +│ varchar │ +├─────────┤ +│ hello │ +│ padded │ +│ │ +│ NULL │ +└─────────┘ +``` + +The 5 is 2 + 3 words; the NULL row contributes none. The empty string has no first word, +so its result is the empty string (the blank row), and the NULL row gives NULL. Use the +library file name and platform string for your system as the +[Quick Start](quick-start.md) lists them. --- @@ -449,10 +536,7 @@ fn word_count_combine() { } ``` -Run all tests with: - -```bash -cargo test --manifest-path examples/hello-ext/Cargo.toml -``` - -See the [Testing Guide](../testing.md) for the full test strategy. +Run all tests with `cargo test`. The harness cannot exercise the DuckDB side of the +callbacks (NULL handling, the window shapes above, the state layout); for that, load the +built extension into a real DuckDB, or use the `bundled-test` features. See the +[Testing Guide](../testing.md) for the full test strategy. diff --git a/book/src/getting-started/installation.md b/book/src/getting-started/installation.md index 317fcf90..36b13b15 100644 --- a/book/src/getting-started/installation.md +++ b/book/src/getting-started/installation.md @@ -10,36 +10,83 @@ Add the following to your extension's `Cargo.toml`: ```toml [dependencies] -quack-rs = "0.19" -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] ``` -> **Why `>=1.4.4, <2`?** -> DuckDB 1.4.4 through 2.0.0 load extensions built for C API version `v1.2.0` (1.5.6 and -> 2.0.0 declare `v1.5.6` and accept every earlier one), so `quack-rs` supports them all with -> a single bounded range. Releases before 1.4.4 are not supported. The range covers the -> bindings of every supported release, 1.4.4 through 1.20000.x. The `<2` upper bound does not exclude the DuckDB -> 2.0 bindings: `libduckdb-sys` numbers DuckDB `vMAJOR.MINOR.PATCH` as -> `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0`, which the range -> admits. The 1.20000.x releases need Rust 1.88. On Rust 1.86 or 1.87, Cargo's default -> (edition 2021, resolver 2) ignores `rust-version` and selects them anyway, so the build -> fails: set `rust-version = "1.86"` and `resolver = "3"` in your `[package]` (as the +Copy this block rather than running `cargo add libduckdb-sys`: `cargo add` writes the +newest version, with the default features on and without `loadable-extension`. + +> **Why `>=1.4.4, <1.20100`?** +> DuckDB 1.4.4 through 2.0.x load extensions built for C API version `v1.2.0` (1.5.6 and +> 2.0.x declare `v1.5.6` and accept every earlier one), so `quack-rs` supports them all with +> a single bounded range (an extension that uses a `duckdb-1-5*` feature is narrower: see +> [ABI Compatibility](../concepts/abi.md#unknown-duckdb-versions)). Releases before 1.4.4 are +> not supported. `libduckdb-sys` numbers +> DuckDB `vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so the range runs +> from the DuckDB 1.4.4 bindings (`1.10404.x`) through the DuckDB 2.0 bindings +> (`1.20000.x`) and stops before DuckDB 2.1 (`1.20100.0`). +> +> **The upper bound.** quack-rs, and every extension that uses its macros, names the C API +> through the crate root: `libduckdb_sys::duckdb_connection`, +> `libduckdb_sys::duckdb_register_scalar_function`, and so on. In the 1.20000 bindings +> those root names exist only as a compatibility re-export of the crate's `v1` module, +> marked `// TODO: Remove in the future` upstream. If a later release removes it, quack-rs +> and your extension stop compiling. So quack-rs widens the range one DuckDB minor at a time, +> after running its tests against the new bindings and the new engine. +> +> **When DuckDB 2.1 bindings (`1.20100.x`) are published**, nothing breaks: Cargo keeps +> resolving `1.20000.x`. An extension built that way keeps the default stamp (`C_STRUCT`, +> `v1.2.0`), which DuckDB's development branch, the 2.1 line, still loads as 2.0 does (its +> C API v1 loader was unchanged from v2.0.0 to `main` when this was checked; a release of 2.1 +> could still change that). If you ask for a newer `libduckdb-sys` yourself, Cargo +> reports a conflict with this range. There is nothing to configure on your side (a +> `[patch]` entry does not change a version requirement): wait for the quack-rs release that +> widens the range; its changelog entry says so. +> +> **Rust 1.88.** The 1.20000.x releases need Rust 1.88. On Rust 1.86 or 1.87, Cargo's +> default (edition 2021, resolver 2) ignores `rust-version` and selects them anyway, so the +> build fails: set `rust-version = "1.86"` and `resolver = "3"` in your `[package]` (as the > [Quick Start](quick-start.md) does), or resolve with > `CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback`. Either keeps the build on > `1.10506.0`, the newest release that builds there. > See [Extension Anatomy](../concepts/anatomy.md#version-support). +> +> **`default-features = false`** is optional. With the 1.20000 bindings it leaves out the +> HTTP, TLS and download tooling their default features compile: `cargo tree` lists 65 crates +> for a bare `libduckdb-sys` dependency with the defaults and 30 without them. With the +> 1.10506 bindings the difference is two crates, because that tooling is not optional +> there. Write it in your own manifest: a `features = [...]` list elsewhere in the graph +> turns the defaults back on. `cargo test` with `bundled-test` or `bundled-test-prebuilt` does +> so too, because the `duckdb` crate asks for them; a plain `cargo build --release` of the +> extension does not. + +### Build requirements + +| What you build | What it needs | +|----------------|---------------| +| The extension (`cargo build --release`) | Rust 1.86 or newer. A C compiler may be needed: the 1.10506 bindings, and the 1.20000 bindings with their default features, have a build dependency (`ring`, through the download tooling) that compiles C. `default-features = false` with the 1.20000 bindings removes it. | +| `append_metadata` (`cargo install quack-rs --bin append_metadata`) | Rust 1.88 or newer, because `cargo install` resolves the newest `libduckdb-sys`. | +| Tests with `bundled-test` | A C++ toolchain; DuckDB compiles from source, about 5 to 10 minutes cold. | +| Tests with `bundled-test-prebuilt` | A prebuilt libduckdb (`DUCKDB_DOWNLOAD_LIB=1` or `DUCKDB_LIB_DIR`) and a C compiler for the small shim that fills the dispatch table (a C++ compiler when the library comes from a DuckDB 1.4 release zip). | +| The community-extension flow (`make configure release test`, [scaffold](scaffold.md)) | Python 3 with `venv`, `pip` and network access to PyPI and GitHub, `git` (with a user identity set, for the first commit), and `make`. | Upgrading an extension from quack-rs 0.16? 0.18 has breaking changes; see -[Upgrading from 0.16 to 0.18](upgrading-0-18.md) for what to change. +[Upgrading from 0.16 to 0.18](upgrading-0-18.md) for what to change. From 0.18 or 0.19, +see [Upgrading from 0.18 to 0.20](upgrading-0-20.md). --- ## Cargo features The default build uses only the C API's stable prefix and works with every DuckDB -release in the `>=1.4.4, <2` range. Each feature below unlocks more API and raises the -`libduckdb-sys` version Cargo must resolve. The floors are the ones the comments in -quack-rs's `Cargo.toml` state. +release in the `>=1.4.4, <1.20100` range. Each feature below unlocks more API and raises +the `libduckdb-sys` version Cargo must resolve. The floors are the ones in the feature +documentation in quack-rs's `Cargo.toml`. | Feature | Requires | Enables | |---------|----------|---------| @@ -103,8 +150,13 @@ as a SQL error, or wrap its body in `quack_rs::callback::catch_ffi_panic` and re behaviour on Rust ≥ 1.81 — the runtime aborts the process — but that is exactly the outcome the guards exist to prevent.) -`validate_release_profile` rejects `panic = "abort"`, and the scaffold generator emits -`panic = "unwind"`. +`validate_release_profile` rejects `panic = "abort"`, the scaffold generator emits +`panic = "unwind"`, and `entry_point!` and `entry_point_v2!` stop the build with an error +that names the fix when the extension is compiled under `panic = "abort"` (a profile +setting, or `-C panic=abort` in `RUSTFLAGS`). An entry point you write by hand around +`init_extension` has no such check. Under `unwind` a few failures still abort the process +whatever quack-rs does: a panic while another panic unwinds, a panic hook that itself +panics, and running out of memory. --- @@ -112,11 +164,11 @@ outcome the guards exist to prevent.) quack-rs requires **Rust ≥ 1.86.0**. -1.86.0 is a ceiling as much as a floor while DuckDB 1.x is built. The +While DuckDB 1.x is built, 1.86.0 is a ceiling as much as a floor. The community-extension build workflow (`_extension_distribution.yml` in `duckdb/extension-ci-tools`) pins Rust **1.86.0** for its WebAssembly jobs on the -`v1.4-andium` and `v1.5-variegata` branches, so an extension — and therefore -quack-rs — must build on it; CI's `msrv-vs-duckdb-ci` job re-derives that pin +`v1.4-andium` and `v1.5-variegata` branches, so an extension, and therefore +quack-rs, must build on it; CI's `msrv-vs-duckdb-ci` job re-derives that pin and fails if the MSRV rises above it. (The 2.0 branch, `v2.0-cyanoptera`, pins 1.97.1 for Windows and WebAssembly.) The `msrv` job checks the crate with `cargo +1.86.0 check`, and the benchmark dev-dependency (`criterion`) needs @@ -145,9 +197,9 @@ features as a dev-dependency: ```toml [dev-dependencies] # Compiles DuckDB from C++ source: no setup, slow cold build. -quack-rs = { version = "0.19", features = ["bundled-test"] } +quack-rs = { version = "0.20", features = ["bundled-test"] } # ...or link a prebuilt libduckdb instead (set DUCKDB_DOWNLOAD_LIB=1 or DUCKDB_LIB_DIR): -# quack-rs = { version = "0.19", features = ["bundled-test-prebuilt"] } +# quack-rs = { version = "0.20", features = ["bundled-test-prebuilt"] } ``` Either one initialises the `loadable-extension` dispatch table from the linked DuckDB, so diff --git a/book/src/getting-started/quick-start.md b/book/src/getting-started/quick-start.md index 6696137d..66f6e723 100644 --- a/book/src/getting-started/quick-start.md +++ b/book/src/getting-started/quick-start.md @@ -21,25 +21,35 @@ In your extension's `Cargo.toml`: name = "my_extension" version = "0.1.0" edition = "2021" -rust-version = "1.86" # with resolver 3, Cargo picks dependency versions that build on it +# With resolver 3, Cargo picks dependency versions that build on this. +rust-version = "1.86" resolver = "3" [dependencies] -quack-rs = "0.19" -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] [lib] -name = "my_extension" # must match your extension name — see Pitfall P1 +# Must match your extension name: see Pitfall P1. +name = "my_extension" crate-type = ["cdylib", "rlib"] [profile.release] -panic = "unwind" # required — quack-rs's panic guards need unwinding (see Installation) +# Required: quack-rs's panic guards need unwinding (see Installation). +panic = "unwind" lto = true opt-level = 3 codegen-units = 1 strip = true ``` +The `libduckdb-sys` range and why it has an upper bound are explained on the +[Installation](installation.md#adding-quack-rs-to-an-existing-extension) page. + > **Starting from scratch?** The [scaffold generator](scaffold.md) generates a complete > community extension project, including this `Cargo.toml`. @@ -88,8 +98,9 @@ cargo build --release # DuckDB refuses to LOAD a bare .so: append the metadata footer first. # quack-rs ships the `append_metadata` binary for this step. Installing it needs -# Rust 1.88 or newer (it resolves the newest libduckdb-sys); use -# `cargo +stable install` if this project pins an older toolchain. +# Rust 1.88 or newer, because `cargo install` resolves the newest libduckdb-sys, +# which requires it; use `cargo +stable install` if this project pins an older +# toolchain. cargo install quack-rs --bin append_metadata append_metadata target/release/libmy_extension.so my_extension.duckdb_extension \ --abi-type C_STRUCT --extension-version v0.1.0 \ diff --git a/book/src/getting-started/scaffold.md b/book/src/getting-started/scaffold.md index 3fed89d6..86a5f4d6 100644 --- a/book/src/getting-started/scaffold.md +++ b/book/src/getting-started/scaffold.md @@ -1,8 +1,10 @@ # Project Scaffold `quack_rs::scaffold::generate_scaffold` generates every file of a new DuckDB community -extension project in Rust — `Cargo.toml`, `Makefile`, CI workflow, `description.yml`, an -example function and its tests — from a single function call. +extension project in Rust — `Cargo.toml`, `Makefile`, CI workflows, `description.yml`, an +example function and its tests, a README, the license text and a `cargo-deny` configuration +— from a single function call, and the `quack_scaffold` command does the same from the +command line. --- @@ -12,10 +14,15 @@ example function and its tests — from a single function call. my_extension/ ├── Cargo.toml # cdylib + rlib crate, MSRV, dependencies, release profile ├── Makefile # delegates to cargo + extension-ci-tools +├── README.md # build, test and try instructions, and what to submit +├── LICENSE # the license text (see "The generated project") +├── deny.toml # cargo-deny configuration ├── src/ -│ ├── lib.rs # entry point, example function, unit test +│ ├── lib.rs # entry point, an example scalar function, unit test │ └── wasm_lib.rs # WASM staticlib shim ├── description.yml # community extension metadata +├── docs/ +│ └── function_descriptions.csv # the function descriptions the community site shows ├── test/ │ └── sql/ │ └── my_extension.test # SQLLogicTest for the example function @@ -72,19 +79,53 @@ fn main() { } ``` -The repository's `scaffold_to_dir` example does the same from the command -line: +`generate_scaffold_with_warnings` returns a `Scaffold`: the same files, plus warnings about +values that were accepted but deserve a second look (a license outside quack-rs's shortlist, +an excluded platform that is opt-in, every platform excluded, an unstable-ABI project the +community build cannot ship as generated). `generate_scaffold` is the same call without the +warnings. + +### The `quack_scaffold` command ```bash -cargo run --example scaffold_to_dir -- my_extension # writes ./my_extension +cargo install quack-rs --bin quack_scaffold # once (0.20.0 or later) +quack_scaffold my_extension \ + --description "What the extension does" \ + --maintainer "Your Name" \ + --github-repo yourorg/duckdb-my-extension # writes ./my_extension + # An extension using the duckdb-1-5* features: C_STRUCT, C API v1.5.6 -cargo run --example scaffold_to_dir -- my_extension ../my_extension --c-api v1.5.6 +quack_scaffold my_extension --description "..." --maintainer "..." \ + --github-repo yourorg/duckdb-my-extension --target-duckdb-version v1.5.6 ``` -`--c-api v1.5.6` sets `target_duckdb_version` to `v1.5.6`: the binary loads into -DuckDB 1.5.6 and 2.0.0. `--unstable ` (`use_unstable_c_api`) pins the -binary to one DuckDB 1.x release instead, such as `--unstable v1.5.5`; DuckDB 2.0 -cannot load such a binary, and the scaffold refuses a 2.x release there. +Every `ScaffoldConfig` field is a flag; `--description`, `--maintainer` and `--github-repo` +are required, and each value can also be written `--flag=VALUE`: + +| Flag | Field | Default | +|------|-------|---------| +| `--version` | `version` | `0.1.0` | +| `--license` | `license` | `MIT` | +| `--excluded-platforms` | `excluded_platforms`, separated by `;` or `,` (an empty value excludes none) | Wasm and MinGW | +| `--target-duckdb-version` | `target_duckdb_version`: the C API version (`v1.2.0`, or `v1.5.6` for the `duckdb-1-5*` features), or with `--unstable-c-api` the exact DuckDB 1.x release | `v1.2.0` | +| `--unstable-c-api` | `use_unstable_c_api` | off | +| `--git-ref` | `git_ref` | the placeholder | +| `--dir` | where to write the project | `./` | + +The command refuses to overwrite a file it generates unless `--force` is given, and writes +nothing when it refuses; files it does not generate are left alone, and it never writes +through a symbolic link inside the project. Warnings go to stderr. When it is done it prints +the next steps: the `git submodule add` command for extension-ci-tools, the build and test +commands, and the exact `append_metadata` invocation for stamping a build without `make`. +(Older releases ship no such command: use the program above in a project that already +depends on `quack-rs`.) + +`--target-duckdb-version v1.5.6` sets `target_duckdb_version` to the C API version `v1.5.6`: +the binary loads into DuckDB 1.5.6 and 2.0.0, and the generated `Cargo.toml` pins the +bindings to `~1.10506.0` and enables the `duckdb-1-5*` feature it describes in a comment. +`--unstable-c-api` with `--target-duckdb-version v1.5.5` pins the binary to one DuckDB 1.x +release instead; DuckDB 2.0 cannot load such a binary, and the scaffold refuses a 2.x release +there. --- @@ -94,11 +135,11 @@ cannot load such a binary, and the scaffold refuses a 2.x release there. |-------|------|-------------| | `name` | `String` | Extension name — must match `[lib] name` in Cargo.toml and `description.yml` | | `description` | `String` | Description for `description.yml` and the `//!` docs of `src/lib.rs`. Quoted and escaped, so `:`, `#` and quotes are fine; must not be empty, padded with whitespace, or contain control characters other than newline and tab | -| `version` | `String` | The extension's version, also the Cargo package version — so it must be semver without a leading `v` (`validate_semver`): `0.1.0`, not `v0.1.0`, `1.0` or a date stamp | -| `license` | `String` | SPDX license identifier or expression (e.g., `"MIT"`, `"MIT OR Apache-2.0"`) | +| `version` | `String` | The extension's version, also the Cargo package version — so it must be semver without a leading `v` (`validate_semver`): `0.1.0`, not `v0.1.0`, `1.0` or a date stamp. A component above `u64::MAX` is refused | +| `license` | `String` | SPDX license identifier or expression (e.g., `"MIT"`, `"MIT OR Apache-2.0"`). An identifier outside quack-rs's shortlist (`BUSL-1.1`, `CC0-1.0`) is accepted with a warning; the wrong case of a listed one (`mit`), or a character SPDX does not allow, is an error. Line breaks and runs of whitespace are normalised to single spaces | | `maintainer` | `String` | Your name or org, listed in `description.yml` — one non-empty line | | `github_repo` | `String` | `"owner/repo"`, in the characters GitHub allows | -| `excluded_platforms` | `Vec` | Platforms the community build skips, written to `description.yml` and the distribution workflow alike. Defaults to `DEFAULT_EXCLUDED_PLATFORMS` (Wasm and MinGW) | +| `excluded_platforms` | `Vec` | Platforms the community build skips, written to `description.yml` and the distribution workflow alike. Defaults to `DEFAULT_EXCLUDED_PLATFORMS` (Wasm and MinGW). An entry that excludes nothing is refused: the community build compares exact names, so a platform group such as `wasm` (the error lists its platforms), a retired platform or a name it does not build is an error | | `git_ref` | `String` | `repo.ref` — **a commit hash** (or tag), not a branch. Defaults to `REF_PLACEHOLDER` so it cannot be submitted unset | | `target_duckdb_version` | `String` | Written as `TARGET_DUCKDB_VERSION` in the Makefile. Without `use_unstable_c_api`, the C API version: `DUCKDB_API_VERSION` (`v1.2.0`, the default) or `DUCKDB_API_VERSION_1_5_6` (`v1.5.6`) for an extension that enables `duckdb-1-5` / `duckdb-1-5-3` / `duckdb-1-5-4`; `v1.5.6` pins `libduckdb-sys` to `~1.10506.0` and loads into DuckDB 1.5.6 and 2.0.0. With `use_unstable_c_api`, an exact DuckDB 1.x release such as `v1.5.5` | | `use_unstable_c_api` | `bool` | Stamps `C_STRUCT_UNSTABLE`, pinning the binary to the one DuckDB 1.x release in `target_duckdb_version`. DuckDB 2.0 cannot load such a binary, and 2.x releases are refused. Defaults to `false` | @@ -114,13 +155,21 @@ Extension names must satisfy all of: Use vendor-prefixed names to avoid collisions: `myorg_analytics`, not `analytics`. -The scaffold generator checks the first two rules before generating any files and returns -an error if the name breaks one. Uniqueness is yours to check. +The scaffold generator checks the first two rules before generating any files, and +refuses the names DuckDB itself reserves (aliases such as `s3` and `md`, and the core +extensions: see [Names DuckDB reserves](../publishing.md#names-duckdb-reserves)). It +returns an error if the name breaks one. Uniqueness among community extensions is yours +to check. --- ## After scaffolding +You need `git` with a user name and email configured (the first commit fails without), +`make`, Python 3 with the `venv` module, and network access to PyPI and GitHub: +`make configure` creates a virtual environment, installs the `duckdb` package from PyPI, +and installs the SQLLogicTest runner from a pinned commit on GitHub. + ```bash cd my_extension git init @@ -139,6 +188,11 @@ commit for every DuckDB release; without a committed lock file, each rebuild resolves its dependencies afresh. (`make release` would write the file too, but after `git add -A`, so it would stay untracked.) +**Leave `CARGO_TARGET_DIR` unset for `make`.** extension-ci-tools' `rust.Makefile` copies +the library from `./target/release/`; with `CARGO_TARGET_DIR` set, Cargo builds elsewhere +and `make release` stops with `FileNotFoundError: './target/release/lib.so'` +(seen with a scaffolded project). + `git submodule add` is not optional in a new project: the scaffold writes `.gitmodules`, but `git submodule update --init` does nothing until the submodule has been added once (Pitfall P4). The generated `Makefile` stops with @@ -151,15 +205,16 @@ community build uses the branch for the DuckDB release it targets, so keep the submodule, `.gitmodules` and the workflow on the same one. **Commit before `make configure`.** `make configure` writes the extension -version into `configure/extension_version.txt` from git — the tag on `HEAD`, -otherwise the short commit hash. With no commit, extension-ci-tools records -git's error message instead, which overflows the 32-byte version field of the -metadata footer, and DuckDB then refuses the file with -`Unknown ABI type for extension`. The generated `Makefile` guards against this -by using the Cargo package version (`v0.1.0`) while there is no commit, and -`make EXTENSION_VERSION=v0.1.0 configure` sets one explicitly. If you configured -before committing, run `make clean_all configure` after the first commit so the -version comes from git. +version into `configure/extension_version.txt`, which the metadata footer +holds in a 32-byte field. extension-ci-tools writes every tag on `HEAD`, one +per line, or the short commit hash, or, with no commit, git's error message; +several tags, a tag over 32 bytes or that message overflow the field, and DuckDB +then refuses the file with `Unknown ABI type for extension`. The generated +`Makefile` chooses the value itself: the first tag on `HEAD` if it fits and holds +only `A-Za-z0-9._+-`, otherwise the short commit hash, and before the first commit +the Cargo package version (`v0.1.0`). `make EXTENSION_VERSION=v0.1.0 configure` +sets one explicitly. The value is cached, so run `make clean_all configure` after +committing or tagging. The project declares `rust-version` (quack-rs's MSRV, 1.86.0; for DuckDB 2.0, extension-ci-tools pins Rust 1.97.1 on Windows and WebAssembly and uses the current @@ -174,6 +229,45 @@ places, and push to GitHub — CI runs automatically. --- +## The generated project + +- **The example function** is a native scalar function, `my_extension_hello(name)`, written + with `ScalarFunctionBuilder::map1_str` as a one-line closure: a NULL argument gives NULL + without calling it, and a panic fails the query instead of aborting. (Before 0.20.0 it was + a `CREATE MACRO`; a macro is written into the user's database, so `LOAD` failed on a + read-only one.) +- **The SQLLogicTest** carries the header lines the upstream tests have (`# name:`, + `# description:`, `# group:`), checks that the function is missing before `require`, and + reads its NULL from a column, because a literal `NULL` is constant-folded before the + function is called. +- **`make test` fails when the runner skips a test file or runs none.** The SQLLogicTest + runner reports a file whose `require` is not met as `SKIPPED` and exits 0, which is what a + rename that leaves the extension name, the `[lib]` name and the test file out of step looks + like, so `make test` would pass having tested nothing. +- **`Cargo.toml`** names `libduckdb-sys` as `>=1.4.4, <1.20100` with + `default-features = false`, and, for a project stamped C API `v1.5.6` (or an unstable + DuckDB 1.5 target), enables the `duckdb-1-5*` feature its comments describe on `quack-rs`. +- **License files.** `LICENSE` holds the text for a single license among MIT, Apache-2.0, + BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, MIT-0 and Unlicense. An expression gets one + `LICENSE-` file per license (`MIT OR Apache-2.0` gives `LICENSE-MIT` and + `LICENSE-APACHE`); any other license gets a `LICENSE.TODO` that names the text to add. +- **`deny.toml`** configures `cargo deny check`. It ignores RUSTSEC-2026-0367, -0368 and -0369, + each with a reason: they concern the `zip` crate before 9, which is only a build-script + dependency of `libduckdb-sys` up to 1.10506.0, the release Cargo resolves for + `rust-version = 1.86`. It also allows the licenses of the dependency graph and the + project's own. +- **`extension-ci.yml`** sets `permissions: contents: read`, checks out with + `persist-credentials: false`, and pins `Swatinem/rust-cache` and `dtolnay/rust-toolchain` + to commit hashes. +- **`README.md`** says how to build, test and try the extension, that a stamped extension must + not be `strip`ped, that pull requests get a reduced CI run, and what to submit. +- **`docs/function_descriptions.csv`** has a row for the example function: the extension page on + duckdb.org takes function descriptions and examples only from this file, which belongs in + the community-extensions repository next to `description.yml` when you submit (see + [Community Extensions](../publishing.md)). + +--- + ## Excluded platforms The community build compiles every extension for nine platforms with diff --git a/book/src/getting-started/upgrading-0-20.md b/book/src/getting-started/upgrading-0-20.md new file mode 100644 index 00000000..158421be --- /dev/null +++ b/book/src/getting-started/upgrading-0-20.md @@ -0,0 +1,144 @@ +# Upgrading from 0.18 to 0.20 + +This page covers the two steps after [0.16 to 0.18](upgrading-0-18.md): 0.18 to 0.19.0, +and 0.19.0 to 0.20.0. The [changelog](../reference/changelog.md) is the full record; this +page says what to look for. Neither 0.19.0 nor 0.20.0 removes a public item that a correct +extension used. 0.20.0 has two changes that can stop a build (see "Changes that stop a +build"); several calls that used to succeed now return an error, which can fail a `LOAD` +that worked before, and a few results that were wrong are now right. + +To upgrade, set the version, build, run your SQL tests, and then load the built extension +into every DuckDB release you support: + +```toml +[dependencies] +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] +``` + +## From 0.18 to 0.19.0 + +You need 0.19.0 to support DuckDB 2.0.0. On 2.0.0, quack-rs 0.18.0 refuses to load a +build that uses the `duckdb-1-5*` features, every casting `Value` getter (`as_i64`, +`as_str`, …) returns `None`, and `bundled-test` no longer builds once Cargo resolves +`libduckdb-sys` 1.20000.x (a fresh resolve on Rust 1.88 or newer). + +### An extension that uses the `duckdb-1-5*` features + +DuckDB 2.0 cannot load a `C_STRUCT_UNSTABLE` binary through the entry point quack-rs +generates, so the stamp 0.18 advised no longer works there. Rebuild against the 1.5.6 +bindings and stamp the binary with the C API version instead: + +| In 0.18 | From 0.19 | +|---------|-----------| +| `append_metadata … --abi-type C_STRUCT_UNSTABLE --duckdb-version v1.5.5` (the exact release) | `append_metadata … --abi-type C_STRUCT --duckdb-version v1.5.6`, built against `libduckdb-sys = "~1.10506.0"`. DuckDB 1.5.6 and 2.0.0 load it; earlier releases refuse it, and the default `Strict` guard refuses a release quack-rs has not listed (2.0.1, for now). | +| `TARGET_DUCKDB_VERSION=v1.5.5` and `USE_UNSTABLE_C_API=1` in the `Makefile` | `TARGET_DUCKDB_VERSION=v1.5.6` and no `USE_UNSTABLE_C_API`, with the `extension-ci-tools` submodule on the `v2.0-cyanoptera` branch | +| `ScaffoldConfig { use_unstable_c_api: true, target_duckdb_version: "v1.5.5", … }` | `target_duckdb_version: quack_rs::DUCKDB_API_VERSION_1_5_6.to_string()` and `use_unstable_c_api: false` | + +An extension that uses only the default features needs no change: its `C_STRUCT` +`v1.2.0` binary loads into every DuckDB from 1.4.4 through 2.0.x. See +[ABI Compatibility](../concepts/abi.md). + +### Behaviour changes + +| Change | What to check | +|--------|---------------| +| `parse_description_yml` and `validate_description_yml_str`: a `repo.ref` that is not a commit hash is a warning; quoted values are no longer trimmed (`name: " my_ext"` is refused, as the community build refuses it); `validate_rust_extension` requires an exact `rust` token in `requires_toolchains` (`"python3; rust"` installs no Rust toolchain). | A `description.yml` that passed before can now fail. Run the validator on it. | +| `generate_scaffold` requires `version` to be semver (`2025120401`, `1.0` and `v1.0.0` are refused), and `ScaffoldConfig::default()` excludes `wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw`. | Pass an explicit `excluded_platforms` if you build those platforms. A project generated earlier is not rewritten; diff a fresh one against yours. | +| `validate::platform`: `osx_amd64` is an opt-in platform, as in `extension-ci-tools` for DuckDB 2.0 (DuckDB 2.1 drops it). | Listing it in `excluded_platforms` has no effect; list it in `opt_in_platforms` to have it built. | +| On DuckDB 2.0.0, an appender error caused by an open stream reports `ResourceInUse` instead of `Invalid`. | Code that matched on the error category. | +| Registering several table functions through the entry point's `Connection` lists the catalog once per `LOAD`, not once per function. | Nothing: faster. A table function registered on the raw connection after that listing is not seen by the check (see [Table Functions](../functions/table-functions.md#registering-a-name-twice)). | +| `DuckDbSecretInfo`'s `Debug` prints the length of `secret_string`, not its text. | A test that compared the `Debug` output. | +| The published crate no longer contains `scripts/`, `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md` or `GOVERNANCE.md`, and `mutants` is a dev-dependency only. | Nothing, unless a build script of yours read those files from the registry source. | + +## From 0.19.0 to 0.20.0 + +### Dependency requirement + +0.20.0 requires `libduckdb-sys` `>=1.4.4, <1.20100`; 0.18.0 and 0.19.0 required +`>=1.4.4, <2`, which also admits bindings for DuckDB releases that do not exist yet. +The new bound stops before the DuckDB 2.1 bindings because `libduckdb-sys` keeps the C API +names at its crate root only through a re-export that upstream marks for removal; the +[Installation](installation.md#adding-quack-rs-to-an-existing-extension) page has the +reasoning and what happens when 2.1 bindings are published. + +| Change | What to do | +|--------|------------| +| A project that asks for `libduckdb-sys` 1.20100 or newer no longer resolves with 0.20.0. | Remove that requirement. Cargo picks 1.20000.x, which is the DuckDB 2.0 bindings. | +| You stay on 0.18.0 or 0.19.0 and want the same ceiling. | Add `libduckdb-sys = ">=1.4.4, <1.20100"` to your own `Cargo.toml`; those releases cannot be changed after publication. | +| `default-features = false` on `libduckdb-sys` is now the documented form (the scaffold, the README and the book write it). | Optional. With the 1.20000 bindings it leaves out the HTTP and TLS download tooling their default features build. Set it in your own manifest too: a dependency elsewhere in the graph that leaves the defaults on turns them back on. | + +### Changes that stop a build + +| Change | What to do | +|--------|------------| +| `AbiCheck`, `AbiPolicy`, `ConfigOptionScope` and `TlsVersion` are `#[non_exhaustive]`, so DuckDB releases that add a variant do not break your `match`. An exhaustive `match` on one of them outside quack-rs no longer compiles (error E0004). Constructing the variants, comparing them and `TlsVersion`'s ordering are unchanged. | Add a `_` arm. | +| `ScalarBindInfo` (`duckdb-1-5`) is no longer `RefUnwindSafe`: it remembers the bind data it stored so that a second `set` can free the first. It matters only if you hold a `&ScalarBindInfo` across your own `catch_unwind`. quack-rs's own guards are unaffected. | Wrap it in `AssertUnwindSafe`. | +| `entry_point!` and `entry_point_v2!` fail to compile when the extension is built with `panic = "abort"` (a profile setting, or `-C panic=abort` in `RUSTFLAGS`), because every panic guard in quack-rs is `catch_unwind`, which does nothing there. The error names the fix. An entry point written by hand around `init_extension` has no such check. | Set `panic = "unwind"` under `[profile.release]` (the scaffold already does). | + +### Behaviour changes + +| Change | What to do | +|--------|------------| +| Registering an aggregate, or an overload in a set, **with no parameter** is an error. `SELECT my_count() FROM t` counted 100 rows on DuckDB 1.5.6 and 0 on 2.0.0. If your registration function propagates the error with `?`, `LOAD` fails. | Give the aggregate one parameter and put a macro in front for the zero-argument spelling: `SqlMacro::scalar("my_count", &[], "my_count_1(1::BIGINT)")`. See [Known Limitations](../reference/known-limitations.md#aggregates-that-take-no-argument-count-no-rows-on-duckdb-20). | +| `TypeId::Variant` (feature `duckdb-1-5-3`) is refused in a function signature: `LogicalType::try_new`, and so every builder's `param`, `returns` and result-column slot, returns an error, and `LogicalType::new` panics. It registered before, and the first query that reached it crashed DuckDB 2.0.0 or invalidated the database on 1.5.x. | Declare the parameter as `TypeId::Any`, which accepts a `VARIANT`. | +| `ListVector::get_entry` and `MapVector::get_entry` return stale bytes for a NULL row (lengths around 7·10¹³ were seen on 2.0.0). | Read with `get_entry_checked`, which returns `None` for a NULL row and for an entry outside the child, or test `is_valid(row)` first. See [Complex Types](../data/complex-types.md). | +| `append_metadata` writes a sibling file, created exclusively under a random name (it was a guessable `..tmp` that a planted symbolic link could redirect), and renames it over the output; it rewrote the output in place, which crashed a host that had the previous build loaded. A failed write removes the temporary file; a re-stamp keeps the output's permissions; an output that is a symbolic link is written through; errors name the output; a file name that is not UTF-8 is accepted; the input is read only if it is a regular file and at most 4 GiB. It also warns when the library does not export `_init_c_api`. | Nothing to change. The warning is not an error: stamping to a temporary name and renaming is ordinary. | +| `parse_description_yml` warns about a platform group name (`wasm`, `windows`, `linux`, `osx`) in `excluded_platforms` or `opt_in_platforms`, and about an `opt_in_platforms` entry that is not an opt-in platform. `DescriptionYml` gains `opt_in_platforms`. | If CI fails on warnings, write the platforms out: `wasm_mvp;wasm_eh;wasm_threads`. A group name excluded nothing in the community build. | +| `TIMESTAMPTZ_NS` (DuckDB 2.0.0) is supported: `TypeId::TimestampTzNs`, `VectorReader::read_timestamp_tz_ns`, `VectorWriter::write_timestamp_tz_ns`, `Value::timestamp_tz_ns` and `as_timestamp_tz_ns`. `TypeId::try_from_duckdb_type(42)` returned `None` before, and `LogicalType::get_type_id` panicked on such a column. | Code that treated id 42 as unknown. Before DuckDB 2.0.0, creating the type returns an error and the getter returns `None`. | +| The scaffold no longer generates `extension_config.cmake` or the `EXT_CONFIG` line of the `Makefile`; the generated `Cargo.toml` carries `license`; the generated `.cargo/config.toml` sets `-C target-feature=-crt-static` for the two musl targets. | In a project generated earlier, delete the two files and the line (no `extension-ci-tools` makefile reads them). Add the musl setting only if you opt into the musl platforms. | +| `VectorReader::is_valid`, `VectorWriter::set_null`, `ListBuilder` and `DuckStringView::as_str` do less work per row. | Nothing: the API is unchanged. | +| The duplicate-overload guard of `ScalarFunctionSetBuilder` and `AggregateFunctionSetBuilder` (and the catalog check of scalar registrations) compares `DECIMAL`, `MAP` and `ENUM` parameters by type alone, also inside `LIST`, `ARRAY`, `STRUCT` and `UNION`, as DuckDB resolves overloads. A set such as `f(DECIMAL(4,1))` with `f(DECIMAL(18,3))` registered before and then failed every call with "Could not choose a best candidate function"; it is now refused at registration, which fails a `LOAD` that propagates the error. | Give the overloads different names, or take `ANY` and branch on the type at run time. See [Scalar Functions](../functions/scalar.md#decimal-parameters-are-not-generic). | +| `WarningCollector` keeps at most 10,000 warnings between drains (`DEFAULT_MAX_WARNINGS`); later ones are counted in `dropped()`, not stored. Before, a collector that was never drained grew without bound. | Code that expects every warning to be stored: read `dropped()`, or build the collector with `WarningCollector::with_limit(n)`. | +| `data_chunk_from_arrow` refuses (`InvalidInput`) arrays that DuckDB would read out of bounds: list, large-list and map offsets that are negative, decrease or end past the child, a negative list-view offset or size, a child shorter than the rows its parent reads, a dictionary index outside the dictionary. | These arrays violate the Arrow specification; fix the producer. See [Arrow Interop](../duckdb-1-5/arrow.md). | +| A panicking `Clone` of `ScalarBindData` fails the query with the panic message; before, DuckDB's copy of the bound call silently had no bind data and the query answered as if none had been set. A second `ScalarBindData::set` in one bind frees the value it replaces (it was leaked). | Nothing, unless a test pinned the old answer. | +| `SqlMacro::register` refuses a `CREATE MACRO` text that does not parse, with DuckDB's parse message, instead of executing it. A body such as `1); DROP TABLE t; SELECT (` made DuckDB 2.0.0 run the `DROP` (1.4.4 and 1.5.6 parse the whole string first and run nothing). | Nothing, unless a macro body of yours did not parse and relied on the error. | +| `BindInfo::set_cardinality` clamps its hint to `i64::MAX` (above it, a join with the function failed with an internal error). | Nothing. | +| A `BIT` value whose bytes are malformed (only the raw `duckdb_create_bit` builds one) is refused by `Value::as_str`, `display_string` and `Debug`, which print `UNRENDERABLE`; DuckDB wrote out of bounds rendering it. | Nothing, unless a test printed such a value. | +| The entry point runs your registration closure in one transaction and rolls it back when the closure returns `Err` or panics, so a failed `LOAD` leaves no function, macro, type or table behind and can be retried in the same database. Casts, replacement scans and configuration options are not rolled back. A statement that fails inside the closure can abort the transaction (always on DuckDB 2.0), so every later statement fails with "Current transaction is aborted". | Register casts, replacement scans and configuration options last, return the first error instead of carrying on, and do not run `BEGIN`, `COMMIT` or `ROLLBACK` on the registration connection. Passing `Registration::Autocommit` through `options =` restores the earlier behaviour. See [The Entry Point](../concepts/entry-point.md#registration-runs-in-one-transaction). | +| `QUACK_RS_TARGET_DUCKDB_VERSION` is forwarded to the build only when it is a `vX.Y.Z` release of digits (a value with a newline once added `cargo:` directives of its own); anything else is a one-line warning and is ignored. | Nothing, unless you set it to something that is not a release version. | +| A `duckdb-1-5*` build under the default `AbiPolicy::Strict` refuses a DuckDB release that quack-rs's layout table does not list. 2.0.1 is not listed yet: a build that loads into 1.5.6 and 2.0.0 refuses 2.0.1. Default-feature builds are unaffected. | Rebuild with `QUACK_RS_TARGET_DUCKDB_VERSION=v2.0.1` once you have checked that release, or upgrade quack-rs to a release that lists it. See [ABI Compatibility](../concepts/abi.md#unknown-duckdb-versions). | +| `generate_scaffold` refuses configurations it used to accept: a name DuckDB uses (an alias such as `s3` or a core extension such as `json`: `INSTALL s3 FROM community` asks for `httpfs`, so such an extension could never be installed under its name), an `excluded_platforms` entry that excludes nothing (a platform group such as `wasm`, a retired platform, a name the community build does not build), a version component above `u64::MAX`, and a license with a character SPDX does not allow or a listed id in the wrong case. `parse_description_yml` refuses a reserved name too. | Rename the extension; write platforms out (`wasm_mvp;wasm_eh;wasm_threads`). See [Names DuckDB reserves](../publishing.md#names-duckdb-reserves). A license id outside quack-rs's shortlist (`BUSL-1.1`) is only a warning. | +| The scaffold generates a different project: a native scalar example function (`ScalarFunctionBuilder::map1_str`) instead of a `CREATE MACRO` that was written into the user's database; a `.test` file with the upstream header lines that reads its NULL from a column; a `Makefile` that fails `make test` when the SQLLogicTest runner skips a test file or runs none, and chooses the extension version itself; `README.md`, `LICENSE`, `deny.toml` and `docs/function_descriptions.csv`; a hardened `extension-ci.yml` (`permissions: contents: read`, `persist-credentials: false`, pinned actions); and, for a `v1.5.6` project, the `duckdb-1-5*` feature on `quack-rs`. | A project generated earlier is not rewritten; diff a fresh one against yours. The skip check and the version rule matter most. See [Project Scaffold](scaffold.md#the-generated-project). | +| `parse_description_yml` and `validate_description_yml_str` read every section by PyYAML's rules and in linear time; they reject text that `yaml.safe_load` rejects (before, `docs:` and any section other than `extension:` and `repo:` were skipped by indentation) and read `repo.variegata` and `repo.andium`. | A `description.yml` that passed before can now fail; run the validator on it. | + +### New API + +- `ScalarFunctionSetBuilder::overload_typed`: closure-built functions as overloads of a set. +- `ChunkWriter::release_row`: give back a claimed row that the source cannot fill. +- `ScalarBindData::try_get`: the bind data, or the message of a `Clone` that panicked. +- `WarningCollector::with_limit` and `dropped`. +- `entry_point::{EntryPointOptions, Registration}`, taken by `options =` in the entry-point + macros. +- `validate::validate_community_extension_name`, with the lists of reserved names, and + `DescriptionYml::git_ref_variegata` and `git_ref_andium`. +- The `quack_scaffold` command (`cargo install quack-rs --bin quack_scaffold`) and + `scaffold::generate_scaffold_with_warnings`, which returns a `Scaffold`: the files plus + warnings about values that were accepted but deserve a second look. See + [Project Scaffold](scaffold.md). +- `Value::try_cast`, `array_items`, `union_tag` (`duckdb-1-5`), `bit`, `as_bit`, `bignum` and + `as_bignum`: see [Values & Parameters](../data/values-and-parameters.md). +- `query::StatementType` with `PreparedStatement::statement_type` and `parameter_type`, + `QueryResult::statement_type`; with `duckdb-1-5`, `PreparedStatement::columns`, + `query::table_names` (and the same method on `OwnedConnection` and `Connection`) and + `Appender::for_query`, an appender whose rows feed an `INSERT … SELECT` or `MERGE INTO`. +- `from_raw` and `into_raw` on `Appender`, `PreparedStatement`, `QueryResult` and + `OwnedConnection`, for handles from a C call quack-rs does not wrap. +- Typed, tag-checked extra info (`extra_info_typed` on the aggregate, cast and `COPY` builders, + read with `extra_info_ref::()`) and typed `COPY TO` state: see + [Copy Functions](../functions/copy-functions.md#typed-state-without-raw-pointers). + `FileHandle` and `ClientContext` are `Send`. `ErrorData` implements `From`. + +### New pitfalls + +The [Pitfall Catalog](../reference/pitfalls.md) gained L20, L21, P16 and P17 (a log-storage +registration that frees its data twice, a chunk row that is published whether or not it was +written, `duckdb_get_bit` aborting on NULL, and `std::fs` bypassing DuckDB's permission +settings) and corrected L3 and L11. If your extension reads files, read +[P17](../reference/pitfalls.md#p17) first: a table function or `COPY` reader that opens a +path from SQL with `std::fs` ignores `SET enable_external_access = false`; open it with +`FileSystem`. diff --git a/book/src/introduction.md b/book/src/introduction.md index 8117483e..a4a647d2 100644 --- a/book/src/introduction.md +++ b/book/src/introduction.md @@ -5,7 +5,7 @@ # quack-rs: DuckDB extensions in Rust -**A Rust SDK for building DuckDB loadable extensions — no C++ required.** +**A Rust SDK for building DuckDB loadable extensions — no C or C++ to write.** [![CI](https://github.com/tomtom215/quack-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/tomtom215/quack-rs/actions/workflows/ci.yml) [![Crates.io](https://img.shields.io/crates/v/quack-rs.svg)](https://crates.io/crates/quack-rs) @@ -18,7 +18,7 @@ ## What is quack-rs? `quack-rs` is a Rust SDK for building [DuckDB](https://duckdb.org/) loadable extensions -without C++ or CMake. It wraps DuckDB's C Extension API — the C interface DuckDB exposes +without writing C++ or CMake. It wraps DuckDB's C Extension API — the C interface DuckDB exposes to loadable extensions — in safe builders and RAII types, and guards against the FFI pitfalls documented in the [Pitfall Catalog](reference/pitfalls.md), so you can focus on extension logic. @@ -28,7 +28,9 @@ extension requires writing glue code in C++", and that a pure-Rust path is in de ([DuckDB Community Extensions FAQ](https://duckdb.org/community_extensions/faq#can-i-write-extensions-in-rust)). That path now exists in two forms. DuckDB's own `extension-template-rs` targets the unstable C API v2 with the experimental `duckdb-neo` crate. **quack-rs** targets the stable C API v1, -so one binary loads across DuckDB releases. No C++. No CMake. No glue code. +so one binary loads across DuckDB releases. Your repository holds Rust only: no C++, no +CMake, and no glue layer of your own between your functions and the C API. (Some of the +tools around it still build C: see [Build requirements](getting-started/installation.md#build-requirements).) --- @@ -57,9 +59,10 @@ so one binary loads across DuckDB releases. No C++. No CMake. No glue code. `quack-rs` was extracted from [duckdb-behavioral](https://github.com/tomtom215/duckdb-behavioral), a production DuckDB -community extension. Building that extension revealed **16 undocumented pitfalls** in DuckDB's -Rust FFI surface — struct layouts, callback contracts, and initialization sequences that -aren't covered anywhere in the DuckDB documentation or `libduckdb-sys` docs. +community extension. Building that extension revealed the first **undocumented pitfalls** +in DuckDB's Rust FFI surface — struct layouts, callback contracts, and initialization +sequences that aren't covered anywhere in the DuckDB documentation or `libduckdb-sys` docs. +The catalogue has grown since, from building and auditing quack-rs itself. Three of those pitfalls caused extension-breaking bugs that passed 435 unit tests before being caught by end-to-end tests: @@ -77,7 +80,7 @@ for it without DuckDB. The full list is in the [Pitfall Catalog](reference/pitfa ## Key features -- **Zero C++** — no `CMakeLists.txt`, no header files, no glue code +- **No C++ in your repository** — no `CMakeLists.txt`, no header files, no glue layer of your own - **Every function kind the C API can register** — scalar, aggregate, table, cast, replacement scan, copy function (`duckdb-1-5`) — plus SQL macros - **Panic-safe FFI** — the entry point and the callbacks quack-rs generates catch panics and report them as SQL errors; registration errors surface via `Result` - **RAII memory management** — `LogicalType` and `FfiState` prevent leaks and double-frees @@ -85,7 +88,7 @@ for it without DuckDB. The full list is in the [Pitfall Catalog](reference/pitfa - **SQL macros** — register `CREATE MACRO` statements without any FFI callbacks - **Testable state** — `AggregateTestHarness` tests aggregate logic without a live DuckDB - **Scaffold generator** — generates a complete community extension project (`Cargo.toml`, `Makefile`, CI, `description.yml`, tests) from one function call -- **34 pitfalls documented** — every known DuckDB Rust FFI pitfall, with symptom, root cause and fix +- **38 pitfalls documented** — every known DuckDB Rust FFI pitfall, with symptom, root cause and fix --- diff --git a/book/src/publishing.md b/book/src/publishing.md index 37502b70..0cf0e5d3 100644 --- a/book/src/publishing.md +++ b/book/src/publishing.md @@ -22,7 +22,10 @@ settings and submission for a community extension built with quack-rs. It returns paths relative to the project root and writes nothing itself, so join them under the project directory — never write them relative to the current directory, which would overwrite its `Cargo.toml` and -`src/lib.rs`: +`src/lib.rs`. From 0.20.0 the `quack_scaffold` command +(`cargo install quack-rs --bin quack_scaffold`) writes the project for you, refuses to +overwrite an existing file without `--force`, and prints the next steps; see +[Project Scaffold](getting-started/scaffold.md). The same call from Rust: ```rust,no_run use quack_rs::scaffold::{ScaffoldConfig, generate_scaffold}; @@ -63,9 +66,13 @@ This generates: my_extension/ ├── Cargo.toml ├── Makefile +├── README.md +├── LICENSE +├── deny.toml ├── src/lib.rs ├── src/wasm_lib.rs ├── description.yml +├── docs/function_descriptions.csv ├── test/sql/my_extension.test ├── .github/workflows/extension-ci.yml ├── .github/workflows/MainDistributionPipeline.yml @@ -78,12 +85,12 @@ my_extension/ ## `description.yml` -This is the file `generate_scaffold` writes for the configuration above, with -`git_ref` set to a commit hash. Every text field is quoted because the community -build reads the file with PyYAML (YAML 1.1), where an unquoted `yes` is a -boolean and a version such as `1.0` is a number. An `excluded_platforms` line -appears only when `excluded_platforms` is not empty, and `extended_description` -repeats `description` until you replace it. +This is the file `generate_scaffold` writes for the configuration above (generated, with +`git_ref` set to a commit hash, `description` to the text shown, and the comments +included). Every text field is quoted because the community build reads the file with +PyYAML (YAML 1.1), where an unquoted `yes` is a boolean and a version such as `1.0` is a +number. An `excluded_platforms` line appears only when `excluded_platforms` is not empty, +and `extended_description` repeats `description` until you replace it. ```yaml extension: @@ -92,7 +99,7 @@ extension: version: "0.1.0" language: Rust build: cargo - license: MIT + license: "MIT" requires_toolchains: rust;python3 maintainers: - "Your Name" @@ -101,8 +108,11 @@ repo: github: "yourorg/duckdb-my-extension" # Must be a commit hash, not a branch. ref: "0a11ddc058beb2d480ccbfa83e16a68400c5d076" - # ref_next: # optional: a revision compatible with DuckDB main, - # # used while a new DuckDB release is being prepared. + # Optional, each a revision to build instead of `ref` (which is built for + # the newest release, v2.0.0): + # variegata: # for DuckDB v1.5.6 + # andium: # for DuckDB v1.4.5 + # ref_next: # for a DuckDB release still in preparation docs: hello_world: | @@ -110,10 +120,10 @@ docs: extended_description: "One-line description of what your extension does" ``` -`my_extension_hello` is the SQL macro the generated `src/lib.rs` registers, so the -`hello_world` example works as generated. +`my_extension_hello` is the native scalar function the generated `src/lib.rs` registers, so +the `hello_world` example works as generated. -### `repo.ref` +### `repo.ref` and the other revision keys DuckDB's [community extension documentation](https://duckdb.org/community_extensions/development) @@ -123,11 +133,68 @@ the result, so a branch name, which moves, would make the build unreproducible. Use a full commit hash. (A few published descriptors pin a tag instead; none pins a branch.) -quack-rs does not check that the value is a hash. `ScaffoldConfig::git_ref` -accepts letters, digits, `-`, `_`, `.` and `/`, and defaults to the placeholder -`REPLACE_WITH_COMMIT_HASH`, which `git` cannot resolve, so an unedited file cannot -be built by accident. `validate_description_yml_str` only requires `repo.ref` to be -non-empty. +The community build also reads three optional keys under `repo`. Its `scripts/build.py` +chooses the revision for the DuckDB version it is building: + +| DuckDB version being built | Revision used | +|----------------------------|---------------| +| the latest stable release (v2.0.0 when this was written) | `ref` | +| v1.5.6 (the `v1.5-variegata` line) | `variegata` if set, else `ref` | +| v1.4.5 (the `v1.4-andium` line) | `andium` if set, else `ref` | +| any other version (the development line) | `ref_next` if set, else `ref`; a prerelease validation run can be configured to skip an extension that has no `ref_next` | + +An extension that uses only the stable C API (the quack-rs default, stamped `C_STRUCT` +with C API `v1.2.0`) loads into all of those releases from one build, so one `ref` +is enough. An extension whose Makefile pins `TARGET_DUCKDB_VERSION` with +`USE_UNSTABLE_C_API=1` (a `ScaffoldConfig` with `use_unstable_c_api = true`) needs one +revision per line: the build compiles whatever revision it is given with the extension's +own Makefile, so a revision that pins DuckDB v1.4.4 is stamped `C_STRUCT_UNSTABLE` for +v1.4.4 whichever release the community build is running, and every other release refuses +to load it (built and loaded: DuckDB 1.5.6 and 2.0.0 both refused it +with "The file was built specifically for DuckDB version 'v1.4.4'"). Put the revision that targets DuckDB 2.0 +under `ref`, the one that targets 1.5.6 under `variegata` and the one that targets 1.4.5 +under `andium`. The scaffold writes a comment for each of `variegata`, `andium` and +`ref_next`, and for a `C_STRUCT_UNSTABLE` project a comment that says the project cannot ship +as generated, with a warning from `generate_scaffold_with_warnings` and from `quack_scaffold`; +the wiring is yours to do. + +`ScaffoldConfig::git_ref` accepts letters, digits, `-`, `_`, `.` and `/`, and defaults +to the placeholder `REPLACE_WITH_COMMIT_HASH`, which `git` cannot resolve, so an unedited +file cannot be built by accident. `parse_description_yml` (and with it +`validate_description_yml_str`) requires `repo.ref` to be non-empty and **warns** when it +is not 7 to 64 hexadecimal digits: a tag is legal, and a branch name does not fail the +build, it only moves. From 0.20.0 the same check applies to `variegata` and `andium` (an +empty value is an error), and both are returned as `DescriptionYml::git_ref_variegata` +and `git_ref_andium`. + +### What the community build reads + +`scripts/build.py` in `duckdb/community-extensions` reads `description.yml` like this: + +- It starts with `yaml.safe_load` (PyYAML). From 0.20.0 `validate_description_yml_str` + and `parse_description_yml` read the whole document by PyYAML's rules (a plain scalar + PyYAML refuses, a tab indent, an unterminated quote or an undefined alias anywhere in the + file is an error), in time linear in the file size; in a few documented cases they are + stricter than PyYAML (a repeated or quoted key under `extension:` or `repo:`, a tag, a + second document). Before 0.20.0 they checked only the fields quack-rs knows. Running + `python3 -c 'import yaml; yaml.safe_load(open("description.yml"))'` yourself is still + the cheapest second opinion. +- It requires the directory name `extensions//` to equal `extension.name`. +- It reads `extension.name`, `repo.github` and `repo.ref`, and, when present, + `repo.canonical_name`, `repo.ref_next`, `repo.variegata`, `repo.andium`, + `extension.excluded_platforms`, `extension.opt_in_platforms`, + `extension.requires_toolchains`, `extension.custom_toolchain_script`, + `extension.vcpkg_url`, `extension.vcpkg_commit` and `extension.test_config`. + **Any other key is ignored without a message**: a misspelt `excluded_platform:` or + `ref_nxt:` does nothing. +- `requires_toolchains` is a `;`-separated list. The workflow looks for each token + between semicolons (`;rust;`), so write `rust;python3` with no spaces: in + `rust; python3` the second token is `" python3"` and is lost. The workflow acts on + `rust` and `go` (checked) and, read in its source and not run, on `python3`, `parser_tools`, + `fortran`, `unixodbc`, `multimedia`, `cuda`, `omp` and `downgraded_aws_cli`; an unknown + token is ignored too. +- `test_config` is a JSON string of environment variables for the test run, for example + `{"test_env_variables": {"SKIP_TESTS": "1"}}`. quack-rs neither generates nor validates it. ### Validating the whole file @@ -192,7 +259,9 @@ Extension names must satisfy **all** of the following: - Be **globally unique** across the entire DuckDB community extensions ecosystem Check existing names at [community-extensions.duckdb.org](https://community-extensions.duckdb.org/) -before choosing. Use vendor-prefixed names to avoid collisions: +before choosing; quack-rs cannot check uniqueness for you (the directory names under +`extensions/` in the community-extensions repository are the authority). Use +vendor-prefixed names to avoid collisions: ```text myorg_analytics ✓ @@ -214,6 +283,51 @@ assert!(validate_function_name("1word").is_err()); // parsed as a number assert!(validate_function_name("order").is_err()); // reserved keyword ``` +### Names DuckDB reserves + +DuckDB rewrites some names before `INSTALL` and `LOAD` look anything up +(`ExtensionHelper::ApplyExtensionAlias`), and it already ships the core extensions. An +extension with one of these names can never be installed by that name from the community +repository. Measured on DuckDB 2.0.0: `INSTALL s3 FROM community;` reports `Failed to download +extension "httpfs" at URL ".../v2.0.0/linux_amd64/httpfs.duckdb_extension.gz" (HTTP 404)`, and +`md` and `sqlite3` fail the same way for `motherduck` and `sqlite_scanner`. + +| Kind | Names | +|------|-------| +| Aliases (`name` → what DuckDB looks up) | `http`, `https`, `s3` → `httpfs`; `md` → `motherduck`; `mysql` → `mysql_scanner`; `odbc` → `odbc_scanner`; `postgres` → `postgres_scanner`; `sqlite`, `sqlite3` → `sqlite_scanner`; `uc_catalog` → `unity_catalog` | +| The 34 core extensions (the union of what DuckDB 1.4.4, 1.5.6 and 2.0.0 build in or autoload, and the DuckDB 2.0 documentation's list) | `autocomplete`, `avro`, `aws`, `azure`, `core_functions`, `delta`, `ducklake`, `encodings`, `excel`, `fts`, `httpfs`, `iceberg`, `icu`, `inet`, `jemalloc`, `json`, `lance`, `legacy_parser`, `motherduck`, `mysql_scanner`, `odbc_scanner`, `parquet`, `postgres_scanner`, `quack`, `sirius`, `spatial`, `sqlite_scanner`, `sqlsmith`, `tpcds`, `tpch`, `ui`, `unity_catalog`, `vortex`, `vss` | + +The community-extensions FAQ says a community extension whose name matches a core +extension "has to be renamed". Since 0.20.0 `generate_scaffold`, `parse_description_yml` +and `quack_rs::validate::validate_community_extension_name` refuse all 44 names (the ten +aliases and the 34 core names) and say which extension an alias stands for; +`validate_extension_name` stays the syntax check, which accepts DuckDB's own names. +Whether another *community* extension already has your name is still yours to look up +(`extensions/` in `duckdb/community-extensions`). + +The list changes with DuckDB releases: `scripts/check-extension-names.py` in the quack-rs +repository compares quack-rs's copy with `src/main/extension/extension_alias.cpp`, +`extension_helper.cpp` and `extension_entries.hpp` in the DuckDB source and with the +core extensions table of the DuckDB documentation. For a name you are about to choose, +read those rather than this page. + +### Function descriptions + +The community extensions site lists the functions your extension adds, with a description, +a comment and examples for each. The C API v1 that quack-rs uses has no call that sets a +function's description (the C API v2 has `duckdb_v2_function_docs_*`), so by default the +page shows a name and a type and nothing else. To fill the text in, add +`extensions//docs/function_descriptions.csv` **to the community-extensions +repository**, in the same pull request as `description.yml`; the file is not read from +your extension's repository. Its header is `function,description,comment,example` (some +published files also carry a `function_type` column, which is ignored). If the file +exists, its columns **replace** the generated ones through a left join on the function +name: a function that is missing from the CSV loses any description, it is not merged with +one. About one extension in eight ships the file (52 of 395 descriptors when this was +checked). The scaffold writes `docs/function_descriptions.csv` with a row for its example +function; copy it to `extensions//docs/` in your pull request to the +community-extensions repository. + > **Pitfall P1** — The `[lib] name` in `Cargo.toml` MUST exactly match the > extension name. If your crate name is `duckdb-my-ext` (producing > `libduckdb_my_ext.so`) but `description.yml` says `name: my_ext`, the @@ -249,6 +363,24 @@ match stability { # } ``` +### Which version DuckDB shows + +`description.yml`'s `version` is documentation: DuckDB does not read it. The version that +`duckdb_extensions()` reports comes from the metadata footer, and `make configure` writes it +from git: **the tag on `HEAD`, if there is one, otherwise the abbreviated hash of the +commit** (`git tag --points-at HEAD`, then `git log -1 --format=%h` in +`extension-ci-tools`' `configure_helper.py`; it is not `git describe`). So the community +build reports a short hash for your extension unless the commit it builds carries a tag. +Two things break the footer: a tag longer than the footer's fixed 32-byte version field +(DuckDB 2.0.0 refuses such a file with an `INTERNAL Error: Unknown ABI +type for extension`), and several tags on one commit (the tool joins them with newlines; +measured: tags `v0.1.0` and `v0.1.0-rc1` on one commit give the two-line value +"v0.1.0" then "v0.1.0-rc1"). Keep tags short and put one tag on a released commit. The +`Makefile` that the scaffold generates since 0.20.0 sets `EXTENSION_VERSION` itself (the +first tag on `HEAD` if it fits and holds only `A-Za-z0-9._+-`, else the short hash, else the +Cargo version before the first commit). A project that does not set it, such as one from an +older scaffold, gets `extension-ci-tools`' value, so the rules above apply to it. + --- ## Platform targets @@ -276,11 +408,13 @@ reports which these are. To have one built, list it in `extension.opt_in_platfor (a `;`-separated string, like `excluded_platforms`). Both fields are matched against the platform names above **exactly**. A group -name such as `wasm` or `windows` is not a platform: the build's -`modify_distribution_matrix.py` tests `arch in excluded_arch_values` (on every -`extension-ci-tools` branch the community build uses), so `excluded_platforms: -"wasm"` leaves all three `wasm_*` platforms building. Write -`wasm_mvp;wasm_eh;wasm_threads`. Published files do contain group names, so +name such as `wasm` or `windows` is not a platform. On the 1.x lines +(`v1.5-variegata`, `v1.4-andium`) the matrix comes from +`modify_distribution_matrix.py`, which splits on `;` only and tests +`arch in excluded_arch_values`; on `v2.0-cyanoptera` it comes from the Go program +`extbuild matrix` (which, read in its source and not run, trims spaces and splits on `,` too). Either way `excluded_platforms: "wasm"` leaves all three `wasm_*` +platforms building. Write `wasm_mvp;wasm_eh;wasm_threads`: separated by `;`, no spaces, +which both accept. Published files do contain group names, so `parse_description_yml` accepts them, with a warning that spells out the platforms the group stands for; it also warns about an `opt_in_platforms` entry that is a group, is not an opt-in platform, or is not a platform at all, all of @@ -330,14 +464,23 @@ assert!(validate_platform("invalid").is_err()); name = "my_extension" version = "0.1.0" edition = "2021" +# With resolver 3, Cargo picks dependency versions that build on this. Without +# these two lines, a fresh lock file selects libduckdb-sys 1.20000.x, which needs +# Rust 1.88: the WebAssembly jobs of the 1.x extension-ci-tools lines pin 1.86.0. +rust-version = "1.86" +resolver = "3" [lib] name = "my_extension" # Must match description.yml `name` crate-type = ["cdylib"] [dependencies] -quack-rs = "0.19" -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +quack-rs = "0.20" + +[dependencies.libduckdb-sys] +version = ">=1.4.4, <1.20100" +default-features = false +features = ["loadable-extension"] [profile.release] panic = "unwind" # Required — "abort" disables quack-rs's panic guards @@ -347,8 +490,10 @@ codegen-units = 1 strip = true ``` -This matches the scaffold's `Cargo.toml` (which also declares a `staticlib` example -target for WebAssembly builds). +The scaffold's `Cargo.toml` has these settings and adds `license`, explanatory comments +and a `staticlib` example target for WebAssembly builds. For a project stamped C API `v1.5.6` +it also enables the `duckdb-1-5*` feature on `quack-rs` and pins `libduckdb-sys` to +`~1.10506.0`. > **[ADR-4](https://github.com/tomtom215/quack-rs/blob/main/docs/architecture.md#adr-4-libduckdb-sys-only-at-run-time-no-duckdb-crate)** > — Do NOT use the `duckdb` crate's `bundled` feature. A @@ -378,9 +523,9 @@ assert!(validate_release_profile("abort", "true", "3", "1").is_err()); crash, and `catch_unwind` cannot catch anything under `panic = "abort"`: the runtime aborts before unwinding starts, killing the user's DuckDB session. -The older advice to set `abort` came from panics escaping an `extern "C"` -boundary once being undefined behavior. They no longer are — Rust defines that -as an abort — and quack-rs catches them before the boundary anyway. +The older advice to set `abort` dates from when a panic escaping an `extern "C"` +function was undefined behavior. It now aborts the process by definition, and +quack-rs catches panics before the boundary anyway. --- @@ -418,6 +563,20 @@ WebAssembly (`wasm_mvp`, `wasm_eh`, `wasm_threads`) and `windows_amd64_mingw` targets by default; set `excluded_platforms` to opt them back in once your extension builds there. +**On a pull request the community build runs a reduced matrix.** The default +`reduced_ci_mode: auto` builds only the platforms marked `run_in_reduced_ci_mode` in +`config/distribution_matrix.json` (`linux_amd64`, `windows_amd64` and `wasm_eh` on +`v2.0-cyanoptera`) and builds the rest after the merge, so a failure that is specific +to `osx_arm64` or `linux_arm64` shows up after your pull request is merged. (That +`auto` means "reduced" for a pull request was read in `extbuild` by the conform +review; the matrix flags and the workflow input are from the upstream files.) + +**Do not run `strip` on a stamped extension.** DuckDB's documentation warns that stripping +removes or corrupts the metadata footer. Measured: `strip` on a stamped extension exits 0, +and `LOAD` then fails with `The file is not a DuckDB extension. The metadata at the end +of the file is invalid`. Cargo's `strip = true` is safe because it runs when the library +is linked, before the footer is appended. + After scaffolding: ```bash @@ -436,8 +595,9 @@ make release ``` **Commit before `make configure`.** `make configure` writes the extension version -from `git describe` into `configure/extension_version.txt`. In a repository with -no commit, that command fails and its error text ("error: malformed object name +into `configure/extension_version.txt` from the tag on `HEAD`, or else the short hash of +the commit (see [Which version DuckDB shows](#which-version-duckdb-shows)). In a repository +with no commit, that command fails and its error text ("error: malformed object name 'HEAD'") is written to the file instead. The text overflows the 32-byte version field of the metadata footer, and DuckDB then refuses the stamped file with an error such as "Unknown ABI type for extension". If this has already happened, @@ -447,9 +607,9 @@ again. **Pin `extension-ci-tools` to the community build's version.** The community repository builds with the `extension-ci-tools` branch for the current DuckDB release: for DuckDB v2.0.0 its build workflow uses -`_extension_distribution.yml@v2.0-cyanoptera`, and `extension-ci-tools` has no -`v2.0.0` branch. (For DuckDB v1.5.6 it used `v1.5-variegata`, and the `v1.5.6` -branch points to the same commit.) Check out the matching branch so your local +`_extension_distribution.yml` from the `v2.0-cyanoptera` branch, and `extension-ci-tools` +has no `v2.0.0` branch. (For DuckDB v1.5.6 it used `v1.5-variegata`; a `v1.5.6` branch +exists too, but it is a different commit.) Check out the matching branch so your local build uses the same scripts. For DuckDB 2.0, `extension-ci-tools` pins Rust 1.97.1 on Windows and for WebAssembly (the `v1.5-variegata` branch pins 1.86.0 for WebAssembly), installs the current stable on Linux (`rustup` inside the build @@ -486,7 +646,7 @@ LOAD my_extension; An extension that uses only the stable C API (the default quack-rs features) is stamped `C_STRUCT` with C API version `v1.2.0`, and one binary loads into every -DuckDB release from 1.4.4 through 2.0.0 for its platform. The `libduckdb-sys = ">=1.4.4, <2"` +DuckDB release from 1.4.4 through 2.0.x for its platform. The `libduckdb-sys = ">=1.4.4, <1.20100"` range above is correct for such an extension. An extension that enables the `duckdb-1-5*` features should be built against the @@ -516,5 +676,9 @@ responsibility is yours: - Never let a panic escape an FFI boundary: quack-rs's callbacks catch panics and report them as SQL errors, which requires `panic = "unwind"` - Validate user inputs at system boundaries (extension entry point is the boundary) +- Open every file path that comes from SQL with the `FileSystem` wrapper, not `std::fs`: + `std::fs` ignores `SET enable_external_access = false` and `allowed_directories`, so a + function that reads user-supplied paths with it lets a user read what the person running + DuckDB meant to keep closed ([Virtual File System](duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings)) - Do not include secrets, API keys, or credentials in your binary - Dynamic SQL in SQL macros must not construct queries from unsanitized user data diff --git a/book/src/reference/changelog.md b/book/src/reference/changelog.md index 978ba79f..d09cb847 100644 --- a/book/src/reference/changelog.md +++ b/book/src/reference/changelog.md @@ -10,6 +10,24 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.20.0] — 2026-10-30 + +This release comes out of an eighth review (`AUDIT.md` section 14): twelve parallel +reviewers (soundness, the diff since 0.19.0, C API gaps, conformance with DuckDB's +community-extension process, documentation, CI and supply chain, hardening, the FFI +boundary, two authors' passes over tables and types, dependencies, and the test suite +itself), then six work packages that fixed what they found, each with the tests that fail +without the fix. It also carries the performance work that review measured against 0.19.0. + +**It contains breaking changes**, marked **Breaking:** under Changed: the entry-point +macros refuse `panic = "abort"`, four public enums are `#[non_exhaustive]`, registration +is transactional, `generate_scaffold` and `parse_description_yml` refuse inputs they used to +accept, and a zero-argument aggregate no longer registers. "Upgrading from 0.18 to 0.20" in +the book lists each with what to do. Two High-severity defects were found and fixed before +release: a segmentation fault in an unpublished `ListBuilder` optimisation, and +`SqlMacro::register` executing statements ahead of a parse error on DuckDB 2.0.0 (see +Security). + ### Added - **`TIMESTAMPTZ_NS`, the type DuckDB 2.0.0 added** (`DUCKDB_TYPE_TIMESTAMP_TZ_NS`, @@ -50,10 +68,144 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). a change to a vector, validity or list path. - `scripts/check-api-coverage.py` and `scripts/api-coverage-allowlist.txt`: every slot of `duckdb_ext_api_v1` is either used under `src/` or listed with a class and - a reason (546 slots at v1.5.6: 447 used, 99 listed: 45 deprecated, 22 host-side, - 11 superseded, 21 candidates to wrap). The `abi-table` CI job runs it, so a slot a + a reason (546 slots at v1.5.6: 457 wrapped, 89 listed: 45 deprecated, 23 host-side, + 13 superseded, 8 candidates to wrap). The `abi-table` CI job runs it, so a slot a DuckDB release adds is noticed instead of left for someone to read the header. +- **`quack_scaffold` binary** (`cargo install quack-rs --bin quack_scaffold`): writes a new + extension project to disk from the command line. Every `ScaffoldConfig` field is a flag + (`--description`, `--maintainer`, `--github-repo`, `--version`, `--license`, + `--excluded-platforms`, `--target-duckdb-version`, `--unstable-c-api`, `--git-ref`, `--dir`). + It refuses to overwrite an existing file unless `--force` is given, and writes nothing if it + refuses. It never writes through a symbolic link inside the project. When it is done it + prints the next steps: the `git submodule add` command for extension-ci-tools, the build and + test commands, and the exact `append_metadata` invocation for stamping without `make`. +- **`scaffold::generate_scaffold_with_warnings`** and **`scaffold::Scaffold`**: the generated + files plus warnings about values that were accepted but deserve a second look. These are: a + license id outside quack-rs's shortlist, an excluded platform that is opt-in, excluding every + platform, and an unstable-ABI project the community build cannot ship as generated. + `generate_scaffold` is unchanged and drops the warnings. +- **Scaffold: files a community submission needs** (CONFORM-14/15, DEPS-1): + - `README.md`: build, test and try instructions, the no-`strip`-after-stamping rule, + reduced CI on pull requests, and what to submit. + - `docs/function_descriptions.csv`: the duckdb.org extension page takes function + descriptions and examples only from this file. It has a row for the example function. + - The license text: `LICENSE` for MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, + MIT-0 and Unlicense. An expression gets one `LICENSE-` per license. Anything else gets + a `LICENSE.TODO` naming the text to add. + - `deny.toml`: ignores RUSTSEC-2026-0367/-0368/-0369 (zip < 9), each with a reason. That + zip is only a build-script dependency of `libduckdb-sys` ≤ 1.10506.0, which is what Cargo + resolves for `rust-version = 1.86`. The file also allows the dependency graph's licenses + and the project's own. `cargo deny check` passes on a freshly generated, locked project; + without the ignores it fails on exactly those three advisories. +- **`validate::validate_community_extension_name`**: rejects a name `DuckDB` itself uses, both + its aliases (`s3`, `md`, `postgres`, ...) and its core extensions (`json`, `parquet`, ...), + as well as a badly formed one. The parser and the scaffold use it. `validate_extension_name` + stays a syntax check only. (CONFORM-3) +- **`description.yml`: `repo.variegata` / `repo.andium`** (the refs built for DuckDB v1.5.6 and + v1.4.5) are read into `ExtensionDescription::git_ref_variegata` / `git_ref_andium`. An + unknown `repo` key is now a warning, and so is a `repo` that is not `owner/repo`. + (CONFORM-10) + +- `ChunkWriter::release_row(row)` gives back the row the last `next_row` claimed, so the + drop does not publish it; the next `next_row` returns the same index. Giving back any + other row panics with the row that could be given back. For the + `while let Some(row) = cw.next_row()` loop whose source runs out (UNSOUND-1). +- `ScalarFunctionSetBuilder::overload_typed(TypedScalarFunctionBuilder)`: a closure-built + function (`map1`, `map2`, `try_map1`, ...) can now be one overload of a set, keeping the + signature its closure fixed. This is the only way to overload closure-built functions on + DuckDB 1.4.x, where a second registration under a taken name is refused. An overload built + under another name than the set's is refused by `register` (AUTHT2-2). +- `ScalarBindData::try_get`: the bind data, or the panic message of a `Clone` that failed + when DuckDB copied the bound call (FFI-6). +- `WarningCollector::with_limit(n)`, `WarningCollector::dropped()` and + `warning::DEFAULT_MAX_WARNINGS` (HARDEN-8, see Changed). + +- `entry_point::Registration` (`Transactional`, the default, or `Autocommit`) and + `entry_point::EntryPointOptions` (an `AbiPolicy` plus a `Registration`, built with + `EntryPointOptions::new().abi_policy(..).registration(..)`; `From`). The entry + point macros take them as `entry_point!(name, options = EntryPointOptions::new()..., |con| ..)` + (and `entry_point_v2!`); hand-written entry points use the new + `init_extension_with_options` / `init_extension_v2_with_options`. The existing + `AbiPolicy` arm and `*_with_policy` functions are unchanged (FFI-2). + +- **`Value::try_cast(&LogicalType)`** — SQL `TRY_CAST` of a value to any + logical type, nested ones included, with no path for a C++ exception to reach + Rust (the cast runs inside `duckdb_create_list_value`'s `try`). Payloads + `DuckDB` cannot convert safely (out-of-range timestamps, over-wide `DECIMAL`s) + are refused first, inside `ARRAY`s too. All engines (1.4.4+), no feature. + (APIGAPS-2) +- **`Value::array_items()`** — the elements of an `ARRAY` (or `LIST`) value; an + `ARRAY` read through `list_items` looks empty because the C API reads `LIST` + only. A fixed-size-list argument of a table function, `COPY` option or + scalar bind is now readable. All engines. (APIGAPS-2) +- **`Value::union_tag()`** — the active member of a `UNION` value (its member + value reads as text through `try_cast` to `VARCHAR`). Feature `duckdb-1-5`. + (APIGAPS-2) +- **`Value::bit(&[bool])`, `Value::as_bit()`, `Value::bignum(&[u8], bool)`, + `Value::as_bignum()`** — `BIT` and `BIGNUM` values. The getters never hand + `DuckDB` a value its `duckdb_get_bit` / `duckdb_get_bignum` would abort the + process on (SQL `NULL`, `'abc'`, a `BLOB`, `inf`; a null handle segfaulted + `duckdb_get_bit`); the constructors build only well-formed, canonical values. + All engines. (APIGAPS-3) +- **`query::StatementType`**, **`PreparedStatement::statement_type()`**, + **`QueryResult::statement_type()`**, **`PreparedStatement::parameter_type(i)`** + — what a statement is and its parameter types before it runs. All engines + (stable slots); `COPY DATABASE` / `UPDATE EXTENSIONS` / `MERGE INTO` are + reported as `Invalid` by DuckDB before 1.5.5. (apigaps P1) +- **`PreparedStatement::columns()`** — result column names and types before + execution (`None` while they depend on unbound parameters). Feature + `duckdb-1-5`. (apigaps P1) +- **`query::table_names(con, sql, qualified)`**, **`OwnedConnection::table_names`**, + **`Connection::table_names`** — the tables a statement touches, without running + it (one statement; `JOIN … USING` is refused by DuckDB's names-only bind). + Feature `duckdb-1-5`. (apigaps P1) +- **`Appender::for_query(con, sql, types, table, columns)`** — an appender whose + rows feed a query (`INSERT … SELECT`, `INSERT OR REPLACE`, `MERGE INTO`, …) + instead of a table. Feature `duckdb-1-5`. (apigaps P1, slot 409) +- **`from_raw` / `into_raw`** on `Appender`, `PreparedStatement`, `QueryResult` + and `OwnedConnection`, so a handle from an unwrapped C call can be adopted + into RAII. All engines. (APIGAPS-9) +- **Typed, tag-checked extra info**: `extra_info_typed(value)` on + `AggregateFunctionBuilder`, `AggregateOverloadBuilder`, `CastFunctionBuilder` + and `CopyFunctionBuilder`, read with `extra_info_ref::()` on + `AggregateFunctionInfo`, `CastFunctionInfo` and the four `Copy*Info` types; a + value of another type reads as `None` instead of reinterpreting memory, and + the destructor is generated and panic-safe. All engines (copy: `duckdb-1-5`). + (APIGAPS-8) +- **Typed `COPY TO` state**: `CopyBindInfo::set_bind_data_typed`, + `CopyGlobalInitInfo::set_global_state_typed`, and `bind_data_ref` / + `global_state_ref` on the later phases — no hand-written `Box::into_raw` / + `extern "C"` destructors. Feature `duckdb-1-5`. (AUTHT-3) +- **`FileHandle` and `ClientContext` are now `Send`** (not `Sync`), and + `FileHandle::into_raw` exists: a file opened in a `COPY` `global_init` can be + kept in the statement's global state (in a `Mutex`) without an + `unsafe impl Send` wrapper; the pattern is documented on `FileHandle`. + Feature `duckdb-1-5`. (AUTHT-2) +- **`From for ErrorData`**, so `?` on a `Value` or builder error + works inside `Appender::row` with `duckdb-1-5`. (AUTHT-13) +- **35 new tests from a review of the test suite itself** (`TESTS-3` to `-7`), each + written against a mutant that the existing suite let live, found by exposing the + bodies of `unsafe fn`s that `cargo-mutants` skips: every fixed-width and string type + echoed over 5,000 distinct rows (`wide_rows`, which kills the `u128` stride mutants); + the `OwnedVector` capacity guard for `MAP` and `UNION`; twelve Arrow layout-walker unit + tests; `AbiPolicy` enforced in-process against a v1.4.4 layout table + (`tests/abi_policy_enforced.rs`); `ScalarBindData::get_from_init`, two `extra_info` + destructors and `abi::check`; the seven `Registrar` methods of `Connection` that no test + executed; randomised cross-checks of `datetime` and `interval` against DuckDB's own SQL + (4,000 values per check, fixed seed); and property tests for `parse_version`, SPDX, + `description.yml` and the scaffold's escaping. They pass on DuckDB 1.5.6 and 2.0.0. +- Four fuzz targets, each with a seed directory: `abi_version`, `append_metadata`, + `description_roundtrip` and `scaffold_text` (60 s each on this tree; the first run of + `description_roundtrip` found its own escaping wrong, because PyYAML refuses U+007F even + inside quotes). The fuzz job runs every target `cargo fuzz list` prints. +- `scripts/mutants-expose-unsafe.py`: prepares a copy of the tree so `cargo-mutants` can + mutate what it skips (`unsafe fn` bodies, integer operands of pointer arithmetic). An + audit aid, not run in CI; see CONTRIBUTING.md, "Mutation testing". [TESTS-2] +- `tests/lsan_probe.rs`: the `leak-check` job's canary. It leaks a Rust allocation, a + `LogicalType` and a `Value` on request, and the job now requires LeakSanitizer to fail + on each. [TESTS-1] + ### Fixed - **A `VARIANT` in a function signature crashed DuckDB.** `TypeId::Variant` @@ -85,6 +237,128 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `Cargo.toml`: `cc = "1"` could not be resolved by `cargo update -Z direct-minimal-versions`; the lower bound is now `1.0.69`. +- CI: `scripts/check-ci-aggregate.py` no longer passes while a gate is removed + (a job demoted to informational and dropped from `needs`, a step-level + `continue-on-error` in a blocking job or in `ci-ok` itself, a job id with a + comment or anchor, a safety pattern kept only in a comment); it also checks + the workflows README's gate table. [CI-7] +- CI: `scaffold-compile` can no longer pass by running zero tests (`--exact`, + one test binary, and the libtest summary must say `1 passed`). [CI-8] +- CI: `gh release edit` now clears a leftover draft flag and sets the + prerelease flag both ways. [CI-9] +- CI: `scripts/check-api-coverage.py` counts a C API slot as wrapped only when + library code uses it (not comments, string literals, `#[cfg(test)]` code or + test-only modules): `duckdb_open` / `duckdb_close` are test-only, so they are + listed as host-side. [APIGAPS-4, CI-14] +- CI: `check-msrv-vs-duckdb-ci.py` reads `# v1.86.0` / `# rust 1.86.0` pin + comments and `toolchain:` inputs as the constraints they are; + `check-autoload-entries.py` refuses a reshaped upstream array instead of + passing on zero parsed names. [CI-14] + +- **Scaffold (HARDEN-5):** a license with a line break or other whitespace (`MIT\nOR + Apache-2.0`) was written raw into `Cargo.toml`, which then was not TOML. A version + component above `u64::MAX` was accepted, and Cargo then refused to read the project. +- **Scaffold:** an `excluded_platforms` entry that excludes nothing is now refused. The + community build compares exact names, so this covers a platform group such as `wasm` (the + error lists its platforms), a retired platform, and a name it does not build. Before, any + of these was written into `description.yml` and the workflow as if it worked. +- **`append_metadata` (HARDEN-3):** the temporary file was `..tmp`, opened with + create-and-truncate. A symbolic link planted at that guessable name made the tool overwrite + the link's target. It is now created exclusively, under a random name. +- **`append_metadata` (LASTDIFF-6, FFI-11):** + - a failed write left the temporary file behind; it is now removed on every failure; + - a re-stamp reset the output's permissions; they are now kept; + - an output that is a symbolic link was replaced by a regular file; it is now written + through, as 0.19.0 did; + - errors named the temporary file, and a missing output directory surfaced as a + temporary-file error. Errors now name the output and say when the directory does not + exist. +- **`append_metadata` (HARDEN-4):** a file name that is not UTF-8 made it panic (exit 101). + Such names are now used as paths; a flag's value must still be UTF-8. +- **`append_metadata`:** the input is read only if it is a regular file, and only up to 4 GiB. + A FIFO blocked it forever, and `/dev/zero` grew its memory until it was killed. +- **Build script (HARDEN-12):** `QUACK_RS_TARGET_DUCKDB_VERSION` holding a newline could add + `cargo:` directives of its own. Only an all-digit `vX.Y.Z` is now forwarded; anything else + is a one-line, quoted warning. +- **`description.yml`:** the `repo.ref` warning now says why a tag is not enough either (a + branch or a tag can be moved). (CONFORM-15) + +- `ChunkWriter`'s documentation said it counts rows *written*; it counts rows *claimed*, + and a claimed row never written was published with stale contents from DuckDB's reused + output chunk (a wrong answer; for a hand-written `LIST`/`MAP` column a stale entry). + The docs now say so, and `release_row` gives such a row back (UNSOUND-1). +- A `Clone` of `ScalarBindData` that panicked while DuckDB copied the bound call (filter + pushdown does) left the copy with no bind data, so the query answered as if none had + been set and the panic message was lost. `ScalarBindData::get` now reports the message as + the call's error (the query fails with it), `get_from_init` likewise (FFI-6). +- The walks over a logical type (`type_string`, `describe`, `contains_type`) are bounded at + 256 levels: a type nested a few thousand levels deep overflowed the stack and aborted the + process (HARDEN-6). +- `data_chunk_from_arrow`'s list-offset arithmetic is checked (`[i64::MIN, 0]` overflowed + `last - first`) (FFI-1). +- **The `leak-check` CI job could not fail for a missing `duckdb_destroy_*`.** + `lsan-suppressions.txt` held `leak:libduckdb.so`, and LeakSanitizer matches a + suppression against every frame of an allocation, so it silenced every handle DuckDB + allocates. Measured with `tests/lsan_probe.rs`: a leaked `LogicalType` or `Value` exited + 0 with the rule and 23 with a leak report without it. The rule was not needed: the + 460 tests of `ffi_roundtrip` exit 0 under LeakSanitizer with no suppression, because + DuckDB's own one-time allocations are still reachable at exit. The file now holds no + rule, and the job runs the canary. [TESTS-1] +- `abi::parse_version` accepted a signed part (`"1.5.+4"`, because `u64::from_str` takes a + leading `+`); it takes ASCII digits only, like the build script's copy of the rule. + [TESTS-9] +- **`parse_description_yml` disagreed with PyYAML 6.0.3 in four places**, three found by the + first sharded run of the mutation job (101 of the 102 mutants that survived were in this + reader) and one by a differential run on flow collections afterwards. Each has a + regression test whose expectation is PyYAML's verdict: + - A block scalar (`description: |`) lost the lines after a blank line when the next line was + a `#` line indented as far as the text, so `l1`, a blank line and `# c` read as `l1` + where PyYAML reads `l1\n\n# c\n`. A comment at the key's own level still ends the scalar. + - An anchor on a line of its own, followed by a sequence written at the key's indentation + (`k:\n &a\n- x`), was refused; PyYAML reads `{k: [x]}`. + - A quoted key was measured without the spaces before its colon, so one PyYAML refuses at + 1,024 characters (`"…" : v`, "mapping values are not allowed here") was accepted. + - Inside a flow collection, a node followed by `:` was taken for a mapping key wherever the + `:` was. PyYAML takes it for one only if the `:` is on the key's own line and at most + 1,024 characters from where the key began, an anchor in front of it included; otherwise + the `:` has no key and the parser stops there ("expected ',' or '}'"). `k: {a` with + `b: c}` on the next line, `k: [a` with `b: c]` on the next, `{"a"` with `: c}` on the + next, and a key of 1,025 characters were accepted in a section the reader only checks + (and in `extension:`, for the mapping form). + The statement above, that the reader never accepts a file PyYAML refuses, held for 118,500 + random mutants of the published descriptors and missed the third and the fourth. On 30,120 + random flow collections (token soup over brackets, braces, quotes, anchors, aliases, + colons and line breaks, plus keys at the 1,024 boundary) the reader disagreed with PyYAML + on 109 documents before the fix, every one a file only PyYAML refuses, and on none after, + with no document newly refused and no panic. On 14,874 targeted + documents (block scalars with comment-like lines, plain, quoted and flow values with + comments below them) the fixed reader's `description` equals PyYAML's wherever both accept + (200 differed before), and it matches PyYAML on 474 more documents than before and on none + fewer. It still refuses two things PyYAML accepts, both unchanged: a tab-only content line + inside a block scalar (a `|` scalar with a line of six spaces and a tab) and a tag, or an + anchor and a tag, alone on a line whose value starts on the next one (`k:\n !!str`). + [MUT-1] + +### Security + +- **`SqlMacro::register` ran the statements that precede a parse error on DuckDB + 2.0.0 (SQLMACRO-1).** It counted the body's statements with + `duckdb_extract_statements` and refused only a count above one; a body that does + not parse counts as zero and was then executed. DuckDB 1.4.4 and 1.5.6 parse the + whole string before running anything, so nothing ran there; DuckDB 2.0.0 runs + the statements it had already parsed. With the body `1); DROP TABLE canary; --` + (the shape of the module documentation's own example) the table was dropped. + `register` now requires exactly one statement, and a count of zero is an error + that carries the parser's message. A macro body is the extension author's code, + not SQL data (the module documentation says never to build one from untrusted + text), so exploiting this needed an author who passed untrusted text; the guard + the documentation described as a defence failed open on 2.0.0 all the same. + Regression test `hostile_input::a_macro_body_cannot_turn_one_statement_into_several`: + on the 0.19.0 code it fails on 2.0.0 with "Table with name canary does not exist"; + it passes on 1.5.6 and 2.0.0 with the fix. Earlier hostile-input sweeps ran on + 1.5.6 only and could not have seen this (`AUDIT.md` section 14.9 lists the + corrections). + ### Performance - **Nullable columns, string reads and list building cost less in the callback @@ -93,31 +367,37 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `duckdb_vector_get_validity`) instead of making one C call per row; `DuckStringView::as_str` checks for ASCII before falling back to full UTF-8 validation; `ListBuilder` keeps its child writer between rows (dropped whenever - a reserve grows the child) and writes the parent entries through the entry array. + a reserve grows the child) and writes the parent entries through the entry array, + which it fetches on every call. An earlier version of this change cached the + array's address and crashed with a segmentation fault when a nested list's reserve + grew the parent (found in review before release, never published); the test + `a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves` fails on that + version and passes on this one. Measured with `benches/engine/compare.py`, DuckDB 1.5.6, one thread, release build with fat LTO, a shared 4-vCPU virtual machine ("Intel(R) Xeon(R) Processor @ 2.10GHz"), - 20 million rows, base and new interleaved, 18 timed runs per cell; median ns per - row, with the hand-written twin in brackets: + 20 million rows (10 million for the long strings), base and new interleaved, 18 + timed runs per cell, load average 1.5 at the start; median ns per row, with the + hand-written twin in brackets: | | 0.19.0 | now | |---|---|---| - | `map1` on a column with 10% NULLs | 7.45 (5.95) | 6.08 | - | `map2`, same column | 9.95 (6.20) | 6.75 | - | `map1` on a column with no NULLs (control) | 1.95 (1.80) | 2.00 | - | `map1_str` returning an `i64`, short strings | 33.15 (21.40) | 24.35 | - | the same, 46 to 50-byte strings | 47.30 (39.40) | 42.15 | - | `map1_str` returning a `String` (trim), short strings | 63.52 (41.70) | 59.02 | - | `ListBuilder`, `[x, x+1, x+2]` | 11.40 (5.45) | 7.45 | - | `ListBuilder`, a NULL fourth element | 57.70 (5.25) | 8.60 | - - The same run on DuckDB 2.0.0 (12 runs per cell) gave 7.47 → 5.75, 9.43 → 6.50, - 2.20 → 2.20, 30.90 → 22.85, 49.60 → 46.50, 65.67 → 57.85, 15.60 → 7.87 and - 66.27 → 9.30. Non-NULL data is unchanged. A difference is claimed only where - the old build's fastest run was slower than the new median; that excludes the - 46 to 50-byte string row on both engines and `trim` on 1.5.6, whose runs overlap. - The remaining gap in `trim` is the `String` that `map1_str` makes you return (one - allocation per row); no API change was made for it. `AUDIT.md` section 13 has the - spread of every cell. + | `map1` on a column with 10% NULLs | 7.80 (6.17) | 5.88 | + | `map2`, same column | 10.15 (6.15) | 6.60 | + | `map1` on a column with no NULLs (control) | 2.05 (1.70) | 2.07 | + | `map1_str` returning an `i64`, short strings | 33.55 (21.73) | 24.40 | + | the same, 46 to 50-byte strings | 44.80 (41.75) | 44.15 | + | `map1_str` returning a `String` (trim), short strings | 61.25 (39.68) | 53.83 | + | `ListBuilder`, `[x, x+1, x+2]` | 11.28 (5.03) | 6.88 | + | `ListBuilder`, a NULL fourth element | 60.30 (5.58) | 10.72 | + + The same run on DuckDB 2.0.0 (12 runs per cell, load average 1.1) gave 7.08 → 6.20, + 10.88 → 6.17, 2.20 → 2.27, 38.38 → 26.08, 61.90 → 54.90, 66.20 → 61.00, + 15.38 → 8.15 and 62.05 → 12.85. Non-NULL data is unchanged. A difference is claimed + only where the old build's fastest run was slower than the new median; that excludes + the control and the 46 to 50-byte string row on both engines, and `trim` on 2.0.0, + whose runs overlap. The remaining gap in `trim` is the `String` that `map1_str` makes + you return (one allocation per row); no API change was made for it. `AUDIT.md` + section 14.7 has the spread of every cell. ### Changed @@ -158,13 +438,245 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). crate types" (reproduced; extension-ci-tools' Rust makefile passes nothing for musl, and the platform is opt-in, so the failure only shows after opting in). +- CI: **the Rust toolchain is pinned to 1.99.1** (`rust-toolchain.toml` and + every job's `toolchain:`), so a new Rust release can no longer turn every + `-D warnings` / clippy-pedantic job red with no change in the repository; an + informational `latest-stable` job runs fmt, clippy, rustdoc and the library + tests on the current stable, and `scripts/check-toolchain-pin.py` keeps the + pins consistent. CONTRIBUTING.md describes how to move the pin. [CI-3] +- CI: the release workflow re-packages the `.crate` with the rustc the + `package` job recorded and **compares it with the attested SHA-256 before + uploading it** to crates.io (the existing comparison ran only after the + irreversible upload). [CI-6, HARDEN-11] +- CI: a release commit whose lockfiles still name the previous version now + fails in minutes: `scripts/check-release-metadata.py` (the `doc` job and the + release `validate` job) checks every committed lockfile, the CHANGELOG + heading and links, `CITATION.cff`, the `SECURITY.md` row, README's "latest" + line, `MIN_RELEASE` and every `quack-rs = "0.X"` snippet against + `Cargo.toml`, and the publish dry-run uses `--locked` like the release. + [CI-1, DOCS-1] +- CI: the five upstream-comparison jobs run through `scripts/upstream-guard.sh`: + a missing guard script fails instead of reading as "upstream unreachable", + and on a release tag an unreachable upstream fails instead of being skipped. + [CI-2] +- CI: new blocking jobs `workflow-lint` (actionlint, checksum-pinned) and + `hygiene` (typos, taplo, cargo-machete, checksum-pinned); cargo-deny now also + checks the example and CI-only crates (`deny.ci-crates.toml`), and the OSV + scan includes `benches/engine/Cargo.lock`, which is now committed. [CI-11, + DEPS-4, DEPS-6] +- CI: actions bumped (Dependabot #145, folded in by hand): `Swatinem/rust-cache` + v2.9.3, `rust-lang/crates-io-auth-action` v1.0.6, `dtolnay/rust-toolchain` + master `683b588`; `scripts/check-action-pins.py` fails when the scaffold's + generated workflows (`src/scaffold/templates_ci.rs`) pin a different commit + than this repository's workflows. [DEPS-15] + +- **`parse_description_yml` reads every section the way PyYAML 6 `safe_load` does**, which is + what the community build uses. On the 395 published descriptors (community-extensions + `845779e`) it agrees with PyYAML 6.0.3 on every one, and on 118,500 random mutants of them + (three seeds) it never accepts a file PyYAML refuses and never panics. Tabs, flow collections, + anchors and aliases, quoted and block scalars, document markers, line breaks (NEL, LS and + PS included) and nesting are handled as PyYAML handles them. The parse runs in linear time: + hostile inputs that took 3.8 to 21.5 s (and over 100 s for a comment run) now take at most + 330 ms. (CONFORM-1, HARDEN-1, HARDEN-2) +- **Scaffold: the example function is a native scalar function**, written with the typed + builder as a one-line closure (`ScalarFunctionBuilder::map1_str`), instead of a + `CREATE MACRO`. A macro is written into the user's database, so `LOAD` failed on a + read-only one. +- **Scaffold: the generated `.test` file** now: + - carries the upstream header lines (`# name:`, `# description:`, `# group:`); + - checks that the function is missing before `require`; + - reads its NULL from a column, since a literal NULL is constant-folded before the function + is called. +- **Scaffold: `make test` fails when the SQLLogicTest runner skips a test file or runs none.** + The runner reports an unmet `require` as `SKIPPED` and exits 0, so a project whose + extension name, `[lib]` name and test file had drifted apart passed having tested nothing. +- **Scaffold: the extension version written into the metadata footer** is now: + - the first tag on HEAD, if it fits the 32-byte field and holds only `A-Za-z0-9._+-`; + - otherwise the short commit hash; + - before the first commit, the `Cargo.toml` version. + + extension-ci-tools wrote every tag (one per line), or git's error message, and DuckDB 2.0 + refused the result. +- **Scaffold: generated `Cargo.toml`.** The `libduckdb-sys` requirement is + `>=1.4.4, <1.20100` with `default-features = false` (it was `<2`, which admits the untested + DuckDB 2.x bindings). With `--target-duckdb-version v1.5.6` (C API v1.5.6), or for an + unstable DuckDB 1.5 target, it enables the `duckdb-1-5*` feature its comments describe. + Before, it enabled none, and the author had to add it. (DEPS-2/3, CONFORM-14) +- **Scaffold: generated `extension-ci.yml`:** + - sets `permissions: contents: read`; + - checks out with `persist-credentials: false`; + - pins `Swatinem/rust-cache` v2.9.3 (`d1527c8f…`) and `dtolnay/rust-toolchain` master + 2026-10-15 (`683b588c…`). + `zizmor --offline` reports nothing for it. +- **Scaffold: a license may be any SPDX expression.** An identifier outside the shortlist + (`BUSL-1.1`, `CC0-1.0`) is a warning instead of an error. A misspelled case of a listed + one (`mit`) or a character SPDX does not allow is still an error. (CONFORM-14) +- **Generated files and `append_metadata` link to the book's pitfalls page** for P2, P4, P5 + and L11, instead of `LESSONS.md`, which the published crate does not contain. (DOCS-8) +- **hello-ext: `gen_series_v2`'s callbacks are written with + `table_bind_callback!` / `table_init_callback!` / `table_scan_callback!`.** A value past + BIGINT is now a query error instead of a wrapped number. New check T30. (FFI-9) + +- **Breaking:** `generate_scaffold` refuses configurations it used to accept: a name DuckDB + uses (an alias such as `s3` or a core extension such as `json`); a platform group, a + retired platform, or a name the community build does not build, in `excluded_platforms`; + a version component above `u64::MAX`; a license with characters SPDX does not allow, or a + listed id in the wrong case. `parse_description_yml` is stricter where PyYAML is (it + rejects files the community build cannot read) and rejects a reserved extension name. + (CONFORM-1, -3, HARDEN-5) +- **Behaviour change:** `WarningCollector` keeps at most 10,000 warnings between drains + (`DEFAULT_MAX_WARNINGS`; `with_limit` for another cap). Warnings past the cap are counted + in `dropped()` instead of stored, and `drain`/`clear` reset the count. Before, every + warning was stored, and a collector never drained grew without bound (2,000,000 + warnings were 381 MB) (HARDEN-8). +- **Behaviour change:** the duplicate-overload guard of `ScalarFunctionSetBuilder` and + `AggregateFunctionSetBuilder`, and the catalog collision check of scalar registrations, + compare types the way DuckDB's overload resolution does: a `DECIMAL`, `MAP` or `ENUM` + parameter by type alone (also inside `LIST`, `ARRAY`, `STRUCT`, `UNION`), and named + `STRUCT` fields / `UNION` members by name in any order and case. Sets such as + `f(DECIMAL(4,1))` + `f(DECIMAL(18,3))`, and a scalar `round(DECIMAL(18,3))` beside the + built-in `round(DECIMAL)`, which registered and then made every such call fail with + "Could not choose a best candidate function", are now refused at registration with the + reason. `LIST(INTEGER)`/`LIST(VARCHAR)`, `INTEGER[3]`/`INTEGER[4]` and other pairs DuckDB + resolves still register (AUTHT2-1). +- **Behaviour change:** `data_chunk_from_arrow` refuses, with `InvalidInput`, Arrow arrays + that break bounds DuckDB takes on trust and reads past later: a list, large-list or map + offset that is negative, decreases or lies past the child; a negative list-view offset + or size; a struct field, list or fixed-size-list child, or union member shorter than the + rows its parent reads; a dictionary index outside the dictionary (in any row the indices' + own validity does not mark NULL); dictionary indices of a non-integer type. Such arrays + violate the Arrow specification; before, they imported and reading the chunk was an + out-of-bounds read (SIGSEGV on 1.5.6 and 2.0.0 for a list offset past its child) (FFI-1). +- `BindInfo::set_cardinality` clamps the hint to `i64::MAX`: above it any join with the + function failed with "INTERNAL Error: Information loss on integer cast" (1.5.6) (HARDEN-7). +- A failed aggregate registration under a taken name now says which catalog entry holds + the name (scalar function, aggregate or macro) and what to do, instead of the generic + "the C API reports no reason" hint; the catalog is consulted only after the failure + (FFI-8). +- `ScalarBindData::set` called twice through the same `ScalarBindInfo` frees the value it + replaces; it used to be leaked (FFI-12). +- The DuckDB-before-1.5.0 "cannot add an overload to an existing name" error names + `overload_typed` beside `overload` (AUTHT2-2). + +- **Behaviour change:** the entry point runs the registration closure in one transaction on + the registration connection: `BEGIN TRANSACTION`, then `COMMIT` when the closure returns + `Ok`, `ROLLBACK` when it returns `Err` or panics. A `LOAD` that fails part-way now leaves + no catalog entry behind (scalar, aggregate and table functions, macros, types, copy + functions, tables the closure created) and can be retried in the same database; before, + everything registered up to the failure stayed and every retry failed at the first name + it registered again ("... already exists"). Validated with a real `LOAD` on DuckDB 1.4.4, + 1.5.6 and 2.0.0. **Not rolled back**, because DuckDB keeps them in the database + configuration: casts, replacement scans and configuration options — register those last. + Inside the closure: a failed statement can abort the transaction (DuckDB 2.0 on every + failed statement), after which later statements fail with "Current transaction is + aborted", so return the first error; do not run `BEGIN`/`COMMIT`/`ROLLBACK` on the + registration connection; what the closure creates is visible to other connections + (including one it opened itself) only once the `LOAD` commits. A `LOAD` issued inside the + host's own transaction does not wait on it and is not undone by the host's `ROLLBACK` + (as before). Cost: one `BEGIN`/`COMMIT` pair, about 0.3 ms; no measurable change in + `LOAD` time. `Registration::Autocommit` restores the old behaviour (FFI-2). +- `CopyFunctionBuilder::register` on the entry point's registration connection runs its + format-name probe (a statement that fails on purpose) on a short-lived connection of its + own and checks formats registered earlier in the same transaction from the entry point's + own record. On DuckDB 2.0, which aborts a transaction on any failed statement, the probe + otherwise made the registration fail with "duckdb_register_copy_function failed for + ''" and aborted the transaction. On a connection of yours inside a transaction you + opened, the probe still runs on that connection (documented) (FFI-3). +- The entry-point macros emit `#[unsafe(no_mangle)]` instead of `#[no_mangle]`. Identical + today on every edition (Rust 1.82+); a bare `#[no_mangle]` in an exported macro would + fail every extension that expands it once quack-rs moves to edition 2024, and + `cargo fix --edition` does not rewrite macro bodies (DEPS-12). +- docs.rs labels every feature-gated item "Available on crate feature `duckdb-1-5` only" + (`#![cfg_attr(docsrs, feature(doc_cfg))]`) (DOCS-5, DEPS-7). +- Docs: the C API struct over-read by `libduckdb-sys` before the ABI guard runs is stated + precisely (87 pointers past the engine's struct on 1.4.x for a 546-slot build) (FFI-4); + the panics no guard can catch (a panic during an unwind, a panicking panic hook, out of + memory) (FFI-10); what the catalog collision scans cost on a large catalog (~0.1 s per + scan with 20,000 user macros; narrowing the query does not help) (FFI-7); what a + `duckdb-1-5` build does on DuckDB v2.0.1 (refused under `AbiPolicy::Strict` until quack-rs + lists it or the build sets `QUACK_RS_TARGET_DUCKDB_VERSION`). + +- **Breaking:** `entry_point!` and `entry_point_v2!` refuse to compile an extension built + with `panic = "abort"` (`[profile.*] panic = "abort"` or `RUSTFLAGS=-C panic=abort`), + with an error that names the fix: under `panic = "abort"` none of quack-rs's + `catch_unwind` guards work and any panic aborts the DuckDB process. Build with + `panic = "unwind"` (what the scaffold and hello-ext already use). The check runs in the + extension's crate; a hand-written entry point around `init_extension_v2` is not + checked. `wasm32-unknown-emscripten` defaults to unwinding and is unaffected (FFI-5). +- **Breaking:** `abi::AbiCheck` is `#[non_exhaustive]`: a `match` on it outside quack-rs + needs a wildcard arm (use `is_compatible` / `error_message` to cover every outcome) + (DEPS-13). +- **Breaking:** `abi::AbiPolicy` is `#[non_exhaustive]`: a `match` on it outside quack-rs + needs a wildcard arm. Passing a policy is unchanged (DEPS-13). +- **Breaking:** `config_option::ConfigOptionScope` is `#[non_exhaustive]` (DEPS-13). +- **Breaking:** `tls::TlsVersion` is `#[non_exhaustive]`; comparisons and ordering are + unchanged (DEPS-13). +- **Breaking (found by `cargo semver-checks` against v0.19.0):** `ScalarBindInfo` + (`duckdb-1-5`) is no longer `RefUnwindSafe`, because it now remembers the bind data it + stored so a second `set` can free the first (FFI-12). Code that holds a `&ScalarBindInfo` + across its own `catch_unwind` needs `AssertUnwindSafe`; quack-rs's guards already do. The + same run reports no other breaking change, and lists the four `#[non_exhaustive]` enums + above. +- **Breaking (behaviour):** see the transactional-registration entry under Changed — an + extension whose closure relied on carrying on after a failed registration, or that runs + `BEGIN`/`COMMIT` on the registration connection itself, needs + `options = EntryPointOptions::new().registration(Registration::Autocommit)` (FFI-2). + +- A `BIT` value whose bytes are malformed (only the raw `duckdb_create_bit` + builds one) is now refused by `Value::as_str`, `display_string` and `Debug` + (`UNRENDERABLE`) instead of being rendered through an out-of-bounds write + inside `DuckDB`; the `UNRENDERABLE` text names it. (APIGAPS-3) +- `QueryProgress` / `InterruptHandle::progress` document why `DuckDB` 1.x reports + `-1.0` for whole queries (any pipeline without progress keeps it there). + (AUTHT-MEAS) +- `scripts/api-coverage-allowlist.txt`: 11 candidate slots are now wrapped; + `duckdb_param_type` / `duckdb_prepared_statement_column_type` are listed as + superseded; the log-storage rows carry the upstream double-free caveat and the + 1.5.2 slot move (APIGAPS-1, APIGAPS-11). +- **Dependency ceiling.** The `libduckdb-sys` and `duckdb` requirements are + `>=1.4.4, <1.20100` (they were `<2`) in the crate and in every manifest in the repository. + The 1.20000.x bindings are the DuckDB 2.0 bindings; the next ones (1.20100.x, DuckDB 2.1) + are not selected until a quack-rs release widens the range, because `<2` also admitted + every release up to 1.99999, whose C API layout nobody had checked. No action for + dependants already on 1.10xxx or 1.20000.x; one who needs a later release must wait for + quack-rs or pin. CONTRIBUTING.md has the procedure for widening the range, and a test fails + if a manifest states a different one. (DEPS-2) +- `cargo package` selects files with an allow-list (`include`) instead of an `exclude` + list, so a new top-level directory is not published by default. (DEPS-7) +- The build script of `bundled-test` / `bundled-test-prebuilt` fails in seconds, naming + the directory, when `libduckdb-sys` reports an include directory that has no + `duckdb.hpp` (a partial build cache); it used to poll for ten minutes before panicking. +- CI: runners are pinned to `ubuntu-24.04`, `macos-26` and `windows-2025` (the images + `-latest` named on 2026-10-30), and `scripts/check-action-pins.py` fails on a floating + label in any workflow. [CI-12] +- CI: the upstream-comparison scripts return 2 ("could not check", a warning) only for a + network failure, an HTTP 5xx or a 429; an HTTP 4xx, an entry without the expected key, + a header the parser cannot read, or every release answering 404 returns 1. Before, a moved + or reshaped upstream file turned each of these jobs into a skipped check for good + (reproduced for the platform list with a nonexistent ref). `scripts/upstream-guard.sh` also + fails a 2 on the weekly schedule, so a gate that has not run for a week is noticed. [TESTS-10] +- CI: the Linux leg of the `test` job installs `python3-yaml`, `make` and `git` and sets + `QUACK_REQUIRE_TOOLS=1`, which turns the scaffold tests' "tool missing, skipping" into a + failure (`cargo test` hides the note, so a runner without PyYAML kept them green while they + checked nothing). macOS and Windows keep the skip. [TESTS-14] +- `.cargo/mutants.toml` no longer excludes `src/callback.rs`: the macros in it are invisible + to `cargo-mutants`, but its four plain functions are mutated (5 mutants, 4 caught, 1 + unviable). CONTRIBUTING.md now says what the mutation gate cannot see: `unsafe fn` bodies + (506 functions, 22.4% of non-test lines by a heuristic count), integer literals and macro + bodies. [TESTS-2, TESTS-11] +- CI: the incremental mutation job runs as eight shards (`cargo mutants --shard k/8 + --sharding round-robin`) and a gate job named "Mutation Testing (incremental)". One job did + the 1,639 mutants of this release's 102 changed files and was stopped by its runner after 33 + and 55 minutes; the eight shards took 22 minutes together. The Miri job's limit is 90 + minutes (the library tests had grown to 41.6 minutes for the native run alone, 18 of them + one parser test on 100,000-level inputs, which now uses 1,000 under Miri). [MUT-2] + ### Documentation - `LESSONS.md` and the book's pitfalls page gain P13 (a `VARIANT` in a signature crashes DuckDB), P14 (an aggregate with no parameter counts no rows on DuckDB 2.0) and P15 (the entry of a NULL `LIST` or `MAP` row holds stale bytes): the three - engine behaviours this release guards against. The pitfall count is now 34 - (L1–L19, P1–P15) wherever it is stated. + engine behaviours this release guards against (four more follow below). - README: the Quick Start manifest now sets `rust-version` and `resolver = "3"` (edition 2021's default resolver ignores `rust-version`, so Rust 1.86 or 1.87 selected the 2.0 bindings, which need 1.88, and failed); the example uses @@ -216,6 +728,166 @@ quack-rs adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). stamp and the enabled features; the pull request template lists the doctest and changelog-sync steps. +- `SECURITY.md`: supported versions for 0.20.0 (0.20.x and 0.19.x; 0.18.x + end-of-life), the two-newest-lines policy, and what releases before 0.16.0 + carry, with version ranges checked against the published crates; no more + rows for 0.1.x / 0.2.x, which were never published. [DOCS-13, HARDEN-10] +- `RELEASING.md`: the release commit updates every lockfile (`cargo update + --workspace` in five directories) because the release packages with + `--locked`; the hotfix branch is merged into `main` with a merge commit and + its tip tagged (the old "tag, then cherry-pick" flow was refused by + `validate`); the crates.io fallback token belongs in the `crates-io` + environment, with a Trusted Publishing set-up order; several statements + corrected to what the workflow does. [DOCS-1, CI-1, CI-5, CI-15, HARDEN-9] +- `CONTRIBUTING.md`: what the sanitizer and valgrind runs cannot see (the + release libduckdb allocates vector data buffers and LIST entry arrays with + its own jemalloc, invisible to valgrind and AddressSanitizer), the toolchain + pin policy, and DuckDB 2.0's CLI "agent mode" (`AI_AGENT` / `CLAUDECODE`; + use `-csv`/`-json`, or `-no-agent` on 2.0 only). [LASTDIFF-3, CI-3] + +- **"Your First Extension" teaches guarded callbacks.** The page walked through + `hello-ext`'s unguarded `unsafe extern "C" fn`s (a panic in one aborts the DuckDB + process) and its `word_count` aggregate, which segfaults under `OVER ()` on DuckDB + 1.4.4 and 1.5.x, with no warning. It now builds `word_count` with the + `aggregate_*_callback!` macros and `first_word` with `ScalarFunctionBuilder::map1_str` + (and shows the `scalar_callback!` form), opens with a warning that lists the crashing + query shapes, and prints output that was produced by building the page's own code and + loading it into DuckDB 1.4.4, 1.5.6 and 2.0.0 (byte-identical). The raw scalar examples in + `scalar.md` use `scalar_callback!` and report overflow with `checked_mul` / `checked_add` + instead of wrapping (DOCS-4). +- **The window shapes that crash a C API aggregate are listed as measured.** On DuckDB 1.4.4 + and 1.5.6 an explicit `ROWS`/`RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` + frame (with or without `PARTITION BY`) crashes like `OVER ()` and `agg(x ORDER BY y)` do; + `OVER (PARTITION BY p)` without an `ORDER BY` crashes on 1.4.4 but not on 1.5.6; every + shape works on 2.0.0 (one process per shape, 5,000 and 300,000 rows). Pitfall L11, + `known-limitations.md`, the aggregate pages and the tutorial say so (AUTHOR-TYPES-4). +- **`std::fs` bypasses `SET enable_external_access = false` and `allowed_directories`** + (AUTHT-1, measured on DuckDB 1.5.6 and 2.0.0: `read_csv` and quack-rs's `FileSystem` get a + Permission Error, a table function using `std::fs::read` returns the file). Stated in the + Virtual File System chapter (new section), the table-function, replacement-scan and + `COPY` chapters, Known Limitations, the FAQ, the publishing checklist and a new + pitfall, P17. +- **Four new pitfalls, 38 in all** (L1-L21, P1-P17): L20 `duckdb_register_log_storage` + frees its extra data twice when the name is taken (DuckDB 1.5.0 to `main`; quack-rs does not + wrap log storage because of it), L21 a `ChunkWriter` row that is claimed but not written is + published with stale contents, P16 `duckdb_get_bit` / `duckdb_get_bignum` abort the process + on NULL and on values that do not cast, P17 `std::fs` (above). `LESSONS.md` L3 no longer + says a raw callback is "made impossible" to panic in (it is not guarded); the six + `LESSONS.md` titles that differed from the book's (P2, P3, P5, P7, P9, P12) match it, and + every place that states the pitfall count says 38 (DOCS-2). +- **`known-limitations.md` is grouped** (what your extension must get right / DuckDB defects / + what the C API does not offer / DuckDB 2.0 changes / resolved) instead of interleaving + resolved history with current limits (DOCS-12), with new entries: no table-in-out functions, + progress callbacks, filter pushdown, logging or secret access in the v1 C API; a quack-rs + extension cannot be linked statically into a DuckDB 2.0 host; five DuckDB 2.0 behaviour + changes that reach C API extensions; panics that cannot be caught (double panic, a panicking + hook, an allocation failure). The C API v2 list no longer cites a function that does not + exist (`duckdb_v2_table_function_set_filter_pushdown` is `..._callback`) and covers the v2 + capabilities an author of the wrapped function kinds would look for, each name grepped in + the 2.0.0 headers (APIGAPS-6, -7, -10). Table function arguments may be constant + expressions, not only literals (AUTHT-10). +- **Corrections found by running the book against DuckDB 1.4.4, 1.5.6 and 2.0.0.** The scalar + bind callback cannot set the return type (the v1 C API has no such function); a `DECIMAL` + parameter rounds its argument and DECIMAL/MAP/ENUM overloads cannot be told apart; a type + registered with `set_alias` is not a `CREATE TYPE` synonym (`sum`, `abs` and `+` do not bind + on it); no callback API accepts an `ErrorData`; a table function's `bind` runs again on every + `EXECUTE` on DuckDB 2.0.0 (4 binds against 1 for a prepared statement executed three times); + the typed table-function builder scans on one thread; without `duckdb-1-5` a zero-column typed + bind is an `INTERNAL Error`; the table-function example no longer hard-codes 2048; a + `COPY ... TO` format cannot declare options (unknown names are accepted silently), sees no + column names, and is chosen from the file extension only when that equals its name; Arrow + import refuses a record batch with a root `offset`; `SET threads` from a callback can abort + the process; a prepared-statement parameter is not foldable at bind time; a SQLLogicTest file + whose `require` fails is reported `SKIPPED` and exits 0. On DuckDB 2.0.0 an + `INTERVAL '1 month' - INTERVAL '1 day'` compares equal to `INTERVAL '29 days'`, a bare + identifier as a table macro argument warns (write `'users'`), `VerifyNullHandling` is compiled + into release builds and runs after `SET debug_verification_mode = 'VERIFY_FUNCTIONS'`, and + `arg_max`'s third parameter is `BIGINT` (DOCS-3, AUTHOR-TYPES-3, -5, -6, -7, -13, AUTHT-4 to -12). +- **Dependency requirement.** Every snippet and sentence shows `libduckdb-sys` as + `>=1.4.4, <1.20100` with `default-features = false`, `quack-rs = "0.20"`, and Installation + explains why the range has an upper bound (the 1.20000 bindings keep the C API names at their + crate root through a re-export that upstream marks for removal), what to do when the DuckDB 2.1 + bindings are published (nothing: Cargo stays on 1.20000.x until quack-rs widens the range), + what `default-features = false` saves (65 crates against 30 with the 1.20000 bindings, + measured with `cargo tree`; 74 against 72 with 1.10506), and the build requirements of each + part (a C compiler may be needed by the bindings' build dependencies, a C++ toolchain for + `bundled-test`, Python 3, git and network access for the community flow). The README no + longer says "no C, no C++, no glue code" without saying what that covers (DEPS-2, DEPS-11). +- **README front page.** Code lines are at most 64 columns (crates.io clips at about that + width), a link row, a maintenance bullet, a "Getting help" section, a pointer to + `examples/hello-ext` with the warning that it is not a template, a "use 0.18.0 or later" + notice, a shorter comparison table that is no longer cut off, and DuckDB support stated as + 1.4.4 through 2.0.x everywhere (DOCS-9, DOCS-22). +- **A `duckdb-1-5*` extension loads only into DuckDB releases whose layout quack-rs has verified** + (README "Supported versions", the ABI chapter, FAQ, Installation, Anatomy, a new Known Limitations + entry). "DuckDB 1.4.4 through 2.0.x" now describes the default features only; an extension that + enables `duckdb-1-5*` loads into 1.5.6 and 2.0.0, and under the default `AbiPolicy::Strict` refuses any + release quack-rs has not listed (2.0.1, for example) with an error that names the fixes: build with + `QUACK_RS_TARGET_DUCKDB_VERSION` set to the release (after checking its header), upgrade quack-rs, or stay + on the default features. The evidence is stated: DuckDB 2.0.0's `duckdb_extension.h` opens with "this API is + not yet stable ... only guaranteed to work for this specific DuckDB version", and 2.0.0 put + `duckdb_create_timestamp_tz_ns` and `duckdb_get_timestamp_tz_ns` into the range that 1.5.6 declared stable + (checked against the 2.0.0 header and the 1.5.6 header). The ABI chapter also says that `libduckdb-sys` + reads every slot of its 546- or 548-slot struct before the guard runs, which is 87 pointers past DuckDB + 1.4.x's smaller struct on every `LOAD` (read from the sources, not measured) (FFI-4). +- **Registration is transactional, `panic = "abort"` does not compile, four enums are `#[non_exhaustive]`** + (The Entry Point chapter, Anatomy, Installation, Panic-Safe Callbacks, Pitfall L3, Known Limitations, the + catalog chapter, and the upgrade guide). The sentence "Registration is not transactional: functions + registered before a failure stay registered" is replaced by what the entry point now does (FFI-2, FFI-5, + FFI-10, DEPS-13): one transaction, what is not rolled back (casts, replacement scans, configuration + options: register them last), what a failed statement inside the closure does, and how to get the old + behaviour (`Registration::Autocommit`). +- **Pitfall P16 is now "prevented in quack-rs"** (`Value::as_bit` and `as_bignum` return `None` instead of + reaching the aborting C getters), and the book covers the new `Value` readers (`try_cast`, `array_items`, + `union_tag`), typed `COPY TO` state and typed extra info, the `Send` `FileHandle`, the catalog-scan cost of a + large database (FFI-7), and the `-1.0` progress of DuckDB 1.x (APIGAPS-2, -3, -8, AUTHT-2, -3, -MEAS). +- **`duckdb_query` runs the statements before a parse error on DuckDB 2.0.0** (Known Limitations, Running SQL, + SQL Macros, the upgrade guide). Measured with a C program on 1.4.4, 1.5.6 and 2.0.0: for the text + `CREATE OR REPLACE MACRO "m"() AS (1); DROP TABLE canary; SELECT ()`, 1.4.4 and 1.5.6 report a parser error and + run nothing; 2.0.0 ran the `DROP TABLE` and returned success. An extension that passes text it did not write to + `duckdb_query` must check that it parses as the expected statement count first (`SqlMacro::register` now + requires exactly one). +- **The Project Scaffold chapter and the publishing page follow the new scaffold**: the `quack_scaffold` command (flags, + `--force`, the printed next steps), `generate_scaffold_with_warnings`, the new generated files (`README.md`, `LICENSE` + files, `deny.toml`, `docs/function_descriptions.csv`), the native example function and `.test` layout, the + `make test` skip check, the extension-version choice, the hardened `extension-ci.yml`, what `generate_scaffold` now + refuses, and the `Cargo.toml` it writes (`<1.20100`, `default-features = false`, the `duckdb-1-5*` feature for a + `v1.5.6` project). +- **Community-extension publishing** (`publishing.md`): the names DuckDB reserves (ten aliases and + the 34 core names; `INSTALL s3 FROM community` fails on 2.0.0; the scaffold and `parse_description_yml` now + refuse them), what the description.yml reader checks (PyYAML's rules, `variegata` and `andium`), + `function_descriptions.csv` + (lives in the community-extensions repository, replaces the generated text), the four revision + keys `ref`, `ref_next`, `variegata` and `andium` with `build.py`'s rule and what a project that + uses the unstable C API must do, which version DuckDB shows (the tag on `HEAD`, else the short + hash; `description.yml`'s `version` is not read; tags over 32 bytes or two tags on one commit + break the footer), the keys the build reads (anything else is ignored), reduced CI on pull + requests, and not running `strip` on a stamped extension. The page's `description.yml` and + `Cargo.toml` examples match the scaffold's output, and the statements that had drifted from + upstream (the `v1.5.6` branch is not the `v1.5-variegata` commit, the matrix script differs + per `extension-ci-tools` branch, the version is not from `git describe`) are corrected + (CONFORM-3, -7, -8, -9, -11, -14). +- **New page "Upgrading from 0.18 to 0.20"** (Installation > Upgrading): every behaviour change + of 0.19.0 and 0.20.0 that can fail a `LOAD` or change a result, with what to do. +- `scaffold.md` says the scaffold example is in the repository, not the published crate, lists + the prerequisites of `make configure`, and warns that `make` ignores `CARGO_TARGET_DIR`; the + Testing Guide explains the DuckDB 2.0 CLI's agent mode (`-no-agent`) and reads the NULL test + case from a column. `GOVERNANCE.md` says there is one maintainer. The publishing page no longer + prints `..._distribution.yml@v2.0-cyanoptera`, which Cloudflare's email obfuscation rewrote to + `[email protected]` on the live site (DOCS-6). `docs/E2E_DOGFOOD_REPORT.md` and + `docs/duckdb-v1.5.1-evaluation.md` moved to `docs/archive/`. + +### Dependencies + +- `Cargo.lock`: `proptest` 1.12.0 (a dev-dependency) and `cc` 1.10.0, the updates Dependabot + proposed in #148 (`cc` 1.9.0 there); `fuzz/Cargo.lock` and `book/doctest/Cargo.lock`: `cc` + 1.10.0 (#149). `examples/hello-ext/Cargo.lock`: `libduckdb-sys` 1.20000.1, so the example + builds against the DuckDB 2.0 bindings (#143 asked for 1.20000.0, whose bundled engine + reports itself as `v2.0.0-dev88614`, which the ABI check refuses). The main lockfile stays + on the 1.5.6 bindings (1.10506.1). `benches/engine/Cargo.lock` is committed, so the + benchmark crate's dependencies are scanned and reproducible. +- GitHub Actions bumps from Dependabot #145 are folded in (listed under Changed). + ## [0.19.0] — 2026-10-22 This section comes out of a sixth review (`AUDIT.md` section 11): seven @@ -3138,7 +3810,7 @@ is not grouped. `-D warnings` makes it an error. `testing::InMemoryDb` was a 59th, only visible once the lint ran with `bundled-test` on. -[C-DEBUG]: https://rust-lang.github.io/api-guidelines/debugging.html +[C-DEBUG]: https://rust-lang.github.io/api-guidelines/debuggability.html#c-debug ### Changed @@ -4218,6 +4890,9 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. ## [0.2.0] — 2026-03-07 +0.1.0 and 0.2.0 were never tagged or published: the oldest git tag and the oldest +crates.io release are 0.3.0, so these two headings have no compare link. + ### Added - **`validate::description_yml` module** — parse and validate a complete `description.yml` @@ -4317,7 +4992,8 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. - CI pipeline: check, test, clippy, fmt, doc, MSRV, bench-compile - `SECURITY.md` vulnerability disclosure policy -[Unreleased]: https://github.com/tomtom215/quack-rs/compare/v0.19.0...HEAD +[Unreleased]: https://github.com/tomtom215/quack-rs/compare/v0.20.0...HEAD +[0.20.0]: https://github.com/tomtom215/quack-rs/compare/v0.19.0...v0.20.0 [0.19.0]: https://github.com/tomtom215/quack-rs/compare/v0.18.0...v0.19.0 [0.18.0]: https://github.com/tomtom215/quack-rs/compare/v0.16.0...v0.18.0 [0.16.0]: https://github.com/tomtom215/quack-rs/compare/v0.15.0...v0.16.0 @@ -4336,6 +5012,4 @@ the workspace `Cargo.lock` and `examples/hello-ext/Cargo.lock`. [0.5.1]: https://github.com/tomtom215/quack-rs/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/tomtom215/quack-rs/compare/v0.4.0...v0.5.0 [0.4.0]: https://github.com/tomtom215/quack-rs/compare/v0.3.0...v0.4.0 -[0.3.0]: https://github.com/tomtom215/quack-rs/compare/v0.2.0...v0.3.0 -[0.2.0]: https://github.com/tomtom215/quack-rs/compare/v0.1.0...v0.2.0 -[0.1.0]: https://github.com/tomtom215/quack-rs/releases/tag/v0.1.0 +[0.3.0]: https://github.com/tomtom215/quack-rs/releases/tag/v0.3.0 diff --git a/book/src/reference/known-limitations.md b/book/src/reference/known-limitations.md index 0a3edd1a..cf32a843 100644 --- a/book/src/reference/known-limitations.md +++ b/book/src/reference/known-limitations.md @@ -1,63 +1,176 @@ # Known Limitations What quack-rs cannot do because the DuckDB C extension API does not allow it, -DuckDB behaviour an extension should plan around, and former limitations that -have since been resolved. +DuckDB defects and behaviour an extension should plan around, things your own code must get +right because nothing can check them for you, and, at the end, former limitations that +have been resolved. Each entry says which DuckDB releases it applies to and whether it was +measured or only read. + +## What your extension must get right + +### An extension that reads files with `std::fs` bypasses DuckDB's permission settings + + +**This is a security-relevant trap.** `SET enable_external_access = false` and +`SET allowed_directories = [...]` restrict what SQL can read and write, and the +restriction applies to everything DuckDB opens itself. It does not apply to code in your +extension that opens a file with `std::fs`: measured on DuckDB 1.5.6 and 2.0.0, with +external access off, `read_csv('data.csv')` fails with "Permission Error: Cannot access +file … file system operations are disabled by configuration", a table function that opens +the same path with quack-rs's `FileSystem` fails the same way, and a table function that +calls `std::fs::read` returns the file's contents. A function that takes a path from SQL +and reads it with `std::fs` therefore lets any user of that DuckDB read any file the +process can, which is what the setting exists to prevent. The same goes for any +network client you add: no DuckDB setting reaches it. + +Open user-supplied paths with the `FileSystem` wrapper (feature `duckdb-1-5`), which +applies the settings and also resolves `s3://` and `http://` paths when `httpfs` is +loaded. See [Virtual File +System](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings) +and [Pitfall P17](pitfalls.md#p17). + +### A `duckdb-1-5*` extension refuses a DuckDB release quack-rs has not listed + +With the default features, one binary loads into every DuckDB from 1.4.4 through 2.0.x. An +extension that enables a `duckdb-1-5*` feature calls the part of the C API that DuckDB +declares unstable, and quack-rs has verified that layout for DuckDB 1.5.2 to 1.5.6 and 2.0.0 +only: built against the 1.5.6 bindings and stamped `C_STRUCT` with C API `v1.5.6`, it +loads into 1.5.6 and 2.0.0. Under the default `AbiPolicy::Strict` it refuses to load into any +other release with an error that names the fixes, and DuckDB 2.0.1 will be one until a quack-rs +release lists it. The layout cannot be extrapolated: DuckDB 2.0.0's `duckdb_extension.h` opens +with "this API is not yet stable, meaning it is only guaranteed to work for this specific +DuckDB version", 2.0.0 itself put two slots (`duckdb_create_timestamp_tz_ns` and +`duckdb_get_timestamp_tz_ns`) into the range that 1.5.6 declared stable, and the 1.5.2 patch +release inserted a slot in the middle of the struct. What to do when a new release appears: +check the release's `duckdb_extension.h` against the one you built with, then rebuild with +`QUACK_RS_TARGET_DUCKDB_VERSION` set to it, or move to a quack-rs release that lists it; or stay +on the default features, which only use the stable prefix and are not affected. See [ABI +Compatibility](../concepts/abi.md#unknown-duckdb-versions). + +### A function that reads a setting must be marked volatile -## Window functions are not available +DuckDB folds a non-volatile function call whose arguments are constants into the +plan, once. A function whose `init` reads a setting +(`ClientContext::config_option`) and is not `.volatile()` then keeps answering +with the value it read first: after `SET v_setting = 9; PREPARE p AS SELECT f(); +SET v_setting = 10; EXECUTE p;` the plain function returned 9 and the same function +marked `.volatile()` returned 10. The C API v1 has only `volatile`, not "stable +within a query". -DuckDB **window functions** (`OVER (...)` clauses) are implemented entirely in -DuckDB's C++ layer and have **no counterpart in the public C extension API**. +### NULL rows of `LIST` and `MAP` vectors hold stale entries -This is not a gap in `quack-rs` or in `libduckdb-sys`: the C API has no function for -registering a window function, and no symbol such as `duckdb_create_window_function` -exists in `duckdb.h` or `duckdb_extension.h`. DuckDB's C++ API does support window -functions, through its own C++ classes rather than a function of that name. +DuckDB writes no list entry for a NULL row, so the `offset` and `length` that +`ListVector::get_entry` and `MapVector::get_entry` return for one are whatever the +reused buffer held. On 2.0.0 a `nullif(l, [])` column had NULL rows with lengths +around 7·10¹³, and a loop that walked them read far outside the child vector: a +segmentation fault in a minority of runs, and no failure at all in most, which is +how the pattern went unnoticed. `get_entry_checked` returns `None` for a NULL row +and for an entry that does not lie inside the child; use it, or test the row's +validity before `get_entry`. -| Kind of function | C API symbol | C API (1.4.4, 1.4.5) | C API (1.5.0+) | C++ API | -|------------------|--------------|----------------------|----------------|---------| -| Window function | *(none)* | **No** | **No** | Yes | -| `COPY` function | `duckdb_create_copy_function` | **No** | **Yes** | Yes | -| Scalar function | `duckdb_create_scalar_function` | Yes | Yes | Yes | -| Aggregate function | `duckdb_create_aggregate_function` | Yes | Yes | Yes | -| Table function | `duckdb_create_table_function` | Yes | Yes | Yes | -| Cast function | `duckdb_create_cast_function` | Yes | Yes | Yes | +### A `VARIANT` cannot be declared in a function signature -**What this means for your extension:** -A custom window operator requires a C++ extension. An ordinary aggregate can be -used *as* a window (`agg(x) OVER (...)`), but see the next section before -recommending that to your users: two of those shapes crash every aggregate -registered through the C API. +The type `duckdb_create_logical_type(VARIANT)` returns has no `STRUCT` layout, because +DuckDB stores a VARIANT as a struct. A scalar function declared with it +registers, and the first query that reaches it takes DuckDB down: on 2.0.0 +`SELECT v_out(1)` is a segmentation fault (`VectorStructBuffer` while +initialising the result vector; reproduced with the guard bypassed), and on 1.5.x +`SELECT v_in({'a':1}::VARIANT)` is an internal error that invalidates the +database. quack-rs therefore refuses `TypeId::Variant` wherever it builds a +logical type (`LogicalType::try_new`, and so every builder's parameter, +return-type and result-column slots), with an error that names `ANY`. A function +that wants to accept a VARIANT declares the parameter as `TypeId::Any`. +`GEOMETRY` is unaffected. See [Pitfall P13](pitfalls.md#p13). -If DuckDB exposes window registration in a future C API version, `quack-rs` -will add wrappers in the corresponding release. +### A caught panic is still printed to stderr + +The macros and typed builders catch a panic at the FFI boundary and report it as a +SQL error, but Rust's default panic hook has already run by then: it writes the +panic message to the host process's stderr (and a backtrace when `RUST_BACKTRACE` +is set), and does so even though the query only fails with an error. quack-rs +installs no hook, because the hook is process-wide and the host (Python, a CLI, a +server) owns it. An extension that wants these lines silent or routed to its own +log can call `std::panic::set_hook` in its entry point, knowing that it replaces +the host's hook for every thread in the process. -## Aggregates crash under `OVER ()` and `ORDER BY` (DuckDB defect, Pitfall L11) +**Some panics cannot be caught at all.** A destructor that panics while another panic is +unwinding, a panic hook that itself panics, and an allocation failure while the panic +message is formatted (Rust's default handler aborts) all end the host process, whatever +the guards do. The first was reproduced: `panic in a destructor during cleanup … aborting`, +exit code 134. Keep `Drop` implementations of your own types from panicking. And under +`panic = "abort"` no guard works at all, which is why `entry_point!` and `entry_point_v2!` +stop the build there. + +### Loading an extension scans the function catalog once + +Every scalar and table registration through quack-rs first lists `duckdb_functions()` +to refuse a signature that would silently replace an existing function (a built-in +included) or make calls ambiguous. Registering through the entry point's +`Connection` does that scan once per `LOAD`, however many functions follow. The cost +is fixed, not per function: for an extension with one `map1` function, `LOAD` took +18 ms longer than the same entry point registering the function through the raw C +API on DuckDB 1.5.6, and 30 ms longer on 2.0.0 (medians of 30 interleaved CLI +runs each, one 4-vCPU virtual machine, release build; the minima differ by 17 and +27 ms). The C API has no catalog lookup for functions, so the scan cannot be +narrowed to a name, and filtering the query to the names being registered does not make +it cheaper (`duckdb_functions()` has no pushdown). The scan grows with the catalog: one +scan took about 20 ms on 1.5.6 and 13 ms on 2.0.0 in an empty database and about 110 ms +on both with 20,000 user macros (CLI measurements on a shared 4-vCPU machine), and a +`LOAD` that registers scalar and table functions scans twice. Registering through a +builder's `register(con)` on the raw connection scans once per call instead of once per +`LOAD`. + +### SQL macros: no defaults or overloads, and they persist + +`SqlMacro` takes plain parameter names; `y := 5` is rejected, and registering the +same name again replaces the first (`CREATE OR REPLACE`). For a default +parameter, run `CREATE OR REPLACE MACRO m(x, y := 5) AS x + y` through +`Connection::execute`. A macro is stored in the user's database, so one whose +body calls an extension function breaks when the database is opened without the +extension loaded ("Table Function with name … does not exist"); a custom type +registered through the C API is not stored. + +## DuckDB defects an extension cannot prevent + +### Aggregates crash under `OVER ()` and `ORDER BY` (DuckDB defect, Pitfall L11) On DuckDB 1.4.x and 1.5.x, every aggregate registered through the C API — -quack-rs's or anyone else's — reads out of bounds when DuckDB runs it as a whole-partition window -(`agg(x) OVER ()`, `agg(x) OVER (PARTITION BY p)`) or as an ordered aggregate -(`agg(x ORDER BY y)`). The usual result is a segmentation fault that takes the -host process down. The book's own example aggregate does it: +quack-rs's or anyone else's — reads out of bounds when DuckDB runs it as a +whole-partition window or as an ordered aggregate, and the usual result is a +segmentation fault that takes the host process down. The book's own example +aggregate does it. Measured with `word_count` from [Your First +Extension](../getting-started/first-extension.md), one query shape per process, over 5,000 +and over 300,000 rows: + +| Query shape | 1.4.4 | 1.5.6 | 2.0.0 | +|-------------|-------|-------|-------| +| `agg(x) OVER ()` | crash | crash | works | +| `agg(x) OVER (ORDER BY i ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` | crash | crash | works | +| the same with `RANGE`, or with a `PARTITION BY` | crash | crash | works | +| `agg(x ORDER BY y)` | crash | crash | works | +| `agg(x) OVER (PARTITION BY p)`, no `ORDER BY` | **crash** | works | works | +| `OVER (ORDER BY i)` (the default frame), `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`, `CURRENT ROW AND UNBOUNDED FOLLOWING`, sliding frames, `EXCLUDE`, `DISTINCT`, `FILTER` | works | works | works | ```sql --- with hello-ext loaded: segfaults on DuckDB 1.4.4 and 1.5.5 +-- with the book's word_count loaded: segfaults on DuckDB 1.4.4 and 1.5.6 SELECT max(w) FROM (SELECT word_count(s) OVER () AS w FROM (SELECT 'a b c' AS s FROM range(5000))); ``` -The cause is in DuckDB (`CAPIAggregateUpdate` hands the callback a constant -state vector), there is no way for an extension to detect or prevent it, and it -is reported upstream as +An explicit `UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame is a whole-partition +frame, and crashes like `OVER ()`. `OVER (PARTITION BY p)` without an `ORDER BY` did not +crash on 1.5.6 in these runs; the code path that reads the state array is DuckDB's, and +whether it reads out of bounds there without a visible effect was not established, so do +not rely on it. The cause is in DuckDB (`CAPIAggregateUpdate` hands the callback a constant +state vector), there is no way for an extension to detect or prevent it, and it is +reported upstream as [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109). DuckDB 2.0.0 fixes it (`CAPIAggregateUpdate` flattens the state vector first); on -older releases, tell your users not to use your aggregates in those two shapes. -Frames that are not whole-partition (`ROWS BETWEEN 5 PRECEDING AND CURRENT ROW`) -and `DISTINCT` windows work. See +older releases, tell your users not to use your aggregates in those shapes. See [Pitfall L11](pitfalls.md#l11-c-api-aggregates-crash-under-aggx-over--and-aggx-order-by-y). -## Aggregate states leak when `finalize` reports an error (DuckDB behaviour) +### Aggregate states leak when `finalize` reports an error (DuckDB behaviour) When an aggregate's `finalize` callback reports an error (`AggregateFunctionInfo::set_error`), DuckDB does not call the destructor @@ -82,7 +195,7 @@ one thread, and not on 2.0.0. The behaviour is pinned by `aggregate_states_after_update_and_combine_errors` in `tests/ffi_roundtrip/lifecycle.rs`. -## Grouped-aggregate states the scan never reaches are never destroyed (DuckDB defect) +### Grouped-aggregate states the scan never reaches are never destroyed (DuckDB defect) DuckDB 1.4.4 to 2.0.0 destroys a grouped aggregate's states as its result scan passes them. When the scan stops early, the states it has not reached are @@ -103,7 +216,7 @@ keep aggregate states small and free of heap allocations where you can. Pinned by `states_a_grouped_scan_never_reaches_leak_no_rust_heap` in `tests/aggregate_leaks.rs`. -## Window frames with `EXCLUDE` never destroy one state per row (DuckDB defect) +### Window frames with `EXCLUDE` never destroy one state per row (DuckDB defect) A C API aggregate in a window whose frame has `EXCLUDE CURRENT ROW`, `GROUP` or `TIES` is evaluated by DuckDB's segment tree in two parts, and before @@ -115,7 +228,21 @@ what `T` owns on the heap. See `docs/upstream-duckdb-reports.md`, item 35; pinned by `a_window_frame_with_exclude_leaves_states_undestroyed` in `tests/ffi_roundtrip/agg_states.rs`. -## An abandoned stream keeps its table-function state (DuckDB 1.x behaviour) +### A failing state initialiser in a window frame crashes the host (DuckDB defect) + +When an aggregate's `state_init` reports an error while the aggregate runs as a +window function with a `ROWS` or `RANGE` frame, DuckDB's +`WindowAggregateStates` destroys its whole table of state pointers, including +entries it never wrote. The destructor is handed garbage pointers. A plain C +aggregate shows it, with no Rust involved: on DuckDB 1.5.6 55 of 68 pointers +passed to the destructor were garbage, and on 2.0.0 65 of 68. quack-rs's +`FfiState` reports a panic in `T::default()` as a SQL error, which is exactly this +trigger; its tag check cannot help, since there is no slot to read. Ungrouped, +grouped, `DISTINCT` and `FILTER` aggregates report a clean error; a window shape +kills the process (`SIGSEGV` or `SIGBUS`). **`T::default()` must not panic** for an +aggregate that can run as a window function. + +### An abandoned stream keeps its table-function state (DuckDB 1.x behaviour) On DuckDB 1.x, dropping a streaming `QueryResult` part-way through, and then its `PreparedStatement`, does not free the query's operator states: DuckDB @@ -136,7 +263,7 @@ pinned by `an_abandoned_stream_keeps_its_table_state_until_the_next_statement` and `another_statement_on_the_connection_ends_a_stream_with_an_error` in `tests/ffi_roundtrip/query_stream.rs`. -## Running out of memory inside a callback aborts the process +### Running out of memory inside a callback aborts the process An allocation failure is not an error quack-rs can report. On the Rust side, the default allocation-error handler aborts. On the DuckDB side, @@ -156,7 +283,238 @@ everything built on it) then reads and frees that statement again. This is undefined behaviour inside DuckDB's C API that no caller can detect; see `docs/upstream-duckdb-reports.md`, item 36. -## COPY functions (resolved in DuckDB 1.5.0; both directions since) +### `duckdb_register_log_storage` frees its extra data twice when the name is taken (DuckDB defect, Pitfall L20) + + +When registering a log storage fails because the name is already registered, DuckDB runs +the extra-data destructor inside the failed call and leaves the pointer on the wrapper, so +the `duckdb_destroy_log_storage` the caller must run frees it again. Observed on DuckDB +1.5.6 and 2.0.0: the extra-data destructor ran twice on the same pointer. With a destructor +that frees its box that is a heap double free, which the eighth review saw abort with +`free(): double free detected in tcache 2`; the code is the same since 1.5.0 and on `main` +(read). It is not a quack-rs defect: quack-rs does not wrap the log-storage functions, for this reason. +An extension that calls them through the raw bindings must never register a name that can +already exist, and must treat a failed registration as having consumed the extra data. See +[Pitfall L20](pitfalls.md#l20-duckdb_register_log_storage-frees-the-extra-data-twice-when-the-name-is-taken). + +### `duckdb_get_bit` and `duckdb_get_bignum` abort on NULL and on values that do not cast (DuckDB defect, Pitfall P16) + + +Called on a SQL `NULL` value, or on a value that cannot be cast to `BIT`, `duckdb_get_bit` +throws a C++ exception through the C boundary, and Rust cannot catch it: +`fatal runtime error: Rust cannot catch foreign exceptions`, `SIGABRT` (reproduced on +DuckDB 1.5.6 and 2.0.0 with a `NULL` and with `'abc'`). `duckdb_get_bignum` runs the same +code (read in the 2.0.0 source, not run). quack-rs's `Value::as_bit()` and +`Value::as_bignum()` check the value's type and its NULL-ness first and return `None` +instead. Code that calls the raw getters through the bindings must do the same. See +[Pitfall P16](pitfalls.md#p16). + +## What the C API does not offer + +### Window functions are not available + +DuckDB **window functions** (`OVER (...)` clauses) are implemented entirely in +DuckDB's C++ layer and have **no counterpart in the public C extension API**. + +This is not a gap in `quack-rs` or in `libduckdb-sys`: the C API has no function for +registering a window function, and no symbol such as `duckdb_create_window_function` +exists in `duckdb.h` or `duckdb_extension.h`. DuckDB's C++ API does support window +functions, through its own C++ classes rather than a function of that name. + +| Kind of function | C API symbol | C API (1.4.4, 1.4.5) | C API (1.5.0+) | C++ API | +|------------------|--------------|----------------------|----------------|---------| +| Window function | *(none)* | **No** | **No** | Yes | +| `COPY` function | `duckdb_create_copy_function` | **No** | **Yes** | Yes | +| Scalar function | `duckdb_create_scalar_function` | Yes | Yes | Yes | +| Aggregate function | `duckdb_create_aggregate_function` | Yes | Yes | Yes | +| Table function | `duckdb_create_table_function` | Yes | Yes | Yes | +| Cast function | `duckdb_create_cast_function` | Yes | Yes | Yes | + +**What this means for your extension:** + +A custom window operator requires a C++ extension. An ordinary aggregate can be +used *as* a window (`agg(x) OVER (...)`), but see the next section before +recommending that to your users: two of those shapes crash every aggregate +registered through the C API. + +If DuckDB exposes window registration in a future C API version, `quack-rs` +will add wrappers in the corresponding release. + +### Table function arguments must be constants, not columns or subqueries + +A table function registered through the C API cannot take a column or a +subquery as an argument. `LATERAL (SELECT * FROM f(t.n))`, a correlated call, a +recursive CTE that references the function and `f((SELECT 4))` all fail in the +binder, on 1.5.6 and 2.0.0, with "does not support lateral join column +parameters" or "cannot contain subqueries". What it accepts is a **constant +expression**: `f(1 + 2)`, `f(3::INTEGER)` and +`f(current_setting('threads')::BIGINT * 0 + 4)` were evaluated once, at bind +time, and passed in (measured on 1.5.6 and 2.0.0). A prepared-statement parameter +(`f($1)`) works. Pass a `LIST` literal, or write the function as a scalar. + +### The C API v1 has no table-in-out functions, progress callbacks, filter pushdown, logging or secret access + + +Checked by searching the 1.5.6 and 2.0.0 `duckdb.h`, `duckdb_extension.h` and, for the +2.0.0 v2 headers, `duckdb_v2.h`: + +- a table function cannot take rows from a subquery (no table-in-table-out function exists + in either C API), has no progress callback, and receives no filters (no filter pushdown + in v1); +- the only logging functions are the log *storage* (sink) functions, such as + `duckdb_register_log_storage` (see the entry on its double free below); nothing writes a log + record. There is no function that reads a secret. quack-rs offers `list_duckdb_secrets`, + which asks SQL (`duckdb_secrets()`) and redacts the value; +- `COPY` has no column names, option schema or file extension (see below). + +DuckDB 2.0's C API v2 has most of these (next entries). + +### Custom COPY formats cannot declare a file extension + +The C API v1 has no way to give a COPY function a file extension, so +`PER_THREAD_OUTPUT`, `PARTITION_BY` and `FILENAME_PATTERN` produce files that end +in a bare dot (`data_0.`, `part_0.`); the built-in formats give `data_0.csv`. +Append the extension to the path in your `global_init` if the output needs one. + +DuckDB does choose a format from the file name, as long as the name is the format's: +`COPY t TO 'out.my_format'` uses a copy function called `my_format` (measured on 1.5.6 and +2.0.0), so name the format after the extension you want recognised. Two more +things a COPY format cannot do in the C API v1: it cannot see the **column names** of a +`COPY … TO` (the bind callback gets the count and the types), so a writer that wants a +header needs a `names` option from the user; and it cannot declare its **options**, so DuckDB +accepts any option name, value count or value type for a `COPY … TO` and the format must +reject what it does not understand (see [Copy +Functions](../functions/copy-functions.md#options)). + +### DuckDB 2.0's C API v2 offers what v1 cannot + +DuckDB 2.0.0 ships a second C API, `duckdb_extension_v2.h` / `duckdb_v2.h`, with +no stable surface yet: a binary built against it loads only into the exact +DuckDB release it was built for, and DuckDB 2.0 sends every +`C_STRUCT_UNSTABLE` binary to its `_init_c_api_v2` entry point. quack-rs +does not generate that entry point, so none of the following is available +through it. Every name below exists in the 2.0.0 `duckdb_v2.h` / +`duckdb_extension_v2.h` and none exists in the v1 headers (checked by `grep` +against the headers of the 2.0.0 release archive); the v2 API has about 860 +`duckdb_v2_*` declarations, and this list is the part an author of the extension kinds +quack-rs wraps would reach for: + +- a scalar function's or aggregate's return type chosen at bind time + (`duckdb_v2_scalar_function_resolve_types_set_return_type`, + `duckdb_v2_aggregate_function_bind_set_return_type`) and the aggregate's row + count (`duckdb_v2_aggregate_function_update_get_row_count`); +- function properties, among them a "consistent within query" stability; +- documentation for `duckdb_functions()`: a description, examples and categories + (`duckdb_v2_function_docs_set_description`, `_add_example`, `_add_category`), which the + community extensions site shows (see [Community Extensions](../publishing.md)); +- an option's update callback (`duckdb_v2_custom_option_set_update_callback`), which + can validate or replace a setting's value; +- for table functions: filter pushdown + (`duckdb_v2_table_function_set_filter_pushdown_callback`), a progress callback + (`duckdb_v2_table_function_progress_*`), order preservation + (`duckdb_v2_table_function_bind_set_order_preservation`) and batch claiming + (`duckdb_v2_table_function_claim_batch_*`); and multi-file readers + (`duckdb_v2_multi_file_function_*`); +- for `COPY`: a file extension (`duckdb_v2_copy_function_set_file_extension`) and file-size + statistics (`duckdb_v2_copy_to_set_statistics_callback`); +- writing log records (`duckdb_v2_context_log`) and registering log sinks; +- more file-system operations: `glob`, `exists`, `expand`, `move`, `is_remote` + (`duckdb_v2_file_system_path_*`); +- query metrics (`duckdb_v2_query_metrics_*`), a SQL tokenizer (`duckdb_v2_token_iterator_*`), + native macro functions, pragma functions and custom types; +- static registration, which also does not work with quack-rs's v1 entry point (next entry). + +The official Rust template and `duckdb-neo` target this API. quack-rs stays on v1 +for the portable binary; see the README's comparison. + +### A quack-rs extension cannot be linked statically into a DuckDB 2.0 host + + +DuckDB 2.0.0 can register an extension compiled into the host program +(`duckdb_register_static_extension`, in `duckdb_static_extension.h`). For a C API v1 entry +point it calls the entry point with `get_api` unset, because the extension is expected to call +DuckDB's linked symbols directly instead of going through the function-pointer table +(`DuckDB::LoadStaticCAPIExtension`, `extension_load.cpp` in the v2.0.0 source). quack-rs's +entry points need the table to initialise the bindings, so they refuse with "DuckDB passed no +get_api function, so the C API cannot be initialised": no panic, no crash, and no extension. +A Rust program that embeds DuckDB 2.0 and wants a quack-rs extension in-process cannot do +that; it can `LOAD` the stamped shared library like any other client. Read in the source, +not run. + +## Behaviour that changed in DuckDB 2.0 + +### Aggregates that take no argument count no rows on DuckDB 2.0 + +The C API gives an aggregate's `update` callback a chunk and the state array and +no row count: the number of rows is the chunk's size. DuckDB 2.0 no longer sets +that cardinality (`CAPIAggregateUpdate`, `aggregate_function-c.cpp`), and a chunk +with no column reports a size of 0. A zero-argument aggregate (`SELECT my_count() +FROM t`) therefore answers 100 over 100 rows on DuckDB 1.5.6 and **0** on 2.0.0 +(measured on both, ungrouped and grouped), and a windowed call of such an +aggregate never reaches `update` on either. quack-rs refuses to register an +aggregate, or an overload in a set, that has no parameter. Give it one, and put a +macro in front of it for a zero-argument spelling: + +```rust,ignore +// my_count_1(x) counts the rows it is called on; `x` is ignored. +AggregateFunctionBuilder::new("my_count_1").param(TypeId::BigInt) /* … */; +SqlMacro::scalar("my_count", &[], "my_count_1(1::BIGINT)")?; // CREATE MACRO my_count() AS my_count_1(1::BIGINT) +``` + +### A `STRUCT` with only empty field names is a `TUPLE` on DuckDB 2.0 + +DuckDB 2.0 has a `TUPLE` type, which the C API reports as a `STRUCT` with empty +child names, and `duckdb_create_struct_type` with all-empty names now builds one. +`LogicalType::struct_type(&[("", …), ("", …)])` is accepted and gives a `TUPLE` on 2.0 +and a different type on 1.x. Name the fields. + +### `duckdb_query` runs the statements before a parse error on DuckDB 2.0 + +Given text that does not parse as a whole, DuckDB 1.4.4 and 1.5.6 report a parser error and +run nothing. DuckDB 2.0.0 runs the statements that come before the error (the call may then +report the error or, as in the measurement below, success). Measured with a C program that sends `duckdb_query` the text +`CREATE OR REPLACE MACRO "m"() AS (1); DROP TABLE canary; SELECT ()` (what +`SqlMacro::register` would have sent for the body `1); DROP TABLE canary; SELECT (`): +1.4.4 and 1.5.6 returned `Parser Error: syntax error at or near ")"` and the table survived; +2.0.0 returned success and the table was gone. An extension that passes text it did not +write itself to `duckdb_query`, `OwnedConnection::query` or `execute` must check first that it +parses as the statements it expects: `duckdb_extract_statements` reports a count of 0 for text +that does not parse as a whole, and the count is the number to require. quack-rs's +`SqlMacro::register` requires exactly 1 since 0.20.0. + +### Other DuckDB 2.0 behaviour changes that reach C API extensions + + +Beyond the entries above, DuckDB 2.0 changes these things an extension can notice. The +first four were measured against the 1.4.4, 1.5.6 and 2.0.0 engines; the rest were read +by comparing `src/main/capi` at v1.5.6 with `src/main/capi/v1` at v2.0.0 and have not been +run. + +- `VerifyNullHandling` is compiled into release builds and runs after + `SET debug_verification_mode = 'VERIFY_FUNCTIONS'` ([Pitfall + L8](pitfalls.md#l8-default_null_handling-does-not-propagate-nulls-for-scalar-functions)). +- A table function's `bind` runs again on every `EXECUTE` of a prepared statement + ([Table Functions](../functions/table-functions.md#lifecycle)). +- `INTERVAL '1 month' - INTERVAL '1 day'` compares equal to `INTERVAL '29 days'`, where + DuckDB 1.4.4 and 1.5.6 compare it greater ([INTERVAL](../data/intervals.md)). +- A bare identifier as a table function argument (`active_users(users)`) prints a + deprecation warning; write `'users'`. +- `duckdb_appender_close` returns an error when an earlier flush or close failed, where 1.5.6 + did not report it again (`appender-c.cpp`). +- `duckdb_data_chunk_set_size` ignores a size DuckDB rejects in a release build instead of + reporting it (`SetChildCardinality` inside `catch (...) { D_ASSERT(false); }`). +- Every C API scalar function is registered fallible, so an error from a function + declared non-fallible is no longer turned into an internal error. +- A scalar function whose inputs are all constants is no longer marked a constant vector + on the way out. +- A `COPY` option given several values of different types arrives as a `TUPLE`. + +## Resolved + +These were limits of earlier releases. They are kept so older changelog entries and +reports still make sense. + +### COPY functions (resolved in DuckDB 1.5.0; both directions since) DuckDB 1.5.0 added `duckdb_create_copy_function` and related symbols to the public C extension API. quack-rs wraps these in the `copy_function` module behind the @@ -170,7 +528,7 @@ format's reader, and a copy function may implement either direction or both — read-only format leaves the writing callbacks unset entirely. See the [Copy Functions](../functions/copy-functions.md) chapter. -## Arrow interop (resolved behind `duckdb-1-5-4`) +### Arrow interop (resolved behind `duckdb-1-5-4`) DuckDB's C API has a family of conversion functions (present since 1.4.4) that move data directly between a `duckdb_data_chunk` and the Arrow C Data @@ -187,7 +545,7 @@ declared the two Arrow ABI records as opaque zero-sized placeholders until quack-rs supports, but because the feature implies `duckdb-1-5`, an extension built with it needs a DuckDB 1.5.0+ engine. -## Callback accessor wrappers (resolved) +### Callback accessor wrappers (resolved) quack-rs wraps the **callback accessor** functions — the C API functions used *inside* your callbacks to retrieve arguments, set errors, access bind data, @@ -213,7 +571,7 @@ init, table function bind, and the four copy-function callbacks — the wrapper exposes it as `get_client_context`, which returns a `ClientContext` (see the `client_context` module). -## Complex type creation (resolved) +### Complex type creation (resolved) `LogicalType` provides constructors for all complex parameterized types: @@ -233,7 +591,7 @@ constructor has a `try_` form that returns an error instead of panicking. Introspection methods (`get_type_id`, `list_child_type`, `struct_child_count`, `decimal_width`, etc.) are also available. -## VARIANT and GEOMETRY types (resolved — exposed behind `duckdb-1-5-3`) +### VARIANT and GEOMETRY types (resolved — exposed behind `duckdb-1-5-3`) The `VARIANT` type (a self-describing nested value, used for example by Iceberg v3) entered the C type enum as `DUCKDB_TYPE_VARIANT` (41) in @@ -250,156 +608,11 @@ is gated with it so that one feature covers both values. Keeping them out of ```toml [dependencies] -quack-rs = { version = "0.19", features = ["duckdb-1-5-3"] } +quack-rs = { version = "0.20", features = ["duckdb-1-5-3"] } ``` Neither type yet has dedicated `VectorReader`/`VectorWriter` helpers; access their data via the raw pointer from `duckdb_vector_get_data` when needed. -**A `VARIANT` cannot be declared in a function signature.** The type -`duckdb_create_logical_type(VARIANT)` returns has no `STRUCT` layout, because -DuckDB stores a VARIANT as a struct. A scalar function declared with it -registers, and the first query that reaches it takes DuckDB down: on 2.0.0 -`SELECT v_out(1)` is a segmentation fault (`VectorStructBuffer` while -initialising the result vector; reproduced with the guard bypassed), and on 1.5.x -`SELECT v_in({'a':1}::VARIANT)` is an internal error that invalidates the -database. quack-rs therefore refuses `TypeId::Variant` wherever it builds a -logical type (`LogicalType::try_new`, and so every builder's parameter, -return-type and result-column slots), with an error that names `ANY`. A function -that wants to accept a VARIANT declares the parameter as `TypeId::Any`. -`GEOMETRY` is unaffected. - -## Aggregates that take no argument count no rows on DuckDB 2.0 - -The C API gives an aggregate's `update` callback a chunk and the state array and -no row count: the number of rows is the chunk's size. DuckDB 2.0 no longer sets -that cardinality (`CAPIAggregateUpdate`, `aggregate_function-c.cpp`), and a chunk -with no column reports a size of 0. A zero-argument aggregate (`SELECT my_count() -FROM t`) therefore answers 100 over 100 rows on DuckDB 1.5.6 and **0** on 2.0.0 -(measured on both, ungrouped and grouped), and a windowed call of such an -aggregate never reaches `update` on either. quack-rs refuses to register an -aggregate, or an overload in a set, that has no parameter. Give it one, and put a -macro in front of it for a zero-argument spelling: - -```rust,ignore -// my_count_1(x) counts the rows it is called on; `x` is ignored. -AggregateFunctionBuilder::new("my_count_1").param(TypeId::BigInt) /* … */; -SqlMacro::scalar("my_count", &[], "my_count_1(1::BIGINT)")?; // CREATE MACRO my_count() AS my_count_1(1::BIGINT) -``` - -## A failing state initialiser in a window frame crashes the host (DuckDB defect) - -When an aggregate's `state_init` reports an error while the aggregate runs as a -window function with a `ROWS` or `RANGE` frame, DuckDB's -`WindowAggregateStates` destroys its whole table of state pointers, including -entries it never wrote. The destructor is handed garbage pointers. A plain C -aggregate shows it, with no Rust involved: on DuckDB 1.5.6 55 of 68 pointers -passed to the destructor were garbage, and on 2.0.0 65 of 68. quack-rs's -`FfiState` reports a panic in `T::default()` as a SQL error, which is exactly this -trigger; its tag check cannot help, since there is no slot to read. Ungrouped, -grouped, `DISTINCT` and `FILTER` aggregates report a clean error; a window shape -kills the process (`SIGSEGV` or `SIGBUS`). **`T::default()` must not panic** for an -aggregate that can run as a window function. - -## NULL rows of `LIST` and `MAP` vectors hold stale entries - -DuckDB writes no list entry for a NULL row, so the `offset` and `length` that -`ListVector::get_entry` and `MapVector::get_entry` return for one are whatever the -reused buffer held. On 2.0.0 a `nullif(l, [])` column had NULL rows with lengths -around 7·10¹³, and a loop that walked them read far outside the child vector: a -segmentation fault in a minority of runs, and no failure at all in most, which is -how the pattern went unnoticed. `get_entry_checked` returns `None` for a NULL row -and for an entry that does not lie inside the child; use it, or test the row's -validity before `get_entry`. - -## Table functions take literals only - -A table function registered through the C API cannot take a column or a -subquery as an argument. `LATERAL (SELECT * FROM f(t.n))`, a correlated call, a -recursive CTE that references the function and `f((SELECT 4))` all fail in the -binder, on 1.5.6 and 2.0.0, with "does not support lateral join column -parameters" or "cannot contain subqueries". There is no table-in-table-out -function in the C API. Pass a `LIST` literal, or write the function as a scalar. - -## A function that reads a setting must be marked volatile - -DuckDB folds a non-volatile function call whose arguments are constants into the -plan, once. A function whose `init` reads a setting -(`ClientContext::config_option`) and is not `.volatile()` then keeps answering -with the value it read first: after `SET v_setting = 9; PREPARE p AS SELECT f(); -SET v_setting = 10; EXECUTE p;` the plain function returned 9 and the same function -marked `.volatile()` returned 10. The C API v1 has only `volatile`, not "stable -within a query". - -## Custom COPY formats cannot declare a file extension - -The C API v1 has no way to give a COPY function a file extension, so -`PER_THREAD_OUTPUT`, `PARTITION_BY` and `FILENAME_PATTERN` produce files that end -in a bare dot (`data_0.`, `part_0.`); the built-in formats give `data_0.csv`. -Append the extension to the path in your `global_init` if the output needs one. - -## SQL macros: no defaults or overloads, and they persist - -`SqlMacro` takes plain parameter names; `y := 5` is rejected, and registering the -same name again replaces the first (`CREATE OR REPLACE`). For a default -parameter, run `CREATE OR REPLACE MACRO m(x, y := 5) AS x + y` through -`Connection::execute`. A macro is stored in the user's database, so one whose -body calls an extension function breaks when the database is opened without the -extension loaded ("Table Function with name … does not exist"); a custom type -registered through the C API is not stored. - -## DuckDB 2.0's C API v2 offers what v1 cannot - -DuckDB 2.0.0 ships a second C API, `duckdb_extension_v2.h` / `duckdb_v2.h`, with -no stable surface yet: a binary built against it loads only into the exact -DuckDB release it was built for, and DuckDB 2.0 sends every -`C_STRUCT_UNSTABLE` binary to its `_init_c_api_v2` entry point. quack-rs -does not generate that entry point, so none of the following is available -through it. Each name below exists in the 2.0.0 header and is absent from the v1 -header (checked by `grep`): - -- a scalar function's or aggregate's return type chosen at bind time - (`duckdb_v2_scalar_function_resolve_types_set_return_type`, - `duckdb_v2_aggregate_function_bind_set_return_type`) and the aggregate's row - count (`duckdb_v2_aggregate_function_update_get_row_count`); -- function properties, among them a "consistent within query" stability; -- an option's update callback (`duckdb_v2_custom_option_set_update_callback`), which - can validate or replace a setting's value; -- filter pushdown for table functions - (`duckdb_v2_table_function_set_filter_pushdown`), and a progress callback; -- a COPY function's file extension (`duckdb_v2_copy_function_set_file_extension`); -- native macro functions, pragma functions, custom types and log sinks. - -The official Rust template and `duckdb-neo` target this API. quack-rs stays on v1 -for the portable binary; see the README's comparison. - -## A `STRUCT` with only empty field names is a `TUPLE` on DuckDB 2.0 - -DuckDB 2.0 has a `TUPLE` type, which the C API reports as a `STRUCT` with empty -child names, and `duckdb_create_struct_type` with all-empty names now builds one. -`LogicalType::struct_type(&[("", …), ("", …)])` is accepted and gives a `TUPLE` on 2.0 -and a different type on 1.x. Name the fields. - -## A caught panic is still printed to stderr - -The macros and typed builders catch a panic at the FFI boundary and report it as a -SQL error, but Rust's default panic hook has already run by then: it writes the -panic message to the host process's stderr (and a backtrace when `RUST_BACKTRACE` -is set), and does so even though the query only fails with an error. quack-rs -installs no hook, because the hook is process-wide and the host (Python, a CLI, a -server) owns it. An extension that wants these lines silent or routed to its own -log can call `std::panic::set_hook` in its entry point, knowing that it replaces -the host's hook for every thread in the process. - -## Loading an extension scans the function catalog once - -Every scalar and table registration through quack-rs first lists `duckdb_functions()` -to refuse a signature that would silently replace an existing function (a built-in -included) or make calls ambiguous. Registering through the entry point's -`Connection` does that scan once per `LOAD`, however many functions follow. The cost -is fixed, not per function: for an extension with one `map1` function, `LOAD` took -18 ms longer than the same entry point registering the function through the raw C -API on DuckDB 1.5.6, and 30 ms longer on 2.0.0 (medians of 30 interleaved CLI -runs each, one 4-vCPU virtual machine, release build; the minima differ by 17 and -27 ms). The C API has no catalog lookup for functions, so the scan cannot be -narrowed to a name. +A `VARIANT` still cannot be declared in a function signature: see +[the entry above](#a-variant-cannot-be-declared-in-a-function-signature). diff --git a/book/src/reference/pitfalls.md b/book/src/reference/pitfalls.md index 0bdab965..1e3737fd 100644 --- a/book/src/reference/pitfalls.md +++ b/book/src/reference/pitfalls.md @@ -1,6 +1,6 @@ # Pitfall Catalog -The 34 known pitfalls of writing a DuckDB extension in Rust against the C +The 38 known pitfalls of writing a DuckDB extension in Rust against the C extension API, with the symptom, root cause and fix for each. The first were found while building [duckdb-behavioral](https://github.com/tomtom215/duckdb-behavioral), a @@ -135,7 +135,8 @@ and report the `Err` it returns yourself. All of these guards require `panic = "unwind"` in the release profile, which is what the scaffold generates and what `validate_release_profile` insists on: -under `panic = "abort"` nothing can be caught. +under `panic = "abort"` nothing can be caught, so `entry_point!` and `entry_point_v2!` +refuse to compile there (an entry point written by hand has no such check). --- @@ -254,8 +255,10 @@ survives review and ships. scalar function registered through the C API it does not, at run time: `CAPIScalarFunction` calls the callback for every row including NULL ones and checks only the *error* flag, and the one NULL check in `ExpressionExecutor` — -`VerifyNullHandling` — has its entire body inside `#ifdef DEBUG`. Every DuckDB a -user installs is a release build. +`VerifyNullHandling` — has its entire body inside `#ifdef DEBUG` through DuckDB +1.5.6, so a release build skips it; DuckDB 2.0.0 compiles it into release builds +but runs it only after `SET debug_verification_mode = 'VERIFY_FUNCTIONS'` (default +`NONE`). Every DuckDB a user installs runs with the default. **Fix**: use the typed closure constructors, which skip NULL rows and write NULL for them, or call `DataChunk::propagate_nulls(&mut writer)` at the end of a @@ -376,8 +379,12 @@ documented on `AggregateFunctionBuilder`, `AggregateFunctionSetBuilder` and **Symptom**: An aggregate that works under `SELECT agg(x) FROM t` and `GROUP BY` segfaults (or corrupts memory, or returns a wrong answer) when used as a window -over a whole-partition frame — `agg(x) OVER ()`, `OVER (PARTITION BY p)` — or as -an ordered aggregate, `agg(x ORDER BY y)`. +over a whole-partition frame — `agg(x) OVER ()`, or an explicit `ROWS` or `RANGE +BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame, with or without a +`PARTITION BY` — or as an ordered aggregate, `agg(x ORDER BY y)`. On 1.4.4 `OVER +(PARTITION BY p)` without an `ORDER BY` crashes too; on 1.5.6 it did not in the shapes +measured (5,000 and 300,000 rows). Whether that is a different code path or luck was not +established, so do not rely on it. **Root cause**: `CAPIAggregateUpdate` (`src/main/capi/aggregate_function-c.cpp`) flattens the input vectors but not the state vector, then hands the callback @@ -394,9 +401,12 @@ callback, called from `CAPIAggregateUpdate`. `duckdb_aggregate_state *` and cannot tell a constant vector from a flat one, and reading `states[1]` to find out is itself the out-of-bounds read. Until you require DuckDB 2.0.0, document for your users that the aggregate must not be used -in those two query shapes. Frames that are not whole-partition (`ROWS BETWEEN 5 -PRECEDING AND CURRENT ROW`, segment-tree windows) and `DISTINCT` windows were -checked and work. +in those query shapes. Frames that are not whole-partition (the default frame of +`OVER (ORDER BY i)`, `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`, `ROWS BETWEEN 5 +PRECEDING AND CURRENT ROW`, `EXCLUDE`) and `DISTINCT` and `FILTER` were measured and work +on 1.4.4, 1.5.6 and 2.0.0 ([Known +Limitations](known-limitations.md#aggregates-crash-under-over--and-order-by-duckdb-defect-pitfall-l11) +has the table). DuckDB 2.0.0 fixes it in `CAPIAggregateUpdate` (`src/main/capi/v1/aggregate_function-c.cpp`, lines 104-108): a state vector @@ -626,6 +636,74 @@ runs this build. --- +## L20: `duckdb_register_log_storage` frees the extra data twice when the name is taken + +**Status**: Documented. quack-rs does not wrap the log-storage functions because of it. +This is a DuckDB defect, present since 1.5.0 and still on `main` (2026-10-30), not a +quack-rs one. + +**Symptom**: registering a log storage under a name that is already taken returns +`DuckDBError`, and the extra-data destructor you gave DuckDB has already run. The +`duckdb_destroy_log_storage` you must still call runs it again on the same pointer. +Observed on DuckDB 1.5.6 and 2.0.0 with a probe extension that registers the same name +twice: the destructor ran twice on one address. With a destructor that frees its box this +is a heap double free (`free(): double free detected in tcache 2`, seen in the eighth +review). The same happens for a name that a built-in storage already holds. + +**Root cause** (`src/main/capi/v1/logging-c.cpp`, identical at 1.5.6, 2.0.0 and `main`): + +```cpp +auto shared_storage_ptr = make_shared_ptr( + wrapper->name, wrapper->write_log_entry, wrapper->extra_data, wrapper->delete_callback); +const auto success = RegisterLogStorage(wrapper->name, storage_ptr); +if (!success) { + return DuckDBError; // shared_storage_ptr dies here and calls delete_callback(extra_data) +} +wrapper->extra_data = nullptr; // only cleared on success +wrapper->delete_callback = nullptr; +``` + +The early failures (a null argument, an empty name, no write callback) return before the +`CallbackLogStorage` exists and do not consume the data, so a caller cannot treat every +`DuckDBError` alike. + +**Workaround**: never register a name that can already exist. After a failure that got +past argument validation, clear the extra data with +`duckdb_log_storage_set_extra_data(ls, NULL, NULL)` before destroying the wrapper (proposed, +not tested). + +**Rule**: a destructor you hand to DuckDB may already have run when the call that took +it reports failure; read the failure path before you free again. + +--- + +## L21: A chunk row you claim is published, written or not + +**Status**: Documented. Since 0.20.0 `ChunkWriter::release_row` gives a claimed row back; +before 0.20.0 every claimed row was published. The rule is about the chunk the +scan callback returns. + +**Symptom**: a table function whose scan claims a row from the writer and then finds +nothing to write (its source ended) still returns that row to DuckDB, holding whatever the +reused output buffer held last. Measured in the eighth review on DuckDB 1.5.6: a scan that +wrote 2 rows returned 3, and the third was the `7` an earlier chunk had left in that +position. For a fixed-width column that is a wrong answer, not undefined behaviour +(valgrind-clean). For a `LIST` or `MAP` column written by hand, a row never written keeps the +`{offset, length}` entry of an earlier chunk, which can point past the child data this chunk +filled (CONJECTURED, not reproduced). + +**Root cause**: the writer counts the rows you *claimed* and sets the chunk's size from that +count when it is dropped. DuckDB reuses output vectors across chunks, so a claimed row you +did not write is published with stale contents. + +**Fix**: claim a row only when you are going to write it: check that the source has another +value first, then claim the row and write it in the same step. A row you have already +claimed can be given back with `ChunkWriter::release_row(row)` before the writer is dropped. + +**Rule**: an output chunk holds the rows you wrote, not the rows you asked for. + +--- + ## P1: Library name must match extension name **Status**: Must be configured in `Cargo.toml`. Scaffold handles this. @@ -1062,6 +1140,57 @@ whatever bytes the allocation had. an entry that does not fit inside the child, or test the row's validity before reading the entry. + +## P16: `duckdb_get_bit` and `duckdb_get_bignum` abort on NULL and on values that do not cast {#p16} + +**Status**: Prevented in quack-rs's own wrappers, a DuckDB defect for everything else. +`Value::as_bit()` and `Value::as_bignum()` return `None` for a value they cannot read +instead of calling the C getter on it, and `Value::bit` and `Value::bignum` build only +well-formed values. Code that calls `duckdb_get_bit` or `duckdb_get_bignum` through the +raw bindings must make the checks below itself. + +**Symptom**: the process aborts with `fatal runtime error: Rust cannot catch foreign +exceptions` (`SIGABRT`, exit code 134) when `duckdb_get_bit` is called on a SQL `NULL` +value or on a value that does not cast to `BIT` (such as the string `'abc'`). Reproduced on +DuckDB 1.5.6 and 2.0.0. `duckdb_get_bignum` is built the same way (read in the 2.0.0 +source, not run). + +**Root cause**: both functions cast the value (`DefaultCastAs(BIT)` or `BIGNUM`) and read +the result with no `try`, so the cast's C++ exception leaves an `extern "C"` function. + +**Fix**: use `Value::as_bit()` / `Value::as_bignum()`. In raw code, before calling either +getter, check that the handle is not null (a null handle is dereferenced), that +`Value::type_id()` is the type you expect (convert with `Value::try_cast`, which cannot +throw, when it is not), and `Value::is_sql_null()`; these are plain accessors that cannot +throw. + +**Rule**: a C API getter that casts can throw; check the type and NULL-ness first, because +Rust cannot catch a C++ exception. + +## P17: `std::fs` bypasses DuckDB's permission settings {#p17} + +**Status**: Documented in the [Virtual File +System](../duckdb-1-5/file-system.md#security-only-filesystem-honours-duckdbs-permission-settings) +chapter, in the table-function, replacement-scan and `COPY` chapters, in [Known +Limitations](known-limitations.md) and in the FAQ. quack-rs's `FileSystem` honours the +settings; nothing can make `std::fs` do so. + +**Symptom**: with `SET enable_external_access = false` (or `allowed_directories` set), +`read_csv('data.csv')` fails with "Permission Error: Cannot access file … file system +operations are disabled by configuration", and so does a table function that opens the same +path through `FileSystem`. A table function that reads it with `std::fs::read` returns the +file. Measured on DuckDB 1.5.6 and 2.0.0. + +**Root cause**: DuckDB enforces the settings in its own file system layer. `std::fs` calls +the operating system directly and never consults it. + +**Fix**: open every path that comes from SQL (arguments, options, `COPY` targets, +replacement-scan names) with `FileSystem::from_client_context(...)` (feature `duckdb-1-5`). +Without that feature, take the data as a `VARCHAR` or `BLOB` argument. + +**Rule**: an extension runs with the process's full privileges; the only restrictions it +inherits are the ones it asks DuckDB to apply. + --- ## Summary @@ -1087,6 +1216,8 @@ reading the entry. | L17: `COPY … FROM` reader declares columns | Prevented (typed) / Documented | Read the target's columns; declare none | | L18: `LIST` reserve moves nested buffers | Documented | Fetch writers again after each reserve | | L19: constant addresses as identities | Fixed | `FfiState` salts its tag with a hash of `TypeId` | +| L20: log-storage registration frees its extra data twice | DuckDB defect; not wrapped | Do not call `duckdb_register_log_storage` with a name that can exist | +| L21: a claimed chunk row is published unwritten | Documented | Claim a row only when you will write it | | P1: lib name mismatch | Scaffold | Set `[lib] name` in `Cargo.toml` | | P2: API version string | Constant | Use `DUCKDB_API_VERSION` | | P3: unit tests insufficient | Documented | Write SQLLogicTest E2E tests | @@ -1095,10 +1226,12 @@ reading the entry. | P6: register set silent fail | Prevented | Builder returns `Err` | | P7: VARCHAR format undocumented | Prevented | Use `VectorReader::read_str` | | P8: INTERVAL layout undocumented | Prevented | Use `DuckInterval` | -| P9: dispatch table uninitialised | Fixed | `InMemoryDb::open()` initialises it via C++ shim | +| P9: dispatch table uninitialised | Fixed | `InMemoryDb::open()` initialises it through a small generated shim | | P10: unstable ABI tail shifts | Prevented | Use `init_extension`; set `QUACK_RS_TARGET_DUCKDB_VERSION` when building | | P11: freeing a borrowed `const char *` | Fixed | Read the C++ impl before `duckdb_free`; prefer quack-rs wrappers | | P12: config-option probe aborts (debug) | Documented | Ask `duckdb_settings()` in SQL instead | | P13: `VARIANT` in a signature crashes | Guarded | Declare the parameter as `ANY` | | P14: zero-parameter aggregate counts no rows (2.0) | Guarded | One parameter, plus a `CREATE MACRO` for the zero-argument spelling | | P15: NULL list/map row has a stale entry | Prevented | Use `get_entry_checked` | +| P16: `duckdb_get_bit` / `duckdb_get_bignum` abort on NULL | DuckDB defect; guarded in `Value::as_bit` / `as_bignum` | Use those, or check `type_id()` and `is_sql_null()` first | +| P17: `std::fs` ignores `enable_external_access` | Documented | Open user-supplied paths with `FileSystem` (`duckdb-1-5`) | diff --git a/book/src/security/warnings.md b/book/src/security/warnings.md index 753a6896..23ddf160 100644 --- a/book/src/security/warnings.md +++ b/book/src/security/warnings.md @@ -38,6 +38,13 @@ across threads via `Arc`. A panic on another thread while it held the lock does not lose warnings: the collector recovers the list from the poisoned lock and keeps using it. +A collector keeps at most 10,000 warnings between drains (`DEFAULT_MAX_WARNINGS`; use +`WarningCollector::with_limit(n)` for another limit). Warnings past the limit are not stored: +`emit` counts them, `dropped()` returns the count, and `drain` and `clear` reset it. The first +warnings are the ones kept. Before 0.20.0 every warning was stored, and a collector that is never +drained grew for as long as the process lived (2,000,000 warnings were about 381 MB), which a +condition reported once per request, such as `TLS_NO_VERIFY`, can cause. + ## Usage ```rust diff --git a/book/src/sitemap.xml b/book/src/sitemap.xml index 6f0e1342..5c750f5d 100644 --- a/book/src/sitemap.xml +++ b/book/src/sitemap.xml @@ -4,6 +4,7 @@ https://quack-rs.com/getting-started/quick-start.htmlmonthly0.9 https://quack-rs.com/getting-started/installation.htmlmonthly0.7 https://quack-rs.com/getting-started/upgrading-0-18.htmlmonthly0.7 + https://quack-rs.com/getting-started/upgrading-0-20.htmlmonthly0.7 https://quack-rs.com/getting-started/first-extension.htmlmonthly0.7 https://quack-rs.com/getting-started/scaffold.htmlmonthly0.7 https://quack-rs.com/concepts/anatomy.htmlmonthly0.7 diff --git a/book/src/testing.md b/book/src/testing.md index f4dc0558..7897823b 100644 --- a/book/src/testing.md +++ b/book/src/testing.md @@ -206,7 +206,7 @@ Two features expose `InMemoryDb`; pick the one that fits your build-time budget: ```toml # Zero-config but slow: compile libduckdb from C++ source (~5–10 min cold). [dev-dependencies] -quack-rs = { version = "0.19", features = ["bundled-test"] } +quack-rs = { version = "0.20", features = ["bundled-test"] } ``` ```toml @@ -218,7 +218,7 @@ quack-rs = { version = "0.19", features = ["bundled-test"] } # or duckdb.hpp. Header discovery needs libduckdb-sys >= 1.10503; with an older # one, also set DUCKDB_INCLUDE_DIR. [dev-dependencies] -quack-rs = { version = "0.19", features = ["bundled-test-prebuilt"] } +quack-rs = { version = "0.20", features = ["bundled-test-prebuilt"] } ``` Both keep `duckdb` out of a plain `cargo test` and out of your published @@ -522,9 +522,6 @@ test/sql/my_extension.test require my_extension -statement ok -LOAD my_extension; - query I SELECT my_function('hello world'); ---- @@ -538,17 +535,23 @@ Directives: | `require` | Load the extension; skip the file if it is not available | | `statement ok` | SQL must succeed | | `statement error` | SQL must fail | -| `query I` | Query returning one INTEGER column | -| `query II` | Query returning two INTEGER columns | -| `query T` | Query returning one TEXT column | +| `query I`, `query II`, `query T`, … | Query whose result has one column per letter; each letter is that column's type (`I` integer, `T` text, `R` real) | | `----` | Expected output follows | +**A skipped file passes.** If `require my_extension` cannot load the extension (the name +does not match `EXTENSION_NAME`, `[lib] name` or the file name, or the extension file was +not built), the runner reports the file as `SKIPPED` and exits 0. The community build +then succeeds having tested nothing, and so does a plain `make test`. The `Makefile` that +the scaffold generates since 0.20.0 runs the runner through a check that fails `make test` +on `SKIPPED` and when no test file ran; in a project from an older scaffold, read the +runner's output for `SKIPPED` and fail the job on it in your own CI. + ### Installing DuckDB (1.4.4 or later) E2E testing needs the DuckDB CLI. Download it with `curl`; no system package -manager is needed. quack-rs supports DuckDB 1.4.4 through 2.0.0, and every one +manager is needed. quack-rs supports DuckDB 1.4.4 through 2.0.x, and every one of them loads an extension stamped with -C API version `v1.2.0` (1.4.4 through 1.5.5 declare that version; 1.5.6 and 2.0.0 +C API version `v1.2.0` (1.4.4 through 1.5.5 declare that version; 1.5.6 and 2.0.x declare `v1.5.6` and accept every earlier one). The quack-rs CI `extension-load` job loads its example extension into 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release (currently 2.0.0). @@ -602,13 +605,22 @@ SELECT my_function('hello world'); " ``` +**Parsing the DuckDB 2.0 CLI's output in a script.** The 2.0 CLI switches to an "agent +mode" (a Markdown-like table with `name:TYPE` headers and a banner line) when the +environment variable `AI_AGENT` is set and its output is not a terminal. Measured with +`AI_AGENT=1` and the output piped: `-csv` overrides it, and `-no-agent` turns it off. `-no-agent` exists only in the 2.0 CLI; the 1.x CLIs reject it ("Unrecognized +option '-no-agent'"), so a script that runs several releases should unset the variable +instead of passing the flag. + The community extension CI runs these SQLLogicTest files automatically. Give each function at least one test, covering NULL, empty and typical input: ```sql -# Test NULL handling +# Test NULL handling. Read the NULL from a column: DuckDB folds a literal +# `my_function(NULL)` into the plan and never calls the function, so that query +# passes even for a callback that gets NULL wrong (Pitfall L8). query I -SELECT my_function(NULL); +SELECT my_function(x) FROM (VALUES (NULL::VARCHAR)) t(x); ---- NULL @@ -675,12 +687,12 @@ quack-rs's own test suite uses proptest for interval conversion and ```toml [dependencies] -quack-rs = "0.19" +quack-rs = "0.20" [dev-dependencies] proptest = "1" # Only for InMemoryDb; enables the feature for test builds alone. -quack-rs = { version = "0.19", features = ["bundled-test"] } +quack-rs = { version = "0.20", features = ["bundled-test"] } ``` The `testing` module is compiled unconditionally (not `#[cfg(test)]`), so crates diff --git a/build.rs b/build.rs index 0cdcfbd9..257e7a31 100644 --- a/build.rs +++ b/build.rs @@ -225,15 +225,15 @@ fn has_shim_headers(dir: &Path) -> bool { /// probeable on pre-1.10503 (the download dir isn't exposed), so users /// on that mode must upgrade `libduckdb-sys` or set the dirs explicitly. fn resolve_duckdb_include(bundled: bool) -> PathBuf { - if let Some(dir) = env::var_os("DEP_DUCKDB_INCLUDE") { - let p = PathBuf::from(dir); - if has_shim_headers(&p) { - return p; + let reported = env::var_os("DEP_DUCKDB_INCLUDE").map(PathBuf::from); + if let Some(p) = &reported { + if has_shim_headers(p) { + return p.clone(); } } if bundled { - find_duckdb_include_via_scan() + find_duckdb_include_via_scan(reported.as_deref()) } else { resolve_prebuilt_include_from_env() } @@ -290,7 +290,16 @@ fn resolve_prebuilt_include_from_env() -> PathBuf { /// compiled. This means `libduckdb-sys` (a regular dependency) may still be /// compiling when this function executes. We therefore poll with a timeout /// to wait for the headers to appear. -fn find_duckdb_include_via_scan() -> PathBuf { +/// +/// `reported` is the directory `libduckdb-sys` itself published as +/// `DEP_DUCKDB_INCLUDE`, if it did. Cargo sets that variable only once the +/// dependency's build script has finished (a fresh unit replays its recorded +/// output), so the headers are not about to appear and waiting for them is +/// pointless: a stale cache that restored the build record without the `out/` +/// directory made this script poll for ten minutes and then fail with a message +/// that said nothing about the path. With `reported` set, the tree is scanned +/// once and the error names the directory that was expected. +fn find_duckdb_include_via_scan(reported: Option<&Path>) -> PathBuf { // OUT_DIR = .../target/{profile}/build/quack-rs-{hash}/out // We want = .../target/{profile}/build/ let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); @@ -302,15 +311,28 @@ fn find_duckdb_include_via_scan() -> PathBuf { // Poll for up to ~10 minutes (libduckdb-sys compiles the full DuckDB C++ // source when the `bundled` feature is active, which can take several // minutes on CI runners). - for attempt in 0..120 { + let attempts: u32 = if reported.is_some() { 1 } else { 120 }; + for attempt in 0..attempts { if let Some(path) = scan_for_duckdb_headers(build_dir) { return path; } - if attempt < 119 { + if attempt + 1 < attempts { std::thread::sleep(std::time::Duration::from_secs(5)); } } + if let Some(dir) = reported { + panic!( + "libduckdb-sys reported its include directory as {}, but neither \ + duckdb_extension.h nor duckdb.hpp is there, and no libduckdb-sys-*/out/duckdb/src/include \ + exists under {}.\n\ + A restored build cache that kept libduckdb-sys's build record but not its `out/` \ + directory looks exactly like this: clear the cache (or `cargo clean -p libduckdb-sys`) \ + and build again.", + dir.display(), + build_dir.display() + ); + } panic!( "Could not find DuckDB headers from libduckdb-sys build output.\n\ Ensure that the `duckdb` dependency is resolved with `features = [\"bundled\"]`\n\ @@ -360,28 +382,37 @@ fn export_built_against_duckdb_version() { if let Ok(version) = env::var("QUACK_RS_TARGET_DUCKDB_VERSION") { let version = version.trim(); if !version.is_empty() { - // `abi::built_against_version` ignores anything but `vX.Y.Z` - // (a development build's `v1.5.6-dev123` included); say so here, + // Anything but `vX.Y.Z` is dropped here, not passed on: this value + // is printed into a `cargo:` directive, and cargo reads those a + // line at a time, so a value holding a newline (`v1.5.6\ncargo: + // rustc-cfg=...`) would add directives of its own (HARDEN-12). + // `abi::built_against_version` ignores such a value anyway (a + // development build's `v1.5.6-dev123` included); say so here, // where the declaration is made, rather than only in a LOAD error. if !is_release_version(version) { println!( - "cargo:warning=QUACK_RS_TARGET_DUCKDB_VERSION='{version}' is not a release \ + "cargo:warning=QUACK_RS_TARGET_DUCKDB_VERSION={version:?} is not a release \ version (vX.Y.Z) and is ignored: the ABI check treats this build as \ undeclared" ); + return; } println!("cargo:rustc-env=QUACK_RS_BUILT_AGAINST_DUCKDB={version}"); } } } -/// `vX.Y.Z` (the `v` optional) with three decimal parts: what -/// `abi::parse_version` accepts. +/// `vX.Y.Z` (the `v` optional) with three parts of ASCII digits, each a +/// `u64`: exactly what `abi::parse_version` accepts. Only such a value reaches +/// a `cargo:` directive. fn is_release_version(version: &str) -> bool { let parts: Vec<&str> = version .strip_prefix('v') .unwrap_or(version) .split('.') .collect(); - parts.len() == 3 && parts.iter().all(|p| p.parse::().is_ok()) + parts.len() == 3 + && parts.iter().all(|p| { + !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) && p.parse::().is_ok() + }) } diff --git a/clippy.toml b/clippy.toml index a845816f..cdf47900 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,16 +1,20 @@ # SPDX-License-Identifier: MIT # Copyright 2026 Tom F. (https://github.com/tomtom215) # -# Clippy configuration — pedantic + nursery lint levels. -# Lint levels are set in Cargo.toml [lints.clippy]; this file holds per-lint overrides. +# Clippy configuration. The lint levels (pedantic, nursery, cargo, and the +# few allowed lints) are set in Cargo.toml [lints.clippy]; this file holds +# per-lint thresholds. +# +# All three values below are clippy's own defaults (clippy_config/src/conf.rs +# at rust-1.99.0: 100, 7, 25), so today this file changes nothing. They are +# written down so that a future clippy changing a default does not silently +# change what this crate is held to; raising one is a decision for review. -# Minimum function line count before suggesting extraction. -# Set to 100 to accommodate FFI registration methods (e.g., builder::register) -# which are inherently sequential and resist extraction. +# Lines in a function before `too_many_lines` fires (pedantic). too-many-lines-threshold = 100 -# Maximum number of function arguments before warning +# Arguments a function may take before `too_many_arguments` fires. too-many-arguments-threshold = 7 -# Cognitive complexity threshold +# Score above which `cognitive_complexity` fires (nursery). cognitive-complexity-threshold = 25 diff --git a/deny.ci-crates.toml b/deny.ci-crates.toml new file mode 100644 index 00000000..b079b50d --- /dev/null +++ b/deny.ci-crates.toml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: MIT +# +# cargo-deny configuration for the repository's other dependency graphs: +# examples/hello-ext, fuzz, book/doctest (and benches/engine, which has no +# committed lockfile). `deny.toml` is written for the published crate and is +# deliberately strict (`unused-allowed-license = "deny"`), which cannot hold for +# graphs with a different set of licenses. Differences, and why: +# +# - NCSA is allowed: libfuzzer-sys is "(MIT OR Apache-2.0) AND NCSA" (LLVM's +# libFuzzer is vendored), so the fuzz graph cannot pass without it. +# - An unused allowed license is not an error here. +# - Crates marked `publish = false` need no license field (doctest, bench). +# +# for m in examples/hello-ext fuzz book/doctest; do +# cargo deny --manifest-path $m/Cargo.toml --config deny.ci-crates.toml check +# done +# +# Advisories, sources and bans are the same policy as deny.toml. + +[graph] +all-features = true + +[licenses] +confidence-threshold = 0.8 +unused-allowed-license = "allow" +private = { ignore = true } +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unicode-3.0", + "Zlib", + "CC0-1.0", + "CDLA-Permissive-2.0", + "NCSA", +] + +[advisories] +db-urls = ["https://github.com/rustsec/advisory-db"] +yanked = "deny" +unmaintained = "all" +unsound = "all" +ignore = [] + +[bans] +multiple-versions = "allow" +wildcards = "deny" +allow-wildcard-paths = true + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] diff --git a/deny.toml b/deny.toml index 911ef342..1587b3e0 100644 --- a/deny.toml +++ b/deny.toml @@ -48,6 +48,13 @@ db-urls = ["https://github.com/rustsec/advisory-db"] # soundness or security bug, and Cargo.lock is what CI and releases build. yanked = "deny" +# Unmaintained and unsound crates anywhere in the graph, not only direct +# dependencies (the default scope is "workspace"). No crate in any of the five +# lockfiles is flagged today, so this costs nothing and catches a transitive +# crate being abandoned. +unmaintained = "all" +unsound = "all" + # IDs of advisories to explicitly ignore (document reason in comment). # Example: ignore = ["RUSTSEC-2023-0001"] # false positive for our usage pattern ignore = [] @@ -59,8 +66,11 @@ ignore = [] # same unsafe crate with different soundness properties). multiple-versions = "warn" -# Wildcard version requirements (e.g., "*") in direct dependencies: warn. -wildcards = "warn" +# Wildcard version requirements (e.g., "*") in direct dependencies: deny. A path +# dependency without a version is not a wildcard in this sense (the examples and +# CI crates depend on the working copy by path). +wildcards = "deny" +allow-wildcard-paths = true # Highlight which dependency paths cause multiple-version warnings. highlight = "all" @@ -87,7 +97,11 @@ unknown-registry = "deny" # Git sources bypass the crates.io review pipeline and complicate auditing. unknown-git = "deny" -allow-registry = ["https://github.com/rust-lang/crates.io-index", "sparse+https://index.crates.io/"] +# Cargo.lock records crates.io as this URL whether the sparse or the git index +# was used, so this is the only form cargo-deny ever sees; listing +# `sparse+https://index.crates.io/` too only produced an `unmatched-source` +# warning on every run. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] # Uncomment to allow specific git sources (e.g., for unreleased patches): # allow-git = ["https://github.com/some-org/some-repo"] diff --git a/docs/architecture.md b/docs/architecture.md index aded3d7d..d2994075 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,7 +28,7 @@ quack_rs ├── callback scalar_callback!, table_scan_callback!, … — panic-safe callback wrapper macros ├── aggregate │ ├── state FfiState — tagged state slot: T inline (≤ 256 bytes, ≤ usize-aligned) or boxed -│ ├── callbacks Type aliases for the six aggregate callbacks (StateSizeFn, StateInitFn, UpdateFn, CombineFn, FinalizeFn, DestroyFn) +│ ├── callbacks Type aliases for the aggregate callbacks: the five required ones (StateSizeFn, StateInitFn, UpdateFn, CombineFn, FinalizeFn) and the optional DestroyFn │ ├── info AggregateFunctionInfo — callback info wrapper │ └── builder/ │ ├── single AggregateFunctionBuilder (single-signature) @@ -257,15 +257,15 @@ callback's `set_error`, or `access.set_error` at load. ### 3. Bounded version range -`libduckdb-sys = ">=1.4.4, <2"` — the range is intentional. Every release from -1.4.4 to 1.5.5 declares C API `v1.2.0` (v1.5.6 and v2.0.0 declare `v1.5.6` and +`libduckdb-sys = ">=1.4.4, <1.20100"` — the range is intentional. Every release from +1.4.4 to 1.5.5 declares C API `v1.2.0` (v1.5.6 and v2.0.x declare `v1.5.6` and still load `v1.2.0` extensions), and the `v1.2.0` stable part (357 functions) is unchanged across them through v2.0.0; the unstable region differs between releases -(459 / 545 / 546 / 548 functions) and is checked at load by `abi.rs`. The upper bound -does not exclude the DuckDB 2.0 bindings: `libduckdb-sys` encodes DuckDB -`vMAJOR.MINOR.PATCH` as `1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is -`1.20000.0` (published 2026-10-22; `rust-version` 1.88), which the range -admits. +(459 / 545 / 546 / 548 functions) and is checked at load by `abi.rs`. +`libduckdb-sys` encodes DuckDB `vMAJOR.MINOR.PATCH` as +`1.(10000·MAJOR + 100·MINOR + PATCH).N`, so DuckDB 2.0.0 is `1.20000.0` (published +2026-10-22; `rust-version` 1.88), which the range admits, and DuckDB 2.1.0 is +`1.20100.0`, the first release it excludes (see ADR-2). ### 4. Testable business logic @@ -289,9 +289,9 @@ on its `ScaffoldConfig` before it generates anything. ``` Extension crate ├── quack_rs (this crate) - │ ├── libduckdb-sys = ">=1.4.4, <2" { loadable-extension } - │ └── duckdb = ">=1.4.4, <2" (optional: only with `bundled-test` / `bundled-test-prebuilt`) - └── libduckdb-sys = ">=1.4.4, <2" { loadable-extension } + │ ├── libduckdb-sys = ">=1.4.4, <1.20100" { loadable-extension } + │ └── duckdb = ">=1.4.4, <1.20100" (optional: only with `bundled-test` / `bundled-test-prebuilt`) + └── libduckdb-sys = ">=1.4.4, <1.20100" { loadable-extension } └── (DuckDB headers only — no linked library) ``` @@ -466,16 +466,26 @@ DuckDB version). A DuckDB release can change the C API in ways that break an extension without a compile error: new function signatures, changed constant values, renamed symbols, or a shifted unstable region. -**Decision**: Use `libduckdb-sys = ">=1.4.4, <2"`. Every DuckDB release from -1.4.4 to 1.5.5 declares C extension API version `v1.2.0`; 1.5.6 and 2.0.0 declare +**Decision**: Use `libduckdb-sys = ">=1.4.4, <1.20100"`. Every DuckDB release from +1.4.4 to 1.5.5 declares C extension API version `v1.2.0`; 1.5.6 and 2.0.x declare `v1.5.6` and still load `v1.2.0` extensions. The stable part of the API (357 functions) is unchanged across all of them. The unstable region differs between -releases and is checked at load by the ABI guard (`abi.rs`). The `<2` upper -bound stops a future `libduckdb-sys` 2.x, but not the DuckDB 2.0 bindings: -`libduckdb-sys` numbers DuckDB 2.0.0 `1.20000.0` (the crate version is -`1.(10000·major + 100·minor + patch).N`), which the range admits. That release was -published on 2026-10-22 and declares `rust-version` 1.88, so a Rust 1.86 -build stays on `1.10506.x` under MSRV-aware resolution. +releases and is checked at load by the ABI guard (`abi.rs`). `libduckdb-sys` numbers +DuckDB 2.0.0 `1.20000.0` (the crate version is `1.(10000·major + 100·minor + patch).N`), +which the range admits. That release was published on 2026-10-22 and declares +`rust-version` 1.88, so a Rust 1.86 build stays on `1.10506.x` under MSRV-aware +resolution. + +The upper bound was `<2` until 0.19, which did not exclude the DuckDB 2.0 bindings and +admitted every later one. It is now `<1.20100`, the DuckDB 2.1 bindings, because the +1.20000 bindings keep the C API names at the crate root only through a compatibility +re-export (`pub use v1::*`, marked `// TODO: Remove in the future` upstream, checked +2026-10-30 in `libduckdb-sys` 1.20000.1). quack-rs mentions those root paths about 500 times +in 108 files, and its exported macros hard-code them (`::libduckdb_sys::duckdb_function_info` +and others), so a release without the re-export breaks quack-rs and every extension that +uses its macros. In the eighth review, a copy of 1.20000.1 with the re-export deleted made +quack-rs fail to compile with 214 errors. Each new DuckDB minor therefore needs a quack-rs release that has +been built and tested against its bindings and engine before the range is widened. CI checks the range in two ways (`.github/workflows/ci.yml`): @@ -543,7 +553,7 @@ changed. `quack-rs` instead uses a hand-written C entry point that calls encapsulate it. The original reason no longer holds: `duckdb-loadable-macros` 1.20000.1 uses -`Connection::open_from_raw` and contains no `extract_raw_connection` (checked +`duckdb::Connection::open_from_raw` and contains no `extract_raw_connection` (checked 2026-10-29 against the published crate). The decision stands for other reasons. The macro's host crate wraps only scalar and table functions; quack-rs wraps more function kinds; it guards callbacks against panics and checks the C API diff --git a/docs/E2E_DOGFOOD_REPORT.md b/docs/archive/E2E_DOGFOOD_REPORT.md similarity index 100% rename from docs/E2E_DOGFOOD_REPORT.md rename to docs/archive/E2E_DOGFOOD_REPORT.md diff --git a/docs/duckdb-v1.5.1-evaluation.md b/docs/archive/duckdb-v1.5.1-evaluation.md similarity index 97% rename from docs/duckdb-v1.5.1-evaluation.md rename to docs/archive/duckdb-v1.5.1-evaluation.md index 14a1eed7..9e6b688d 100644 --- a/docs/duckdb-v1.5.1-evaluation.md +++ b/docs/archive/duckdb-v1.5.1-evaluation.md @@ -11,19 +11,20 @@ > does not keep out the DuckDB 2.0 bindings, as the risk table below assumes: > `libduckdb-sys` numbers DuckDB 2.0.0 `1.20000.0`, which the range admits. The > C API version the extension's `v1.2.0` stamp targets is still accepted by -> DuckDB 2.0.0. +> DuckDB 2.0.0. *Note added with 0.20.0:* the bound is now `<1.20100`, which keeps +> out the DuckDB 2.1 bindings (see the Installation chapter of the book). > > For the current state, see: > > - Supported DuckDB releases, the `libduckdb-sys` range and the MSRV: the -> [README](../README.md#supported-versions) and `Cargo.toml`. +> [README](../../README.md#supported-versions) and `Cargo.toml`. > - The `libduckdb-sys` release the tests build against: `Cargo.lock` (CI > derives the DuckDB release from it with `scripts/duckdb-version-from-lock.sh`). > - The DuckDB releases CI tests and loads into: the `test-older-engines` and > `extension-load` jobs in `.github/workflows/ci.yml`. > - The C extension API version: `quack_rs::DUCKDB_API_VERSION` (`src/lib.rs`) -> and the [ABI Compatibility](../book/src/concepts/abi.md) chapter. -> - Open limitations: the [Known Limitations](../book/src/reference/known-limitations.md) +> and the [ABI Compatibility](../../book/src/concepts/abi.md) chapter. +> - Open limitations: the [Known Limitations](../../book/src/reference/known-limitations.md) > reference. **Date:** 2026-03-27 diff --git a/docs/ffi-reference.md b/docs/ffi-reference.md index 2d7d0bea..08db3fda 100644 --- a/docs/ffi-reference.md +++ b/docs/ffi-reference.md @@ -1,5 +1,8 @@ # DuckDB C API Reference for Extension Authors +> **These code blocks are sketches, not compiled.** They show the C API shapes the SDK wraps; +> the book's examples are compiled on every pull request. + This page documents the DuckDB C API surface used by `quack-rs`, including correct callback signatures, memory ownership rules, and gotchas discovered through production use. The [LESSONS.md](../LESSONS.md) companion file documents @@ -22,7 +25,7 @@ each pitfall in detail; this page is the quick-reference. ## Extension Entry Point -```rust +```rust,ignore // The entry point function name must be: {extension_name}_init_c_api // (all lowercase, underscores, no "lib" prefix) #[unsafe(no_mangle)] @@ -56,7 +59,7 @@ duckdb_disconnect(&mut con) *C API version* (`"v1.2.0"`), not the DuckDB release version (`"v1.4.4"`). DuckDB 1.4.4 to 1.5.5 declare C API `v1.2.0`; 1.5.6 and 2.0.0 declare `v1.5.6` and still load `v1.2.0` extensions. `quack_rs::DUCKDB_API_VERSION` holds `"v1.2.0"`. See -[Pitfall P2](../LESSONS.md#p2-extension-metadata-version-is-c-api-version-not-duckdb-version). +[Pitfall P2](../LESSONS.md#p2-metadata-version-is-c-api-version-not-duckdb-version). --- @@ -64,7 +67,7 @@ duckdb_disconnect(&mut con) ### Single function -```rust +```rust,ignore unsafe { ScalarFunctionBuilder::new("double_it") .param(TypeId::BigInt) @@ -76,7 +79,7 @@ unsafe { ### Function set (multiple overloads) -```rust +```rust,ignore unsafe { ScalarFunctionSetBuilder::new("my_add") .overload( @@ -97,7 +100,7 @@ unsafe { ### Complex parameter and return types -```rust +```rust,ignore use quack_rs::types::{LogicalType, TypeId}; ScalarFunctionBuilder::new("flatten_list") @@ -107,7 +110,7 @@ ScalarFunctionBuilder::new("flatten_list") .register(con)? ``` -```rust +```rust,ignore ScalarOverloadBuilder::new() .param(TypeId::Varchar) .returns_logical(LogicalType::list(TypeId::Timestamp)) // LIST(TIMESTAMP) return @@ -116,7 +119,7 @@ ScalarOverloadBuilder::new() ### NULL handling -```rust +```rust,ignore ScalarFunctionBuilder::new("my_fn") // Declares the function may return non-NULL for NULL input. The callback // receives NULL rows under either setting (Pitfall L8). @@ -130,7 +133,7 @@ ScalarFunctionBuilder::new("my_fn") ### Single function -```rust +```rust,ignore unsafe { AggregateFunctionBuilder::new("my_agg") .param(TypeId::Varchar) // one call per input parameter @@ -147,7 +150,7 @@ unsafe { ### Function set (multiple arities) -```rust +```rust,ignore unsafe { AggregateFunctionSetBuilder::new("retention") .returns_logical(LogicalType::list(TypeId::Boolean)) // LIST(BOOLEAN) return @@ -179,7 +182,7 @@ The following are the correct Rust signatures for DuckDB aggregate callbacks. Note that `update`, `combine`, and `finalize` take `*mut duckdb_aggregate_state` (a pointer to an **array** of state pointers), **not** `duckdb_aggregate_state`. -```rust +```text // state_size: called once; returns sizeof(your FFI state struct) unsafe extern "C" fn state_size( info: duckdb_function_info, @@ -231,7 +234,7 @@ unsafe extern "C" fn destroy( ### Reading input vectors (inside `update`) -```rust +```rust,ignore // Get column i from the input chunk let row_count = duckdb_data_chunk_get_size(chunk) as usize; let vector = duckdb_data_chunk_get_vector(chunk, col_idx as idx_t); @@ -252,7 +255,7 @@ let b = *data.add(row) != 0; ### Writing output vectors (inside `finalize`) -```rust +```rust,ignore let data = duckdb_vector_get_data(result) as *mut u8; // Write a value @@ -290,7 +293,7 @@ bytes 8..16 : pointer to heap-allocated string data (*const u8), native may hold stale bytes ``` -```rust +```rust,ignore // Correct reading: let bytes: [u8; 16] = ptr::read(data.add(row * 16) as *const [u8; 16]); let len = u32::from_ne_bytes(bytes[..4].try_into().unwrap()) as usize; @@ -355,7 +358,7 @@ dealing with these constants directly. ## Common Mistakes Quick Reference This page does not keep its own copy of the pitfall list. Every known pitfall -(L1–L19 and P1–P15), with symptom, root cause and fix, is in +(L1–L21 and P1–P17), with symptom, root cause and fix, is in [`LESSONS.md`](../LESSONS.md); the book's [Pitfall Catalog](https://quack-rs.com/reference/pitfalls.html) has the same entries with compiled examples and a summary table. diff --git a/examples/hello-ext/Cargo.lock b/examples/hello-ext/Cargo.lock index 283e9ca4..d6a861e5 100644 --- a/examples/hello-ext/Cargo.lock +++ b/examples/hello-ext/Cargo.lock @@ -8,30 +8,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - [[package]] name = "cc" version = "1.8.0" @@ -48,12 +30,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "crc32fast" version = "1.5.2" @@ -63,12 +39,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "errno" version = "0.3.14" @@ -76,7 +46,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -106,23 +76,6 @@ dependencies = [ "zlib-rs", ] -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - [[package]] name = "hello-ext" version = "0.1.0" @@ -131,32 +84,6 @@ dependencies = [ "quack-rs", ] -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "indexmap" -version = "2.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", - "hashbrown", -] - [[package]] name = "itoa" version = "1.0.18" @@ -171,21 +98,17 @@ checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libduckdb-sys" -version = "1.10506.1" +version = "1.20000.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113d1c26c9f518405a17b5729bb6ab6abf029fdc46626406922cf9a8dc9b4609" +checksum = "0140448c472968ee5cf5ba697435deebe558b366ef1e5c080d24d9696b600725" dependencies = [ "flate2", - "pkg-config", "prettyplease", "quote", "serde", "serde_json", "syn 2.0.119", "tar", - "ureq", - "vcpkg", - "zip", ] [[package]] @@ -194,12 +117,6 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - [[package]] name = "memchr" version = "2.8.3" @@ -216,36 +133,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - [[package]] name = "prettyplease" version = "0.2.37" @@ -267,7 +154,7 @@ dependencies = [ [[package]] name = "quack-rs" -version = "0.19.0" +version = "0.20.0" dependencies = [ "cc", "libduckdb-sys", @@ -282,20 +169,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rustix" version = "1.1.5" @@ -306,42 +179,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" -dependencies = [ - "log", - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", + "windows-sys", ] [[package]] @@ -399,12 +237,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.119" @@ -438,94 +270,18 @@ dependencies = [ "xattr", ] -[[package]] -name = "typed-path" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" - [[package]] name = "unicode-ident" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "ureq" -version = "3.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" -dependencies = [ - "base64", - "log", - "percent-encoding", - "rustls", - "rustls-pki-types", - "ureq-proto", - "utf8-zero", - "webpki-roots", -] - -[[package]] -name = "ureq-proto" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" -dependencies = [ - "base64", - "http", - "httparse", - "log", -] - -[[package]] -name = "utf8-zero" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "webpki-roots" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -535,70 +291,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "xattr" version = "1.6.1" @@ -609,27 +301,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "zeroize" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" - -[[package]] -name = "zip" -version = "9.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" -dependencies = [ - "crc32fast", - "flate2", - "indexmap", - "memchr", - "nix", - "typed-path", - "zopfli", -] - [[package]] name = "zlib-rs" version = "0.6.8" @@ -641,15 +312,3 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] diff --git a/examples/hello-ext/Cargo.toml b/examples/hello-ext/Cargo.toml index dc880398..d19b1c2f 100644 --- a/examples/hello-ext/Cargo.toml +++ b/examples/hello-ext/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" rust-version = "1.86.0" authors = ["Tom F"] license = "MIT" +publish = false description = "Example DuckDB community extension built with quack-rs: word_count, first_word, generate_series_ext, VARCHAR→INTEGER cast" [lib] @@ -13,7 +14,7 @@ crate-type = ["cdylib"] [dependencies] quack_rs = { path = "../..", package = "quack-rs" } -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } [profile.release] # `unwind`, not `abort`. quack-rs catches panics at every FFI boundary with diff --git a/examples/hello-ext/README.md b/examples/hello-ext/README.md index d769d945..04d0435a 100644 --- a/examples/hello-ext/README.md +++ b/examples/hello-ext/README.md @@ -7,18 +7,20 @@ implementation or copy-paste starting point for your own extension. Features that need the `duckdb-1-5*` flags (copy functions, Arrow, config options, …) are not in it; their chapters in the book have examples. -> **This example uses the raw callback style on purpose.** Its 28 +> **This example mostly uses the raw callback style on purpose.** Its 24 > `unsafe extern "C" fn`s are installed exactly as written and run with no > `catch_unwind`: a panic in one aborts the DuckDB process. It shows what the > C API calls look like, not how to write new code. For that, use the typed > builders (`ScalarFunctionBuilder::map1` / `try_map1`, which return a SQL error > for a panic) and the `*_callback!` macros, as the -> [Quick Start](https://quack-rs.com/getting-started/quick-start.html) does. +> [Quick Start](https://quack-rs.com/getting-started/quick-start.html) does — +> and as `gen_series_v2` does here: its four callbacks are written with +> `table_bind_callback!`, `table_init_callback!` and `table_scan_callback!`. > **The aggregates here (`word_count`, `typed_sum`) must not be used as > `agg(x) OVER ()` or `agg(x ORDER BY y)`.** A DuckDB defect crashes every C API > aggregate in those two shapes — see -> [Pitfall L11](../../LESSONS.md#l11-c-api-aggregates-crash-under-aggx-over--and-aggx-order-by-y). +> [Pitfall L11](https://quack-rs.com/reference/pitfalls.html#l11-c-api-aggregates-crash-under-aggx-over--and-aggx-order-by-y). ## What it registers @@ -45,7 +47,7 @@ are not in it; their chapters in the book have examples. All functions use `entry_point_v2!` with `Connection`/`Registrar` for type-safe registration. -All **29 numbered SQL checks** (31 statements, listed under "Load in DuckDB CLI" below) +All **30 numbered SQL checks** (32 statements, listed under "Load in DuckDB CLI" below) must return their expected results. CI's `extension-load` job runs every one of them with `scripts/check-hello-ext.py` on **DuckDB 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and the latest release** (2.0.0 at the time of writing). @@ -185,7 +187,7 @@ duckdb -unsigned ```sql LOAD 'hello_ext.duckdb_extension'; --- 29 numbered checks (31 statements); CI runs them on DuckDB 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and latest: +-- 30 numbered checks (32 statements); CI runs them on DuckDB 1.4.4, 1.5.0, 1.5.5, 1.5.6, 2.0.0 and latest: -- T01: word_count aggregate SELECT word_count(sentence) AS wc FROM ( @@ -265,6 +267,9 @@ SELECT make_kv_map(NULL, 42); -- NULL -- T29: gen_series_v2 projection pushdown (value column only) SELECT value FROM gen_series_v2(3); -- 0, 1, 2 + +-- T30: gen_series_v2 refuses a value past BIGINT (2 * 2^62) instead of wrapping +SELECT * FROM gen_series_v2(3, step := 4611686018427387904); -- error ``` ## Adapting this for your own extension diff --git a/examples/hello-ext/sql_checks.txt b/examples/hello-ext/sql_checks.txt index 41c5fc00..654cdf43 100644 --- a/examples/hello-ext/sql_checks.txt +++ b/examples/hello-ext/sql_checks.txt @@ -170,3 +170,8 @@ SELECT value FROM gen_series_v2(3); 0 1 2 + +== T30 +SELECT * FROM gen_series_v2(3, step := 4611686018427387904); +---- +ERROR: Invalid Input Error: gen_series_v2: value 2 * step 4611686018427387904 is out of range for BIGINT diff --git a/examples/hello-ext/src/lib.rs b/examples/hello-ext/src/lib.rs index 4f64e86d..4ff676bb 100644 --- a/examples/hello-ext/src/lib.rs +++ b/examples/hello-ext/src/lib.rs @@ -48,7 +48,7 @@ use quack_rs::interval::DuckInterval; use quack_rs::replacement_scan::ReplacementScanInfo; use quack_rs::scalar::{ScalarFunctionBuilder, ScalarFunctionSetBuilder, ScalarOverloadBuilder}; use quack_rs::sql_macro::SqlMacro; -use quack_rs::table::{BindInfo, FfiBindData, FfiInitData, FfiLocalInitData, InitInfo, TableFunctionBuilder}; +use quack_rs::table::{BindInfo, FfiBindData, FfiInitData, FfiLocalInitData, FunctionInfo, InitInfo, TableFunctionBuilder}; use quack_rs::types::{LogicalType, NullHandling, TypeId}; use quack_rs::vector::complex::{ListVector, MapVector, StructVector}; use quack_rs::vector::validity::ValidityBitmap; @@ -296,93 +296,137 @@ struct GsV2LocalState { emitted: i64, } -unsafe extern "C" fn gs_v2_bind(info: duckdb_bind_info) { - unsafe { - let bind_info = BindInfo::new(info); - // Value is RAII — automatically destroyed when dropped. - // A NULL argument (or a handle DuckDB did not supply) yields 0 rows. - let n = bind_info.get_parameter_value(0).as_i64_or(0); - - // Read named param "step" if provided, default to 1. `as_i64` is - // `None` when the parameter is absent or `step := NULL`. - let step = match bind_info.get_named_parameter_value("step").as_i64() { - Some(0) | None => 1i64, - Some(s) => s, - }; - - let total = n.max(0); - bind_info - .add_result_column("value", TypeId::BigInt) - .add_result_column("step_used", TypeId::BigInt) - .set_cardinality(total as u64, true); - FfiBindData::::set(info, GsV2BindData { total, step }); - } +/// The `offset`th value of `gen_series_v2` with step `step`, or `None` when it +/// is past BIGINT. Wrapping (the release-mode `*`) returned a wrong number. +fn gs_v2_value(offset: i64, step: i64) -> Option { + offset.checked_mul(step) } -unsafe extern "C" fn gs_v2_init(info: duckdb_init_info) { - unsafe { - let bind_data = FfiBindData::::get_from_init(info); - let (total, step) = bind_data.map_or((0, 1), |d| (d.total, d.step)); - - // Demonstrate InitInfo: set_max_threads - let init_info = InitInfo::new(info); - init_info.set_max_threads(1); +// The callbacks go through quack-rs's guard macros: a panic in one becomes a +// query error instead of unwinding into DuckDB (which aborts the process). + +quack_rs::table_bind_callback!(gs_v2_bind, |info| { + // SAFETY: `info` is the bind info DuckDB passed to this callback. + let bind_info = unsafe { BindInfo::new(info) }; + // Value is RAII — automatically destroyed when dropped. + // A NULL argument (or a handle DuckDB did not supply) yields 0 rows. + // SAFETY: the function declares one positional parameter, so index 0 exists. + let n = unsafe { bind_info.get_parameter_value(0) }.as_i64_or(0); + + // Read named param "step" if provided, default to 1. `as_i64` is + // `None` when the parameter is absent or `step := NULL`. + // SAFETY: "step" is the named parameter the function declares. + let step = match unsafe { bind_info.get_named_parameter_value("step") }.as_i64() { + Some(0) | None => 1i64, + Some(s) => s, + }; + let total = n.max(0); + bind_info + .add_result_column("value", TypeId::BigInt) + .add_result_column("step_used", TypeId::BigInt) + .set_cardinality(total.unsigned_abs(), true); + // SAFETY: `info` is the bind info of this callback. + unsafe { FfiBindData::::set(info, GsV2BindData { total, step }) }; +}); + +quack_rs::table_init_callback!(gs_v2_init, |info| { + // SAFETY: `info` is the init info DuckDB passed to this callback, and the + // bind data was set by `gs_v2_bind` as a `GsV2BindData`. + let bind_data = unsafe { FfiBindData::::get_from_init(info) }; + let (total, step) = bind_data.map_or((0, 1), |d| (d.total, d.step)); + + // Demonstrate InitInfo: set_max_threads + // SAFETY: as above. + let init_info = unsafe { InitInfo::new(info) }; + init_info.set_max_threads(1); + + // SAFETY: as above. + unsafe { FfiInitData::::set(info, GsV2GlobalState { next_offset: 0, total, step, }); } -} +}); -unsafe extern "C" fn gs_v2_local_init(info: duckdb_init_info) { +quack_rs::table_init_callback!(gs_v2_local_init, |info| { + // Demonstrates FfiLocalInitData for per-thread state. + // In single-threaded mode this is called once. + // SAFETY: `info` is the local init info DuckDB passed to this callback. unsafe { - // Demonstrates FfiLocalInitData for per-thread state. - // In single-threaded mode this is called once. FfiLocalInitData::::set(info, GsV2LocalState { local_start: 0, local_count: 0, emitted: 0, }); } -} +}); + +quack_rs::table_scan_callback!(gs_v2_scan, |info, output| { + // SAFETY: `output` is the chunk DuckDB passed to this scan callback. + let chunk = unsafe { DataChunk::from_raw(output) }; + // SAFETY: `info` is this scan's function info; the global state was set + // by `gs_v2_init` as a `GsV2GlobalState`. + let Some(global) = (unsafe { FfiInitData::::get_mut(info) }) else { + // SAFETY: `output` is valid for this call. + unsafe { chunk.set_size(0) }; + return; + }; -unsafe extern "C" fn gs_v2_scan(info: duckdb_function_info, output: duckdb_data_chunk) { - unsafe { - let chunk = DataChunk::from_raw(output); - let Some(global) = FfiInitData::::get_mut(info) else { - chunk.set_size(0); + // SAFETY: a plain query of DuckDB's vector size. + let vector_size = unsafe { duckdb_vector_size() } as i64; + let remaining = global.total - global.next_offset; + if remaining <= 0 { + // SAFETY: `output` is valid for this call. + unsafe { chunk.set_size(0) }; + return; + } + let batch_size = remaining.min(vector_size) as usize; + let start = global.next_offset; + let step = global.step; + + // Every value first: `start + i` stays below `total` (an i64), but the + // product can leave BIGINT, which is an error, not a wrapped number. + let mut values = Vec::with_capacity(batch_size); + for i in 0..batch_size as i64 { + let Some(value) = gs_v2_value(start + i, step) else { + // SAFETY: `info` is this scan's function info. + let function_info = unsafe { FunctionInfo::new(info) }; + function_info.set_error(&format!( + "gen_series_v2: value {} * step {step} is out of range for BIGINT", + start + i + )); + // SAFETY: `output` is valid for this call. + unsafe { chunk.set_size(0) }; return; }; + values.push(value); + } + global.next_offset += batch_size as i64; - let vector_size = duckdb_vector_size() as i64; - let remaining = global.total - global.next_offset; - if remaining <= 0 { - chunk.set_size(0); - return; - } - let batch_size = remaining.min(vector_size) as usize; - let start = global.next_offset; - let step = global.step; - global.next_offset += batch_size as i64; - - // Demonstrate FfiLocalInitData::get_mut in scan - if let Some(local) = FfiLocalInitData::::get_mut(info) { - local.local_start = start; - local.local_count = batch_size as i64; - local.emitted += batch_size as i64; - } + // Demonstrate FfiLocalInitData::get_mut in scan + // SAFETY: `info` is this scan's function info; the local state was set by + // `gs_v2_local_init` as a `GsV2LocalState`. + if let Some(local) = unsafe { FfiLocalInitData::::get_mut(info) } { + local.local_start = start; + local.local_count = batch_size as i64; + local.emitted += batch_size as i64; + } + // SAFETY: the bind callback declared two BIGINT columns, and every row + // index is below the vector size. + unsafe { let mut writer0 = chunk.writer(0); let mut writer1 = chunk.writer(1); - for i in 0..batch_size { - writer0.write_i64(i, (start + i as i64) * step); + for (i, value) in values.into_iter().enumerate() { + writer0.write_i64(i, value); writer1.write_i64(i, step); } chunk.set_size(batch_size); } -} +}); // ============================================================================ // Cast function: CAST(VARCHAR AS INTEGER) / TRY_CAST(VARCHAR AS INTEGER) @@ -1441,9 +1485,8 @@ mod tests { #[test] fn gen_series_v2_step_logic() { // With step=2, values should be 0, 2, 4 - let step = 2i64; - let values: Vec = (0..3).map(|i| i * step).collect(); - assert_eq!(values, vec![0, 2, 4]); + let values: Vec> = (0..3).map(|i| gs_v2_value(i, 2)).collect(); + assert_eq!(values, vec![Some(0), Some(2), Some(4)]); } #[test] @@ -1452,4 +1495,16 @@ mod tests { let effective_step = if step == 0 { 1 } else { step }; assert_eq!(effective_step, 1); } + + /// FFI-9: `(start + i) * step` wrapped in a release build, so + /// `gen_series_v2(3, step := 4611686018427387904)` returned + /// -9223372036854775808 as its third value. + #[test] + fn gen_series_v2_refuses_a_value_past_bigint() { + let half = 1i64 << 62; + assert_eq!(gs_v2_value(1, half), Some(half)); + assert_eq!(gs_v2_value(2, half), None); + assert_eq!(gs_v2_value(i64::MAX, -1), Some(-i64::MAX)); + assert_eq!(gs_v2_value(2, i64::MIN), None); + } } diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 3b4790bc..5234222b 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -262,9 +262,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "jobserver", @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "quack-rs" -version = "0.19.0" +version = "0.20.0" dependencies = [ "cc", "duckdb", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 84098a53..01cf44ca 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -19,7 +19,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" # The same bindings quack-rs uses (fuzz/Cargo.lock pins the release), for the # raw `duckdb_create_*` constructors in `value_render`. -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } [dependencies.quack-rs] path = ".." @@ -57,5 +57,33 @@ test = false doc = false bench = false +[[bin]] +name = "append_metadata" +path = "fuzz_targets/append_metadata.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "scaffold_text" +path = "fuzz_targets/scaffold_text.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "description_roundtrip" +path = "fuzz_targets/description_roundtrip.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "abi_version" +path = "fuzz_targets/abi_version.rs" +test = false +doc = false +bench = false + [profile.release] debug = true diff --git a/fuzz/fuzz_targets/abi_version.rs b/fuzz/fuzz_targets/abi_version.rs new file mode 100644 index 00000000..6ed80e23 --- /dev/null +++ b/fuzz/fuzz_targets/abi_version.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +//! `abi::parse_version` against a strict model of its documented grammar, and +//! the table lookups built on it. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use quack_rs::abi::{expected_slot_count, parse_version}; + +fn model(s: &str) -> Option<(u64, u64, u64)> { + let body = s.strip_prefix('v').unwrap_or(s); + let parts: Vec<&str> = body.split('.').collect(); + if parts.len() != 3 { + return None; + } + let mut n = [0u64; 3]; + for (slot, p) in n.iter_mut().zip(&parts) { + if p.is_empty() || !p.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + *slot = p.parse().ok()?; + } + Some((n[0], n[1], n[2])) +} + +fuzz_target!(|data: &[u8]| { + let Ok(s) = std::str::from_utf8(data) else { + return; + }; + assert_eq!(parse_version(s), model(s), "{s:?}"); + let _ = expected_slot_count(s); +}); diff --git a/fuzz/fuzz_targets/append_metadata.rs b/fuzz/fuzz_targets/append_metadata.rs new file mode 100644 index 00000000..e859372a --- /dev/null +++ b/fuzz/fuzz_targets/append_metadata.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MIT +//! Fuzzes `append_metadata`'s input handling: the footer detector, the +//! entry-point string-table scan, the stamp round trip and the command-line +//! parser. The binary's modules are included by path (it has no library). +#![no_main] +#![allow(dead_code)] + +#[path = "../../src/bin/append_metadata/cli.rs"] +mod cli; +#[path = "../../src/bin/append_metadata/entry.rs"] +mod entry; +#[path = "../../src/bin/append_metadata/footer.rs"] +mod footer; + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + // 1. Any bytes: the detector and the scan must return, and agree with themselves. + if let Some(n) = footer::existing_stamp_len(data) { + assert!( + n == footer::METADATA_SIZE + || n == footer::METADATA_SIZE + footer::WASM_SECTION_HEADER.len() + ); + assert!(n <= data.len()); + } + let names = entry::entry_point_names(data); + assert!(names.len() <= 8); + for n in &names { + assert!( + !n.is_empty() && n.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'), + "{n:?}" + ); + } + let _ = entry::entry_point_note(data, std::path::Path::new("x.duckdb_extension")); + + // 2. Stamp round trip: a stamped file is recognised, with the length stamped. + let block = + footer::build_metadata("C_STRUCT", "v0.1.0", "v1.2.0", "linux_amd64").expect("valid"); + let mut stamped = data.to_vec(); + stamped.extend_from_slice(&block); + // 534, not 512, when the payload itself ends in the 22-byte WebAssembly + // section header (the detector reads that header as part of the stamp). + match footer::existing_stamp_len(&stamped) { + Some(n) if n == footer::METADATA_SIZE => {} + Some(n) if n == footer::METADATA_SIZE + footer::WASM_SECTION_HEADER.len() => { + assert!(data.ends_with(&footer::WASM_SECTION_HEADER)); + } + other => panic!("a stamped file is not recognised as stamped: {other:?}"), + } + // The names found before stamping are still found after (the footer adds none). + assert!(entry::entry_point_names(&stamped).len() >= names.len().min(8)); + + // 3. Arguments: NUL-separated strings from the input. + let args: Vec = String::from_utf8_lossy(data) + .split('\0') + .map(str::to_owned) + .collect(); + let _ = cli::parse(&args); +}); diff --git a/fuzz/fuzz_targets/description_roundtrip.rs b/fuzz/fuzz_targets/description_roundtrip.rs new file mode 100644 index 00000000..cdc754d4 --- /dev/null +++ b/fuzz/fuzz_targets/description_roundtrip.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +//! A double-quoted YAML scalar, encoded the way JSON encodes a string, must +//! decode to the text it encodes through `parse_description_yml`. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use quack_rs::validate::description_yml::parse_description_yml; + +fn dq(s: &str) -> String { + let mut out = String::from("\""); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + // PyYAML refuses a file holding a character outside its printable + // set (C0 controls but TAB/LF/CR, DEL, the C1 controls but NEL, + // U+FFFE, U+FFFF) even inside quotes; an escape is fine. + c if (c as u32) < 0x20 + || matches!( + c, + '\u{7f}'..='\u{9f}' | '\u{fffe}' | '\u{ffff}' | '\u{2028}' | '\u{2029}' + ) => + { + out.push_str(&format!("\\u{:04x}", c as u32)) + } + c => out.push(c), + } + } + out.push('"'); + out +} + +fuzz_target!(|data: &[u8]| { + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + if text.trim().is_empty() { + return; + } + let yml = format!( + "extension:\n name: my_ext\n description: {}\n version: 0.1.0\n language: Rust\n build: cargo\n license: MIT\n maintainers:\n - {}\nrepo:\n github: a/b\n ref: 4f1c2e9a7b3d5c8e1f0a2b4c6d8e0f1a3b5c7d9e\n", + dq(text), dq(text) + ); + let parsed = parse_description_yml(&yml).unwrap_or_else(|e| panic!("{text:?}: {e}")); + assert_eq!(parsed.description, text); + // Maintainer names are trimmed by the reader; the description is not. + assert_eq!(parsed.maintainers, vec![text.trim().to_string()]); +}); diff --git a/fuzz/fuzz_targets/scaffold_text.rs b/fuzz/fuzz_targets/scaffold_text.rs new file mode 100644 index 00000000..67b1c29f --- /dev/null +++ b/fuzz/fuzz_targets/scaffold_text.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: MIT +//! Fuzzes the scaffold's escaping: for any free text `generate_scaffold` +//! accepts, `description.yml` must parse back to exactly that text, and every +//! description line must stay inside the `//!` header of `src/lib.rs`. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use quack_rs::scaffold::{generate_scaffold, ScaffoldConfig}; +use quack_rs::validate::description_yml::parse_description_yml; + +fuzz_target!(|data: &[u8]| { + let Ok(text) = std::str::from_utf8(data) else { + return; + }; + let (description, maintainer) = text.split_once('\u{1}').unwrap_or((text, "Jane Doe")); + let config = ScaffoldConfig { + name: "my_ext".to_string(), + description: description.to_string(), + maintainer: maintainer.to_string(), + github_repo: "janedoe/duckdb-my-ext".to_string(), + ..ScaffoldConfig::default() + }; + let Ok(files) = generate_scaffold(&config) else { + return; + }; + let file = |p: &str| files.iter().find(|f| f.path == p).expect(p).content.clone(); + let yml = file("description.yml"); + let parsed = + parse_description_yml(&yml).unwrap_or_else(|e| panic!("{description:?}: {e}\n{yml}")); + assert_eq!(parsed.description, description, "{yml}"); + assert_eq!(parsed.maintainers, vec![maintainer.to_string()], "{yml}"); + let lib = file("src/lib.rs"); + let header: Vec<&str> = lib + .split('\n') + .take_while(|l| !l.starts_with("use ")) + .filter(|l| !l.is_empty()) + .collect(); + assert!( + header.iter().all(|l| l.starts_with("//!")), + "{description:?}\n{lib}" + ); + assert!(!lib.contains('\r')); +}); diff --git a/fuzz/seeds/abi_version/development_suffix b/fuzz/seeds/abi_version/development_suffix new file mode 100644 index 00000000..8d4a4573 --- /dev/null +++ b/fuzz/seeds/abi_version/development_suffix @@ -0,0 +1 @@ +v1.5.6-dev \ No newline at end of file diff --git a/fuzz/seeds/abi_version/empty_part b/fuzz/seeds/abi_version/empty_part new file mode 100644 index 00000000..b3c95063 --- /dev/null +++ b/fuzz/seeds/abi_version/empty_part @@ -0,0 +1 @@ +1..2 \ No newline at end of file diff --git a/fuzz/seeds/abi_version/largest_part b/fuzz/seeds/abi_version/largest_part new file mode 100644 index 00000000..5c194565 --- /dev/null +++ b/fuzz/seeds/abi_version/largest_part @@ -0,0 +1 @@ +v18446744073709551615.0.1 \ No newline at end of file diff --git a/fuzz/seeds/abi_version/release_with_v b/fuzz/seeds/abi_version/release_with_v new file mode 100644 index 00000000..620d7574 --- /dev/null +++ b/fuzz/seeds/abi_version/release_with_v @@ -0,0 +1 @@ +v1.5.6 \ No newline at end of file diff --git a/fuzz/seeds/abi_version/release_without_v b/fuzz/seeds/abi_version/release_without_v new file mode 100644 index 00000000..359a5b95 --- /dev/null +++ b/fuzz/seeds/abi_version/release_without_v @@ -0,0 +1 @@ +2.0.0 \ No newline at end of file diff --git a/fuzz/seeds/abi_version/signed_part b/fuzz/seeds/abi_version/signed_part new file mode 100644 index 00000000..ecd2be91 --- /dev/null +++ b/fuzz/seeds/abi_version/signed_part @@ -0,0 +1 @@ +1.5.+4 \ No newline at end of file diff --git a/fuzz/seeds/append_metadata/arguments_nul_separated b/fuzz/seeds/append_metadata/arguments_nul_separated new file mode 100644 index 00000000..a5102767 Binary files /dev/null and b/fuzz/seeds/append_metadata/arguments_nul_separated differ diff --git a/fuzz/seeds/append_metadata/elf_symbols b/fuzz/seeds/append_metadata/elf_symbols new file mode 100644 index 00000000..4960fddb Binary files /dev/null and b/fuzz/seeds/append_metadata/elf_symbols differ diff --git a/fuzz/seeds/append_metadata/full_command_line b/fuzz/seeds/append_metadata/full_command_line new file mode 100644 index 00000000..65589695 Binary files /dev/null and b/fuzz/seeds/append_metadata/full_command_line differ diff --git a/fuzz/seeds/append_metadata/macho_symbols b/fuzz/seeds/append_metadata/macho_symbols new file mode 100644 index 00000000..5e0c2a5b Binary files /dev/null and b/fuzz/seeds/append_metadata/macho_symbols differ diff --git a/fuzz/seeds/description_roundtrip/plain b/fuzz/seeds/description_roundtrip/plain new file mode 100644 index 00000000..4acb2cd2 --- /dev/null +++ b/fuzz/seeds/description_roundtrip/plain @@ -0,0 +1 @@ +Adds a scalar function \ No newline at end of file diff --git a/fuzz/seeds/description_roundtrip/quoting b/fuzz/seeds/description_roundtrip/quoting new file mode 100644 index 00000000..a019fe2d --- /dev/null +++ b/fuzz/seeds/description_roundtrip/quoting @@ -0,0 +1 @@ +Quote " backslash \ colon: hash # tab here \ No newline at end of file diff --git a/fuzz/seeds/description_roundtrip/unicode b/fuzz/seeds/description_roundtrip/unicode new file mode 100644 index 00000000..d08e7017 --- /dev/null +++ b/fuzz/seeds/description_roundtrip/unicode @@ -0,0 +1 @@ +café — 🦆 \ No newline at end of file diff --git a/fuzz/seeds/scaffold_text/description_and_maintainer b/fuzz/seeds/scaffold_text/description_and_maintainer new file mode 100644 index 00000000..1195bf69 --- /dev/null +++ b/fuzz/seeds/scaffold_text/description_and_maintainer @@ -0,0 +1 @@ +Does a thingJane Doe \ No newline at end of file diff --git a/fuzz/seeds/scaffold_text/quoting b/fuzz/seeds/scaffold_text/quoting new file mode 100644 index 00000000..4d10da13 --- /dev/null +++ b/fuzz/seeds/scaffold_text/quoting @@ -0,0 +1 @@ +Quote " and \ and # and: xO'Brien \ No newline at end of file diff --git a/fuzz/seeds/scaffold_text/unicode b/fuzz/seeds/scaffold_text/unicode new file mode 100644 index 00000000..b8ae4047 --- /dev/null +++ b/fuzz/seeds/scaffold_text/unicode @@ -0,0 +1 @@ +café 🦆 \ No newline at end of file diff --git a/lsan-suppressions.txt b/lsan-suppressions.txt index 3f394b99..ef423abc 100644 --- a/lsan-suppressions.txt +++ b/lsan-suppressions.txt @@ -1,13 +1,16 @@ # LeakSanitizer suppressions for the `leak-check` CI job. # -# The job exists to catch a missing `duckdb_destroy_*` in one of quack-rs's RAII -# wrappers. DuckDB itself keeps a number of process-lifetime allocations that are -# still reachable at exit and are not ours to free, so they are suppressed by -# the library that made them -- never by symbol name inside quack-rs, which would -# defeat the point. +# There are none, on purpose. This file used to hold `leak:libduckdb.so`, with a +# comment saying it covered only DuckDB's process-lifetime allocations. LeakSanitizer +# matches a suppression against EVERY frame of a leak's allocation stack, and every +# allocation DuckDB makes for a handle has a libduckdb.so frame, so that one line +# silenced exactly what this job exists to find: a wrapper whose `Drop` never calls +# `duckdb_destroy_*`. Checked (fifth review of the safety net): a leaked LogicalType +# or Value exits 0 with the suppression ("Suppressions used: libduckdb.so") and 23 +# without it, and the whole `--test ffi_roundtrip` suite (404 tests) reports no leak +# at all with no suppression, because DuckDB's one-time allocations are still +# reachable at exit and LeakSanitizer does not report reachable memory. # -# Anything allocated inside libduckdb and freed by a quack-rs destructor is still -# reported when that destructor is missing, because the *allocating* frame is -# what LeakSanitizer attributes and these entries only cover allocations that are -# never freed by design. -leak:libduckdb.so +# If a future DuckDB does leak a one-time allocation that is unreachable at exit, add +# a suppression for that function by name (`leak:duckdb::SomeClass::Method`), never +# for the library. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 81ee5d5b..2209f777 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ # SPDX-License-Identifier: MIT [toolchain] -channel = "stable" +channel = "1.99.1" components = ["rustfmt", "clippy"] diff --git a/scripts/api-coverage-allowlist.txt b/scripts/api-coverage-allowlist.txt index eb6a45ac..bc3b4ef0 100644 --- a/scripts/api-coverage-allowlist.txt +++ b/scripts/api-coverage-allowlist.txt @@ -63,12 +63,16 @@ duckdb_create_varchar superseded duckdb.h: superseded by duckdb_create_varchar_l duckdb_vector_assign_string_element superseded the _len variant is wrapped duckdb_appender_begin_row superseded duckdb.h: a no-op kept for compatibility; only duckdb_appender_end_row is needed duckdb_validity_set_row_validity superseded duckdb_validity_set_row_valid and _invalid are the calls used +duckdb_param_type superseded duckdb_param_logical_type is wrapped (PreparedStatement::parameter_type); this is that call plus duckdb_get_type_id +duckdb_prepared_statement_column_type superseded duckdb_prepared_statement_column_logical_type is wrapped (PreparedStatement::columns); this is that call plus duckdb_get_type_id # --- host-side: an embedding application's concern, not a loadable extension's --- duckdb_get_profiling_info host-side query profiling of the host's own queries duckdb_profiling_info_get_value host-side query profiling of the host's own queries duckdb_profiling_info_get_metrics host-side query profiling of the host's own queries duckdb_profiling_info_get_child_count host-side query profiling of the host's own queries +duckdb_open host-side opening a database is the host's job; quack-rs names it only in tests (the `InMemoryDb` fixture), where the corrected coverage script no longer counts it as wrapped +duckdb_close host-side closing a database is the host's job; quack-rs names it only in tests (the `InMemoryDb` fixture), where the corrected coverage script no longer counts it as wrapped duckdb_profiling_info_get_child host-side query profiling of the host's own queries duckdb_pending_prepared host-side incremental execution; execute_streaming plus InterruptHandle cover an extension's need duckdb_destroy_pending host-side incremental execution; execute_streaming plus InterruptHandle cover an extension's need @@ -85,30 +89,18 @@ duckdb_finish_execution host-side lets an application run DuckDB's scheduler tas duckdb_task_state_is_finished host-side lets an application run DuckDB's scheduler tasks on its own threads duckdb_destroy_task_state host-side lets an application run DuckDB's scheduler tasks on its own threads duckdb_execution_is_finished host-side lets an application run DuckDB's scheduler tasks on its own threads -duckdb_extract_statements_error host-side statement extraction for an application's own multi-statement scripts duckdb_prepare_extracted_statement host-side statement extraction for an application's own multi-statement scripts # --- candidate: worth wrapping, not done yet --- -duckdb_param_type candidate PreparedStatement::parameter_type -duckdb_param_logical_type candidate PreparedStatement::parameter_type -duckdb_prepared_statement_type candidate PreparedStatement::statement_type -duckdb_result_statement_type candidate QueryResult::statement_type -duckdb_prepared_statement_column_count candidate PreparedStatement::column_count (the name is strdup'd: free it with duckdb_free) -duckdb_prepared_statement_column_name candidate PreparedStatement::column_name (the name is strdup'd: free it with duckdb_free) -duckdb_prepared_statement_column_logical_type candidate PreparedStatement::column_type -duckdb_prepared_statement_column_type candidate PreparedStatement::column_type -duckdb_get_table_names candidate Connection::table_names -duckdb_appender_create_query candidate Appender::for_query -duckdb_create_bignum candidate Value::bignum -duckdb_get_bignum candidate Value::as_bignum (a null data pointer on allocation failure must be handled) -duckdb_get_bit candidate Value::as_bit +# (duckdb_get_bit / duckdb_get_bignum, now wrapped as Value::as_bit / as_bignum, abort the +# process on a SQL NULL or a value that does not cast: a wrapper must check type and NULL first.) duckdb_geometry_type_get_crs candidate LogicalType::geometry_crs (the string is strdup'd) -duckdb_unsafe_vector_assign_string_element_len candidate VectorWriter::write_varchar could skip the UTF-8 re-check of a &str (1.5.0+) -duckdb_create_log_storage candidate a safe LogStorage builder (1.5.0+) -duckdb_destroy_log_storage candidate a safe LogStorage builder (1.5.0+) -duckdb_log_storage_set_write_log_entry candidate a safe LogStorage builder (1.5.0+) -duckdb_log_storage_set_extra_data candidate a safe LogStorage builder (1.5.0+) -duckdb_log_storage_set_name candidate a safe LogStorage builder (1.5.0+) -duckdb_register_log_storage candidate a safe LogStorage builder (1.5.0+) +duckdb_unsafe_vector_assign_string_element_len candidate VectorWriter::write_varchar could skip the UTF-8 re-check of a &str (1.5.0+; the slot moved 544 -> 545 at 1.5.2, so gate on the 546-slot layout features, not on 1.5.0) +duckdb_create_log_storage candidate a safe LogStorage builder (1.5.0+, slots moved at 1.5.2; see duckdb_register_log_storage) +duckdb_destroy_log_storage candidate a safe LogStorage builder (1.5.0+, slots moved at 1.5.2; see duckdb_register_log_storage) +duckdb_log_storage_set_write_log_entry candidate a safe LogStorage builder (1.5.0+, slots moved at 1.5.2; see duckdb_register_log_storage) +duckdb_log_storage_set_extra_data candidate a safe LogStorage builder (1.5.0+, slots moved at 1.5.2; see duckdb_register_log_storage) +duckdb_log_storage_set_name candidate a safe LogStorage builder (1.5.0+, slots moved at 1.5.2; see duckdb_register_log_storage) +duckdb_register_log_storage candidate a safe LogStorage builder (1.5.0+). CAVEAT (upstream defect, 1.5.0 to 2.0.0): when the name is taken it runs the extra-data destructor and leaves it set, so the mandatory duckdb_destroy_log_storage frees it twice; validate in Rust first and clear the extra data on a later DuckDBError duckdb_create_timestamp_tz_ns candidate wrapped differently on purpose: calling it raises the engine floor to 2.0.0 (Value::timestamp_tz_ns casts instead); absent from the 546-slot header duckdb_get_timestamp_tz_ns candidate wrapped differently on purpose: calling it raises the engine floor to 2.0.0 (Value::as_timestamp_tz_ns reads the TIMESTAMP_NS payload); absent from the 546-slot header diff --git a/scripts/build-versioned-site.py b/scripts/build-versioned-site.py index 82600e49..1c608008 100755 --- a/scripts/build-versioned-site.py +++ b/scripts/build-versioned-site.py @@ -64,10 +64,11 @@ SITE_URL = "https://quack-rs.com" DOCS_RS_LATEST = "https://docs.rs/quack-rs/latest/" -# The oldest release line published: 0.18, the oldest line SECURITY.md still -# lists as supported (0.16 is end-of-life there). Raise this when a line reaches -# end of life. -MIN_RELEASE = (0, 18, 0) +# The oldest release line published: 0.19, the oldest line SECURITY.md still +# lists as supported (0.18 is end-of-life there since 0.20.0). Raise this when a +# line reaches end of life; scripts/check-release-metadata.py fails while the +# two disagree. +MIN_RELEASE = (0, 19, 0) # Every tag up to and including v0.18.0 was built by docs.yml with this # version and has no book/mdbook-version file (mdBook 0.5 rejects their diff --git a/scripts/check-abi-table.py b/scripts/check-abi-table.py index 28f6090c..7bdcdfc0 100755 --- a/scripts/check-abi-table.py +++ b/scripts/check-abi-table.py @@ -31,7 +31,8 @@ Exit codes: 0 src/abi.rs matches upstream - 1 it has genuinely drifted, or a header is inconsistent + 1 it has genuinely drifted, a header is inconsistent, or no release has a + header at HEADER_URL (the file moved) 2 the release tags could not be listed, or at least one release header could not be fetched, so the comparison would be unsound -- treat as "could not check", not as a failure @@ -353,8 +354,13 @@ def main() -> int: print(f" {tag}: {len(full)} slots ({len(stable)} stable) layout={digest}") if not rows: - print("::warning::no release headers could be downloaded") - return 2 + if unfetchable: + print("::warning::no release headers could be downloaded") + return 2 + # Every release answered 404: the header is not where HEADER_URL says. + where = HEADER_URL.replace("{tag}", "") + print(f"::error::no release has a header at {where}: the file moved") + return 1 derived = collapse(rows) derived_prefixes = derive_prefixes(declarations_by_slots) diff --git a/scripts/check-action-pins.py b/scripts/check-action-pins.py new file mode 100755 index 00000000..cb4cb2fc --- /dev/null +++ b/scripts/check-action-pins.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright 2026 Tom F. +"""Check that each GitHub Action is pinned to one commit wherever it is used. + +Every remote `uses:` in .github/workflows/ names its action by full commit SHA, +with the release (or branch) in a trailing comment: +`Swatinem/rust-cache@d1527c8f... # v2.9.3`. Dependabot moves those together. +The scaffold carries its own copies of the same pins in +src/scaffold/templates_ci.rs, for the workflows it generates for an extension, +and Dependabot never sees them: when it bumped this repository's actions +(#145), the generated workflows would have kept the old commits with no test +failing (DEPS-15). This fails when + +* a remote `uses:` is not pinned to a 40-hex commit; +* two uses of one action repository name different commits, or comments with + a different first word (`# v2.9.2` beside `# v2.9.3`); +* a pin in src/scaffold/templates_ci.rs names an action repository this + repository's workflows use, at another commit or with another version; +* a workflow names a floating runner image (`ubuntu-latest`, `macos-latest`, + `windows-latest`): GitHub moves those labels (`macos-latest` and + `windows-latest` already have), and a job must not change under a pull + request that did not touch it. The pinned labels are `ubuntu-24.04`, + `macos-26` and `windows-2025` (each was the `-latest` image on 2026-10-30; + actions/runner-images lists them); +* either side yields no pins at all (a layout change must not read as "all + consistent"). + +A comment's first word is its version (`v2.9.3`, `master`); anything after it +(a date) is not compared. An action only the scaffold uses (setup-python) is +listed, not compared. + +Usage: + python3 scripts/check-action-pins.py +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS = ROOT / ".github" / "workflows" +TEMPLATES = ROOT / "src" / "scaffold" / "templates_ci.rs" + +USES = re.compile( + r"^\s*(?:-\s+)?uses:\s*(?P[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+)@(?P[^\s#]+)" + r"(?:\s+#\s*(?P.*?))?\s*$" +) +# `const NAME: &str = "owner/repo@sha # comment";`, possibly wrapped after `=`. +TEMPLATE_PIN = re.compile( + r"const\s+(?P\w+)\s*:\s*&str\s*=\s*\"(?P[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+)@(?P[^\s\"#]+)" + r"(?:\s+#\s*(?P[^\"]*?))?\s*\"\s*;" +) +SHA = re.compile(r"^[0-9a-f]{40}$") +FLOATING_RUNNER = re.compile(r"\b(?:ubuntu|macos|windows)-latest\b") + + +def repo_of(action: str) -> str: + """`actions/cache/restore` -> `actions/cache`: one repository, one pin.""" + return "/".join(action.split("/")[:2]) + + +def version_of(comment: str | None) -> str: + return comment.split()[0] if comment and comment.split() else "(no comment)" + + +def main() -> int: + problems: list[str] = [] + # repo -> {(sha, version): [where, ...]} + pins: dict[str, dict[tuple[str, str], list[str]]] = {} + count = 0 + for path in sorted(WORKFLOWS.glob("*.yml")): + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if line.lstrip().startswith("#"): + continue + floating = FLOATING_RUNNER.search(line) + if floating: + problems.append( + f"{path.name}:{number}: `{floating.group(0)}` is a floating runner image; " + "pin it (ubuntu-24.04, macos-26, windows-2025)" + ) + m = USES.match(line) + if not m: + continue + count += 1 + where = f"{path.name}:{number}" + if not SHA.match(m.group("ref")): + problems.append(f"{where}: {m.group('action')}@{m.group('ref')} is not pinned to a full commit SHA") + continue + key = (m.group("ref"), version_of(m.group("comment"))) + pins.setdefault(repo_of(m.group("action")), {}).setdefault(key, []).append(where) + if count == 0: + problems.append(f"no `uses:` found under {WORKFLOWS.relative_to(ROOT)}; has the layout changed?") + for repo, variants in sorted(pins.items()): + if len(variants) > 1: + listed = "; ".join(f"{sha[:12]} # {ver} at {', '.join(w)}" for (sha, ver), w in sorted(variants.items())) + problems.append(f"{repo} is pinned {len(variants)} ways: {listed}") + + template_text = TEMPLATES.read_text(encoding="utf-8") if TEMPLATES.is_file() else "" + template_pins = list(TEMPLATE_PIN.finditer(template_text)) + if not template_pins: + problems.append(f"no action pins found in {TEMPLATES.relative_to(ROOT)}; has the layout changed?") + compared = [] + for m in template_pins: + repo, sha, ver = repo_of(m.group("action")), m.group("ref"), version_of(m.group("comment")) + where = f"{TEMPLATES.relative_to(ROOT)} {m.group('const')}" + if not SHA.match(sha): + problems.append(f"{where}: {m.group('action')}@{sha} is not pinned to a full commit SHA") + continue + variants = pins.get(repo) + if not variants: + compared.append(f"{repo} (scaffold only, not compared)") + continue + if len(variants) == 1: + (want_sha, want_ver), _ = next(iter(variants.items())) + if (sha, ver) != (want_sha, want_ver): + problems.append( + f"{where} pins {repo}@{sha[:12]} # {ver}; the workflows use " + f"{want_sha[:12]} # {want_ver}: update the scaffold's pin to `{repo}@{want_sha} # {want_ver}`" + ) + continue + compared.append(repo) + + for problem in problems: + print(f"error: {problem}") + if problems: + return 1 + print( + f"{count} workflow `uses:` pin {len(pins)} action repositories, one commit each; " + f"the scaffold's pins agree: {', '.join(compared)}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-api-coverage.py b/scripts/check-api-coverage.py index d0e53057..0f821bd4 100755 --- a/scripts/check-api-coverage.py +++ b/scripts/check-api-coverage.py @@ -8,20 +8,33 @@ itself (AUDIT.md section 5.1 records 98, 99 and 105) and a slot a new DuckDB release added went unnoticed until someone read the header. -This script makes the answer mechanical. A slot is **wrapped** when its name -appears in the code (not a comment) of any file under `src/`. Every other slot -must be listed in `scripts/api-coverage-allowlist.txt` with a class and a reason: +This script makes the answer mechanical. A slot is **used** when its name +appears in the *library code* under `src/`: not in a comment, not in a string +or character literal, not in an item under `#[cfg(test)]` (or a `cfg` that +requires `test`, such as `all(test, feature = "...")`), not in a file that is +only reached through such a `mod x;`, and not under `src/bin/`. A slot only a +test, a string or a comment names is not wrapped: counting those as wrapped +put `duckdb_open` and `duckdb_close` (used only by tests) in the total, and a +name kept in an error message satisfied the check after its wrapper was +deleted. A used slot is **wrapped** unless the allowlist classes it +`internal`. Every other slot must be listed in `scripts/api-coverage-allowlist.txt` +with a class and a reason: deprecated DuckDB marks it deprecated in `duckdb.h` superseded a newer call does the same job, or it is an internal helper host-side only a host application (not an extension) has a use for it candidate worth wrapping; not done yet + internal library code calls it, but no public API gives an extension + author what it does (it is used, so it must stay used) It fails when - * a slot is neither wrapped nor listed (a new DuckDB release added it, or a - wrapper was deleted): wrap it, or list it with a reason; - * a listed slot is wrapped after all: delete its line, so the list stays true. + * a slot is neither used nor listed (a new DuckDB release added it, or a + wrapper was deleted, or only tests use it): wrap it, or list it; + * a slot listed in any class but `internal` is used after all: delete its + line (or class it `internal` if no public API exposes it); + * a slot listed `internal` is not used: it is unused, so give it another + class. A listed name the header does not declare is ignored (the same list serves the 546-slot and the 548-slot header). @@ -38,7 +51,8 @@ Usage: python3 scripts/check-api-coverage.py python3 scripts/check-api-coverage.py --header path/to/duckdb_extension.h - python3 scripts/check-api-coverage.py --list # print the unwrapped slots + python3 scripts/check-api-coverage.py --list # print the slots library code does not use + python3 scripts/check-api-coverage.py --self-test # the source scanner, on fixed snippets """ from __future__ import annotations @@ -52,7 +66,8 @@ ROOT = Path(__file__).resolve().parent.parent ALLOWLIST = ROOT / "scripts" / "api-coverage-allowlist.txt" -CLASSES = {"deprecated", "superseded", "host-side", "candidate"} +CLASSES = {"deprecated", "superseded", "host-side", "candidate", "internal"} +NAME = re.compile(r"\bduckdb_\w+\b") def load_abi_table_module(): @@ -65,24 +80,297 @@ def load_abi_table_module(): return module -def code_only(text: str) -> str: - """`text` with `/* */` and `//` comments (doc comments included) removed. +# --------------------------------------------------------------------- scanner - A `//` inside a string literal is not told apart from a comment; the names - searched for never sit in one after it. +CHAR_LITERAL = re.compile(r"'(?:[^'\\\n]|\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F]{1,6}\}))'") +RAW_STRING = re.compile(r'(?:b|c)?r(#*)"') + + +def strip(text: str) -> tuple[str, str]: + """`text` without comments, and without comments or literals. + + Returns (code with string literals kept, code with every string, raw + string, byte string and character literal replaced by `""`). Comments + (`//`, `///`, `//!`, nested `/* */`) become a space; newlines inside them + are kept so positions stay roughly comparable. Lifetimes (`'a`) are code. """ - text = re.sub(r"/\*.*?\*/", " ", text, flags=re.S) - return re.sub(r"//[^\n]*", " ", text) + keep: list[str] = [] + bare: list[str] = [] + i, n = 0, len(text) + while i < n: + c = text[i] + if text.startswith("//", i): + j = text.find("\n", i) + j = n if j < 0 else j + keep.append(" ") + bare.append(" ") + i = j + continue + if text.startswith("/*", i): + depth, j = 1, i + 2 + while j < n and depth: + if text.startswith("/*", j): + depth, j = depth + 1, j + 2 + elif text.startswith("*/", j): + depth, j = depth - 1, j + 2 + else: + j += 1 + blank = " " + "\n" * text.count("\n", i, j) + keep.append(blank) + bare.append(blank) + i = j + continue + prev_ident = i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_") + raw = None if prev_ident else RAW_STRING.match(text, i) + if raw: + close = '"' + raw.group(1) + j = text.find(close, raw.end()) + j = n if j < 0 else j + len(close) + keep.append(text[i:j]) + bare.append('""') + i = j + continue + if c == '"' or (c in "bc" and not prev_ident and text.startswith('"', i + 1)): + j = i + (2 if c != '"' else 1) + while j < n and text[j] != '"': + j += 2 if text[j] == "\\" else 1 + j = min(j + 1, n) + keep.append(text[i:j]) + bare.append('""') + i = j + continue + if c == "'": + m = CHAR_LITERAL.match(text, i) + if m: + keep.append(m.group(0)) + bare.append("''") + i = m.end() + continue + keep.append(c) + bare.append(c) + i += 1 + return "".join(keep), "".join(bare) + + +def split_args(body: str) -> list[str]: + """The comma-separated arguments of a cfg predicate list, at depth 0.""" + args, depth, start = [], 0, 0 + for k, ch in enumerate(body): + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "," and depth == 0: + args.append(body[start:k]) + start = k + 1 + args.append(body[start:]) + return [a.strip() for a in args if a.strip()] + + +def requires_test(predicate: str) -> bool: + """Whether a `cfg(...)` predicate can only hold when compiling tests.""" + p = predicate.strip() + if p == "test": + return True + m = re.fullmatch(r"(all|any|not)\s*\((.*)\)", p, re.DOTALL) + if not m: + return False + args = split_args(m.group(2)) + if m.group(1) == "all": + return any(requires_test(a) for a in args) + if m.group(1) == "any": + return bool(args) and all(requires_test(a) for a in args) + return False # not(...): never test-only + + +CFG_ATTR = re.compile(r"#\[\s*cfg\s*\(") + + +def cfg_attrs(code: str): + """(start, end, predicate) of each outer `#[cfg(...)]` attribute in `code`.""" + for m in CFG_ATTR.finditer(code): + depth, k = 1, m.end() + while k < len(code) and depth: + depth += {"(": 1, ")": -1}.get(code[k], 0) + k += 1 + close = code.find("]", k) + if close < 0: + continue + yield m.start(), close + 1, code[m.end() : k - 1] + + +def item_end(code: str, start: int) -> int: + """End of the item that begins at `start` (after its attributes): the + matching `}` of its first top-level `{`, or its first top-level `;`.""" + k, n = start, len(code) + # Further attributes on the same item. + while True: + while k < n and code[k].isspace(): + k += 1 + if code.startswith("#[", k): + depth = 0 + while k < n: + depth += {"[": 1, "]": -1}.get(code[k], 0) + k += 1 + if depth == 0: + break + continue + break + parens = 0 + while k < n: + ch = code[k] + if ch in "([": + parens += 1 + elif ch in ")]": + parens -= 1 + elif ch == ";" and parens == 0: + return k + 1 + elif ch == "{" and parens == 0: + depth = 0 + while k < n: + depth += {"{": 1, "}": -1}.get(code[k], 0) + k += 1 + if depth == 0: + return k + return n + k += 1 + return n + + +# What may follow a test-only cfg attribute for it to be a `mod x;` declaration: +# other attributes, at most one of them `#[path = "..."]`, then `mod name;`. +MOD_DECL = re.compile( + r"(?:\s*#\[(?!\s*path\b)[^\]]*\])*" + r"(?:\s*#\[\s*path\s*=\s*\"(?P[^\"]+)\"\s*\])?" + r"(?:\s*#\[[^\]]*\])*" + r"\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+(?P\w+)\s*;" +) + + +def library_code(path: Path, text: str, test_files: set[Path]) -> str: + """The library code of one file: comments, literals and test-only items removed. + Adds to `test_files` the files its test-only `mod x;` declarations load.""" + keep, bare = strip(text) + if re.search(r"#!\[\s*cfg\s*\(\s*test\s*\)\s*\]", bare): + return "" + # Test-only `mod x;` declarations, read where string literals are intact + # (a `#[path = "..."]` needs its string). A declaration inside a string (the + # scaffold's templates hold some) resolves to no file, so it changes nothing. + for _, end, predicate in cfg_attrs(keep): + decl = MOD_DECL.match(keep, end) if requires_test(predicate) else None + if decl is None: + continue + name, explicit = decl.group("name"), decl.group("path") + if explicit: + candidates = [path.parent / explicit] + elif path.name in ("mod.rs", "lib.rs", "main.rs"): + candidates = [path.parent / f"{name}.rs", path.parent / name / "mod.rs"] + else: + candidates = [path.parent / path.stem / f"{name}.rs", path.parent / path.stem / name / "mod.rs"] + test_files.update(c.resolve() for c in candidates if c.exists()) + # Test-only items, cut from the literal-free code. + cut = [(start, item_end(bare, end)) for start, end, predicate in cfg_attrs(bare) if requires_test(predicate)] + out, last = [], 0 + for start, stop in sorted(cut): + if start >= last: + out.append(bare[last:start]) + last = stop + out.append(bare[last:]) + return "".join(out) + + +def scan(src: Path) -> tuple[dict[str, set[str]], set[str]]: + """(name -> library files using it, names used only outside library code).""" + files = sorted(p for p in src.rglob("*.rs") if "bin" not in p.relative_to(src).parts[:1]) + test_files: set[Path] = set() + code = {p.resolve(): library_code(p, p.read_text(encoding="utf-8"), test_files) for p in files} + # A test module's own children are test code too. + changed = True + while changed: + changed = False + for p in code: + if p in test_files: + continue + for t in list(test_files): + tdir = t.parent if t.name == "mod.rs" else t.parent / t.stem + if tdir in p.parents: + test_files.add(p) + changed = True + break + used: dict[str, set[str]] = {} + for p, text in code.items(): + if p in test_files: + continue + for name in set(NAME.findall(text)): + used.setdefault(name, set()).add(p.relative_to(src.resolve()).as_posix()) + elsewhere = set() + for p in src.rglob("*.rs"): + elsewhere |= set(NAME.findall(p.read_text(encoding="utf-8"))) + return used, elsewhere - set(used) + + +SELF_TEST = [ + # (file name, source, names that must count as used, names that must not) + ("a.rs", 'fn f() { unsafe { duckdb_a() } }', {"duckdb_a"}, set()), + ("b.rs", "// duckdb_b()\n/* duckdb_c /* nested */ duckdb_d */\n/// duckdb_e\nfn g() {}", + set(), {"duckdb_b", "duckdb_c", "duckdb_d", "duckdb_e"}), + ("c.rs", ("fn h() -> &'static str { \"duckdb_f\" }\n" + 'fn i() { let _ = r#"duckdb_g"#; let _ = b"duckdb_h"; }'), + set(), {"duckdb_f", "duckdb_g", "duckdb_h"}), + ("d.rs", "fn j() { let q = '\"'; duckdb_i(); let r = '\\''; }", {"duckdb_i"}, set()), + ("e.rs", "fn k<'a>(x: &'a u8) { duckdb_j() }\n#[cfg(test)]\nmod tests { fn t() { duckdb_k() } }", + {"duckdb_j"}, {"duckdb_k"}), + ("f.rs", ('#[cfg(all(feature = "x", test))]\nfn t() { duckdb_l() }\n' + '#[cfg(any(test, feature = "y"))]\nfn u() { duckdb_m() }\n#[cfg(not(test))]\nfn v() { duckdb_n() }'), + {"duckdb_m", "duckdb_n"}, {"duckdb_l"}), + ("g.rs", ("#[cfg(test)]\n#[allow(dead_code)]\nconst T: [u8; 2] = [1, 2];\nfn w() { duckdb_o() }\n" + "#[cfg(test)]\nuse x::duckdb_p;"), + {"duckdb_o"}, {"duckdb_p"}), + ("h.rs", 'fn s() { let _ = "a \\" duckdb_q"; duckdb_r() }', {"duckdb_r"}, {"duckdb_q"}), +] + + +def self_test() -> int: + import tempfile + + failures = total = 0 + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "src" + (src / "m").mkdir(parents=True) + for name, source, _, _ in SELF_TEST: + (src / name).write_text(source, encoding="utf-8") + # A test-only module file and its child, reached through `mod`. + (src / "m.rs").write_text("#[cfg(test)]\nmod live;\nfn x() { duckdb_s() }", encoding="utf-8") + (src / "m" / "live.rs").write_text("fn y() { duckdb_t() }", encoding="utf-8") + (src / "m" / "live").mkdir() + (src / "m" / "live" / "deep.rs").write_text("fn z() { duckdb_u() }", encoding="utf-8") + (src / "bin").mkdir() + (src / "bin" / "tool.rs").write_text("fn main() { duckdb_v() }", encoding="utf-8") + used, elsewhere = scan(src) + cases = [(n, s, u, nu) for n, s, u, nu in SELF_TEST] + [ + ("m.rs", "", {"duckdb_s"}, {"duckdb_t", "duckdb_u"}), + ("bin/tool.rs", "", set(), {"duckdb_v"}), + ] + for name, _, want, not_want in cases: + for w in want: + total += 1 + if w not in used: + print(f"self-test: {name}: {w} should count as used") + failures += 1 + for w in not_want: + total += 1 + if w in used: + print(f"self-test: {name}: {w} should not count as used (found in {sorted(used[w])})") + failures += 1 + total += 1 + if "duckdb_k" not in elsewhere: + print("self-test: a name used only by a test should be reported as such") + failures += 1 + print(f"self-test: {total - failures}/{total} expectations hold") + return 1 if failures else 0 -def wrapped_names(names: list[str]) -> set[str]: - """The names in `names` that appear in the code of some file under `src/`.""" - wanted = set(names) - found: set[str] = set() - for path in sorted((ROOT / "src").rglob("*.rs")): - code = code_only(path.read_text(encoding="utf-8")) - found |= wanted & set(re.findall(r"\bduckdb_\w+\b", code)) - return found +# ------------------------------------------------------------------ allowlist def read_allowlist() -> dict[str, tuple[str, str]]: @@ -104,8 +392,11 @@ def read_allowlist() -> dict[str, tuple[str, str]]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--header", type=Path, help="duckdb_extension.h to read instead of fetching one") - parser.add_argument("--list", action="store_true", help="print the slots no src/ code uses, and exit") + parser.add_argument("--list", action="store_true", help="print the slots library code does not use, and exit") + parser.add_argument("--self-test", action="store_true", help="check the source scanner on fixed snippets, and exit") args = parser.parse_args() + if args.self_test: + return self_test() abi = load_abi_table_module() if args.header: @@ -123,26 +414,39 @@ def main() -> int: label = f"DuckDB {tag}" slots = abi.struct_fields(header, unstable=True, api=None) - wrapped = wrapped_names(slots) - unwrapped = [name for name in slots if name not in wrapped] + used_in, not_library = scan(ROOT / "src") + used = [name for name in slots if name in used_in] + unused = [name for name in slots if name not in used_in] if args.list: - print("\n".join(unwrapped)) + print("\n".join(unused)) return 0 allowed = read_allowlist() problems: list[str] = [] - for name in unwrapped: + for name in unused: if name not in allowed: - problems.append(f"unclassified: {name} (wrap it, or list it in {ALLOWLIST.name} with a reason)") - for name, (cls, _) in allowed.items(): - if name in wrapped: - problems.append(f"stale: {name} is listed as {cls} but src/ uses it; delete its line") + hint = " (only tests, strings or comments under src/ name it)" if name in not_library else "" + problems.append(f"unclassified: {name}{hint} (wrap it, or list it in {ALLOWLIST.name} with a reason)") + elif allowed[name][0] == "internal": + problems.append(f"stale: {name} is listed as internal but library code does not use it; give it another class") + for name in used: + cls = allowed.get(name, ("",))[0] + if cls and cls != "internal": + problems.append( + f"stale: {name} is listed as {cls} but library code uses it ({', '.join(sorted(used_in[name]))}); " + "delete its line, or class it `internal` if no public API exposes it" + ) - counts = {cls: sum(1 for n in unwrapped if allowed.get(n, ("",))[0] == cls) for cls in sorted(CLASSES)} + internal = [n for n in used if allowed.get(n, ("",))[0] == "internal"] + wrapped = len(used) - len(internal) + counts = {cls: sum(1 for n in unused if allowed.get(n, ("",))[0] == cls) for cls in sorted(CLASSES - {"internal"})} print( - f"{label}: {len(slots)} slots, {len(wrapped)} wrapped, {len(unwrapped)} not " - + ", ".join(f"{n} {cls}" for cls, n in counts.items()) + f"{label}: {len(slots)} slots, {wrapped} wrapped, {len(internal)} used internally only, " + f"{len(unused)} not used: " + ", ".join(f"{n} {cls}" for cls, n in counts.items()) ) + test_only = sorted(n for n in unused if n in not_library) + if test_only: + print(f" named under src/ only by tests, strings or comments: {' '.join(test_only)}") for problem in problems: print(f" {problem}") return 1 if problems else 0 diff --git a/scripts/check-autoload-entries.py b/scripts/check-autoload-entries.py index c91e6fc5..fc7aa38b 100755 --- a/scripts/check-autoload-entries.py +++ b/scripts/check-autoload-entries.py @@ -26,7 +26,10 @@ Exit codes: 0 the lists cover every supported release 1 a release autoloads for a name quack-rs does not list - 2 upstream could not be reached or parsed + 2 upstream could not be reached (network failure, HTTP 5xx or 429) + +An HTTP 4xx answer, or a header the script cannot read, exits 1 as well: the +file moved or changed shape and this script needs updating. Usage: python3 scripts/check-autoload-entries.py @@ -39,6 +42,7 @@ import re import subprocess import sys +import time import urllib.error import urllib.request from pathlib import Path @@ -77,7 +81,30 @@ def upstream_names(header: str, array: str) -> set[str]: m = re.search(rf"{array}\[\]\s*=\s*\{{(.*?)\}};", header, re.DOTALL) if not m: raise ValueError(f"{array} not found") - return {name.lower() for name, _ in re.findall(r'\{"([^"]+)",\s*"([^"]+)"\}', m.group(1))} + entries = re.findall(r'\{"([^"]+)",\s*"([^"]+)"\}', m.group(1)) + # Each `{` in the array body opens one entry. If upstream changes the entry + # shape (a third field, a macro), the pattern above matches fewer of them, + # and an empty or partial set would read as "src/catalog.rs covers every + # name": refuse instead (exit 1: the file changed shape and this script + # needs updating). + opened = m.group(1).count("{") + if not entries or len(entries) != opened: + raise ValueError(f"{array}: {opened} entries, {len(entries)} parsed; has the entry layout changed?") + return {name.lower() for name, _ in entries} + + +def fetch(url: str, attempts: int = 3) -> str: + """The body at `url`, retried: this script makes two requests per release, + and a release run (QUACK_STRICT_UPSTREAM=1) fails on one dropped connection.""" + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(url, timeout=60) as r: + return r.read().decode("utf-8") + except (urllib.error.URLError, OSError): + if attempt == attempts: + raise + time.sleep(5 * attempt) + raise AssertionError("unreachable") def rust_names(text: str, const: str) -> set[str]: @@ -107,13 +134,28 @@ def main() -> int: union: dict[str, set[str]] = {array: set() for array in LISTS} for tag in tags: try: - with urllib.request.urlopen(HEADER_URL.format(tag=tag), timeout=60) as r: - header = r.read().decode("utf-8") + header = fetch(HEADER_URL.format(tag=tag)) for array in LISTS: names = upstream_names(header, array) union[array] |= names print(f"{tag}: {array} {len(names)} names") - except (urllib.error.URLError, OSError, ValueError) as e: + except urllib.error.HTTPError as e: + if 400 <= e.code < 500 and e.code != 429: + print( + f"::error::{tag}: extension_entries.hpp answered HTTP {e.code}: " + "the file moved, so this check cannot read it", + file=sys.stderr, + ) + return 1 + print(f"{tag}: could not read extension_entries.hpp: {e}", file=sys.stderr) + return 2 + except ValueError as e: + print( + f"::error::{tag}: extension_entries.hpp has a different shape: {e}", + file=sys.stderr, + ) + return 1 + except (urllib.error.URLError, OSError) as e: print(f"{tag}: could not read extension_entries.hpp: {e}", file=sys.stderr) return 2 diff --git a/scripts/check-ci-aggregate.py b/scripts/check-ci-aggregate.py index 92e689ca..ff2ce089 100755 --- a/scripts/check-ci-aggregate.py +++ b/scripts/check-ci-aggregate.py @@ -13,6 +13,24 @@ required check counts as passing), and its step must compare every result to `success`. +More holes are closed, each of which let a blocking job stop blocking without +this check noticing: + +* the set of informational jobs is fixed in INFORMATIONAL below. A job that gains + `continue-on-error: true` and drops out of `needs:` is no longer silently + demoted: the script fails until INFORMATIONAL names it, so it takes a + deliberate edit here, visible in review; +* a step-level `continue-on-error: true` in a blocking job, or in `ci-ok` itself + (the job stays green while the step fails), must be listed in + INFORMATIONAL_STEPS; +* a job id written with a trailing comment or a YAML anchor (` fuzz: # smoke`) + is still a job. It used to be invisible to the parser; +* the safety patterns are matched against `ci-ok` with its comments removed, so + a pattern kept in a comment after the code that enforced it went does not + count; +* the quality-gate table in .github/workflows/README.md must list exactly + ci.yml's jobs, each with the blocking / informational kind ci.yml gives it. + Usage: scripts/check-ci-aggregate.py # exit 1 on a mismatch """ @@ -24,8 +42,25 @@ from pathlib import Path CI = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "ci.yml" +README = CI.parent / "README.md" AGGREGATE = "ci-ok" +# The jobs that are allowed to be informational (`continue-on-error: true`, and +# outside `ci-ok`'s needs). Adding one is a decision; make it here. +INFORMATIONAL = {"clippy-beta", "latest-stable", "nightly", "test-bundled-duckdb-2-0"} + +# (job, step name) pairs allowed to carry a step-level `continue-on-error`. +INFORMATIONAL_STEPS = { + ("semver", "List public API changes since origin/main (informational)"), +} + +JOB_ID = re.compile(r"^ ([A-Za-z0-9_-]+):[ \t]*(?:[&#].*)?$") +# A step's own keys: on its `- ` line, or indented eight spaces under it. +STEP_CONTINUE_ON_ERROR = re.compile(r"^(?: - | )continue-on-error:\s*(?!false\b)\S") +STEP_NAME = re.compile(r"^(?: - | )name:\s*(.+?)\s*$") +# A row of the README's quality-gate table: | `job-id` | Name | blocking... / informational | +README_ROW = re.compile(r"^\| `([A-Za-z0-9_-]+)` \|.*\| (blocking[^|]*?|informational) \|\s*$", re.MULTILINE) + def parse_jobs(text: str) -> dict[str, dict]: """The `jobs:` map of a workflow, reduced to what this check reads. @@ -44,13 +79,14 @@ def parse_jobs(text: str) -> dict[str, dict]: continue if line and not line.startswith(" ") and not line.startswith("#"): break - m = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + m = JOB_ID.match(line) if m: - current = jobs.setdefault(m.group(1), {"needs": []}) + current = jobs.setdefault(m.group(1), {"needs": [], "lines": []}) in_needs = False continue if current is None: continue + current["lines"].append(line) if re.match(r"^ continue-on-error:\s*true\b", line): current["continue-on-error"] = True m = re.match(r"^ needs:\s*(.*)$", line) @@ -73,12 +109,15 @@ def parse_jobs(text: str) -> dict[str, dict]: def aggregate_block(text: str) -> str: """The text of the `ci-ok` job: from its id to the next job id or EOF.""" - start = re.search(rf"^ {AGGREGATE}:\s*$", text, re.MULTILINE) + start = re.search(rf"^ {AGGREGATE}:[ \t]*(?:[&#].*)?$", text, re.MULTILINE) if start is None: return "" rest = text[start.end() :] - nxt = re.search(r"^ [A-Za-z0-9_-]+:\s*$", rest, re.MULTILINE) - return rest[: nxt.start()] if nxt else rest + nxt = re.search(r"^ [A-Za-z0-9_-]+:[ \t]*(?:[&#].*)?$", rest, re.MULTILINE) + block = rest[: nxt.start()] if nxt else rest + # Comments do not count: a pattern kept in a comment while the code that + # enforced it is gone must not satisfy the check below. + return "\n".join(re.sub(r"(^|\s)#.*$", "", line) for line in block.splitlines()) def safety_errors(block: str) -> list[str]: @@ -97,6 +136,52 @@ def safety_errors(block: str) -> list[str]: return errors +def step_blocks(lines: list[str]) -> list[list[str]]: + """A job's lines split into its steps (each starts with ` - `).""" + blocks: list[list[str]] = [] + for line in lines: + if line.startswith(" - "): + blocks.append([]) + if blocks: + blocks[-1].append(line) + return blocks + + +def step_errors(job: str, lines: list[str]) -> list[str]: + """Steps of `job` that can fail without failing it.""" + errors = [] + for block in step_blocks(lines): + if not any(STEP_CONTINUE_ON_ERROR.match(line) for line in block): + continue + name = next( + (m.group(1).strip().strip("\"'") for line in block if (m := STEP_NAME.match(line))), + "(unnamed)", + ) + if (job, name) not in INFORMATIONAL_STEPS: + errors.append( + f"step `{name}` of blocking job `{job}` has `continue-on-error`: the job " + "stays green when it fails (list it in INFORMATIONAL_STEPS if that is intended)" + ) + return errors + + +def readme_errors(jobs: dict[str, dict]) -> list[str]: + """The README's quality-gate table must list exactly ci.yml's jobs, with the + blocking / informational split ci.yml declares (it is documented as + generated, and nothing regenerated it).""" + rows = {m.group(1): m.group(2) for m in README_ROW.finditer(README.read_text(encoding="utf-8"))} + errors = [] + for job in sorted(set(jobs) - set(rows)): + errors.append(f"{README.name}: job `{job}` is missing from the quality-gate table") + for job in sorted(set(rows) - set(jobs)): + errors.append(f"{README.name}: the table lists `{job}`, which is not a job in {CI.name}") + for job in sorted(set(rows) & set(jobs)): + kind = "informational" if jobs[job].get("continue-on-error") else "blocking" + if not rows[job].startswith(kind): + errors.append(f"{README.name}: `{job}` is `{rows[job]}` in the table, `{kind}` in {CI.name}") + return errors + + def main() -> int: text = CI.read_text(encoding="utf-8") jobs = parse_jobs(text) @@ -110,12 +195,28 @@ def main() -> int: for job in sorted(blocking - needs): print(f"error: blocking job `{job}` is not in `{AGGREGATE}`.needs") failed = True + for job in sorted(informational - INFORMATIONAL): + print(f"error: job `{job}` has `continue-on-error: true` but is not in INFORMATIONAL; " + "a blocking job must not be demoted by an edit to ci.yml alone") + failed = True + for job in sorted(INFORMATIONAL - informational): + print(f"error: INFORMATIONAL names `{job}`, which is not (or no longer) `continue-on-error`") + failed = True + # `ci-ok` itself included: a `continue-on-error` on its own step would make + # the one required check pass whatever its dependencies did. + for job in sorted(set(jobs) - informational): + for message in step_errors(job, jobs[job]["lines"]): + print(f"error: {message}") + failed = True for job in sorted(needs & informational): print(f"error: informational job `{job}` is in `{AGGREGATE}`.needs") failed = True for job in sorted(needs - set(jobs)): print(f"error: `{AGGREGATE}`.needs names `{job}`, which is not a job") failed = True + for message in readme_errors(jobs): + print(f"error: {message}") + failed = True for message in safety_errors(aggregate_block(text)): print(f"error: {message}") failed = True diff --git a/scripts/check-extension-names.py b/scripts/check-extension-names.py new file mode 100755 index 00000000..fbe0ca8d --- /dev/null +++ b/scripts/check-extension-names.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Check quack-rs's reserved extension names against upstream DuckDB. + +`validate_community_extension_name` refuses a name `DuckDB` itself uses: one of +its built-in aliases (`INSTALL s3` really installs `httpfs`, so a community +extension called `s3` can never be installed) or a core extension's name (the +community-extensions FAQ: such an extension "has to be renamed"). Those lists +live in `src/validate/extension_name.rs` and go stale when DuckDB adds a core +extension or an alias, so this script re-derives them. + +Checked: + +- `DUCKDB_EXTENSION_ALIASES` must equal `internal_aliases` in + `src/main/extension/extension_alias.cpp` (pairs, exactly). +- `DUCKDB_CORE_EXTENSION_NAMES` must contain every name in + `internal_extensions[]` (`src/main/extension/extension_helper.cpp`), + `AUTOLOADABLE_EXTENSIONS` (`src/include/duckdb/main/extension_entries.hpp`) + and the core-extension table of the current DuckDB documentation + (`duckdb/duckdb-web`, `docs//core_extensions/ + overview.md`), minus the aliases — and nothing else but the names in + `HISTORICAL` below, each with its reason. +- `DUCKDB_RESERVED_EXTENSION_NAMES` must be exactly the two combined. +- No published community extension (`_includes/list_of_community_extensions.md` + in `duckdb/duckdb-web`) may have a reserved name: if one does, either the + list is wrong or DuckDB allowed the name, and it needs looking at. + +Exit codes: + 0 the lists agree + 1 they diverge + 2 an upstream file could not be fetched or parsed (network, layout change) + +Usage: + python3 scripts/check-extension-names.py + python3 scripts/check-extension-names.py --ref v2.0.0 +""" + +from __future__ import annotations + +import argparse +import re +import sys +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +NAMES_RS = REPO_ROOT / "src" / "validate" / "extension_name.rs" + +RAW = "https://raw.githubusercontent.com" +DUCKDB = RAW + "/duckdb/duckdb/{ref}/" +WEB = RAW + "/duckdb/duckdb-web/main/" + +# Names kept although current DuckDB no longer lists them, with why. +HISTORICAL = { + "jemalloc": "a core extension up to DuckDB 1.4.x (internal_extensions[] at v1.4.4)", +} + + +class Unavailable(Exception): + """An upstream file could not be fetched or no longer has the expected shape.""" + + +def fetch(url: str) -> str: + try: + with urllib.request.urlopen(url, timeout=60) as response: + return response.read().decode("utf-8") + except Exception as error: + raise Unavailable(f"could not fetch {url}: {error}") from error + + +def section(text: str, start: str, end: str, url: str) -> str: + at = text.find(start) + stop = text.find(end, at + 1) if at >= 0 else -1 + if at < 0 or stop < 0: + raise Unavailable(f"{url}: no `{start}` ... `{end}` block (layout changed?)") + return text[at:stop] + + +def upstream_aliases(ref: str) -> set[tuple[str, str]]: + url = DUCKDB.format(ref=ref) + "src/main/extension/extension_alias.cpp" + body = section(fetch(url), "internal_aliases[]", "{nullptr, nullptr}", url) + pairs = set(re.findall(r'\{"([a-z0-9_]+)",\s*"([a-z0-9_]+)"\}', body)) + if not pairs: + raise Unavailable(f"{url}: no aliases found") + return pairs + + +def upstream_internal(ref: str) -> set[str]: + url = DUCKDB.format(ref=ref) + "src/main/extension/extension_helper.cpp" + body = section(fetch(url), "internal_extensions[]", "{nullptr, nullptr", url) + names = set(re.findall(r'\{"([a-z0-9_]+)",', body)) + if len(names) < 10: + raise Unavailable(f"{url}: only {len(names)} internal extensions found") + return names + + +def upstream_autoloadable(ref: str) -> set[str]: + url = DUCKDB.format(ref=ref) + "src/include/duckdb/main/extension_entries.hpp" + body = section(fetch(url), "AUTOLOADABLE_EXTENSIONS[]", "END_OF_AUTOLOADABLE", url) + names = set(re.findall(r'"([a-z0-9_]+)"', body)) + if len(names) < 10: + raise Unavailable(f"{url}: only {len(names)} autoloadable extensions found") + return names + + +def latest_docs_version() -> str: + url = WEB + "_config.yml" + match = re.search( + r'^latest_short_duckdb_version:\s*"([0-9.]+)"', fetch(url), re.MULTILINE + ) + if not match: + raise Unavailable(f"{url}: no latest_short_duckdb_version") + return match.group(1) + + +def documented_core(version: str) -> set[str]: + url = WEB + f"docs/{version}/core_extensions/overview.md" + names = set(re.findall(r"^\|\s*\[([a-z0-9_]+)\]\(", fetch(url), re.MULTILINE)) + if len(names) < 10: + raise Unavailable(f"{url}: only {len(names)} core extensions in the table") + return names + + +def published_community() -> set[str]: + url = WEB + "_includes/list_of_community_extensions.md" + names = set(re.findall(r"^\|\s*\[([a-z0-9_]+)\]\(", fetch(url), re.MULTILINE)) + if len(names) < 100: + raise Unavailable(f"{url}: only {len(names)} community extensions listed") + return names + + +def rust_names(text: str, name: str) -> list[str] | None: + match = re.search( + rf"pub const {name}:\s*&\[&str\]\s*=\s*&\[(.*?)\];", text, re.DOTALL + ) + return re.findall(r'"([^"]+)"', match.group(1)) if match else None + + +def rust_aliases(text: str) -> list[tuple[str, str]] | None: + match = re.search( + r"pub const DUCKDB_EXTENSION_ALIASES:\s*&\[\(&str, &str\)\]\s*=\s*&\[(.*?)\];", + text, + re.DOTALL, + ) + return re.findall(r'\("([^"]+)",\s*"([^"]+)"\)', match.group(1)) if match else None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--ref", default="main", help="DuckDB git ref (default: main)") + parser.add_argument( + "--docs-version", + help="DuckDB documentation version (default: latest_short_duckdb_version in duckdb-web)", + ) + parser.add_argument( + "--rust-file", + type=Path, + default=NAMES_RS, + help="the Rust file holding the lists", + ) + args = parser.parse_args() + + text = args.rust_file.read_text(encoding="utf-8") + ours_aliases = rust_aliases(text) + ours_core = rust_names(text, "DUCKDB_CORE_EXTENSION_NAMES") + ours_reserved = rust_names(text, "DUCKDB_RESERVED_EXTENSION_NAMES") + if ours_aliases is None or ours_core is None or ours_reserved is None: + sys.exit(f"could not parse the name lists out of {args.rust_file}") + + try: + aliases = upstream_aliases(args.ref) + docs_version = args.docs_version or latest_docs_version() + sources = { + f"internal_extensions[] at {args.ref}": upstream_internal(args.ref), + f"AUTOLOADABLE_EXTENSIONS at {args.ref}": upstream_autoloadable(args.ref), + f"the DuckDB {docs_version} core-extension table": documented_core( + docs_version + ), + } + community = published_community() + except Unavailable as error: + print(f"::warning::{error}") + return 2 + + alias_names = {alias for alias, _ in aliases} + upstream_core = set().union(*sources.values()) - alias_names + problems: list[tuple[str, list[str]]] = [] + + def fail(label: str, names) -> None: + if names: + problems.append((label, sorted(names))) + + fail( + "an upstream alias is missing from DUCKDB_EXTENSION_ALIASES", + [f"{a} -> {t}" for a, t in aliases - set(ours_aliases)], + ) + fail( + "DUCKDB_EXTENSION_ALIASES has an alias upstream does not", + [f"{a} -> {t}" for a, t in set(ours_aliases) - aliases], + ) + for label, names in sources.items(): + fail( + f"{label} names a core extension DUCKDB_CORE_EXTENSION_NAMES lacks", + (names - alias_names) - set(ours_core), + ) + fail( + "DUCKDB_CORE_EXTENSION_NAMES lists a name no upstream source has (and HISTORICAL " + "does not explain)", + set(ours_core) - upstream_core - set(HISTORICAL), + ) + fail( + "HISTORICAL lists a name DUCKDB_CORE_EXTENSION_NAMES does not have", + set(HISTORICAL) - set(ours_core), + ) + expected_reserved = set(ours_core) | {alias for alias, _ in ours_aliases} + fail( + "DUCKDB_RESERVED_EXTENSION_NAMES is not exactly the aliases and the core names", + set(ours_reserved) ^ expected_reserved, + ) + if ours_reserved != sorted(ours_reserved): + problems.append(("DUCKDB_RESERVED_EXTENSION_NAMES is not sorted", [])) + fail( + "a published community extension has a reserved name (is the list wrong?)", + community & expected_reserved, + ) + + print( + f"upstream ({args.ref}, docs {docs_version}): {len(aliases)} aliases, " + f"{len(upstream_core)} core extension names; {len(community)} published community " + "extensions" + ) + print( + f"quack-rs: {len(ours_aliases)} aliases, {len(ours_core)} core names " + f"({len(HISTORICAL)} historical), {len(ours_reserved)} reserved" + ) + for label, names in problems: + print(f"\nFAIL: {label}" + (f": {', '.join(names)}" if names else "")) + if problems: + print( + f"\nUpdate the lists in {args.rust_file} (and HISTORICAL here if a name is kept)." + ) + return 1 + print(f"\nOK: {args.rust_file.name} matches upstream") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-hello-ext.py b/scripts/check-hello-ext.py index 4ac3f22f..c68b3482 100755 --- a/scripts/check-hello-ext.py +++ b/scripts/check-hello-ext.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT """Run hello-ext's documented SQL checks against a real DuckDB. -`examples/hello-ext/README.md` lists 31 statements with the answer each should +`examples/hello-ext/README.md` lists 32 statements with the answer each should give once the extension is loaded. CI used to run two of them. This runs all of `examples/hello-ext/sql_checks.txt` — the same statements, with the exact CLI output each must produce — and also fails if the README's statements and the diff --git a/scripts/check-matrix.sh b/scripts/check-matrix.sh index 14e792a6..872b55d2 100755 --- a/scripts/check-matrix.sh +++ b/scripts/check-matrix.sh @@ -21,7 +21,7 @@ # rustup target add wasm32-unknown-emscripten set -uo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/.." || exit 1 # CI sets these globally, which promotes every warning to an error. Without # them a plain `cargo check` here passes on code CI rejects -- an unnecessary diff --git a/scripts/check-msrv-vs-duckdb-ci.py b/scripts/check-msrv-vs-duckdb-ci.py index 93444e49..1394c8c1 100755 --- a/scripts/check-msrv-vs-duckdb-ci.py +++ b/scripts/check-msrv-vs-duckdb-ci.py @@ -15,8 +15,10 @@ Exit codes: 0 every pinned toolchain can build quack-rs - 1 a pinned toolchain is older than quack-rs's MSRV - 2 the workflow could not be fetched or parsed (network, layout change) + 1 a pinned toolchain is older than quack-rs's MSRV, or the workflow answered + HTTP 4xx or holds no toolchain pin (it moved or changed shape, and this + script needs updating) + 2 the workflow could not be fetched (network failure, HTTP 5xx or 429) By default it checks `main`, the branch the community build uses for the newest DuckDB release (DISTRIBUTION_REF, kept equal to @@ -36,6 +38,7 @@ import argparse import re import sys +import urllib.error import urllib.request from pathlib import Path @@ -65,21 +68,50 @@ # anchor it found nothing, and it never saw the `@1.86.0` pin at all. TOOLCHAIN_PIN = re.compile( r"uses:\s*(?:&[\w-]+\s+)?dtolnay/rust-toolchain@" - r"(?:[0-9a-f]{40}\s*#\s*(?P\S+)|(?P[\w.-]+))" + r"(?:[0-9a-f]{40}[ \t]*(?:#[ \t]*(?P[^\n]*))?|(?P[\w.-]+))" ) +# A SHA pin's comment names its toolchain in any of these styles: `# 1.86.0`, +# `# v1.86.0`, `# rust 1.86.0`. Read only the first word, a `v1.86.0` became +# `v1.86.0` (not a version) and `rust 1.86.0` became `rust`, both "floating": +# an upstream switch to either style would have silenced this guard. +COMMENT_VERSION = re.compile(r"(? # master` then +# `toolchain: 1.86.0`. Only the input says which Rust that is. +TOOLCHAIN_INPUT = re.compile(r"(?m)^[ \t]*toolchain:[ \t]*[\"']?(?P[^\"'\s#]+)") SELF_TEST = { - " uses: dtolnay/rust-toolchain@1.86.0": "1.86.0", - " uses: dtolnay/rust-toolchain@stable": "stable", + " uses: dtolnay/rust-toolchain@1.86.0": ["1.86.0"], + " uses: dtolnay/rust-toolchain@stable": ["stable"], " uses: &rust-toolchain-action dtolnay/rust-toolchain@" - "46511b1c83438f0dd37c02d843619ece5a4abb5b # 1.97.1": "1.97.1", + "46511b1c83438f0dd37c02d843619ece5a4abb5b # 1.97.1": ["1.97.1"], " - uses: dtolnay/rust-toolchain@" - "e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08": "master", + "e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master 2026-10-08": ["master"], + " - uses: dtolnay/rust-toolchain@" + "46511b1c83438f0dd37c02d843619ece5a4abb5b # v1.86.0": ["1.86.0"], + " - uses: dtolnay/rust-toolchain@" + "46511b1c83438f0dd37c02d843619ece5a4abb5b # rust 1.86.0": ["1.86.0"], + " - uses: dtolnay/rust-toolchain@46511b1c83438f0dd37c02d843619ece5a4abb5b": ["(no comment)"], + " - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master\n" + " with:\n toolchain: 1.86.0": ["master", "1.86.0"], + " with:\n toolchain: \"1.85\" # wasm": ["1.85"], } +def pin_of(match: re.Match[str]) -> str: + if match.group("ref") is not None: + return match.group("ref") + comment = (match.group("commented") or "").strip() + version = COMMENT_VERSION.search(comment) + if version: + return version.group(1) + return comment.split()[0] if comment else "(no comment)" + + def pins_in(workflow: str) -> list[str]: - return [m.group("commented") or m.group("ref") for m in TOOLCHAIN_PIN.finditer(workflow)] + found = [(m.start(), pin_of(m)) for m in TOOLCHAIN_PIN.finditer(workflow)] + found += [(m.start(), m.group("value")) for m in TOOLCHAIN_INPUT.finditer(workflow)] + return [pin for _, pin in sorted(found)] SEMVER = re.compile(r"^(\d+)\.(\d+)(?:\.(\d+))?$") @@ -91,7 +123,7 @@ def msrv() -> tuple[int, int, int]: def parse(version: str) -> tuple[int, int, int] | None: - match = SEMVER.match(version) + match = SEMVER.match(version.removeprefix("v")) if not match: return None major, minor, patch = match.groups() @@ -117,8 +149,13 @@ def main() -> int: for line, want in SELF_TEST.items(): got = pins_in(line) - if got != [want]: - print(f"self-test failed: {line.strip()!r} gave {got}, expected [{want!r}]") + if got != want: + print(f"self-test failed: {line.strip()!r} gave {got}, expected {want!r}") + return 1 + # A pin that names a release is a constraint, however it is spelled. + for text, want in {"1.86.0": (1, 86, 0), "v1.86.0": (1, 86, 0), "1.85": (1, 85, 0), "stable": None}.items(): + if parse(text) != want: + print(f"self-test failed: parse({text!r}) gave {parse(text)}, expected {want}") return 1 ours = msrv() @@ -130,16 +167,22 @@ def main() -> int: try: with urllib.request.urlopen(url, timeout=60) as response: workflow = response.read().decode("utf-8") - except Exception as error: # noqa: BLE001 - any failure is "could not check" + except urllib.error.HTTPError as error: + if 400 <= error.code < 500 and error.code != 429: + print(f"::error::{url} answered HTTP {error.code}: the workflow moved") + return 1 + print(f"::warning::could not fetch {url}: {error}") + return 2 + except Exception as error: # noqa: BLE001 - network, DNS, TLS, timeouts: "could not check" print(f"::warning::could not fetch {url}: {error}") return 2 found = pins_in(workflow) if not found: print( - "::warning::found no dtolnay/rust-toolchain pins in " - f"{url} — the workflow layout may have changed" + "::error::found no dtolnay/rust-toolchain pins in " + f"{url}: the workflow layout changed" ) - return 2 + return 1 print(f"{ref}: {', '.join(sorted(set(found)))}") pins.extend(found) diff --git a/scripts/check-pitfalls-parity.py b/scripts/check-pitfalls-parity.py new file mode 100755 index 00000000..a2e95d83 --- /dev/null +++ b/scripts/check-pitfalls-parity.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Check that LESSONS.md and the book's pitfall page describe the same pitfalls. + +`LESSONS.md` (the long form, read on GitHub) and `book/src/reference/pitfalls.md` +(the condensed form, read at quack-rs.com) are two hand-written copies. They are +meant to differ in length, not in content, and they had drifted: a pitfall added +to one and not the other, a title reworded, a count updated in one place. + +This script does not compare prose. It fails when the two files disagree on: + +- the set and order of the pitfall ids (the `L` and `P` series); +- a pitfall's title, ignoring backticks and the `DuckDB` / `DuckDB` spelling; +- the "The N known pitfalls" count in the book page's introduction. + +With `--identifiers` it also lists backticked identifiers (types, functions, +`duckdb_*` C functions, test names) that a pitfall's book section uses and its +LESSONS.md section does not. The condensed form should not make claims the long +form lacks. That check is informational (exit status unaffected). + +Exit codes: 0 consistent, 1 inconsistent, 2 a file could not be read. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +LESSONS = ROOT / "LESSONS.md" +BOOK = ROOT / "book" / "src" / "reference" / "pitfalls.md" + +HEADING = re.compile(r"^## ((?:L|P)\d+): (.*?)(?: \{#[\w-]+\})?\s*$", re.M) + + +def sections(text: str) -> dict[str, tuple[str, str]]: + """Map pitfall id -> (title, body).""" + found = list(HEADING.finditer(text)) + out: dict[str, tuple[str, str]] = {} + for i, m in enumerate(found): + end = found[i + 1].start() if i + 1 < len(found) else len(text) + # A following non-pitfall `## ` heading ends the last section. + nxt = re.search(r"^## (?!(?:L|P)\d+:)", text[m.end():end], re.M) + if nxt: + end = m.end() + nxt.start() + out[m.group(1)] = (m.group(2), text[m.end():end]) + return out + + +def norm_title(title: str) -> str: + return re.sub(r"\s+", " ", title.replace("`", "")).strip().lower() + + +def idents(body: str) -> set[str]: + # Hidden doctest lines (`# use ...`) are scaffolding, not claims. + body = "\n".join(l for l in body.split("\n") if not l.startswith("# ") and l != "#") + found = set() + for tok in re.findall(r"`([^`\n]+)`", body): + for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]{3,}", tok): + if "_" in w or any(c.isupper() for c in w[1:]): + found.add(w) + return found + + +def main() -> int: + show_idents = "--identifiers" in sys.argv[1:] + try: + long_text = LESSONS.read_text(encoding="utf-8") + book_text = BOOK.read_text(encoding="utf-8") + except OSError as e: + print(f"cannot read {e.filename}: {e.strerror}", file=sys.stderr) + return 2 + + a, b = sections(long_text), sections(book_text) + problems: list[str] = [] + + if list(a) != list(b): + only_a = [k for k in a if k not in b] + only_b = [k for k in b if k not in a] + problems.append( + f"pitfall ids differ: only in LESSONS.md {only_a or '-'}, " + f"only in the book {only_b or '-'}" + + ("" if sorted(a) != sorted(b) else " (same ids, different order)") + ) + for k in a: + if k in b and norm_title(a[k][0]) != norm_title(b[k][0]): + problems.append(f"{k}: titles differ\n LESSONS.md: {a[k][0]}\n book: {b[k][0]}") + + m = re.search(r"The (\d+) known pitfalls", book_text) + if not m: + problems.append("the book page no longer says 'The N known pitfalls'") + elif int(m.group(1)) != len(b): + problems.append(f"the book page says {m.group(1)} pitfalls but has {len(b)}") + + for p in problems: + print(f"error: {p}") + + if show_idents: + for k in b: + if k in a: + extra = sorted(idents(b[k][1]) - idents(a[k][1])) + if extra: + print(f"note: {k}: in the book section but not in LESSONS.md: {', '.join(extra)}") + + if problems: + return 1 + print(f"LESSONS.md and the book agree on {len(a)} pitfalls ({', '.join(a)})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-platform-table.py b/scripts/check-platform-table.py index 50cb62aa..7fe9c143 100755 --- a/scripts/check-platform-table.py +++ b/scripts/check-platform-table.py @@ -22,8 +22,9 @@ Exit codes: 0 the lists agree - 1 they diverge - 2 the matrix could not be fetched or parsed (network, layout change) + 1 they diverge, or the matrix is not where or what it was (HTTP 4xx, an + entry without `duckdb_arch`, no entries): the script needs updating + 2 the matrix could not be fetched (network failure, HTTP 5xx or 429) Usage: python3 scripts/check-platform-table.py @@ -36,6 +37,7 @@ import json import re import sys +import urllib.error import urllib.request from pathlib import Path @@ -83,7 +85,16 @@ def main() -> int: try: with urllib.request.urlopen(url, timeout=60) as response: matrix = json.loads(response.read().decode("utf-8")) - except Exception as error: # noqa: BLE001 - any failure is "could not check" + except urllib.error.HTTPError as error: + if 400 <= error.code < 500 and error.code != 429: + print( + f"::error::{url} answered HTTP {error.code}: the file moved or the ref " + "does not exist, so this check cannot read the distribution matrix" + ) + return 1 + print(f"::warning::could not fetch {url}: {error}") + return 2 + except Exception as error: # noqa: BLE001 - network, DNS, TLS, timeouts: "could not check" print(f"::warning::could not fetch {url}: {error}") return 2 @@ -92,13 +103,13 @@ def main() -> int: for entry in group.get("include", []): arch = entry.get("duckdb_arch") if not arch: - print(f"::warning::an entry in {url} has no duckdb_arch") - return 2 + print(f"::error::an entry in {url} has no duckdb_arch: the layout changed") + return 1 upstream[arch] = bool(entry.get("opt_in", False)) if not upstream: - print(f"::warning::{url} listed no architectures — layout may have changed") - return 2 + print(f"::error::{url} listed no architectures: the layout changed") + return 1 upstream_opt_in = {arch for arch, is_opt_in in upstream.items() if is_opt_in} diff --git a/scripts/check-release-metadata.py b/scripts/check-release-metadata.py new file mode 100755 index 00000000..fd5b8863 --- /dev/null +++ b/scripts/check-release-metadata.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright 2026 Tom F. +"""Check that everything a release bumps agrees with Cargo.toml's version. + +A release bumps one version in about twenty places: Cargo.toml, the committed +lockfiles (Cargo.lock and those of examples/hello-ext, fuzz, book/doctest and +benches/engine, which all record quack-rs itself), the +CHANGELOG heading and its two link definitions, CITATION.cff, SECURITY.md's +supported-versions table, README.md's "The latest is ..." line, and every +`quack-rs = "0.X"` snippet in the README, the book and the rustdoc. +RELEASING.md Step 3b used to tell the maintainer to `git grep` for the old +version. A place missed was found after the tag, in the published crate, where +it cannot be corrected: a stale Cargo.lock fails `cargo package --locked` in +release.yml only after the whole of ci.yml has run, and a stale snippet ships. + +This check is dependency-free and runs in two places: ci.yml's `doc` job (so the +release PR cannot merge with a place missed) and release.yml's `validate` job +(so a tag on a commit that slipped through fails in a minute, not after two +hours of CI). + +For a pre-release version (`0.20.0-rc.1`) only the lockfile and CHANGELOG +checks apply: the docs keep naming the latest stable release. + +Exit codes: 0 consistent, 1 a mismatch (listed), 2 a file could not be read. + +Usage: + python3 scripts/check-release-metadata.py +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import tomllib + +ROOT = Path(__file__).resolve().parent.parent + +# (lockfile, must exist). +LOCKFILES = [ + ("Cargo.lock", True), + ("examples/hello-ext/Cargo.lock", True), + ("fuzz/Cargo.lock", True), + ("book/doctest/Cargo.lock", True), + ("benches/engine/Cargo.lock", True), +] + +# Where a "quack-rs = ..." dependency snippet is something a reader copies. +SNIPPET_GLOBS = ["README.md", "Cargo.toml", "src/**/*.rs", "book/src/**/*.md", "examples/**/README.md"] +# Pages that name an old version on purpose. +SNIPPET_SKIP = re.compile(r"(?:^|/)(?:changelog\.md|upgrading-[^/]*\.md)$") +SNIPPET = re.compile( + r"""quack-rs\s*=\s*(?:"(?P\d+\.\d+)(?:\.\d+)?"|\{[^}\n]*?version\s*=\s*"(?P\d+\.\d+)(?:\.\d+)?")""" +) + + +def read(path: str) -> str: + try: + return (ROOT / path).read_text(encoding="utf-8") + except OSError as err: + print(f"error: cannot read {path}: {err}", file=sys.stderr) + raise SystemExit(2) from err + + +def main() -> int: + version = tomllib.loads(read("Cargo.toml"))["package"]["version"] + major, minor, *_ = version.split("-")[0].split(".") + line = f"{major}.{minor}" + prerelease = "-" in version + problems: list[str] = [] + + # Every committed lockfile records quack-rs itself (the root package, or the + # path dependency of the example and CI crates). `cargo package --locked` + # refuses a stale root Cargo.lock; the others go stale the same way and are + # bumped in the same commit (RELEASING.md Step 3). + for path, required in LOCKFILES: + if not required and not (ROOT / path).is_file(): + continue + lock = tomllib.loads(read(path)) + locked = [p["version"] for p in lock.get("package", []) if p["name"] == "quack-rs" and "source" not in p] + if locked != [version]: + directory = str(Path(path).parent) + why = ( + "release.yml packages with --locked" if directory == "." + else "the release commit updates every lockfile, RELEASING.md Step 3" + ) + problems.append( + f"{path} has quack-rs {locked}, Cargo.toml says {version}: run `cargo update --workspace` " + f"in {'the repository root' if directory == '.' else directory} and commit {path} ({why})" + ) + + # CHANGELOG: a dated section, and the two link definitions that go with it. + changelog = read("CHANGELOG.md") + heading = re.search(rf"^## \[{re.escape(version)}\] - (\d{{4}}-\d{{2}}-\d{{2}})\s*$", changelog, re.MULTILINE) + if not heading: + problems.append(f"CHANGELOG.md has no `## [{version}] - YYYY-MM-DD` section") + if not re.search(rf"^\[{re.escape(version)}\]: \S+", changelog, re.MULTILINE): + problems.append(f"CHANGELOG.md has no `[{version}]: ...` link definition at the bottom") + unreleased = re.search(r"^\[Unreleased\]: \S+/compare/(\S+?)\.\.\.HEAD\s*$", changelog, re.MULTILINE) + if not unreleased or unreleased.group(1) != f"v{version}": + problems.append( + f"CHANGELOG.md's `[Unreleased]:` link should compare `v{version}...HEAD`, " + f"it compares `{unreleased.group(1) if unreleased else '(missing)'}...HEAD`" + ) + + # The versioned book publishes every release line from MIN_RELEASE on; + # RELEASING.md Step 2b raises it in the commit that ends a line's support. + supported = [ + (int(a), int(b)) + for a, b in re.findall(r"^\|\s*(\d+)\.(\d+)\.x\s*\|\s*Yes\b", read("SECURITY.md"), re.MULTILINE) + ] + site = re.search(r"^MIN_RELEASE\s*=\s*\((\d+),\s*(\d+),\s*(\d+)\)", read("scripts/build-versioned-site.py"), re.MULTILINE) + if not supported: + problems.append("SECURITY.md's supported-versions table has no `| X.Y.x | Yes` row") + elif not site or (int(site[1]), int(site[2]), int(site[3])) != (*min(supported), 0): + oldest = "{}.{}".format(*min(supported)) + problems.append( + f"scripts/build-versioned-site.py MIN_RELEASE is {site.group(0) if site else '(missing)'}, " + f"but the oldest line SECURITY.md supports is {oldest}.x: set it to ({oldest.replace('.', ', ')}, 0)" + ) + + if not prerelease: + date = heading.group(1) if heading else None + citation = read("CITATION.cff") + m = re.search(r"^version:\s*(\S+)\s*$", citation, re.MULTILINE) + if not m or m.group(1) != version: + problems.append(f"CITATION.cff version is {m.group(1) if m else '(missing)'}, expected {version}") + m = re.search(r"^date-released:\s*(\S+)\s*$", citation, re.MULTILINE) + if date and (not m or m.group(1) != date): + problems.append(f"CITATION.cff date-released is {m.group(1) if m else '(missing)'}, CHANGELOG says {date}") + + if not re.search(rf"^\|\s*{re.escape(line)}\.x\s*\|\s*Yes\b", read("SECURITY.md"), re.MULTILINE): + problems.append(f"SECURITY.md's supported-versions table has no `| {line}.x | Yes` row") + + m = re.search(r"The latest is (\d+\.\d+\.\d+) \((\d{4}-\d{2}-\d{2})\)", read("README.md")) + if not m or m.group(1) != version or (date and m.group(2) != date): + problems.append( + f"README.md's \"The latest is ...\" line says {m.groups() if m else '(missing)'}, " + f"expected ({version!r}, {date!r})" + ) + + for pattern in SNIPPET_GLOBS: + for path in sorted(ROOT.glob(pattern)): + rel = path.relative_to(ROOT).as_posix() + if SNIPPET_SKIP.search(rel): + continue + for number, text in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + for m in SNIPPET.finditer(text): + seen = m.group("plain") or m.group("table") + if seen != line: + problems.append(f"{rel}:{number}: snippet names quack-rs {seen}, expected {line}") + + for problem in problems: + print(f"error: {problem}") + if problems: + print(f"\n{len(problems)} place(s) disagree with Cargo.toml's version {version}.") + return 1 + print(f"release metadata agrees with Cargo.toml's version {version}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-spdx-list.py b/scripts/check-spdx-list.py index f511c046..f7ec1283 100755 --- a/scripts/check-spdx-list.py +++ b/scripts/check-spdx-list.py @@ -20,7 +20,10 @@ Exit codes: 0 both lists are correct 1 an entry is unknown, deprecated or unsorted, or an exception is missing - 2 the registry could not be fetched or parsed (network, layout change) + 2 the registry could not be fetched (network failure, HTTP 5xx or 429) + +An HTTP 4xx answer, or a registry without the expected key, exits 1 as well: the +URL or the layout changed and this script needs updating. Usage: python3 scripts/check-spdx-list.py @@ -31,6 +34,7 @@ import json import re import sys +import urllib.error import urllib.request from pathlib import Path @@ -57,13 +61,19 @@ def fetch(url: str, key: str) -> tuple[list[dict], str] | None: try: with urllib.request.urlopen(url, timeout=60) as response: registry = json.loads(response.read().decode("utf-8")) - except Exception as error: # noqa: BLE001 - any failure is "could not check" + except urllib.error.HTTPError as error: + if 400 <= error.code < 500 and error.code != 429: + print(f"::error::{url} answered HTTP {error.code}: the registry moved") + sys.exit(1) + print(f"::warning::could not fetch {url}: {error}") + return None + except Exception as error: # noqa: BLE001 - network, DNS, TLS, timeouts: "could not check" print(f"::warning::could not fetch {url}: {error}") return None entries = registry.get(key) if not entries: - print(f"::warning::{url} listed no {key} — layout may have changed") - return None + print(f"::error::{url} listed no {key}: the registry layout changed") + sys.exit(1) return entries, registry.get("licenseListVersion", "unknown") diff --git a/scripts/check-toolchain-pin.py b/scripts/check-toolchain-pin.py new file mode 100755 index 00000000..52e5d8f3 --- /dev/null +++ b/scripts/check-toolchain-pin.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright 2026 Tom F. +"""Check that every workflow builds with the Rust toolchain rust-toolchain.toml pins. + +CI used to install `stable` everywhere, so a new Rust release (every six weeks) +changed the compiler, rustfmt and clippy under every job at once, with no change +in this repository: 1.99.0's `clippy::assert_is_empty` turned a release pull +request red (AUDIT.md 12.7), and `RUSTFLAGS: -D warnings` makes any new rustc +warning fatal in all of them. The toolchain is now pinned in one place, +rust-toolchain.toml, and moved by a pull request; the informational +`latest-stable`, `clippy-beta` and `nightly` jobs show what the next releases +will report. + +Every toolchain a workflow names is checked, wherever it is named: a +`toolchain:` input, `cargo +X` / `rustc +X` / `rustdoc +X`, `rustup run X`, or a +`RUSTUP_TOOLCHAIN` value. Each must be + +* the pin (rust-toolchain.toml's `channel`, which must be an exact `X.Y.Z`); +* Cargo.toml's `rust-version` (the MSRV job); +* `nightly` or a dated `nightly-YYYY-MM-DD` (Miri, the sanitizers and the + fuzzer need one); +* `stable` or `beta`, only inside an informational job (job-level + `continue-on-error: true`, which scripts/check-ci-aggregate.py ties to its + INFORMATIONAL set), where seeing the next release is the point; +* an expression (`${{ ... }}`) or a shell variable (`"+$msrv"`), which cannot + be checked here, only in a (workflow, job) listed in DYNAMIC below. + +Lines that are comments (YAML or shell) are not read. + +Usage: + python3 scripts/check-toolchain-pin.py +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import tomllib + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS = ROOT / ".github" / "workflows" +EXACT = re.compile(r"^\d+\.\d+\.\d+$") +NIGHTLY = re.compile(r"^nightly(?:-\d{4}-\d{2}-\d{2})?$") + +# Toolchains that only exist at run time, each a deliberate choice: +DYNAMIC = { + # The rustc the `package` job recorded; the step after it checks they agree. + ("release.yml", "publish"), + # The generated project's own `rust-version`, read from its Cargo.toml. + ("ci.yml", "scaffold-e2e"), +} + +JOB_ID = re.compile(r"^ ([A-Za-z0-9_-]+):[ \t]*(?:[&#].*)?$") +REFERENCES = [ + # The `toolchain:` input of dtolnay/rust-toolchain. + re.compile(r"^\s*(?:-\s+)?toolchain:\s*(?P.+?)\s*(?:\s#.*)?$"), + # `cargo +X`, `rustc +X`, `rustdoc +X` (also `cargo "+$msrv"`). + re.compile(r"\b(?:cargo|rustc|rustdoc)\s+[\"']?\+(?P[^\s\"']+)"), + re.compile(r"\brustup\s+run\s+[\"']?(?P[^\s\"']+)"), + # An env key, or a shell assignment. + re.compile(r"\bRUSTUP_TOOLCHAIN(?::\s*|=)[\"']?(?P[^\s\"']+)"), +] + + +def jobs_of(text: str) -> list[tuple[str, bool, list[tuple[int, str]]]]: + """(job id, informational, [(line number, line)]) for each job; lines before + `jobs:` belong to a pseudo-job "(workflow)", which is never informational.""" + jobs: list[tuple[str, bool, list[tuple[int, str]]]] = [("(workflow)", False, [])] + in_jobs = False + for number, line in enumerate(text.splitlines(), 1): + if line.startswith("jobs:"): + in_jobs = True + continue + if in_jobs and line and not line.startswith((" ", "#")): + in_jobs = False + m = JOB_ID.match(line) if in_jobs else None + if m: + jobs.append((m.group(1), False, [])) + continue + job, _, lines = jobs[-1] + if in_jobs and re.match(r"^ continue-on-error:\s*true\b", line): + jobs[-1] = (job, True, lines) + lines.append((number, line)) + return jobs + + +def main() -> int: + pin = tomllib.loads((ROOT / "rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"] + msrv = tomllib.loads((ROOT / "Cargo.toml").read_text(encoding="utf-8"))["package"]["rust-version"] + problems: list[str] = [] + if not EXACT.match(pin): + problems.append(f"rust-toolchain.toml channel is {pin!r}, not an exact X.Y.Z release") + counts = {"pin": 0, "msrv": 0, "nightly": 0, "informational": 0, "dynamic": 0} + for path in sorted(WORKFLOWS.glob("*.yml")): + for job, informational, lines in jobs_of(path.read_text(encoding="utf-8")): + for number, line in lines: + if line.lstrip().startswith("#"): + continue + for pattern in REFERENCES: + for m in pattern.finditer(line): + value = m.group("value").strip("\"'") + where = f"{path.name}:{number} (job `{job}`)" + if value == pin: + counts["pin"] += 1 + elif value == msrv: + counts["msrv"] += 1 + elif NIGHTLY.match(value): + counts["nightly"] += 1 + elif value in {"stable", "beta"}: + if informational: + counts["informational"] += 1 + else: + problems.append( + f"{where}: `{value}` in a blocking job; use {pin} (rust-toolchain.toml), " + "or make the job informational" + ) + elif value.startswith("$"): + if (path.name, job) in DYNAMIC: + counts["dynamic"] += 1 + else: + problems.append( + f"{where}: toolchain `{value}` is only known at run time; " + "list the job in DYNAMIC if that is deliberate" + ) + else: + problems.append(f"{where}: toolchain `{value}`; expected {pin} (rust-toolchain.toml)") + for problem in problems: + print(f"error: {problem}") + if problems: + return 1 + print( + f"every workflow toolchain agrees with rust-toolchain.toml ({pin}): " + + ", ".join(f"{n} {kind}" for kind, n in counts.items()) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mutants-expose-unsafe.py b/scripts/mutants-expose-unsafe.py new file mode 100755 index 00000000..2df645f1 --- /dev/null +++ b/scripts/mutants-expose-unsafe.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Prepare a COPY of the tree so cargo-mutants can see code it skips by design. + +cargo-mutants 27.1.0 generates no mutants inside `unsafe fn` bodies (free, +associated, `const`, trait-default and `unsafe extern "C"` functions alike) and +never mutates integer literals. In this crate that hides the pointer arithmetic: +the strides in `VectorReader` / `VectorWriter`, the bit tests of the validity +mask, `ListBuilder`'s growth arithmetic, the Arrow layout walker, `FfiState`'s +callbacks. `cargo mutants --list` on `src/vector/reader.rs` and `writer.rs` lists +6 mutants; with the `unsafe` keyword removed from the signatures it lists 192, +and with the literals wrapped as well 344 (measured on 0.20.0 development). + +The crate denies `unsafe_op_in_unsafe_fn`, so every unsafe operation inside an +`unsafe fn` is already in its own `unsafe { }` block: dropping the keyword from +the signature still compiles (callers get `unused_unsafe` warnings). With +`--literals`, an integer literal that is an operand of `+ - * / % << >> &`, or +the argument of `.add()` / `.sub()` / `.offset()`, is rewritten as `(L - 1 + 1)` +(`(0 + 1 - 1)` for 0), which gives cargo-mutants operators to flip. + +This is a heuristic over text (comments and strings are blanked, braces matched, +`#[cfg(test)]` modules skipped), for an audit, not for CI: run it on a copy, +check that the copy still builds, then run the live suite against the mutants: + + cp -r . /tmp/mutants-copy && cd /tmp/mutants-copy + python3 scripts/mutants-expose-unsafe.py . --literals src/vector/reader.rs src/vector/writer.rs + cargo check --lib --features duckdb-1-5-4 + cargo mutants --in-place --no-config --features duckdb-1-5-4 \\ + --file src/vector/reader.rs --file src/vector/writer.rs \\ + --cargo-arg=--test --cargo-arg=ffi_roundtrip # live: DUCKDB_LIB_DIR, bundled-test-prebuilt + +Usage: mutants-expose-unsafe.py TREE [--literals] [--sites FILE] FILE... +FILE is relative to TREE and edited in place; --sites writes a JSON list of the +rewritten literal sites (file, line). +""" +import json +import os +import re +import sys + + +def strip_comments_strings(src): + """Replace comments and string/char literal contents with spaces (keeps offsets).""" + out = list(src) + i, n = 0, len(src) + while i < n: + c = src[i] + if src.startswith("//", i): + j = src.find("\n", i) + j = n if j < 0 else j + for k in range(i, j): out[k] = " " + i = j + elif src.startswith("/*", i): + depth, j = 1, i + 2 + while j < n and depth: + if src.startswith("/*", j): depth += 1; j += 2 + elif src.startswith("*/", j): depth -= 1; j += 2 + else: j += 1 + for k in range(i, j): + if out[k] != "\n": out[k] = " " + i = j + elif c == 'r' and re.match(r'r#*"', src[i:i+10]): + m = re.match(r'r(#*)"', src[i:]) + hashes = m.group(1) + end = '"' + hashes + j = src.find(end, i + len(m.group(0))) + j = n if j < 0 else j + len(end) + for k in range(i + len(m.group(0)), j - len(end)): + if out[k] != "\n": out[k] = " " + i = j + elif c == '"': + j = i + 1 + while j < n and src[j] != '"': + j += 2 if src[j] == '\\' else 1 + for k in range(i + 1, min(j, n)): + if out[k] != "\n": out[k] = " " + i = j + 1 + elif c == "'": + m = re.match(r"'(\\.[^']*|[^'\\])'", src[i:i+12]) + if m: + for k in range(i + 1, i + len(m.group(0)) - 1): out[k] = " " + i += len(m.group(0)) + else: + i += 1 + else: + i += 1 + return "".join(out) + + +tree = sys.argv[1] +args = sys.argv[2:] +literals = "--literals" in args +sites_file = None +if "--sites" in args: + sites_file = args[args.index("--sites") + 1] + del args[args.index("--sites"):args.index("--sites") + 2] +files = [a for a in args if not a.startswith("--")] +sites = [] +stats = {} + +def test_regions(clean): + regs = [] + for m in re.finditer(r'#\[cfg\((?:all\()?test[^\]]*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub\s+)?mod\s+\w+\s*\{', clean): + i = clean.index("{", m.start()); depth = 0 + for j in range(i, len(clean)): + if clean[j] == "{": depth += 1 + elif clean[j] == "}": + depth -= 1 + if depth == 0: + regs.append((m.start(), j)); break + return regs + +LIT = r'(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|[0-9][0-9_]*)(?:_?(?:u8|u16|u32|u64|u128|usize|i8|i16|i32|i64|i128|isize))?' +for rel in files: + path = os.path.join(tree, rel) + src = open(path, encoding="utf-8").read() + clean = strip_comments_strings(src) + regs = test_regions(clean) + in_test = lambda pos: any(a <= pos <= b for a, b in regs) + edits = [] # (start, end, replacement) + n_unsafe = 0 + for m in re.finditer(r'\bunsafe\s+(?=(?:extern\s+"[^"]*"\s+)?fn\s+\w)', clean): + if in_test(m.start()): continue + # skip trait method declarations/impls inside `unsafe trait`/`unsafe impl` blocks: detect `unsafe impl`/`unsafe trait` ancestors crudely + before = clean[:m.start()] + last_impl = max(before.rfind("unsafe impl"), before.rfind("unsafe trait")) + if last_impl >= 0: + # still inside that block if braces are unbalanced since + seg = clean[last_impl:m.start()] + if seg.count("{") > seg.count("}"): continue + edits.append((m.start(), m.end(), "")); n_unsafe += 1 + n_lit = 0 + if literals: + # candidates: literal preceded by `.add(`/.sub(/.offset( or by a binary operator after an operand + for m in re.finditer(LIT, clean): + s, e = m.span() + if in_test(s): continue + if s > 0 and (clean[s-1].isalnum() or clean[s-1] in "_."): continue + if e < len(clean) and (clean[e].isalnum() or clean[e] in "_."): continue + # reject floats like 1.5 / ranges + if clean[e:e+1] == "." and clean[e+1:e+2].isdigit(): continue + # find previous non-space char(s) + k = s - 1 + while k >= 0 and clean[k] in " \t\n": k -= 1 + prev = clean[k] + ok = False + if prev == "(": + head = clean[max(0, k-14):k+1] + if re.search(r'\.(add|sub|offset|wrapping_add|wrapping_sub|byte_add|byte_sub)\($', head): ok = True + elif prev in "*/%&+-<>" : + # binary operator? look at what precedes the operator + op_end = k + op_start = k + while op_start > 0 and clean[op_start-1] in "*/%&+-<>=": op_start -= 1 + op = clean[op_start:op_end+1] + if op in ("*", "/", "%", "&", "+", "-", "<<", ">>"): + j = op_start - 1 + while j >= 0 and clean[j] in " \t\n": j -= 1 + if j >= 0 and (clean[j].isalnum() or clean[j] in ")]_"): + # exclude `as` casts / keywords before op (e.g. `return -1`) + word = re.search(r'([A-Za-z_]\w*)$', clean[max(0, j-12):j+1]) + if not (word and word.group(1) in ("return", "in", "if", "else", "match", "break", "as", "mut", "let")): + ok = True + if not ok: continue + # skip pattern/attribute contexts: line starts with #[ or contains '=>' before literal on same line + line_start = clean.rfind("\n", 0, s) + 1 + line = clean[line_start:s] + if line.lstrip().startswith("#") or "=>" in line.split("{")[-1] and re.match(r'\s*[^=]*=>\s*$', line) : continue + text = src[s:e] + val = text.replace("_", "") + lit_val = None + try: + core = re.sub(r'(u8|u16|u32|u64|u128|usize|i8|i16|i32|i64|i128|isize)$', '', val) + lit_val = int(core, 0) + except ValueError: + pass + rep = f"({text} + 1 - 1)" if lit_val == 0 else f"({text} - 1 + 1)" + edits.append((s, e, rep)); n_lit += 1 + sites.append((rel, src.count("\n", 0, s) + 1)) + edits.sort(key=lambda t: t[0], reverse=True) + out = src + for s, e, rep in edits: + out = out[:s] + rep + out[e:] + open(path, "w", encoding="utf-8").write(out) + stats[rel] = (n_unsafe, n_lit) +if sites_file: + with open(sites_file, "w", encoding="utf-8") as handle: + json.dump(sites, handle) +for k, v in stats.items(): print(f"{k}: unsafe-stripped={v[0]} literals-wrapped={v[1]}") diff --git a/scripts/upstream-guard.sh b/scripts/upstream-guard.sh new file mode 100755 index 00000000..6e0a1632 --- /dev/null +++ b/scripts/upstream-guard.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# +# Runs one of the guard scripts that compare a checked-in table with upstream +# (check-abi-table.py, check-platform-table.py, ...), mapping their documented +# exit codes onto a job result: +# +# 0 the table matches upstream -> pass +# 1 it has drifted -> fail +# 2 upstream could not be reached -> pass with a warning, UNLESS +# QUACK_STRICT_UPSTREAM=1 (set by ci.yml on a tag, i.e. for a release) or +# the run is the weekly schedule (GITHUB_EVENT_NAME=schedule), where +# "could not check" must not read as "checked": fail, and re-run. A pull +# request run that cannot reach upstream is not blamed on the pull request; +# a gate that has not actually run for a week is noticed. +# +# The guard scripts return 2 only for a network failure, a 5xx or a 429. An HTTP +# 4xx (a file that moved) or a file that no longer has the expected shape is 1: +# the script needs updating, and it must not degrade into a skipped check. +# +# It also refuses a guard script that does not exist. Python itself exits 2 for +# "can't open file", which the inline `|| status=$?; [ $status -eq 2 ]` wrapper +# this replaces read as "upstream unreachable": renaming or deleting a guard +# left its job green, saying only "check skipped". +# +# Usage: scripts/upstream-guard.sh scripts/check-abi-table.py [args...] +# scripts/upstream-guard.sh --self-test # the mapping above, on stub scripts +set -uo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 [args...] | --self-test" >&2 + exit 64 +fi + +# The mapping is what makes these jobs mean anything, so it is tested on every +# run (ci.yml's `doc` job): stubs that exit 0..3, a missing script, strict and not. +if [ "$1" = "--self-test" ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + for code in 0 1 2 3; do + printf 'import sys\nsys.exit(%s)\n' "$code" > "$tmp/exit$code.py" + done + failed=0 + cases=0 + expect() { #