From 734bb15564a3fa94a849d426acc2eef2cb5bce9c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 01:55:48 +0000 Subject: [PATCH 001/111] Fix the ListBuilder entry-array regression and five smaller issues found re-reviewing the seventh review ListBuilder (High, regression vs 0.19.0): the seventh review made ListBuilder cache its parent's entry array. A builder on a LIST that sits inside another LIST's child (LIST(LIST(T)), or a LIST field of a list's STRUCT elements), kept across the outer list's rows, then wrote through a dangling pointer after the outer list grew, because an outer reserve reallocates that vector's data buffer (LESSONS L18). The reviewer measured a segmentation fault on DuckDB 1.5.6 and 2.0.0 and a pass with only the set_entry line reverted. set_entry fetches the array on every row again, as 0.19.0 did; the child-writer cache is kept. append_metadata entry-point note: a Mach-O symbol table spells C symbols with a leading underscore, so the note fired on every correct macOS build and advised a rename that would break LOAD. The extension name is now DuckDB's (the file name up to its first '.', lowercased) and the quoted error is DuckDB's. LogicalType: the TIMESTAMPTZ_NS engine check moves from precheck, which MockRegistrar runs with no DuckDB behind it, into try_new, so the offline registrar no longer panics; a 1.x development build is judged by the release it names. VectorWriter::set_null returns early on a vector with no validity mask, as the C function it replaced did. Verified so far: lib and bin unit tests (867 and 49) pass with default features. The new live regression test (inline_paths::a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves) was written and validated by the reviewer; I have not yet run it myself, and I have not yet shown that it fails with list_builder.rs reverted. That check, and the live suites on 1.5.6 and 2.0.0, are still to do. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EAuNU9xPW522tZ37enhFzP --- src/bin/append_metadata/entry.rs | 43 ++++++++++++----- src/bin/append_metadata/tests.rs | 35 ++++++++++++++ src/types/logical_type/construct.rs | 72 +++++++++++++++++++---------- src/vector/list_builder.rs | 27 ++++++----- src/vector/writer.rs | 10 ++++ tests/ffi_roundtrip/inline_paths.rs | 57 +++++++++++++++++++++++ 6 files changed, 193 insertions(+), 51 deletions(-) diff --git a/src/bin/append_metadata/entry.rs b/src/bin/append_metadata/entry.rs index 40d389d..8dc4509 100644 --- a/src/bin/append_metadata/entry.rs +++ b/src/bin/append_metadata/entry.rs @@ -6,7 +6,11 @@ //! `DuckDB` takes the extension's name from the *file name* and, for a C API //! extension, calls the symbol `_init_c_api`. A library that exports //! `a_init_c_api` stamped into `b.duckdb_extension` stamps without complaint -//! and then fails at `LOAD` with `undefined symbol: b_init_c_api`. +//! and then fails at `LOAD` with `did not contain function "b_init_c_api"`. +//! +//! The name is `FileSystem::ExtractBaseName` of the path, lowercased +//! (`extension_load.cpp`, 1.5.6 and 2.0.0): the file name up to its first +//! `.`. A Mach-O symbol table spells the C symbol with a leading `_`. use std::fs; use std::path::{Path, PathBuf}; @@ -15,17 +19,25 @@ use std::path::{Path, PathBuf}; /// string table of an ELF, Mach-O or PE file. const ENTRY_SUFFIX: &[u8] = b"_init_c_api\0"; -/// The extension name `DuckDB` derives from `output`: its file name without -/// the `.duckdb_extension` suffix (or, failing that, without any extension). +/// The extension name `DuckDB` derives from `output`: its file name up to the +/// first `.`, lowercased, as `ExtractBaseName` and `StringUtil::Lower` make it. pub fn extension_name(output: &Path) -> Option { let name = output.file_name()?.to_str()?; - let stem = name.strip_suffix(".duckdb_extension").unwrap_or_else(|| { - Path::new(name) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or(name) - }); - (!stem.is_empty()).then(|| stem.to_owned()) + let stem = name.split('.').next().unwrap_or(name); + (!stem.is_empty()).then(|| stem.to_lowercase()) +} + +/// Whether `data` is a Mach-O image (thin or universal), whose symbol and +/// export tables carry each C symbol with a leading underscore. +fn is_macho(data: &[u8]) -> bool { + matches!( + data.get(..4), + Some( + [0xfe, 0xed, 0xfa, 0xce | 0xcf] + | [0xce | 0xcf, 0xfa, 0xed, 0xfe] + | [0xca, 0xfe, 0xba, 0xbe] + ) + ) } /// Every `X` for which `data` holds the NUL-terminated string `X_init_c_api`, @@ -65,7 +77,14 @@ fn find(haystack: &[u8], needle: &[u8]) -> Option { /// all `DuckDB` goes by. pub fn entry_point_note(data: &[u8], output: &Path) -> Option { let expected = extension_name(output)?; - let found = entry_point_names(data); + let mut found = entry_point_names(data); + if is_macho(data) { + for name in &mut found { + if let Some(c_name) = name.strip_prefix('_') { + *name = c_name.to_owned(); + } + } + } if found.contains(&expected) { return None; } @@ -84,7 +103,7 @@ pub fn entry_point_note(data: &[u8], output: &Path) -> Option { format!( "the input exports {list}, but DuckDB takes the extension name from the file \ name and will look for `{expected}_init_c_api` in `{}`: LOAD fails with \ - \"undefined symbol\" unless the file is named `{}.duckdb_extension`", + \"did not contain function\" unless the file is named `{}.duckdb_extension`", output.display(), found[0] ) diff --git a/src/bin/append_metadata/tests.rs b/src/bin/append_metadata/tests.rs index fdb0dfd..a8654b9 100644 --- a/src/bin/append_metadata/tests.rs +++ b/src/bin/append_metadata/tests.rs @@ -689,3 +689,38 @@ fn writing_replaces_the_file_instead_of_rewriting_it() { assert!(leftovers.is_empty(), "{leftovers:?}"); let _ = fs::remove_dir_all(&dir); } + +/// `DuckDB` looks for `_init_c_api`, and a +/// Mach-O symbol table spells a C symbol with a leading `_`. Each case below +/// was checked against `LOAD` in the 1.5.6 and 2.0.0 CLIs (reviewer `lastdiff`). +#[test] +fn the_name_is_duckdbs_lowercased_base_name_and_a_mach_o_underscore_is_not_part_of_it() { + use std::path::Path; + let name = |p: &str| entry::extension_name(Path::new(p)); + assert_eq!(name("My_Ext.duckdb_extension").as_deref(), Some("my_ext")); + assert_eq!( + name("my_ext.v1.duckdb_extension").as_deref(), + Some("my_ext") + ); + let out = Path::new("my_ext.duckdb_extension"); + // A thin 64-bit Mach-O: the symbol is `_my_ext_init_c_api`. + let mut macho = vec![0xcf, 0xfa, 0xed, 0xfe]; + macho.extend_from_slice(b"\0\x01_my_ext_init_c_api\0"); + assert_eq!(entry::entry_point_note(&macho, out), None); + // An ELF symbol with a leading underscore is another name. + assert!(entry::entry_point_note(&symbols(&["_my_ext_init_c_api"]), out).is_some()); + // DuckDB lowercases the file name, not the symbol. + let upper = Path::new("My_Ext.duckdb_extension"); + assert_eq!( + entry::entry_point_note(&symbols(&["my_ext_init_c_api"]), upper), + None + ); + assert!(entry::entry_point_note(&symbols(&["My_Ext_init_c_api"]), upper).is_some()); + assert_eq!( + entry::entry_point_note( + &symbols(&["my_ext_init_c_api"]), + Path::new("my_ext.v1.duckdb_extension") + ), + None + ); +} diff --git a/src/types/logical_type/construct.rs b/src/types/logical_type/construct.rs index 54b9bef..0254d03 100644 --- a/src/types/logical_type/construct.rs +++ b/src/types/logical_type/construct.rs @@ -189,6 +189,7 @@ impl LogicalType { /// `DuckDB` built with assertions aborts on a type id it does not know). pub fn try_new(type_id: TypeId) -> Result { Self::precheck(type_id)?; + Self::refuse_where_the_engine_lacks(type_id)?; // SAFETY: any DUCKDB_TYPE value is accepted; DuckDB returns an owned // handle or null. let inner = unsafe { duckdb_create_logical_type(type_id.to_duckdb_type()) }; @@ -210,9 +211,34 @@ impl LogicalType { Ok(created) } - /// The checks [`try_new`][Self::try_new] makes before it creates a type: a - /// composite id, the two literal types, (with `duckdb-1-5-3`) `VARIANT` - /// and `TIMESTAMPTZ_NS` on an engine older than 2.0.0 are refused. + /// The engine-dependent refusal [`try_new`][Self::try_new] makes before it + /// sends an id: `TIMESTAMPTZ_NS` on an engine older than 2.0.0. Kept out of + /// [`precheck`][Self::precheck], which `check_slot_offline` (and so + /// `MockRegistrar`) runs with no `DuckDB` behind the dispatch table. + fn refuse_where_the_engine_lacks(type_id: TypeId) -> Result<(), LogicalTypeError> { + if type_id == TypeId::TimestampTzNs { + // SAFETY: only `try_new` calls this, right before it creates the type + // with `duckdb_create_logical_type`, which needs the dispatch table + // initialised too. + let engine = unsafe { crate::abi::engine_version() }; + if let Some(engine) = engine.filter(|v| predates_timestamp_tz_ns(v)) { + return Err(LogicalTypeError::with_detail( + "duckdb_create_logical_type", + format!( + "{type_id} (type id 42) was added in DuckDB 2.0.0 and this is DuckDB \ + {engine}. The id is not sent: a DuckDB built with assertions aborts \ + the process on a type id it does not know, and one built without \ + them answers with an INVALID type" + ), + )); + } + } + Ok(()) + } + + /// The checks [`try_new`][Self::try_new] makes before it creates a type + /// that need no engine: a composite id, the two literal types and (with + /// `duckdb-1-5-3`) `VARIANT` are refused. fn precheck(type_id: TypeId) -> Result<(), LogicalTypeError> { if type_id.is_composite() { return Err(LogicalTypeError::with_detail( @@ -232,23 +258,6 @@ impl LogicalType { ), )); } - if type_id == TypeId::TimestampTzNs { - // SAFETY: `try_new` creates the type with `duckdb_create_logical_type` - // right after this, so the dispatch table is initialised whenever - // this runs. - let engine = unsafe { crate::abi::engine_version() }; - if let Some(engine) = engine.filter(|v| predates_timestamp_tz_ns(v)) { - return Err(LogicalTypeError::with_detail( - "duckdb_create_logical_type", - format!( - "{type_id} (type id 42) was added in DuckDB 2.0.0 and this is DuckDB \ - {engine}. The id is not sent: a DuckDB built with assertions aborts \ - the process on a type id it does not know, and one built without \ - them answers with an INVALID type" - ), - )); - } - } #[cfg(feature = "duckdb-1-5-3")] if type_id == TypeId::Variant { return Err(LogicalTypeError::with_detail( @@ -736,10 +745,14 @@ fn check_array_size(size: u64) -> Result<(), LogicalTypeError> { } /// Whether `engine`, a `duckdb_library_version()` string, names a release older -/// than 2.0.0, which has no `TIMESTAMPTZ_NS`. A development build (`v2.0.0-dev1`) -/// or anything else `parse_version` cannot read is not known to predate it. +/// than 2.0.0, which has no `TIMESTAMPTZ_NS`. A development build is judged by +/// the release it names (`v1.5.7-dev12` predates it, `v2.0.0-dev88614` does +/// not); anything else `parse_version` cannot read is not known to predate it. fn predates_timestamp_tz_ns(engine: &str) -> bool { - crate::abi::parse_version(engine).is_some_and(|version| version < (2, 0, 0)) + let release = engine + .split_once('-') + .map_or(engine, |(release, _)| release); + crate::abi::parse_version(release).is_some_and(|version| version < (2, 0, 0)) } #[cfg(test)] @@ -753,14 +766,23 @@ mod tests { /// "less than", on each of major, minor and patch. #[test] fn only_a_release_before_2_0_0_predates_timestamp_tz_ns() { - for older in ["v1.4.4", "v1.5.0", "v1.5.6", "v1.99.99", "1.5.6", "v0.9.9"] { + for older in [ + "v1.4.4", + "v1.5.0", + "v1.5.6", + "v1.99.99", + "1.5.6", + "v0.9.9", + "v1.5.6-dev1", + "v1.5.7-dev12", + ] { assert!(predates_timestamp_tz_ns(older), "{older}"); } for newer in ["v2.0.0", "v2.0.1", "v2.1.0", "v3.0.0", "2.0.0"] { assert!(!predates_timestamp_tz_ns(newer), "{newer}"); } // Unreadable versions are let through: the version is not known. - for unknown in ["", "v2.0.0-dev88614", "v1.5.6-dev1", "garbage", "v1.5"] { + for unknown in ["", "v2.0.0-dev88614", "garbage", "v1.5"] { assert!(!predates_timestamp_tz_ns(unknown), "{unknown:?}"); } } diff --git a/src/vector/list_builder.rs b/src/vector/list_builder.rs index e98a08a..775c84d 100644 --- a/src/vector/list_builder.rs +++ b/src/vector/list_builder.rs @@ -326,9 +326,6 @@ pub struct ListBuilder { /// for NULL elements, its NULL state). Dropped whenever a reserve grows the /// child, because that reallocates the child's buffers. child: Option, - /// The parent's entry array, fetched on first use. A reserve of the child - /// does not move it. - entries: *mut duckdb_list_entry, } /// What to reserve for `capacity` elements under `limit` (with @@ -372,7 +369,6 @@ impl ListBuilder { ceiling: MAX_CHILD_CAPACITY_USIZE, overflowed: false, child: None, - entries: core::ptr::null_mut(), } } @@ -506,22 +502,25 @@ impl ListBuilder { self.written = base + len; } - /// Writes the `{offset, length}` entry of parent row `row_idx`, through the - /// entry array fetched once instead of one `duckdb_list_vector_*` call per row. + /// Writes the `{offset, length}` entry of parent row `row_idx`. + /// + /// The entry array is fetched on every call, not cached: when this LIST is + /// itself inside the child of another LIST or MAP (a `LIST(LIST(T))`, a LIST + /// field of a list's STRUCT elements), a reserve on that outer list + /// reallocates this vector's data buffer (L18), and a builder kept across + /// the outer list's rows would write through a dangling pointer. /// /// # Safety /// /// `row_idx` must be within the parent vector's capacity. unsafe fn set_entry(&mut self, row_idx: usize, offset: usize, length: usize) { - if self.entries.is_null() { - // SAFETY: `self.vector` is a valid LIST vector per the constructor's - // contract, and its data is the array of entries. - self.entries = unsafe { duckdb_vector_get_data(self.vector) }.cast(); - } - // SAFETY: `self.entries` addresses the parent's entry array, and - // `row_idx` is in bounds per this function's contract. + // SAFETY: `self.vector` is a valid LIST vector per the constructor's + // contract, and its data is the array of entries. + let entries = unsafe { duckdb_vector_get_data(self.vector) }.cast::(); + // SAFETY: `entries` addresses the parent's entry array, and `row_idx` is + // in bounds per this function's contract. unsafe { - *self.entries.add(row_idx) = duckdb_list_entry { + *entries.add(row_idx) = duckdb_list_entry { offset: offset as u64, length: length as u64, }; diff --git a/src/vector/writer.rs b/src/vector/writer.rs index c86ef7c..3dce44c 100644 --- a/src/vector/writer.rs +++ b/src/vector/writer.rs @@ -681,6 +681,16 @@ impl VectorWriter { pub unsafe fn set_null(&mut self, idx: usize) { // SAFETY: self.vector is valid per constructor's contract. let validity = unsafe { self.writable_validity() }; + // `duckdb_vector_get_validity` has no mask to give for a vector that is + // not flat or constant (a sliced or dictionary vector); the C function + // this replaced returned early on a null mask, and so does this. + debug_assert!( + !validity.is_null(), + "set_null on a vector with no validity mask" + ); + if validity.is_null() { + return; + } // Clears bit `idx % 64` of word `idx / 64`, which is all // `duckdb_validity_set_row_invalid` does, without the FFI call (see // `VectorReader::is_valid`). diff --git a/tests/ffi_roundtrip/inline_paths.rs b/tests/ffi_roundtrip/inline_paths.rs index 9dc2ff6..b2d2ba0 100644 --- a/tests/ffi_roundtrip/inline_paths.rs +++ b/tests/ffi_roundtrip/inline_paths.rs @@ -195,3 +195,60 @@ fn a_list_builder_keeps_correct_elements_across_reserves_and_nulls() { "{sql}" ); } + +// A `ListBuilder` on the inner LIST of a `LIST(LIST(BIGINT))`, made once per +// chunk and kept across the outer builder's rows. Each outer reserve that grows +// reallocates the inner LIST vector's entry array (it is the outer child's +// data, L18), so a builder that cached that array wrote into freed memory +// (SIGSEGV on DuckDB 1.5.6 at 20,000 rows). +quack_rs::scalar_callback!(hoisted_inner_list, |_info, input, output| { + let chunk = unsafe { DataChunk::from_raw(input) }; + let reader = unsafe { chunk.reader(0) }; + let inner_vec = unsafe { quack_rs::vector::complex::ListVector::get_child(output) }; + let mut outer = unsafe { ListBuilder::new(output) }; + let mut inner = unsafe { ListBuilder::new(inner_vec) }; + for row in 0..chunk.size() { + let x = unsafe { reader.read_i64(row) }; + let n = usize::try_from(x % 7).expect("non-negative"); + unsafe { + outer.push_row(row, n, |_w, base| { + for j in 0..n { + let v = x * 100 + j as i64; + inner.push_row(base + j, 2, |w, b| { + w.write_i64(b, v); + w.write_i64(b + 1, -v); + }); + } + }); + } + } + unsafe { inner.finish() }; + unsafe { outer.finish() }; +}); + +#[test] +fn a_list_builder_on_an_inner_list_survives_the_outer_lists_reserves() { + let fx = Fixture::open(); + // SAFETY: `con` is open; the callback matches the declared signature. + unsafe { + ScalarFunctionBuilder::try_new("hoisted_inner_list") + .expect("name") + .param(TypeId::BigInt) + .returns_logical(LogicalType::list_from_logical(&LogicalType::list( + TypeId::BigInt, + ))) + .function(hoisted_inner_list) + .register(fx.con()) + .expect("register"); + } + let sql = "WITH t AS (SELECT i AS x FROM range(20000) t(i)) \ + SELECT (count(*) FILTER (WHERE hoisted_inner_list(x) IS DISTINCT FROM \ + list_transform(range(x % 7), j -> [x * 100 + j, -(x * 100 + j)])))::VARCHAR \ + || ',' || count(*)::VARCHAR FROM t"; + assert_eq!( + fx.scalar(sql, |r, i| unsafe { r.read_str(i).to_owned() }) + .as_deref(), + Some("0,20000"), + "{sql}" + ); +} From 24055fc7964827fadd7ecf21d6f5b131679594d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 05:00:33 +0000 Subject: [PATCH 002/111] Cap the DuckDB bindings range below 2.1 and ship an allow-list package libduckdb-sys / duckdb: ">=1.4.4, <2" becomes ">=1.4.4, <1.20100". libduckdb-sys numbers DuckDB vX.Y.Z as 1.(10000*X + 100*Y + Z).N, so the new bound admits every DuckDB 1.4.4 - 2.0.x binding and stops before DuckDB 2.1 (1.20100.0). The old range admitted bindings this crate cannot be compiled against: libduckdb-sys 1.20000.x keeps the root-level re-exports quack-rs (and the names its exported macros expand to) behind the default feature `capi-v1`, next to a "TODO: Remove in the future". The deps reviewer deleted only that re-export from a copy of 1.20000.1 and quack-rs then failed with 214 errors; a fresh `cargo add libduckdb-sys` already writes 1.20000.1. Published 0.18.0 and 0.19.0 cannot be narrowed; this only protects new releases. default-features = false for libduckdb-sys: from 1.20000.x the default features pull an HTTP/TLS/zip toolchain into every consumer's build (the reviewer measured 67 crates / 36 s CPU against 35 crates / 22 s when both quack-rs and the consumer turn them off). Up to 1.10506.x those build dependencies are not optional, so it changes nothing there. `loadable-extension` implies `capi-v1`. Package: `exclude` (which ships any new top-level file by default) becomes an `include` allow-list. `cargo package --list --locked` lists 247 files before and after, which I re-ran: the two examples under examples/*.rs ship, and the "ignoring example" warning on every package run goes away. cargo-machete ignore list for `cc` and `criterion`, both used outside the places it looks. Verified so far: `cargo package --list --locked` = 247 files and Cargo.lock unchanged. NOT yet verified by me: a build against the 1.4.4 floor and the 1.20000.1 bindings with the lean feature set (a local matrix is running; CI's dependency-floor, msrv and test-duckdb-2-0 jobs will also exercise it). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EAuNU9xPW522tZ37enhFzP --- Cargo.toml | 58 +++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7549653..a726589 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,31 +13,22 @@ readme = "README.md" keywords = ["duckdb", "extension", "ffi", "sql", "analytics"] categories = ["database", "api-bindings", "development-tools::ffi"] autobenches = false -exclude = [ - "docs/", - "examples/", - "benches/engine/", - "fuzz/", - ".github/", - "book/", - "book.toml", - "assets/", - "deny.toml", - "osv-scanner.toml", - "AUDIT.md", - "RELEASING.md", - "LESSONS.md", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "GOVERNANCE.md", - # Repository tooling (CI checks, the book and release scripts); nothing in - # the build uses it. - "scripts/", - "clippy.toml", - "codecov.yml", - ".cargo/", - "rust-toolchain.toml", - "lsan-suppressions.txt", +# An allow-list: a new top-level file or directory is NOT published unless it is +# named here (the previous `exclude` list shipped anything new by default). +# `examples/*.rs` ship because Cargo packages every auto-discovered example and +# warns ("ignoring example ...") when its file is missing; `examples/hello-ext` +# is a separate package and is not matched. +include = [ + "/src/**", + "/tests/**", + "/benches/interval_bench.rs", + "/examples/*.rs", + "/build.rs", + "/README.md", + "/LICENSE", + "/CHANGELOG.md", + "/SECURITY.md", + "/CITATION.cff", ] [lib] @@ -185,8 +176,16 @@ duckdb-1-5-4 = ["duckdb-1-5-3"] # so it does not keep out the DuckDB 2.0 bindings. [dependencies] -libduckdb-sys = { version = ">=1.4.4, <2", features = ["loadable-extension"] } -duckdb = { version = ">=1.4.4, <2", optional = true } +# `libduckdb-sys` numbers DuckDB vX.Y.Z as 1.(10000*X + 100*Y + Z).N, so `<1.20100` +# admits every DuckDB 1.4.4 - 2.0.x binding and stops before DuckDB 2.1 (1.20100.0). +# The bound is deliberate: 1.20000.x keeps the root-level re-exports quack-rs and its +# exported macros name (`libduckdb_sys::duckdb_connection`, ...) behind the default +# feature `capi-v1`, next to a `// TODO: Remove in the future`. If a later binding +# drops them, an unbounded range would resolve to bindings this crate cannot compile +# against (214 errors, reproduced). Widen it in a patch release once a new DuckDB +# minor has been run through CI (AUDIT.md section 14). +libduckdb-sys = { version = ">=1.4.4, <1.20100", default-features = false, features = ["loadable-extension"] } +duckdb = { version = ">=1.4.4, <1.20100", optional = true } [build-dependencies] # Used only when `bundled-test` / `bundled-test-prebuilt` is active: compiles the @@ -265,3 +264,8 @@ undocumented_unsafe_blocks = "warn" # feature-gated item would be missing. `bundled-test` is # intentionally excluded (it compiles DuckDB from source). features = ["duckdb-1-5-4"] + +[package.metadata.cargo-machete] +# Used, but outside the places cargo-machete looks: `cc` by build.rs, `criterion` +# by benches/interval_bench.rs (a `[[bench]]` with `autobenches = false`). +ignored = ["cc", "criterion"] From bc704b9fd3ae278f2b254c30e942cfd3e8404087 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 05:04:14 +0000 Subject: [PATCH 003/111] ci: run the six upstream-comparison guards through scripts/upstream-guard.sh (CI-2) What was wrong: abi-table (two steps), platform-table, spdx-list, autoload-entries and msrv-vs-duckdb-ci each ran `python3 scripts/check-X.py || status=$?` and mapped exit 2 to "upstream unreachable; check skipped". Python itself exits 2 for "can't open file", so renaming or deleting a guard script left its job green; and on a release (release.yml calls ci.yml) an upstream outage skipped all six silently. Evidence (ci reviewer, re-run here): the old inline wrapper around a missing script printed the "skipped" warning and exited 0. Change: the ci reviewer's patch 01 (scripts/upstream-guard.sh: refuse a missing script; 0/1 pass through; 2 is a warning, or a failure when QUACK_STRICT_UPSTREAM=1, which ci.yml sets when github.ref_type is 'tag'). Added on top: `upstream-guard.sh --self-test` (9 cases on stub scripts: exit 0/1/2/3, strict and not, missing script), run by the `doc` job, so the mapping is itself under test. Ran: shellcheck 0.11.0 clean; self-test 9/9; five mutants of the wrapper (no exists-check, strict ignored, 2 -> fail, status not passed through, `|| true` on the python call) each turn the self-test red; the six real guards through the wrapper: abi-table, api-coverage, platform-table, spdx-list, msrv-vs-duckdb-ci exit 0, autoload-entries exit 0 with the "skipped" warning after a transient "Remote end closed connection" (the case exit 2 is for); a missing script exits 1 with ::error::. actionlint 1.7.12 clean on ci.yml (-ignore SC2129). Not run: the workflow on GitHub. --- .github/workflows/ci.yml | 57 ++++++++-------------------- scripts/upstream-guard.sh | 80 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 42 deletions(-) create mode 100755 scripts/upstream-guard.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 453d28e..a620a90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,10 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + # On a tag (a release, which reaches this file through workflow_call) the + # upstream-comparison guards (scripts/upstream-guard.sh) fail instead of + # skipping when upstream cannot be reached. + QUACK_STRICT_UPSTREAM: ${{ github.ref_type == 'tag' && '1' || '' }} RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" # Harden against transient crates.io/CDN download flakes — notably @@ -545,6 +549,11 @@ jobs: # `ci-ok` is the one required check; it is only as good as its `needs`. - name: The ci-ok job waits on every blocking job run: scripts/check-ci-aggregate.py + # The six upstream-comparison steps run through this wrapper; a mapping + # that read a missing script, or exit 2 on a release, as a pass would turn + # each of them into a check of nothing. + - name: The upstream-guard wrapper maps exit codes as documented + run: scripts/upstream-guard.sh --self-test msrv: name: MSRV (1.86.0) @@ -723,24 +732,12 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- a DuckDB release header was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-abi-table.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-abi-table.py # Every slot of the C extension API is either used under src/ or listed, # with a reason, in scripts/api-coverage-allowlist.txt, so a slot a DuckDB # release adds is noticed instead of left for someone to read the header. - name: Classify the C API surface - run: | - python3 scripts/check-api-coverage.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-api-coverage.py platform-table: name: Platform list (vs upstream distribution matrix) @@ -759,13 +756,7 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the distribution matrix was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-platform-table.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-platform-table.py spdx-list: name: SPDX shortlist (vs official registry) @@ -783,13 +774,7 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the SPDX registry was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-spdx-list.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-spdx-list.py autoload-entries: name: Catalog autoload names (vs every supported DuckDB) @@ -807,13 +792,7 @@ jobs: - name: Verify src/catalog.rs covers every supported release # Exit 2 is the documented "could not check" code (upstream # unreachable) and must not be read as drift. - run: | - python3 scripts/check-autoload-entries.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-autoload-entries.py book: name: Book builds (mdBook) @@ -871,13 +850,7 @@ jobs: # These guards fetch from upstream. Exit 2 is their documented # "could not check" code -- the pinned-toolchain config was unreachable -- and must not # be read as drift. Only exit 1 means the checked-in data is wrong. - run: | - python3 scripts/check-msrv-vs-duckdb-ci.py || status=$? - if [ "${status:-0}" -eq 2 ]; then - echo "::warning::upstream unreachable; check skipped, not failed" - exit 0 - fi - exit "${status:-0}" + run: scripts/upstream-guard.sh scripts/check-msrv-vs-duckdb-ci.py extension-load: name: Extension load test (DuckDB ${{ matrix.duckdb }}) diff --git a/scripts/upstream-guard.sh b/scripts/upstream-guard.sh new file mode 100755 index 0000000..d80c7cc --- /dev/null +++ b/scripts/upstream-guard.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# +# Runs one of the guard scripts that compare a checked-in table with upstream +# (check-abi-table.py, check-platform-table.py, ...), mapping their documented +# exit codes onto a job result: +# +# 0 the table matches upstream -> pass +# 1 it has drifted -> fail +# 2 upstream could not be reached/parsed -> pass with a warning, UNLESS +# QUACK_STRICT_UPSTREAM=1 (set by ci.yml on a tag, i.e. for a release), +# where "could not check" must not read as "checked": fail, and re-run. +# +# It also refuses a guard script that does not exist. Python itself exits 2 for +# "can't open file", which the inline `|| status=$?; [ $status -eq 2 ]` wrapper +# this replaces read as "upstream unreachable": renaming or deleting a guard +# left its job green, saying only "check skipped". +# +# Usage: scripts/upstream-guard.sh scripts/check-abi-table.py [args...] +# scripts/upstream-guard.sh --self-test # the mapping above, on stub scripts +set -uo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 [args...] | --self-test" >&2 + exit 64 +fi + +# The mapping is what makes these jobs mean anything, so it is tested on every +# run (ci.yml's `doc` job): stubs that exit 0..3, a missing script, strict and not. +if [ "$1" = "--self-test" ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + for code in 0 1 2 3; do + printf 'import sys\nsys.exit(%s)\n' "$code" > "$tmp/exit$code.py" + done + failed=0 + cases=0 + expect() { #