diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c375393..1bdd6c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + if: runner.os == 'Linux' && contains(matrix.flags, '--all-features') + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config # Pin a complete, present Xcode on macOS: the floating `macos-latest` # image intermittently leaves a default selection whose clang runtime # (`libclang_rt.osx.a`) is missing, breaking the link with @@ -93,6 +96,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + if: runner.os == 'Linux' && contains(matrix.flags, '--all-features') + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config # See the `check` job: deterministic Xcode so macOS linking is stable. - if: runner.os == 'macOS' id: xcode @@ -133,6 +139,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + if: runner.os == 'Linux' && contains(matrix.flags, '--all-features') + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config # See the `check` job: deterministic Xcode so macOS linking is stable. - if: runner.os == 'macOS' id: xcode @@ -212,6 +221,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo doc --no-deps --all-features @@ -230,6 +241,8 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - uses: dtolnay/rust-toolchain@stable with: components: llvm-tools-preview @@ -325,6 +338,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + # cargo-semver-checks enables public features, including system-openblas. + - run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@v2 @@ -365,6 +380,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install optional OpenBLAS backend + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@v2 @@ -524,3 +541,13 @@ jobs: - run: | sudo apt-get update && sudo apt-get install -y minisign - run: ./scripts/sign-models.sh --dry-run + + linux-blas: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config + - name: Verify Rust default and explicit OpenBLAS artifacts + run: bash scripts/check-linux-blas.sh diff --git a/.github/workflows/linux-cpu-der.yml b/.github/workflows/linux-cpu-der.yml index b15d6e4..828639f 100644 --- a/.github/workflows/linux-cpu-der.yml +++ b/.github/workflows/linux-cpu-der.yml @@ -79,15 +79,14 @@ jobs: env: OUT: benchmarks/results/linux-cpu-native-der-ci EP: cpu - FEATURES: cli + # Match release assets: Rust kernels, no system-openblas. + FEATURES: cli-native POLYVOICE_POWERSET_BATCH_SIZE: "8" POLYVOICE_VBX_PLDA_DIR: ${{ github.workspace }}/fixtures/vbx-plda ASSERT_BASELINE: "1" BASELINE_JSON: ${{ github.workspace }}/tests/der_baseline.json run: | set -euo pipefail - sudo apt-get update -qq - sudo apt-get install -y -qq libopenblas-dev pkg-config if [ "$MODE" = "full" ]; then export MAX_VOX=0 MAX_AMI=0 else diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 68cde70..03dc6b4 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -25,6 +25,7 @@ jobs: - uses: taiki-e/install-action@v2 with: tool: cargo-mutants + - run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - run: cargo mutants --all-features --timeout 300 - if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5443923..2fc5dbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,8 @@ jobs: - run: bash scripts/download-ami-test-single.sh - run: bash scripts/download-voxconverse-test.sh - run: cargo run --features cli --bin polyvoice -- download-models --profile balanced + - name: Install OpenBLAS for all-features validation + run: sudo apt-get update -qq && sudo apt-get install -y libopenblas-dev pkg-config - run: bash scripts/release-check.sh cli-binaries: @@ -105,6 +107,7 @@ jobs: ~/Library/Caches/ort.pyke.io ~/AppData/Local/ort.pyke.io key: ort-dfbin-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + # Release assets use Rust Linux kernels; system-openblas is not enabled. - run: cargo build --release --features cli --target ${{ matrix.target }} - name: Rename binary (unix) if: runner.os != 'Windows' diff --git a/CHANGELOG.md b/CHANGELOG.md index 687ddcc..ebc7b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Linux native builds now use Rust kernels regardless of installed BLAS. + Enable `system-openblas` for the optional LP64 OpenBLAS backend; missing + development files fail clearly instead of silently selecting a fallback. + `--all-features` includes this opt-in. Darwin remains unchanged. + - `PipelineBuilder::validate` rejects out-of-range public settings (NaN, infinity, `max_speakers == 0`, `min_cluster_size == 0`, non-positive embed windows, AHC thresholds outside [-1, 1], AS-norm `top_n < 2`, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5eb1652..af55e10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,3 +113,8 @@ Library docs: [docs/API.md](docs/API.md). ## License By contributing, you agree that your contributions will be licensed under MIT. + +On Linux, `--all-features` and the release checks require LP64 OpenBLAS +development files and `pkg-config` because they enable `system-openblas`. +Ordinary `--features cli` builds use Rust kernels without BLAS. See the +[backend contract](docs/strategy/zero-deps.md#linux-backend-selection). diff --git a/Cargo.toml b/Cargo.toml index f41ed02..0b012d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,8 @@ crate-type = ["cdylib", "rlib"] [features] default = [] +# Opt-in Linux LP64 OpenBLAS for enabled native kernels; no effect on Darwin. +system-openblas = ["polyvoice-kernels?/system-openblas"] # InferenceRuntime surface (tensors, factory, session trait). No native dylib. # Enabled by `backend-tract`. Stages that load ONNX *files* via tract depend # on this, not on a concrete engine. The product CLI does not enable `infer`. diff --git a/PRODUCTION-READINESS.md b/PRODUCTION-READINESS.md index e1a7c1f..3938bfe 100644 --- a/PRODUCTION-READINESS.md +++ b/PRODUCTION-READINESS.md @@ -42,8 +42,8 @@ mean no C/C++ code, no system libraries, or zero Rust crates. - Empty default features provide the BYO core with normal Rust dependencies. - Native Darwin inference compiles C shims and links Accelerate/BNNS. -- Native Linux uses Rust kernels and currently auto-detects installed BLAS. - Making that choice explicit and reproducible remains a release requirement. +- Native Linux uses Rust kernels by default; `system-openblas` explicitly + enables the optional LP64 OpenBLAS backend. - Download-enabled builds pull TLS dependencies, including `ring` native code. `pipeline-local` omits the downloader and its TLS graph. - Python also depends on the Python runtime/ABI. diff --git a/benchmarks/results/linux-blas-selection-2026-09-24/NOTES.md b/benchmarks/results/linux-blas-selection-2026-09-24/NOTES.md new file mode 100644 index 0000000..075d9a3 --- /dev/null +++ b/benchmarks/results/linux-blas-selection-2026-09-24/NOTES.md @@ -0,0 +1,97 @@ +# Explicit Linux OpenBLAS selection: before/after evidence + +Host: AMD Ryzen AI 9 HX 370, Linux x86_64, 24 logical CPUs, performance +CPU governor. Date: 2026-09-24. Release profile, balanced INT8, v2 + VBx, +CPU, powerset batch 8, collar 0, overlap included, checked-in PLDA fixtures. + +## Revisions and artifacts + +- `baseline`: master `a8461d42f7793f2f21409b41f7619e881b4c5a43`, `--features cli`. + Its auto-detection found no BLAS; the saved binary has no BLAS imports. +- `rust`: `64f13c8433c684b6ff73f8cc5e5f5597baa278ec`, `--features cli`. + Default Rust backend even with OpenBLAS available to pkg-config. +- `openblas`: same implementation revision, `--features cli,system-openblas`. + Ubuntu LP64 pthread OpenBLAS 0.3.32, extracted locally with its Fortran + runtime dependencies; no system installation changes. pkg-config points + to that prefix and the runtime loader path points to its shared libraries. + +[Measurements](measurements.json) include binary SHA-256, source revision, +DER, wall RTFx and process peak RSS for every completed run. Source revision +is recorded explicitly: the bench program itself queries the current checkout +at runtime, which cannot identify a saved baseline executable. + +Model pair unchanged, **8,414,314 bytes**: + +- powerset_int8: `175896d26f639933cd86906d2dd3e6796eddb23c1f719925a3949052da76183b` +- resnet34_int8: `24b58559fefb2af624a5d371c43ebae891a9a8ca363b2f9e7c31fd8e440a36b3` + +## Scope and results + +Vox-3 is euqef/fuzfh/msbyq. Vox-10 is the first ten filenames in sorted +VoxConverse-test order; AMI-1 is the first sorted AMI-test recording. +The exact filenames and all per-file quality fields are in +[quality.json](quality.json). These are fixed diagnostic subsets, not the +published full-split Vox/AMI reference scores. + +Vox-3 runs alternate baseline/rust/openblas five times for each jobs setting. +Vox-10 has one run per backend; AMI-1 repeats baseline/rust three times and +OpenBLAS once. Table timing/RSS values are medians, **not release floors**. + +| Dataset | Jobs | Backend | Runs | DER₀ micro % | Wall RTFx | Peak RSS MiB | +|---------|------|---------|------|--------------|-----------|--------------| +| vox3 | 1 | baseline | 5 | 7.0295 | 91.7 | 316.5 | +| vox3 | 1 | rust | 5 | 7.0295 | 89.4 | 314.8 | +| vox3 | 1 | openblas | 5 | 7.0295 | 93.0 | 321.2 | +| vox3 | 3 | baseline | 5 | 7.0295 | 117.0 | 469.5 | +| vox3 | 3 | rust | 5 | 7.0295 | 127.7 | 480.4 | +| vox3 | 3 | openblas | 5 | 7.0295 | 132.3 | 480.5 | +| vox10 | 3 | baseline | 1 | 16.4243 | 134.8 | 2283.0 | +| vox10 | 3 | rust | 1 | 16.4243 | 95.6 | 2224.9 | +| vox10 | 3 | openblas | 1 | 15.6938 | 165.1 | 2242.2 | +| ami1 | 3 | baseline | 3 | 32.5584 | 144.7 | 1263.9 | +| ami1 | 3 | rust | 3 | 32.5584 | 150.0 | 1292.6 | +| ami1 | 3 | openblas | 1 | 31.7272 | 163.5 | 1299.3 | + +Default Rust matches the baseline on every recorded per-file quality field, +including speaker counts and turn counts. OpenBLAS matches Vox-3 quality but +changes clustering on Vox-10/AMI-1; its lower DER here is subset evidence, +not a universal improvement or numerical-equivalence claim. + +**Performance limitation:** unrelated builds and CPU-heavy tests were active +on the host. Throughput fluctuated substantially, including between repeated +runs of the same binary. Full-corpus comparisons were stopped after detecting +that contention; no incomplete full-corpus result is included. These runs +measure before/after behavior but do not certify isolated performance parity +or establish new speed/RSS baselines. Default median AMI RSS is slightly +higher than baseline, with overlapping observed ranges; no memory improvement +is claimed. Re-run on an idle reference host for performance qualification. + +Darwin implementation and all five locked scoreboard limits are unchanged. +Darwin scoreboard CI passed; its shared-runner result does not substitute for +an isolated Darwin RTF/RSS measurement. Linux subset RSS is not the Darwin +Vox-3 memory protocol. + +## Reproduction and verification + +Save a release `polyvoice-bench` from each revision/feature selection. Prepare +subset directories with `audio/` and `rttm/` symlinks for the recorded filenames. +For each binary and dataset: + +```bash +POLYVOICE_VBX_PLDA_DIR="$PWD/fixtures/vbx-plda" POLYVOICE_POWERSET_BATCH_SIZE=8 \ + /usr/bin/time -v ./saved-polyvoice-bench DATASET \ + --profile balanced --pipeline v2 --clusterer vbx --execution-provider cpu \ + --collar 0 --jobs 3 --output result.json +``` + +For the OpenBLAS binary, install the runtime or set `LD_LIBRARY_PATH` to its +local prefix. Read process RSS from `/usr/bin/time -v`, not summed workers. + +Validation: + +- `scripts/check-linux-blas.sh`: passes for both artifacts with OpenBLAS + available, including missing-prerequisite failure and `readelf`/`ldd` checks. +- Kernel tests: 58 Rust / 59 OpenBLAS passed; three manual tests ignored each. +- Product CLI/FFI/OpenBLAS library tests: 943 passed; local-only library: 875. +- Clippy: product CLI/FFI and all-features with `-D warnings` passed. +- Dependency invariants, standalone lockfiles, formatting and doc links passed. diff --git a/benchmarks/results/linux-blas-selection-2026-09-24/measurements.json b/benchmarks/results/linux-blas-selection-2026-09-24/measurements.json new file mode 100644 index 0000000..d707d84 --- /dev/null +++ b/benchmarks/results/linux-blas-selection-2026-09-24/measurements.json @@ -0,0 +1,522 @@ +[ + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 1, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 86.30462118107206, + "peak_rss_mib": 308.16796875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 1, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 89.3779940223416, + "peak_rss_mib": 318.234375 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 1, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 92.79848403146411, + "peak_rss_mib": 320.51953125 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 1, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 95.49843811766628, + "peak_rss_mib": 316.54296875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 1, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 95.32097467446278, + "peak_rss_mib": 314.84375 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 1, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 81.6842355680697, + "peak_rss_mib": 324.82421875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 1, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 91.6953217517525, + "peak_rss_mib": 323.1328125 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 1, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 88.49674343739304, + "peak_rss_mib": 318.92578125 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 1, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 96.19175376673773, + "peak_rss_mib": 326.85546875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 1, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 89.6340889644593, + "peak_rss_mib": 322.046875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 1, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 90.30177227801774, + "peak_rss_mib": 312.6640625 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 1, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 92.96886394808092, + "peak_rss_mib": 321.1640625 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 1, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 95.0808138192513, + "peak_rss_mib": 314.671875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 1, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 88.96880905659827, + "peak_rss_mib": 313.609375 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 1, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 96.20365853341073, + "peak_rss_mib": 320.7578125 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 3, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 116.68261759822745, + "peak_rss_mib": 456.796875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 3, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 127.73771849893299, + "peak_rss_mib": 476.68359375 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 3, + "iteration": 0, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 115.51626820559969, + "peak_rss_mib": 480.48046875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 3, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 136.70235389329676, + "peak_rss_mib": 469.37890625 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 3, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 135.42843244669814, + "peak_rss_mib": 480.390625 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 3, + "iteration": 1, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 125.43875750170528, + "peak_rss_mib": 485.046875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 3, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 114.59635188133215, + "peak_rss_mib": 491.53125 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 3, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 114.57125403673248, + "peak_rss_mib": 474.31640625 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 3, + "iteration": 2, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 132.3101401016383, + "peak_rss_mib": 465.765625 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 3, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 116.9869061768595, + "peak_rss_mib": 469.52734375 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 3, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 128.1026052119163, + "peak_rss_mib": 481.22265625 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 3, + "iteration": 3, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 139.97436119650803, + "peak_rss_mib": 469.83203125 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox3", + "jobs": 3, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 133.51803902611059, + "peak_rss_mib": 474.05078125 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox3", + "jobs": 3, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 113.19373279510963, + "peak_rss_mib": 486.65234375 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox3", + "jobs": 3, + "iteration": 4, + "files": 3, + "der_micro": 7.029478458049887, + "der_macro": 7.355872297558546, + "rt_factor_wall": 136.2547124656419, + "peak_rss_mib": 488.8359375 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "vox10", + "jobs": 3, + "iteration": 0, + "files": 10, + "der_micro": 16.424309448287445, + "der_macro": 16.858423118333395, + "rt_factor_wall": 134.75609187118857, + "peak_rss_mib": 2283.03515625 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "vox10", + "jobs": 3, + "iteration": 0, + "files": 10, + "der_micro": 16.424309448287445, + "der_macro": 16.858423118333395, + "rt_factor_wall": 95.62115447926982, + "peak_rss_mib": 2224.9296875 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "vox10", + "jobs": 3, + "iteration": 0, + "files": 10, + "der_micro": 15.693752158716778, + "der_macro": 16.63064277993919, + "rt_factor_wall": 165.09869598127923, + "peak_rss_mib": 2242.2421875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "ami1", + "jobs": 3, + "iteration": 0, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 171.43832500574453, + "peak_rss_mib": 1248.75390625 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "ami1", + "jobs": 3, + "iteration": 0, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 166.94216960615816, + "peak_rss_mib": 1326.75390625 + }, + { + "backend": "openblas", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "c1c1c5442e4de5ed5e54ad44404aa55300f8a0c0a6f48b70570ffdc695374f55", + "dataset": "ami1", + "jobs": 3, + "iteration": 0, + "files": 1, + "der_micro": 31.727213829719993, + "der_macro": 31.727213829719993, + "rt_factor_wall": 163.49136075398107, + "peak_rss_mib": 1299.32421875 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "ami1", + "jobs": 3, + "iteration": 1, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 99.14897700932826, + "peak_rss_mib": 1278.0078125 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "ami1", + "jobs": 3, + "iteration": 1, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 149.95434277304227, + "peak_rss_mib": 1292.59765625 + }, + { + "backend": "baseline", + "source_revision": "a8461d42f7793f2f21409b41f7619e881b4c5a43", + "binary_sha256": "4a060aff13e90a9200b0fe8c03e36a3f639c44fa29c162f32e86bd9da1503a3d", + "dataset": "ami1", + "jobs": 3, + "iteration": 2, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 144.748781663104, + "peak_rss_mib": 1263.921875 + }, + { + "backend": "rust", + "source_revision": "64f13c8433c684b6ff73f8cc5e5f5597baa278ec", + "binary_sha256": "7f6855638c6c68cf7360d6be30d54401ad4de1a18748f8d8d1ff7518812757f8", + "dataset": "ami1", + "jobs": 3, + "iteration": 2, + "files": 1, + "der_micro": 32.558387586119714, + "der_macro": 32.558387586119714, + "rt_factor_wall": 63.9768888331367, + "peak_rss_mib": 1246.25390625 + } +] diff --git a/benchmarks/results/linux-blas-selection-2026-09-24/quality.json b/benchmarks/results/linux-blas-selection-2026-09-24/quality.json new file mode 100644 index 0000000..9d3c20d --- /dev/null +++ b/benchmarks/results/linux-blas-selection-2026-09-24/quality.json @@ -0,0 +1,1694 @@ +{ + "vox3": { + "baseline": [ + { + "filename": "euqef", + "der_collar": 5.5027711797308, + "der_no_collar": 5.5027711797308, + "miss_rate": 4.513064133016627, + "false_alarm_rate": 0.9897070467141725, + "confusion_rate": 0.0, + "der_single_speaker": 5.5027711797308, + "der_overlap": 0.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9548693586698337 + } + ], + "ref_speakers": 1, + "hyp_speakers": 1, + "num_turns": 3, + "audio_duration_secs": 41.152 + }, + { + "filename": "fuzfh", + "der_collar": 11.651628089446842, + "der_no_collar": 11.651628089446842, + "miss_rate": 3.5700274617497056, + "false_alarm_rate": 4.354648881914476, + "confusion_rate": 3.7269517457826598, + "der_single_speaker": 9.954936501433838, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8890577507598785 + }, + { + "speaker": 1, + "recall": 0.8181818181818182 + }, + { + "speaker": 2, + "recall": 0.996135265700483 + } + ], + "ref_speakers": 3, + "hyp_speakers": 3, + "num_turns": 3, + "audio_duration_secs": 26.048 + }, + { + "filename": "msbyq", + "der_collar": 4.913217623497998, + "der_no_collar": 4.913217623497998, + "miss_rate": 1.9492656875834447, + "false_alarm_rate": 0.4806408544726302, + "confusion_rate": 2.4833110814419226, + "der_single_speaker": 4.70966015520471, + "der_overlap": 100.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9768392370572208 + }, + { + "speaker": 1, + "recall": 0.9782321899736148 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.9910179640718563 + } + ], + "ref_speakers": 4, + "hyp_speakers": 3, + "num_turns": 5, + "audio_duration_secs": 38.272 + } + ], + "rust": [ + { + "filename": "euqef", + "der_collar": 5.5027711797308, + "der_no_collar": 5.5027711797308, + "miss_rate": 4.513064133016627, + "false_alarm_rate": 0.9897070467141725, + "confusion_rate": 0.0, + "der_single_speaker": 5.5027711797308, + "der_overlap": 0.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9548693586698337 + } + ], + "ref_speakers": 1, + "hyp_speakers": 1, + "num_turns": 3, + "audio_duration_secs": 41.152 + }, + { + "filename": "fuzfh", + "der_collar": 11.651628089446842, + "der_no_collar": 11.651628089446842, + "miss_rate": 3.5700274617497056, + "false_alarm_rate": 4.354648881914476, + "confusion_rate": 3.7269517457826598, + "der_single_speaker": 9.954936501433838, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8890577507598785 + }, + { + "speaker": 1, + "recall": 0.8181818181818182 + }, + { + "speaker": 2, + "recall": 0.996135265700483 + } + ], + "ref_speakers": 3, + "hyp_speakers": 3, + "num_turns": 3, + "audio_duration_secs": 26.048 + }, + { + "filename": "msbyq", + "der_collar": 4.913217623497998, + "der_no_collar": 4.913217623497998, + "miss_rate": 1.9492656875834447, + "false_alarm_rate": 0.4806408544726302, + "confusion_rate": 2.4833110814419226, + "der_single_speaker": 4.70966015520471, + "der_overlap": 100.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9768392370572208 + }, + { + "speaker": 1, + "recall": 0.9782321899736148 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.9910179640718563 + } + ], + "ref_speakers": 4, + "hyp_speakers": 3, + "num_turns": 5, + "audio_duration_secs": 38.272 + } + ], + "openblas": [ + { + "filename": "euqef", + "der_collar": 5.5027711797308, + "der_no_collar": 5.5027711797308, + "miss_rate": 4.513064133016627, + "false_alarm_rate": 0.9897070467141725, + "confusion_rate": 0.0, + "der_single_speaker": 5.5027711797308, + "der_overlap": 0.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9548693586698337 + } + ], + "ref_speakers": 1, + "hyp_speakers": 1, + "num_turns": 3, + "audio_duration_secs": 41.152 + }, + { + "filename": "fuzfh", + "der_collar": 11.651628089446842, + "der_no_collar": 11.651628089446842, + "miss_rate": 3.5700274617497056, + "false_alarm_rate": 4.354648881914476, + "confusion_rate": 3.7269517457826598, + "der_single_speaker": 9.954936501433838, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8890577507598785 + }, + { + "speaker": 1, + "recall": 0.8181818181818182 + }, + { + "speaker": 2, + "recall": 0.996135265700483 + } + ], + "ref_speakers": 3, + "hyp_speakers": 3, + "num_turns": 3, + "audio_duration_secs": 26.048 + }, + { + "filename": "msbyq", + "der_collar": 4.913217623497998, + "der_no_collar": 4.913217623497998, + "miss_rate": 1.9492656875834447, + "false_alarm_rate": 0.4806408544726302, + "confusion_rate": 2.4833110814419226, + "der_single_speaker": 4.70966015520471, + "der_overlap": 100.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9768392370572208 + }, + { + "speaker": 1, + "recall": 0.9782321899736148 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.9910179640718563 + } + ], + "ref_speakers": 4, + "hyp_speakers": 3, + "num_turns": 5, + "audio_duration_secs": 38.272 + } + ] + }, + "vox10": { + "baseline": [ + { + "filename": "aepyx", + "der_collar": 15.761089389240935, + "der_no_collar": 15.761089389240935, + "miss_rate": 3.6200620196845086, + "false_alarm_rate": 5.480652554941351, + "confusion_rate": 6.660374814615073, + "der_single_speaker": 14.121220683808986, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8495903539959808 + }, + { + "speaker": 1, + "recall": 0.9518407038666358 + }, + { + "speaker": 2, + "recall": 0.9823982398239824 + }, + { + "speaker": 3, + "recall": 0.8599640933572711 + } + ], + "ref_speakers": 4, + "hyp_speakers": 4, + "num_turns": 29, + "audio_duration_secs": 169.152 + }, + { + "filename": "aggyz", + "der_collar": 18.025561921551343, + "der_no_collar": 18.025561921551343, + "miss_rate": 0.34456508674225733, + "false_alarm_rate": 2.704435273849113, + "confusion_rate": 14.976561560959976, + "der_single_speaker": 17.411899220057986, + "der_overlap": 15.319148936170212, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 1.0 + }, + { + "speaker": 1, + "recall": 0.0 + }, + { + "speaker": 2, + "recall": 0.8731998595012294 + }, + { + "speaker": 3, + "recall": 0.993075228202707 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 1.0 + }, + { + "speaker": 6, + "recall": 0.5270935960591133 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 1.0 + }, + { + "speaker": 9, + "recall": 1.0 + }, + { + "speaker": 10, + "recall": 1.0 + }, + { + "speaker": 11, + "recall": 1.0 + }, + { + "speaker": 12, + "recall": 1.0 + } + ], + "ref_speakers": 13, + "hyp_speakers": 13, + "num_turns": 27, + "audio_duration_secs": 262.592 + }, + { + "filename": "aiqwk", + "der_collar": 12.395950506186727, + "der_no_collar": 12.395950506186727, + "miss_rate": 2.2159730033745784, + "false_alarm_rate": 5.821147356580427, + "confusion_rate": 4.358830146231721, + "der_single_speaker": 11.423955688899145, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.867901972504483 + }, + { + "speaker": 1, + "recall": 0.9968150265414455 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.6268382352941176 + }, + { + "speaker": 4, + "recall": 0.7909604519774012 + }, + { + "speaker": 5, + "recall": 0.5355329949238579 + }, + { + "speaker": 6, + "recall": 1.0 + } + ], + "ref_speakers": 7, + "hyp_speakers": 9, + "num_turns": 22, + "audio_duration_secs": 198.848 + }, + { + "filename": "aorju", + "der_collar": 10.848308421504413, + "der_no_collar": 10.848308421504413, + "miss_rate": 1.930433484483756, + "false_alarm_rate": 4.02694582843494, + "confusion_rate": 4.890929108585717, + "der_single_speaker": 10.636043065611284, + "der_overlap": 36.36363636363637, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.025653923541247486 + }, + { + "speaker": 1, + "recall": 0.945908137094894 + }, + { + "speaker": 2, + "recall": 0.9945992960880294 + }, + { + "speaker": 3, + "recall": 0.9895406213079435 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 0.9527235354573484 + }, + { + "speaker": 6, + "recall": 0.9791422239603702 + }, + { + "speaker": 7, + "recall": 0.9062317996505533 + }, + { + "speaker": 8, + "recall": 0.9613992762364294 + }, + { + "speaker": 9, + "recall": 1.0 + }, + { + "speaker": 10, + "recall": 0.9807480748074807 + }, + { + "speaker": 11, + "recall": 1.0 + } + ], + "ref_speakers": 12, + "hyp_speakers": 12, + "num_turns": 102, + "audio_duration_secs": 1200.064 + }, + { + "filename": "auzru", + "der_collar": 4.231525157232705, + "der_no_collar": 4.231525157232705, + "miss_rate": 0.6308962264150944, + "false_alarm_rate": 1.4799528301886793, + "confusion_rate": 2.1206761006289305, + "der_single_speaker": 3.7476129853596434, + "der_overlap": 44.243421052631575, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8855665104250848 + }, + { + "speaker": 1, + "recall": 0.9850891017092981 + }, + { + "speaker": 2, + "recall": 0.9986569294048518 + }, + { + "speaker": 3, + "recall": 0.9987364430872907 + }, + { + "speaker": 4, + "recall": 0.7848101265822784 + }, + { + "speaker": 5, + "recall": 0.9665254237288136 + }, + { + "speaker": 6, + "recall": 0.953386744355426 + }, + { + "speaker": 7, + "recall": 0.9996791786974655 + } + ], + "ref_speakers": 8, + "hyp_speakers": 8, + "num_turns": 27, + "audio_duration_secs": 515.904 + }, + { + "filename": "bgvvt", + "der_collar": 9.717425275458101, + "der_no_collar": 9.717425275458101, + "miss_rate": 5.512148025759189, + "false_alarm_rate": 3.9405189660492295, + "confusion_rate": 0.264758283649683, + "der_single_speaker": 9.688016425459463, + "der_overlap": 14.423076923076922, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9760991606914569 + }, + { + "speaker": 1, + "recall": 0.9348432309221854 + } + ], + "ref_speakers": 2, + "hyp_speakers": 3, + "num_turns": 244, + "audio_duration_secs": 1200.064 + }, + { + "filename": "bidnq", + "der_collar": 24.311260202329695, + "der_no_collar": 24.311260202329695, + "miss_rate": 9.752854891395264, + "false_alarm_rate": 5.442122965036935, + "confusion_rate": 9.116282345897497, + "der_single_speaker": 13.301337384618925, + "der_overlap": 55.325765239219216, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7739699403990671 + }, + { + "speaker": 1, + "recall": 0.9542556857339766 + }, + { + "speaker": 2, + "recall": 0.42072025942993685 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.47384615384615386 + }, + { + "speaker": 5, + "recall": 0.4412278997474257 + }, + { + "speaker": 6, + "recall": 0.046511627906976744 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 0.7969804047542564 + }, + { + "speaker": 9, + "recall": 0.0 + }, + { + "speaker": 10, + "recall": 0.0 + } + ], + "ref_speakers": 11, + "hyp_speakers": 9, + "num_turns": 312, + "audio_duration_secs": 1029.888 + }, + { + "filename": "bjruf", + "der_collar": 24.88120486456012, + "der_no_collar": 24.88120486456012, + "miss_rate": 10.926467824639587, + "false_alarm_rate": 4.244409245885795, + "confusion_rate": 9.710327794034738, + "der_single_speaker": 17.594772160348523, + "der_overlap": 48.486228090143406, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8055615434360853 + }, + { + "speaker": 1, + "recall": 0.7817899748568983 + } + ], + "ref_speakers": 2, + "hyp_speakers": 8, + "num_turns": 159, + "audio_duration_secs": 360.512 + }, + { + "filename": "bmsyn", + "der_collar": 36.924353019243526, + "der_no_collar": 36.924353019243526, + "miss_rate": 11.678832116788321, + "false_alarm_rate": 12.54976775049768, + "confusion_rate": 12.69575315195753, + "der_single_speaker": 31.29219377957277, + "der_overlap": 61.3573548981472, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8178500331785004 + }, + { + "speaker": 1, + "recall": 0.8179977320589664 + }, + { + "speaker": 2, + "recall": 0.8244316253817442 + }, + { + "speaker": 3, + "recall": 0.5955015019415342 + }, + { + "speaker": 4, + "recall": 0.2878581173260573 + } + ], + "ref_speakers": 5, + "hyp_speakers": 12, + "num_turns": 267, + "audio_duration_secs": 710.912 + }, + { + "filename": "bpzsc", + "der_collar": 11.487552426026395, + "der_no_collar": 11.487552426026395, + "miss_rate": 2.7261533725807072, + "false_alarm_rate": 2.9041132786948323, + "confusion_rate": 5.857285774750856, + "der_single_speaker": 10.634659102759297, + "der_overlap": 21.63234929297941, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8020400079223609 + }, + { + "speaker": 1, + "recall": 0.9482027427052498 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.4927797833935018 + }, + { + "speaker": 5, + "recall": 0.0 + }, + { + "speaker": 6, + "recall": 0.3605150214592275 + } + ], + "ref_speakers": 7, + "hyp_speakers": 5, + "num_turns": 201, + "audio_duration_secs": 1052.992 + } + ], + "rust": [ + { + "filename": "aepyx", + "der_collar": 15.761089389240935, + "der_no_collar": 15.761089389240935, + "miss_rate": 3.6200620196845086, + "false_alarm_rate": 5.480652554941351, + "confusion_rate": 6.660374814615073, + "der_single_speaker": 14.121220683808986, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8495903539959808 + }, + { + "speaker": 1, + "recall": 0.9518407038666358 + }, + { + "speaker": 2, + "recall": 0.9823982398239824 + }, + { + "speaker": 3, + "recall": 0.8599640933572711 + } + ], + "ref_speakers": 4, + "hyp_speakers": 4, + "num_turns": 29, + "audio_duration_secs": 169.152 + }, + { + "filename": "aggyz", + "der_collar": 18.025561921551343, + "der_no_collar": 18.025561921551343, + "miss_rate": 0.34456508674225733, + "false_alarm_rate": 2.704435273849113, + "confusion_rate": 14.976561560959976, + "der_single_speaker": 17.411899220057986, + "der_overlap": 15.319148936170212, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 1.0 + }, + { + "speaker": 1, + "recall": 0.0 + }, + { + "speaker": 2, + "recall": 0.8731998595012294 + }, + { + "speaker": 3, + "recall": 0.993075228202707 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 1.0 + }, + { + "speaker": 6, + "recall": 0.5270935960591133 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 1.0 + }, + { + "speaker": 9, + "recall": 1.0 + }, + { + "speaker": 10, + "recall": 1.0 + }, + { + "speaker": 11, + "recall": 1.0 + }, + { + "speaker": 12, + "recall": 1.0 + } + ], + "ref_speakers": 13, + "hyp_speakers": 13, + "num_turns": 27, + "audio_duration_secs": 262.592 + }, + { + "filename": "aiqwk", + "der_collar": 12.395950506186727, + "der_no_collar": 12.395950506186727, + "miss_rate": 2.2159730033745784, + "false_alarm_rate": 5.821147356580427, + "confusion_rate": 4.358830146231721, + "der_single_speaker": 11.423955688899145, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.867901972504483 + }, + { + "speaker": 1, + "recall": 0.9968150265414455 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.6268382352941176 + }, + { + "speaker": 4, + "recall": 0.7909604519774012 + }, + { + "speaker": 5, + "recall": 0.5355329949238579 + }, + { + "speaker": 6, + "recall": 1.0 + } + ], + "ref_speakers": 7, + "hyp_speakers": 9, + "num_turns": 22, + "audio_duration_secs": 198.848 + }, + { + "filename": "aorju", + "der_collar": 10.848308421504413, + "der_no_collar": 10.848308421504413, + "miss_rate": 1.930433484483756, + "false_alarm_rate": 4.02694582843494, + "confusion_rate": 4.890929108585717, + "der_single_speaker": 10.636043065611284, + "der_overlap": 36.36363636363637, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.025653923541247486 + }, + { + "speaker": 1, + "recall": 0.945908137094894 + }, + { + "speaker": 2, + "recall": 0.9945992960880294 + }, + { + "speaker": 3, + "recall": 0.9895406213079435 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 0.9527235354573484 + }, + { + "speaker": 6, + "recall": 0.9791422239603702 + }, + { + "speaker": 7, + "recall": 0.9062317996505533 + }, + { + "speaker": 8, + "recall": 0.9613992762364294 + }, + { + "speaker": 9, + "recall": 1.0 + }, + { + "speaker": 10, + "recall": 0.9807480748074807 + }, + { + "speaker": 11, + "recall": 1.0 + } + ], + "ref_speakers": 12, + "hyp_speakers": 12, + "num_turns": 102, + "audio_duration_secs": 1200.064 + }, + { + "filename": "auzru", + "der_collar": 4.231525157232705, + "der_no_collar": 4.231525157232705, + "miss_rate": 0.6308962264150944, + "false_alarm_rate": 1.4799528301886793, + "confusion_rate": 2.1206761006289305, + "der_single_speaker": 3.7476129853596434, + "der_overlap": 44.243421052631575, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8855665104250848 + }, + { + "speaker": 1, + "recall": 0.9850891017092981 + }, + { + "speaker": 2, + "recall": 0.9986569294048518 + }, + { + "speaker": 3, + "recall": 0.9987364430872907 + }, + { + "speaker": 4, + "recall": 0.7848101265822784 + }, + { + "speaker": 5, + "recall": 0.9665254237288136 + }, + { + "speaker": 6, + "recall": 0.953386744355426 + }, + { + "speaker": 7, + "recall": 0.9996791786974655 + } + ], + "ref_speakers": 8, + "hyp_speakers": 8, + "num_turns": 27, + "audio_duration_secs": 515.904 + }, + { + "filename": "bgvvt", + "der_collar": 9.717425275458101, + "der_no_collar": 9.717425275458101, + "miss_rate": 5.512148025759189, + "false_alarm_rate": 3.9405189660492295, + "confusion_rate": 0.264758283649683, + "der_single_speaker": 9.688016425459463, + "der_overlap": 14.423076923076922, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9760991606914569 + }, + { + "speaker": 1, + "recall": 0.9348432309221854 + } + ], + "ref_speakers": 2, + "hyp_speakers": 3, + "num_turns": 244, + "audio_duration_secs": 1200.064 + }, + { + "filename": "bidnq", + "der_collar": 24.311260202329695, + "der_no_collar": 24.311260202329695, + "miss_rate": 9.752854891395264, + "false_alarm_rate": 5.442122965036935, + "confusion_rate": 9.116282345897497, + "der_single_speaker": 13.301337384618925, + "der_overlap": 55.325765239219216, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7739699403990671 + }, + { + "speaker": 1, + "recall": 0.9542556857339766 + }, + { + "speaker": 2, + "recall": 0.42072025942993685 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.47384615384615386 + }, + { + "speaker": 5, + "recall": 0.4412278997474257 + }, + { + "speaker": 6, + "recall": 0.046511627906976744 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 0.7969804047542564 + }, + { + "speaker": 9, + "recall": 0.0 + }, + { + "speaker": 10, + "recall": 0.0 + } + ], + "ref_speakers": 11, + "hyp_speakers": 9, + "num_turns": 312, + "audio_duration_secs": 1029.888 + }, + { + "filename": "bjruf", + "der_collar": 24.88120486456012, + "der_no_collar": 24.88120486456012, + "miss_rate": 10.926467824639587, + "false_alarm_rate": 4.244409245885795, + "confusion_rate": 9.710327794034738, + "der_single_speaker": 17.594772160348523, + "der_overlap": 48.486228090143406, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8055615434360853 + }, + { + "speaker": 1, + "recall": 0.7817899748568983 + } + ], + "ref_speakers": 2, + "hyp_speakers": 8, + "num_turns": 159, + "audio_duration_secs": 360.512 + }, + { + "filename": "bmsyn", + "der_collar": 36.924353019243526, + "der_no_collar": 36.924353019243526, + "miss_rate": 11.678832116788321, + "false_alarm_rate": 12.54976775049768, + "confusion_rate": 12.69575315195753, + "der_single_speaker": 31.29219377957277, + "der_overlap": 61.3573548981472, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8178500331785004 + }, + { + "speaker": 1, + "recall": 0.8179977320589664 + }, + { + "speaker": 2, + "recall": 0.8244316253817442 + }, + { + "speaker": 3, + "recall": 0.5955015019415342 + }, + { + "speaker": 4, + "recall": 0.2878581173260573 + } + ], + "ref_speakers": 5, + "hyp_speakers": 12, + "num_turns": 267, + "audio_duration_secs": 710.912 + }, + { + "filename": "bpzsc", + "der_collar": 11.487552426026395, + "der_no_collar": 11.487552426026395, + "miss_rate": 2.7261533725807072, + "false_alarm_rate": 2.9041132786948323, + "confusion_rate": 5.857285774750856, + "der_single_speaker": 10.634659102759297, + "der_overlap": 21.63234929297941, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8020400079223609 + }, + { + "speaker": 1, + "recall": 0.9482027427052498 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.4927797833935018 + }, + { + "speaker": 5, + "recall": 0.0 + }, + { + "speaker": 6, + "recall": 0.3605150214592275 + } + ], + "ref_speakers": 7, + "hyp_speakers": 5, + "num_turns": 201, + "audio_duration_secs": 1052.992 + } + ], + "openblas": [ + { + "filename": "aepyx", + "der_collar": 16.057705271673186, + "der_no_collar": 16.057705271673186, + "miss_rate": 3.647027099905622, + "false_alarm_rate": 5.487393824996629, + "confusion_rate": 6.923284346770932, + "der_single_speaker": 14.43204294998587, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8495903539959808 + }, + { + "speaker": 1, + "recall": 0.941884695531373 + }, + { + "speaker": 2, + "recall": 0.9823982398239824 + }, + { + "speaker": 3, + "recall": 0.8599640933572711 + } + ], + "ref_speakers": 4, + "hyp_speakers": 4, + "num_turns": 31, + "audio_duration_secs": 169.152 + }, + { + "filename": "aggyz", + "der_collar": 19.64020994430867, + "der_no_collar": 19.64020994430867, + "miss_rate": 0.34456508674225733, + "false_alarm_rate": 2.716454986177331, + "confusion_rate": 16.579189871389076, + "der_single_speaker": 18.30209481808159, + "der_overlap": 19.361702127659576, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 1.0 + }, + { + "speaker": 1, + "recall": 0.0 + }, + { + "speaker": 2, + "recall": 0.8731998595012294 + }, + { + "speaker": 3, + "recall": 0.9244570349386213 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 1.0 + }, + { + "speaker": 6, + "recall": 0.7009852216748769 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 1.0 + }, + { + "speaker": 9, + "recall": 0.0 + }, + { + "speaker": 10, + "recall": 1.0 + }, + { + "speaker": 11, + "recall": 1.0 + }, + { + "speaker": 12, + "recall": 1.0 + } + ], + "ref_speakers": 13, + "hyp_speakers": 11, + "num_turns": 29, + "audio_duration_secs": 262.592 + }, + { + "filename": "aiqwk", + "der_collar": 13.11586051743532, + "der_no_collar": 13.11586051743532, + "miss_rate": 2.2159730033745784, + "false_alarm_rate": 5.821147356580427, + "confusion_rate": 5.078740157480315, + "der_single_speaker": 12.162474036464344, + "der_overlap": 50.0, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8296473401075911 + }, + { + "speaker": 1, + "recall": 0.9968150265414455 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.6268382352941176 + }, + { + "speaker": 4, + "recall": 0.7909604519774012 + }, + { + "speaker": 5, + "recall": 0.5355329949238579 + }, + { + "speaker": 6, + "recall": 1.0 + } + ], + "ref_speakers": 7, + "hyp_speakers": 9, + "num_turns": 22, + "audio_duration_secs": 198.848 + }, + { + "filename": "aorju", + "der_collar": 10.211983802646065, + "der_no_collar": 10.211983802646065, + "miss_rate": 1.9425628393886805, + "false_alarm_rate": 4.02694582843494, + "confusion_rate": 4.2424751348224445, + "der_single_speaker": 9.996436208125445, + "der_overlap": 36.36363636363637, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.1130114017437961 + }, + { + "speaker": 1, + "recall": 0.945908137094894 + }, + { + "speaker": 2, + "recall": 0.9945992960880294 + }, + { + "speaker": 3, + "recall": 0.9895406213079435 + }, + { + "speaker": 4, + "recall": 0.0 + }, + { + "speaker": 5, + "recall": 0.9527235354573484 + }, + { + "speaker": 6, + "recall": 0.9791422239603702 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 0.9613992762364294 + }, + { + "speaker": 9, + "recall": 1.0 + }, + { + "speaker": 10, + "recall": 0.9807480748074807 + }, + { + "speaker": 11, + "recall": 1.0 + } + ], + "ref_speakers": 12, + "hyp_speakers": 13, + "num_turns": 103, + "audio_duration_secs": 1200.064 + }, + { + "filename": "auzru", + "der_collar": 4.231525157232705, + "der_no_collar": 4.231525157232705, + "miss_rate": 0.6308962264150944, + "false_alarm_rate": 1.4799528301886793, + "confusion_rate": 2.1206761006289305, + "der_single_speaker": 3.7476129853596434, + "der_overlap": 44.243421052631575, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8855665104250848 + }, + { + "speaker": 1, + "recall": 0.9850891017092981 + }, + { + "speaker": 2, + "recall": 0.9986569294048518 + }, + { + "speaker": 3, + "recall": 0.9987364430872907 + }, + { + "speaker": 4, + "recall": 0.7848101265822784 + }, + { + "speaker": 5, + "recall": 0.9665254237288136 + }, + { + "speaker": 6, + "recall": 0.953386744355426 + }, + { + "speaker": 7, + "recall": 0.9996791786974655 + } + ], + "ref_speakers": 8, + "hyp_speakers": 8, + "num_turns": 27, + "audio_duration_secs": 515.904 + }, + { + "filename": "bgvvt", + "der_collar": 9.710457952204163, + "der_no_collar": 9.710457952204163, + "miss_rate": 5.521106012799968, + "false_alarm_rate": 3.9335516427952903, + "confusion_rate": 0.25580029660890424, + "der_single_speaker": 9.681005558615855, + "der_overlap": 14.423076923076922, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9760991606914569 + }, + { + "speaker": 1, + "recall": 0.9348432309221854 + } + ], + "ref_speakers": 2, + "hyp_speakers": 3, + "num_turns": 245, + "audio_duration_secs": 1200.064 + }, + { + "filename": "bidnq", + "der_collar": 22.75372654674802, + "der_no_collar": 22.75372654674802, + "miss_rate": 9.741408912015636, + "false_alarm_rate": 5.4341988254664235, + "confusion_rate": 7.57811880926596, + "der_single_speaker": 12.939944294823103, + "der_overlap": 54.35914727810872, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7739699403990671 + }, + { + "speaker": 1, + "recall": 0.9564266023432115 + }, + { + "speaker": 2, + "recall": 0.5380611025772316 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.47384615384615386 + }, + { + "speaker": 5, + "recall": 0.4412278997474257 + }, + { + "speaker": 6, + "recall": 0.5940803382663847 + }, + { + "speaker": 7, + "recall": 1.0 + }, + { + "speaker": 8, + "recall": 0.7969804047542564 + }, + { + "speaker": 9, + "recall": 0.0 + }, + { + "speaker": 10, + "recall": 0.0 + } + ], + "ref_speakers": 11, + "hyp_speakers": 10, + "num_turns": 309, + "audio_duration_secs": 1029.888 + }, + { + "filename": "bjruf", + "der_collar": 24.736234529786035, + "der_no_collar": 24.736234529786035, + "miss_rate": 10.926467824639587, + "false_alarm_rate": 4.241724610056646, + "confusion_rate": 9.568042095089801, + "der_single_speaker": 17.23641218423919, + "der_overlap": 49.03255178693376, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8055615434360853 + }, + { + "speaker": 1, + "recall": 0.7846252607928101 + } + ], + "ref_speakers": 2, + "hyp_speakers": 8, + "num_turns": 159, + "audio_duration_secs": 360.512 + }, + { + "filename": "bmsyn", + "der_collar": 36.94591904445919, + "der_no_collar": 36.94591904445919, + "miss_rate": 11.647312541473125, + "false_alarm_rate": 12.631055076310552, + "confusion_rate": 12.667551426675514, + "der_single_speaker": 31.319910514541387, + "der_overlap": 60.917187020165834, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.8178500331785004 + }, + { + "speaker": 1, + "recall": 0.8028511258707274 + }, + { + "speaker": 2, + "recall": 0.827281981676281 + }, + { + "speaker": 3, + "recall": 0.613158473148216 + }, + { + "speaker": 4, + "recall": 0.24693042291950887 + } + ], + "ref_speakers": 5, + "hyp_speakers": 13, + "num_turns": 263, + "audio_duration_secs": 710.912 + }, + { + "filename": "bpzsc", + "der_collar": 8.902805032898534, + "der_no_collar": 8.902805032898534, + "miss_rate": 2.647273846627419, + "false_alarm_rate": 2.8723690792258263, + "confusion_rate": 3.3831621070452886, + "der_single_speaker": 7.964001918785326, + "der_overlap": 20.069461672041676, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.9203307585660527 + }, + { + "speaker": 1, + "recall": 0.9525886546245393 + }, + { + "speaker": 2, + "recall": 0.0 + }, + { + "speaker": 3, + "recall": 0.0 + }, + { + "speaker": 4, + "recall": 0.315884476534296 + }, + { + "speaker": 5, + "recall": 0.0 + }, + { + "speaker": 6, + "recall": 0.3605150214592275 + } + ], + "ref_speakers": 7, + "hyp_speakers": 4, + "num_turns": 192, + "audio_duration_secs": 1052.992 + } + ] + }, + "ami1": { + "baseline": [ + { + "filename": "EN2002a.Mix-Headset", + "der_collar": 32.558387586119714, + "der_no_collar": 32.558387586119714, + "miss_rate": 18.03536438910309, + "false_alarm_rate": 2.4492762783641995, + "confusion_rate": 12.073746918652423, + "der_single_speaker": 14.398848092152628, + "der_overlap": 54.15524324324325, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7410377266237703 + }, + { + "speaker": 1, + "recall": 0.619253482984763 + }, + { + "speaker": 2, + "recall": 0.8448318974335659 + }, + { + "speaker": 3, + "recall": 0.5071597913703176 + } + ], + "ref_speakers": 4, + "hyp_speakers": 10, + "num_turns": 844, + "audio_duration_secs": 2142.709375 + } + ], + "rust": [ + { + "filename": "EN2002a.Mix-Headset", + "der_collar": 32.558387586119714, + "der_no_collar": 32.558387586119714, + "miss_rate": 18.03536438910309, + "false_alarm_rate": 2.4492762783641995, + "confusion_rate": 12.073746918652423, + "der_single_speaker": 14.398848092152628, + "der_overlap": 54.15524324324325, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7410377266237703 + }, + { + "speaker": 1, + "recall": 0.619253482984763 + }, + { + "speaker": 2, + "recall": 0.8448318974335659 + }, + { + "speaker": 3, + "recall": 0.5071597913703176 + } + ], + "ref_speakers": 4, + "hyp_speakers": 10, + "num_turns": 844, + "audio_duration_secs": 2142.709375 + } + ], + "openblas": [ + { + "filename": "EN2002a.Mix-Headset", + "der_collar": 31.727213829719993, + "der_no_collar": 31.727213829719993, + "miss_rate": 18.02746349788256, + "false_alarm_rate": 2.448881233803173, + "confusion_rate": 11.250869098034258, + "der_single_speaker": 13.310207910639877, + "der_overlap": 53.63027027027027, + "per_speaker_recall": [ + { + "speaker": 0, + "recall": 0.7400402886702783 + }, + { + "speaker": 1, + "recall": 0.6207869751052236 + }, + { + "speaker": 2, + "recall": 0.8444113568684504 + }, + { + "speaker": 3, + "recall": 0.546932195353248 + } + ], + "ref_speakers": 4, + "hyp_speakers": 12, + "num_turns": 845, + "audio_duration_secs": 2142.709375 + } + ] + } +} diff --git a/docs/strategy/zero-deps.md b/docs/strategy/zero-deps.md index 6617105..f8eb288 100644 --- a/docs/strategy/zero-deps.md +++ b/docs/strategy/zero-deps.md @@ -10,7 +10,7 @@ but the product is not literally zero-dependency or entirely pure Rust. |-------|---------|----------------| | No ONNX Runtime | No `ort` / `ort-sys` or `libonnxruntime` in the selected product graph | Core diarization meets this; companion ASR is separate | | Pure Rust implementation | Selected implementation and transitive dependencies do not compile/link C/C++ or assembly components | BYO algorithm core qualifies; full native/download-enabled product does not | -| No native build/runtime dependencies | No extra native toolchain components or linked native libraries beyond the documented platform runtime | Not a blanket product claim: Darwin C shims/Accelerate, optional Linux BLAS and downloader crypto matter | +| No native build/runtime dependencies | No extra native toolchain components or linked native libraries beyond the documented platform runtime | Not a blanket product claim: Darwin C shims/Accelerate, opt-in Linux OpenBLAS and downloader crypto matter | | Zero external Rust crates | No third-party Cargo dependencies | Not achieved, including with empty default features | Pure Rust does not mean a static executable, absence of libc, or freedom from @@ -40,13 +40,14 @@ Native kernels have platform-specific implementations: - **Darwin:** C shims compiled by `cc`, linked to system Accelerate/BNNS. No ONNX Runtime, but not a pure-Rust inference stack. -- **Linux:** `rten-gemm` and in-crate kernels, with system BLAS currently - selected automatically through `pkg-config` when present. That implicit - host-dependent choice must become explicit before 1.0. +- **Linux:** `rten-gemm` and in-crate kernels, with no system BLAS by default. + `system-openblas` explicitly enables LP64 OpenBLAS via `pkg-config`; missing + prerequisites fail the build. - **Windows:** Rust kernel path; platform/runtime linking and download crypto still need to be considered for the complete artifact. -The kernel crate also has build dependencies such as `cc` and `pkg-config`. +The kernel crate uses `cc` for Darwin shims; `pkg-config` is an optional build +dependency enabled by `system-openblas`. Download-enabled features add `ureq` → `rustls` → `ring`, which includes native crypto code/toolchain requirements. Removing HTTP/TLS via `pipeline-local` does not remove Darwin's C shims or system frameworks. @@ -55,8 +56,8 @@ does not remove Darwin's C shims or system frameworks. 1. Preserve the native product's no-ORT/no-tract graph and the downloader-free local graph. Do not reintroduce a generic ML runtime to reduce one kernel. -2. Make Linux BLAS selection explicit; build/test the Rust fallback and any - opt-in system backend independently with documented linking requirements. +2. Keep Linux backend selection explicit; test Rust kernels and opt-in + OpenBLAS independently, including actual artifact linkage. 3. Reduce avoidable dependencies using existing code or the standard library; account for build dependencies and transitive costs as well as direct crates. 4. Evaluate a fully Rust Darwin kernel path against the same accuracy, speed, @@ -106,3 +107,41 @@ preparation are retained in: - [Library mode and feature inventory](../library-mode.md) These reports describe measured revisions, not current release certification. + +## Linux backend selection + +`cli`, `cli-native`, `ffi` and `pipeline-local` default to Rust Linux kernels, +even when OpenBLAS is installed. Release CLI assets, Python wheels and the +scheduled Linux quality gate use this default. Darwin keeps Accelerate/BNNS. + +To opt into Linux system OpenBLAS: + +```bash +# Debian/Ubuntu prerequisites (LP64, not openblas64 / ILP64): +sudo apt-get install libopenblas-dev pkg-config +cargo build --release --features cli,system-openblas +# Direct kernel consumers: polyvoice-kernels feature system-openblas. +``` + +The feature forwards to enabled native kernels; it does not enable a pipeline +by itself. It has no backend effect on non-Linux targets. Cargo features are +additive: `--all-features` includes this opt-in and requires OpenBLAS on Linux. +For a custom install use `PKG_CONFIG_PATH`; for cross-compilation configure +pkg-config for the target libraries. Generic BLAS is not supported because +the adapter uses `openblas_set_num_threads` and 32-bit CBLAS integer arguments. +The optional backend retains Rust GEMM/INT8 routing where that already wins; +it enables OpenBLAS for the existing CBLAS branches, not every multiplication. +Floating-point differences can change clustering and DER; qualify the selected +backend on your audio rather than assuming bit-identical results. + +CI runs `scripts/check-linux-blas.sh` with OpenBLAS installed for both builds: +kernel numerical tests, unavailable-pkg-config failure, and `readelf`/`ldd` +inspection of actual CLI artifacts. Release Rust artifacts must not import +BLAS/OpenBLAS. Benchmark reports must record `FEATURES`; use +`FEATURES=cli-native,system-openblas` only for explicitly labeled OpenBLAS +comparisons. Historical auto-detected builds need their original linkage +record before assigning them a backend label. + +[Linux backend comparison](../../benchmarks/results/linux-blas-selection-2026-09-24/NOTES.md) +records fixed-subset quality, artifact hashes and the limits of the measured +throughput/RSS evidence. diff --git a/polyvoice-kernels/Cargo.toml b/polyvoice-kernels/Cargo.toml index b82c01d..da464ec 100644 --- a/polyvoice-kernels/Cargo.toml +++ b/polyvoice-kernels/Cargo.toml @@ -7,6 +7,11 @@ description = "Hand-written WeSpeaker ResNet34 kernels for polyvoice — not a g license = "MIT" repository = "https://github.com/ekhodzitsky/polyvoice" +[features] +default = [] +# Linux-only LP64 OpenBLAS; other targets keep their native implementation. +system-openblas = ["dep:pkg-config"] + [dependencies] # Pure-Rust GEMM (INT8 + FP32, NEON/dotprod/AVX). Justified: Linux has no # BNNS/AMX; OpenBLAS in the Desktop VM is ~10 GFLOP/s. Not a second ONNX @@ -22,7 +27,7 @@ memmap2 = "0.9" [build-dependencies] cc = "1" -pkg-config = "0.3" +pkg-config = { version = "0.3", optional = true } [lints.clippy] unwrap_used = "deny" diff --git a/polyvoice-kernels/build.rs b/polyvoice-kernels/build.rs index 8934660..249a73e 100644 --- a/polyvoice-kernels/build.rs +++ b/polyvoice-kernels/build.rs @@ -10,16 +10,20 @@ fn main() { .compile("pv_bnns_conv"); println!("cargo:rustc-link-lib=framework=Accelerate"); } + #[cfg(feature = "system-openblas")] if std::env::var("CARGO_CFG_TARGET_OS").ok().as_deref() == Some("linux") { - // System OpenBLAS — same role as Accelerate on Apple. Missing pkg is - // fine: GEMM stays on the in-crate kernel. - let mut probed = pkg_config::Config::new(); - probed.cargo_metadata(true); - if probed.probe("openblas").is_ok() - || probed.probe("openblas64").is_ok() - || probed.probe("blas").is_ok() - { - println!("cargo:rustc-cfg=linux_cblas"); + // Only the LP64 OpenBLAS ABI provides the i32 CBLAS arguments and + // openblas_set_num_threads used by linux_cblas. Generic BLAS and + // ILP64 openblas64 are not interchangeable with this interface. + if let Err(error) = pkg_config::Config::new().probe("openblas") { + eprintln!( + "system-openblas requires LP64 OpenBLAS development files and pkg-config \ + (for example, libopenblas-dev and pkg-config on Debian/Ubuntu). \ + Set PKG_CONFIG_PATH for a non-system installation, or disable \ + system-openblas to use Rust kernels.\n{error}" + ); + std::process::exit(1); } + println!("cargo:rustc-cfg=linux_cblas"); } } diff --git a/polyvoice-kernels/src/linux_cblas.rs b/polyvoice-kernels/src/linux_cblas.rs index 321ebe3..6d9fc22 100644 --- a/polyvoice-kernels/src/linux_cblas.rs +++ b/polyvoice-kernels/src/linux_cblas.rs @@ -1,6 +1,6 @@ //! System OpenBLAS `cblas_sgemm` on Linux (same role as Accelerate on Apple). //! -//! Linked only when `build.rs` finds OpenBLAS via pkg-config. Callers pin +//! Linked only with `system-openblas` and LP64 OpenBLAS via pkg-config. Callers pin //! `OPENBLAS_NUM_THREADS=1` so the BLAS pool does not fight window/embed workers. #![cfg(linux_cblas)] @@ -77,3 +77,18 @@ pub fn pin_to_one_thread() { } }); } + +#[cfg(test)] +mod tests { + #[test] + fn lp64_sgemm_preserves_row_major_layout_and_scaling() { + let a = [1.0, -2.0, 3.0, 4.0, 5.0, -6.0]; + let b = [1.0, 2.0, 3.0, 4.0, -5.0, 6.0]; + let mut c = [2.0, 4.0, 6.0, 8.0]; + // SAFETY: A is 2x3, B is 3x2, and C is 2x2; all dimensions fit i32. + unsafe { + super::sgemm_rowmajor(a.as_ptr(), b.as_ptr(), c.as_mut_ptr(), 2, 2, 3, 0.5, 2.0); + } + assert_eq!(c, [-6.0, 14.0, 36.5, 12.0]); + } +} diff --git a/python/Cargo.lock b/python/Cargo.lock index 1299280..ec771e2 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -327,12 +327,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - [[package]] name = "polyvoice" version = "0.22.0" @@ -359,7 +353,6 @@ version = "0.1.2" dependencies = [ "cc", "memmap2", - "pkg-config", "rten-gemm", "rten-tensor", "thiserror", diff --git a/scripts/check-linux-blas.sh b/scripts/check-linux-blas.sh new file mode 100755 index 0000000..edaa88b --- /dev/null +++ b/scripts/check-linux-blas.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Linux backend contract. Requires LP64 OpenBLAS development files, pkg-config, +# readelf and ldd. Checks the actual CLI artifact as well as kernel arithmetic. +set -euo pipefail +cd "$(dirname "$0")/.." +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# No pkg-config executable is needed for the default Rust kernel build. +PKG_CONFIG=/bin/false cargo check --locked -p polyvoice-kernels + +# An explicit backend must fail, rather than silently change implementation. +if PKG_CONFIG=/bin/false cargo check --locked -p polyvoice-kernels \ + --features system-openblas >"$work/missing.log" 2>&1; then + echo 'FAIL: system-openblas succeeded without pkg-config' >&2 + exit 1 +fi +if ! grep -q 'system-openblas requires LP64 OpenBLAS' "$work/missing.log"; then + cat "$work/missing.log" >&2 + exit 1 +fi + +# OpenBLAS is installed for BOTH builds: its presence must not affect default. +pkg-config --exists openblas +for backend in rust openblas; do + features=cli + kernel_features=() + if [[ "$backend" == openblas ]]; then + features+=,system-openblas + kernel_features=(--features system-openblas) + fi + cargo test --locked --release -p polyvoice-kernels "${kernel_features[@]}" + cargo build --locked --release --no-default-features --features "$features" --bin polyvoice + binary="${CARGO_TARGET_DIR:-target}/release/polyvoice" + readelf -d "$binary" >"$work/dynamic.txt" + ldd "$binary" >"$work/ldd.txt" + cat "$work/dynamic.txt" "$work/ldd.txt" >"$work/linkage.txt" + if [[ "$backend" == rust ]]; then + if grep -Eiq 'blas|lapack' "$work/linkage.txt"; then + cat "$work/linkage.txt" >&2 + echo 'FAIL: default artifact links BLAS' >&2 + exit 1 + fi + else + grep -q 'libopenblas' "$work/dynamic.txt" + if grep -q 'not found' "$work/ldd.txt"; then + cat "$work/ldd.txt" >&2 + exit 1 + fi + fi + "$binary" --version + echo "OK: $backend kernel tests and CLI linkage" +done diff --git a/scripts/linux-cpu-native-der-gate.sh b/scripts/linux-cpu-native-der-gate.sh index fd0d4ac..55c7cdf 100755 --- a/scripts/linux-cpu-native-der-gate.sh +++ b/scripts/linux-cpu-native-der-gate.sh @@ -3,8 +3,9 @@ # # Same protocol as scripts/linux-cpu-der-gate.sh (v2+VBx, INT8 pair, EP=cpu, # powerset N=8). Builds `--features cli-native` so libonnxruntime is not -# linked. On Linux, GEMM uses system OpenBLAS when pkg-config finds it -# (Docker image installs libopenblas-dev). +# linked. Linux defaults to Rust kernels, matching release assets. To measure +# the optional LP64 OpenBLAS path, set FEATURES=cli-native,system-openblas; +# install its development files and pkg-config first. Reports record FEATURES. # # Usage: # bash scripts/linux-cpu-native-der-gate.sh