feat(cli): audio answers beats, downbeats, bpm, sections and describe through a provider ladder

docs/packages/cli.mdx

@@ -540,9 +540,57 @@ For a file not in the film yet, `--place-at <s>` answers as if it started at tha
 film second; `--skip-file-seconds <s>` cuts that much from the start of the file
 first, and the result warns when the cut part held the file's loudest moment.
 
-Every measurement comes from a provider ladder. Today the only provider is
-local signal processing; `--quality fast|local|best` limits which providers may
-run, `--provider onsets=signal` pins one, and `--full` shows each ladder step.
+`--only` also asks for the music's structure, which is not measured unless asked:
+
+- **beats**: beat times, or none with a `reason` (`no steady beat`, or too short
+  to tell) when the music has no beat to follow.
+- **downbeats**: the first beat of each bar, with the `meter` (3 or 4). From
+  signal processing these are a guess (`confidence: "low"`).
+- **bpm**: the tempo read off the beats (median interval), with a `curve` every
+  4 s for music that speeds up or slows down. It always comes from the provider
+  that answered `beats`.
+- **sections**: spans where the sound stays alike, each with a letter; a part
+  that returns gets its earlier letter.
+- **describe**: labels for what the audio holds, each with a score.
+
+Word timing is not here: use [`transcribe`](#transcribe).
+
+Every measurement comes from a provider ladder, tried in this order until one
+answers:
+
+| Capability | Ladder |
+| --- | --- |
+| moments, onsets, loudness, silence, envelope, kind | `signal` |
+| beats, downbeats (and so bpm) | `api`, `apple-mu`, `beat-this`, `signal` |
+| sections | `api`, `apple-mu`, `signal` |
+| describe | `api`, `apple-sound` |
+
+- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and
+  `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,
+  downbeat or section boundary it answers moves onto the nearest beat signal
+  processing measured, when one is within a quarter beat; the rest are listed in
+  `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.
+- `apple-mu` is Apple's Music Understanding, which needs macOS 27. It is not
+  available yet.
+- `beat-this` is [Beat This!](https://github.com/CPJKU/beat_this), used when
+  `HYPERFRAMES_BEAT_THIS_PYTHON` names a Python with `beat-this` installed and
+  its `small0` weights are on disk (`HYPERFRAMES_BEAT_THIS_CHECKPOINT`, or
+  `beat_this-small0.ckpt` in the Torch hub cache). The CLI never downloads them;
+  to set it up once:
+
+  ```bash
+  uv init --bare ~/.beat-this
+  uv add --project ~/.beat-this "beat-this @ git+https://github.com/CPJKU/beat_this" torch torchaudio soundfile
+  ~/.beat-this/.venv/bin/python -c "from beat_this.inference import load_checkpoint; load_checkpoint('small0')"
+  export HYPERFRAMES_BEAT_THIS_PYTHON=~/.beat-this/.venv/bin/python
+  ```
+- `apple-sound` is macOS's built-in sound classifier, through a small Swift
+  helper compiled once with the Xcode command line tools.
+
+A provider that cannot run, or fails, is passed over with a warning that says
+why. `--quality fast|local|best` limits which providers may run (`local` keeps
+the audio on this machine, `fast` allows only signal processing),
+`--provider beats=signal` pins one, and `--full` shows each ladder step.
 Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`,
 `--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needs
 FFmpeg.

packages/cli/src/audio/analysis/analyze.test.ts

@@ -38,10 +38,48 @@ describe("analyzeSignal", () => {
     expect(analysis.values.moments).toEqual({ onset: 1.1, steepest: 1.1, peak: 1.1, tailEnd: 1.2 });
   });
 
-  it("warns, and answers nothing, for a capability no provider offers yet", async () => {
-    const analysis = await analyzeSignal(tone, { capabilities: ["beats"] });
-    expect(analysis.values.beats).toBeNull();
-    expect(analysis.warnings).toEqual(["beats: no provider answered (none offers it)"]);
+  it("warns, and answers nothing, for a capability no provider offers", async () => {
+    const analysis = await analyzeSignal(tone, { capabilities: ["describe"] }, [signalProvider]);
+    expect(analysis.values.describe).toBeNull();
+    expect(analysis.warnings).toEqual(["describe: no provider answered (none offers it)"]);
+  });
+
+  it("reads tempo off the beats of whichever provider placed them, in file seconds", async () => {
+    const grid: AudioAnalysisProvider<AudioSignal> = {
+      id: "grid",
+      local: true,
+      capabilities: ["beats"],
+      analyze: async () => ({ times: [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4] }),
+    };
+    const analysis = await analyzeSignal(tone, { capabilities: ["bpm", "beats"], fileOffset: 10 }, [
+      grid,
+      signalProvider,
+    ]);
+    expect(analysis.provenance).toEqual({ bpm: "grid", beats: "grid" });
+    expect(analysis.values.bpm).toEqual({
+      value: 120,
+      curve: [
+        { t: 10, bpm: 120 },
+        { t: 14, bpm: 120 },
+      ],
+    });
+    expect((analysis.values.beats as { times: number[] }).times[0]).toBe(10);
+  });
+
+  it("keeps a service off the ladder at quality fast", async () => {
+    const service: AudioAnalysisProvider<AudioSignal> = {
+      id: "service",
+      local: false,
+      capabilities: ["beats"],
+      analyze: async () => ({ times: [1] }),
+    };
+    const analysis = await analyzeSignal(
+      tone,
+      { capabilities: ["beats"], quality: "fast", options: {} },
+      [service, signalProvider],
+    );
+    expect(analysis.provenance.beats).toBe("signal");
+    expect(analysis.warnings).toEqual([]);
   });
 });
 

packages/cli/src/audio/analysis/analyze.ts

@@ -2,12 +2,19 @@ import {
   runLadder,
   signalDuration,
   signalProvider,
+  tempoOf,
   type AnalysisQuality,
   type AudioAnalysisProvider,
   type AudioCapability,
   type AudioSignal,
+  type Beats,
+  type LadderResult,
   type LadderStep,
 } from "@hyperframes/core/audio-analysis";
+import { apiProvider } from "./providers/api.js";
+import { appleMuProvider, appleSoundProvider } from "./providers/apple.js";
+import { beatThisProvider } from "./providers/beatThis.js";
+import { shiftedTimes } from "./fileTime.js";
 
 /** What `hyperframes audio` asks for when `--only` is not given. */
 export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
@@ -19,9 +26,12 @@ export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
   "envelope",
 ];
 
-/** Every provider the CLI knows, in default ladder order. Model and service providers join this list. */
+/**
+ * Every provider the CLI knows, in default ladder order: each capability tries the ones that offer it in this order.
+ * beats, downbeats: api, apple-mu, beat-this, signal. sections: api, apple-mu, signal. describe: api, apple-sound.
+ */
 function audioAnalysisProviders(): AudioAnalysisProvider<AudioSignal>[] {
-  return [signalProvider];
+  return [apiProvider, appleMuProvider, beatThisProvider, appleSoundProvider, signalProvider];
 }
 
 export interface AnalyzeOptions {
@@ -54,36 +64,6 @@ export function envelopeHopMs(seconds: number, asked?: number): number {
 
 const round = (t: number) => Math.round(t * 1000) / 1000;
 
-/** Move a capability's times from signal seconds to file seconds. */
-function toFileTime(capability: AudioCapability, value: unknown, offset: number): unknown {
-  if (value === null || typeof value !== "object") return value;
-  const at = (t: number) => round(t + offset);
-  switch (capability) {
-    case "moments": {
-      const m = value as Record<string, number>;
-      return Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)]));
-    }
-    case "onsets": {
-      const o = value as { events: { t: number; strength: number }[] };
-      return { ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) };
-    }
-    case "silence":
-      return (value as { start: number; end: number }[]).map((s) => ({
-        start: at(s.start),
-        end: at(s.end),
-      }));
-    case "loudness": {
-      const l = value as { clipping: { count: number; firstAt: number | null } };
-      const firstAt = l.clipping.firstAt === null ? null : at(l.clipping.firstAt);
-      return { ...l, clipping: { ...l.clipping, firstAt } };
-    }
-    case "envelope":
-      return { startsAt: round(offset), ...(value as object) };
-    default:
-      return value;
-  }
-}
-
 function within(value: unknown, from: number, to: number): unknown {
   if (!value || typeof value !== "object" || !("events" in value)) return value;
   const onsets = value as { events: { t: number }[] };
@@ -123,6 +103,9 @@ export async function analyzeSignal(
   request: AnalyzeOptions,
   providers: readonly AudioAnalysisProvider<AudioSignal>[] = audioAnalysisProviders(),
 ): Promise<Analysis> {
+  const { beats, bpm } = request.provider ?? {};
+  if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider");
+  const beatProvider = beats ?? bpm;
   const framing = frame(signal, request.context);
   const analysis: Analysis = {
     analyzed: round(signalDuration(framing.windowed)),
@@ -131,20 +114,37 @@ export async function analyzeSignal(
     ladder: {},
     warnings: [],
   };
+  const ladders = new Map<AudioCapability, Promise<LadderResult>>();
+  const ladderFor = (capability: AudioCapability) => {
+    let result = ladders.get(capability);
+    if (!result) {
+      const withContext = capability === "onsets" && framing.cut;
+      result = runLadder(capability, withContext ? signal : framing.windowed, providers, {
+        quality: request.quality,
+        provider: capability === "beats" ? beatProvider : request.provider?.[capability],
+        options: optionsFor(capability, request, analysis.analyzed),
+      });
+      ladders.set(capability, result);
+    }
+    return result;
+  };
   for (const capability of request.capabilities) {
-    // Onsets see the context, so a window's edge is not mistaken for a hit, then keep only the window's own.
     const withContext = capability === "onsets" && framing.cut;
-    const result = await runLadder(capability, withContext ? signal : framing.windowed, providers, {
-      quality: request.quality,
-      provider: request.provider?.[capability],
-      options: optionsFor(capability, request, analysis.analyzed),
-    });
+    const result =
+      capability === "bpm" ? tempoFromBeats(await ladderFor("beats")) : await ladderFor(capability);
     const value = withContext ? within(result.value, framing.before, framing.end) : result.value;
     const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);
-    analysis.values[capability] = toFileTime(capability, value, offset);
+    analysis.values[capability] = shiftedTimes(capability, value, offset);
     analysis.provenance[capability] = result.provenance;
     analysis.ladder[capability] = result.steps;
-    analysis.warnings.push(...result.warnings);
+    const said = new Set(analysis.warnings);
+    analysis.warnings.push(...result.warnings.filter((w) => !said.has(w)));
   }
   return analysis;
 }
+
+/** Tempo is read off the beats of whichever provider placed them, so the two never disagree. */
+function tempoFromBeats(beats: LadderResult): LadderResult {
+  const times = (beats.value as Beats | null)?.times;
+  return { ...beats, value: times ? tempoOf(times) : null };
+}

packages/cli/src/audio/analysis/fileTime.ts

@@ -0,0 +1,57 @@
+import type {
+  AudioCapability,
+  Onsets,
+  SnapReport,
+  Span,
+  Tempo,
+} from "@hyperframes/core/audio-analysis";
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+type At = (t: number) => number;
+
+/** A snap report's kept times are times too. */
+function withSnap<V extends { snap?: SnapReport }>(value: V, at: At): V {
+  return value.snap
+    ? { ...value, snap: { ...value.snap, unsnapped: value.snap.unsnapped.map(at) } }
+    : value;
+}
+
+const SHIFTS: Partial<Record<AudioCapability, (value: never, at: At, offset: number) => unknown>> =
+  {
+    moments: (m: Record<string, number>, at) =>
+      Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)])),
+    onsets: (o: Onsets, at) => ({ ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) }),
+    silence: (spans: Span[], at) => spans.map((s) => ({ start: at(s.start), end: at(s.end) })),
+    loudness: (l: { clipping: { count: number; firstAt: number | null } }, at) => ({
+      ...l,
+      clipping: {
+        ...l.clipping,
+        firstAt: l.clipping.firstAt === null ? null : at(l.clipping.firstAt),
+      },
+    }),
+    envelope: (e: object, _at, offset) => ({ startsAt: round(offset), ...e }),
+    beats: (b: { times: number[]; snap?: SnapReport }, at) =>
+      withSnap({ ...b, times: b.times.map(at) }, at),
+    downbeats: (b: { times: number[]; snap?: SnapReport }, at) =>
+      withSnap({ ...b, times: b.times.map(at) }, at),
+    bpm: (tempo: Tempo, at) => ({
+      ...tempo,
+      curve: tempo.curve.map((p) => ({ ...p, t: at(p.t) })),
+    }),
+    sections: (s: { spans: Span[]; snap?: SnapReport }, at) =>
+      withSnap(
+        {
+          ...s,
+          spans: s.spans.map((span) => ({ ...span, start: at(span.start), end: at(span.end) })),
+        },
+        at,
+      ),
+  };
+
+/** Move a capability's times from signal seconds to file seconds. */
+export function shiftedTimes(capability: AudioCapability, value: unknown, offset: number): unknown {
+  if (value === null || typeof value !== "object") return value;
+  const shift = SHIFTS[capability];
+  return shift ? shift(value as never, (t) => round(t + offset), offset) : value;
+}

packages/cli/src/audio/analysis/format.ts

@@ -1,10 +1,15 @@
 import type {
+  Beats,
+  Downbeats,
   Envelope,
   KindGuess,
   Loudness,
   Moments,
   Onsets,
+  Sections,
+  SnapReport,
   Span,
+  Tempo,
 } from "@hyperframes/core/audio-analysis";
 import type { AudioItem, AudioReport } from "./run.js";
 import { filmTime } from "./targets.js";
@@ -60,13 +65,62 @@ function silenceLine(silence: Span[], at: At): string | null {
     : null;
 }
 
+function snapNote(snap: SnapReport | undefined): string {
+  if (!snap) return "";
+  const total = snap.snapped + snap.unsnapped.length;
+  const median = snap.medianShiftMs === null ? "" : `, median ${snap.medianShiftMs} ms`;
+  return `; ${snap.snapped}/${total} snapped to ${snap.reference}${median}`;
+}
+
+const listed = (times: number[], at: At) => {
+  const shown = times.slice(0, ONSETS_SHOWN).map(at).join(" ");
+  return times.length > ONSETS_SHOWN ? `${shown} +${times.length - ONSETS_SHOWN}` : shown;
+};
+
+function beatsLine(beats: Beats & { snap?: SnapReport }, at: At): string {
+  if (!beats.times.length) return `  beats none${beats.reason ? ` (${beats.reason})` : ""}`;
+  return `  beats ${beats.times.length}: ${listed(beats.times, at)}${snapNote(beats.snap)}`;
+}
+
+function downbeatsLine(downbeats: Downbeats & { snap?: SnapReport }, at: At): string | null {
+  if (!downbeats.times.length) return null;
+  const meter = downbeats.meter ? ` in ${downbeats.meter}` : "";
+  const low = downbeats.confidence === "low" ? ", a guess" : "";
+  return `  downbeats${meter}${low}: ${listed(downbeats.times, at)}${snapNote(downbeats.snap)}`;
+}
+
+function tempoLine(tempo: Tempo): string | null {
+  if (tempo.value === null) return null;
+  const bpms = tempo.curve.map((p) => p.bpm);
+  const spread =
+    bpms.length && Math.max(...bpms) - Math.min(...bpms) >= 2
+      ? ` (${Math.min(...bpms)}–${Math.max(...bpms)})`
+      : "";
+  return `  tempo ${tempo.value} BPM${spread}`;
+}
+
+function sectionsLine(sections: Sections, at: At): string {
+  const spans = sections.spans.map((s) => `${s.label} ${at(s.start)}–${at(s.end)}`).join("  ");
+  return `  sections ${spans}${snapNote(sections.snap)}`;
+}
+
+function describeLine(description: { text?: string; tags: { label: string; score: number }[] }) {
+  const tags = description.tags.map((t) => `${t.label} ${t.score.toFixed(2)}`).join(", ");
+  return `  sounds like ${[description.text, tags].filter(Boolean).join(" — ")}`;
+}
+
 /** One line per measurement the item holds, in reading order. */
 const LINES: [string, (value: never, at: At) => string | null][] = [
   ["moments", momentsLine],
   ["onsets", hitsLine],
   ["loudness", loudnessLine],
   ["silence", silenceLine],
   ["envelope", (envelope: Envelope) => `  ${sparkline(envelope)}`],
+  ["describe", describeLine],
+  ["bpm", tempoLine],
+  ["beats", beatsLine],
+  ["downbeats", downbeatsLine],
+  ["sections", sectionsLine],
 ];
 
 function headline(item: AudioItem, where: string): string {