diff --git a/electron/media/audioPeaks.test.ts b/electron/media/audioPeaks.test.ts index 485fbb034..0376651c2 100644 --- a/electron/media/audioPeaks.test.ts +++ b/electron/media/audioPeaks.test.ts @@ -1,9 +1,17 @@ // @vitest-environment node +import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + countAudioStreams, + decodePeaks, + ffmpegCandidates, + mediaClockAudioArgs, + peakBlockCount, + resolveFfmpeg, +} from "./audioPeaks"; const ROOT = path.resolve(__dirname, "..", ".."); @@ -311,3 +319,110 @@ describe.runIf(staged)("decoding a real file", () => { expect(mn).toBeCloseTo(-0.205, 1); }, 120_000); }); + +describe("countAudioStreams", () => { + it("counts the audio streams in ffmpeg's listing, and nothing else", () => { + // The layout of an older macOS recording, as ffmpeg 8.1 lists it. + const listing = [ + "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4':", + " Duration: 00:00:12.00, start: 0.000000, bitrate: 5012 kb/s", + " Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p, 60 fps (default)", + " Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo (default)", + " Metadata:", + " handler_name : Audio: system", + " Stream #0:2[0x3](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, mono (default)", + "At least one output file must be specified", + ].join("\n"); + expect(countAudioStreams(listing)).toBe(2); + expect(countAudioStreams(" Stream #0:0: Video: ffv1, 16x16")).toBe(0); + }); +}); + +const ffmpegOnPath = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0; + +describe.skipIf(!ffmpegOnPath)("peaks on the media clock, with real FFmpeg", () => { + let fixtureDir: string; + let delayed: string; + let twoTracks: string; + + /** A four-second video from zero, muxed with the given audio inputs, one stream each. */ + function makeFixture(filename: string, audioInputs: string[][]): string { + const filePath = path.join(fixtureDir, filename); + const result = spawnSync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:r=1:d=4", + ...audioInputs.flat(), + "-map", + "0:v", + ...audioInputs.flatMap((_, i) => ["-map", `${i + 1}:a`]), + "-c:v", + "ffv1", + "-c:a", + "pcm_s16le", + filePath, + ], + { encoding: "utf8", timeout: 10_000 }, + ); + if (result.status !== 0) throw new Error(result.error?.message ?? result.stderr); + return filePath; + } + + /** Seconds into the clip of the first block with signal in it, or -1. */ + function firstAudibleSec(peaks: Float32Array): number { + for (let i = 0; i < peaks.length; i += 2) { + if (peaks[i + 1] > 0.01) return i / 2 / 200; + } + return -1; + } + + beforeAll(() => { + fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-peaks-clock-")); + const delayedTone = (sec: number, duration: number) => [ + "-itsoffset", + String(sec), + "-f", + "lavfi", + "-i", + `sine=frequency=440:sample_rate=16000:duration=${duration}`, + ]; + // Video from zero, audio from two seconds, no encoded silence in between. + delayed = makeFixture("delayed.mkv", [delayedTone(2, 2)]); + // Older macOS layout: a silent stereo system track, which is the stream ffmpeg + // picks on its own, and a mono microphone speaking from the first second. + twoTracks = makeFixture("two-tracks.mkv", [ + ["-f", "lavfi", "-i", "anullsrc=r=16000:cl=stereo:d=4"], + delayedTone(1, 3), + ]); + }); + + afterAll(() => { + if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true }); + }); + + it("draws audio that starts after the video where it plays, not at zero", async () => { + // Pulled to zero, the waveform runs two seconds ahead of the transcript and of + // the export, both of which keep the offset. + const peaks = await decodePeaks("ffmpeg", delayed, 4); + expect(peaks.length).toBe(peakBlockCount(4) * 2); + expect(firstAudibleSec(peaks)).toBe(2); + }); + + it("draws every audio stream, each on the media clock, as the export mixes them", async () => { + const peaks = await decodePeaks("ffmpeg", twoTracks, 4); + expect(firstAudibleSec(peaks)).toBe(1); + }); + + it("kills the stream probe on abort instead of waiting it out", async () => { + // Killed before it lists anything, the probe counts no stream at all. + const args = await mediaClockAudioArgs("ffmpeg", twoTracks, AbortSignal.abort()); + expect(args).not.toContain("-filter_complex"); + }); +}); diff --git a/electron/media/audioPeaks.ts b/electron/media/audioPeaks.ts index 604afd051..317747edf 100644 --- a/electron/media/audioPeaks.ts +++ b/electron/media/audioPeaks.ts @@ -147,6 +147,69 @@ export function resolveFfmpeg(here?: string): string | null { return found; } +/** Listing streams is only opening the file: past this, the open is wedged. */ +const PROBE_TIMEOUT_MS = 10_000; + +/** Pads audio that starts late, so sample zero stays on the media clock. */ +const ON_MEDIA_CLOCK = "aresample=first_pts=0"; + +/** Audio streams in ffmpeg's listing of its input (the stderr of `ffmpeg -i `). */ +export function countAudioStreams(listing: string): number { + return listing.match(/^\s*Stream #0:\d+\S*: Audio:/gm)?.length ?? 0; +} + +/** Any failure counts as zero, an abort included: the caller then checks its signal, + * and otherwise the decode runs as it always did and reports the problem itself. */ +function probeAudioStreams( + ffmpeg: string, + filePath: string, + signal?: AbortSignal, +): Promise { + return new Promise((resolve) => { + let listing = ""; + const child = spawn(ffmpeg, ["-hide_banner", "-i", filePath], { + stdio: ["ignore", "ignore", "pipe"], + timeout: PROBE_TIMEOUT_MS, + killSignal: "SIGKILL", + signal, + }); + child.stderr.on("data", (c: Buffer) => { + listing += c.toString(); + }); + child.once("error", () => resolve(0)); + child.once("close", () => resolve(countAudioStreams(listing))); + }); +} + +/** + * The ffmpeg arguments, after `-i`, that decode a recording's audio the way the + * compositor exports it, so the waveform and the transcript line up with the export: + * + * - On the media clock. Audio that starts after the video is padded, not pulled to + * zero, or everything drawn or transcribed from it lands that much early. + * - Every audio stream, summed. Older macOS recordings carry system audio and the + * microphone as two streams; ffmpeg alone decodes the one it picks, usually the + * system track, silent whenever nothing plays, while the export mixes both. `amix` + * mixes by sample count, not timestamp, so each stream is aligned first, and + * `normalize=0` sums like the compositor instead of averaging. + */ +export async function mediaClockAudioArgs( + ffmpeg: string, + filePath: string, + signal?: AbortSignal, +): Promise { + const streams = await probeAudioStreams(ffmpeg, filePath, signal); + if (streams < 2) return ["-vn", "-af", ON_MEDIA_CLOCK]; + const aligned = Array.from({ length: streams }, (_, i) => `[0:a:${i}]${ON_MEDIA_CLOCK}[a${i}]`); + const labels = aligned.map((_, i) => `[a${i}]`).join(""); + return [ + "-filter_complex", + `${aligned.join(";")};${labels}amix=inputs=${streams}:normalize=0[a]`, + "-map", + "[a]", + ]; +} + /** Number of min/max blocks for a clip of `durationSec`. */ export function peakBlockCount(durationSec: number): number { return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC))); @@ -205,13 +268,14 @@ class PeakFolder { } /** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */ -async function decodePeaks( +export async function decodePeaks( ffmpeg: string, filePath: string, durationSec: number, ): Promise { const blocks = peakBlockCount(durationSec); const folder = new PeakFolder(blocks, durationSec * PCM_RATE); + const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath); const child = spawn( ffmpeg, [ @@ -220,7 +284,7 @@ async function decodePeaks( "error", "-i", filePath, - "-vn", + ...audioArgs, "-ac", "1", "-ar", @@ -265,11 +329,14 @@ async function decodePeaks( * Cache key: path plus size plus mtime. A recording is immutable in practice, * but keying on identity alone would serve stale peaks for a re-encoded or * replaced file, and that failure is silent and confusing. + * + * The leading version changes whenever the decode does: v2 put the peaks on the + * media clock and mixed every audio stream, which the v1 entries do not. */ async function cacheKey(filePath: string): Promise { const info = await stat(filePath); return createHash("sha1") - .update(`${filePath}:${info.size}:${info.mtimeMs}`) + .update(`v2:${filePath}:${info.size}:${info.mtimeMs}`) .digest("hex") .slice(0, 32); } diff --git a/electron/stt/extractAudio.integration.test.ts b/electron/stt/extractAudio.integration.test.ts index 2301f6fd3..f37059243 100644 --- a/electron/stt/extractAudio.integration.test.ts +++ b/electron/stt/extractAudio.integration.test.ts @@ -5,7 +5,10 @@ import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; // Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg. -vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" })); +vi.mock("../media/audioPeaks", async (importOriginal) => ({ + ...(await importOriginal()), + resolveFfmpeg: () => "ffmpeg", +})); import { extractMono16kPcm } from "./extractAudio"; @@ -19,6 +22,7 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => { let fixtureDir: string; let delayedVideo: string; let standaloneAudio: string; + let twoTracks: string; function makeFixture(filename: string, args: string[]): string { const filePath = path.join(fixtureDir, filename); @@ -59,6 +63,35 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => { "pcm_s16le", ]); standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]); + // An older macOS recording: system audio and microphone as two streams. The + // system track is stereo, so it is the one ffmpeg picks on its own, and it is + // silent; the microphone, mono, speaks from the first second. + twoTracks = makeFixture("two-tracks.mkv", [ + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:r=1:d=4", + "-f", + "lavfi", + "-i", + "anullsrc=r=16000:cl=stereo:d=4", + "-itsoffset", + "1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=16000:duration=3", + "-map", + "0:v", + "-map", + "1:a", + "-map", + "2:a", + "-c:v", + "ffv1", + "-c:a", + "pcm_s16le", + ]); }); afterAll(() => { @@ -85,4 +118,14 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => { expect(firstAudibleSample).toBeGreaterThanOrEqual(0); expect(firstAudibleSample).toBeLessThan(16); }); + + it("transcribes every audio stream, each on the media clock, as the export mixes them", async () => { + // Decoding only ffmpeg's pick yields the silent system track: no transcript, + // while the exported video has the voice in it. + const samples = await extractMono16kPcm(twoTracks); + expect(samples).toHaveLength(4 * 16_000); + const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01); + expect(firstAudibleSample).toBeGreaterThanOrEqual(16_000); + expect(firstAudibleSample).toBeLessThan(16_000 + 16); + }); }); diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts index c437b1ae9..d87217668 100644 --- a/electron/stt/extractAudio.test.ts +++ b/electron/stt/extractAudio.test.ts @@ -4,9 +4,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const spawnMock = vi.fn(); const resolveFfmpegMock = vi.fn<() => string | null>(); +const audioArgsMock = vi.fn(async (..._args: unknown[]) => ["-vn", "-af", "aresample=first_pts=0"]); vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) })); -vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() })); +vi.mock("../media/audioPeaks", () => ({ + resolveFfmpeg: () => resolveFfmpegMock(), + mediaClockAudioArgs: (...args: unknown[]) => audioArgsMock(...args), +})); const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } = await import("./extractAudio"); @@ -27,6 +31,11 @@ function fakeChild() { return child; } +/** Extraction counts the audio streams before it spawns the decode: let it get there. */ +async function spawned(): Promise { + for (let i = 0; i < 100 && spawnMock.mock.calls.length === 0; i++) await Promise.resolve(); +} + /** The little-endian float32 bytes ffmpeg would emit for `values`. */ function f32le(values: number[]): Buffer { const buf = Buffer.alloc(values.length * 4); @@ -44,6 +53,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/tmp/a.mp3"); + await spawned(); child.stdout.end(f32le([0.5])); child.emit("close", 0); await promise; @@ -61,6 +71,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/tmp/a.mp3"); + await spawned(); child.stdout.end(f32le([0, 0.5, -0.25])); child.emit("close", 0); @@ -75,6 +86,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/tmp/a.mp3"); + await spawned(); const bytes = f32le([0.25, -0.75, 1]); child.stdout.write(bytes.subarray(0, 6)); // one whole float + half of the next child.stdout.write(bytes.subarray(6)); @@ -89,6 +101,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/tmp/silent.mp4"); + await spawned(); child.stderr.end("Stream map '0:a' matches no streams"); child.stdout.end(); child.emit("close", 1); @@ -102,6 +115,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + await spawned(); child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted"); child.stdout.end(); child.emit("close", 1); @@ -116,6 +130,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + await spawned(); const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE); await vi.advanceTimersByTimeAsync(60_000); await settled; @@ -132,6 +147,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + await spawned(); const settled = promise.then( () => { throw new Error("expected a rejection"); @@ -156,6 +172,7 @@ describe("extractMono16kPcm", () => { const child = fakeChild(); spawnMock.mockReturnValue(child); const promise = extractMono16kPcm("/tmp/truncated.mp3"); + await spawned(); child.stdout.end(f32le([0.1, 0.2])); child.emit("close", 1); @@ -177,6 +194,7 @@ describe("extractMono16kPcm", () => { spawnMock.mockReturnValue(child); const controller = new AbortController(); const promise = extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }); + await spawned(); controller.abort(); await expect(promise).rejects.toMatchObject({ name: "AbortError" }); @@ -185,6 +203,20 @@ describe("extractMono16kPcm", () => { expect(child.kill).toHaveBeenCalled(); }); + it("hands the abort to the stream probe, and does not decode after it", async () => { + // The probe is a process too: a cancel must not wait out a wedged one. + const controller = new AbortController(); + audioArgsMock.mockImplementationOnce(async (..._args: unknown[]) => { + controller.abort(); + return ["-vn"]; + }); + await expect( + extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(audioArgsMock).toHaveBeenCalledWith("/usr/bin/ffmpeg", "/tmp/a.mp3", controller.signal); + expect(spawnMock).not.toHaveBeenCalled(); + }); + it("does not spawn at all when the signal is already aborted", async () => { const controller = new AbortController(); controller.abort(); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 901078669..2ffebf559 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -20,7 +20,7 @@ // that is already never repeated. import { spawn } from "node:child_process"; -import { resolveFfmpeg } from "../media/audioPeaks"; +import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks"; import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ @@ -82,6 +82,9 @@ export async function extractMono16kPcm( const ffmpeg = resolveFfmpeg(); if (!ffmpeg) throw new FfmpegUnavailableError(); if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath, options.signal); + // Again: an abort during the probe (which it kills) fired before the listener below. + if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError"); const child = spawn( ffmpeg, @@ -91,14 +94,11 @@ export async function extractMono16kPcm( "error", "-i", filePath, - "-vn", + ...audioArgs, "-ac", "1", "-ar", String(SAMPLE_RATE), - // Pad delayed audio to keep sample zero on the media clock. - "-af", - "aresample=first_pts=0", "-f", "f32le", "-",