fix(audio): align waveform and STT with every audio track on the media clock

electron/media/audioPeaks.test.ts

@@ -1,9 +1,17 @@
 // @vitest-environment node
+import { spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import {
+	countAudioStreams,
+	decodePeaks,
+	ffmpegCandidates,
+	mediaClockAudioArgs,
+	peakBlockCount,
+	resolveFfmpeg,
+} from "./audioPeaks";
 
 const ROOT = path.resolve(__dirname, "..", "..");
 
@@ -311,3 +319,110 @@ describe.runIf(staged)("decoding a real file", () => {
 		expect(mn).toBeCloseTo(-0.205, 1);
 	}, 120_000);
 });
+
+describe("countAudioStreams", () => {
+	it("counts the audio streams in ffmpeg's listing, and nothing else", () => {
+		// The layout of an older macOS recording, as ffmpeg 8.1 lists it.
+		const listing = [
+			"Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4':",
+			"  Duration: 00:00:12.00, start: 0.000000, bitrate: 5012 kb/s",
+			"  Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p, 60 fps (default)",
+			"  Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo (default)",
+			"    Metadata:",
+			"      handler_name    : Audio: system",
+			"  Stream #0:2[0x3](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, mono (default)",
+			"At least one output file must be specified",
+		].join("\n");
+		expect(countAudioStreams(listing)).toBe(2);
+		expect(countAudioStreams("  Stream #0:0: Video: ffv1, 16x16")).toBe(0);
+	});
+});
+
+const ffmpegOnPath = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0;
+
+describe.skipIf(!ffmpegOnPath)("peaks on the media clock, with real FFmpeg", () => {
+	let fixtureDir: string;
+	let delayed: string;
+	let twoTracks: string;
+
+	/** A four-second video from zero, muxed with the given audio inputs, one stream each. */
+	function makeFixture(filename: string, audioInputs: string[][]): string {
+		const filePath = path.join(fixtureDir, filename);
+		const result = spawnSync(
+			"ffmpeg",
+			[
+				"-hide_banner",
+				"-loglevel",
+				"error",
+				"-y",
+				"-f",
+				"lavfi",
+				"-i",
+				"color=c=black:s=16x16:r=1:d=4",
+				...audioInputs.flat(),
+				"-map",
+				"0:v",
+				...audioInputs.flatMap((_, i) => ["-map", `${i + 1}:a`]),
+				"-c:v",
+				"ffv1",
+				"-c:a",
+				"pcm_s16le",
+				filePath,
+			],
+			{ encoding: "utf8", timeout: 10_000 },
+		);
+		if (result.status !== 0) throw new Error(result.error?.message ?? result.stderr);
+		return filePath;
+	}
+
+	/** Seconds into the clip of the first block with signal in it, or -1. */
+	function firstAudibleSec(peaks: Float32Array): number {
+		for (let i = 0; i < peaks.length; i += 2) {
+			if (peaks[i + 1] > 0.01) return i / 2 / 200;
+		}
+		return -1;
+	}
+
+	beforeAll(() => {
+		fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-peaks-clock-"));
+		const delayedTone = (sec: number, duration: number) => [
+			"-itsoffset",
+			String(sec),
+			"-f",
+			"lavfi",
+			"-i",
+			`sine=frequency=440:sample_rate=16000:duration=${duration}`,
+		];
+		// Video from zero, audio from two seconds, no encoded silence in between.
+		delayed = makeFixture("delayed.mkv", [delayedTone(2, 2)]);
+		// Older macOS layout: a silent stereo system track, which is the stream ffmpeg
+		// picks on its own, and a mono microphone speaking from the first second.
+		twoTracks = makeFixture("two-tracks.mkv", [
+			["-f", "lavfi", "-i", "anullsrc=r=16000:cl=stereo:d=4"],
+			delayedTone(1, 3),
+		]);
+	});
+
+	afterAll(() => {
+		if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true });
+	});
+
+	it("draws audio that starts after the video where it plays, not at zero", async () => {
+		// Pulled to zero, the waveform runs two seconds ahead of the transcript and of
+		// the export, both of which keep the offset.
+		const peaks = await decodePeaks("ffmpeg", delayed, 4);
+		expect(peaks.length).toBe(peakBlockCount(4) * 2);
+		expect(firstAudibleSec(peaks)).toBe(2);
+	});
+
+	it("draws every audio stream, each on the media clock, as the export mixes them", async () => {
+		const peaks = await decodePeaks("ffmpeg", twoTracks, 4);
+		expect(firstAudibleSec(peaks)).toBe(1);
+	});
+
+	it("kills the stream probe on abort instead of waiting it out", async () => {
+		// Killed before it lists anything, the probe counts no stream at all.
+		const args = await mediaClockAudioArgs("ffmpeg", twoTracks, AbortSignal.abort());
+		expect(args).not.toContain("-filter_complex");
+	});
+});

electron/media/audioPeaks.ts

@@ -147,6 +147,69 @@ export function resolveFfmpeg(here?: string): string | null {
 	return found;
 }
 
+/** Listing streams is only opening the file: past this, the open is wedged. */
+const PROBE_TIMEOUT_MS = 10_000;
+
+/** Pads audio that starts late, so sample zero stays on the media clock. */
+const ON_MEDIA_CLOCK = "aresample=first_pts=0";
+
+/** Audio streams in ffmpeg's listing of its input (the stderr of `ffmpeg -i <file>`). */
+export function countAudioStreams(listing: string): number {
+	return listing.match(/^\s*Stream #0:\d+\S*: Audio:/gm)?.length ?? 0;
+}
+
+/** Any failure counts as zero, an abort included: the caller then checks its signal,
+ *  and otherwise the decode runs as it always did and reports the problem itself. */
+function probeAudioStreams(
+	ffmpeg: string,
+	filePath: string,
+	signal?: AbortSignal,
+): Promise<number> {
+	return new Promise((resolve) => {
+		let listing = "";
+		const child = spawn(ffmpeg, ["-hide_banner", "-i", filePath], {
+			stdio: ["ignore", "ignore", "pipe"],
+			timeout: PROBE_TIMEOUT_MS,
+			killSignal: "SIGKILL",
+			signal,
+		});
+		child.stderr.on("data", (c: Buffer) => {
+			listing += c.toString();
+		});
+		child.once("error", () => resolve(0));
+		child.once("close", () => resolve(countAudioStreams(listing)));
+	});
+}
+
+/**
+ * The ffmpeg arguments, after `-i`, that decode a recording's audio the way the
+ * compositor exports it, so the waveform and the transcript line up with the export:
+ *
+ * - On the media clock. Audio that starts after the video is padded, not pulled to
+ *   zero, or everything drawn or transcribed from it lands that much early.
+ * - Every audio stream, summed. Older macOS recordings carry system audio and the
+ *   microphone as two streams; ffmpeg alone decodes the one it picks, usually the
+ *   system track, silent whenever nothing plays, while the export mixes both. `amix`
+ *   mixes by sample count, not timestamp, so each stream is aligned first, and
+ *   `normalize=0` sums like the compositor instead of averaging.
+ */
+export async function mediaClockAudioArgs(
+	ffmpeg: string,
+	filePath: string,
+	signal?: AbortSignal,
+): Promise<string[]> {
+	const streams = await probeAudioStreams(ffmpeg, filePath, signal);
+	if (streams < 2) return ["-vn", "-af", ON_MEDIA_CLOCK];
+	const aligned = Array.from({ length: streams }, (_, i) => `[0:a:${i}]${ON_MEDIA_CLOCK}[a${i}]`);
+	const labels = aligned.map((_, i) => `[a${i}]`).join("");
+	return [
+		"-filter_complex",
+		`${aligned.join(";")};${labels}amix=inputs=${streams}:normalize=0[a]`,
+		"-map",
+		"[a]",
+	];
+}
+
 /** Number of min/max blocks for a clip of `durationSec`. */
 export function peakBlockCount(durationSec: number): number {
 	return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC)));
@@ -205,13 +268,14 @@ class PeakFolder {
 }
 
 /** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */
-async function decodePeaks(
+export async function decodePeaks(
 	ffmpeg: string,
 	filePath: string,
 	durationSec: number,
 ): Promise<Float32Array> {
 	const blocks = peakBlockCount(durationSec);
 	const folder = new PeakFolder(blocks, durationSec * PCM_RATE);
+	const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath);
 	const child = spawn(
 		ffmpeg,
 		[
@@ -220,7 +284,7 @@ async function decodePeaks(
 			"error",
 			"-i",
 			filePath,
-			"-vn",
+			...audioArgs,
 			"-ac",
 			"1",
 			"-ar",
@@ -265,11 +329,14 @@ async function decodePeaks(
  * Cache key: path plus size plus mtime. A recording is immutable in practice,
  * but keying on identity alone would serve stale peaks for a re-encoded or
  * replaced file, and that failure is silent and confusing.
+ *
+ * The leading version changes whenever the decode does: v2 put the peaks on the
+ * media clock and mixed every audio stream, which the v1 entries do not.
  */
 async function cacheKey(filePath: string): Promise<string> {
 	const info = await stat(filePath);
 	return createHash("sha1")
-		.update(`${filePath}:${info.size}:${info.mtimeMs}`)
+		.update(`v2:${filePath}:${info.size}:${info.mtimeMs}`)
 		.digest("hex")
 		.slice(0, 32);
 }

electron/stt/extractAudio.integration.test.ts

@@ -5,7 +5,10 @@ import path from "node:path";
 import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 
 // Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg.
-vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" }));
+vi.mock("../media/audioPeaks", async (importOriginal) => ({
+	...(await importOriginal<typeof import("../media/audioPeaks")>()),
+	resolveFfmpeg: () => "ffmpeg",
+}));
 
 import { extractMono16kPcm } from "./extractAudio";
 
@@ -19,6 +22,7 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
 	let fixtureDir: string;
 	let delayedVideo: string;
 	let standaloneAudio: string;
+	let twoTracks: string;
 
 	function makeFixture(filename: string, args: string[]): string {
 		const filePath = path.join(fixtureDir, filename);
@@ -59,6 +63,35 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
 			"pcm_s16le",
 		]);
 		standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]);
+		// An older macOS recording: system audio and microphone as two streams. The
+		// system track is stereo, so it is the one ffmpeg picks on its own, and it is
+		// silent; the microphone, mono, speaks from the first second.
+		twoTracks = makeFixture("two-tracks.mkv", [
+			"-f",
+			"lavfi",
+			"-i",
+			"color=c=black:s=16x16:r=1:d=4",
+			"-f",
+			"lavfi",
+			"-i",
+			"anullsrc=r=16000:cl=stereo:d=4",
+			"-itsoffset",
+			"1",
+			"-f",
+			"lavfi",
+			"-i",
+			"sine=frequency=440:sample_rate=16000:duration=3",
+			"-map",
+			"0:v",
+			"-map",
+			"1:a",
+			"-map",
+			"2:a",
+			"-c:v",
+			"ffv1",
+			"-c:a",
+			"pcm_s16le",
+		]);
 	});
 
 	afterAll(() => {
@@ -85,4 +118,14 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
 		expect(firstAudibleSample).toBeGreaterThanOrEqual(0);
 		expect(firstAudibleSample).toBeLessThan(16);
 	});
+
+	it("transcribes every audio stream, each on the media clock, as the export mixes them", async () => {
+		// Decoding only ffmpeg's pick yields the silent system track: no transcript,
+		// while the exported video has the voice in it.
+		const samples = await extractMono16kPcm(twoTracks);
+		expect(samples).toHaveLength(4 * 16_000);
+		const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01);
+		expect(firstAudibleSample).toBeGreaterThanOrEqual(16_000);
+		expect(firstAudibleSample).toBeLessThan(16_000 + 16);
+	});
 });