diff --git a/packages/cli/src/media-use/lib/manifest.mjs b/packages/cli/src/media-use/lib/manifest.mjs index 56b533aaa3..49e387690f 100644 --- a/packages/cli/src/media-use/lib/manifest.mjs +++ b/packages/cli/src/media-use/lib/manifest.mjs @@ -79,6 +79,9 @@ export function appendRecord(projectDir, record) { appendFileSync(p, line); } +/** Sources that mean the agent made or fetched the file; any other file is the person's own. */ +export const AGENT_SOURCES = ["generated", "search", "bundled"]; + /** The record a path has now: the manifest only appends, so the last one for a path wins. */ export function latestRecordFor(projectDir, path) { return readManifest(projectDir).findLast((record) => record.path === path); diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs index a529fd313d..80bc42ec2e 100644 --- a/packages/cli/src/media-use/resolve.mjs +++ b/packages/cli/src/media-use/resolve.mjs @@ -5,6 +5,7 @@ import { existsSync, statSync, writeFileSync, renameSync, rmSync, realpathSync } import { resolve, join, extname, basename, relative, isAbsolute, sep } from "node:path"; import { parseArgs } from "node:util"; import { + AGENT_SOURCES, appendRecord, latestRecordFor, recordInPlace, @@ -60,7 +61,6 @@ import { } from "./lib/local-media-search.mjs"; const INGEST_TYPES = listTypes(); -const RECORDED_SOURCES = ["generated", "search", "bundled"]; const DEFAULT_EXT = { bgm: ".wav", sfx: ".mp3", @@ -130,7 +130,7 @@ Options: --reuse Import a specific global-cache asset (by content sha/prefix, from --candidates) into this project --from Freeze a local file or direct public URL (ingest) - --source With --from: how the file was made (${RECORDED_SOURCES.join(" | ")}). + --source With --from: how the file was made (${AGENT_SOURCES.join(" | ")}). A file already inside the project is then recorded where it is --params Build an explicit parametric LUT (lut/grade only) --for Analyze a local image/video and add measured grade adjust @@ -918,8 +918,8 @@ async function ingest(src) { console.error(`error: refusing to ingest a 0-byte file: ${src}`); process.exit(2); } - if (args.source && !RECORDED_SOURCES.includes(args.source)) { - console.error(`error: --source takes one of: ${RECORDED_SOURCES.join(", ")}`); + if (args.source && !AGENT_SOURCES.includes(args.source)) { + console.error(`error: --source takes one of: ${AGENT_SOURCES.join(", ")}`); process.exit(2); } if (args.source && (type === "lut" || type === "grade")) { diff --git a/skills-manifest.json b/skills-manifest.json index 33fd821fbe..da195b39f1 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -54,7 +54,7 @@ "files": 1 }, "media-use": { - "hash": "1871ad36628ac54b", + "hash": "0744050e3114e60a", "files": 104 }, "motion-graphics": { diff --git a/skills/media-use/audio/references/bgm.md b/skills/media-use/audio/references/bgm.md index 9c43da908a..e6483479d4 100644 --- a/skills/media-use/audio/references/bgm.md +++ b/skills/media-use/audio/references/bgm.md @@ -15,6 +15,8 @@ One music bed per composition, produced by the shared audio engine (`scripts/aud - **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`). - **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference. +Both routes keep a file of yours already at the output name: the engine writes the next free name (`track-2.mp3`), reports it as an anomaly, and `bgm.path` carries the real path. + ## HeyGen retrieval (default) `searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`: diff --git a/skills/media-use/audio/references/sfx.md b/skills/media-use/audio/references/sfx.md index 0fa1d28e77..9de60a3a8d 100644 --- a/skills/media-use/audio/references/sfx.md +++ b/skills/media-use/audio/references/sfx.md @@ -16,15 +16,14 @@ Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The "id": "3", // joins the cue to the caller's model (frame / scene / segment) "name": "whoosh", "file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root - "source": "heygen" | "local" | "project", // which route resolved it; "project": a file already - // in assets/sfx under that name that is not the library's copy + "source": "heygen" | "local", // which route resolved it "offset_s": 0, // delay from the line's start "duration_s": 0.57, "volume": 0.35 // SFX sit UNDER voice + BGM } ``` -A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. +A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. Neither route replaces a file of yours already at the output name: the cue gets the next free name (`whoosh-2.mp3`), reported as an anomaly, and `file` carries the real path. ## HeyGen retrieval (credentialed) diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md index 468ca78a14..cb295e53dd 100644 --- a/skills/media-use/audio/references/tts.md +++ b/skills/media-use/audio/references/tts.md @@ -135,7 +135,7 @@ node /audio/scripts/audio.mjs \ --request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts ``` -The engine saves `assets/voice/intro.wav`, measures its duration, and transcribes +The engine saves `assets/voice/intro.wav` (or `intro-2.wav` when a file of yours already has that name; `voices[].path` says which), measures its duration, and transcribes it into `voices[].words` in `audio_meta.json`. Check that every requested line has audio and nonempty word timings before building a captioned video. Review the timings against the actual audio; transcription is estimated alignment, diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index d0437028e7..3f84aede4d 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -55,7 +55,7 @@ import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs" import { resolveSfx } from "./lib/sfx.mjs"; import { mapWithConcurrency } from "./lib/concurrency.mjs"; import { openAudioMeta } from "./lib/audio-meta.mjs"; -import { recordInManifest, writtenAssets } from "./lib/media-record.mjs"; +import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const argv = process.argv.slice(2); @@ -138,6 +138,7 @@ if (only.has("tts") && lines.length) { lang, }); console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`); + const paths = voicePaths(hyperframesDir, lines, anomalies); const synthLine = async (line) => { const id = String(line.id); const text = String(line.text ?? "").trim(); @@ -145,7 +146,7 @@ if (only.has("tts") && lines.length) { anomalies.push(`line ${id}: empty text — skipped`); return null; } - const rel = `assets/voice/${id}.wav`; + const rel = paths.get(id); const abs = join(hyperframesDir, rel); const { ok, words, error } = await synthesizeOne({ provider: ttsProvider, @@ -217,6 +218,7 @@ if (only.has("bgm")) { headers: heygenAuthHeaders(), hyperframesDir, hasVoice, + anomalies, }); if (bgm) { bgmFields.bgm_provider = "heygen"; @@ -243,6 +245,7 @@ if (only.has("bgm")) { lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null, seedSeconds, hasVoice, + anomalies, }); if (gen.disabled) { anomalies.push(`bgm: ${gen.reason}`); diff --git a/skills/media-use/audio/scripts/audio.test.mjs b/skills/media-use/audio/scripts/audio.test.mjs index 3d7de20316..505cdecae7 100644 --- a/skills/media-use/audio/scripts/audio.test.mjs +++ b/skills/media-use/audio/scripts/audio.test.mjs @@ -1,9 +1,10 @@ import { strict as assert } from "node:assert"; import { test } from "node:test"; -import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs"; import { join, dirname } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; +import { recordInManifest } from "./lib/media-record.mjs"; import { resolveSfx } from "./lib/sfx.mjs"; // Proves the relocated engine (skills/media-use/audio/) still resolves its @@ -48,9 +49,12 @@ test("an unknown cue is reported, not fatal", async () => { } }); -test("a person's own file under a bundled name is not labelled as the library's", async () => { +test("a person's own file under a bundled name survives, and the cue gets the next free name", async () => { const dir = mkdtempSync(join(tmpdir(), "mu-audio-")); try { + const own = join(dir, "assets", "sfx", "whoosh.mp3"); + mkdirSync(dirname(own), { recursive: true }); + writeFileSync(own, "the person's own whoosh"); const resolve = () => resolveSfx({ cues: [{ id: "1", name: "whoosh" }], @@ -58,16 +62,65 @@ test("a person's own file under a bundled name is not labelled as the library's" hyperframesDir: dir, sfxLibDir, }); - const copied = await resolve(); - const reused = await resolve(); - writeFileSync(join(dir, copied.sfx[0].file), "the person's own whoosh"); - const own = await resolve(); + const first = await resolve(); + const second = await resolve(); + + assert.equal(readFileSync(own, "utf8"), "the person's own whoosh"); + assert.deepEqual( + [first, second].map(({ sfx }) => [sfx[0].file, sfx[0].source]), + [ + ["assets/sfx/whoosh-2.mp3", "local"], + ["assets/sfx/whoosh-2.mp3", "local"], + ], + ); assert.deepEqual( - [copied, reused, own].map(({ sfx }) => sfx[0].source), - ["local", "local", "project"], + readFileSync(join(dir, "assets/sfx/whoosh-2.mp3")), + readFileSync(join(sfxLibDir, "whoosh.mp3")), ); } finally { rmSync(dir, { recursive: true, force: true }); } }); + +test("two effects never share a file when one's name is taken by the person, run after run", async () => { + const dir = mkdtempSync(join(tmpdir(), "mu-audio-")); + try { + mkdirSync(join(dir, "assets", "sfx"), { recursive: true }); + writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch"); + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const query = new URL(url).searchParams.get("query"); + if (query) + return Response.json({ data: [{ audio_url: `https://sound.test/${query}`, score: 0.6 }] }); + return new Response(`bytes of ${new URL(url).pathname}`); + }; + const run = async (names) => { + const { sfx } = await resolveSfx({ + cues: names.map((name, index) => ({ id: String(index), name })), + heygenOK: true, + headers: {}, + hyperframesDir: dir, + sfxLibDir, + }); + const files = sfx.map(({ file }) => file); + recordInManifest( + dir, + [...new Set(files)].map((path) => ({ path, type: "sfx", source: "search" })), + ); + return files; + }; + try { + assert.deepEqual(await run(["glitch"]), ["assets/sfx/glitch-2.mp3"]); + assert.deepEqual(await run(["glitch", "glitch 2", "glitch"]), [ + "assets/sfx/glitch-2.mp3", + "assets/sfx/glitch-2-2.mp3", + "assets/sfx/glitch-2.mp3", + ]); + } finally { + globalThis.fetch = realFetch; + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/audio/scripts/gemini-pipeline.test.mjs b/skills/media-use/audio/scripts/gemini-pipeline.test.mjs index 8658807a89..842e5085e6 100644 --- a/skills/media-use/audio/scripts/gemini-pipeline.test.mjs +++ b/skills/media-use/audio/scripts/gemini-pipeline.test.mjs @@ -9,9 +9,15 @@ import { spawnSync } from "node:child_process"; // external audio tools are fixtures. This is not a live synthesis/render test. for (const expired of [false, true]) { for (const only of ["tts", "tts,bgm,sfx"]) { + // With expired credentials the person already has an assets/voice/intro.wav, which the engine must keep. + const voicePath = expired ? "assets/voice/intro-2.wav" : "assets/voice/intro.wav"; test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => { const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-")); t.after(() => rmSync(dir, { recursive: true, force: true })); + if (expired) { + mkdirSync(join(dir, "assets/voice"), { recursive: true }); + writeFileSync(join(dir, "assets/voice/intro.wav"), "the person's own intro"); + } const config = join(dir, "heygen"); mkdirSync(config); if (expired) @@ -48,7 +54,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const args = process.argv.slice(2); -assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', 'assets/voice/intro.wav']); +assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', '${voicePath}']); assert.ok(fs.existsSync(args[2])); assert.equal(args[args.indexOf('--model')+1], 'small.en'); fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([ @@ -99,7 +105,7 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS assert.deepEqual(meta.voices, [ { id: "intro", - path: "assets/voice/intro.wav", + path: voicePath, duration_s: 1.25, words: [ { id: "w0", text: "Hello", start: 0.1, end: 0.4 }, @@ -113,8 +119,13 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS manifest .map((line) => JSON.parse(line)) .map(({ path, type, source }) => [path, type, source]), - [["assets/voice/intro.wav", "voice", "generated"]], + [[voicePath, "voice", "generated"]], ); + if (expired) + assert.equal( + readFileSync(join(dir, "assets/voice/intro.wav"), "utf8"), + "the person's own intro", + ); }); } } diff --git a/skills/media-use/audio/scripts/lib/bgm.mjs b/skills/media-use/audio/scripts/lib/bgm.mjs index 0e2cc5b545..d23c8c0b65 100644 --- a/skills/media-use/audio/scripts/lib/bgm.mjs +++ b/skills/media-use/audio/scripts/lib/bgm.mjs @@ -12,9 +12,10 @@ // Missing/failed BGM never blocks a render. import { spawn, spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, openSync, closeSync } from "node:fs"; +import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs"; import { join } from "node:path"; import { downloadTo, searchSounds } from "./heygen.mjs"; +import { agentWritePath } from "./media-record.mjs"; import { pythonInvocation } from "./python.mjs"; const r3 = (x) => Number(x.toFixed(3)); @@ -45,12 +46,12 @@ function pipInstall(deps) { } // ── retrieval (HeyGen music library) ────────────────────────────────────────── -export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice }) { +export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) { const q = query || "calm cinematic underscore"; const results = await searchSounds(q, "music", headers, { limit: 5 }); if (!results.length) return null; const top = results[0]; - const rel = "assets/bgm/track.mp3"; + const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies }); await downloadTo(top.audio_url, join(hyperframesDir, rel)); return { path: rel, @@ -113,8 +114,9 @@ export function generateBgmDetached({ lyriaRecipe, seedSeconds = 28, hasVoice, + anomalies, }) { - const rel = "assets/bgm/track.wav"; + const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies }); const abs = join(hyperframesDir, rel); mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true }); const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`); @@ -141,6 +143,7 @@ export function generateBgmDetached({ "--prompt", prompt, ]); + rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] }); proc.unref(); closeSync(fd); @@ -159,6 +162,7 @@ export function generateBgmDetached({ const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1; const script = musicgenScript({ prompt, abs, targetS, seedS }); const { cmd, args } = pythonInvocation(["-c", script]); + rmSync(abs, { force: true }); const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] }); proc.unref(); closeSync(fd); diff --git a/skills/media-use/audio/scripts/lib/bgm.test.mjs b/skills/media-use/audio/scripts/lib/bgm.test.mjs index 46967605c7..a31f6e72c6 100644 --- a/skills/media-use/audio/scripts/lib/bgm.test.mjs +++ b/skills/media-use/audio/scripts/lib/bgm.test.mjs @@ -1,6 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm.mjs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { appendRecord } from "../../../scripts/lib/manifest.mjs"; +import { + BGM_BED_VOLUME, + BGM_SILENT_VOLUME, + bgmDefaultVolume, + generateBgmDetached, +} from "./bgm.mjs"; // Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB // hotter than a music bed under a voice should be. The default under narration @@ -28,3 +37,38 @@ test("the narrated default is well below the voice (≈ 0 dBFS)", () => { `bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`, ); }); + +test( + "generating music again clears the engine's old track, so waiting cannot mistake it for the new one", + { skip: process.platform === "win32" && "the fake python is a shell script" }, + (t) => { + const dir = mkdtempSync(join(tmpdir(), "hf-bgm-")); + const path = process.env.PATH; + t.after(() => { + process.env.PATH = path; + rmSync(dir, { recursive: true, force: true }); + }); + mkdirSync(join(dir, "bin")); + for (const name of ["python3", "python"]) + writeFileSync(join(dir, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + process.env.PATH = `${join(dir, "bin")}:${path}`; + mkdirSync(join(dir, "assets/bgm"), { recursive: true }); + writeFileSync(join(dir, "assets/bgm/track.wav"), "last run's track"); + appendRecord(dir, { + id: "bgm_001", + type: "bgm", + path: "assets/bgm/track.wav", + source: "generated", + }); + + const gen = generateBgmDetached({ + prompt: "calm", + durationS: 5, + hyperframesDir: dir, + anomalies: [], + }); + + assert.equal(gen.path, "assets/bgm/track.wav"); + assert.equal(existsSync(join(dir, "assets/bgm/track.wav")), false); + }, +); diff --git a/skills/media-use/audio/scripts/lib/media-record.mjs b/skills/media-use/audio/scripts/lib/media-record.mjs index ed94a691db..bbc2f91888 100644 --- a/skills/media-use/audio/scripts/lib/media-record.mjs +++ b/skills/media-use/audio/scripts/lib/media-record.mjs @@ -1,6 +1,48 @@ -import { recordInPlace } from "../../../scripts/lib/manifest.mjs"; +import { existsSync } from "node:fs"; +import { extname, join } from "node:path"; +import { AGENT_SOURCES, latestRecordFor, recordInPlace } from "../../../scripts/lib/manifest.mjs"; import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs"; +/** + * Where the engine may write `rel`: there, unless another file of this run has `taken` it or the file there is + * the person's (not `reusable` and not recorded as agent-made). Then the first free `name-2.ext`, with an anomaly. + */ +export function agentWritePath( + hyperframesDir, + rel, + { anomalies, taken = new Set(), reusable = () => false }, +) { + const free = (path) => + !taken.has(path) && + (!existsSync(join(hyperframesDir, path)) || + reusable(path) || + AGENT_SOURCES.includes(latestRecordFor(hyperframesDir, path)?.source)); + if (free(rel)) return rel; + const ext = extname(rel); + const stem = rel.slice(0, rel.length - ext.length); + let n = 2; + while (!free(`${stem}-${n}${ext}`)) n++; + const path = `${stem}-${n}${ext}`; + const why = taken.has(rel) + ? "another file of this run goes there" + : "the file there is yours (the media manifest does not record it as made by the engine)"; + const note = `${rel}: kept, because ${why}; writing ${path} instead (audio_meta.json has the path used)`; + if (!anomalies.includes(note)) anomalies.push(note); + return path; +} + +/** Each spoken line's file, picked before lines synthesize concurrently so two never land on one free name. */ +export function voicePaths(hyperframesDir, lines, anomalies) { + const taken = new Set(); + const paths = new Map(); + for (const line of lines.filter((l) => String(l.text ?? "").trim())) { + const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken }); + taken.add(rel); + paths.set(String(line.id), rel); + } + return paths; +} + const SFX_SOURCES = { heygen: "search", local: "bundled" }; // The files one engine run wrote, each with how it was made, for the project's media manifest. @@ -32,7 +74,6 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields if (only.has("sfx")) { for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) { const source = SFX_SOURCES[cue.source]; - if (!source) continue; const provider = source === "search" ? "heygen" : "bundled.sfx"; assets.push({ path: cue.file, diff --git a/skills/media-use/audio/scripts/lib/media-record.test.mjs b/skills/media-use/audio/scripts/lib/media-record.test.mjs index b6ef637267..0325fa9479 100644 --- a/skills/media-use/audio/scripts/lib/media-record.test.mjs +++ b/skills/media-use/audio/scripts/lib/media-record.test.mjs @@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import { join } from "node:path"; import { tmpdir } from "node:os"; import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs"; -import { recordInManifest, writtenAssets } from "./media-record.mjs"; +import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs"; const noBgm = { bgm: null, bgmFields: { bgm_pending: false } }; @@ -98,7 +98,6 @@ test("music and sound effects are marked by where they came from", () => { { file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" }, { file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" }, { file: "assets/sfx/glass.mp3", name: "glass", source: "heygen" }, - { file: "assets/sfx/pop.mp3", name: "pop", source: "project" }, ], }); @@ -139,6 +138,59 @@ test("a file that cannot be recorded becomes an anomaly, not a failure", (t) => assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/); }); +test("the engine writes over only its own files, else the next free name", (t) => { + const dir = project(t); + mkdirSync(join(dir, "assets/sfx"), { recursive: true }); + const names = ["mine", "made", "adopted", "twice", "twice-2", "kept", "kept-2"]; + for (const name of names) writeFileSync(join(dir, `assets/sfx/${name}.mp3`), name); + for (const path of ["assets/sfx/made.mp3", "assets/sfx/kept-2.mp3"]) + recordInManifest(dir, [{ path, type: "sfx", source: "search" }]); + recordInManifest(dir, [{ path: "assets/sfx/adopted.mp3", type: "sfx", source: "existing" }]); + const anomalies = []; + const at = (name, reusable) => + agentWritePath(dir, `assets/sfx/${name}.mp3`, { anomalies, reusable }); + + assert.equal(at("new"), "assets/sfx/new.mp3"); + assert.equal(at("made"), "assets/sfx/made.mp3"); + assert.equal( + at("mine", (path) => path === "assets/sfx/mine.mp3"), + "assets/sfx/mine.mp3", + ); + assert.deepEqual(anomalies, []); + assert.equal(at("mine"), "assets/sfx/mine-2.mp3"); + assert.equal(at("adopted"), "assets/sfx/adopted-2.mp3"); + assert.equal(at("twice"), "assets/sfx/twice-3.mp3"); + assert.equal(at("kept"), "assets/sfx/kept-2.mp3"); + assert.equal(anomalies.length, 4); + assert.match( + anomalies[0], + /^assets\/sfx\/mine\.mp3: kept, .* writing assets\/sfx\/mine-2\.mp3 instead/, + ); +}); + +test("two spoken lines never share a file when one's name is taken by the person", (t) => { + const dir = project(t); + mkdirSync(join(dir, "assets/voice"), { recursive: true }); + writeFileSync(join(dir, "assets/voice/hook.wav"), "the person's own hook"); + const anomalies = []; + + const paths = voicePaths( + dir, + [ + { id: "hook", text: "First" }, + { id: "hook-2", text: "Second" }, + { id: "blank", text: " " }, + ], + anomalies, + ); + + assert.deepEqual(Object.fromEntries(paths), { + hook: "assets/voice/hook-2.wav", + "hook-2": "assets/voice/hook-2-2.wav", + }); + assert.equal(anomalies.length, 2); +}); + test("a record never takes the id of a download still in flight", (t) => { const dir = project(t); mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true }); diff --git a/skills/media-use/audio/scripts/lib/sfx.mjs b/skills/media-use/audio/scripts/lib/sfx.mjs index 7cc3487c34..7f746f149b 100644 --- a/skills/media-use/audio/scripts/lib/sfx.mjs +++ b/skills/media-use/audio/scripts/lib/sfx.mjs @@ -15,6 +15,8 @@ import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { downloadTo, searchSounds } from "./heygen.mjs"; +import { latestRecordFor } from "../../../scripts/lib/manifest.mjs"; +import { agentWritePath } from "./media-record.mjs"; const SFX_VOLUME = 0.35; const slug = (s) => @@ -31,6 +33,8 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL const sfx = []; const anomalies = []; const destDir = join(hyperframesDir, "assets", "sfx"); + // Each effect's file this run: one effect named twice gets one file, and two effects never share one. + const fileFor = new Map(); // Dedupe identical (id,name) cues — the same effect named twice in one line // downloads/copies once. @@ -57,8 +61,14 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL continue; } const top = results[0]; - const file = `assets/sfx/${slug(name)}.mp3`; + const file = + fileFor.get(slug(name)) ?? + agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}.mp3`, { + anomalies, + taken: new Set(fileFor.values()), + }); await downloadTo(top.audio_url, join(hyperframesDir, file)); + fileFor.set(slug(name), file); sfx.push({ id, name, @@ -111,32 +121,41 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL continue; } const src = join(sfxLibDir, hit.file); - const destRel = `assets/sfx/${hit.file}`; + const library = existsSync(src) ? readFileSync(src) : null; + const isLibraryCopy = (rel) => + library && + existsSync(join(hyperframesDir, rel)) && + readFileSync(join(hyperframesDir, rel)).equals(library); + const destRel = + fileFor.get(hit.file) ?? + agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, { + anomalies, + taken: new Set(fileFor.values()), + // An unrecorded copy of the library file is one an engine run made before the manifest had it. + reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel), + }); const dest = join(hyperframesDir, destRel); // The bundled library may be incomplete: some installs of the skill ship // manifest.json without the actual mp3s. Pushing an sfx entry that points at // a file we never copied produces a dangling reference that silently drops // downstream ("not on disk"). Surface it as a loud anomaly and skip the cue // instead, so the audio_meta never references a missing file. - const reused = existsSync(dest); - if (!reused) { - if (!existsSync(src)) { - anomalies.push( - `sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` + - `library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` + - `restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`, - ); - continue; - } - copyFileSync(src, dest); - } - // A file already there that is not the library's copy is the person's own. - const own = reused && !(existsSync(src) && readFileSync(dest).equals(readFileSync(src))); + if (library) { + if (!isLibraryCopy(destRel)) copyFileSync(src, dest); + } else if (!existsSync(dest)) { + anomalies.push( + `sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` + + `library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` + + `restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`, + ); + continue; + } // else the engine's earlier copy at dest stands in for the file this install lacks + fileFor.set(hit.file, destRel); sfx.push({ id, name, file: destRel, - source: own ? "project" : "local", + source: "local", offset_s: 0, duration_s: r3(hit.duration), volume: SFX_VOLUME, diff --git a/skills/media-use/audio/scripts/lib/sfx.test.mjs b/skills/media-use/audio/scripts/lib/sfx.test.mjs index 3c89d58748..ef692962e1 100644 --- a/skills/media-use/audio/scripts/lib/sfx.test.mjs +++ b/skills/media-use/audio/scripts/lib/sfx.test.mjs @@ -1,8 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync } from "node:fs"; +import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; +import { appendRecord } from "../../../scripts/lib/manifest.mjs"; import { resolveSfx } from "./sfx.mjs"; // Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json @@ -67,3 +68,54 @@ test("offline: a matched-but-missing bundled file yields an anomaly and NO dangl assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied"); }); }); + +const whooshLibrary = (libDir, withFile) => { + writeFileSync( + join(libDir, "manifest.json"), + JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }), + ); + if (withFile) writeFileSync(join(libDir, "whoosh.mp3"), "ID3-library-bytes"); +}; +const record = (projDir, source) => + appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source }); +const offline = (libDir, projDir) => + resolveSfx({ + cues: [{ id: "s1", name: "whoosh" }], + heygenOK: false, + hyperframesDir: projDir, + sfxLibDir: libDir, + }); + +test("offline: the engine's earlier copy stands in for a library file this install lacks", async () => { + await withDirs(async ({ libDir, projDir }) => { + whooshLibrary(libDir, false); + mkdirSync(join(projDir, "assets/sfx"), { recursive: true }); + writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy"); + record(projDir, "bundled"); + + const { sfx, anomalies } = await offline(libDir, projDir); + + assert.deepEqual( + sfx.map(({ file }) => file), + ["assets/sfx/whoosh.mp3"], + ); + assert.deepEqual(anomalies, []); + }); +}); + +test("offline: a person's file with the library's bytes stays theirs once recorded as their own", async () => { + await withDirs(async ({ libDir, projDir }) => { + whooshLibrary(libDir, true); + mkdirSync(join(projDir, "assets/sfx"), { recursive: true }); + writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes"); + record(projDir, "existing"); + + const { sfx } = await offline(libDir, projDir); + + assert.deepEqual( + sfx.map(({ file }) => file), + ["assets/sfx/whoosh-2.mp3"], + ); + assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes"); + }); +}); diff --git a/skills/media-use/scripts/lib/manifest.mjs b/skills/media-use/scripts/lib/manifest.mjs index 56b533aaa3..49e387690f 100644 --- a/skills/media-use/scripts/lib/manifest.mjs +++ b/skills/media-use/scripts/lib/manifest.mjs @@ -79,6 +79,9 @@ export function appendRecord(projectDir, record) { appendFileSync(p, line); } +/** Sources that mean the agent made or fetched the file; any other file is the person's own. */ +export const AGENT_SOURCES = ["generated", "search", "bundled"]; + /** The record a path has now: the manifest only appends, so the last one for a path wins. */ export function latestRecordFor(projectDir, path) { return readManifest(projectDir).findLast((record) => record.path === path);