Skip to content

Commit d0d3a39

Browse files
committed
fix(producer): preserve runtime media playback rates
1 parent 69f3306 commit d0d3a39

6 files changed

Lines changed: 190 additions & 1 deletion

File tree

‎packages/producer/src/services/htmlCompiler.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ describe("discoverMediaFromBrowser", () => {
3636
currentSrcById: Record<string, string>,
3737
serializeCallback = false,
3838
intrinsicDurationById: Record<string, number> = {},
39+
defaultPlaybackRateById: Record<string, number> = {},
3940
) {
4041
const { document } = parseHTML(html);
4142
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
@@ -46,6 +47,13 @@ describe("discoverMediaFromBrowser", () => {
4647
const element = document.getElementById(id);
4748
if (element) Object.defineProperty(element, "duration", { value: duration });
4849
}
50+
for (const [id, defaultPlaybackRate] of Object.entries(defaultPlaybackRateById)) {
51+
const element = document.getElementById(id);
52+
if (element) {
53+
Object.defineProperty(element, "defaultPlaybackRate", { value: defaultPlaybackRate });
54+
Object.defineProperty(element, "playbackRate", { value: defaultPlaybackRate * 2 });
55+
}
56+
}
4957
const previousDocument = Reflect.get(globalThis, "document");
5058
Reflect.set(globalThis, "document", document);
5159
try {
@@ -129,6 +137,22 @@ describe("discoverMediaFromBrowser", () => {
129137
expect(media[0]).toMatchObject({ start: 0, end: 2, duration: 2, mediaStart: 1 });
130138
});
131139

140+
it("reads authored rates and native defaults after the browser callback is serialized", async () => {
141+
const media = await discover(
142+
`<video id="authored" src="clip.mp4" data-start="0" data-duration="2" data-playback-rate="1.5"></video>
143+
<audio id="native" src="clip.wav" data-start="0" data-duration="2"></audio>
144+
<video id="invalid" src="clip.mp4" data-start="0" data-duration="2" data-playback-rate="0"></video>
145+
<audio id="bounded" src="clip.wav" data-start="0" data-duration="2" data-playback-rate="20"></audio>
146+
<video id="default" src="clip.mp4" data-start="0" data-duration="2"></video>`,
147+
{},
148+
true,
149+
{},
150+
{ authored: 0.5, native: 0.5, invalid: 1.25 },
151+
);
152+
153+
expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5, 1.25, 10, 1]);
154+
});
155+
132156
it("reports colliding empty-src videos by render id, not author id", async () => {
133157
const media = await discover(
134158
`<video id="clip" data-hf-render-id="clip" src="" data-start="0" data-end="4" data-media-start="10"></video>` +

‎packages/producer/src/services/htmlCompiler.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
type UnresolvedElement,
3737
} from "@hyperframes/core";
3838
import { MAX_AUDIO_GAIN } from "@hyperframes/core/audio-gain";
39+
import { readPlaybackRate } from "@hyperframes/parsers/media-duration";
3940
import { gsapCdnDist } from "@hyperframes/core/gsap-cdn";
4041
import {
4142
assignBundledRuntimeCompositionIds,
@@ -2235,6 +2236,7 @@ export interface BrowserMediaElement {
22352236
/** True when compilation inferred duration from the fallback source. */
22362237
durationInferred: boolean;
22372238
mediaStart: number;
2239+
playbackRate: number;
22382240
loop: boolean;
22392241
hasAudio: boolean;
22402242
volume: number;
@@ -2260,6 +2262,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
22602262
durationInferred: boolean;
22612263
playbackStartRaw: string | null;
22622264
mediaStartRaw: string | null;
2265+
playbackRateRaw: string | null;
2266+
defaultPlaybackRate: number;
22632267
loop: boolean;
22642268
hasAudio: boolean;
22652269
volume: number;
@@ -2310,6 +2314,11 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
23102314
: (htmlEl as HTMLVideoElement | HTMLAudioElement).duration;
23112315
const playbackStartRaw = htmlEl.getAttribute("data-playback-start");
23122316
const mediaStartRaw = htmlEl.getAttribute("data-media-start");
2317+
const playbackRateRaw = htmlEl.getAttribute("data-playback-rate");
2318+
// playbackRate can include the transport speed; defaultPlaybackRate is the clip's own rate.
2319+
const defaultPlaybackRate = isImage
2320+
? 1
2321+
: (htmlEl as HTMLVideoElement | HTMLAudioElement).defaultPlaybackRate;
23132322
const loop = htmlEl.hasAttribute("loop");
23142323
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
23152324
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
@@ -2328,6 +2337,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
23282337
durationInferred,
23292338
playbackStartRaw,
23302339
mediaStartRaw,
2340+
playbackRateRaw,
2341+
defaultPlaybackRate,
23312342
loop,
23322343
hasAudio,
23332344
volume,
@@ -2339,7 +2350,16 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
23392350
});
23402351

23412352
return elements.map(
2342-
({ endRaw, durationRaw, intrinsicDuration, playbackStartRaw, mediaStartRaw, ...element }) => ({
2353+
({
2354+
endRaw,
2355+
durationRaw,
2356+
intrinsicDuration,
2357+
playbackStartRaw,
2358+
mediaStartRaw,
2359+
playbackRateRaw,
2360+
defaultPlaybackRate,
2361+
...element
2362+
}) => ({
23432363
...element,
23442364
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
23452365
duration:
@@ -2353,6 +2373,10 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
23532373
return null;
23542374
},
23552375
}),
2376+
playbackRate: readPlaybackRate(
2377+
(name) => (name === "data-playback-rate" ? playbackRateRaw : null),
2378+
defaultPlaybackRate,
2379+
),
23562380
}),
23572381
);
23582382
}

‎packages/producer/src/services/render/stages/extractVideosStage.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,34 @@ describe("appendAutoDetectedVideoAudio", () => {
105105
});
106106
});
107107

108+
it.each([
109+
1.5,
110+
{
111+
target: "rate",
112+
points: [
113+
{ t: 0, v: 1 },
114+
{ t: 3, v: 2 },
115+
],
116+
},
117+
] satisfies NonNullable<VideoElement["playbackRate"]>[])(
118+
"keeps the video's rate and trim in its detected audio (%j)",
119+
(playbackRate) => {
120+
const composition: Parameters<typeof appendAutoDetectedVideoAudio>[0] = {
121+
videos: [makeVideo({ start: 2, end: 5, mediaStart: 0.5, playbackRate })],
122+
audios: [],
123+
};
124+
125+
appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]);
126+
127+
expect(composition.audios[0]).toMatchObject({
128+
start: 2,
129+
end: 5,
130+
mediaStart: 0.5,
131+
playbackRate,
132+
});
133+
},
134+
);
135+
108136
it("skips a muted video even when the source file has audio", () => {
109137
const composition = {
110138
videos: [makeVideo({ hasAudio: false })],

‎packages/producer/src/services/render/stages/extractVideosStage.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,7 @@ export function appendAutoDetectedVideoAudio(
578578
start: video.start,
579579
end: video.end,
580580
mediaStart: video.mediaStart,
581+
playbackRate: video.playbackRate,
581582
layer: 0,
582583
volume: 1.0,
583584
type: "video",

‎packages/producer/src/services/render/stages/probeStage.test.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { afterAll, describe, expect, it, mock } from "bun:test";
22
import { mkdtempSync, rmSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import { join } from "node:path";
5+
import type { RateSpec } from "@hyperframes/core";
56

67
const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-"));
78
afterAll(() => rmSync(workDir, { recursive: true, force: true }));
@@ -413,6 +414,99 @@ describe("runProbeStage — forceScreenshot threading", () => {
413414
});
414415

415416
// Scene in-point 5 and clip trim 1: the collector starts picture and sound at source 6.
417+
it.each([
418+
{ tagName: "video", existing: false },
419+
{ tagName: "audio", existing: false },
420+
{ tagName: "video", existing: true },
421+
{ tagName: "audio", existing: true },
422+
])("retains a runtime $tagName rate (existing: $existing)", async ({ tagName, existing }) => {
423+
resetRetryMocks();
424+
browserMediaResults = [
425+
{
426+
id: "clip",
427+
tagName,
428+
src: "clip.mp4",
429+
start: 0,
430+
end: 3,
431+
duration: 3,
432+
mediaStart: 0.5,
433+
playbackRate: 1.5,
434+
loop: false,
435+
hasAudio: true,
436+
volume: 1,
437+
muted: false,
438+
},
439+
];
440+
const { runProbeStage } = await import("./probeStage.js");
441+
const input = makeProbeInput({});
442+
const clip = {
443+
id: "clip",
444+
src: "clip.mp4",
445+
start: 0,
446+
end: 3,
447+
mediaStart: 0.5,
448+
playbackRate: 1,
449+
};
450+
if (existing && tagName === "video") {
451+
input.composition.videos.push({ ...clip, loop: false, hasAudio: true });
452+
input.composition.audios.push({ ...clip, id: "clip-audio", layer: 0, type: "video" });
453+
}
454+
if (existing && tagName === "audio") {
455+
input.composition.audios.push({ ...clip, layer: 0, type: "audio" });
456+
}
457+
458+
await runProbeStage(input);
459+
460+
const media = tagName === "video" ? input.composition.videos : input.composition.audios;
461+
expect(media[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 });
462+
if (existing && tagName === "video") {
463+
expect(input.composition.audios[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 });
464+
}
465+
});
466+
467+
it("preserves a collected speed ramp during scalar browser reconciliation", async () => {
468+
resetRetryMocks();
469+
browserMediaResults = [
470+
{
471+
id: "clip",
472+
tagName: "video",
473+
src: "clip.mp4",
474+
start: 0,
475+
end: 3,
476+
duration: 3,
477+
mediaStart: 0,
478+
playbackRate: 1,
479+
loop: false,
480+
hasAudio: false,
481+
volume: 1,
482+
muted: false,
483+
},
484+
];
485+
const { runProbeStage } = await import("./probeStage.js");
486+
const input = makeProbeInput({});
487+
const playbackRate = {
488+
target: "rate",
489+
points: [
490+
{ t: 0, v: 1 },
491+
{ t: 3, v: 2 },
492+
],
493+
} satisfies RateSpec;
494+
input.composition.videos.push({
495+
id: "clip",
496+
src: "clip.mp4",
497+
start: 0,
498+
end: 3,
499+
mediaStart: 0,
500+
playbackRate,
501+
loop: false,
502+
hasAudio: false,
503+
});
504+
505+
await runProbeStage(input);
506+
507+
expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate);
508+
});
509+
416510
it.each([
417511
{ pageStart: 1, expected: 6 },
418512
{ pageStart: 3, expected: 8 },

‎packages/producer/src/services/render/stages/probeStage.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,19 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
599599
el,
600600
authoredStarts.get(el.id) ?? existing.mediaStart,
601601
);
602+
// A collected rate lane already includes scene trimming; retain its phase.
603+
if (typeof existing.playbackRate !== "object") {
604+
existing.playbackRate = el.playbackRate;
605+
for (const audio of composition.audios) {
606+
if (
607+
audio.type === "video" &&
608+
audio.id === `${el.id}-audio` &&
609+
typeof audio.playbackRate !== "object"
610+
) {
611+
audio.playbackRate = el.playbackRate;
612+
}
613+
}
614+
}
602615
if (el.hasAudio && !el.muted && !existing.hasAudio) {
603616
existing.hasAudio = true;
604617
}
@@ -614,6 +627,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
614627
start: el.start,
615628
end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
616629
mediaStart: el.mediaStart,
630+
playbackRate: el.playbackRate,
617631
loop: el.loop,
618632
hasAudio: el.hasAudio && !el.muted,
619633
});
@@ -644,6 +658,9 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
644658
el,
645659
authoredStarts.get(el.id) ?? existing.mediaStart,
646660
);
661+
if (typeof existing.playbackRate !== "object") {
662+
existing.playbackRate = el.playbackRate;
663+
}
647664
if (
648665
el.volume > 0 &&
649666
Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
@@ -658,6 +675,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
658675
start: el.start,
659676
end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
660677
mediaStart: el.mediaStart,
678+
playbackRate: el.playbackRate,
661679
layer: 0,
662680
volume: el.volume,
663681
type: "audio",

0 commit comments

Comments
 (0)