fix(producer): retain scene trims for discovered speed ramps

Author: AksharP5

docs/reference/speed-ramps.mdx

@@ -39,6 +39,9 @@ which the preview and the renderer both use, so the render shows the frames the
 preview showed. Set `data-duration` on the clip to fix how long it holds the
 timeline; without it the length follows from the source and the speed.
 
+If a browser probe discovers a rate lane on an existing clip, export preserves
+the scene's trim in both the ramp's position and the source media in-point.
+
 Pitch is kept. The preview leaves the browser's default pitch correction on.
 The render time-stretches audio in short constant-speed slices (a quarter
 second, coarser on clips longer than a minute), which approximates the curve

packages/engine/src/index.ts

@@ -266,6 +266,7 @@ export {
   parseAudioElements,
   parseMasterFxChain,
   processCompositionAudio,
+  videoAudioId,
 } from "./services/audioMixer.js";
 export { audioFailureDiagnostics } from "./services/audioFailureDiagnostic.js";
 export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js";

packages/engine/src/services/audioMixer.ts

@@ -557,6 +557,10 @@ export function parseMasterFxChain(html: string): string | undefined {
   return readMasterFxChain(parseHTML(unwrapTemplate(html)).document);
 }
 
+export function videoAudioId(videoId: string): string {
+  return `${videoId}-audio`;
+}
+
 export function parseAudioElements(html: string): AudioElement[] {
   const elements: AudioElement[] = [];
   const { document } = parseHTML(unwrapTemplate(html));
@@ -656,7 +660,7 @@ export function parseAudioElements(html: string): AudioElement[] {
     const src = resolveMediaElementSrc(el);
     if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;
     if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
-    elements.push(build(el, `${id}-audio`, src, "video", true));
+    elements.push(build(el, videoAudioId(id), src, "video", true));
   }
 
   return elements;

packages/producer/src/services/render/stages/extractVideosStage.ts

@@ -53,6 +53,7 @@ import {
   resolveProjectRelativeSrc,
   runVideoExtractionWithRetry,
   safeVideoExtractionSourceIdentity,
+  videoAudioId,
 } from "@hyperframes/engine";
 import {
   collectVideoMetadataHints,
@@ -579,7 +580,7 @@ export function appendAutoDetectedVideoAudio(
     const video = composition.videos.find((v) => v.id === ext.videoId);
     if (!video || !video.hasAudio || video.hidden || existingAudioSrcs.has(video.src)) continue;
     composition.audios.push({
-      id: `${video.id}-audio`,
+      id: videoAudioId(video.id),
       src: video.src,
       start: video.start,
       end: video.end,

packages/producer/src/services/render/stages/probeStage.test.ts

@@ -3,6 +3,13 @@ import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { RateSpec } from "@hyperframes/core";
+import { parseAudioElements, videoAudioId } from "../../../../../engine/src/services/audioMixer.js";
+import {
+  parseVideoElements,
+  parseImageElements,
+} from "../../../../../engine/src/services/videoFrameExtractor.js";
+import { resolveReferencedStart } from "../../../../../engine/src/services/referenceResolver.js";
+import { collectRenderMedia } from "../../renderMediaCollector.js";
 
 const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-"));
 afterAll(() => rmSync(workDir, { recursive: true, force: true }));
@@ -100,6 +107,11 @@ mock.module("../../assetMediaType.js", () => ({
 }));
 
 mock.module("@hyperframes/engine", () => ({
+  parseAudioElements,
+  parseVideoElements,
+  parseImageElements,
+  resolveReferencedStart,
+  videoAudioId,
   createCaptureSession: async (
     _url: string,
     _dir: string,
@@ -529,49 +541,124 @@ describe("runProbeStage — forceScreenshot threading", () => {
     },
   );
 
-  it.each(runtimeRates)(
-    "preserves a collected speed ramp during browser rate %j reconciliation",
-    async (browserRate) => {
+  it.each([
+    { tagName: "video", pageStart: 1, expected: 3 },
+    { tagName: "video", pageStart: 3, expected: 5 },
+    { tagName: "audio", pageStart: 1, expected: 3 },
+    { tagName: "audio", pageStart: 3, expected: 5 },
+  ])(
+    "applies a scene trim when adopting a runtime $tagName lane (page start $pageStart)",
+    async ({ tagName, pageStart, expected }) => {
       resetRetryMocks();
+      const playbackRate = {
+        target: "rate",
+        points: [
+          { t: 0, v: 1 },
+          { t: 2, v: 1 },
+          { t: 4, v: 2 },
+        ],
+      } satisfies RateSpec;
       browserMediaResults = [
         {
           id: "clip",
-          tagName: "video",
+          tagName,
           src: "clip.mp4",
           start: 0,
-          end: 3,
-          duration: 3,
-          mediaStart: 0,
-          playbackRate: browserRate,
+          end: 5,
+          duration: 5,
+          mediaStart: pageStart,
+          playbackRate,
           loop: false,
-          hasAudio: false,
+          hasAudio: true,
           volume: 1,
           muted: false,
         },
       ];
       const { runProbeStage } = await import("./probeStage.js");
       const input = makeProbeInput({});
-      const playbackRate = {
+      input.compiled.html = `<div data-composition-id="scene" data-start="5" data-duration="3" data-playback-start="2">
+        <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-playback-rate="2" data-has-audio="true"></${tagName}>
+      </div>`;
+      const collected = collectRenderMedia(input.compiled.html);
+      input.composition.videos = collected.videos;
+      input.composition.audios = collected.audios;
+      const before = tagName === "video" ? collected.videos : collected.audios;
+      expect(before[0]).toMatchObject({ start: 5, end: 8, mediaStart: 5, playbackRate: 2 });
+
+      await runProbeStage(input);
+
+      const shiftedRate = {
         target: "rate",
         points: [
-          { t: 0, v: 1.25 },
+          { t: 0, v: 1 },
           { t: 2, v: 2 },
         ],
-      } satisfies RateSpec;
-      input.composition.videos.push({
-        id: "clip",
-        src: "clip.mp4",
-        start: 0,
-        end: 3,
-        mediaStart: 0,
-        playbackRate,
-        loop: false,
-        hasAudio: false,
+      };
+      const media = tagName === "video" ? input.composition.videos : input.composition.audios;
+      expect(media[0]).toMatchObject({
+        start: 5,
+        end: 8,
+        mediaStart: expected,
+        playbackRate: shiftedRate,
       });
+      if (tagName === "video") {
+        expect(input.composition.audios[0]).toMatchObject({
+          mediaStart: expected,
+          playbackRate: shiftedRate,
+        });
+      }
+    },
+  );
+
+  it.each(
+    runtimeRates.flatMap((playbackRate) => [
+      { tagName: "video", playbackRate },
+      { tagName: "audio", playbackRate },
+    ]),
+  )(
+    "preserves a collected $tagName speed ramp during reconciliation (browser rate: $playbackRate)",
+    async ({ tagName, playbackRate }) => {
+      resetRetryMocks();
+      browserMediaResults = [
+        {
+          id: "clip",
+          tagName,
+          src: "clip.mp4",
+          start: 0,
+          end: 5,
+          duration: 5,
+          mediaStart: 1,
+          playbackRate,
+          loop: false,
+          hasAudio: true,
+          volume: 1,
+          muted: false,
+        },
+      ];
+      const { runProbeStage } = await import("./probeStage.js");
+      const input = makeProbeInput({});
+      const authoredRate = {
+        target: "rate",
+        points: [
+          { t: 0, v: 1 },
+          { t: 2, v: 1 },
+          { t: 4, v: 2 },
+        ],
+      } satisfies RateSpec;
+      input.compiled.html = `<div data-composition-id="scene" data-start="5" data-duration="3" data-playback-start="2">
+        <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-has-audio="true" data-automation='${JSON.stringify({ version: 1, lanes: [authoredRate] })}'></${tagName}>
+      </div>`;
+      const collected = collectRenderMedia(input.compiled.html);
+      input.composition.videos = collected.videos;
+      input.composition.audios = collected.audios;
+      const before = structuredClone({ videos: collected.videos, audios: collected.audios });
+      const media = tagName === "video" ? collected.videos : collected.audios;
+      expect(media[0]).toMatchObject({ mediaStart: 3 });
 
       await runProbeStage(input);
 
-      expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate);
+      expect(input.composition.videos).toEqual(before.videos);
+      expect(input.composition.audios).toEqual(before.audios);
     },
   );
 

packages/producer/src/services/render/stages/probeStage.ts

@@ -43,8 +43,17 @@ import {
   initializeSession,
   isTransientBrowserError,
   probeBeginFrameLiveness,
+  videoAudioId,
 } from "@hyperframes/engine";
-import { MEDIA_RENDER_ID_ATTR, fpsToNumber, readMediaStart } from "@hyperframes/core";
+import {
+  MEDIA_RENDER_ID_ATTR,
+  fpsToNumber,
+  normalizeRateSpec,
+  readMediaStart,
+  shiftRateLane,
+  sourceTimeAt,
+  timeAtSourceTime,
+} from "@hyperframes/core";
 import { extractMediaSrcMutations } from "@hyperframes/parsers";
 import type { BrowserMediaElement, CompiledComposition } from "../../htmlCompiler.js";
 import {
@@ -193,28 +202,39 @@ function authoredMediaStarts(html: string): Map<string, number> {
 }
 
 function isVideoAudio(audio: CompositionMetadata["audios"][number], videoId: string): boolean {
-  return audio.type === "video" && audio.id === `${videoId}-audio`;
+  return audio.type === "video" && audio.id === videoAudioId(videoId);
 }
 
 /**
- * Take a start point the page changed by moving the collected one by the same amount, so the
- * in-point its scene added survives, and a video's sound track moves with its picture.
+ * Adopt browser timing without losing the scene trim the collector applied to picture and sound.
  */
-function adoptRuntimeMediaStart(
+function adoptRuntimeMediaTiming(
   composition: Pick<CompositionMetadata, "videos" | "audios">,
   el: BrowserMediaElement,
   authored: number,
 ): void {
   const shift = el.mediaStart - authored;
-  if (el.mediaStart <= 0 || Math.abs(shift) <= BROWSER_MEDIA_EPSILON) return;
   const entries =
     el.tagName === "video"
       ? [
           ...composition.videos.filter((video) => video.id === el.id),
           ...composition.audios.filter((audio) => isVideoAudio(audio, el.id)),
         ]
       : composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id);
-  for (const entry of entries) entry.mediaStart += shift;
+  for (const entry of entries) {
+    // Collected lanes already carry their scene trim. Only newly discovered lanes need it.
+    if (typeof entry.playbackRate !== "object") {
+      if (typeof el.playbackRate === "object") {
+        const consumed = Math.max(0, entry.mediaStart - authored);
+        const head = timeAtSourceTime(normalizeRateSpec(entry.playbackRate), consumed);
+        entry.mediaStart += sourceTimeAt(el.playbackRate, head) - consumed;
+        entry.playbackRate = shiftRateLane(el.playbackRate, head);
+      } else {
+        entry.playbackRate = el.playbackRate;
+      }
+    }
+    if (el.mediaStart > 0 && Math.abs(shift) > BROWSER_MEDIA_EPSILON) entry.mediaStart += shift;
+  }
 }
 
 function reconcileBrowserMediaEnd(
@@ -597,20 +617,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
                   sourceChanged,
                   el.durationInferred,
                 );
-                adoptRuntimeMediaStart(
+                adoptRuntimeMediaTiming(
                   composition,
                   el,
                   authoredStarts.get(el.id) ?? existing.mediaStart,
                 );
-                // A collected rate lane already includes scene trimming; retain its phase.
-                if (typeof existing.playbackRate !== "object") {
-                  existing.playbackRate = el.playbackRate;
-                  for (const audio of composition.audios) {
-                    if (isVideoAudio(audio, el.id) && typeof audio.playbackRate !== "object") {
-                      audio.playbackRate = el.playbackRate;
-                    }
-                  }
-                }
                 if (el.hasAudio && !el.muted && !existing.hasAudio) {
                   existing.hasAudio = true;
                 }
@@ -656,15 +667,12 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
                   sourceChanged,
                   el.durationInferred,
                 );
-                adoptRuntimeMediaStart(
+                adoptRuntimeMediaTiming(
                   composition,
                   el,
                   authoredStarts.get(el.id) ?? existing.mediaStart,
                 );
                 existing.loop = el.loop;
-                if (typeof existing.playbackRate !== "object") {
-                  existing.playbackRate = el.playbackRate;
-                }
                 if (
                   el.volume > 0 &&
                   Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON