From 608dc220aea988ad5dc114e2762eeff5eb746e16 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 14:21:29 -0700 Subject: [PATCH 01/13] feat(core): support deterministic async frame sources --- docs/docs.json | 1 + docs/guides/frame-sources.md | 27 +++ packages/core/src/runtime/entry.test.ts | 31 +++ packages/core/src/runtime/entry.ts | 3 + .../core/src/runtime/frameSources.test.ts | 218 ++++++++++++++++++ packages/core/src/runtime/frameSources.ts | 131 +++++++++++ packages/core/src/runtime/init.ts | 5 + packages/core/src/runtime/window.d.ts | 1 + 8 files changed, 417 insertions(+) create mode 100644 docs/guides/frame-sources.md create mode 100644 packages/core/src/runtime/frameSources.test.ts create mode 100644 packages/core/src/runtime/frameSources.ts diff --git a/docs/docs.json b/docs/docs.json index 92cecc2104..7fbe3b2cd7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -913,6 +913,7 @@ "concepts/variables", "concepts/data-attributes", "guides/gsap-animation", + "guides/frame-sources", "concepts/frame-adapters", "concepts/determinism", "guides/html-in-canvas", diff --git a/docs/guides/frame-sources.md b/docs/guides/frame-sources.md new file mode 100644 index 0000000000..b0fde5f38b --- /dev/null +++ b/docs/guides/frame-sources.md @@ -0,0 +1,27 @@ +--- +title: "Frame sources" +description: "Run deterministic scene code on the HyperFrames timeline." +--- + +HyperFrames can drive scene code that exposes a function of time without converting it to GSAP. Register that function against a timed host using `window.__hyperframes.registerFrameSource` before runtime initialization completes. + +```javascript +const unregister = window.__hyperframes.registerFrameSource({ + element: document.getElementById("scene"), + ready: initializeScene(), + render: (sourceTime, signal) => drawScene(sourceTime, signal), + dispose: () => releaseScene(), +}); +``` + +The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. + +The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. + +`ready` holds initialization and export capture. Return a promise from `render` when drawing is asynchronous: capture waits for it before taking pixels. Repeated and out-of-order times must produce the same frame. HyperFrames owns playback; do not start a second clock. + +Draws are serialized per source. Rapid preview seeks coalesce while a draw is in flight; sequential export seeks draw every requested frame. Setup and draw failures are delivered to the capture completion barrier. A later successful seek can recover from a frame failure. + +Call `unregister()` when replacing a source. Removal of its host or runtime teardown also unregisters it. The callback's `AbortSignal` is aborted, queued work is cancelled, capture waiters are released, and `dispose` runs once. Stop any underlying work in response to the signal or in `dispose`; cancellation cannot forcibly interrupt synchronous JavaScript. + +Registration does not expose arbitrary scene code as editable keyframes. Manual element edits need stable identities and persistent parameters or overrides. Imported iframe sources also need screenshot capture and an explicit message bridge; this API does not grant access to sandboxed DOM. diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index 9398089799..63b7afd2dc 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -96,6 +96,37 @@ const neverDecodes = (clip: HTMLElement) => { describe("runtime entry", () => { afterEach(resetRuntimeGlobals); + it("drives an async frame source without a GSAP timeline and waits before capture", async () => { + const root = mountRoot(); + root.setAttribute("data-duration", "10"); + root.appendChild(document.createElement("div")); + window.__timelines = {}; + Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" }); + await evaluateRuntime(); + let finish!: () => void; + const first = new Promise((resolve) => { + finish = resolve; + }); + const render = vi + .fn() + .mockImplementationOnce(() => first) + .mockResolvedValue(undefined); + window.__hyperframes!.registerFrameSource({ element: root, render }); + delete (document as { readyState?: unknown }).readyState; + document.dispatchEvent(new Event("DOMContentLoaded")); + await vi.waitFor(() => expect(render).toHaveBeenCalled()); + window.__player!.seek(1.2); + let captured = false; + const capture = window.__hfWaitForSeekCompletion!().then(() => { + captured = true; + }); + await Promise.resolve(); + expect(captured).toBe(false); + finish(); + await capture; + expect(render).toHaveBeenLastCalledWith(1.2, expect.any(AbortSignal)); + }); + it("paints no timed clip, from script evaluation until the first visibility pass decides it", async () => { servePreview(); const root = mountRoot(); diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 666e872a93..6903414623 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -14,6 +14,7 @@ import { getVariables } from "./getVariables"; import { clearRuntimeData, registerRuntimeDataHandler, setRuntimeData } from "./runtimeData"; import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; +import { registerFrameSource } from "./frameSources"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -25,6 +26,7 @@ type HyperframeWindow = Window & { registerRuntimeDataHandler: typeof registerRuntimeDataHandler; setRuntimeData: typeof setRuntimeData; clearRuntimeData: typeof clearRuntimeData; + registerFrameSource: typeof registerFrameSource; }; }; @@ -53,6 +55,7 @@ deferMediaUntilDue(); registerRuntimeDataHandler, setRuntimeData, clearRuntimeData, + registerFrameSource, }; function bootstrapHyperframeRuntime(): void { diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts new file mode 100644 index 0000000000..7e9c2ede79 --- /dev/null +++ b/packages/core/src/runtime/frameSources.test.ts @@ -0,0 +1,218 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createFrameSourceAdapter, registerFrameSource } from "./frameSources"; +import { resetSeekDispatchState, waitForSeekCompletion } from "./adapters/seek-dispatch"; +import { createRuntimeStartTimeResolver } from "./startResolver"; + +function deferred() { + let resolve!: () => void; + let reject!: (reason: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +const adapters: ReturnType[] = []; +const adapter = () => { + const runtime = createFrameSourceAdapter({ + start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0), + duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element), + }); + adapters.push(runtime); + return runtime; +}; +const disposers: Array<() => void> = []; +function mount(start = "0", duration = "10") { + const element = document.createElement("section"); + element.setAttribute("data-start", start); + element.setAttribute("data-duration", duration); + document.body.append(element); + return element; +} +beforeEach(() => resetSeekDispatchState()); +afterEach(() => { + for (const runtime of adapters.splice(0)) runtime.revert?.(); + for (const dispose of disposers.splice(0)) dispose(); + document.body.innerHTML = ""; + resetSeekDispatchState(); +}); + +describe("frame sources", () => { + it("holds capture through setup and asynchronous drawing", async () => { + const setup = deferred(); + const frame = deferred(); + const render = vi.fn(() => frame.promise); + disposers.push(registerFrameSource({ element: mount(), ready: setup.promise, render })); + const runtime = adapter(); + runtime.seek({ time: 2 }); + let captured = false; + const capture = waitForSeekCompletion().then(() => { + captured = true; + }); + await Promise.resolve(); + expect(render).not.toHaveBeenCalled(); + expect(captured).toBe(false); + setup.resolve(); + await vi.waitFor(() => expect(render).toHaveBeenCalledWith(2, expect.any(AbortSignal))); + expect(captured).toBe(false); + frame.resolve(); + await capture; + expect(captured).toBe(true); + }); + + it("serializes drawing and coalesces queued scrubs to the latest time", async () => { + const frame = deferred(); + const render = vi.fn().mockImplementationOnce(() => frame.promise); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + runtime.seek({ time: 1 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + runtime.seek({ time: 2 }); + runtime.seek({ time: 3 }); + expect(render).toHaveBeenCalledTimes(1); + frame.resolve(); + await waitForSeekCompletion(); + expect(render.mock.calls.map(([time]) => time)).toEqual([1, 3]); + }); + + it("renders every sequential export request, including repeated and reverse time", async () => { + const render = vi.fn(); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + for (const time of [7, 2, 2, 0, 9]) { + runtime.seek({ time }); + await waitForSeekCompletion(); + } + expect(render.mock.calls.map(([time]) => time)).toEqual([7, 2, 2, 0, 9]); + }); + + it("rereads clip timing, trim inpoints and rates after edits", async () => { + const element = mount("3", "4"); + element.setAttribute("data-playback-start", "1"); + element.setAttribute("data-playback-rate", "2"); + const render = vi.fn(); + disposers.push(registerFrameSource({ element, render })); + const runtime = adapter(); + runtime.seek({ time: 4 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(3, expect.any(AbortSignal)); + element.setAttribute("data-start", "0"); + element.setAttribute("data-playback-start", "2"); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(4, expect.any(AbortSignal)); + runtime.seek({ time: 8 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(2); + }); + + it("surfaces failed setup even when its clip has not started", async () => { + const render = vi.fn(); + disposers.push( + registerFrameSource({ + element: mount("5"), + ready: Promise.reject(new Error("font failed")), + render, + }), + ); + const runtime = adapter(); + runtime.discover(); + await expect(waitForSeekCompletion()).rejects.toThrow("font failed"); + expect(render).not.toHaveBeenCalled(); + }); + + it("surfaces draw failure and allows the next seek to recover", async () => { + const render = vi + .fn() + .mockRejectedValueOnce(new Error("draw failed")) + .mockResolvedValue(undefined); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + runtime.seek({ time: 2 }); + await expect(waitForSeekCompletion()).rejects.toThrow("draw failed"); + runtime.seek({ time: 3 }); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(render).toHaveBeenLastCalledWith(3, expect.any(AbortSignal)); + }); + + it("unregisters stalled setup and allows replacing the same host", async () => { + const element = mount(); + const cleanup = vi.fn(); + const unregister = registerFrameSource({ + element, + ready: deferred().promise, + render: vi.fn(), + dispose: cleanup, + }); + const runtime = adapter(); + runtime.seek({ time: 0 }); + expect(() => registerFrameSource({ element, render: vi.fn() })).toThrow("already has"); + unregister(); + unregister(); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(cleanup).toHaveBeenCalledTimes(1); + const render = vi.fn(); + disposers.push(registerFrameSource({ element, render })); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(1); + }); + + it("removal aborts an active draw and releases capture", async () => { + const element = mount(); + let signal: AbortSignal | undefined; + const render = vi.fn((_time: number, abort: AbortSignal) => { + signal = abort; + return deferred().promise; + }); + const cleanup = vi.fn(); + disposers.push(registerFrameSource({ element, render, dispose: cleanup })); + const runtime = adapter(); + runtime.seek({ time: 0 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + element.remove(); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(signal?.aborted).toBe(true); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("keeps readiness promises stable and releases sources on teardown", async () => { + const ready = deferred(); + const cleanup = vi.fn(); + disposers.push( + registerFrameSource({ + element: mount(), + ready: ready.promise, + render: vi.fn(), + dispose: cleanup, + }), + ); + const runtime = adapter(); + const first = runtime.getReadyPromise?.(); + expect(runtime.getReadyPromise?.()).toBe(first); + runtime.revert?.(); + await expect(first).resolves.toBeDefined(); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(runtime.getReadyPromise?.()).toBeNull(); + }); + + it("draws overlapping independent sources without serializing them together", async () => { + const a = deferred(); + const b = deferred(); + const renderA = vi.fn(() => a.promise); + const renderB = vi.fn(() => b.promise); + disposers.push( + registerFrameSource({ element: mount(), render: renderA }), + registerFrameSource({ element: mount(), render: renderB }), + ); + adapter().seek({ time: 1 }); + await vi.waitFor(() => { + expect(renderA).toHaveBeenCalled(); + expect(renderB).toHaveBeenCalled(); + }); + a.resolve(); + b.resolve(); + await waitForSeekCompletion(); + }); +}); diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts new file mode 100644 index 0000000000..a8dc2140ef --- /dev/null +++ b/packages/core/src/runtime/frameSources.ts @@ -0,0 +1,131 @@ +import { sourceTimeAt } from "../speedRamp"; +import { readElementRateSpec, readMediaStart } from "./playbackRate"; +import { registerSeekCompletion } from "./adapters/seek-dispatch"; +import type { RuntimeDeterministicAdapter } from "./types"; + +export interface FrameSource { + element: Element; + ready?: PromiseLike; + render: (sourceTime: number, signal: AbortSignal) => void | PromiseLike; + dispose?: () => void; +} + +interface RegisteredSource { + element: Element; + ready: Promise; + seek: (time: number) => Promise; + dispose: () => void; +} + +const sources = new Map(); + +/** Bind a frame source to a timed host. Unregister before replacing its source. */ +export function registerFrameSource(source: FrameSource): () => void { + if (sources.has(source.element)) throw new Error("This element already has a frame source"); + const controller = new AbortController(); + const cancelled = new Promise((resolve) => { + controller.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + const ready = Promise.race([Promise.resolve(source.ready), cancelled]); + void ready.catch(() => {}); + let pending: number | null = null; + let work: Promise | null = null; + const drain = async () => { + try { + await ready; + while (!controller.signal.aborted && pending !== null) { + const time = pending; + pending = null; + await Promise.race([source.render(time, controller.signal), cancelled]); + } + } finally { + pending = null; + work = null; + } + }; + const registered: RegisteredSource = { + element: source.element, + ready, + seek(time) { + pending = time; + work ??= drain(); + return work; + }, + dispose() { + if (controller.signal.aborted) return; + controller.abort(); + pending = null; + sources.delete(source.element); + source.dispose?.(); + }, + }; + sources.set(source.element, registered); + return registered.dispose; +} + +export function createFrameSourceAdapter(timing: { + start: (element: Element) => number; + duration: (element: Element) => number | null; +}): RuntimeDeterministicAdapter { + const owned = new Set(); + let readySources: RegisteredSource[] = []; + let readiness: Promise | null = null; + const current = () => { + for (const source of owned) { + if (sources.get(source.element) !== source) owned.delete(source); + } + const connected: Array<[Element, RegisteredSource]> = []; + for (const [element, source] of sources) { + if (!element.isConnected) { + source.dispose(); + continue; + } + if (!owned.has(source)) { + owned.add(source); + // Readiness trackers settle rejected setup; capture must also receive that failure. + registerSeekCompletion(source.ready); + } + connected.push([element, source]); + } + return connected; + }; + const removals = new MutationObserver(() => { + current(); + }); + removals.observe(document, { childList: true, subtree: true }); + return { + name: "frame-source", + discover: () => { + current(); + }, + pause: () => {}, + seek: ({ time }) => { + for (const [element, source] of current()) { + const start = timing.start(element); + const duration = timing.duration(element); + if (time < start || (duration !== null && time > start + duration)) continue; + const localTime = Math.max(0, time - start); + const sourceTime = + readMediaStart(element) + sourceTimeAt(readElementRateSpec(element), localTime); + registerSeekCompletion(source.seek(sourceTime)); + } + }, + getReadyPromise: () => { + const next = current().map(([, source]) => source); + if (next.length === 0) return null; + if ( + next.length !== readySources.length || + next.some((source, i) => source !== readySources[i]) + ) { + readySources = next; + readiness = Promise.all(next.map((source) => source.ready)); + } + return readiness; + }, + revert: () => { + removals.disconnect(); + for (const source of owned) source.dispose(); + owned.clear(); + }, + }; +} diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index fa9731725a..137fa3106d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -20,6 +20,7 @@ import { createGoogleMapsAdapter } from "./adapters/google-maps"; import { createMaplibreAdapter } from "./adapters/maplibre"; import { createD3Adapter } from "./adapters/d3"; import { createTypegpuAdapter } from "./adapters/typegpu"; +import { createFrameSourceAdapter } from "./frameSources"; import { patchVideoTextureCompat, patchWebGLVideoTextureCompat, @@ -3979,6 +3980,10 @@ export function initSandboxRuntimeModular(): void { }); state.deterministicAdapters = [ + createFrameSourceAdapter({ + start: (element) => resolveStartForElement(element, 0), + duration: (element) => resolveDurationForElement(element), + }), createWaapiAdapter(), createCssAdapter({ resolveStartSeconds: (element) => resolveStartForElement(element, 0), diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 76a30e0e85..4a802d4208 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -38,6 +38,7 @@ declare global { __timelines: Record; __player?: PlayerAPI; __hyperframes?: { + registerFrameSource: typeof import("./frameSources").registerFrameSource; /** A path the calling composition wrote relative to its own file, as a URL the page can load. */ assetUrl?: (path: string) => string; registerRuntimeDataHandler?: ( From 30f6c08aaa2c09828c30c30a5723d8eaa93a9a8f Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 14:27:21 -0700 Subject: [PATCH 02/13] feat(core): bridge preserved film HTML to native capture --- docs/docs.json | 1 + docs/guides/imported-film-html.md | 32 ++++ packages/core/src/runtime/entry.test.ts | 1 + packages/core/src/runtime/entry.ts | 3 + packages/core/src/runtime/filmBridge.test.ts | 145 +++++++++++++++++++ packages/core/src/runtime/filmBridge.ts | 110 ++++++++++++++ packages/core/src/runtime/window.d.ts | 1 + 7 files changed, 293 insertions(+) create mode 100644 docs/guides/imported-film-html.md create mode 100644 packages/core/src/runtime/filmBridge.test.ts create mode 100644 packages/core/src/runtime/filmBridge.ts diff --git a/docs/docs.json b/docs/docs.json index 7fbe3b2cd7..ecf87fde00 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -914,6 +914,7 @@ "concepts/data-attributes", "guides/gsap-animation", "guides/frame-sources", + "guides/imported-film-html", "concepts/frame-adapters", "concepts/determinism", "guides/html-in-canvas", diff --git a/docs/guides/imported-film-html.md b/docs/guides/imported-film-html.md new file mode 100644 index 0000000000..5b7513c679 --- /dev/null +++ b/docs/guides/imported-film-html.md @@ -0,0 +1,32 @@ +--- +title: "Imported film HTML" +description: "Connect preserved film-runner HTML to HyperFrames capture." +--- + +The runtime supports the `appifact-film:` message protocol used by current Claude Motion HTML exports. Keep the original `film-runner` HTML and scene modules intact. The MCP importer packages them with a HyperFrames wrapper; HyperFrames controls time and captures the resulting pixels using its existing renderer. + +Create and mount an iframe with `sandbox="allow-scripts"`, explicit picture dimensions, and no player controls. Then connect it before assigning any runner HTML yourself: + +```javascript +const bridge = window.__hyperframes.createFilmBridge({ + iframe: document.getElementById("film-stage"), + runnerHtml: preservedRunnerHtml, + load: { script, modules, assets, look, faces }, +}); +const unregister = window.__hyperframes.registerFrameSource({ + element: document.getElementById("film-clip"), + ready: bridge.ready, + render: bridge.render, + dispose: bridge.dispose, +}); +``` + +`preservedRunnerHtml` is the decoded JSON value of the original `film-runner` script, not the outer HTML player. `load` carries the original entry script, modules, look, fonts and asset records. Supply the existing protocol's asset buffers/URLs; the bridge structured-clones them without transferring ownership, allowing separate runner instances to reuse them. Asset packaging and HTML extraction belong to the importer. + +The bridge installs its listener before setting `srcdoc`. It accepts messages only from that iframe's window with opaque origin `"null"`. It sends `load` after `hello`, waits for `ready`, and sends explicit `frame` requests with increasing sequence IDs. Capture waits for the matching frame acknowledgement; stale replies do not release it. The wrapper never calls the original player's autoplay path. + +Startup has a 20-second deadline and a frame has a 15-second deadline. Startup, runner and timeout failures reject capture. An individual `frame-error` can recover on the next seek. Unregistering rejects outstanding bridge work and removes its listener. Use a separate bridge and runner iframe for each independently timed or overlapping scene. + +Use screenshot capture for iframe pixels. This bridge does not read the sandbox DOM, expose internal scene nodes as Studio layers, or provide audio mixing and video extraction. Font/asset readiness depends on the preserved runner's `ready` and `frame` guarantees and must be checked for each supported export version. + +The compatibility target is the current protocol, not every arbitrary animated web page. Visual comparison against the partner preview remains an integration acceptance step after MCP packaging. diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index 63b7afd2dc..dae405e6c2 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -103,6 +103,7 @@ describe("runtime entry", () => { window.__timelines = {}; Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" }); await evaluateRuntime(); + expect(window.__hyperframes!.createFilmBridge).toEqual(expect.any(Function)); let finish!: () => void; const first = new Promise((resolve) => { finish = resolve; diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 6903414623..4b7c7599f0 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -15,6 +15,7 @@ import { clearRuntimeData, registerRuntimeDataHandler, setRuntimeData } from "./ import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; import { registerFrameSource } from "./frameSources"; +import { createFilmBridge } from "./filmBridge"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -27,6 +28,7 @@ type HyperframeWindow = Window & { setRuntimeData: typeof setRuntimeData; clearRuntimeData: typeof clearRuntimeData; registerFrameSource: typeof registerFrameSource; + createFilmBridge: typeof createFilmBridge; }; }; @@ -56,6 +58,7 @@ deferMediaUntilDue(); setRuntimeData, clearRuntimeData, registerFrameSource, + createFilmBridge, }; function bootstrapHyperframeRuntime(): void { diff --git a/packages/core/src/runtime/filmBridge.test.ts b/packages/core/src/runtime/filmBridge.test.ts new file mode 100644 index 0000000000..f4825c07a6 --- /dev/null +++ b/packages/core/src/runtime/filmBridge.test.ts @@ -0,0 +1,145 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createFilmBridge, type FilmBridge } from "./filmBridge"; + +const bridges: FilmBridge[] = []; +afterEach(() => { + for (const bridge of bridges.splice(0)) bridge.dispose(); + document.body.replaceChildren(); + vi.useRealTimers(); +}); +function setup() { + const iframe = document.createElement("iframe"); + iframe.setAttribute("sandbox", "allow-scripts"); + document.body.appendChild(iframe); + const send = vi.spyOn(iframe.contentWindow!, "postMessage").mockImplementation(() => {}); + const load = { script: "window.CUT = kit.scenes()", modules: [], assets: [] }; + const bridge = createFilmBridge({ iframe, runnerHtml: "
fixture runner
", load }); + bridges.push(bridge); + const receive = ( + type: string, + data: Record = {}, + origin = "null", + source = iframe.contentWindow, + ) => { + window.dispatchEvent( + new MessageEvent("message", { + source, + origin, + data: { ...data, type: "appifact-film:" + type }, + }), + ); + }; + const initialize = () => { + receive("hello"); + receive("ready"); + }; + return { iframe, send, load, bridge, receive, initialize }; +} + +describe("film runner bridge", () => { + it("loads the original protocol payload once and waits for the matching frame", async () => { + const { iframe, bridge, receive, send, load, initialize } = setup(); + expect(iframe.srcdoc).toContain("fixture runner"); + initialize(); + receive("hello"); + await bridge.ready; + const frame = bridge.render(3.25); + await Promise.resolve(); + expect(send).toHaveBeenNthCalledWith(1, { ...load, type: "appifact-film:load" }, "*"); + expect(send).toHaveBeenNthCalledWith(2, { type: "appifact-film:frame", t: 3.25, seq: 1 }, "*"); + let complete = false; + void frame.then(() => { + complete = true; + }); + receive("frame", { seq: 2 }); + await Promise.resolve(); + expect(complete).toBe(false); + receive("frame", { seq: 1 }); + await frame; + expect(complete).toBe(true); + }); + + it("rejects sandbox configurations that expose a same-origin document", () => { + const iframe = document.createElement("iframe"); + for (const sandbox of ["", "allow-scripts allow-same-origin"]) { + iframe.setAttribute("sandbox", sandbox); + expect(() => createFilmBridge({ iframe, runnerHtml: "", load: {} })).toThrow("sandbox"); + } + }); + + it("ignores foreign sources, origins, and ready messages before hello", async () => { + const { bridge, receive, send } = setup(); + let ready = false; + void bridge.ready.then(() => { + ready = true; + }); + receive("hello", {}, "https://example.com"); + receive("hello", {}, "null", window); + receive("ready"); + await Promise.resolve(); + expect(ready).toBe(false); + expect(send).not.toHaveBeenCalled(); + receive("hello"); + receive("ready"); + await bridge.ready; + }); + + it("can seek again after an individual frame error", async () => { + const { bridge, initialize, receive } = setup(); + initialize(); + const failed = bridge.render(2); + const rejection = expect(failed).rejects.toThrow("bad frame"); + await Promise.resolve(); + receive("frame-error", { seq: 1, message: "bad frame" }); + await rejection; + const next = bridge.render(0); + await Promise.resolve(); + receive("frame", { seq: 1 }); + receive("frame", { seq: 2 }); + await next; + }); + + it("propagates startup failures and keeps fatal errors for later seeks", async () => { + const { bridge, receive } = setup(); + receive("error", { message: "module missing" }); + await expect(bridge.ready).rejects.toThrow("module missing"); + await expect(bridge.render(0)).rejects.toThrow("module missing"); + const second = setup(); + second.initialize(); + await second.bridge.ready; + second.receive("error", { message: "runner crashed" }); + await expect(second.bridge.render(0)).rejects.toThrow("runner crashed"); + }); + + it("rejects stalled initialization and frames with bounded timeouts", async () => { + vi.useFakeTimers(); + const first = setup(); + const rejected = expect(first.bridge.ready).rejects.toThrow("20 s"); + await vi.advanceTimersByTimeAsync(20_000); + await rejected; + const second = setup(); + second.initialize(); + const frame = second.bridge.render(0); + const rejection = expect(frame).rejects.toThrow("15 s"); + await vi.advanceTimersByTimeAsync(15_000); + await rejection; + await expect(second.bridge.render(1)).rejects.toThrow("15 s"); + }); + + it("disposes before startup or during a frame and ignores late messages", async () => { + const first = setup(); + first.bridge.dispose(); + first.bridge.dispose(); + first.initialize(); + await expect(first.bridge.ready).rejects.toThrow("disposed"); + expect(first.send).not.toHaveBeenCalled(); + const second = setup(); + second.initialize(); + const frame = second.bridge.render(0); + await Promise.resolve(); + second.bridge.dispose(); + second.receive("frame", { seq: 1 }); + await expect(frame).rejects.toThrow("disposed"); + await expect(second.bridge.render(1)).rejects.toThrow("disposed"); + }); +}); diff --git a/packages/core/src/runtime/filmBridge.ts b/packages/core/src/runtime/filmBridge.ts new file mode 100644 index 0000000000..3da9b5805a --- /dev/null +++ b/packages/core/src/runtime/filmBridge.ts @@ -0,0 +1,110 @@ +const PREFIX = "appifact-film:"; + +export interface FilmBridge { + ready: Promise; + render: (time: number) => Promise; + dispose: () => void; +} + +/** Install the listener before loading the preserved runner into its opaque sandbox. */ +export function createFilmBridge(options: { + iframe: HTMLIFrameElement; + runnerHtml: string; + load: Record; +}): FilmBridge { + const { iframe, runnerHtml, load } = options; + const permissions = (iframe.getAttribute("sandbox") ?? "").split(/\s+/); + if (!permissions.includes("allow-scripts") || permissions.includes("allow-same-origin")) { + throw new Error('Film runners require sandbox="allow-scripts" without allow-same-origin'); + } + let sequence = 0; + let loaded = false; + let disposed = false; + let failure: Error | null = null; + let resolveReady!: () => void; + let rejectReady!: (reason: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + void ready.catch(() => {}); + let pending: { + sequence: number; + resolve: () => void; + reject: (reason: Error) => void; + timer: ReturnType; + } | null = null; + const fail = (error: Error) => { + failure = error; + clearTimeout(startupTimer); + rejectReady(error); + if (pending) { + clearTimeout(pending.timer); + pending.reject(error); + pending = null; + } + }; + const startupTimer = setTimeout( + () => fail(new Error("Film runner did not become ready within 20 s")), + 20_000, + ); + const completeFrame = (data: { type: unknown }) => { + if ( + (data.type !== PREFIX + "frame" && data.type !== PREFIX + "frame-error") || + !("seq" in data) || + pending === null || + pending.sequence !== data.seq + ) + return; + const request = pending; + pending = null; + clearTimeout(request.timer); + if (data.type === PREFIX + "frame-error") { + request.reject(new Error("message" in data ? String(data.message) : "Film frame failed")); + } else request.resolve(); + }; + const receiveStartup = (data: { type: unknown }) => { + if (data.type === PREFIX + "hello" && !loaded) { + loaded = true; + iframe.contentWindow?.postMessage({ ...load, type: PREFIX + "load" }, "*"); + } else if (data.type === PREFIX + "ready" && loaded) { + clearTimeout(startupTimer); + resolveReady(); + } else if (data.type === PREFIX + "error") { + fail(new Error("message" in data ? String(data.message) : "Film runner failed")); + } + }; + const receive = (event: MessageEvent) => { + if (disposed || failure || event.source !== iframe.contentWindow || event.origin !== "null") { + return; + } + const data = event.data; + if (!data || typeof data !== "object" || !("type" in data)) return; + receiveStartup(data); + completeFrame(data); + }; + window.addEventListener("message", receive); + iframe.srcdoc = runnerHtml; + return { + ready, + async render(time) { + await ready; + if (failure) throw failure; + if (pending) throw new Error("Film seeks must be serialized; use registerFrameSource"); + const seq = ++sequence; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + fail(new Error("Film frame did not arrive within 15 s")); + }, 15_000); + pending = { sequence: seq, resolve, reject, timer }; + iframe.contentWindow?.postMessage({ type: PREFIX + "frame", t: time, seq }, "*"); + }); + }, + dispose() { + if (disposed) return; + disposed = true; + window.removeEventListener("message", receive); + fail(new Error("Film bridge was disposed")); + }, + }; +} diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 4a802d4208..3aa2885d6e 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -39,6 +39,7 @@ declare global { __player?: PlayerAPI; __hyperframes?: { registerFrameSource: typeof import("./frameSources").registerFrameSource; + createFilmBridge: typeof import("./filmBridge").createFilmBridge; /** A path the calling composition wrote relative to its own file, as a URL the page can load. */ assetUrl?: (path: string) => string; registerRuntimeDataHandler?: ( From 789b2ee3e2977b0f7b9abe1aa1b7a7ca1125f3a7 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 14:30:51 -0700 Subject: [PATCH 03/13] feat(core): bound imported scene timing to source intervals --- docs/guides/frame-sources.md | 2 +- docs/guides/imported-film-html.md | 24 +++++- packages/core/src/runtime/filmBridge.test.ts | 1 + packages/core/src/runtime/filmBridge.ts | 1 + .../core/src/runtime/frameSources.test.ts | 79 ++++++++++++++++++- packages/core/src/runtime/frameSources.ts | 29 ++++++- packages/core/src/runtime/init.ts | 1 + packages/sdk/src/session.film-scenes.test.ts | 75 ++++++++++++++++++ 8 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 packages/sdk/src/session.film-scenes.test.ts diff --git a/docs/guides/frame-sources.md b/docs/guides/frame-sources.md index b0fde5f38b..06c7f3cf10 100644 --- a/docs/guides/frame-sources.md +++ b/docs/guides/frame-sources.md @@ -16,7 +16,7 @@ const unregister = window.__hyperframes.registerFrameSource({ The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. -The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. +The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Optional `sourceRange: { start, duration, fps }` adds an original source offset and clamps this time to the last source frame within that interval. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. `ready` holds initialization and export capture. Return a promise from `render` when drawing is asynchronous: capture waits for it before taking pixels. Repeated and out-of-order times must produce the same frame. HyperFrames owns playback; do not start a second clock. diff --git a/docs/guides/imported-film-html.md b/docs/guides/imported-film-html.md index 5b7513c679..e4efe7b10e 100644 --- a/docs/guides/imported-film-html.md +++ b/docs/guides/imported-film-html.md @@ -25,8 +25,30 @@ const unregister = window.__hyperframes.registerFrameSource({ The bridge installs its listener before setting `srcdoc`. It accepts messages only from that iframe's window with opaque origin `"null"`. It sends `load` after `hello`, waits for `ready`, and sends explicit `frame` requests with increasing sequence IDs. Capture waits for the matching frame acknowledgement; stale replies do not release it. The wrapper never calls the original player's autoplay path. -Startup has a 20-second deadline and a frame has a 15-second deadline. Startup, runner and timeout failures reject capture. An individual `frame-error` can recover on the next seek. Unregistering rejects outstanding bridge work and removes its listener. Use a separate bridge and runner iframe for each independently timed or overlapping scene. +Startup has a 20-second deadline and a frame has a 15-second deadline. Startup, runner and timeout failures reject capture. An individual `frame-error` can recover on the next seek. Unregistering rejects outstanding bridge work, removes its listener and clears the runner document. Use a separate bridge and runner iframe for each independently timed or overlapping scene. Use screenshot capture for iframe pixels. This bridge does not read the sandbox DOM, expose internal scene nodes as Studio layers, or provide audio mixing and video extraction. Font/asset readiness depends on the preserved runner's `ready` and `frame` guarantees and must be checked for each supported export version. The compatibility target is the current protocol, not every arbitrary animated web page. Visual comparison against the partner preview remains an integration acceptance step after MCP packaging. + +## Editable scene timing + +To expose individual scenes on the HyperFrames timeline, mount one timed host and independent runner iframe per scene. Each runner can load the preserved full film. Register the original scene's immutable source interval separately from its editable host timing: + +```javascript +window.__hyperframes.registerFrameSource({ + element: sceneHost, + ready: bridge.ready, + render: bridge.render, + dispose: bridge.dispose, + sourceRange: { start: 3, duration: 4, fps: 60 }, +}); +``` + +For a scene originally covering `[3, 7)`, source time is `3 + data-playback-start + rateAdjustedLocalTime`. The source interval clamps that result between 3 and the last source frame before 7. Moving the host changes `data-start`; trimming changes its duration and source inpoint; stretching uses the existing playback-rate attributes. Extending beyond available source footage holds the last frame. Changing duration alone does not automatically stretch the animation. + +Adjacent clips use half-open timeline windows. At the final composition instant, the final scene holds its last frame. Different source instances allow reordered or overlapping scenes to request different original film times without sharing mutable runner state. + +Persist the preserved runner/load payload and `sourceRange` in the wrapper's initialization data. Host timing stays in ordinary HyperFrames attributes, so serialization, save/reopen, undo and redo preserve it without rewriting scene modules. Do not use the original player's `recut` field as a replacement for host timing: that protocol does not represent arbitrary trim, reorder and overlap edits. + +This provides scene-level timing and placement. Text, colors and individual animation changes still require editing the preserved scene modules. The runtime does not expose iframe internals as manually editable Studio objects or keyframes. diff --git a/packages/core/src/runtime/filmBridge.test.ts b/packages/core/src/runtime/filmBridge.test.ts index f4825c07a6..f10cd8e48f 100644 --- a/packages/core/src/runtime/filmBridge.test.ts +++ b/packages/core/src/runtime/filmBridge.test.ts @@ -132,6 +132,7 @@ describe("film runner bridge", () => { first.bridge.dispose(); first.initialize(); await expect(first.bridge.ready).rejects.toThrow("disposed"); + expect(first.iframe.srcdoc).toBe(""); expect(first.send).not.toHaveBeenCalled(); const second = setup(); second.initialize(); diff --git a/packages/core/src/runtime/filmBridge.ts b/packages/core/src/runtime/filmBridge.ts index 3da9b5805a..59457da241 100644 --- a/packages/core/src/runtime/filmBridge.ts +++ b/packages/core/src/runtime/filmBridge.ts @@ -105,6 +105,7 @@ export function createFilmBridge(options: { disposed = true; window.removeEventListener("message", receive); fail(new Error("Film bridge was disposed")); + iframe.srcdoc = ""; }, }; } diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts index 7e9c2ede79..bf642f976c 100644 --- a/packages/core/src/runtime/frameSources.test.ts +++ b/packages/core/src/runtime/frameSources.test.ts @@ -14,10 +14,11 @@ function deferred() { } const adapters: ReturnType[] = []; -const adapter = () => { +const adapter = (compositionDuration = 20) => { const runtime = createFrameSourceAdapter({ start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0), duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element), + compositionDuration: () => compositionDuration, }); adapters.push(runtime); return runtime; @@ -215,4 +216,80 @@ describe("frame sources", () => { b.resolve(); await waitForSeekCompletion(); }); + it("keeps trimmed, sped-up and extended clips inside their original scene", async () => { + const host = mount("5", "8"); + host.setAttribute("data-playback-start", "0.5"); + host.setAttribute("data-playback-rate", "2"); + const render = vi.fn(); + disposers.push( + registerFrameSource({ + element: host, + render, + sourceRange: { start: 3, duration: 4, fps: 60 }, + }), + ); + const runtime = adapter(); + runtime.seek({ time: 5 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(3.5, expect.any(AbortSignal)); + runtime.seek({ time: 6 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(5.5, expect.any(AbortSignal)); + runtime.seek({ time: 12 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(7 - 1 / 60, expect.any(AbortSignal)); + host.setAttribute("data-start", "0"); + host.setAttribute("data-playback-start", "0"); + runtime.seek({ time: 0 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(3, expect.any(AbortSignal)); + }); + + it("uses half-open cut boundaries and holds the terminal scene's last source frame", async () => { + const first = vi.fn(); + const last = vi.fn(); + disposers.push(registerFrameSource({ element: mount("0", "3"), render: first })); + disposers.push( + registerFrameSource({ + element: mount("3", "4"), + render: last, + sourceRange: { start: 10, duration: 4, fps: 60 }, + }), + ); + const runtime = adapter(7); + runtime.seek({ time: 3 }); + await waitForSeekCompletion(); + expect(first).not.toHaveBeenCalled(); + expect(last).toHaveBeenLastCalledWith(10, expect.any(AbortSignal)); + runtime.seek({ time: 7 }); + await waitForSeekCompletion(); + expect(last.mock.lastCall?.[0]).toBeCloseTo(14 - 1 / 60, 10); + }); + + it("does not redraw an ended scene across a gap or seek backward before its start", async () => { + const render = vi.fn(); + disposers.push(registerFrameSource({ element: mount("2", "2"), render })); + const runtime = adapter(8); + for (const time of [0, 1, 4, 6, 8]) runtime.seek({ time }); + await waitForSeekCompletion(); + expect(render).not.toHaveBeenCalled(); + runtime.seek({ time: 2 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(1); + }); + + it("rejects invalid original ranges before registering a source", () => { + const element = mount(); + for (const sourceRange of [ + { start: -1, duration: 4, fps: 60 }, + { start: 0, duration: 0, fps: 60 }, + { start: 0, duration: 4, fps: NaN }, + { start: Infinity, duration: 4, fps: 60 }, + ]) { + expect(() => registerFrameSource({ element, render: vi.fn(), sourceRange })).toThrow( + "ranges", + ); + } + disposers.push(registerFrameSource({ element, render: vi.fn() })); + }); }); diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts index a8dc2140ef..b515526a50 100644 --- a/packages/core/src/runtime/frameSources.ts +++ b/packages/core/src/runtime/frameSources.ts @@ -1,11 +1,13 @@ import { sourceTimeAt } from "../speedRamp"; import { readElementRateSpec, readMediaStart } from "./playbackRate"; import { registerSeekCompletion } from "./adapters/seek-dispatch"; +import { isClipVisibleAt } from "./clipWindow"; import type { RuntimeDeterministicAdapter } from "./types"; export interface FrameSource { element: Element; ready?: PromiseLike; + sourceRange?: { start: number; duration: number; fps: number }; render: (sourceTime: number, signal: AbortSignal) => void | PromiseLike; dispose?: () => void; } @@ -22,6 +24,18 @@ const sources = new Map(); /** Bind a frame source to a timed host. Unregister before replacing its source. */ export function registerFrameSource(source: FrameSource): () => void { if (sources.has(source.element)) throw new Error("This element already has a frame source"); + const range = source.sourceRange; + if ( + range && + (!Number.isFinite(range.start) || + range.start < 0 || + !Number.isFinite(range.duration) || + range.duration <= 0 || + !Number.isFinite(range.fps) || + range.fps <= 0) + ) { + throw new Error("Frame source ranges require a nonnegative start, positive duration and FPS"); + } const controller = new AbortController(); const cancelled = new Promise((resolve) => { controller.signal.addEventListener("abort", () => resolve(), { once: true }); @@ -47,7 +61,9 @@ export function registerFrameSource(source: FrameSource): () => void { element: source.element, ready, seek(time) { - pending = time; + pending = range + ? range.start + Math.min(Math.max(0, time), Math.max(0, range.duration - 1 / range.fps)) + : time; work ??= drain(); return work; }, @@ -66,6 +82,7 @@ export function registerFrameSource(source: FrameSource): () => void { export function createFrameSourceAdapter(timing: { start: (element: Element) => number; duration: (element: Element) => number | null; + compositionDuration: () => number; }): RuntimeDeterministicAdapter { const owned = new Set(); let readySources: RegisteredSource[] = []; @@ -103,7 +120,15 @@ export function createFrameSourceAdapter(timing: { for (const [element, source] of current()) { const start = timing.start(element); const duration = timing.duration(element); - if (time < start || (duration !== null && time > start + duration)) continue; + if ( + !isClipVisibleAt( + time, + start, + start + (duration ?? Infinity), + timing.compositionDuration(), + ) + ) + continue; const localTime = Math.max(0, time - start); const sourceTime = readMediaStart(element) + sourceTimeAt(readElementRateSpec(element), localTime); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 137fa3106d..92de290194 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3983,6 +3983,7 @@ export function initSandboxRuntimeModular(): void { createFrameSourceAdapter({ start: (element) => resolveStartForElement(element, 0), duration: (element) => resolveDurationForElement(element), + compositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0), }), createWaapiAdapter(), createCssAdapter({ diff --git a/packages/sdk/src/session.film-scenes.test.ts b/packages/sdk/src/session.film-scenes.test.ts new file mode 100644 index 0000000000..696a7feee5 --- /dev/null +++ b/packages/sdk/src/session.film-scenes.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { openComposition } from "./session.js"; + +const SOURCE = "const originalScene = (t) => ({ x: 200 * t });"; +const LOAD = JSON.stringify({ + script: SOURCE, + modules: [], + assets: [], + sourceRange: { start: 3, duration: 4, fps: 60 }, +}); +const HTML = ` +
+
+ +
+ +
`; + +function timing(html: string) { + const tag = /]*data-hf-id="hf-boat"[^>]*>/.exec(html)?.[0] ?? ""; + const attr = (name: string) => new RegExp(`${name}="([^"]*)"`).exec(tag)?.[1]; + return { + start: attr("data-start"), + duration: attr("data-duration"), + inpoint: attr("data-playback-start"), + rate: attr("data-playback-rate"), + track: attr("data-track-index"), + }; +} + +function expectPreserved(html: string) { + expect(html).toContain(LOAD); + expect(html).toContain('sandbox="allow-scripts"'); + expect(html).not.toContain("allow-same-origin"); + expect(html).not.toContain("gsap.timeline"); +} + +describe("preserved film scene editing", () => { + it("moves and resizes a no-GSAP scene without rewriting its source", async () => { + const comp = await openComposition(HTML); + comp.setTiming("hf-boat", { start: 7, duration: 6, trackIndex: 2 }); + const exported = comp.serialize(); + expect(timing(exported)).toMatchObject({ start: "7", duration: "6", track: "2" }); + expectPreserved(exported); + const reopened = await openComposition(exported); + expect(timing(reopened.serialize())).toEqual(timing(exported)); + expectPreserved(reopened.serialize()); + }); + + it("persists an independent source inpoint and rate across save/reopen", async () => { + const comp = await openComposition(HTML); + comp.setAttribute("hf-boat", "data-playback-start", "0.5"); + comp.setAttribute("hf-boat", "data-playback-rate", "2"); + comp.setTiming("hf-boat", { start: 0, duration: 1.5 }); + const reopened = await openComposition(comp.serialize()); + expect(timing(reopened.serialize())).toMatchObject({ + start: "0", + duration: "1.5", + inpoint: "0.5", + rate: "2", + }); + expectPreserved(reopened.serialize()); + }); + + it("undoes and redoes timing edits while retaining the original scene payload", async () => { + const comp = await openComposition(HTML); + comp.setTiming("hf-boat", { start: 0, duration: 8 }); + comp.undo(); + expect(timing(comp.serialize())).toMatchObject({ start: "3", duration: "4" }); + expectPreserved(comp.serialize()); + comp.redo(); + expect(timing(comp.serialize())).toMatchObject({ start: "0", duration: "8" }); + expectPreserved(comp.serialize()); + }); +}); From 7503e04fa750d2c2f0da9c8170757fd4ce6bb7cd Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 15:17:58 -0700 Subject: [PATCH 04/13] test(producer): verify film bridge pixels in real browser --- .../src/services/coreRuntimeBrowser.test.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 30e5bde580..2485e441ee 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -4,6 +4,8 @@ import { tmpdir } from "node:os"; import { bundleToSingleHtml } from "@hyperframes/core/compiler"; import { resolve } from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer"; +import type {} from "../../../core/src/runtime/window"; +import { waitForPendingSeekCompletion } from "../../../engine/src/services/frameCapture"; const RUNTIME_PATH = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js"); const PNG_1PX = @@ -1008,3 +1010,105 @@ describe("core runtime browser contract", () => { expect(result).toEqual({ hadTeardown: true, teardownCleared: true, isPlaying: false }); }); }); + +// This is a first-party protocol fixture, not the partner runner or its animation code. +const FILM_RUNNER_FIXTURE = ` +`; + +function filmRuntimeFixture(runtime: string): string { + const runnerLiteral = JSON.stringify(FILM_RUNNER_FIXTURE).replaceAll("<", "\\u003c"); + return ` + + +
+
+ +
+`; +} + +describe("film bridge browser capture contract", () => { + let browser: Browser; + let html: string; + beforeAll(async () => { + html = filmRuntimeFixture(readFileSync(RUNTIME_PATH, "utf8")); + browser = await puppeteer.launch({ + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); + }, 30_000); + afterAll(async () => { + await browser?.close(); + }); + + async function openFilm(): Promise { + const page = await browser.newPage(); + await page.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); + await page.setContent(html); + await page.waitForFunction( + () => window.__playerReady === true && window.__renderReady === true, + ); + return page; + } + + it("captures the acknowledged frame for reverse, repeated and fresh-page seeks", async () => { + const page = await openFilm(); + const reference = await browser.newPage(); + await reference.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + const captures = new Map(); + try { + for (const time of [0, 0.5, 0.2, 0.5, 0.9, 1, 0]) { + await page.evaluate((t) => window.__player?.renderSeek?.(t), time); + await waitForPendingSeekCompletion(page); + const actual = await page.screenshot(); + const sourceTime = 3 + Math.min(time, 1 - 1 / 30); + await reference.setContent( + ``, + ); + expect(Buffer.from(actual)).toEqual(Buffer.from(await reference.screenshot())); + const previous = captures.get(time); + if (previous) expect(Buffer.from(actual)).toEqual(Buffer.from(previous)); + captures.set(time, actual); + } + const fresh = await openFilm(); + try { + await fresh.evaluate(() => window.__player?.renderSeek?.(0.5)); + await waitForPendingSeekCompletion(fresh); + expect(Buffer.from(await fresh.screenshot())).toEqual(Buffer.from(captures.get(0.5)!)); + } finally { + await fresh.close(); + } + await page.evaluate(() => { + const scene = document.getElementById("scene")!; + scene.setAttribute("data-playback-start", "0.25"); + scene.setAttribute("data-playback-rate", "2"); + window.__player?.renderSeek?.(0.2); + }); + await waitForPendingSeekCompletion(page); + await reference.setContent(""); + expect(Buffer.from(await page.screenshot())).toEqual( + Buffer.from(await reference.screenshot()), + ); + expect(errors).toEqual([]); + } finally { + await page.close(); + await reference.close(); + } + }, 30_000); +}); From 37b9f4c302fb977de9dc38c60ff05f627aaeb934 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 16:26:35 -0700 Subject: [PATCH 05/13] fix(runtime): retain queued draws and disable unsafe dedup --- packages/core/src/runtime/entry.ts | 3 +- .../core/src/runtime/frameSources.test.ts | 24 ++++++++- packages/core/src/runtime/frameSources.ts | 10 +++- packages/core/src/runtime/window.d.ts | 1 + packages/engine/src/services/frameCapture.ts | 12 +++++ .../src/services/coreRuntimeBrowser.test.ts | 53 +++++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 6903414623..23e43ac7e6 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -14,7 +14,7 @@ import { getVariables } from "./getVariables"; import { clearRuntimeData, registerRuntimeDataHandler, setRuntimeData } from "./runtimeData"; import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; -import { registerFrameSource } from "./frameSources"; +import { hasFrameSources, registerFrameSource } from "./frameSources"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -40,6 +40,7 @@ type HyperframeWindow = Window & { installAuthoredOpacityCapture(); installAuthoredMediaCapture(); installFlatGsapTransforms(); +window.__hfHasFrameSources = hasFrameSources; hideTimedClipsUntilFirstPass(); deferMediaUntilDue(); diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts index 7e9c2ede79..36a27145b3 100644 --- a/packages/core/src/runtime/frameSources.test.ts +++ b/packages/core/src/runtime/frameSources.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createFrameSourceAdapter, registerFrameSource } from "./frameSources"; +import { createFrameSourceAdapter, hasFrameSources, registerFrameSource } from "./frameSources"; import { resetSeekDispatchState, waitForSeekCompletion } from "./adapters/seek-dispatch"; import { createRuntimeStartTimeResolver } from "./startResolver"; @@ -215,4 +215,26 @@ describe("frame sources", () => { b.resolve(); await waitForSeekCompletion(); }); + it("drains a queued seek after an earlier draw fails and retains the failure for capture", async () => { + const first = deferred(); + const render = vi + .fn() + .mockImplementationOnce(() => first.promise) + .mockResolvedValue(undefined); + const unregister = registerFrameSource({ element: mount(), render }); + disposers.push(unregister); + expect(hasFrameSources()).toBe(true); + const runtime = adapter(); + runtime.seek({ time: 1 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + runtime.seek({ time: 2 }); + const capture = expect(waitForSeekCompletion()).rejects.toThrow("first draw failed"); + first.reject(new Error("first draw failed")); + await capture; + expect(render.mock.calls.map(([time]) => time)).toEqual([1, 2]); + runtime.seek({ time: 3 }); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + unregister(); + expect(hasFrameSources()).toBe(false); + }); }); diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts index a8dc2140ef..67f9414c69 100644 --- a/packages/core/src/runtime/frameSources.ts +++ b/packages/core/src/runtime/frameSources.ts @@ -19,6 +19,8 @@ interface RegisteredSource { const sources = new Map(); +export const hasFrameSources = (): boolean => sources.size > 0; + /** Bind a frame source to a timed host. Unregister before replacing its source. */ export function registerFrameSource(source: FrameSource): () => void { if (sources.has(source.element)) throw new Error("This element already has a frame source"); @@ -31,13 +33,19 @@ export function registerFrameSource(source: FrameSource): () => void { let pending: number | null = null; let work: Promise | null = null; const drain = async () => { + let failure: { reason: unknown } | undefined; try { await ready; while (!controller.signal.aborted && pending !== null) { const time = pending; pending = null; - await Promise.race([source.render(time, controller.signal), cancelled]); + try { + await Promise.race([source.render(time, controller.signal), cancelled]); + } catch (reason) { + failure ??= { reason }; + } } + if (failure) throw failure.reason; } finally { pending = null; work = null; diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 4a802d4208..4b0c3b465f 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -35,6 +35,7 @@ type ThreeLike = { declare global { interface Window { + __hfHasFrameSources?: () => boolean; __timelines: Record; __player?: PlayerAPI; __hyperframes?: { diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 42a40db396..f29c87f227 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -3032,12 +3032,15 @@ export async function computeStaticFrameSet( const w = window as unknown as { __timelines?: Record; __hf?: { duration?: number }; + __hfHasFrameSources?: () => boolean; }; for (const tl of Object.values(w.__timelines || {})) { if (tl && typeof tl.getChildren === "function") walk(tl, 0); } const hasVideo = !!document.querySelector("video"); const hasCanvas = !!document.querySelector("canvas"); + const hasFrameSources = w.__hfHasFrameSources?.() ?? false; + const hasIframe = !!document.querySelector("iframe"); // A non-numeric data-start (reference expression like "intro+0.5") can't be turned // into a clip-cut boundary by computeClipBoundaryFrames' parseFloat, so the cut goes // unprotected and could be deduped into the previous scene. Disqualify the comp. @@ -3067,6 +3070,8 @@ export async function computeStaticFrameSet( duration: w.__hf?.duration ?? 0, hasVideo, hasCanvas, + hasFrameSources, + hasIframe, hasNonGsapAnim, hasUnresolvableClipStart, hasTimelineCall, @@ -3079,6 +3084,8 @@ export async function computeStaticFrameSet( duration, hasVideo, hasCanvas, + hasFrameSources, + hasIframe, hasNonGsapAnim, hasUnresolvableClipStart, hasTimelineCall, @@ -3088,6 +3095,8 @@ export async function computeStaticFrameSet( duration: number; hasVideo: boolean; hasCanvas: boolean; + hasFrameSources: boolean; + hasIframe: boolean; hasNonGsapAnim: boolean; hasUnresolvableClipStart: boolean; hasTimelineCall: boolean; @@ -3121,6 +3130,9 @@ export async function computeStaticFrameSet( if (!(duration > 0)) reasons.push("unknown/zero duration"); if (hasVideo) reasons.push("video"); if (hasCanvas) reasons.push("canvas/webgl"); + // GSAP intervals cannot predict frame-source or opaque iframe draws. + if (hasFrameSources) reasons.push("registered frame source"); + if (hasIframe) reasons.push("iframe"); if (tweenCount === 0) reasons.push("no GSAP tweens (non-GSAP animation)"); if (hasNonGsapAnim) reasons.push("running CSS/WAAPI animation"); // tl.call() side effects are not seek-idempotent (see hasTimelineCall detection diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 30e5bde580..1ec2ba200b 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -4,6 +4,11 @@ import { tmpdir } from "node:os"; import { bundleToSingleHtml } from "@hyperframes/core/compiler"; import { resolve } from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer"; +import type {} from "../../../core/src/runtime/window"; +import { + computeStaticFrameSet, + waitForPendingSeekCompletion, +} from "../../../engine/src/services/frameCapture"; const RUNTIME_PATH = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js"); const PNG_1PX = @@ -39,6 +44,54 @@ describe("core runtime browser contract", () => { await browser?.close(); }); + it("disables static dedup for an async frame source even alongside a GSAP tween", async () => { + const sourcePage = await browser.newPage(); + try { + await sourcePage.setContent( + `
`, + ); + await sourcePage.addScriptTag({ + path: resolve(import.meta.dirname, "../../../core/node_modules/gsap/dist/gsap.min.js"), + }); + await sourcePage.addScriptTag({ + content: + 'window.__timelines = {root:gsap.timeline({paused:true}).to("#probe",{opacity:0.5,duration:0.1})};', + }); + await sourcePage.addScriptTag({ content: readFileSync(RUNTIME_PATH, "utf8") }); + await sourcePage.waitForFunction(() => window.__playerReady && window.__renderReady); + await sourcePage.evaluate(() => { + window.__hf = { duration: window.__player!.getDuration!() }; + }); + const baseline = await computeStaticFrameSet(sourcePage, 30); + expect(baseline.reason).toBe("eligible"); + expect(baseline.eligible).toBe(true); + await sourcePage.evaluate(() => { + const element = document.getElementById("source")!; + window.__hyperframes!.registerFrameSource({ + element, + render: async (time) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + element.setAttribute("data-rendered-time", String(time)); + }, + }); + window.__player!.renderSeek!(0.5); + }); + await waitForPendingSeekCompletion(sourcePage); + expect(await sourcePage.$eval("#source", (el) => el.getAttribute("data-rendered-time"))).toBe( + "0.5", + ); + const analysis = await computeStaticFrameSet(sourcePage, 30); + expect(analysis.tweenCount).toBeGreaterThan(0); + expect(analysis.eligible).toBe(false); + expect(analysis.staticFrameSet.size).toBe(0); + expect(analysis.reason).toContain("registered frame source"); + await sourcePage.evaluate(() => document.body.appendChild(document.createElement("iframe"))); + expect((await computeStaticFrameSet(sourcePage, 30)).reason).toContain("iframe"); + } finally { + await sourcePage.close(); + } + }); + it("initializes the public player contract and seeks the CSS adapter", async () => { const result = await page.evaluate(() => { const runtimeWindow = window as unknown as { From d3e7bcb5234b11ad41b835caba1456e7a99d7d39 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 16:28:15 -0700 Subject: [PATCH 06/13] fix(runtime): fail promptly on runner reloads and send errors --- packages/core/src/runtime/filmBridge.test.ts | 59 +++++++++++++++++++- packages/core/src/runtime/filmBridge.ts | 19 ++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime/filmBridge.test.ts b/packages/core/src/runtime/filmBridge.test.ts index f4825c07a6..f0ad23c4f8 100644 --- a/packages/core/src/runtime/filmBridge.test.ts +++ b/packages/core/src/runtime/filmBridge.test.ts @@ -41,7 +41,6 @@ describe("film runner bridge", () => { const { iframe, bridge, receive, send, load, initialize } = setup(); expect(iframe.srcdoc).toContain("fixture runner"); initialize(); - receive("hello"); await bridge.ready; const frame = bridge.render(3.25); await Promise.resolve(); @@ -59,6 +58,64 @@ describe("film runner bridge", () => { expect(complete).toBe(true); }); + it("rejects a runner reload during a frame and ignores old acknowledgements", async () => { + vi.useFakeTimers(); + const { bridge, initialize, receive, send } = setup(); + initialize(); + const frame = bridge.render(1); + const rejected = expect(frame).rejects.toThrow("reloaded"); + await Promise.resolve(); + receive("hello"); + receive("ready"); + receive("frame", { seq: 1 }); + await rejected; + await expect(bridge.render(2)).rejects.toThrow("reloaded"); + expect(send).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects a reload before readiness instead of leaving startup pending", async () => { + vi.useFakeTimers(); + const { bridge, receive } = setup(); + receive("hello"); + receive("hello"); + await expect(bridge.ready).rejects.toThrow("reloaded"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("settles startup immediately when the load payload cannot be cloned", async () => { + vi.useFakeTimers(); + const { bridge, receive, send } = setup(); + send.mockImplementation(() => { + throw new DOMException("uncloneable load", "DataCloneError"); + }); + receive("hello"); + await expect(bridge.ready).rejects.toThrow("uncloneable load"); + await expect(bridge.render(0)).rejects.toThrow("uncloneable load"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("settles frame sends immediately when postMessage throws", async () => { + vi.useFakeTimers(); + const { bridge, initialize, send } = setup(); + initialize(); + send.mockImplementation(() => { + throw new Error("frame send failed"); + }); + await expect(bridge.render(0)).rejects.toThrow("frame send failed"); + await expect(bridge.render(1)).rejects.toThrow("frame send failed"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects immediately when the runner has no content window", async () => { + vi.useFakeTimers(); + const { iframe, bridge, initialize } = setup(); + initialize(); + Object.defineProperty(iframe, "contentWindow", { value: null }); + await expect(bridge.render(0)).rejects.toThrow("window is unavailable"); + expect(vi.getTimerCount()).toBe(0); + }); + it("rejects sandbox configurations that expose a same-origin document", () => { const iframe = document.createElement("iframe"); for (const sandbox of ["", "allow-scripts allow-same-origin"]) { diff --git a/packages/core/src/runtime/filmBridge.ts b/packages/core/src/runtime/filmBridge.ts index 3da9b5805a..5a24f49d92 100644 --- a/packages/core/src/runtime/filmBridge.ts +++ b/packages/core/src/runtime/filmBridge.ts @@ -63,10 +63,23 @@ export function createFilmBridge(options: { request.reject(new Error("message" in data ? String(data.message) : "Film frame failed")); } else request.resolve(); }; + const send = (message: Record) => { + try { + const target = iframe.contentWindow; + if (!target) throw new Error("Film runner window is unavailable"); + target.postMessage(message, "*"); + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + }; const receiveStartup = (data: { type: unknown }) => { - if (data.type === PREFIX + "hello" && !loaded) { + if (data.type === PREFIX + "hello") { + if (loaded) { + fail(new Error("Film runner reloaded; recreate the bridge before seeking")); + return; + } loaded = true; - iframe.contentWindow?.postMessage({ ...load, type: PREFIX + "load" }, "*"); + send({ ...load, type: PREFIX + "load" }); } else if (data.type === PREFIX + "ready" && loaded) { clearTimeout(startupTimer); resolveReady(); @@ -97,7 +110,7 @@ export function createFilmBridge(options: { fail(new Error("Film frame did not arrive within 15 s")); }, 15_000); pending = { sequence: seq, resolve, reject, timer }; - iframe.contentWindow?.postMessage({ type: PREFIX + "frame", t: time, seq }, "*"); + send({ type: PREFIX + "frame", t: time, seq }); }); }, dispose() { From b29b84abb7af6b01fb0c522ce2b52fa60e46cf96 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 16:32:14 -0700 Subject: [PATCH 07/13] fix(runtime): align film draws with export windows --- .../core/src/runtime/frameSources.test.ts | 31 ++++++- packages/core/src/runtime/frameSources.ts | 16 ++-- packages/core/src/runtime/init.ts | 2 + .../src/services/coreRuntimeBrowser.test.ts | 89 +++++++++++++++++-- 4 files changed, 124 insertions(+), 14 deletions(-) diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts index 46cb25ac6c..e6bd4c16fc 100644 --- a/packages/core/src/runtime/frameSources.test.ts +++ b/packages/core/src/runtime/frameSources.test.ts @@ -14,11 +14,13 @@ function deferred() { } const adapters: ReturnType[] = []; -const adapter = (compositionDuration = 20) => { +const adapter = (compositionDuration = 20, exportRenderSeek = false) => { const runtime = createFrameSourceAdapter({ start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0), duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element), compositionDuration: () => compositionDuration, + canonicalFps: () => 30, + exportRenderSeek: () => exportRenderSeek, }); adapters.push(runtime); return runtime; @@ -278,6 +280,33 @@ describe("frame sources", () => { expect(render).toHaveBeenCalledTimes(1); }); + it("matches snapped export visibility at near-frame starts and ends", async () => { + const render = vi.fn(); + disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render })); + const runtime = adapter(3, true); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(0, expect.any(AbortSignal)); + runtime.seek({ time: 59 / 30 }); + await waitForSeekCompletion(); + expect(render.mock.lastCall?.[0]).toBeCloseTo(59 / 30 - 1.00001, 10); + runtime.seek({ time: 2 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(2); + }); + + it("retains unsnapped authored timing during interactive preview", async () => { + const render = vi.fn(); + disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render })); + const runtime = adapter(3); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).not.toHaveBeenCalled(); + runtime.seek({ time: 2 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(1); + }); + it("rejects invalid original ranges before registering a source", () => { const element = mount(); for (const sourceRange of [ diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts index ce18eb44bc..6b944e9d65 100644 --- a/packages/core/src/runtime/frameSources.ts +++ b/packages/core/src/runtime/frameSources.ts @@ -1,3 +1,4 @@ +import { exportClipWindow } from "../inline-scripts/parityContract"; import { sourceTimeAt } from "../speedRamp"; import { readElementRateSpec, readMediaStart } from "./playbackRate"; import { registerSeekCompletion } from "./adapters/seek-dispatch"; @@ -91,6 +92,8 @@ export function createFrameSourceAdapter(timing: { start: (element: Element) => number; duration: (element: Element) => number | null; compositionDuration: () => number; + canonicalFps: () => number; + exportRenderSeek: () => boolean; }): RuntimeDeterministicAdapter { const owned = new Set(); let readySources: RegisteredSource[] = []; @@ -128,14 +131,11 @@ export function createFrameSourceAdapter(timing: { for (const [element, source] of current()) { const start = timing.start(element); const duration = timing.duration(element); - if ( - !isClipVisibleAt( - time, - start, - start + (duration ?? Infinity), - timing.compositionDuration(), - ) - ) + const end = start + (duration ?? Infinity); + const clipWindow = timing.exportRenderSeek() + ? exportClipWindow(start, end, timing.canonicalFps()) + : { start, end }; + if (!isClipVisibleAt(time, clipWindow.start, clipWindow.end, timing.compositionDuration())) continue; const localTime = Math.max(0, time - start); const sourceTime = diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 92de290194..78ed4f1f74 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3984,6 +3984,8 @@ export function initSandboxRuntimeModular(): void { start: (element) => resolveStartForElement(element, 0), duration: (element) => resolveDurationForElement(element), compositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0), + canonicalFps: () => state.canonicalFps, + exportRenderSeek: () => Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG), }), createWaapiAdapter(), createCssAdapter({ diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 7e09786b6f..1086ee7e62 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -5,6 +5,7 @@ import { bundleToSingleHtml } from "@hyperframes/core/compiler"; import { resolve } from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer"; import type {} from "../../../core/src/runtime/window"; +import { openComposition } from "../../../sdk/src/session"; import { computeStaticFrameSet, waitForPendingSeekCompletion, @@ -1082,13 +1083,13 @@ function filmRuntimeFixture(runtime: string): string { return ` -
-
+
+
`; } @@ -1106,16 +1107,94 @@ describe("film bridge browser capture contract", () => { await browser?.close(); }); - async function openFilm(): Promise { + async function openFilm(source = html): Promise { const page = await browser.newPage(); await page.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); - await page.setContent(html); + await page.setContent(source); await page.waitForFunction( () => window.__playerReady === true && window.__renderReady === true, ); return page; } + it("renders the executable wrapper after SDK edits, save and reopen", async () => { + const composition = await openComposition(html); + composition.setTiming("hf-scene", { start: 0.2, duration: 0.6 }); + composition.setAttribute("hf-scene", "data-playback-start", "0.1"); + composition.setAttribute("hf-scene", "data-playback-rate", "2"); + const reopened = await openComposition(composition.serialize()); + const page = await openFilm(reopened.serialize()); + const reference = await browser.newPage(); + try { + await page.evaluate(() => window.__player?.renderSeek?.(0.4)); + await waitForPendingSeekCompletion(page); + expect( + Number(await page.$eval("#scene", (el) => el.getAttribute("data-rendered-source-time"))), + ).toBeCloseTo(3.5, 10); + await reference.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); + await reference.setContent(""); + expect(Buffer.from(await page.screenshot())).toEqual( + Buffer.from(await reference.screenshot()), + ); + } finally { + await page.close(); + await reference.close(); + } + }); + + it("draws the first visible export frame at a near-frame start and stops at its snapped end", async () => { + const composition = await openComposition(html); + composition.setTiming("hf-root", { duration: 3 }); + composition.setTiming("hf-scene", { start: 1.00001, duration: 1 }); + const source = composition + .serialize() + .replace( + "", + '', + ); + const page = await openFilm(source); + const reference = await browser.newPage(); + await reference.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); + try { + for (const [time, sourceTime] of [ + [1, 3], + [59 / 30, 3 + 59 / 30 - 1.00001], + ]) { + await page.evaluate((t) => window.__player?.renderSeek?.(t), time); + await waitForPendingSeekCompletion(page); + expect( + Number(await page.$eval("#scene", (el) => el.getAttribute("data-rendered-source-time"))), + ).toBeCloseTo(sourceTime!, 10); + await reference.setContent( + ``, + ); + expect(Buffer.from(await page.screenshot())).toEqual( + Buffer.from(await reference.screenshot()), + ); + } + const last = await page.$eval("#scene", (el) => el.getAttribute("data-rendered-source-time")); + await page.evaluate(() => window.__player?.renderSeek?.(2)); + await waitForPendingSeekCompletion(page); + expect(await page.$eval("#scene", (el) => el.getAttribute("data-rendered-source-time"))).toBe( + last, + ); + expect( + await page.evaluate(() => ({ + duration: window.__player?.getDuration?.(), + visibility: getComputedStyle(document.getElementById("scene")!).visibility, + background: getComputedStyle(document.body).backgroundColor, + })), + ).toEqual({ duration: 3, visibility: "hidden", background: "rgba(0, 0, 0, 0)" }); + await reference.setContent(""); + expect(Buffer.from(await page.screenshot())).toEqual( + Buffer.from(await reference.screenshot()), + ); + } finally { + await page.close(); + await reference.close(); + } + }); + it("captures the acknowledged frame for reverse, repeated and fresh-page seeks", async () => { const page = await openFilm(); const reference = await browser.newPage(); From 31c2d2c1cee296c1adadb0a4346484aa1f924da3 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 17:43:20 -0700 Subject: [PATCH 08/13] docs(core): mark frame-source hosts with data-no-timeline A composition that follows the frame-source contract registers no GSAP timeline, so lint reported missing_timeline_registry and every render waited 45 s for timeline registration (17.6 s -> 1 min 47.9 s for the 14 s imported film sample). Document the existing data-no-timeline opt-out for the host and a timeline-less root. Co-Authored-By: Claude Opus 5.5 --- docs/guides/frame-sources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/frame-sources.md b/docs/guides/frame-sources.md index b0fde5f38b..1c600a5fb0 100644 --- a/docs/guides/frame-sources.md +++ b/docs/guides/frame-sources.md @@ -14,7 +14,7 @@ const unregister = window.__hyperframes.registerFrameSource({ }); ``` -The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. +The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. Add `data-no-timeline` to the host, and to a root that registers no timeline; otherwise `hyperframes lint` reports `missing_timeline_registry` and each render waits 45 seconds for timeline registration. The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. From 42414f2b15e649e243a69eae69f89cc0a3360134 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 17:43:35 -0700 Subject: [PATCH 09/13] docs(core): require data-no-timeline on imported film hosts Imported film wrappers have no GSAP timeline. Without data-no-timeline on the scene hosts and root, lint fails with missing_timeline_registry and each render waits 45 s for timeline registration. Co-Authored-By: Claude Opus 5.5 --- docs/guides/imported-film-html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/imported-film-html.md b/docs/guides/imported-film-html.md index 5b7513c679..575256612d 100644 --- a/docs/guides/imported-film-html.md +++ b/docs/guides/imported-film-html.md @@ -5,7 +5,7 @@ description: "Connect preserved film-runner HTML to HyperFrames capture." The runtime supports the `appifact-film:` message protocol used by current Claude Motion HTML exports. Keep the original `film-runner` HTML and scene modules intact. The MCP importer packages them with a HyperFrames wrapper; HyperFrames controls time and captures the resulting pixels using its existing renderer. -Create and mount an iframe with `sandbox="allow-scripts"`, explicit picture dimensions, and no player controls. Then connect it before assigning any runner HTML yourself: +Create and mount an iframe with `sandbox="allow-scripts"`, explicit picture dimensions, and no player controls. Give its timed host, and a root without a GSAP timeline, `data-no-timeline` as described in Frame sources. Then connect it before assigning any runner HTML yourself: ```javascript const bridge = window.__hyperframes.createFilmBridge({ From 29a5c02b4566334ab08b195abb0fdd0473433ff1 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 18:28:36 -0700 Subject: [PATCH 10/13] fix(runtime): avoid transport redraws after render seeks --- packages/core/src/runtime/init.ts | 3 + .../src/services/coreRuntimeBrowser.test.ts | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 137fa3106d..37268c3a7d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3902,6 +3902,9 @@ export function initSandboxRuntimeModular(): void { activateChildren: true, suppressEvents: options?.suppressEvents, }); + // The explicit seek owns this paused frame; the transport must not redraw after capture waits. + lastTransportSeekTime = state.currentTime; + lastTransportSeekTimeline = state.capturedTimeline; runAdapters("pause", 0, pageAnimations); syncMediaForCurrentState(); colorGrading.redraw(); diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 1ec2ba200b..7eb7cc15a4 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -92,6 +92,61 @@ describe("core runtime browser contract", () => { } }); + it.each(["preview", "export"])( + "does not redraw async sources on transport ticks after an explicit render seek (%s)", + async (mode) => { + const sourcePage = await browser.newPage(); + try { + await sourcePage.setContent( + `
`, + ); + if (mode === "export") + await sourcePage.addScriptTag({ + content: "window.__HF_EXPORT_RENDER_SEEK_CONFIG={fps:30};", + }); + await sourcePage.addScriptTag({ content: readFileSync(RUNTIME_PATH, "utf8") }); + await sourcePage.waitForFunction(() => window.__playerReady && window.__renderReady); + await sourcePage.evaluate(() => { + const element = document.getElementById("source")!; + window.__hyperframes!.registerFrameSource({ + element, + render: async (time) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + element.setAttribute( + "data-draws", + (element.getAttribute("data-draws") ?? "") + time + ",", + ); + }, + }); + }); + for (const [index, time] of [0.5, 0.5, 0.2].entries()) { + await sourcePage.evaluate((t) => window.__player!.renderSeek!(t), time); + await waitForPendingSeekCompletion(sourcePage); + // Let the transport run after the capture barrier has already settled. + await sourcePage.evaluate(async () => { + for (let frame = 0; frame < 4; frame++) + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + await window.__hfWaitForSeekCompletion!(); + }); + expect(await sourcePage.$eval("#source", (el) => el.getAttribute("data-draws"))).toBe( + [0.5, 0.5, 0.2].slice(0, index + 1).join(",") + ",", + ); + } + await sourcePage.evaluate(() => window.__player!.play()); + await sourcePage.waitForFunction( + () => + (document.getElementById("source")!.getAttribute("data-draws") ?? "") + .split(",") + .filter(Boolean).length > 3, + ); + await sourcePage.evaluate(() => window.__player!.pause()); + await waitForPendingSeekCompletion(sourcePage); + } finally { + await sourcePage.close(); + } + }, + ); + it("initializes the public player contract and seeks the CSS adapter", async () => { const result = await page.evaluate(() => { const runtimeWindow = window as unknown as { From 1a134218bf48a476a21024c433c60e862bb2c26b Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 21:28:04 -0700 Subject: [PATCH 11/13] test: set explicit film fixture canvas background --- packages/producer/src/services/coreRuntimeBrowser.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 676ec7de07..560c90a4f3 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -1136,7 +1136,7 @@ parent.postMessage({type:"appifact-film:hello"}, "*"); function filmRuntimeFixture(runtime: string): string { const runnerLiteral = JSON.stringify(FILM_RUNNER_FIXTURE).replaceAll("<", "\\u003c"); return ` - +
@@ -1240,7 +1240,7 @@ describe("film bridge browser capture contract", () => { background: getComputedStyle(document.body).backgroundColor, })), ).toEqual({ duration: 3, visibility: "hidden", background: "rgba(0, 0, 0, 0)" }); - await reference.setContent(""); + await reference.setContent(""); expect(Buffer.from(await page.screenshot())).toEqual( Buffer.from(await reference.screenshot()), ); From 3b471b866e1fbc7ce2a1910a610d8d33ee27a4fd Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 21:36:12 -0700 Subject: [PATCH 12/13] test: diagnose film capture timeout phase --- .../src/services/coreRuntimeBrowser.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 560c90a4f3..5065839a26 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, onTestFailed } from "vitest"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { bundleToSingleHtml } from "@hyperframes/core/compiler"; @@ -1251,7 +1251,10 @@ describe("film bridge browser capture contract", () => { }); it("captures the acknowledged frame for reverse, repeated and fresh-page seeks", async () => { + let phase = "opening film"; + onTestFailed(() => console.error(`Film capture failed during: ${phase}`)); const page = await openFilm(); + phase = "opening reference"; const reference = await browser.newPage(); await reference.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); const errors: string[] = []; @@ -1259,40 +1262,56 @@ describe("film bridge browser capture contract", () => { const captures = new Map(); try { for (const time of [0, 0.5, 0.2, 0.5, 0.9, 1, 0]) { + phase = `seeking ${time}`; await page.evaluate((t) => window.__player?.renderSeek?.(t), time); + phase = `waiting for frame ${time}`; await waitForPendingSeekCompletion(page); + phase = `capturing frame ${time}`; const actual = await page.screenshot(); const sourceTime = 3 + Math.min(time, 1 - 1 / 30); + phase = `setting reference ${time}`; await reference.setContent( ``, ); + phase = `capturing reference ${time}`; expect(Buffer.from(actual)).toEqual(Buffer.from(await reference.screenshot())); const previous = captures.get(time); if (previous) expect(Buffer.from(actual)).toEqual(Buffer.from(previous)); captures.set(time, actual); } + phase = "opening fresh film"; const fresh = await openFilm(); try { + phase = "seeking fresh film"; await fresh.evaluate(() => window.__player?.renderSeek?.(0.5)); + phase = "waiting for fresh frame"; await waitForPendingSeekCompletion(fresh); + phase = "capturing fresh frame"; expect(Buffer.from(await fresh.screenshot())).toEqual(Buffer.from(captures.get(0.5)!)); } finally { + phase = "closing fresh film"; await fresh.close(); } + phase = "seeking edited film"; await page.evaluate(() => { const scene = document.getElementById("scene")!; scene.setAttribute("data-playback-start", "0.25"); scene.setAttribute("data-playback-rate", "2"); window.__player?.renderSeek?.(0.2); }); + phase = "waiting for edited frame"; await waitForPendingSeekCompletion(page); + phase = "setting edited reference"; await reference.setContent(""); + phase = "capturing edited frame"; expect(Buffer.from(await page.screenshot())).toEqual( Buffer.from(await reference.screenshot()), ); expect(errors).toEqual([]); } finally { + phase = "closing film"; await page.close(); + phase = "closing reference"; await reference.close(); } }, 30_000); From 7a3d335876d0bf6b27b9a464824be37305d7704d Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 21:47:06 -0700 Subject: [PATCH 13/13] test: foreground film pages before screenshot capture --- .../src/services/coreRuntimeBrowser.test.ts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 5065839a26..c052c3ac77 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -1172,6 +1172,12 @@ describe("film bridge browser capture contract", () => { return page; } + async function capture(page: Page): Promise { + // Linux headless Chrome can stall capture when the reference tab is foreground. + await page.bringToFront(); + return Buffer.from(await page.screenshot()); + } + it("renders the executable wrapper after SDK edits, save and reopen", async () => { const composition = await openComposition(html); composition.setTiming("hf-scene", { start: 0.2, duration: 0.6 }); @@ -1188,9 +1194,7 @@ describe("film bridge browser capture contract", () => { ).toBeCloseTo(3.5, 10); await reference.setViewport({ width: 320, height: 180, deviceScaleFactor: 1 }); await reference.setContent(""); - expect(Buffer.from(await page.screenshot())).toEqual( - Buffer.from(await reference.screenshot()), - ); + expect(await capture(page)).toEqual(await capture(reference)); } finally { await page.close(); await reference.close(); @@ -1223,9 +1227,7 @@ describe("film bridge browser capture contract", () => { await reference.setContent( ``, ); - expect(Buffer.from(await page.screenshot())).toEqual( - Buffer.from(await reference.screenshot()), - ); + expect(await capture(page)).toEqual(await capture(reference)); } const last = await page.$eval("#scene", (el) => el.getAttribute("data-rendered-source-time")); await page.evaluate(() => window.__player?.renderSeek?.(2)); @@ -1241,9 +1243,7 @@ describe("film bridge browser capture contract", () => { })), ).toEqual({ duration: 3, visibility: "hidden", background: "rgba(0, 0, 0, 0)" }); await reference.setContent(""); - expect(Buffer.from(await page.screenshot())).toEqual( - Buffer.from(await reference.screenshot()), - ); + expect(await capture(page)).toEqual(await capture(reference)); } finally { await page.close(); await reference.close(); @@ -1267,14 +1267,14 @@ describe("film bridge browser capture contract", () => { phase = `waiting for frame ${time}`; await waitForPendingSeekCompletion(page); phase = `capturing frame ${time}`; - const actual = await page.screenshot(); + const actual = await capture(page); const sourceTime = 3 + Math.min(time, 1 - 1 / 30); phase = `setting reference ${time}`; await reference.setContent( ``, ); phase = `capturing reference ${time}`; - expect(Buffer.from(actual)).toEqual(Buffer.from(await reference.screenshot())); + expect(Buffer.from(actual)).toEqual(await capture(reference)); const previous = captures.get(time); if (previous) expect(Buffer.from(actual)).toEqual(Buffer.from(previous)); captures.set(time, actual); @@ -1287,7 +1287,7 @@ describe("film bridge browser capture contract", () => { phase = "waiting for fresh frame"; await waitForPendingSeekCompletion(fresh); phase = "capturing fresh frame"; - expect(Buffer.from(await fresh.screenshot())).toEqual(Buffer.from(captures.get(0.5)!)); + expect(await capture(fresh)).toEqual(Buffer.from(captures.get(0.5)!)); } finally { phase = "closing fresh film"; await fresh.close(); @@ -1304,9 +1304,7 @@ describe("film bridge browser capture contract", () => { phase = "setting edited reference"; await reference.setContent(""); phase = "capturing edited frame"; - expect(Buffer.from(await page.screenshot())).toEqual( - Buffer.from(await reference.screenshot()), - ); + expect(await capture(page)).toEqual(await capture(reference)); expect(errors).toEqual([]); } finally { phase = "closing film";