From 608dc220aea988ad5dc114e2762eeff5eb746e16 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 14:21:29 -0700 Subject: [PATCH 1/4] feat(core): support deterministic async frame sources --- docs/docs.json | 1 + docs/guides/frame-sources.md | 27 +++ packages/core/src/runtime/entry.test.ts | 31 +++ packages/core/src/runtime/entry.ts | 3 + .../core/src/runtime/frameSources.test.ts | 218 ++++++++++++++++++ packages/core/src/runtime/frameSources.ts | 131 +++++++++++ packages/core/src/runtime/init.ts | 5 + packages/core/src/runtime/window.d.ts | 1 + 8 files changed, 417 insertions(+) create mode 100644 docs/guides/frame-sources.md create mode 100644 packages/core/src/runtime/frameSources.test.ts create mode 100644 packages/core/src/runtime/frameSources.ts diff --git a/docs/docs.json b/docs/docs.json index 92cecc2104..7fbe3b2cd7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -913,6 +913,7 @@ "concepts/variables", "concepts/data-attributes", "guides/gsap-animation", + "guides/frame-sources", "concepts/frame-adapters", "concepts/determinism", "guides/html-in-canvas", diff --git a/docs/guides/frame-sources.md b/docs/guides/frame-sources.md new file mode 100644 index 0000000000..b0fde5f38b --- /dev/null +++ b/docs/guides/frame-sources.md @@ -0,0 +1,27 @@ +--- +title: "Frame sources" +description: "Run deterministic scene code on the HyperFrames timeline." +--- + +HyperFrames can drive scene code that exposes a function of time without converting it to GSAP. Register that function against a timed host using `window.__hyperframes.registerFrameSource` before runtime initialization completes. + +```javascript +const unregister = window.__hyperframes.registerFrameSource({ + element: document.getElementById("scene"), + ready: initializeScene(), + render: (sourceTime, signal) => drawScene(sourceTime, signal), + dispose: () => releaseScene(), +}); +``` + +The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. + +The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. + +`ready` holds initialization and export capture. Return a promise from `render` when drawing is asynchronous: capture waits for it before taking pixels. Repeated and out-of-order times must produce the same frame. HyperFrames owns playback; do not start a second clock. + +Draws are serialized per source. Rapid preview seeks coalesce while a draw is in flight; sequential export seeks draw every requested frame. Setup and draw failures are delivered to the capture completion barrier. A later successful seek can recover from a frame failure. + +Call `unregister()` when replacing a source. Removal of its host or runtime teardown also unregisters it. The callback's `AbortSignal` is aborted, queued work is cancelled, capture waiters are released, and `dispose` runs once. Stop any underlying work in response to the signal or in `dispose`; cancellation cannot forcibly interrupt synchronous JavaScript. + +Registration does not expose arbitrary scene code as editable keyframes. Manual element edits need stable identities and persistent parameters or overrides. Imported iframe sources also need screenshot capture and an explicit message bridge; this API does not grant access to sandboxed DOM. diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index 9398089799..63b7afd2dc 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -96,6 +96,37 @@ const neverDecodes = (clip: HTMLElement) => { describe("runtime entry", () => { afterEach(resetRuntimeGlobals); + it("drives an async frame source without a GSAP timeline and waits before capture", async () => { + const root = mountRoot(); + root.setAttribute("data-duration", "10"); + root.appendChild(document.createElement("div")); + window.__timelines = {}; + Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" }); + await evaluateRuntime(); + let finish!: () => void; + const first = new Promise((resolve) => { + finish = resolve; + }); + const render = vi + .fn() + .mockImplementationOnce(() => first) + .mockResolvedValue(undefined); + window.__hyperframes!.registerFrameSource({ element: root, render }); + delete (document as { readyState?: unknown }).readyState; + document.dispatchEvent(new Event("DOMContentLoaded")); + await vi.waitFor(() => expect(render).toHaveBeenCalled()); + window.__player!.seek(1.2); + let captured = false; + const capture = window.__hfWaitForSeekCompletion!().then(() => { + captured = true; + }); + await Promise.resolve(); + expect(captured).toBe(false); + finish(); + await capture; + expect(render).toHaveBeenLastCalledWith(1.2, expect.any(AbortSignal)); + }); + it("paints no timed clip, from script evaluation until the first visibility pass decides it", async () => { servePreview(); const root = mountRoot(); diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 666e872a93..6903414623 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -14,6 +14,7 @@ import { getVariables } from "./getVariables"; import { clearRuntimeData, registerRuntimeDataHandler, setRuntimeData } from "./runtimeData"; import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; +import { registerFrameSource } from "./frameSources"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -25,6 +26,7 @@ type HyperframeWindow = Window & { registerRuntimeDataHandler: typeof registerRuntimeDataHandler; setRuntimeData: typeof setRuntimeData; clearRuntimeData: typeof clearRuntimeData; + registerFrameSource: typeof registerFrameSource; }; }; @@ -53,6 +55,7 @@ deferMediaUntilDue(); registerRuntimeDataHandler, setRuntimeData, clearRuntimeData, + registerFrameSource, }; function bootstrapHyperframeRuntime(): void { diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts new file mode 100644 index 0000000000..7e9c2ede79 --- /dev/null +++ b/packages/core/src/runtime/frameSources.test.ts @@ -0,0 +1,218 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createFrameSourceAdapter, registerFrameSource } from "./frameSources"; +import { resetSeekDispatchState, waitForSeekCompletion } from "./adapters/seek-dispatch"; +import { createRuntimeStartTimeResolver } from "./startResolver"; + +function deferred() { + let resolve!: () => void; + let reject!: (reason: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +const adapters: ReturnType[] = []; +const adapter = () => { + const runtime = createFrameSourceAdapter({ + start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0), + duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element), + }); + adapters.push(runtime); + return runtime; +}; +const disposers: Array<() => void> = []; +function mount(start = "0", duration = "10") { + const element = document.createElement("section"); + element.setAttribute("data-start", start); + element.setAttribute("data-duration", duration); + document.body.append(element); + return element; +} +beforeEach(() => resetSeekDispatchState()); +afterEach(() => { + for (const runtime of adapters.splice(0)) runtime.revert?.(); + for (const dispose of disposers.splice(0)) dispose(); + document.body.innerHTML = ""; + resetSeekDispatchState(); +}); + +describe("frame sources", () => { + it("holds capture through setup and asynchronous drawing", async () => { + const setup = deferred(); + const frame = deferred(); + const render = vi.fn(() => frame.promise); + disposers.push(registerFrameSource({ element: mount(), ready: setup.promise, render })); + const runtime = adapter(); + runtime.seek({ time: 2 }); + let captured = false; + const capture = waitForSeekCompletion().then(() => { + captured = true; + }); + await Promise.resolve(); + expect(render).not.toHaveBeenCalled(); + expect(captured).toBe(false); + setup.resolve(); + await vi.waitFor(() => expect(render).toHaveBeenCalledWith(2, expect.any(AbortSignal))); + expect(captured).toBe(false); + frame.resolve(); + await capture; + expect(captured).toBe(true); + }); + + it("serializes drawing and coalesces queued scrubs to the latest time", async () => { + const frame = deferred(); + const render = vi.fn().mockImplementationOnce(() => frame.promise); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + runtime.seek({ time: 1 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + runtime.seek({ time: 2 }); + runtime.seek({ time: 3 }); + expect(render).toHaveBeenCalledTimes(1); + frame.resolve(); + await waitForSeekCompletion(); + expect(render.mock.calls.map(([time]) => time)).toEqual([1, 3]); + }); + + it("renders every sequential export request, including repeated and reverse time", async () => { + const render = vi.fn(); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + for (const time of [7, 2, 2, 0, 9]) { + runtime.seek({ time }); + await waitForSeekCompletion(); + } + expect(render.mock.calls.map(([time]) => time)).toEqual([7, 2, 2, 0, 9]); + }); + + it("rereads clip timing, trim inpoints and rates after edits", async () => { + const element = mount("3", "4"); + element.setAttribute("data-playback-start", "1"); + element.setAttribute("data-playback-rate", "2"); + const render = vi.fn(); + disposers.push(registerFrameSource({ element, render })); + const runtime = adapter(); + runtime.seek({ time: 4 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(3, expect.any(AbortSignal)); + element.setAttribute("data-start", "0"); + element.setAttribute("data-playback-start", "2"); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenLastCalledWith(4, expect.any(AbortSignal)); + runtime.seek({ time: 8 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(2); + }); + + it("surfaces failed setup even when its clip has not started", async () => { + const render = vi.fn(); + disposers.push( + registerFrameSource({ + element: mount("5"), + ready: Promise.reject(new Error("font failed")), + render, + }), + ); + const runtime = adapter(); + runtime.discover(); + await expect(waitForSeekCompletion()).rejects.toThrow("font failed"); + expect(render).not.toHaveBeenCalled(); + }); + + it("surfaces draw failure and allows the next seek to recover", async () => { + const render = vi + .fn() + .mockRejectedValueOnce(new Error("draw failed")) + .mockResolvedValue(undefined); + disposers.push(registerFrameSource({ element: mount(), render })); + const runtime = adapter(); + runtime.seek({ time: 2 }); + await expect(waitForSeekCompletion()).rejects.toThrow("draw failed"); + runtime.seek({ time: 3 }); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(render).toHaveBeenLastCalledWith(3, expect.any(AbortSignal)); + }); + + it("unregisters stalled setup and allows replacing the same host", async () => { + const element = mount(); + const cleanup = vi.fn(); + const unregister = registerFrameSource({ + element, + ready: deferred().promise, + render: vi.fn(), + dispose: cleanup, + }); + const runtime = adapter(); + runtime.seek({ time: 0 }); + expect(() => registerFrameSource({ element, render: vi.fn() })).toThrow("already has"); + unregister(); + unregister(); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(cleanup).toHaveBeenCalledTimes(1); + const render = vi.fn(); + disposers.push(registerFrameSource({ element, render })); + runtime.seek({ time: 1 }); + await waitForSeekCompletion(); + expect(render).toHaveBeenCalledTimes(1); + }); + + it("removal aborts an active draw and releases capture", async () => { + const element = mount(); + let signal: AbortSignal | undefined; + const render = vi.fn((_time: number, abort: AbortSignal) => { + signal = abort; + return deferred().promise; + }); + const cleanup = vi.fn(); + disposers.push(registerFrameSource({ element, render, dispose: cleanup })); + const runtime = adapter(); + runtime.seek({ time: 0 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + element.remove(); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + expect(signal?.aborted).toBe(true); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("keeps readiness promises stable and releases sources on teardown", async () => { + const ready = deferred(); + const cleanup = vi.fn(); + disposers.push( + registerFrameSource({ + element: mount(), + ready: ready.promise, + render: vi.fn(), + dispose: cleanup, + }), + ); + const runtime = adapter(); + const first = runtime.getReadyPromise?.(); + expect(runtime.getReadyPromise?.()).toBe(first); + runtime.revert?.(); + await expect(first).resolves.toBeDefined(); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(runtime.getReadyPromise?.()).toBeNull(); + }); + + it("draws overlapping independent sources without serializing them together", async () => { + const a = deferred(); + const b = deferred(); + const renderA = vi.fn(() => a.promise); + const renderB = vi.fn(() => b.promise); + disposers.push( + registerFrameSource({ element: mount(), render: renderA }), + registerFrameSource({ element: mount(), render: renderB }), + ); + adapter().seek({ time: 1 }); + await vi.waitFor(() => { + expect(renderA).toHaveBeenCalled(); + expect(renderB).toHaveBeenCalled(); + }); + a.resolve(); + b.resolve(); + await waitForSeekCompletion(); + }); +}); diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts new file mode 100644 index 0000000000..a8dc2140ef --- /dev/null +++ b/packages/core/src/runtime/frameSources.ts @@ -0,0 +1,131 @@ +import { sourceTimeAt } from "../speedRamp"; +import { readElementRateSpec, readMediaStart } from "./playbackRate"; +import { registerSeekCompletion } from "./adapters/seek-dispatch"; +import type { RuntimeDeterministicAdapter } from "./types"; + +export interface FrameSource { + element: Element; + ready?: PromiseLike; + render: (sourceTime: number, signal: AbortSignal) => void | PromiseLike; + dispose?: () => void; +} + +interface RegisteredSource { + element: Element; + ready: Promise; + seek: (time: number) => Promise; + dispose: () => void; +} + +const sources = new Map(); + +/** Bind a frame source to a timed host. Unregister before replacing its source. */ +export function registerFrameSource(source: FrameSource): () => void { + if (sources.has(source.element)) throw new Error("This element already has a frame source"); + const controller = new AbortController(); + const cancelled = new Promise((resolve) => { + controller.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + const ready = Promise.race([Promise.resolve(source.ready), cancelled]); + void ready.catch(() => {}); + let pending: number | null = null; + let work: Promise | null = null; + const drain = async () => { + try { + await ready; + while (!controller.signal.aborted && pending !== null) { + const time = pending; + pending = null; + await Promise.race([source.render(time, controller.signal), cancelled]); + } + } finally { + pending = null; + work = null; + } + }; + const registered: RegisteredSource = { + element: source.element, + ready, + seek(time) { + pending = time; + work ??= drain(); + return work; + }, + dispose() { + if (controller.signal.aborted) return; + controller.abort(); + pending = null; + sources.delete(source.element); + source.dispose?.(); + }, + }; + sources.set(source.element, registered); + return registered.dispose; +} + +export function createFrameSourceAdapter(timing: { + start: (element: Element) => number; + duration: (element: Element) => number | null; +}): RuntimeDeterministicAdapter { + const owned = new Set(); + let readySources: RegisteredSource[] = []; + let readiness: Promise | null = null; + const current = () => { + for (const source of owned) { + if (sources.get(source.element) !== source) owned.delete(source); + } + const connected: Array<[Element, RegisteredSource]> = []; + for (const [element, source] of sources) { + if (!element.isConnected) { + source.dispose(); + continue; + } + if (!owned.has(source)) { + owned.add(source); + // Readiness trackers settle rejected setup; capture must also receive that failure. + registerSeekCompletion(source.ready); + } + connected.push([element, source]); + } + return connected; + }; + const removals = new MutationObserver(() => { + current(); + }); + removals.observe(document, { childList: true, subtree: true }); + return { + name: "frame-source", + discover: () => { + current(); + }, + pause: () => {}, + seek: ({ time }) => { + for (const [element, source] of current()) { + const start = timing.start(element); + const duration = timing.duration(element); + if (time < start || (duration !== null && time > start + duration)) continue; + const localTime = Math.max(0, time - start); + const sourceTime = + readMediaStart(element) + sourceTimeAt(readElementRateSpec(element), localTime); + registerSeekCompletion(source.seek(sourceTime)); + } + }, + getReadyPromise: () => { + const next = current().map(([, source]) => source); + if (next.length === 0) return null; + if ( + next.length !== readySources.length || + next.some((source, i) => source !== readySources[i]) + ) { + readySources = next; + readiness = Promise.all(next.map((source) => source.ready)); + } + return readiness; + }, + revert: () => { + removals.disconnect(); + for (const source of owned) source.dispose(); + owned.clear(); + }, + }; +} diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index fa9731725a..137fa3106d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -20,6 +20,7 @@ import { createGoogleMapsAdapter } from "./adapters/google-maps"; import { createMaplibreAdapter } from "./adapters/maplibre"; import { createD3Adapter } from "./adapters/d3"; import { createTypegpuAdapter } from "./adapters/typegpu"; +import { createFrameSourceAdapter } from "./frameSources"; import { patchVideoTextureCompat, patchWebGLVideoTextureCompat, @@ -3979,6 +3980,10 @@ export function initSandboxRuntimeModular(): void { }); state.deterministicAdapters = [ + createFrameSourceAdapter({ + start: (element) => resolveStartForElement(element, 0), + duration: (element) => resolveDurationForElement(element), + }), createWaapiAdapter(), createCssAdapter({ resolveStartSeconds: (element) => resolveStartForElement(element, 0), diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 76a30e0e85..4a802d4208 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -38,6 +38,7 @@ declare global { __timelines: Record; __player?: PlayerAPI; __hyperframes?: { + registerFrameSource: typeof import("./frameSources").registerFrameSource; /** A path the calling composition wrote relative to its own file, as a URL the page can load. */ assetUrl?: (path: string) => string; registerRuntimeDataHandler?: ( From 37b9f4c302fb977de9dc38c60ff05f627aaeb934 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 16:26:35 -0700 Subject: [PATCH 2/4] fix(runtime): retain queued draws and disable unsafe dedup --- packages/core/src/runtime/entry.ts | 3 +- .../core/src/runtime/frameSources.test.ts | 24 ++++++++- packages/core/src/runtime/frameSources.ts | 10 +++- packages/core/src/runtime/window.d.ts | 1 + packages/engine/src/services/frameCapture.ts | 12 +++++ .../src/services/coreRuntimeBrowser.test.ts | 53 +++++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 6903414623..23e43ac7e6 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -14,7 +14,7 @@ import { getVariables } from "./getVariables"; import { clearRuntimeData, registerRuntimeDataHandler, setRuntimeData } from "./runtimeData"; import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; -import { registerFrameSource } from "./frameSources"; +import { hasFrameSources, registerFrameSource } from "./frameSources"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -40,6 +40,7 @@ type HyperframeWindow = Window & { installAuthoredOpacityCapture(); installAuthoredMediaCapture(); installFlatGsapTransforms(); +window.__hfHasFrameSources = hasFrameSources; hideTimedClipsUntilFirstPass(); deferMediaUntilDue(); diff --git a/packages/core/src/runtime/frameSources.test.ts b/packages/core/src/runtime/frameSources.test.ts index 7e9c2ede79..36a27145b3 100644 --- a/packages/core/src/runtime/frameSources.test.ts +++ b/packages/core/src/runtime/frameSources.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createFrameSourceAdapter, registerFrameSource } from "./frameSources"; +import { createFrameSourceAdapter, hasFrameSources, registerFrameSource } from "./frameSources"; import { resetSeekDispatchState, waitForSeekCompletion } from "./adapters/seek-dispatch"; import { createRuntimeStartTimeResolver } from "./startResolver"; @@ -215,4 +215,26 @@ describe("frame sources", () => { b.resolve(); await waitForSeekCompletion(); }); + it("drains a queued seek after an earlier draw fails and retains the failure for capture", async () => { + const first = deferred(); + const render = vi + .fn() + .mockImplementationOnce(() => first.promise) + .mockResolvedValue(undefined); + const unregister = registerFrameSource({ element: mount(), render }); + disposers.push(unregister); + expect(hasFrameSources()).toBe(true); + const runtime = adapter(); + runtime.seek({ time: 1 }); + await vi.waitFor(() => expect(render).toHaveBeenCalledTimes(1)); + runtime.seek({ time: 2 }); + const capture = expect(waitForSeekCompletion()).rejects.toThrow("first draw failed"); + first.reject(new Error("first draw failed")); + await capture; + expect(render.mock.calls.map(([time]) => time)).toEqual([1, 2]); + runtime.seek({ time: 3 }); + await expect(waitForSeekCompletion()).resolves.toBeUndefined(); + unregister(); + expect(hasFrameSources()).toBe(false); + }); }); diff --git a/packages/core/src/runtime/frameSources.ts b/packages/core/src/runtime/frameSources.ts index a8dc2140ef..67f9414c69 100644 --- a/packages/core/src/runtime/frameSources.ts +++ b/packages/core/src/runtime/frameSources.ts @@ -19,6 +19,8 @@ interface RegisteredSource { const sources = new Map(); +export const hasFrameSources = (): boolean => sources.size > 0; + /** Bind a frame source to a timed host. Unregister before replacing its source. */ export function registerFrameSource(source: FrameSource): () => void { if (sources.has(source.element)) throw new Error("This element already has a frame source"); @@ -31,13 +33,19 @@ export function registerFrameSource(source: FrameSource): () => void { let pending: number | null = null; let work: Promise | null = null; const drain = async () => { + let failure: { reason: unknown } | undefined; try { await ready; while (!controller.signal.aborted && pending !== null) { const time = pending; pending = null; - await Promise.race([source.render(time, controller.signal), cancelled]); + try { + await Promise.race([source.render(time, controller.signal), cancelled]); + } catch (reason) { + failure ??= { reason }; + } } + if (failure) throw failure.reason; } finally { pending = null; work = null; diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 4a802d4208..4b0c3b465f 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -35,6 +35,7 @@ type ThreeLike = { declare global { interface Window { + __hfHasFrameSources?: () => boolean; __timelines: Record; __player?: PlayerAPI; __hyperframes?: { diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 42a40db396..f29c87f227 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -3032,12 +3032,15 @@ export async function computeStaticFrameSet( const w = window as unknown as { __timelines?: Record; __hf?: { duration?: number }; + __hfHasFrameSources?: () => boolean; }; for (const tl of Object.values(w.__timelines || {})) { if (tl && typeof tl.getChildren === "function") walk(tl, 0); } const hasVideo = !!document.querySelector("video"); const hasCanvas = !!document.querySelector("canvas"); + const hasFrameSources = w.__hfHasFrameSources?.() ?? false; + const hasIframe = !!document.querySelector("iframe"); // A non-numeric data-start (reference expression like "intro+0.5") can't be turned // into a clip-cut boundary by computeClipBoundaryFrames' parseFloat, so the cut goes // unprotected and could be deduped into the previous scene. Disqualify the comp. @@ -3067,6 +3070,8 @@ export async function computeStaticFrameSet( duration: w.__hf?.duration ?? 0, hasVideo, hasCanvas, + hasFrameSources, + hasIframe, hasNonGsapAnim, hasUnresolvableClipStart, hasTimelineCall, @@ -3079,6 +3084,8 @@ export async function computeStaticFrameSet( duration, hasVideo, hasCanvas, + hasFrameSources, + hasIframe, hasNonGsapAnim, hasUnresolvableClipStart, hasTimelineCall, @@ -3088,6 +3095,8 @@ export async function computeStaticFrameSet( duration: number; hasVideo: boolean; hasCanvas: boolean; + hasFrameSources: boolean; + hasIframe: boolean; hasNonGsapAnim: boolean; hasUnresolvableClipStart: boolean; hasTimelineCall: boolean; @@ -3121,6 +3130,9 @@ export async function computeStaticFrameSet( if (!(duration > 0)) reasons.push("unknown/zero duration"); if (hasVideo) reasons.push("video"); if (hasCanvas) reasons.push("canvas/webgl"); + // GSAP intervals cannot predict frame-source or opaque iframe draws. + if (hasFrameSources) reasons.push("registered frame source"); + if (hasIframe) reasons.push("iframe"); if (tweenCount === 0) reasons.push("no GSAP tweens (non-GSAP animation)"); if (hasNonGsapAnim) reasons.push("running CSS/WAAPI animation"); // tl.call() side effects are not seek-idempotent (see hasTimelineCall detection diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 30e5bde580..1ec2ba200b 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -4,6 +4,11 @@ import { tmpdir } from "node:os"; import { bundleToSingleHtml } from "@hyperframes/core/compiler"; import { resolve } from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer"; +import type {} from "../../../core/src/runtime/window"; +import { + computeStaticFrameSet, + waitForPendingSeekCompletion, +} from "../../../engine/src/services/frameCapture"; const RUNTIME_PATH = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js"); const PNG_1PX = @@ -39,6 +44,54 @@ describe("core runtime browser contract", () => { await browser?.close(); }); + it("disables static dedup for an async frame source even alongside a GSAP tween", async () => { + const sourcePage = await browser.newPage(); + try { + await sourcePage.setContent( + `
`, + ); + await sourcePage.addScriptTag({ + path: resolve(import.meta.dirname, "../../../core/node_modules/gsap/dist/gsap.min.js"), + }); + await sourcePage.addScriptTag({ + content: + 'window.__timelines = {root:gsap.timeline({paused:true}).to("#probe",{opacity:0.5,duration:0.1})};', + }); + await sourcePage.addScriptTag({ content: readFileSync(RUNTIME_PATH, "utf8") }); + await sourcePage.waitForFunction(() => window.__playerReady && window.__renderReady); + await sourcePage.evaluate(() => { + window.__hf = { duration: window.__player!.getDuration!() }; + }); + const baseline = await computeStaticFrameSet(sourcePage, 30); + expect(baseline.reason).toBe("eligible"); + expect(baseline.eligible).toBe(true); + await sourcePage.evaluate(() => { + const element = document.getElementById("source")!; + window.__hyperframes!.registerFrameSource({ + element, + render: async (time) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + element.setAttribute("data-rendered-time", String(time)); + }, + }); + window.__player!.renderSeek!(0.5); + }); + await waitForPendingSeekCompletion(sourcePage); + expect(await sourcePage.$eval("#source", (el) => el.getAttribute("data-rendered-time"))).toBe( + "0.5", + ); + const analysis = await computeStaticFrameSet(sourcePage, 30); + expect(analysis.tweenCount).toBeGreaterThan(0); + expect(analysis.eligible).toBe(false); + expect(analysis.staticFrameSet.size).toBe(0); + expect(analysis.reason).toContain("registered frame source"); + await sourcePage.evaluate(() => document.body.appendChild(document.createElement("iframe"))); + expect((await computeStaticFrameSet(sourcePage, 30)).reason).toContain("iframe"); + } finally { + await sourcePage.close(); + } + }); + it("initializes the public player contract and seeks the CSS adapter", async () => { const result = await page.evaluate(() => { const runtimeWindow = window as unknown as { From 31c2d2c1cee296c1adadb0a4346484aa1f924da3 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 17:43:20 -0700 Subject: [PATCH 3/4] docs(core): mark frame-source hosts with data-no-timeline A composition that follows the frame-source contract registers no GSAP timeline, so lint reported missing_timeline_registry and every render waited 45 s for timeline registration (17.6 s -> 1 min 47.9 s for the 14 s imported film sample). Document the existing data-no-timeline opt-out for the host and a timeline-less root. Co-Authored-By: Claude Opus 5.5 --- docs/guides/frame-sources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/frame-sources.md b/docs/guides/frame-sources.md index b0fde5f38b..1c600a5fb0 100644 --- a/docs/guides/frame-sources.md +++ b/docs/guides/frame-sources.md @@ -14,7 +14,7 @@ const unregister = window.__hyperframes.registerFrameSource({ }); ``` -The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. +The host uses the usual composition attributes: `data-composition-id`, `data-start`, `data-duration`, and `data-track-index`. Declare the root's dimensions, FPS, and duration. A frame source does not need a dummy GSAP timeline. Add `data-no-timeline` to the host, and to a root that registers no timeline; otherwise `hyperframes lint` reports `missing_timeline_registry` and each render waits 45 seconds for timeline registration. The callback receives seconds in source time: playback inpoint (`data-playback-start`) plus the rate-adjusted time since the host's resolved start. Existing playback rate and speed-ramp semantics apply. Timing is read again on every seek, so move, trim and rate edits do not require rewriting the animation code. Each overlapping scene needs its own source instance. From 29a5c02b4566334ab08b195abb0fdd0473433ff1 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 6 Oct 2026 18:28:36 -0700 Subject: [PATCH 4/4] fix(runtime): avoid transport redraws after render seeks --- packages/core/src/runtime/init.ts | 3 + .../src/services/coreRuntimeBrowser.test.ts | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 137fa3106d..37268c3a7d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3902,6 +3902,9 @@ export function initSandboxRuntimeModular(): void { activateChildren: true, suppressEvents: options?.suppressEvents, }); + // The explicit seek owns this paused frame; the transport must not redraw after capture waits. + lastTransportSeekTime = state.currentTime; + lastTransportSeekTimeline = state.capturedTimeline; runAdapters("pause", 0, pageAnimations); syncMediaForCurrentState(); colorGrading.redraw(); diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts index 1ec2ba200b..7eb7cc15a4 100644 --- a/packages/producer/src/services/coreRuntimeBrowser.test.ts +++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts @@ -92,6 +92,61 @@ describe("core runtime browser contract", () => { } }); + it.each(["preview", "export"])( + "does not redraw async sources on transport ticks after an explicit render seek (%s)", + async (mode) => { + const sourcePage = await browser.newPage(); + try { + await sourcePage.setContent( + `
`, + ); + if (mode === "export") + await sourcePage.addScriptTag({ + content: "window.__HF_EXPORT_RENDER_SEEK_CONFIG={fps:30};", + }); + await sourcePage.addScriptTag({ content: readFileSync(RUNTIME_PATH, "utf8") }); + await sourcePage.waitForFunction(() => window.__playerReady && window.__renderReady); + await sourcePage.evaluate(() => { + const element = document.getElementById("source")!; + window.__hyperframes!.registerFrameSource({ + element, + render: async (time) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + element.setAttribute( + "data-draws", + (element.getAttribute("data-draws") ?? "") + time + ",", + ); + }, + }); + }); + for (const [index, time] of [0.5, 0.5, 0.2].entries()) { + await sourcePage.evaluate((t) => window.__player!.renderSeek!(t), time); + await waitForPendingSeekCompletion(sourcePage); + // Let the transport run after the capture barrier has already settled. + await sourcePage.evaluate(async () => { + for (let frame = 0; frame < 4; frame++) + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + await window.__hfWaitForSeekCompletion!(); + }); + expect(await sourcePage.$eval("#source", (el) => el.getAttribute("data-draws"))).toBe( + [0.5, 0.5, 0.2].slice(0, index + 1).join(",") + ",", + ); + } + await sourcePage.evaluate(() => window.__player!.play()); + await sourcePage.waitForFunction( + () => + (document.getElementById("source")!.getAttribute("data-draws") ?? "") + .split(",") + .filter(Boolean).length > 3, + ); + await sourcePage.evaluate(() => window.__player!.pause()); + await waitForPendingSeekCompletion(sourcePage); + } finally { + await sourcePage.close(); + } + }, + ); + it("initializes the public player contract and seeks the CSS adapter", async () => { const result = await page.evaluate(() => { const runtimeWindow = window as unknown as {