From 2c577671734a1dd21ea15768e91d83683f74f06c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 19:03:02 -0400 Subject: [PATCH 01/11] perf(studio): reuse filmstrip frames when a zoom changes a strip's width A zoom that changed a strip's frame count decoded the whole strip again on the main thread, and showed the clip's single poster frame until it landed. Frames are now sampled at each slice's left edge, so a strip twice as long holds every frame of the shorter one; decoded frames are shared per source and time, so zooming out decodes nothing and zooming in decodes only the new half. The strip on screen stays until the new width is ready. --- .../player/components/VideoThumbnail.test.tsx | 25 ++ .../src/player/components/VideoThumbnail.tsx | 40 +++- .../src/player/components/thumbnailUtils.ts | 16 ++ .../player/lib/thumbnailVideoDecoder.test.ts | 85 +++++-- .../src/player/lib/thumbnailVideoDecoder.ts | 224 +++++++++++++----- 5 files changed, 305 insertions(+), 85 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index 3f1801f438d..fb14e5c1026 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -280,6 +280,31 @@ describe("VideoThumbnail during a zoom", () => { ); }); + it("shows the strip it has, not the poster, while a zoom's new width decodes", async () => { + vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) => + frameCount === 1 + ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 } + : { + value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, + weight: 2, + }, + ); + await render(440); + expect( + new Set([...host.querySelectorAll("img")].map((img) => img.getAttribute("src"))), + ).toEqual(new Set(["blob:a", "blob:b"])); + // The new width's decode never lands during this test. + vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {})); + await act(async () => { + reportResize(880, 40); + await Promise.resolve(); + }); + await rest(); + const shown = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src")); + expect(shown.length).toBeGreaterThan(0); + expect(shown).not.toContain("blob:poster"); + }); + it("keeps its filmstrip while the zoom resizes it, and decodes the new width at rest", async () => { await render(440); vi.mocked(decodeVideoThumbnail).mockClear(); diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index a5e00b8a4b5..09bbdf7c115 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -1,4 +1,4 @@ -import { memo, useMemo } from "react"; +import { memo, useMemo, useState } from "react"; import { useThumbnailLease } from "../../hooks/useThumbnailLease"; import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize"; import { @@ -8,7 +8,11 @@ import { } from "../lib/thumbnailScheduler"; import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder"; import { ThumbnailTiles } from "./ThumbnailTiles"; -import { computeThumbnailStrip, quantizeThumbnailFrameCount } from "./thumbnailUtils"; +import { + computeThumbnailStrip, + quantizeThumbnailFrameCount, + thumbnailFrameForTile, +} from "./thumbnailUtils"; import { useValueAtRest } from "./timelineMotion"; interface VideoThumbnailProps { @@ -68,13 +72,35 @@ function createVideoThumbnailRequest( function selectThumbnailSnapshot( poster: ThumbnailSnapshot, rich: ThumbnailSnapshot, + shown: ThumbnailSnapshot, ): ThumbnailSnapshot { if (rich.status === "ready") return rich; + if (shown.status === "ready") return shown; if (poster.status === "ready") return poster; if (rich.status === "loading" || poster.status === "loading") return { status: "loading" }; return poster; } +type VideoThumbnailRequest = ReturnType; + +/** + * The strip to show: the requested width once ready, else the strip already on screen, kept leased + * while a zoom's new width decodes, else the poster. + */ +function useVideoThumbnailSnapshot( + poster: VideoThumbnailRequest | null, + rich: VideoThumbnailRequest | null, + props: object, +): ThumbnailSnapshot { + const posterSnapshot = useThumbnailLease(poster); + const richSnapshot = useThumbnailLease(rich); + const [shown, setShown] = useState({ props, request: rich }); + if (richSnapshot.status === "ready" && shown.request !== rich) setShown({ props, request: rich }); + const holdsShown = rich !== null && shown.props === props && shown.request !== rich; + const shownSnapshot = useThumbnailLease(holdsShown ? shown.request : null); + return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot); +} + /** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */ export const VideoThumbnail = memo(function VideoThumbnail({ videoSrc, @@ -114,9 +140,11 @@ export const VideoThumbnail = memo(function VideoThumbnail({ [requestFrameCount, requestProps], ); const measured = useValueAtRest(container.width > 0); - const posterSnapshot = useThumbnailLease(measured ? posterRequest : null); - const richSnapshot = useThumbnailLease(measured && requestFrameCount > 1 ? richRequest : null); - const snapshot = selectThumbnailSnapshot(posterSnapshot, richSnapshot); + const snapshot = useVideoThumbnailSnapshot( + measured ? posterRequest : null, + measured && requestFrameCount > 1 ? richRequest : null, + requestProps, + ); const value = snapshot.status === "ready" ? snapshot.value : null; const urls = value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : []; @@ -133,7 +161,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({ watchGap={watchGap} > {(index) => { - const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))]; + const src = urls[thumbnailFrameForTile(index, frameCount, urls.length)]; return (
{ + source = `/clip-${++sources}.mp4`; vi.clearAllMocks(); getKeyPacket.mockImplementation(async () => null); vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two"); @@ -55,9 +60,15 @@ beforeEach(() => { }); describe("videoThumbnailTimestamps", () => { - it("uses the midpoint for a poster and sorted sparse points for a strip", () => { + it("uses the midpoint for a poster and each frame's left edge for a strip", () => { expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]); - expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 4, 6, 8]); + expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5]); + }); + + it("nests each strip in the strip twice as long, and the poster in every strip", () => { + const strip = videoThumbnailTimestamps(0, 8, 4); + expect(videoThumbnailTimestamps(0, 8, 8).filter((_, i) => i % 2 === 0)).toEqual(strip); + expect(strip).toContain(videoThumbnailTimestamps(0, 8, 1)[0]); }); it("clamps invalid source ranges", () => { @@ -71,11 +82,11 @@ describe("decodeVideoThumbnail", () => { const decoded: number[][] = []; recordDecodes(decoded); const result = await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 }, + { source, sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 }, new AbortController().signal, ); - expect(decoded).toEqual([[2, 8]]); + expect(decoded).toEqual([[2, 5]]); expect(result.value).toEqual({ kind: "filmstrip", urls: ["blob:one", "blob:two"], @@ -88,14 +99,14 @@ describe("decodeVideoThumbnail", () => { }); it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => { - getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 })); + getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 0.5 })); const decoded: number[][] = []; recordDecodes(decoded); await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 }, + { source, sourceStart: 2, sourceRangeDuration: 8, frameCount: 4 }, new AbortController().signal, ); - expect(decoded).toEqual([[2, 4, 8]]); + expect(decoded).toEqual([[2, 3.5, 5.5, 7.5]]); }); it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => { @@ -103,14 +114,10 @@ describe("decodeVideoThumbnail", () => { const decoded: number[][] = []; recordDecodes(decoded); await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 4 }, new AbortController().signal, ); - await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 }, - new AbortController().signal, - ); - expect(decoded).toEqual([[0, 5, 10], [5]]); + expect(decoded).toEqual([[0, 2.5, 5, 7.5]]); }); it("looks up each keyframe just before decoding it", async () => { @@ -126,10 +133,10 @@ describe("decodeVideoThumbnail", () => { } }); await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 }, new AbortController().signal, ); - expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]); + expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5"]); }); it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => { @@ -146,7 +153,7 @@ describe("decodeVideoThumbnail", () => { }); await expect( decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, controller.signal, ), ).rejects.toThrow("Aborted"); @@ -164,10 +171,45 @@ describe("decodeVideoThumbnail", () => { const decoded: number[][] = []; recordDecodes(decoded); await decodeVideoThumbnail( - { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 }, + { source, sourceStart: 5, sourceRangeDuration: 5, frameCount: 2 }, new AbortController().signal, ); - expect(decoded).toEqual([[5, 7.5, 10]]); + expect(decoded).toEqual([[5, 7.5]]); + }); + + it("reuses the frames a strip of the same source shows, so a zoom decodes only new times", async () => { + vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); + const decoded: number[][] = []; + recordDecodes(decoded); + const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; + const signal = new AbortController().signal; + const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); + // Twice the frames decodes only the new half; half the frames decodes nothing. + const eight = await decodeVideoThumbnail({ ...range, frameCount: 8 }, signal); + const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); + expect(decoded).toEqual([ + [0, 2, 4, 6], + [1, 3, 5, 7], + ]); + const urls = (result: typeof four) => + result.value.kind === "filmstrip" ? result.value.urls : []; + expect(urls(eight).filter((_, i) => i % 2 === 0)).toEqual(urls(four)); + expect(urls(two)).toEqual([urls(four)[0], urls(four)[2]]); + for (const result of [four, eight, two]) result.dispose?.(); + }); + + it("revokes a shared frame only once no strip shows it", async () => { + vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); + recordDecodes([]); + const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; + const signal = new AbortController().signal; + const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); + const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); + four.dispose?.(); + // The two frames the 2-frame strip still shows stay; the other two go. + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2); + two.dispose?.(); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(4); }); it("releases input and degrades when the source has no video track", async () => { @@ -187,7 +229,7 @@ describe("decodeVideoThumbnail", () => { yield { canvas, timestamp: 2, duration: 1 }; }); await expect( - decodeVideoThumbnail({ source: "/clip.mp4", frameCount: 2 }, controller.signal), + decodeVideoThumbnail({ source, frameCount: 2 }, controller.signal), ).rejects.toMatchObject({ name: "AbortError" }); expect(URL.revokeObjectURL).toHaveBeenCalledTimes(1); expect(dispose).toHaveBeenCalledTimes(1); @@ -207,10 +249,7 @@ describe("decodeVideoThumbnail", () => { getDurationFromMetadata, }); - const decoding = decodeVideoThumbnail( - { source: "/clip.mp4", frameCount: 1 }, - controller.signal, - ); + const decoding = decodeVideoThumbnail({ source, frameCount: 1 }, controller.signal); await vi.waitFor(() => expect(getDisplayWidth).toHaveBeenCalledOnce()); controller.abort(); resolveWidth(1080); diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index b892c6cc943..5053e42d3cf 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -9,6 +9,10 @@ export interface VideoThumbnailDecodeRequest { fit?: "contain" | "cover"; } +/** + * The times a strip of `frameCount` frames shows: each frame's left edge, so a strip twice as long + * holds every frame of this one and a zoom decodes only the new half. A poster is the midpoint. + */ export function videoThumbnailTimestamps( start: number, duration: number, @@ -18,10 +22,42 @@ export function videoThumbnailTimestamps( const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0); const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1); if (count === 1) return [safeStart + safeDuration / 2]; - return Array.from( - { length: count }, - (_, index) => safeStart + (safeDuration * index) / (count - 1), - ); + return Array.from({ length: count }, (_, index) => safeStart + (safeDuration * index) / count); +} + +/** A decoded frame shared by every strip of its source that shows that time. */ +interface SharedFrame { + url: string; + users: number; +} + +const sharedFrames = new Map(); +const sourceInfos = new Map(); + +function takeSharedFrame(key: string): string | undefined { + const frame = sharedFrames.get(key); + if (frame) frame.users += 1; + return frame?.url; +} + +/** Shares a newly decoded frame, or the copy another strip decoded meanwhile. */ +function shareFrame(key: string, url: string): string { + const existing = takeSharedFrame(key); + if (existing === undefined) { + sharedFrames.set(key, { url, users: 1 }); + return url; + } + URL.revokeObjectURL(url); + return existing; +} + +function releaseSharedFrames(keys: string[]): void { + for (const key of keys.splice(0)) { + const frame = sharedFrames.get(key); + if (!frame || --frame.users > 0) continue; + sharedFrames.delete(key); + URL.revokeObjectURL(frame.url); + } } async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise { @@ -38,7 +74,10 @@ async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promis } interface DecodedResources { - urls: string[]; + /** One frame per slot of the strip; a slot the source could not decode stays empty. */ + urls: (string | undefined)[]; + /** The shared frames this strip holds, released when the strip is. */ + keys: string[]; canvases: Set; } @@ -48,12 +87,19 @@ interface ThumbnailCanvasSink { ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>; } +/** What a strip of a source needs before it can name its frames. */ +interface SourceInfo { + aspect: number; + metadataDuration: number | null; +} + function throwIfAborted(signal: AbortSignal): void { if (signal.aborted) throw new DOMException("Aborted", "AbortError"); } function releaseDecodedResources(resources: DecodedResources): void { - for (const url of resources.urls.splice(0)) URL.revokeObjectURL(url); + resources.urls.length = 0; + releaseSharedFrames(resources.keys); for (const canvas of resources.canvases) { canvas.width = 0; canvas.height = 0; @@ -75,19 +121,65 @@ function targetDimensions( }; } +/** The strip's source range and frame times, clamped to what the file holds. */ +function stripTimes( + request: VideoThumbnailDecodeRequest, + info: SourceInfo, + budgets: Readonly, +) { + const requestedStart = Math.max(0, request.sourceStart ?? 0); + const sourceDuration = Math.max( + 0, + info.metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0), + ); + const sourceStart = Math.min(requestedStart, sourceDuration); + const requestedDuration = + request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart); + const duration = Math.min( + Math.max(0, requestedDuration), + Math.max(0, sourceDuration - sourceStart), + ); + const timestamps = videoThumbnailTimestamps( + sourceStart, + duration, + Math.min(request.frameCount, budgets.richPreviewFrameCount), + ); + return { sourceStart, duration, timestamps }; +} + +/** Holds every frame of the strip already decoded; returns the slots still to decode. */ +function takeDecodedFrames( + timestamps: number[], + keyOf: (time: number) => string, + resources: DecodedResources, +): number[] { + const missing: number[] = []; + timestamps.forEach((time, slot) => { + const url = takeSharedFrame(keyOf(time)); + if (url === undefined) return void missing.push(slot); + resources.urls[slot] = url; + resources.keys.push(keyOf(time)); + }); + return missing; +} + async function decodeFrames( sink: ThumbnailCanvasSink, - timestamps: AsyncIterable, + times: AsyncIterable, + slots: { slot: number; key: string }[], signal: AbortSignal, resources: DecodedResources, ): Promise { - for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) { + let next = 0; + for await (const wrapped of sink.canvasesAtTimestamps(times)) { throwIfAborted(signal); - if (!wrapped) continue; + const target = slots[next++]; + if (!wrapped || !target) continue; resources.canvases.add(wrapped.canvas); const blob = await canvasToBlob(wrapped.canvas); throwIfAborted(signal); - resources.urls.push(URL.createObjectURL(blob)); + resources.urls[target.slot] = shareFrame(target.key, URL.createObjectURL(blob)); + resources.keys.push(target.key); } throwIfAborted(signal); } @@ -95,27 +187,58 @@ async function decodeFrames( function loadedResult( resources: DecodedResources, aspect: number, - width: number, - height: number, + budgets: Readonly, ): ThumbnailLoadedResult { - const firstUrl = resources.urls[0]; + const urls = resources.urls.filter((url): url is string => url !== undefined); + const firstUrl = urls[0]; if (!firstUrl) throw new Error("Video source returned no thumbnail frames"); + const { width, height } = targetDimensions(aspect, budgets); const value: ThumbnailValue = - resources.urls.length === 1 + urls.length === 1 ? { kind: "image", url: firstUrl, aspect } - : { kind: "filmstrip", urls: [...resources.urls], aspect }; + : { kind: "filmstrip", urls, aspect }; return { value, - weight: width * height * 4 * resources.urls.length, + weight: width * height * 4 * urls.length, dispose: () => releaseDecodedResources(resources), }; } -/** Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. */ +/** + * Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. Frames another strip + * of the source already shows are reused, so a zoom decodes only the frames that are new. + */ export async function decodeVideoThumbnail( request: VideoThumbnailDecodeRequest, signal: AbortSignal, budgets: Readonly = TIMELINE_VIEWPORT_BUDGETS, +): Promise { + const fit = request.fit ?? "cover"; + const keyOf = (time: number) => `${request.source}\u0000${fit}\u0000${time}`; + const resources: DecodedResources = { urls: [], keys: [], canvases: new Set() }; + const known = sourceInfos.get(request.source); + try { + if (known) { + const { timestamps } = stripTimes(request, known, budgets); + if (takeDecodedFrames(timestamps, keyOf, resources).length === 0) { + throwIfAborted(signal); + return loadedResult(resources, known.aspect, budgets); + } + releaseDecodedResources(resources); + } + return await decodeMissingFrames(request, signal, budgets, keyOf, resources); + } catch (error) { + releaseDecodedResources(resources); + throw error; + } +} + +async function decodeMissingFrames( + request: VideoThumbnailDecodeRequest, + signal: AbortSignal, + budgets: Readonly, + keyOf: (time: number) => string, + resources: DecodedResources, ): Promise { const mediabunny = await import("mediabunny"); throwIfAborted(signal); @@ -124,7 +247,6 @@ export async function decodeVideoThumbnail( source: new mediabunny.UrlSource(request.source), formats: mediabunny.ALL_FORMATS, }); - const resources: DecodedResources = { urls: [], canvases: new Set() }; try { const track = await input.getPrimaryVideoTrack(); throwIfAborted(signal); @@ -139,46 +261,36 @@ export async function decodeVideoThumbnail( } const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true }); throwIfAborted(signal); - const requestedStart = Math.max(0, request.sourceStart ?? 0); - const sourceDuration = Math.max( - 0, - metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0), - ); - const sourceStart = Math.min(requestedStart, sourceDuration); - const requestedDuration = - request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart); - const duration = Math.min( - Math.max(0, requestedDuration), - Math.max(0, sourceDuration - sourceStart), - ); - const timestamps = videoThumbnailTimestamps( - sourceStart, - duration, - Math.min(request.frameCount, budgets.richPreviewFrameCount), - ); - const keys = new mediabunny.EncodedPacketSink(track); - const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2; - async function* decodeTimesAtNearbyKeyframes() { - for (const time of timestamps) { - const key = await keys.getKeyPacket(time, { metadataOnly: true }); - if (signal.aborted) return; - const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead; - yield near ? key.timestamp : time; + const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration }; + sourceInfos.set(request.source, info); + const { sourceStart, duration, timestamps } = stripTimes(request, info, budgets); + const slots = takeDecodedFrames(timestamps, keyOf, resources).map((slot) => ({ + slot, + key: keyOf(timestamps[slot]!), + })); + if (slots.length > 0) { + const keys = new mediabunny.EncodedPacketSink(track); + const maxKeyframeLead = duration / Math.max(2, timestamps.length) / 2; + async function* decodeTimesAtNearbyKeyframes() { + for (const { slot } of slots) { + const time = timestamps[slot]!; + const key = await keys.getKeyPacket(time, { metadataOnly: true }); + if (signal.aborted) return; + const near = + key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead; + yield near ? key.timestamp : time; + } } + const target = targetDimensions(info.aspect, budgets); + const sink = new mediabunny.CanvasSink(track, { + width: target.width, + height: target.height, + fit: request.fit ?? "cover", + poolSize: 1, + }); + await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources); } - const aspect = displayWidth / displayHeight; - const target = targetDimensions(aspect, budgets); - const sink = new mediabunny.CanvasSink(track, { - width: target.width, - height: target.height, - fit: request.fit ?? "cover", - poolSize: 1, - }); - await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources); - return loadedResult(resources, aspect, target.width, target.height); - } catch (error) { - releaseDecodedResources(resources); - throw error; + return loadedResult(resources, info.aspect, budgets); } finally { input.dispose(); } From c490325b8b18aa93e7997ce7196eb5ec3bc066ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 19:12:20 -0400 Subject: [PATCH 02/11] perf(studio): redraw only the waveforms on or near the screen after a zoom A zoom resizes every mounted clip, and every mounted waveform redrew in that same frame, on screen or not. A waveform now draws only while it is on or near the screen (one shared observer, a screen of margin each side) and draws when it comes near if a zoom changed it meanwhile. --- .../player/components/AudioWaveform.test.tsx | 41 +++++++++++++++++ .../src/player/components/AudioWaveform.tsx | 45 ++++++++++++++++--- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/packages/studio/src/player/components/AudioWaveform.test.tsx b/packages/studio/src/player/components/AudioWaveform.test.tsx index 1652e04b9ab..20819a8c97d 100644 --- a/packages/studio/src/player/components/AudioWaveform.test.tsx +++ b/packages/studio/src/player/components/AudioWaveform.test.tsx @@ -11,6 +11,29 @@ vi.mock("../../hooks/useThumbnailLease", () => ({ useThumbnailLease: leaseSpy, })); +// One observer serves every waveform; it reports each one near as it is watched, unless a test +// reports by hand. +const nearScreen = { + auto: true, + watched: [] as Element[], + report: (_target: Element, _near: boolean) => {}, +}; +globalThis.IntersectionObserver = class { + constructor(callback: IntersectionObserverCallback) { + nearScreen.report = (target, near) => + callback( + [{ target, isIntersecting: near } as IntersectionObserverEntry], + this as unknown as IntersectionObserver, + ); + } + observe(target: Element) { + nearScreen.watched.push(target); + if (nearScreen.auto) nearScreen.report(target, true); + } + unobserve() {} + disconnect() {} +} as unknown as typeof IntersectionObserver; + const EMPTY_STRIP = { width: 0, height: 0, inViewStart: 0, inViewEnd: 0 }; const watchGap = vi.hoisted(() => vi.fn()); const strip = vi.hoisted(() => ({ @@ -52,6 +75,7 @@ afterEach(() => { leaseSpy.mockReset(); leaseSpy.mockImplementation(() => ({ status: "loading" as const })); strip.size = EMPTY_STRIP; + nearScreen.auto = true; watchGap.mockClear(); document.body.innerHTML = ""; }); @@ -141,6 +165,23 @@ describe("AudioWaveform", () => { expect(watchGap.mock.calls.filter(([gap]) => gap)).toEqual([]); }); + it("leaves a waveform off screen undrawn through a zoom, and draws it once it comes near", () => { + nearScreen.auto = false; + const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 }); + renderReadyWaveform(); + const root = nearScreen.watched.at(-1)!; + expect(getContext).not.toHaveBeenCalled(); + act(() => nearScreen.report(root, true)); + const drawn = getContext.mock.calls.length; + expect(drawn).toBeGreaterThan(0); + act(() => nearScreen.report(root, false)); + setStrip({ width: 2000, height: 40, inViewStart: 0, inViewEnd: 2000 }); + expect(getContext.mock.calls.length).toBe(drawn); + act(() => nearScreen.report(root, true)); + expect(getContext.mock.calls.length).toBe(drawn + 1); + }); + it("redraws a short clip, drawn whole, when a zoom changes its width", () => { const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 }); diff --git a/packages/studio/src/player/components/AudioWaveform.tsx b/packages/studio/src/player/components/AudioWaveform.tsx index d7e226ba1b5..32155125d45 100644 --- a/packages/studio/src/player/components/AudioWaveform.tsx +++ b/packages/studio/src/player/components/AudioWaveform.tsx @@ -1,4 +1,4 @@ -import { memo, useCallback, useContext, useEffect, useMemo, useRef } from "react"; +import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { fadeGain } from "@hyperframes/core/audio-fade"; import { useThumbnailLease } from "../../hooks/useThumbnailLease"; import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize"; @@ -158,6 +158,35 @@ async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise void>(); +let nearScreen: IntersectionObserver | null = null; + +/** Whether `element` is on or near the screen, kept current by one observer for all waveforms. */ +function useNearScreen(element: Element | null): boolean { + const [near, setNear] = useState(() => typeof IntersectionObserver === "undefined"); + useEffect(() => { + if (!element || typeof IntersectionObserver === "undefined") return; + nearScreen ??= new IntersectionObserver( + (entries) => + entries.forEach((entry) => nearScreenListeners.get(entry.target)?.(entry.isIntersecting)), + NEAR_SCREEN, + ); + nearScreenListeners.set(element, setNear); + nearScreen.observe(element); + return () => { + nearScreenListeners.delete(element); + nearScreen?.unobserve(element); + }; + }, [element]); + return near; +} + export const AudioWaveform = memo(function AudioWaveform({ audioUrl, waveformUrl, @@ -171,7 +200,7 @@ export const AudioWaveform = memo(function AudioWaveform({ muted = false, labelInset = 16, }: AudioWaveformProps) { - const rootRef = useRef(null); + const [root, setRoot] = useState(null); const canvasRef = useRef(null); const cacheKey = waveformUrl ?? audioUrl; const request = useMemo( @@ -218,7 +247,11 @@ export const AudioWaveform = memo(function AudioWaveform({ span, ); }, [fades, from, muted, peaks, to, trimEndFraction, trimStartFraction]); - useEffect(draw, [draw, strip.width, strip.height]); + // A waveform off screen draws when it comes near, not at every zoom that resizes it. + const near = useNearScreen(root); + useEffect(() => { + if (near) draw(); + }, [draw, near, strip.width, strip.height]); useEffect(() => { const root = document.documentElement; @@ -231,15 +264,15 @@ export const AudioWaveform = memo(function AudioWaveform({ }, [draw]); useEffect(() => { - const clip = rootRef.current?.closest(".timeline-clip"); + const clip = root?.closest(".timeline-clip"); if (!(clip instanceof HTMLElement)) return; if (muted) clip.setAttribute("data-audio-muted", "true"); else clip.removeAttribute("data-audio-muted"); return () => clip.removeAttribute("data-audio-muted"); - }, [muted]); + }, [muted, root]); return ( -
+
Date: Tue, 6 Oct 2026 19:24:06 -0400 Subject: [PATCH 03/11] perf(studio): never start a beat-analysis stage while the timeline zooms Analyzing a long music track runs for over a second on the main thread, and when it started late it landed inside a zoom. The analysis now awaits an optional pause before each long stage; Studio's waits for the timeline to rest and the browser to go idle. Results are unchanged. The tempo detector's stage is one call, so a zoom that starts inside it still waits for it. --- packages/core/src/beats/beatDetection.test.ts | 30 +++++++++++++++++-- packages/core/src/beats/beatDetection.ts | 20 +++++++++++-- .../studio/src/hooks/useMusicBeatAnalysis.ts | 4 ++- .../player/components/timelineMotion.test.ts | 27 +++++++++++++++++ .../src/player/components/timelineMotion.ts | 20 +++++++++++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 packages/studio/src/player/components/timelineMotion.test.ts diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.ts index 31bd1675351..3037facc7a2 100644 --- a/packages/core/src/beats/beatDetection.test.ts +++ b/packages/core/src/beats/beatDetection.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { loadBpmDetective } from "./beatDetection"; +import { analyzeMusicFromBuffer, loadBpmDetective } from "./beatDetection"; describe("loadBpmDetective", () => { it("resolves to null when the import is unavailable", async () => { @@ -39,3 +39,29 @@ describe("loadBpmDetective", () => { expect(detect).toBe(fn); }); }); + +describe("analyzeMusicFromBuffer", () => { + it("waits on the caller's pause before each long stage", async () => { + const sampleRate = 8000; + const channel = new Float32Array(sampleRate * 2); + for (let beat = 0; beat < 4; beat++) channel.fill(0.9, beat * 4000, beat * 4000 + 80); + const audio = { + getChannelData: () => channel, + sampleRate, + duration: 2, + } as unknown as AudioBuffer; + const held: (() => void)[] = []; + const pause = () => new Promise((resolve) => held.push(resolve)); + let done = false; + const analysis = analyzeMusicFromBuffer(audio, { pause }).then((result) => { + done = true; + return result; + }); + for (let stage = 1; stage <= 3; stage++) { + await vi.waitFor(() => expect(held).toHaveLength(stage)); + expect(done).toBe(false); + held[stage - 1]!(); + } + await expect(analysis).resolves.toMatchObject({ sampleRate }); + }); +}); diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts index 59bc109f6b3..cd080722dd6 100644 --- a/packages/core/src/beats/beatDetection.ts +++ b/packages/core/src/beats/beatDetection.ts @@ -213,15 +213,25 @@ function gateBeatsBySilence( } // fallow-ignore-next-line complexity -export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise { +export interface MusicAnalysisOptions { + /** Awaited before each long stage, so a caller can hold the analysis while its page is busy. */ + pause?: () => Promise; +} + +export async function analyzeMusicFromBuffer( + audioBuffer: AudioBuffer, + { pause }: MusicAnalysisOptions = {}, +): Promise { const channelData = audioBuffer.getChannelData(0); const sampleRate = audioBuffer.sampleRate; const duration = audioBuffer.duration; + await pause?.(); const rawBeats = await detectBeats(audioBuffer); const onsetBpm = computeBpmFromBeats(rawBeats); let detectiveBpm: number | null = null; + await pause?.(); try { const detect = await loadBpmDetective(); if (detect) detectiveBpm = detect(audioBuffer); @@ -261,6 +271,7 @@ export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise< regularizeBpm = detectiveBpm; } + await pause?.(); const gridBeats = regularizeBpm !== null ? regularizeBeats(rawBeats, regularizeBpm, duration) : rawBeats; const gated = gateBeatsBySilence(gridBeats, channelData, sampleRate); @@ -288,13 +299,16 @@ export async function detectBeatsFromUrl(url: string): Promise { } } -export async function analyzeMusicFromUrl(url: string): Promise { +export async function analyzeMusicFromUrl( + url: string, + options: MusicAnalysisOptions = {}, +): Promise { const audioContext = new AudioContext(); try { const response = await fetch(url); const arrayBuffer = await response.arrayBuffer(); const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - return analyzeMusicFromBuffer(audioBuffer); + return analyzeMusicFromBuffer(audioBuffer, options); } finally { await audioContext.close(); } diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.ts index 8d0f2dc585f..10831949309 100644 --- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts +++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts @@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef } from "react"; import { usePlayerStore } from "../player/store/playerStore"; import { resolveBeatSourceTrack } from "../utils/timelineInspector"; import { analyzeMusicFromUrl } from "@hyperframes/core/beats"; +import { whenTimelineIdle } from "../player/components/timelineMotion"; import { useFileManagerContextOptional } from "../contexts/FileManagerContext"; import { mergeUserBeats } from "../utils/beatEditing"; import { getTimelineElementIndexes } from "../player/lib/timelineElementIndexes"; @@ -79,7 +80,8 @@ async function loadBeatAnalysis( ): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> { let promise = analysisCache.get(musicSrc); if (!promise) { - promise = analyzeMusicFromUrl(musicSrc); + // Each long stage of the analysis waits for the timeline to rest, so it never lands mid-zoom. + promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle }); cacheAnalysis(musicSrc, promise); } try { diff --git a/packages/studio/src/player/components/timelineMotion.test.ts b/packages/studio/src/player/components/timelineMotion.test.ts new file mode 100644 index 00000000000..33d4c2ee65a --- /dev/null +++ b/packages/studio/src/player/components/timelineMotion.test.ts @@ -0,0 +1,27 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { markTimelineMotion, TIMELINE_REST_MS, whenTimelineIdle } from "./timelineMotion"; + +afterEach(() => { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); +}); + +it("waits for a zoom to rest before resolving, and resolves at once when nothing moves", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "requestIdleCallback"] }); + markTimelineMotion(); + let idle = false; + void whenTimelineIdle().then(() => (idle = true)); + await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50); + expect(idle).toBe(false); + // Another zoom step pushes the rest back. + markTimelineMotion(); + await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50); + expect(idle).toBe(false); + await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS); + expect(idle).toBe(true); + + let again = false; + void whenTimelineIdle().then(() => (again = true)); + await vi.advanceTimersByTimeAsync(1); + expect(again).toBe(true); +}); diff --git a/packages/studio/src/player/components/timelineMotion.ts b/packages/studio/src/player/components/timelineMotion.ts index 2fda7a3b527..78ebb29d82a 100644 --- a/packages/studio/src/player/components/timelineMotion.ts +++ b/packages/studio/src/player/components/timelineMotion.ts @@ -23,6 +23,26 @@ export function markTimelineMotion(): void { }, TIMELINE_REST_MS); } +const onIdle = (callback: () => void) => { + if (typeof requestIdleCallback === "function") requestIdleCallback(callback, { timeout: 1000 }); + else setTimeout(callback, 0); +}; + +/** Resolves at an idle moment with no zoom moving the timeline: long work waits there, not mid-zoom. */ +export function whenTimelineIdle(): Promise { + return new Promise((resolve) => { + const check = () => { + if (!moving) return resolve(); + const stop = subscribeTimelineMotion(() => { + if (moving) return; + stop(); + onIdle(check); + }); + }; + onIdle(check); + }); +} + export function isTimelineMoving(): boolean { return moving; } From 036fefc1816520d66153ee0edf0fdc19ca355061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 20:01:11 -0400 Subject: [PATCH 04/11] fix(studio): reuse a filmstrip frame only where its own strip would show it A shared frame was keyed by the time a strip asked for, not the time it was decoded at, so a coarse strip's keyframe-early frame could land in a finer strip and run its frames backwards. A frame now carries its decoded time and is reused only when the strip would have accepted that keyframe itself. The frame-count ladder now tops out at a power of two (32), so its top step shares frames with the one below instead of none. --- .../player/components/thumbnailUtils.test.ts | 9 +- .../src/player/components/thumbnailUtils.ts | 4 +- .../player/lib/thumbnailVideoDecoder.test.ts | 28 ++++- .../src/player/lib/thumbnailVideoDecoder.ts | 111 +++++++++++------- 4 files changed, 99 insertions(+), 53 deletions(-) diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.ts index e9fd1e73a4d..a17da47a88d 100644 --- a/packages/studio/src/player/components/thumbnailUtils.test.ts +++ b/packages/studio/src/player/components/thumbnailUtils.test.ts @@ -5,6 +5,7 @@ import { resolveMediaPreviewUrl, quantizeThumbnailFrameCount, } from "./thumbnailUtils"; +import { MAX_VISIBLE_THUMBNAIL_FRAMES } from "../lib/timelineViewportBudgets"; describe("computeThumbnailStrip", () => { it("sizes tiles by aspect ratio at the clip height", () => { @@ -55,11 +56,13 @@ describe("quantizeThumbnailFrameCount", () => { it("uses doubling buckets and never exceeds the 4K geometry ceiling", () => { expect(quantizeThumbnailFrameCount(5)).toBe(8); expect(quantizeThumbnailFrameCount(32)).toBe(32); - expect(quantizeThumbnailFrameCount(34)).toBe(33); }); - it("caps decode requests at the shared visible-frame budget", () => { - expect(quantizeThumbnailFrameCount(124)).toBe(33); + it("caps decode requests at the largest step within the visible-frame budget", () => { + // A step that is not a power of two would share no frames with the step below it. + expect(MAX_VISIBLE_THUMBNAIL_FRAMES).toBeGreaterThan(32); + expect(quantizeThumbnailFrameCount(34)).toBe(32); + expect(quantizeThumbnailFrameCount(124)).toBe(32); }); }); diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts index d3d614088bb..497bd8d2dfd 100644 --- a/packages/studio/src/player/components/thumbnailUtils.ts +++ b/packages/studio/src/player/components/thumbnailUtils.ts @@ -11,7 +11,9 @@ export interface ThumbnailStripLayout { /** Quantize request identities so a pixel-by-pixel resize does not thrash the cache. */ export function quantizeThumbnailFrameCount(frameCount: number): number { const safeCount = Math.max(1, Number.isFinite(frameCount) ? Math.ceil(frameCount) : 1); - return Math.min(MAX_VISIBLE_THUMBNAIL_FRAMES, 2 ** Math.ceil(Math.log2(safeCount))); + // Capped at a power of two too, so the top step still holds every frame of the one below it. + const cap = 2 ** Math.floor(Math.log2(MAX_VISIBLE_THUMBNAIL_FRAMES)); + return Math.min(cap, 2 ** Math.ceil(Math.log2(safeCount))); } /** diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index 5c79ec87105..12b5f8a8632 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -198,6 +198,24 @@ describe("decodeVideoThumbnail", () => { for (const result of [four, eight, two]) result.dispose?.(); }); + it("decodes a frame again when the one shared for its time sits too far before it", async () => { + // Every keyframe is 2 s before the time asked: a 2-frame strip of 8 s accepts that (its + // frames stand for 4 s each), a 4-frame strip does not, or its frames would run backwards. + getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2 })); + const decoded: number[][] = []; + recordDecodes(decoded); + const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; + const signal = new AbortController().signal; + const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); + const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); + expect(decoded).toEqual([ + [0, 2], + [2, 4, 6], + ]); + two.dispose?.(); + four.dispose?.(); + }); + it("revokes a shared frame only once no strip shows it", async () => { vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); recordDecodes([]); @@ -223,10 +241,12 @@ describe("decodeVideoThumbnail", () => { it("revokes partial results when cancellation lands during extraction", async () => { const controller = new AbortController(); const canvas = document.createElement("canvas"); - canvasesAtTimestamps.mockImplementation(async function* () { - yield { canvas, timestamp: 1, duration: 1 }; - controller.abort(); - yield { canvas, timestamp: 2, duration: 1 }; + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + let frames = 0; + for await (const timestamp of timestamps) { + if (frames++ === 1) controller.abort(); + yield { canvas, timestamp, duration: 1 }; + } }); await expect( decodeVideoThumbnail({ source, frameCount: 2 }, controller.signal), diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 5053e42d3cf..3c1d2a45cca 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -25,37 +25,56 @@ export function videoThumbnailTimestamps( return Array.from({ length: count }, (_, index) => safeStart + (safeDuration * index) / count); } -/** A decoded frame shared by every strip of its source that shows that time. */ +/** A decoded frame, shared by every strip of its source that would accept it for a time. */ interface SharedFrame { url: string; + decodedAt: number; users: number; } -const sharedFrames = new Map(); +interface HeldFrame { + key: string; + frame: SharedFrame; +} + +/** A strip takes a frame decoded inside its range, at most `lead` before the time it shows. */ +interface FrameWindow { + sourceStart: number; + lead: number; +} + +const sharedFrames = new Map(); const sourceInfos = new Map(); -function takeSharedFrame(key: string): string | undefined { - const frame = sharedFrames.get(key); +const accepts = (window: FrameWindow, time: number, decodedAt: number) => + decodedAt >= window.sourceStart && time - decodedAt <= window.lead; + +function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined { + const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt)); if (frame) frame.users += 1; - return frame?.url; + return frame; } -/** Shares a newly decoded frame, or the copy another strip decoded meanwhile. */ -function shareFrame(key: string, url: string): string { - const existing = takeSharedFrame(key); - if (existing === undefined) { - sharedFrames.set(key, { url, users: 1 }); - return url; +/** Shares a newly decoded frame, or the same frame another strip decoded meanwhile. */ +function shareFrame(key: string, decodedAt: number, url: string): SharedFrame { + const frames = sharedFrames.get(key) ?? []; + const existing = frames.find((shared) => shared.decodedAt === decodedAt); + if (existing) { + existing.users += 1; + URL.revokeObjectURL(url); + return existing; } - URL.revokeObjectURL(url); - return existing; + const frame = { url, decodedAt, users: 1 }; + sharedFrames.set(key, [...frames, frame]); + return frame; } -function releaseSharedFrames(keys: string[]): void { - for (const key of keys.splice(0)) { - const frame = sharedFrames.get(key); - if (!frame || --frame.users > 0) continue; - sharedFrames.delete(key); +function releaseSharedFrames(held: HeldFrame[]): void { + for (const { key, frame } of held.splice(0)) { + if (--frame.users > 0) continue; + const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? []; + if (rest.length > 0) sharedFrames.set(key, rest); + else sharedFrames.delete(key); URL.revokeObjectURL(frame.url); } } @@ -77,7 +96,7 @@ interface DecodedResources { /** One frame per slot of the strip; a slot the source could not decode stays empty. */ urls: (string | undefined)[]; /** The shared frames this strip holds, released when the strip is. */ - keys: string[]; + held: HeldFrame[]; canvases: Set; } @@ -99,7 +118,7 @@ function throwIfAborted(signal: AbortSignal): void { function releaseDecodedResources(resources: DecodedResources): void { resources.urls.length = 0; - releaseSharedFrames(resources.keys); + releaseSharedFrames(resources.held); for (const canvas of resources.canvases) { canvas.width = 0; canvas.height = 0; @@ -121,7 +140,7 @@ function targetDimensions( }; } -/** The strip's source range and frame times, clamped to what the file holds. */ +/** The strip's frame times, clamped to what the file holds, and the frames it accepts for them. */ function stripTimes( request: VideoThumbnailDecodeRequest, info: SourceInfo, @@ -144,21 +163,23 @@ function stripTimes( duration, Math.min(request.frameCount, budgets.richPreviewFrameCount), ); - return { sourceStart, duration, timestamps }; + const window: FrameWindow = { sourceStart, lead: duration / Math.max(2, timestamps.length) / 2 }; + return { window, timestamps }; } /** Holds every frame of the strip already decoded; returns the slots still to decode. */ function takeDecodedFrames( - timestamps: number[], + { timestamps, window }: ReturnType, keyOf: (time: number) => string, resources: DecodedResources, ): number[] { const missing: number[] = []; timestamps.forEach((time, slot) => { - const url = takeSharedFrame(keyOf(time)); - if (url === undefined) return void missing.push(slot); - resources.urls[slot] = url; - resources.keys.push(keyOf(time)); + const key = keyOf(time); + const frame = takeSharedFrame(key, time, window); + if (!frame) return void missing.push(slot); + resources.urls[slot] = frame.url; + resources.held.push({ key, frame }); }); return missing; } @@ -166,7 +187,7 @@ function takeDecodedFrames( async function decodeFrames( sink: ThumbnailCanvasSink, times: AsyncIterable, - slots: { slot: number; key: string }[], + slots: { slot: number; key: string; decodedAt?: number }[], signal: AbortSignal, resources: DecodedResources, ): Promise { @@ -174,12 +195,13 @@ async function decodeFrames( for await (const wrapped of sink.canvasesAtTimestamps(times)) { throwIfAborted(signal); const target = slots[next++]; - if (!wrapped || !target) continue; + if (!wrapped || target?.decodedAt === undefined) continue; resources.canvases.add(wrapped.canvas); const blob = await canvasToBlob(wrapped.canvas); throwIfAborted(signal); - resources.urls[target.slot] = shareFrame(target.key, URL.createObjectURL(blob)); - resources.keys.push(target.key); + const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob)); + resources.urls[target.slot] = frame.url; + resources.held.push({ key: target.key, frame }); } throwIfAborted(signal); } @@ -215,12 +237,11 @@ export async function decodeVideoThumbnail( ): Promise { const fit = request.fit ?? "cover"; const keyOf = (time: number) => `${request.source}\u0000${fit}\u0000${time}`; - const resources: DecodedResources = { urls: [], keys: [], canvases: new Set() }; + const resources: DecodedResources = { urls: [], held: [], canvases: new Set() }; const known = sourceInfos.get(request.source); try { if (known) { - const { timestamps } = stripTimes(request, known, budgets); - if (takeDecodedFrames(timestamps, keyOf, resources).length === 0) { + if (takeDecodedFrames(stripTimes(request, known, budgets), keyOf, resources).length === 0) { throwIfAborted(signal); return loadedResult(resources, known.aspect, budgets); } @@ -263,22 +284,22 @@ async function decodeMissingFrames( throwIfAborted(signal); const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration }; sourceInfos.set(request.source, info); - const { sourceStart, duration, timestamps } = stripTimes(request, info, budgets); - const slots = takeDecodedFrames(timestamps, keyOf, resources).map((slot) => ({ - slot, - key: keyOf(timestamps[slot]!), - })); + const strip = stripTimes(request, info, budgets); + const { timestamps, window } = strip; + const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames( + strip, + keyOf, + resources, + ).map((slot) => ({ slot, key: keyOf(timestamps[slot]!) })); if (slots.length > 0) { const keys = new mediabunny.EncodedPacketSink(track); - const maxKeyframeLead = duration / Math.max(2, timestamps.length) / 2; async function* decodeTimesAtNearbyKeyframes() { - for (const { slot } of slots) { - const time = timestamps[slot]!; + for (const target of slots) { + const time = timestamps[target.slot]!; const key = await keys.getKeyPacket(time, { metadataOnly: true }); if (signal.aborted) return; - const near = - key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead; - yield near ? key.timestamp : time; + target.decodedAt = key && accepts(window, time, key.timestamp) ? key.timestamp : time; + yield target.decodedAt; } } const target = targetDimensions(info.aspect, budgets); From a3a89a47bd8c846b1b7a72bcde1505046af6b3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 20:30:49 -0400 Subject: [PATCH 05/11] feat(studio): show the clip's last frame at the end of every filmstrip Left-edge sampling never showed a clip's final slice, so an end card or a fade-out was missing from its strip. Each strip now also decodes the clip's last frame (at its own time, never an earlier keyframe), shared across zoom steps like the others, and the last tile shows it. The strip on screen is now held per clip and range, not per props: a clip crossing between on and off screen keeps it, and a clip narrowed to one frame lets it go, so widening again never re-decodes a dropped width. --- packages/core/src/beats/beatDetection.ts | 2 +- .../player/components/VideoThumbnail.test.tsx | 92 ++++++++++++++++--- .../src/player/components/VideoThumbnail.tsx | 11 ++- .../player/components/thumbnailUtils.test.ts | 11 +++ .../src/player/components/thumbnailUtils.ts | 13 ++- .../player/lib/thumbnailVideoDecoder.test.ts | 78 ++++++++++++---- .../src/player/lib/thumbnailVideoDecoder.ts | 24 ++++- 7 files changed, 186 insertions(+), 45 deletions(-) diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts index cd080722dd6..250136a211c 100644 --- a/packages/core/src/beats/beatDetection.ts +++ b/packages/core/src/beats/beatDetection.ts @@ -212,12 +212,12 @@ function gateBeatsBySilence( return { times, strengths, peak }; } -// fallow-ignore-next-line complexity export interface MusicAnalysisOptions { /** Awaited before each long stage, so a caller can hold the analysis while its page is busy. */ pause?: () => Promise; } +// fallow-ignore-next-line complexity export async function analyzeMusicFromBuffer( audioBuffer: AudioBuffer, { pause }: MusicAnalysisOptions = {}, diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index fb14e5c1026..dd02004f62b 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -30,25 +30,35 @@ afterEach(() => { // Registered after the reset above, so each test's strip unmounts before the cache is cleared. const harness = createHappyDomRootHarness(); +function clip( + videoSrc = "/api/projects/p/preview/assets/clip.mp4", + priority: "visible" | "overscan" = "visible", +) { + return ( + + ); +} + async function render(width = 0, height = 40) { Object.defineProperty(host, "clientWidth", { configurable: true, value: width }); Object.defineProperty(host, "clientHeight", { configurable: true, value: height }); const root = harness.mount(host); await act(async () => { - root.render( - , - ); + root.render(clip()); await Promise.resolve(); }); + return root; } +const shownSources = () => [...host.querySelectorAll("img")].map((img) => img.getAttribute("src")); + describe("VideoThumbnail", () => { it("does not acquire a thumbnail lease before the clip is measured", async () => { vi.mocked(decodeVideoThumbnail).mockResolvedValue({ @@ -91,7 +101,8 @@ describe("VideoThumbnail", () => { await render(300); const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src")); - expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]); + // Seven slices and the end frame: the last tile shows the end. + expect(tiles).toEqual(["blob:0", "blob:2", "blob:3", "blob:4", "blob:7"]); }); describe("on a 10-minute clip at full zoom", () => { @@ -305,6 +316,65 @@ describe("VideoThumbnail during a zoom", () => { expect(shown).not.toContain("blob:poster"); }); + it("keeps holding the strip when the clip moves between on and off screen mid-decode", async () => { + vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) => + frameCount === 1 + ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 } + : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 }, + ); + const root = await render(440); + vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {})); + await act(async () => { + reportResize(880, 40); + root.render(clip(undefined, "overscan")); + await Promise.resolve(); + }); + await rest(); + expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"])); + }); + + it("never holds one clip's strip for another while the other decodes", async () => { + vi.mocked(decodeVideoThumbnail).mockImplementation(({ source }) => + source.endsWith("clip.mp4") + ? Promise.resolve({ + value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, + weight: 2, + }) + : new Promise(() => {}), + ); + const root = await render(440); + expect(shownSources()).toContain("blob:a"); + await act(async () => { + root.render(clip("/api/projects/p/preview/assets/other.mp4")); + await Promise.resolve(); + }); + await rest(); + expect(shownSources()).toEqual([]); + }); + + it("lets go of the strip it held once the clip narrows to one frame", async () => { + vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) => + frameCount === 1 + ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 } + : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 }, + ); + await render(440); + await act(async () => { + reportResize(50, 40); + await Promise.resolve(); + }); + await rest(); + expect(new Set(shownSources())).toEqual(new Set(["blob:poster"])); + // Widening again shows the poster until the new width lands, not the strip it once held. + vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {})); + await act(async () => { + reportResize(880, 40); + await Promise.resolve(); + }); + await rest(); + expect(new Set(shownSources())).toEqual(new Set(["blob:poster"])); + }); + it("keeps its filmstrip while the zoom resizes it, and decodes the new width at rest", async () => { await render(440); vi.mocked(decodeVideoThumbnail).mockClear(); diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 09bbdf7c115..b452902bd4c 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -90,13 +90,14 @@ type VideoThumbnailRequest = ReturnType; function useVideoThumbnailSnapshot( poster: VideoThumbnailRequest | null, rich: VideoThumbnailRequest | null, - props: object, + media: string, ): ThumbnailSnapshot { const posterSnapshot = useThumbnailLease(poster); const richSnapshot = useThumbnailLease(rich); - const [shown, setShown] = useState({ props, request: rich }); - if (richSnapshot.status === "ready" && shown.request !== rich) setShown({ props, request: rich }); - const holdsShown = rich !== null && shown.props === props && shown.request !== rich; + const [shown, setShown] = useState({ media, request: rich }); + const settled = richSnapshot.status === "ready" || rich === null; + if (settled && shown.request !== rich) setShown({ media, request: rich }); + const holdsShown = rich !== null && shown.media === media && shown.request !== rich; const shownSnapshot = useThumbnailLease(holdsShown ? shown.request : null); return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot); } @@ -143,7 +144,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({ const snapshot = useVideoThumbnailSnapshot( measured ? posterRequest : null, measured && requestFrameCount > 1 ? richRequest : null, - requestProps, + posterRequest.key, ); const value = snapshot.status === "ready" ? snapshot.value : null; const urls = diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.ts index a17da47a88d..3c67e344afe 100644 --- a/packages/studio/src/player/components/thumbnailUtils.test.ts +++ b/packages/studio/src/player/components/thumbnailUtils.test.ts @@ -4,6 +4,7 @@ import { encodePreviewPath, resolveMediaPreviewUrl, quantizeThumbnailFrameCount, + thumbnailFrameForTile, } from "./thumbnailUtils"; import { MAX_VISIBLE_THUMBNAIL_FRAMES } from "../lib/timelineViewportBudgets"; @@ -52,6 +53,16 @@ describe("computeThumbnailStrip", () => { }); }); +describe("thumbnailFrameForTile", () => { + it("shows the clip's last frame in the last tile, and the slice under each other tile's centre", () => { + // 8 slices and the end frame. + expect([0, 1, 2].map((tile) => thumbnailFrameForTile(tile, 3, 9))).toEqual([1, 4, 8]); + // 2 slices and the end frame across 4 tiles. + expect([0, 1, 2, 3].map((tile) => thumbnailFrameForTile(tile, 4, 3))).toEqual([0, 0, 1, 2]); + expect(thumbnailFrameForTile(0, 1, 9)).toBe(4); + }); +}); + describe("quantizeThumbnailFrameCount", () => { it("uses doubling buckets and never exceeds the 4K geometry ceiling", () => { expect(quantizeThumbnailFrameCount(5)).toBe(8); diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts index 497bd8d2dfd..7e5fef625ce 100644 --- a/packages/studio/src/player/components/thumbnailUtils.ts +++ b/packages/studio/src/player/components/thumbnailUtils.ts @@ -17,19 +17,18 @@ export function quantizeThumbnailFrameCount(frameCount: number): number { } /** - * The decoded frame tile `index` of `tileCount` shows. Frame `i` of `frameCount` stands for its - * slice of the clip from its left-edge time (videoThumbnailTimestamps); a tile shows the frame - * whose slice holds the tile's centre. + * The decoded frame tile `index` of `tileCount` shows, of a strip's `frameCount`: the last tile + * shows the clip's last frame; the others the slice (videoThumbnailTimestamps) holding their centre. */ export function thumbnailFrameForTile( index: number, tileCount: number, frameCount: number, ): number { - return Math.min( - frameCount - 1, - Math.floor(((index + 0.5) * frameCount) / Math.max(1, tileCount)), - ); + if (frameCount < 2) return 0; + if (tileCount > 1 && index >= tileCount - 1) return frameCount - 1; + const slices = frameCount - 1; + return Math.min(slices - 1, Math.floor(((index + 0.5) * slices) / tileCount)); } /** diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index 12b5f8a8632..e96047e4132 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -39,6 +39,8 @@ function recordDecodes(decoded: number[][]): void { }); } +const endOf = (start: number, duration: number) => start + Math.max(0, duration - 0.001); + // Decoded frames are shared per source for the module's life, so each test decodes its own file. let sources = 0; let source = ""; @@ -47,7 +49,10 @@ beforeEach(() => { source = `/clip-${++sources}.mp4`; vi.clearAllMocks(); getKeyPacket.mockImplementation(async () => null); - vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two"); + vi.spyOn(URL, "createObjectURL") + .mockReturnValueOnce("blob:one") + .mockReturnValueOnce("blob:two") + .mockReturnValueOnce("blob:end"); vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {}); HTMLCanvasElement.prototype.toBlob = function toBlob(callback) { callback(new Blob(["frame"], { type: "image/jpeg" })); @@ -60,9 +65,9 @@ beforeEach(() => { }); describe("videoThumbnailTimestamps", () => { - it("uses the midpoint for a poster and each frame's left edge for a strip", () => { + it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => { expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]); - expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5]); + expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5, endOf(2, 6)]); }); it("nests each strip in the strip twice as long, and the poster in every strip", () => { @@ -86,15 +91,15 @@ describe("decodeVideoThumbnail", () => { new AbortController().signal, ); - expect(decoded).toEqual([[2, 5]]); + expect(decoded).toEqual([[2, 5, endOf(2, 6)]]); expect(result.value).toEqual({ kind: "filmstrip", - urls: ["blob:one", "blob:two"], + urls: ["blob:one", "blob:two", "blob:end"], aspect: 9 / 16, }); result.dispose?.(); result.dispose?.(); - expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3); expect(dispose).toHaveBeenCalledTimes(1); }); @@ -106,7 +111,7 @@ describe("decodeVideoThumbnail", () => { { source, sourceStart: 2, sourceRangeDuration: 8, frameCount: 4 }, new AbortController().signal, ); - expect(decoded).toEqual([[2, 3.5, 5.5, 7.5]]); + expect(decoded).toEqual([[2, 3.5, 5.5, 7.5, endOf(2, 8)]]); }); it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => { @@ -117,7 +122,7 @@ describe("decodeVideoThumbnail", () => { { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 4 }, new AbortController().signal, ); - expect(decoded).toEqual([[0, 2.5, 5, 7.5]]); + expect(decoded).toEqual([[0, 2.5, 5, 7.5, endOf(0, 10)]]); }); it("looks up each keyframe just before decoding it", async () => { @@ -136,7 +141,8 @@ describe("decodeVideoThumbnail", () => { { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 }, new AbortController().signal, ); - expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5"]); + const end = endOf(0, 10); + expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", `key ${end}`, `frame ${end}`]); }); it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => { @@ -174,7 +180,7 @@ describe("decodeVideoThumbnail", () => { { source, sourceStart: 5, sourceRangeDuration: 5, frameCount: 2 }, new AbortController().signal, ); - expect(decoded).toEqual([[5, 7.5]]); + expect(decoded).toEqual([[5, 7.5, endOf(5, 5)]]); }); it("reuses the frames a strip of the same source shows, so a zoom decodes only new times", async () => { @@ -184,17 +190,17 @@ describe("decodeVideoThumbnail", () => { const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; const signal = new AbortController().signal; const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); - // Twice the frames decodes only the new half; half the frames decodes nothing. + // Twice the frames decodes only the new half; half the frames decodes nothing. All share the end. const eight = await decodeVideoThumbnail({ ...range, frameCount: 8 }, signal); const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); expect(decoded).toEqual([ - [0, 2, 4, 6], + [0, 2, 4, 6, endOf(0, 8)], [1, 3, 5, 7], ]); const urls = (result: typeof four) => result.value.kind === "filmstrip" ? result.value.urls : []; expect(urls(eight).filter((_, i) => i % 2 === 0)).toEqual(urls(four)); - expect(urls(two)).toEqual([urls(four)[0], urls(four)[2]]); + expect(urls(two)).toEqual([urls(four)[0], urls(four)[2], urls(four)[4]]); for (const result of [four, eight, two]) result.dispose?.(); }); @@ -209,13 +215,53 @@ describe("decodeVideoThumbnail", () => { const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); expect(decoded).toEqual([ - [0, 2], + [0, 2, endOf(0, 8)], [2, 4, 6], ]); two.dispose?.(); four.dispose?.(); }); + it("returns a strip whose frames are all shared without opening the file again", async () => { + recordDecodes([]); + const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; + const signal = new AbortController().signal; + const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); + const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); + expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1); + four.dispose?.(); + two.dispose?.(); + }); + + it("keeps one copy of a frame two strips decode at once, and revokes the other", async () => { + vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); + // The first strip holds its first frame until the second is decoding too. + let release!: () => void; + const bothDecoding = new Promise((resolve) => (release = resolve)); + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + const first = canvasesAtTimestamps.mock.calls.length === 1; + for await (const _time of timestamps) { + if (first) await bothDecoding; + else release(); + yield { canvas: document.createElement("canvas") }; + } + }); + const range = { source, sourceStart: 0, sourceRangeDuration: 8 }; + const signal = new AbortController().signal; + const decodingTwo = decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); + await vi.waitFor(() => expect(canvasesAtTimestamps).toHaveBeenCalledOnce()); + const [two, four] = await Promise.all([ + decodingTwo, + decodeVideoThumbnail({ ...range, frameCount: 4 }, signal), + ]); + // Both decoded the frames at 0 s, 4 s and the end; the second copy of each goes at once. + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3); + two.dispose?.(); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3); + four.dispose?.(); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(8); + }); + it("revokes a shared frame only once no strip shows it", async () => { vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); recordDecodes([]); @@ -224,10 +270,10 @@ describe("decodeVideoThumbnail", () => { const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal); const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal); four.dispose?.(); - // The two frames the 2-frame strip still shows stay; the other two go. + // The three frames the 2-slice strip still shows stay; the other two go. expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2); two.dispose?.(); - expect(URL.revokeObjectURL).toHaveBeenCalledTimes(4); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(5); }); it("releases input and degrades when the source has no video track", async () => { diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 3c1d2a45cca..a6de6a4d100 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -10,9 +10,12 @@ export interface VideoThumbnailDecodeRequest { } /** - * The times a strip of `frameCount` frames shows: each frame's left edge, so a strip twice as long - * holds every frame of this one and a zoom decodes only the new half. A poster is the midpoint. + * The times a strip of `frameCount` slices shows: each slice's left edge, so a strip twice as long + * holds every frame of this one, then the clip's last frame. A poster is the midpoint. */ +// Inside the last frame: a time at the very end of a file can decode to nothing. +const END_FRAME_LEAD_S = 0.001; + export function videoThumbnailTimestamps( start: number, duration: number, @@ -22,7 +25,11 @@ export function videoThumbnailTimestamps( const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0); const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1); if (count === 1) return [safeStart + safeDuration / 2]; - return Array.from({ length: count }, (_, index) => safeStart + (safeDuration * index) / count); + const edges = Array.from( + { length: count }, + (_, index) => safeStart + (safeDuration * index) / count, + ); + return [...edges, safeStart + Math.max(0, safeDuration - END_FRAME_LEAD_S)]; } /** A decoded frame, shared by every strip of its source that would accept it for a time. */ @@ -41,13 +48,15 @@ interface HeldFrame { interface FrameWindow { sourceStart: number; lead: number; + /** The clip's last frame, shown as decoded at its own time, never from an earlier keyframe. */ + end: number; } const sharedFrames = new Map(); const sourceInfos = new Map(); const accepts = (window: FrameWindow, time: number, decodedAt: number) => - decodedAt >= window.sourceStart && time - decodedAt <= window.lead; + decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead); function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined { const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt)); @@ -163,7 +172,12 @@ function stripTimes( duration, Math.min(request.frameCount, budgets.richPreviewFrameCount), ); - const window: FrameWindow = { sourceStart, lead: duration / Math.max(2, timestamps.length) / 2 }; + const slices = Math.max(1, timestamps.length - 1); + const window: FrameWindow = { + sourceStart, + lead: duration / Math.max(2, slices) / 2, + end: timestamps.length > 1 ? timestamps.at(-1)! : Infinity, + }; return { window, timestamps }; } From c773581b0dbd9eb1089cbe365ecc2e766a9c8703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 21:09:56 -0400 Subject: [PATCH 06/11] docs(studio): keep the filmstrip files at or under main's comment share The comment ratchet allows no file a higher share of comment lines than on main; the added explanations move to the PR description. --- .../studio/src/hooks/useMusicBeatAnalysis.ts | 1 - .../src/player/components/AudioWaveform.tsx | 4 ---- .../src/player/components/VideoThumbnail.tsx | 4 ---- .../src/player/components/thumbnailUtils.ts | 1 - .../src/player/lib/thumbnailVideoDecoder.ts | 23 +++---------------- 5 files changed, 3 insertions(+), 30 deletions(-) diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.ts index 10831949309..e71339ba996 100644 --- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts +++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts @@ -80,7 +80,6 @@ async function loadBeatAnalysis( ): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> { let promise = analysisCache.get(musicSrc); if (!promise) { - // Each long stage of the analysis waits for the timeline to rest, so it never lands mid-zoom. promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle }); cacheAnalysis(musicSrc, promise); } diff --git a/packages/studio/src/player/components/AudioWaveform.tsx b/packages/studio/src/player/components/AudioWaveform.tsx index 32155125d45..ef5b2f55140 100644 --- a/packages/studio/src/player/components/AudioWaveform.tsx +++ b/packages/studio/src/player/components/AudioWaveform.tsx @@ -158,8 +158,6 @@ async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise void>(); let nearScreen: IntersectionObserver | null = null; -/** Whether `element` is on or near the screen, kept current by one observer for all waveforms. */ function useNearScreen(element: Element | null): boolean { const [near, setNear] = useState(() => typeof IntersectionObserver === "undefined"); useEffect(() => { @@ -247,7 +244,6 @@ export const AudioWaveform = memo(function AudioWaveform({ span, ); }, [fades, from, muted, peaks, to, trimEndFraction, trimStartFraction]); - // A waveform off screen draws when it comes near, not at every zoom that resizes it. const near = useNearScreen(root); useEffect(() => { if (near) draw(); diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index b452902bd4c..a52c5d5bd9f 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -83,10 +83,6 @@ function selectThumbnailSnapshot( type VideoThumbnailRequest = ReturnType; -/** - * The strip to show: the requested width once ready, else the strip already on screen, kept leased - * while a zoom's new width decodes, else the poster. - */ function useVideoThumbnailSnapshot( poster: VideoThumbnailRequest | null, rich: VideoThumbnailRequest | null, diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts index 7e5fef625ce..65290e44feb 100644 --- a/packages/studio/src/player/components/thumbnailUtils.ts +++ b/packages/studio/src/player/components/thumbnailUtils.ts @@ -11,7 +11,6 @@ export interface ThumbnailStripLayout { /** Quantize request identities so a pixel-by-pixel resize does not thrash the cache. */ export function quantizeThumbnailFrameCount(frameCount: number): number { const safeCount = Math.max(1, Number.isFinite(frameCount) ? Math.ceil(frameCount) : 1); - // Capped at a power of two too, so the top step still holds every frame of the one below it. const cap = 2 ** Math.floor(Math.log2(MAX_VISIBLE_THUMBNAIL_FRAMES)); return Math.min(cap, 2 ** Math.ceil(Math.log2(safeCount))); } diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index a6de6a4d100..3b51a85e5d6 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -9,12 +9,7 @@ export interface VideoThumbnailDecodeRequest { fit?: "contain" | "cover"; } -/** - * The times a strip of `frameCount` slices shows: each slice's left edge, so a strip twice as long - * holds every frame of this one, then the clip's last frame. A poster is the midpoint. - */ -// Inside the last frame: a time at the very end of a file can decode to nothing. -const END_FRAME_LEAD_S = 0.001; +const LAST_FRAME_INSET_S = 0.001; export function videoThumbnailTimestamps( start: number, @@ -29,10 +24,9 @@ export function videoThumbnailTimestamps( { length: count }, (_, index) => safeStart + (safeDuration * index) / count, ); - return [...edges, safeStart + Math.max(0, safeDuration - END_FRAME_LEAD_S)]; + return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)]; } -/** A decoded frame, shared by every strip of its source that would accept it for a time. */ interface SharedFrame { url: string; decodedAt: number; @@ -44,11 +38,9 @@ interface HeldFrame { frame: SharedFrame; } -/** A strip takes a frame decoded inside its range, at most `lead` before the time it shows. */ interface FrameWindow { sourceStart: number; lead: number; - /** The clip's last frame, shown as decoded at its own time, never from an earlier keyframe. */ end: number; } @@ -64,7 +56,6 @@ function takeSharedFrame(key: string, time: number, window: FrameWindow): Shared return frame; } -/** Shares a newly decoded frame, or the same frame another strip decoded meanwhile. */ function shareFrame(key: string, decodedAt: number, url: string): SharedFrame { const frames = sharedFrames.get(key) ?? []; const existing = frames.find((shared) => shared.decodedAt === decodedAt); @@ -102,9 +93,7 @@ async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promis } interface DecodedResources { - /** One frame per slot of the strip; a slot the source could not decode stays empty. */ urls: (string | undefined)[]; - /** The shared frames this strip holds, released when the strip is. */ held: HeldFrame[]; canvases: Set; } @@ -115,7 +104,6 @@ interface ThumbnailCanvasSink { ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>; } -/** What a strip of a source needs before it can name its frames. */ interface SourceInfo { aspect: number; metadataDuration: number | null; @@ -149,7 +137,6 @@ function targetDimensions( }; } -/** The strip's frame times, clamped to what the file holds, and the frames it accepts for them. */ function stripTimes( request: VideoThumbnailDecodeRequest, info: SourceInfo, @@ -181,7 +168,6 @@ function stripTimes( return { window, timestamps }; } -/** Holds every frame of the strip already decoded; returns the slots still to decode. */ function takeDecodedFrames( { timestamps, window }: ReturnType, keyOf: (time: number) => string, @@ -240,10 +226,7 @@ function loadedResult( }; } -/** - * Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. Frames another strip - * of the source already shows are reused, so a zoom decodes only the frames that are new. - */ +/** Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. */ export async function decodeVideoThumbnail( request: VideoThumbnailDecodeRequest, signal: AbortSignal, From b278809179aab414afa147f498472b1445a9505a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 22:23:09 -0400 Subject: [PATCH 07/11] fix(studio): keep a held filmstrip leased so a full cache cannot evict it mid-zoom The hold leased the strip on screen only once a new width was requested, so for one commit it had no lease; with the cache over budget it was evicted, the poster showed, and the old width decoded again. The strip on screen is now leased for as long as it is the clip's, and shown only while held. A frame that fails to decode takes the frame before it, so every slot keeps its time and the last tile still shows the end. --- .../player/components/VideoThumbnail.test.tsx | 50 +++++++++++++++++++ .../src/player/components/VideoThumbnail.tsx | 6 ++- .../player/lib/thumbnailVideoDecoder.test.ts | 43 ++++++++++++++++ .../src/player/lib/thumbnailVideoDecoder.ts | 14 ++++-- 4 files changed, 107 insertions(+), 6 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index dd02004f62b..3324a2c1d8e 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -333,6 +333,56 @@ describe("VideoThumbnail during a zoom", () => { expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"])); }); + it("holds the strip, and decodes only the new width, when the cache is over its budget", async () => { + const disposed: number[] = []; + vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) => + frameCount === 1 + ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 } + : { + value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, + weight: 2, + dispose: () => disposed.push(frameCount), + }, + ); + await render(440); + const held = vi.mocked(decodeVideoThumbnail).mock.calls.at(-1)![0].frameCount; + // Fill the project's cache past its entry budget, so a strip without a lease is evicted. + await act(async () => { + for (let i = 0; i < 100; i++) { + const lease = thumbnailScheduler.acquire( + { + key: `filler-${i}`, + projectId: "p", + sessionEpoch: 1, + kind: "image", + priority: "visible", + load: async () => ({ + value: { kind: "image", url: `blob:f${i}`, aspect: 1 }, + weight: 1, + }), + }, + () => {}, + ); + for (let tick = 0; tick < 5; tick++) await Promise.resolve(); + lease.release(); + } + }); + const decodes = vi.mocked(decodeVideoThumbnail).mock.calls.length; + vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {})); + await act(async () => { + reportResize(880, 40); + await Promise.resolve(); + }); + await rest(); + const newWidths = vi + .mocked(decodeVideoThumbnail) + .mock.calls.slice(decodes) + .map(([r]) => r.frameCount); + expect(newWidths).not.toContain(held); + expect(disposed).toEqual([]); + expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"])); + }); + it("never holds one clip's strip for another while the other decodes", async () => { vi.mocked(decodeVideoThumbnail).mockImplementation(({ source }) => source.endsWith("clip.mp4") diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index a52c5d5bd9f..5beb59a7ba3 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -83,6 +83,8 @@ function selectThumbnailSnapshot( type VideoThumbnailRequest = ReturnType; +const IDLE: ThumbnailSnapshot = { status: "idle" }; + function useVideoThumbnailSnapshot( poster: VideoThumbnailRequest | null, rich: VideoThumbnailRequest | null, @@ -93,9 +95,9 @@ function useVideoThumbnailSnapshot( const [shown, setShown] = useState({ media, request: rich }); const settled = richSnapshot.status === "ready" || rich === null; if (settled && shown.request !== rich) setShown({ media, request: rich }); + const shownSnapshot = useThumbnailLease(shown.media === media ? shown.request : null); const holdsShown = rich !== null && shown.media === media && shown.request !== rich; - const shownSnapshot = useThumbnailLease(holdsShown ? shown.request : null); - return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot); + return selectThumbnailSnapshot(posterSnapshot, richSnapshot, holdsShown ? shownSnapshot : IDLE); } /** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */ diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index e96047e4132..839162c6d33 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -262,6 +262,49 @@ describe("decodeVideoThumbnail", () => { expect(URL.revokeObjectURL).toHaveBeenCalledTimes(8); }); + it("fills a frame that failed to decode from the one before it, so tiles stay on their times", async () => { + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + let slot = 0; + for await (const _time of timestamps) { + yield slot++ === 2 ? null : { canvas: document.createElement("canvas") }; + } + }); + const result = await decodeVideoThumbnail( + { source, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 }, + new AbortController().signal, + ); + expect(result.value).toMatchObject({ + kind: "filmstrip", + urls: ["blob:one", "blob:two", "blob:two"], + }); + result.dispose?.(); + }); + + it("snaps a poster up to a quarter of its range and a strip frame up to half a slice", async () => { + const decoded: number[][] = []; + recordDecodes(decoded); + const signal = new AbortController().signal; + getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.4 })); + await decodeVideoThumbnail( + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 }, + signal, + ); + getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.6 })); + await decodeVideoThumbnail( + { source: `${source}?b`, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 }, + signal, + ); + // Four slices of 8 s: a keyframe 0.9 s early snaps, one 1.1 s early does not. + getKeyPacket.mockImplementation(async (time) => ({ + timestamp: time - (time === 4 ? 0.9 : 1.1), + })); + await decodeVideoThumbnail( + { source: `${source}?c`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 4 }, + signal, + ); + expect(decoded).toEqual([[5 - 2.4], [5], [0, 2, 4 - 0.9, 6, endOf(0, 8)]]); + }); + it("revokes a shared frame only once no strip shows it", async () => { vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); recordDecodes([]); diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 3b51a85e5d6..74f68171763 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -210,10 +210,16 @@ function loadedResult( resources: DecodedResources, aspect: number, budgets: Readonly, + frameCount = resources.urls.length, ): ThumbnailLoadedResult { - const urls = resources.urls.filter((url): url is string => url !== undefined); - const firstUrl = urls[0]; + const decoded = resources.urls.filter((url): url is string => url !== undefined); + const firstUrl = decoded[0]; if (!firstUrl) throw new Error("Video source returned no thumbnail frames"); + let nearest = firstUrl; + const urls = Array.from( + { length: frameCount }, + (_, slot) => (nearest = resources.urls[slot] ?? nearest), + ); const { width, height } = targetDimensions(aspect, budgets); const value: ThumbnailValue = urls.length === 1 @@ -221,7 +227,7 @@ function loadedResult( : { kind: "filmstrip", urls, aspect }; return { value, - weight: width * height * 4 * urls.length, + weight: width * height * 4 * decoded.length, dispose: () => releaseDecodedResources(resources), }; } @@ -308,7 +314,7 @@ async function decodeMissingFrames( }); await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources); } - return loadedResult(resources, info.aspect, budgets); + return loadedResult(resources, info.aspect, budgets, timestamps.length); } finally { input.dispose(); } From 6910922aa2228e48a9586776abbca2d2aebef22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 6 Oct 2026 22:47:06 -0400 Subject: [PATCH 08/11] refactor(studio): one owner for which clip a held filmstrip belongs to The hold checked the clip twice, in its lease and again before showing it; the lease alone decides now, and the cross-clip test pins it. A test now also pins that a filled frame is not charged to the cache. --- .../studio/src/player/components/VideoThumbnail.tsx | 5 +---- .../src/player/lib/thumbnailVideoDecoder.test.ts | 11 ++++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 5beb59a7ba3..79330a6df21 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -83,8 +83,6 @@ function selectThumbnailSnapshot( type VideoThumbnailRequest = ReturnType; -const IDLE: ThumbnailSnapshot = { status: "idle" }; - function useVideoThumbnailSnapshot( poster: VideoThumbnailRequest | null, rich: VideoThumbnailRequest | null, @@ -96,8 +94,7 @@ function useVideoThumbnailSnapshot( const settled = richSnapshot.status === "ready" || rich === null; if (settled && shown.request !== rich) setShown({ media, request: rich }); const shownSnapshot = useThumbnailLease(shown.media === media ? shown.request : null); - const holdsShown = rich !== null && shown.media === media && shown.request !== rich; - return selectThumbnailSnapshot(posterSnapshot, richSnapshot, holdsShown ? shownSnapshot : IDLE); + return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot); } /** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */ diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index 839162c6d33..89244895893 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -269,15 +269,24 @@ describe("decodeVideoThumbnail", () => { yield slot++ === 2 ? null : { canvas: document.createElement("canvas") }; } }); + const signal = new AbortController().signal; const result = await decodeVideoThumbnail( { source, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 }, - new AbortController().signal, + signal, ); expect(result.value).toMatchObject({ kind: "filmstrip", urls: ["blob:one", "blob:two", "blob:two"], }); + // The cache is charged for the two decoded frames, not the filled copy. + recordDecodes([]); + const whole = await decodeVideoThumbnail( + { source: `${source}?whole`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 }, + signal, + ); + expect(result.weight * 3).toBe(whole.weight * 2); result.dispose?.(); + whole.dispose?.(); }); it("snaps a poster up to a quarter of its range and a strip frame up to half a slice", async () => { From 2065b15fa91605fb67c4048c09b8b441015db9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 7 Oct 2026 01:23:26 -0400 Subject: [PATCH 09/11] fix(core): hold the tempo detector until idle after a slow import --- packages/core/src/beats/beatDetection.test.ts | 43 +++++++++++++++++++ packages/core/src/beats/beatDetection.ts | 3 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.ts index 3037facc7a2..f0a6ec72fc4 100644 --- a/packages/core/src/beats/beatDetection.test.ts +++ b/packages/core/src/beats/beatDetection.test.ts @@ -64,4 +64,47 @@ describe("analyzeMusicFromBuffer", () => { } await expect(analysis).resolves.toMatchObject({ sampleRate }); }); + + it("rechecks pause after a slow tempo-detector import", async () => { + let releaseImport!: () => void; + const importGate = new Promise((resolve) => (releaseImport = resolve)); + let importStarted = false; + let moving = false; + const movingAtDetect: boolean[] = []; + const detect = vi.fn(() => { + movingAtDetect.push(moving); + return 120; + }); + vi.resetModules(); + vi.doMock("bpm-detective", async () => { + importStarted = true; + await importGate; + return { default: detect }; + }); + try { + const { analyzeMusicFromBuffer: analyze } = await import("./beatDetection"); + const sampleRate = 8000; + const audio = { + getChannelData: () => new Float32Array(sampleRate * 2), + sampleRate, + duration: 2, + } as unknown as AudioBuffer; + const waiters: (() => void)[] = []; + const pause = () => + moving ? new Promise((resolve) => waiters.push(resolve)) : Promise.resolve(); + const analysis = analyze(audio, { pause }); + await vi.waitFor(() => expect(importStarted).toBe(true)); + moving = true; + releaseImport(); + await vi.waitFor(() => expect(waiters).toHaveLength(1)); + expect(detect).not.toHaveBeenCalled(); + moving = false; + waiters.shift()!(); + await analysis; + expect(movingAtDetect).toEqual([false]); + } finally { + vi.doUnmock("bpm-detective"); + vi.resetModules(); + } + }); }); diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts index 250136a211c..af6e41f0839 100644 --- a/packages/core/src/beats/beatDetection.ts +++ b/packages/core/src/beats/beatDetection.ts @@ -231,9 +231,10 @@ export async function analyzeMusicFromBuffer( const onsetBpm = computeBpmFromBeats(rawBeats); let detectiveBpm: number | null = null; + // Pause after the import: a cold import can resolve mid-interaction. + const detect = await loadBpmDetective(); await pause?.(); try { - const detect = await loadBpmDetective(); if (detect) detectiveBpm = detect(audioBuffer); } catch { // Not enough peaks or browser context unavailable From f800a70fc9f6723f1b59d6623e9f894be33803d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 7 Oct 2026 02:28:47 -0400 Subject: [PATCH 10/11] fix(studio): scope shared video frames to timeline sessions --- .../player/components/VideoThumbnail.test.tsx | 2 +- .../src/player/components/VideoThumbnail.tsx | 1 + .../player/lib/thumbnailVideoDecoder.test.ts | 43 +++++++++++++++++++ .../src/player/lib/thumbnailVideoDecoder.ts | 17 +++++--- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index 3324a2c1d8e..d9103555f45 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -81,7 +81,7 @@ describe("VideoThumbnail", () => { expect(decodeVideoThumbnail).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ frameCount: 1 }), + expect.objectContaining({ frameCount: 1, contentVersion: "project=p&session=1" }), expect.any(AbortSignal), ); expect(decodeVideoThumbnail).toHaveBeenCalledWith( diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 79330a6df21..dcb0eb776b6 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -59,6 +59,7 @@ function createVideoThumbnailRequest( decodeVideoThumbnail( { source: videoSrc, + contentVersion: createThumbnailKey({ project: projectId, session: sessionEpoch }), sourceStart, sourceRangeDuration: sourceRangeDuration ?? duration, frameCount, diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index 89244895893..cbac2487a4f 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -233,6 +233,49 @@ describe("decodeVideoThumbnail", () => { two.dispose?.(); }); + it("reopens the same asset after a project round trip and keeps new-session frames separate", async () => { + let frame = 0; + vi.mocked(URL.createObjectURL) + .mockReset() + .mockImplementation(() => `blob:version-${++frame}`); + const decoded: number[][] = []; + recordDecodes(decoded); + const signal = new AbortController().signal; + const original = await decodeVideoThumbnail( + { source, contentVersion: "A:1", frameCount: 1 }, + signal, + ); + const other = await decodeVideoThumbnail( + { source: `${source}?B`, contentVersion: "B:2", frameCount: 1 }, + signal, + ); + input.getPrimaryVideoTrack.mockResolvedValue({ + getDisplayWidth: vi.fn(async () => 1280), + getDisplayHeight: vi.fn(async () => 360), + getDurationFromMetadata: vi.fn(async () => 10), + }); + const reopened = await decodeVideoThumbnail( + { source, contentVersion: "A:3", frameCount: 1 }, + signal, + ); + const shared = await decodeVideoThumbnail( + { source, contentVersion: "A:3", frameCount: 1 }, + signal, + ); + expect(reopened.value).toEqual({ kind: "image", url: "blob:version-3", aspect: 32 / 9 }); + expect(shared.value).toEqual(reopened.value); + expect(decoded).toEqual([[5], [5], [5]]); + expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(3); + original.dispose?.(); + expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:version-1"); + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3"); + other.dispose?.(); + reopened.dispose?.(); + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3"); + shared.dispose?.(); + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3); + }); + it("keeps one copy of a frame two strips decode at once, and revokes the other", async () => { vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`); // The first strip holds its first frame until the second is decoding too. diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 74f68171763..0bf417dd56c 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -1,8 +1,13 @@ import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets"; -import type { ThumbnailLoadedResult, ThumbnailValue } from "./thumbnailScheduler"; +import { + createThumbnailKey, + type ThumbnailLoadedResult, + type ThumbnailValue, +} from "./thumbnailScheduler"; export interface VideoThumbnailDecodeRequest { source: string; + contentVersion?: string; sourceStart?: number; sourceRangeDuration?: number; frameCount: number; @@ -45,7 +50,7 @@ interface FrameWindow { } const sharedFrames = new Map(); -const sourceInfos = new Map(); +const sourceInfos = new Map(); const accepts = (window: FrameWindow, time: number, decodedAt: number) => decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead); @@ -239,9 +244,11 @@ export async function decodeVideoThumbnail( budgets: Readonly = TIMELINE_VIEWPORT_BUDGETS, ): Promise { const fit = request.fit ?? "cover"; - const keyOf = (time: number) => `${request.source}\u0000${fit}\u0000${time}`; + const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion }); + const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`; const resources: DecodedResources = { urls: [], held: [], canvases: new Set() }; - const known = sourceInfos.get(request.source); + const entry = sourceInfos.get(request.source); + const known = entry?.version === request.contentVersion ? entry?.info : undefined; try { if (known) { if (takeDecodedFrames(stripTimes(request, known, budgets), keyOf, resources).length === 0) { @@ -286,7 +293,7 @@ async function decodeMissingFrames( const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true }); throwIfAborted(signal); const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration }; - sourceInfos.set(request.source, info); + sourceInfos.set(request.source, { version: request.contentVersion, info }); const strip = stripTimes(request, info, budgets); const { timestamps, window } = strip; const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames( From 341337d5fa5d2a50d68d32066d28467d48db2bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 7 Oct 2026 03:06:13 -0400 Subject: [PATCH 11/11] chore(core): keep beat analysis comments within the ratchet --- packages/core/src/beats/beatDetection.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts index af6e41f0839..db2eb2399a7 100644 --- a/packages/core/src/beats/beatDetection.ts +++ b/packages/core/src/beats/beatDetection.ts @@ -213,7 +213,6 @@ function gateBeatsBySilence( } export interface MusicAnalysisOptions { - /** Awaited before each long stage, so a caller can hold the analysis while its page is busy. */ pause?: () => Promise; } @@ -231,7 +230,6 @@ export async function analyzeMusicFromBuffer( const onsetBpm = computeBpmFromBeats(rawBeats); let detectiveBpm: number | null = null; - // Pause after the import: a cold import can resolve mid-interaction. const detect = await loadBpmDetective(); await pause?.(); try {