perf(studio): zooming a long timeline re-decodes and redraws less

packages/core/src/beats/beatDetection.test.ts

@@ -1,6 +1,6 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 
-import { loadBpmDetective } from "./beatDetection";
+import { analyzeMusicFromBuffer, loadBpmDetective } from "./beatDetection";
 
 describe("loadBpmDetective", () => {
   it("resolves to null when the import is unavailable", async () => {
@@ -39,3 +39,72 @@ describe("loadBpmDetective", () => {
     expect(detect).toBe(fn);
   });
 });
+
+describe("analyzeMusicFromBuffer", () => {
+  it("waits on the caller's pause before each long stage", async () => {
+    const sampleRate = 8000;
+    const channel = new Float32Array(sampleRate * 2);
+    for (let beat = 0; beat < 4; beat++) channel.fill(0.9, beat * 4000, beat * 4000 + 80);
+    const audio = {
+      getChannelData: () => channel,
+      sampleRate,
+      duration: 2,
+    } as unknown as AudioBuffer;
+    const held: (() => void)[] = [];
+    const pause = () => new Promise<void>((resolve) => held.push(resolve));
+    let done = false;
+    const analysis = analyzeMusicFromBuffer(audio, { pause }).then((result) => {
+      done = true;
+      return result;
+    });
+    for (let stage = 1; stage <= 3; stage++) {
+      await vi.waitFor(() => expect(held).toHaveLength(stage));
+      expect(done).toBe(false);
+      held[stage - 1]!();
+    }
+    await expect(analysis).resolves.toMatchObject({ sampleRate });
+  });
+
+  it("rechecks pause after a slow tempo-detector import", async () => {
+    let releaseImport!: () => void;
+    const importGate = new Promise<void>((resolve) => (releaseImport = resolve));
+    let importStarted = false;
+    let moving = false;
+    const movingAtDetect: boolean[] = [];
+    const detect = vi.fn(() => {
+      movingAtDetect.push(moving);
+      return 120;
+    });
+    vi.resetModules();
+    vi.doMock("bpm-detective", async () => {
+      importStarted = true;
+      await importGate;
+      return { default: detect };
+    });
+    try {
+      const { analyzeMusicFromBuffer: analyze } = await import("./beatDetection");
+      const sampleRate = 8000;
+      const audio = {
+        getChannelData: () => new Float32Array(sampleRate * 2),
+        sampleRate,
+        duration: 2,
+      } as unknown as AudioBuffer;
+      const waiters: (() => void)[] = [];
+      const pause = () =>
+        moving ? new Promise<void>((resolve) => waiters.push(resolve)) : Promise.resolve();
+      const analysis = analyze(audio, { pause });
+      await vi.waitFor(() => expect(importStarted).toBe(true));
+      moving = true;
+      releaseImport();
+      await vi.waitFor(() => expect(waiters).toHaveLength(1));
+      expect(detect).not.toHaveBeenCalled();
+      moving = false;
+      waiters.shift()!();
+      await analysis;
+      expect(movingAtDetect).toEqual([false]);
+    } finally {
+      vi.doUnmock("bpm-detective");
+      vi.resetModules();
+    }
+  });
+});

packages/core/src/beats/beatDetection.ts

@@ -212,18 +212,27 @@ function gateBeatsBySilence(
   return { times, strengths, peak };
 }
 
+export interface MusicAnalysisOptions {
+  pause?: () => Promise<void>;
+}
+
 // fallow-ignore-next-line complexity
-export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<MusicBeatAnalysis> {
+export async function analyzeMusicFromBuffer(
+  audioBuffer: AudioBuffer,
+  { pause }: MusicAnalysisOptions = {},
+): Promise<MusicBeatAnalysis> {
   const channelData = audioBuffer.getChannelData(0);
   const sampleRate = audioBuffer.sampleRate;
   const duration = audioBuffer.duration;
 
+  await pause?.();
   const rawBeats = await detectBeats(audioBuffer);
   const onsetBpm = computeBpmFromBeats(rawBeats);
 
   let detectiveBpm: number | null = null;
+  const detect = await loadBpmDetective();
+  await pause?.();
   try {
-    const detect = await loadBpmDetective();
     if (detect) detectiveBpm = detect(audioBuffer);
   } catch {
     // Not enough peaks or browser context unavailable
@@ -261,6 +270,7 @@ export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<
     regularizeBpm = detectiveBpm;
   }
 
+  await pause?.();
   const gridBeats =
     regularizeBpm !== null ? regularizeBeats(rawBeats, regularizeBpm, duration) : rawBeats;
   const gated = gateBeatsBySilence(gridBeats, channelData, sampleRate);
@@ -288,13 +298,16 @@ export async function detectBeatsFromUrl(url: string): Promise<number[]> {
   }
 }
 
-export async function analyzeMusicFromUrl(url: string): Promise<MusicBeatAnalysis> {
+export async function analyzeMusicFromUrl(
+  url: string,
+  options: MusicAnalysisOptions = {},
+): Promise<MusicBeatAnalysis> {
   const audioContext = new AudioContext();
   try {
     const response = await fetch(url);
     const arrayBuffer = await response.arrayBuffer();
     const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-    return analyzeMusicFromBuffer(audioBuffer);
+    return analyzeMusicFromBuffer(audioBuffer, options);
   } finally {
     await audioContext.close();
   }

packages/studio/src/hooks/useMusicBeatAnalysis.ts

@@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef } from "react";
 import { usePlayerStore } from "../player/store/playerStore";
 import { resolveBeatSourceTrack } from "../utils/timelineInspector";
 import { analyzeMusicFromUrl } from "@hyperframes/core/beats";
+import { whenTimelineIdle } from "../player/components/timelineMotion";
 import { useFileManagerContextOptional } from "../contexts/FileManagerContext";
 import { mergeUserBeats } from "../utils/beatEditing";
 import { getTimelineElementIndexes } from "../player/lib/timelineElementIndexes";
@@ -79,7 +80,7 @@ async function loadBeatAnalysis(
 ): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {
   let promise = analysisCache.get(musicSrc);
   if (!promise) {
-    promise = analyzeMusicFromUrl(musicSrc);
+    promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });
     cacheAnalysis(musicSrc, promise);
   }
   try {

packages/studio/src/player/components/AudioWaveform.test.tsx

@@ -11,6 +11,29 @@ vi.mock("../../hooks/useThumbnailLease", () => ({
   useThumbnailLease: leaseSpy,
 }));
 
+// One observer serves every waveform; it reports each one near as it is watched, unless a test
+// reports by hand.
+const nearScreen = {
+  auto: true,
+  watched: [] as Element[],
+  report: (_target: Element, _near: boolean) => {},
+};
+globalThis.IntersectionObserver = class {
+  constructor(callback: IntersectionObserverCallback) {
+    nearScreen.report = (target, near) =>
+      callback(
+        [{ target, isIntersecting: near } as IntersectionObserverEntry],
+        this as unknown as IntersectionObserver,
+      );
+  }
+  observe(target: Element) {
+    nearScreen.watched.push(target);
+    if (nearScreen.auto) nearScreen.report(target, true);
+  }
+  unobserve() {}
+  disconnect() {}
+} as unknown as typeof IntersectionObserver;
+
 const EMPTY_STRIP = { width: 0, height: 0, inViewStart: 0, inViewEnd: 0 };
 const watchGap = vi.hoisted(() => vi.fn());
 const strip = vi.hoisted(() => ({
@@ -52,6 +75,7 @@ afterEach(() => {
   leaseSpy.mockReset();
   leaseSpy.mockImplementation(() => ({ status: "loading" as const }));
   strip.size = EMPTY_STRIP;
+  nearScreen.auto = true;
   watchGap.mockClear();
   document.body.innerHTML = "";
 });
@@ -141,6 +165,23 @@ describe("AudioWaveform", () => {
     expect(watchGap.mock.calls.filter(([gap]) => gap)).toEqual([]);
   });
 
+  it("leaves a waveform off screen undrawn through a zoom, and draws it once it comes near", () => {
+    nearScreen.auto = false;
+    const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
+    setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });
+    renderReadyWaveform();
+    const root = nearScreen.watched.at(-1)!;
+    expect(getContext).not.toHaveBeenCalled();
+    act(() => nearScreen.report(root, true));
+    const drawn = getContext.mock.calls.length;
+    expect(drawn).toBeGreaterThan(0);
+    act(() => nearScreen.report(root, false));
+    setStrip({ width: 2000, height: 40, inViewStart: 0, inViewEnd: 2000 });
+    expect(getContext.mock.calls.length).toBe(drawn);
+    act(() => nearScreen.report(root, true));
+    expect(getContext.mock.calls.length).toBe(drawn + 1);
+  });
+
   it("redraws a short clip, drawn whole, when a zoom changes its width", () => {
     const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
     setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });

packages/studio/src/player/components/AudioWaveform.tsx

@@ -1,4 +1,4 @@
-import { memo, useCallback, useContext, useEffect, useMemo, useRef } from "react";
+import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
 import { fadeGain } from "@hyperframes/core/audio-fade";
 import { useThumbnailLease } from "../../hooks/useThumbnailLease";
 import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";
@@ -158,6 +158,32 @@ async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise<nu
 }
 
 /** Bounded waveform subscriber; cache, cancellation and dedupe live in one scheduler. */
+const NEAR_SCREEN: IntersectionObserverInit & { scrollMargin: string } = {
+  rootMargin: "50% 100%",
+  scrollMargin: "50% 100%",
+};
+const nearScreenListeners = new Map<Element, (near: boolean) => void>();
+let nearScreen: IntersectionObserver | null = null;
+
+function useNearScreen(element: Element | null): boolean {
+  const [near, setNear] = useState(() => typeof IntersectionObserver === "undefined");
+  useEffect(() => {
+    if (!element || typeof IntersectionObserver === "undefined") return;
+    nearScreen ??= new IntersectionObserver(
+      (entries) =>
+        entries.forEach((entry) => nearScreenListeners.get(entry.target)?.(entry.isIntersecting)),
+      NEAR_SCREEN,
+    );
+    nearScreenListeners.set(element, setNear);
+    nearScreen.observe(element);
+    return () => {
+      nearScreenListeners.delete(element);
+      nearScreen?.unobserve(element);
+    };
+  }, [element]);
+  return near;
+}
+
 export const AudioWaveform = memo(function AudioWaveform({
   audioUrl,
   waveformUrl,
@@ -171,7 +197,7 @@ export const AudioWaveform = memo(function AudioWaveform({
   muted = false,
   labelInset = 16,
 }: AudioWaveformProps) {
-  const rootRef = useRef<HTMLDivElement | null>(null);
+  const [root, setRoot] = useState<HTMLDivElement | null>(null);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const cacheKey = waveformUrl ?? audioUrl;
   const request = useMemo(
@@ -218,7 +244,10 @@ export const AudioWaveform = memo(function AudioWaveform({
       span,
     );
   }, [fades, from, muted, peaks, to, trimEndFraction, trimStartFraction]);
-  useEffect(draw, [draw, strip.width, strip.height]);
+  const near = useNearScreen(root);
+  useEffect(() => {
+    if (near) draw();
+  }, [draw, near, strip.width, strip.height]);
 
   useEffect(() => {
     const root = document.documentElement;
@@ -231,15 +260,15 @@ export const AudioWaveform = memo(function AudioWaveform({
   }, [draw]);
 
   useEffect(() => {
-    const clip = rootRef.current?.closest(".timeline-clip");
+    const clip = root?.closest(".timeline-clip");
     if (!(clip instanceof HTMLElement)) return;
     if (muted) clip.setAttribute("data-audio-muted", "true");
     else clip.removeAttribute("data-audio-muted");
     return () => clip.removeAttribute("data-audio-muted");
-  }, [muted]);
+  }, [muted, root]);
 
   return (
-    <div ref={rootRef} className="absolute inset-0">
+    <div ref={setRoot} className="absolute inset-0">
       <div
         ref={setStripRef}
         className="absolute inset-0 overflow-hidden"