perf(core): a paused preview stops running its audio output (#4534)

* perf(core): a paused preview stops running its audio output

The preview runtime opened its AudioContext at load and left it running for the whole
session, so a paused Studio kept an audio output stream rendering silence in Chrome's
audio service and output threads, and on Chrome's fake sink once a context has been silent
for 30 s.

The transport now suspends the context once it has been paused for a second with no
captured track playing, and resumes it for Play and for a captured track that plays while
paused (the scrub route). The delay keeps a stop-and-reschedule inside one play (a rate
change, a hidden track, a seek while playing) from bouncing the output. A Play that lands
while a suspend is pending resumes anyway, and a Pause that lands while Play waits on
resume() lets the context go back to sleep.

* fix(core): decoded audio after an idle pause starts where the picture is

Review follow-ups to the idle suspend:
- A decoded track read its start time before waiting for the context to wake, so every
  Play after an idle pause anchored the clock that far back and the picture stepped back.
  The transport now reads the time after the wake.
- The transport kept every element it ever captured. It now holds only captured tracks
  that are playing, so a replaced track is free once it pauses.
- A failed suspend and destroy() both clear the pending-suspend flag, so the next pause,
  or a fresh init(), can suspend again.
- A test pins the one-second idle delay; the redundant "ended" listener is gone, since a
  track that ends fires "pause" first.

* fix(core): a preview whose browser starts audio late still goes quiet when idle

The idle timer was armed only if the context was already running when the transport opened
it or resumed it, so a browser that starts the context a moment after construction left it
running until the first pause. The transport now arms the timer from the context's own
statechange event, which Chrome fires after construction and on every resume, so the
separate arming in init() and after a stale Play are gone.

* fix(core): a reloaded track no longer keeps a paused preview's audio awake

Reloading a playing captured track (el.load(), a src swap) stops it without a "pause" event,
so the transport kept counting it as playing and a paused preview never went quiet. The
idle check now drops tracks that are already paused before it counts, and "emptied" is
handled like "pause" so the idle timer is armed when a reload stops a track.

* perf(core): a paused preview waits 10 s before its audio output goes to sleep

* docs(core): say why a paused preview's audio waits 10 s before it sleeps

* fix(core): a browser-side audio suspend drops the preview's idle timer

Author: miguel-heygen

packages/core/src/runtime/capturedMediaIdleRoute.test.ts

@@ -50,6 +50,9 @@ class GraphAudioContext {
   resume() {
     return pendingResume ?? Promise.resolve();
   }
+  suspend() {
+    return Promise.resolve();
+  }
   close() {
     return Promise.resolve();
   }

packages/core/src/runtime/init.test.ts

@@ -4311,6 +4311,9 @@ describe("initSandboxRuntimeModular", () => {
       resume() {
         return Promise.resolve();
       }
+      suspend() {
+        return Promise.resolve();
+      }
       createGain() {
         return { gain: { value: 1 }, connect() {}, disconnect() {} };
       }

packages/core/src/runtime/init.ts

@@ -4366,7 +4366,7 @@ export function initSandboxRuntimeModular(): void {
             buffer,
             compStart,
             mediaStart,
-            clock.now(),
+            () => clock.now(),
             vol,
             gen,
             state.playbackRate,

packages/core/src/runtime/webAudioTransport.test.ts

@@ -39,6 +39,7 @@ function createMockAudioContext(currentTime = 100) {
     currentTime,
     state: "running",
     resume: vi.fn(),
+    suspend: vi.fn(() => Promise.resolve()),
     createBufferSource: vi.fn(() => sourceNode),
     createMediaElementSource: vi.fn(() => mediaElementSourceNode),
     createGain: vi.fn(() => gainNode),
@@ -62,6 +63,8 @@ const mockBuffer = {} as AudioBuffer;
 const mockEl = {
   muted: false,
   volume: 0.4,
+  paused: true,
+  addEventListener: () => {},
   getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
 } as unknown as HTMLMediaElement;
 
@@ -435,7 +438,7 @@ describe("WebAudioTransport", () => {
     it("keeps author boost above unity on the per-element gain node", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, () => 0, 1, gen);
       transport.setElementVolume(mockEl, 3.98);
 
       expect(mock.gainNode.gain.value).toBeCloseTo(3.98, 5);
@@ -444,39 +447,39 @@ describe("WebAudioTransport", () => {
     it("starts in-progress clips immediately with correct buffer offset", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen);
 
       expect(mock.startFn).toHaveBeenCalledWith(0, 3);
     });
 
     it("starts in-progress clips with mediaStart offset", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 2, 8, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 2, () => 8, 1, gen);
 
       expect(mock.startFn).toHaveBeenCalledWith(0, 5);
     });
 
     it("schedules future clips with delay instead of playing immediately", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, 2, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, () => 2, 1, gen);
 
       expect(mock.startFn).toHaveBeenCalledWith(108, 0);
     });
 
     it("schedules future clips with correct mediaStart", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 10, 1.5, 2, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 1.5, () => 2, 1, gen);
 
       expect(mock.startFn).toHaveBeenCalledWith(108, 1.5);
     });
 
     it("starts clips at exact composition start time immediately", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 5, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 5, 1, gen);
 
       expect(mock.startFn).toHaveBeenCalledWith(0, 0);
     });
@@ -486,35 +489,45 @@ describe("WebAudioTransport", () => {
     it("bounds an in-progress clip to its remaining authored window", async () => {
       const { transport, mock, gen } = setupTransport(100);
       // compStart=5, mediaStart=0, compTime=8 → elapsed=3; clipDuration=10 → 7 left
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1, 10);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 1, 10);
       expect(mock.startFn).toHaveBeenCalledWith(0, 3, 7);
     });
 
     it("bounds a future clip to its full authored window", async () => {
       const { transport, mock, gen } = setupTransport(100);
       // compStart=10, mediaStart=1.5, compTime=2 → elapsed=-8 → delay 8; clipDuration=4
-      await transport.schedulePlayback(mockEl, mockBuffer, 10, 1.5, 2, 1, gen, 1, 4);
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 1.5, () => 2, 1, gen, 1, 4);
       expect(mock.startFn).toHaveBeenCalledWith(108, 1.5, 4);
     });
 
     it("does not schedule a clip whose window has already elapsed", async () => {
       const { transport, mock, gen } = setupTransport(100);
       // elapsed=15 > clipDuration=10 → nothing to play
-      const result = await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 20, 1, gen, 1, 10);
+      const result = await transport.schedulePlayback(
+        mockEl,
+        mockBuffer,
+        5,
+        0,
+        () => 20,
+        1,
+        gen,
+        1,
+        10,
+      );
       expect(result).toBeNull();
       expect(mock.startFn).not.toHaveBeenCalled();
     });
 
     it("keeps source bounds in authored media time when global rate changes", async () => {
       const { transport, mock, gen } = setupTransport(100);
       // Global rate=2 changes wallclock speed, not source-time span.
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2, 10);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2, 10);
       expect(mock.startFn).toHaveBeenCalledWith(0, 3, 7);
     });
 
     it("plays unbounded when clipDuration is omitted (legacy behavior)", async () => {
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen);
       expect(mock.startFn).toHaveBeenCalledWith(0, 3);
     });
   });
@@ -530,7 +543,7 @@ describe("WebAudioTransport", () => {
       // At composition t=0.5 with mediaStart=1, authored 2x has consumed one
       // source second. Global 0.5x makes the source node's wallclock rate 1x,
       // but the composition clock must still advance at global 0.5x.
-      await transport.schedulePlayback(el, mockBuffer, 0, 1, 0.5, 1, gen, 0.5, 2);
+      await transport.schedulePlayback(el, mockBuffer, 0, 1, () => 0.5, 1, gen, 0.5, 2);
 
       expect(mock.sourceNode.playbackRate.value).toBe(1);
       expect(mock.startFn).toHaveBeenCalledWith(0, 2, 3);
@@ -544,15 +557,15 @@ describe("WebAudioTransport", () => {
     it("sets sourceNode.playbackRate.value when rate is provided", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2);
 
       expect(mock.sourceNode.playbackRate.value).toBe(2);
     });
 
     it("defaults rate to 1 when not provided", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen);
 
       expect(mock.sourceNode.playbackRate.value).toBe(1);
     });
@@ -561,23 +574,23 @@ describe("WebAudioTransport", () => {
       const { transport, mock, gen } = setupTransport(100);
 
       // compStart=10, compositionTime=2, rate=2 → 8s of comp time = 4s wallclock
-      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, 2, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, () => 2, 1, gen, 2);
 
       expect(mock.startFn).toHaveBeenCalledWith(104, 0);
     });
 
     it("keeps in-progress buffer offset at elapsed + mediaStart regardless of rate", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2);
 
       expect(mock.startFn).toHaveBeenCalledWith(0, 3);
     });
 
     it("setRate updates active sources in place", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 1);
       expect(mock.sourceNode.playbackRate.value).toBe(1);
 
       transport.setRate(2);
@@ -590,7 +603,7 @@ describe("WebAudioTransport", () => {
       // scheduled, so bumping playbackRate alone left every automated parameter
       // running its original plan over audio moving at a different speed.
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 1);
       const active = (transport as unknown as { _activeSources: { fx?: unknown }[] })
         ._activeSources;
       const setRate = vi.fn();
@@ -609,7 +622,7 @@ describe("WebAudioTransport", () => {
 
     it("setRate is a no-op when the rate is unchanged", async () => {
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2);
 
       mock.ctx.currentTime = 100.5;
       const timeBefore = transport.getTime();
@@ -624,7 +637,7 @@ describe("WebAudioTransport", () => {
 
     it("setRate clamps non-finite or non-positive values to 1", async () => {
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2);
       expect(mock.sourceNode.playbackRate.value).toBe(2);
 
       transport.setRate(Number.NaN);
@@ -642,7 +655,7 @@ describe("WebAudioTransport", () => {
     it("getTime advances at the configured rate", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 2);
 
       // At schedule time, ctx.currentTime=100, compositionTime=8.
       expect(transport.getTime()).toBeCloseTo(8, 10);
@@ -656,7 +669,7 @@ describe("WebAudioTransport", () => {
     it("getTime tracks composition time after a mid-playback setRate", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, () => 8, 1, gen, 1);
       expect(transport.getTime()).toBeCloseTo(8, 10);
 
       // 0.5s passes at rate=1 → composition time = 8.5
@@ -681,7 +694,7 @@ describe("WebAudioTransport", () => {
         getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
       } as unknown as HTMLMediaElement;
 
-      await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen);
       expect(transport.ownsClock()).toBe(true);
       expect(el.muted).toBe(true);
 
@@ -698,7 +711,7 @@ describe("WebAudioTransport", () => {
         getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
       } as unknown as HTMLMediaElement;
 
-      await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen);
       expect(el.muted).toBe(true);
 
       mock.sourceNode._fireEnded();
@@ -712,7 +725,7 @@ describe("WebAudioTransport", () => {
       // already removed this entry — so the handle, its MutationObserver and any
       // running LFO survived the clip for the rest of the session.
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, () => 0, 1, gen);
 
       mock.sourceNode._fireEnded();
 
@@ -723,7 +736,7 @@ describe("WebAudioTransport", () => {
     it("registers onended listener on the sourceNode", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
-      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, () => 0, 1, gen);
 
       expect(mock.sourceNode.addEventListener).toHaveBeenCalledWith("ended", expect.any(Function));
     });
@@ -735,7 +748,7 @@ describe("WebAudioTransport", () => {
         getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
       } as unknown as HTMLMediaElement;
 
-      await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen);
       expect(el.muted).toBe(true);
 
       transport.stopAll();
@@ -774,6 +787,7 @@ describe("WebAudioTransport", () => {
         currentTime,
         state: "running",
         resume: vi.fn(),
+        suspend: vi.fn(() => Promise.resolve()),
         createBufferSource: vi.fn(() => ({
           buffer: null as AudioBuffer | null,
           playbackRate: { value: 1 },
@@ -853,7 +867,7 @@ describe("WebAudioTransport", () => {
       groupId?: string,
     ): Promise<HTMLMediaElement> {
       const el = groupedAudioEl(id, groupId);
-      await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen);
+      await transport.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen);
       return el;
     }