perf(producer): chunks read remote videos from their link instead of copying them

docs/deploy/aws-lambda.mdx

@@ -225,8 +225,11 @@ The render bucket is created with CloudFormation `Retain` on delete — `hyperfr
 - The distributed AWS path is SDR-only.
 - Each render chunk extracts only the video frames it shows, so the plan step
   no longer grows with video length. On a 10,240 MB function, a 26-minute
-  576×1024 24 fps H.264 clip plans in about 20 seconds. Each chunk downloads
-  the whole source file of every video it shows. Variable-frame-rate and HDR
+  576×1024 24 fps H.264 clip plans in about 20 seconds. A `<video src>` that is
+  an MP4, MOV, Matroska or WebM file at a public HTTPS URL that serves byte
+  ranges with a strong ETag is read in place: each chunk reads only the parts it
+  shows. Any other source is downloaded
+  whole by each chunk that shows it. Variable-frame-rate and HDR
   sources are the exception: the plan step still extracts every frame of those,
   inside Lambda's 15-minute limit, in 2-minute segments, so
   `FFMPEG_PROCESS_TIMEOUT_MS` applies to each segment, not to the whole clip.

packages/engine/src/index.ts

@@ -329,6 +329,13 @@ export {
   isNotMediaPayload,
 } from "./utils/notMediaPayload.js";
 
+export {
+  createInPlaceSources,
+  relayInPlaceSource,
+  type InPlaceSource,
+  type InPlaceSources,
+} from "./utils/inPlaceSources.js";
+
 export {
   assertPublicHttpsUrl,
   isBlockedNetworkHost,

packages/engine/src/services/audioMixer.ts

@@ -19,6 +19,7 @@ import {
   UrlDownloadError,
   writeUrlDownloadTelemetry,
 } from "../utils/urlDownloader.js";
+import type { InPlaceSources } from "../utils/inPlaceSources.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
 import { unwrapTemplate } from "../utils/htmlTemplate.js";
@@ -1118,6 +1119,7 @@ export async function processCompositionAudio(
   signal?: AbortSignal,
   config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
   compiledDir?: string,
+  options?: { inPlaceSources?: InPlaceSources },
 ): Promise<MixResult> {
   const startMs = Date.now();
   const tracks: AudioTrack[] = [];
@@ -1171,7 +1173,10 @@ export async function processCompositionAudio(
           srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);
         }
 
-        if (isHttpUrl(srcPath)) {
+        const inPlace = await options?.inPlaceSources?.(srcPath);
+        if (inPlace) {
+          srcPath = inPlace.readUrl;
+        } else if (isHttpUrl(srcPath)) {
           try {
             srcPath = await downloadToTemp(
               srcPath,
@@ -1189,7 +1194,7 @@ export async function processCompositionAudio(
           }
         }
 
-        if (!existsSync(srcPath)) {
+        if (!inPlace && !existsSync(srcPath)) {
           failures.push({
             stage: "source",
             reason: "source_not_found",
@@ -1207,7 +1212,7 @@ export async function processCompositionAudio(
         // authored duration or `loop`. It then fails inside ffmpeg as
         // `prepare/ffmpeg_failed` with owner "system" — an authoring bug paged
         // as a platform fault, after every frame has already been captured.
-        if (await isNotMediaPayload(srcPath)) {
+        if (!inPlace && (await isNotMediaPayload(srcPath))) {
           failures.push({
             stage: "source",
             reason: "invalid_media",

packages/engine/src/services/frameCapture.ts

@@ -1384,6 +1384,13 @@ async function constructCaptureSessionWithRollback(
   }
 }
 
+/** A URLPattern string matching exactly this URL, as `Network.setBlockedURLs` takes. */
+function exactUrlPattern(url: string): string {
+  const { protocol, host, pathname, search } = new URL(url);
+  const escape = (part: string) => part.replace(/[\\:*?+(){}]/g, "\\$&");
+  return `${protocol}//${host}${escape(pathname)}\\?${escape(search.slice(1))}`;
+}
+
 async function constructCaptureSession(
   input: CaptureSessionConstructionInput & { onPageCreated(page: Page): void },
 ): Promise<CaptureSession> {
@@ -1438,6 +1445,16 @@ async function constructCaptureSession(
   // Media elements are addressed by a document-unique render id, not by the
   // per-file element id. Install the resolvers before any page script runs.
   await installMediaRenderIdBridge(page);
+  if (options.blockedMediaUrls?.length) {
+    const cdp = await getCdpSession(page);
+    await cdp.send("Network.enable");
+    await cdp.send("Network.setBlockedURLs", {
+      urlPatterns: options.blockedMediaUrls.map((url) => ({
+        urlPattern: exactUrlPattern(url),
+        block: true,
+      })),
+    });
+  }
   // Fast capture: record accelerated canvases (webgl/webgl2/webgpu) and force
   // preserveDrawingBuffer before any page script can create a context — their
   // paint records freeze at the first frame, so captureDrawElementFrame

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -56,6 +56,7 @@ import {
   type VideoMetadata,
 } from "../utils/ffprobe.js";
 import { runFfmpeg } from "../utils/runFfmpeg.js";
+import { relayInPlaceSource } from "../utils/inPlaceSources.js";
 import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js";
 import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js";
 import { compileTimingAttrs, sourceTimeAt } from "@hyperframes/core";
@@ -1993,6 +1994,153 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
       60_000,
     );
 
+    /**
+     * Serves `file` at https://media.example/gop.mp4 as version "v1", by range. Reads from
+     * `failFrom.offset` on get `failFrom.status`.
+     */
+    function stubGopSourceUrl(
+      file = gopSource,
+      failFrom?: { offset: number; status: number },
+    ): string {
+      const bytes = readFileSync(file);
+      vi.stubGlobal(
+        "fetch",
+        vi.fn(async (_url: string, init: RequestInit) => {
+          const headers = init.headers as Record<string, string>;
+          if (headers["if-match"] !== '"v1"') return new Response(null, { status: 412 });
+          const { start, end, ranged } = requestedBytes(headers.range, bytes.length);
+          if (failFrom && start >= failFrom.offset) {
+            return new Response(null, { status: failFrom.status });
+          }
+          return new Response(bytes.subarray(start, end + 1), {
+            status: ranged ? 206 : 200,
+            headers: {
+              "accept-ranges": "bytes",
+              "content-length": String(end - start + 1),
+              ...(ranged ? { "content-range": `bytes ${start}-${end}/${bytes.length}` } : {}),
+            },
+          });
+        }),
+      );
+      return "https://media.example/gop.mp4";
+    }
+
+    function requestedBytes(header: string | undefined, length: number) {
+      const range = /^bytes=(\d+)-(\d*)$/.exec(header ?? "");
+      const end = range?.[2] ? Number(range[2]) : Number.POSITIVE_INFINITY;
+      return {
+        start: range ? Number(range[1]) : 0,
+        end: Math.min(end, length - 1),
+        ranged: range !== null,
+      };
+    }
+
+    it("frameRanges read in place write the file's frames, and a changed source fails", async () => {
+      const url = stubGopSourceUrl();
+      const options = {
+        fps: 24,
+        outputDir: FIXTURE_DIR,
+        format: "jpg" as const,
+        frameRanges: [{ firstFrame: 97, frames: 40 }],
+      };
+      const readUrl = async (etag: string) => (await relayInPlaceSource(url, etag)).readUrl;
+      try {
+        const local = await extractVideoFramesRange(gopSource, "url-local", 1.3, 17, options);
+        const remote = await extractVideoFramesRange(
+          await readUrl('"v1"'),
+          "url-remote",
+          1.3,
+          17,
+          options,
+        );
+        expect(remote.framePaths.size).toBe(40);
+        expect(frameBytes(remote.outputDir)).toEqual(frameBytes(local.outputDir));
+        await expect(
+          extractVideoFramesRange(await readUrl('"v0"'), "url-stale", 1.3, 17, options),
+        ).rejects.toMatchObject({ diagnostic: expect.stringContaining("4XX") });
+      } finally {
+        vi.unstubAllGlobals();
+      }
+    }, 60_000);
+
+    it.each([503, 412])(
+      "a source that answers %i partway through a read fails instead of losing frames",
+      async (status) => {
+        const source = join(FIXTURE_DIR, "past-one-window.mp4");
+        if (!existsSync(source)) {
+          const synth = await runFfmpeg([
+            "-y",
+            "-loglevel",
+            "error",
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=s=640x360:r=30:d=12",
+            "-c:v",
+            "libx264",
+            "-crf",
+            "10",
+            "-movflags",
+            "+faststart",
+            source,
+          ]);
+          expect(synth.success, synth.stderr).toBe(true);
+        }
+        expect(statSync(source).size).toBeGreaterThan(2 * 2 ** 20);
+        const url = stubGopSourceUrl(source, { offset: 2 ** 20, status });
+        const video: VideoElement = {
+          id: `fails-at-${status}`,
+          src: url,
+          start: 0,
+          end: Infinity,
+          mediaStart: 0,
+          loop: false,
+          hasAudio: false,
+        };
+        try {
+          const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
+            fps: 24,
+            outputDir: join(FIXTURE_DIR, `fails-at-${status}`),
+            inPlaceSources: async (src) => relayInPlaceSource(src, '"v1"'),
+          }).catch((error: unknown) => error);
+          expect(result).not.toMatchObject({ errors: [] });
+        } finally {
+          vi.unstubAllGlobals();
+        }
+      },
+      60_000,
+    );
+
+    it("a deferred source replaced since it was checked fails instead of being read", async () => {
+      const url = stubGopSourceUrl();
+      try {
+        await expect(
+          extractAllVideoFrames(
+            [
+              {
+                id: "url-replaced",
+                src: url,
+                start: 0,
+                end: Infinity,
+                mediaStart: 0,
+                loop: false,
+                hasAudio: false,
+              },
+            ],
+            FIXTURE_DIR,
+            {
+              fps: 24,
+              outputDir: join(FIXTURE_DIR, "url-replaced"),
+              deferRangeExtraction: true,
+              inPlaceSources: async (src) => relayInPlaceSource(src, '"v0"'),
+            },
+          ),
+        ).rejects.toThrow(/4XX/);
+      } finally {
+        vi.unstubAllGlobals();
+      }
+    }, 60_000);
+
     it.each([
       { fps: 24, mediaStart: 3.3 },
       { fps: 30, mediaStart: 0 },

packages/engine/src/services/videoFrameExtractor.ts

@@ -57,6 +57,7 @@ import {
   UrlDownloadError,
   writeUrlDownloadTelemetry,
 } from "../utils/urlDownloader.js";
+import type { InPlaceSource, InPlaceSources } from "../utils/inPlaceSources.js";
 import { runFfmpeg, runFfmpegPipeline, type RunFfmpegResult } from "../utils/runFfmpeg.js";
 import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
@@ -208,6 +209,8 @@ export interface DeferredFrameRange {
   startTime: number;
   durationSeconds: number;
   format: "jpg" | "png";
+  /** Present when the source was read in place; another process reads it at the same version. */
+  source?: InPlaceSource;
 }
 
 export interface ExtractionOptions {
@@ -245,6 +248,12 @@ export interface ExtractionOptions {
    * `totalFrames` so another process can extract just the frames it needs with `frameRanges`.
    */
   deferRangeExtraction?: boolean;
+  /**
+   * Read every remote source this answers for in place instead of downloading it, so another
+   * process can read the same URL. Windows extracted here then skip the frame cache, which is keyed
+   * by local files.
+   */
+  inPlaceSources?: InPlaceSources;
   /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
   contentKeyedDirs?: readonly string[];
 }
@@ -1881,7 +1890,10 @@ export async function extractAllVideoFrames(
         videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
       }
 
-      if (isHttpUrl(videoPath)) {
+      const inPlace = await options.inPlaceSources?.(videoPath);
+      if (inPlace) {
+        videoPath = inPlace.readUrl;
+      } else if (isHttpUrl(videoPath)) {
         const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);
         mkdirSync(downloadDir, { recursive: true });
         videoPath = await downloadToTemp(
@@ -1894,7 +1906,7 @@ export async function extractAllVideoFrames(
         );
       }
 
-      if (!existsSync(videoPath)) {
+      if (!inPlace && !existsSync(videoPath)) {
         // Loud: silent miss leaves the rendered video frozen at frame 0 with
         // no error in stdout — extremely confusing for authors. Dedupe by
         // src so 50 broken videos pointing at the same path don't spam.
@@ -2297,6 +2309,7 @@ export async function extractAllVideoFrames(
     );
     const tailFrames = Math.min(expectedFrames, Math.ceil(fps));
     if (expectedFrames <= tailFrames) return null;
+    const source = await options.inPlaceSources?.(work.video.src);
     const probeDir = join(options.outputDir, `${work.video.id}.tail-probe`);
     try {
       const tail = await extractVideoFramesRange(
@@ -2330,6 +2343,7 @@ export async function extractAllVideoFrames(
           startTime: work.extractionMediaStart,
           durationSeconds: work.videoDuration,
           format: work.format,
+          source,
         },
       };
     } catch {

packages/engine/src/types.ts

@@ -195,6 +195,12 @@ export interface CaptureOptions {
    * intrinsic media dimensions.
    */
   skipReadinessVideoIds?: readonly string[];
+  /**
+   * Remote media URLs the page must not fetch, because their frames are supplied out-of-band.
+   * Each URL is blocked exactly, query included. A blocked video errors, which readiness treats as
+   * settled.
+   */
+  blockedMediaUrls?: readonly string[];
   /** Pass one object to every session of a render; see `SubTimelineWaitMemo`. */
   subTimelineWaitMemo?: SubTimelineWaitMemo;
   /**

packages/engine/src/utils/ffprobe.ts

@@ -8,6 +8,7 @@ import { redactTelemetryString } from "@hyperframes/core";
 import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
 import { ManagedChildProcess } from "./managedChildProcess.js";
 import { trackChildProcess } from "./processTracker.js";
+import { watchRelayedReads } from "./inPlaceSources.js";
 
 const FFPROBE_STDERR_MAX_BYTES = 8 * 1024;
 /** Bound on collected stdout. Generous — real -show_streams JSON is well
@@ -65,6 +66,7 @@ async function runFfprobe(
   }
 
   const command = getFfprobeBinary();
+  const relayedReadFailure = watchRelayedReads([filePath]);
   const proc = spawn(command, ["-v", "error", ...argsWithoutInput, "--", filePath], {
     // Nothing is ever written to the child's stdin; leaving it as a pipe is
     // what lets a stdin-reading invocation block indefinitely.
@@ -129,6 +131,8 @@ async function runFfprobe(
       `[FFmpeg] ffprobe ${outcome.reason} with code ${outcome.exitCode}: ${diagnostic}`,
     );
   }
+  const failure = relayedReadFailure();
+  if (failure) throw new Error(`[FFmpeg] ffprobe: ${failure}`);
   return stdout;
 }