fix(studio): the timeline viewport gate judges p95 over every measured scroll step

packages/studio/src/player/lib/timelineViewportBudgets.test.ts

@@ -23,7 +23,7 @@ describe("timeline viewport budgets", () => {
       constrainedLongTaskLimitMs: 300,
       posterCoverageRatio: 0.9,
       supportedFixtureFallbackRatio: 0.02,
-      scrollSamplesPerRun: 21,
+      scrollSamplesPerRun: 63,
       warmupRuns: 3,
       measuredRuns: 5,
       requiredPassingRuns: 4,

packages/studio/src/player/lib/timelineViewportBudgets.ts

@@ -99,7 +99,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
   richPreviewP95Ms: 750,
   constrainedRichPreviewP95Ms: 1_200,
   supportedFixtureFallbackRatio: 0.02,
-  scrollSamplesPerRun: 21,
+  scrollSamplesPerRun: 63,
   warmupRuns: 3,
   measuredRuns: 5,
   requiredPassingRuns: 4,

packages/studio/tests/e2e/chrome-executable.mjs

@@ -1,6 +1,7 @@
 import { existsSync, readdirSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+import puppeteer from "puppeteer-core";
 
 /** Resolve the same Chrome binary for every local Studio browser acceptance test. */
 export function resolveChromeExecutable() {
@@ -22,3 +23,18 @@ export function resolveChromeExecutable() {
     ...installed,
   ].find((candidate) => candidate && existsSync(candidate));
 }
+
+/** Headless Chrome for a Studio browser gate; exits 2 when no Chrome is installed. */
+export async function launchStudioChrome() {
+  const executablePath = resolveChromeExecutable();
+  if (!executablePath) {
+    console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
+    process.exit(2);
+  }
+  const browser = await puppeteer.launch({
+    executablePath,
+    headless: true,
+    args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
+  });
+  return { browser, executablePath };
+}

packages/studio/tests/e2e/fade-handles.mjs

@@ -1,8 +1,7 @@
 #!/usr/bin/env node
 // On a clip narrower than two fade hit boxes, each dot must still lay out at 10 x 10.
 // Flex shrinking only happens in real layout, so this measures it in Chrome.
-import puppeteer from "puppeteer-core";
-import { resolveChromeExecutable } from "./chrome-executable.mjs";
+import { launchStudioChrome } from "./chrome-executable.mjs";
 
 const STUDIO_URL = process.env.STUDIO_URL;
 const DOT_PX = 10;
@@ -12,17 +11,7 @@ if (!STUDIO_URL) {
   console.error("STUDIO_URL is required and must point at the fade-handles fixture");
   process.exit(2);
 }
-const executablePath = resolveChromeExecutable();
-if (!executablePath) {
-  console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
-  process.exit(2);
-}
-
-const browser = await puppeteer.launch({
-  executablePath,
-  headless: true,
-  args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
-});
+const { browser } = await launchStudioChrome();
 const failures = [];
 let evidence = {};
 try {

packages/studio/tests/e2e/timeline-viewport-verdict.mjs

@@ -0,0 +1,54 @@
+/**
+ * The timeline viewport gate's verdict, kept apart from the script that drives Chrome so it can be tested.
+ */
+
+/** Nearest-rank percentile: `ratio` 0.95 of 315 values is the 16th-worst. */
+export function percentile(values, ratio) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];
+}
+
+function assertSampleCount(expected, interactions, frameIntervals) {
+  if (expected > 0 && interactions.length === expected && frameIntervals.length === expected)
+    return;
+  throw new Error(
+    `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` +
+      `and ${frameIntervals.length} frame intervals`,
+  );
+}
+
+/**
+ * p95 over every measured step pooled: one run's p95 is only its 4th-worst step, so a brief runner stall failed it.
+ * Throws when any run is short of samples, so a missing measurement cannot read as a fast one.
+ */
+export function judgeResponsiveness(
+  runs,
+  { samplesPerRun, interactionLimitMs, frameIntervalLimitMs },
+) {
+  const interactions = runs.flatMap((run) => run.interactions);
+  const frameIntervals = runs.flatMap((run) => run.frameIntervals);
+  assertSampleCount(runs.length * samplesPerRun, interactions, frameIntervals);
+  const interactionP95Ms = percentile(interactions, 0.95);
+  const frameIntervalP95Ms = percentile(frameIntervals, 0.95);
+  return {
+    interactionP95Ms,
+    frameIntervalP95Ms,
+    passed: interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs,
+  };
+}
+
+export function gatePassed({
+  directScrollApproved,
+  responsivenessPassed,
+  passingRuns,
+  requiredPassingRuns,
+  memoryReturned,
+}) {
+  return (
+    directScrollApproved &&
+    responsivenessPassed &&
+    passingRuns >= requiredPassingRuns &&
+    memoryReturned
+  );
+}

packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs

@@ -0,0 +1,88 @@
+import { describe, expect, it } from "vitest";
+import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs";
+
+const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 };
+const FAST = 49;
+const SLOW = 83;
+
+/** Five runs of 63 steps; `slowAt(run, step)` marks the steps that take five frames. */
+function runs(slowAt) {
+  return Array.from({ length: 5 }, (_, run) => {
+    const interactions = Array.from({ length: 63 }, (_, step) => (slowAt(run, step) ? SLOW : FAST));
+    return { interactions, frameIntervals: interactions.map(() => 33.3) };
+  });
+}
+
+const slowSteps = (count) => (run, step) => run * 63 + step < count;
+
+describe("percentile", () => {
+  it("takes the nearest rank, so the p95 of 315 steps is the 16th-worst", () => {
+    const values = Array.from({ length: 315 }, (_, index) => index);
+    expect(percentile(values, 0.95)).toBe(299);
+    expect(percentile([3, 1, 2], 0.95)).toBe(3);
+  });
+});
+
+describe("judgeResponsiveness", () => {
+  it("fails 16 slow steps of 315 and passes 15", () => {
+    expect(judgeResponsiveness(runs(slowSteps(16)), LIMITS)).toMatchObject({
+      interactionP95Ms: SLOW,
+      passed: false,
+    });
+    expect(judgeResponsiveness(runs(slowSteps(15)), LIMITS)).toMatchObject({
+      interactionP95Ms: FAST,
+      passed: true,
+    });
+  });
+
+  it("passes two slow steps in every run, 10 of 315", () => {
+    expect(
+      judgeResponsiveness(
+        runs((_, step) => step < 2),
+        LIMITS,
+      ).passed,
+    ).toBe(true);
+  });
+
+  it("fails one run that is slow throughout", () => {
+    expect(
+      judgeResponsiveness(
+        runs((run) => run === 2),
+        LIMITS,
+      ).passed,
+    ).toBe(false);
+  });
+
+  it("fails on the frame interval alone", () => {
+    const measured = runs(() => false).map((run) => ({
+      ...run,
+      frameIntervals: run.frameIntervals.map(() => 83),
+    }));
+    expect(judgeResponsiveness(measured, LIMITS).passed).toBe(false);
+  });
+
+  it("throws on a run short of samples instead of reading it as fast", () => {
+    const measured = runs(() => false);
+    measured[1] = { interactions: [], frameIntervals: [] };
+    expect(() => judgeResponsiveness(measured, LIMITS)).toThrow("Expected 315 scroll samples");
+    expect(() => judgeResponsiveness([], LIMITS)).toThrow("Expected 0 scroll samples");
+  });
+});
+
+describe("gatePassed", () => {
+  const passing = {
+    directScrollApproved: true,
+    responsivenessPassed: true,
+    passingRuns: 5,
+    requiredPassingRuns: 4,
+    memoryReturned: true,
+  };
+
+  it("passes only when every check holds", () => {
+    expect(gatePassed(passing)).toBe(true);
+    expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false);
+    expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false);
+    expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false);
+    expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false);
+  });
+});

packages/studio/tests/e2e/timeline-virtualization.mjs

@@ -22,8 +22,8 @@
  * it, and a mismatch would otherwise pass silently against the wrong build.
  */
 import { platform, arch } from "node:os";
-import puppeteer from "puppeteer-core";
-import { resolveChromeExecutable } from "./chrome-executable.mjs";
+import { launchStudioChrome } from "./chrome-executable.mjs";
+import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs";
 
 const STUDIO_URL = process.env.STUDIO_URL;
 const PROFILE = process.env.TIMELINE_PROFILE || "dense-short";
@@ -61,12 +61,6 @@ if (ROW_VIRTUALIZATION === "off" && ELEMENT_COUNT === 50_000) {
   process.exit(2);
 }
 
-function percentile(values, ratio) {
-  if (values.length === 0) return 0;
-  const sorted = [...values].sort((a, b) => a - b);
-  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];
-}
-
 async function collectHeapBytes(client) {
   const usage = await client.send("Runtime.getHeapUsage");
   return usage.usedSize;
@@ -90,6 +84,8 @@ async function collectRun(page, injectedLongTaskMs = 0) {
       interactionP95Ms: percentileInPage(interactions, 0.95),
       frameIntervalP95Ms: percentileInPage(frameIntervals, 0.95),
       scrollSampleCount: interactions.length,
+      interactions,
+      frameIntervals,
       longestTaskMs: Math.max(0, ...longTasks),
       scrollWidth: scroller.scrollWidth,
       scrollHeight: scroller.scrollHeight,
@@ -227,17 +223,7 @@ async function measureMaximumReliableScrollWidth(page) {
   });
 }
 
-const executablePath = resolveChromeExecutable();
-if (!executablePath) {
-  console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
-  process.exit(2);
-}
-
-const browser = await puppeteer.launch({
-  executablePath,
-  headless: true,
-  args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
-});
+const { browser, executablePath } = await launchStudioChrome();
 let exitCode = 1;
 try {
   const version = await browser.version();
@@ -319,19 +305,21 @@ try {
   // so a skipped budget never reads as a passed one.
   const domBudgetsApply = ROW_VIRTUALIZATION === "on";
   for (const run of runs) {
-    run.responsivenessPassed =
-      run.interactionP95Ms <= interactionLimitMs &&
-      run.frameIntervalP95Ms <= frameIntervalLimitMs &&
-      run.longestTaskMs <= longTaskLimitMs;
+    run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;
     run.timelineMounted = run.diagnostics.timelineRoots === 1;
     run.domSizePassed = domBudgetsApply
       ? run.diagnostics.mountedRows <= budgets.maxMountedRows &&
         run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots &&
         run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&
         run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants
       : null;
-    run.passed = run.responsivenessPassed && run.timelineMounted && run.domSizePassed !== false;
+    run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;
   }
+  const responsiveness = judgeResponsiveness(runs, {
+    samplesPerRun: budgets.scrollSamplesPerRun,
+    interactionLimitMs,
+    frameIntervalLimitMs,
+  });
 
   await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture());
   await page.waitForFunction(
@@ -386,27 +374,25 @@ try {
     directScrollGate,
     runs,
     aggregate: {
-      interactionP95Ms: percentile(
-        runs.map((run) => run.interactionP95Ms),
-        0.95,
-      ),
-      frameIntervalP95Ms: percentile(
-        runs.map((run) => run.frameIntervalP95Ms),
-        0.95,
-      ),
+      interactionP95Ms: responsiveness.interactionP95Ms,
+      frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,
+      responsivenessPassed: responsiveness.passed,
       passingRuns,
       baselineHeapBytes,
       returnedHeapBytes,
       memoryReturned,
     },
   };
   console.log(JSON.stringify(evidence, null, 2));
-  exitCode =
-    directScrollGate.decision === "approved" &&
-    passingRuns >= budgets.requiredPassingRuns &&
-    memoryReturned
-      ? 0
-      : 1;
+  exitCode = gatePassed({
+    directScrollApproved: directScrollGate.decision === "approved",
+    responsivenessPassed: responsiveness.passed,
+    passingRuns,
+    requiredPassingRuns: budgets.requiredPassingRuns,
+    memoryReturned,
+  })
+    ? 0
+    : 1;
 } finally {
   await browser.close();
 }