From 100bb06cbb752703b428b0a3b1835611c9d76ad0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 1 Oct 2026 02:24:26 -0700 Subject: [PATCH 1/4] fix(studio): the timeline viewport gate judges p95 over every measured scroll step --- .../lib/timelineViewportBudgets.test.ts | 2 +- .../src/player/lib/timelineViewportBudgets.ts | 2 +- .../tests/e2e/timeline-virtualization.mjs | 26 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts index 5689b5609a4..a75d630ebed 100644 --- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts +++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts @@ -23,7 +23,7 @@ describe("timeline viewport budgets", () => { constrainedLongTaskLimitMs: 300, posterCoverageRatio: 0.9, supportedFixtureFallbackRatio: 0.02, - scrollSamplesPerRun: 21, + scrollSamplesPerRun: 63, warmupRuns: 3, measuredRuns: 5, requiredPassingRuns: 4, diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.ts index b57bf325ec0..771a37cd32b 100644 --- a/packages/studio/src/player/lib/timelineViewportBudgets.ts +++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts @@ -99,7 +99,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly = Obje richPreviewP95Ms: 750, constrainedRichPreviewP95Ms: 1_200, supportedFixtureFallbackRatio: 0.02, - scrollSamplesPerRun: 21, + scrollSamplesPerRun: 63, warmupRuns: 3, measuredRuns: 5, requiredPassingRuns: 4, diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index 521da8ffee0..011d54fca5d 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -90,6 +90,8 @@ async function collectRun(page, injectedLongTaskMs = 0) { interactionP95Ms: percentileInPage(interactions, 0.95), frameIntervalP95Ms: percentileInPage(frameIntervals, 0.95), scrollSampleCount: interactions.length, + interactions, + frameIntervals, longestTaskMs: Math.max(0, ...longTasks), scrollWidth: scroller.scrollWidth, scrollHeight: scroller.scrollHeight, @@ -319,10 +321,7 @@ try { // so a skipped budget never reads as a passed one. const domBudgetsApply = ROW_VIRTUALIZATION === "on"; for (const run of runs) { - run.responsivenessPassed = - run.interactionP95Ms <= interactionLimitMs && - run.frameIntervalP95Ms <= frameIntervalLimitMs && - run.longestTaskMs <= longTaskLimitMs; + run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs; run.timelineMounted = run.diagnostics.timelineRoots === 1; run.domSizePassed = domBudgetsApply ? run.diagnostics.mountedRows <= budgets.maxMountedRows && @@ -330,8 +329,13 @@ try { run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow && run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants : null; - run.passed = run.responsivenessPassed && run.timelineMounted && run.domSizePassed !== false; + run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false; } + // Pooled over every measured step: one run's p95 is only its 4th-worst step, so a brief runner stall failed it. + const interactionP95Ms = percentile(runs.flatMap((run) => run.interactions), 0.95); + const frameIntervalP95Ms = percentile(runs.flatMap((run) => run.frameIntervals), 0.95); + const responsivenessPassed = + interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs; await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture()); await page.waitForFunction( @@ -386,14 +390,9 @@ try { directScrollGate, runs, aggregate: { - interactionP95Ms: percentile( - runs.map((run) => run.interactionP95Ms), - 0.95, - ), - frameIntervalP95Ms: percentile( - runs.map((run) => run.frameIntervalP95Ms), - 0.95, - ), + interactionP95Ms, + frameIntervalP95Ms, + responsivenessPassed, passingRuns, baselineHeapBytes, returnedHeapBytes, @@ -403,6 +402,7 @@ try { console.log(JSON.stringify(evidence, null, 2)); exitCode = directScrollGate.decision === "approved" && + responsivenessPassed && passingRuns >= budgets.requiredPassingRuns && memoryReturned ? 0 From c48d70de16a9c226e14294c8c8b963150a9ad8fd Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 1 Oct 2026 03:16:39 -0700 Subject: [PATCH 2/4] fix(studio): format the pooled p95 lines in the timeline viewport gate --- packages/studio/tests/e2e/timeline-virtualization.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index 011d54fca5d..f8190d30337 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -332,8 +332,9 @@ try { run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false; } // Pooled over every measured step: one run's p95 is only its 4th-worst step, so a brief runner stall failed it. - const interactionP95Ms = percentile(runs.flatMap((run) => run.interactions), 0.95); - const frameIntervalP95Ms = percentile(runs.flatMap((run) => run.frameIntervals), 0.95); + const pooled = (key) => runs.flatMap((run) => run[key]); + const interactionP95Ms = percentile(pooled("interactions"), 0.95); + const frameIntervalP95Ms = percentile(pooled("frameIntervals"), 0.95); const responsivenessPassed = interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs; From b3df43b0726a0d069da1fe9be05894a50bbf994b Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 1 Oct 2026 04:52:26 -0700 Subject: [PATCH 3/4] test(studio): pin the timeline viewport gate's pooled verdict with tests --- .../tests/e2e/timeline-viewport-verdict.mjs | 51 +++++++++++ .../e2e/timeline-viewport-verdict.test.mjs | 88 +++++++++++++++++++ .../tests/e2e/timeline-virtualization.mjs | 40 ++++----- 3 files changed, 157 insertions(+), 22 deletions(-) create mode 100644 packages/studio/tests/e2e/timeline-viewport-verdict.mjs create mode 100644 packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs new file mode 100644 index 00000000000..8576dc951f1 --- /dev/null +++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs @@ -0,0 +1,51 @@ +/** + * The timeline viewport gate's verdict, kept apart from the script that drives Chrome so it can be tested. + */ + +/** Nearest-rank percentile: `ratio` 0.95 of 315 values is the 16th-worst. */ +export function percentile(values, ratio) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; +} + +/** + * p95 over every measured step pooled: one run's p95 is only its 4th-worst step, so a brief runner stall failed it. + * Throws when any run is short of samples, so a missing measurement cannot read as a fast one. + */ +export function judgeResponsiveness( + runs, + { samplesPerRun, interactionLimitMs, frameIntervalLimitMs }, +) { + const interactions = runs.flatMap((run) => run.interactions); + const frameIntervals = runs.flatMap((run) => run.frameIntervals); + const expected = runs.length * samplesPerRun; + if (expected === 0 || interactions.length !== expected || frameIntervals.length !== expected) { + throw new Error( + `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` + + `and ${frameIntervals.length} frame intervals`, + ); + } + const interactionP95Ms = percentile(interactions, 0.95); + const frameIntervalP95Ms = percentile(frameIntervals, 0.95); + return { + interactionP95Ms, + frameIntervalP95Ms, + passed: interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs, + }; +} + +export function gatePassed({ + directScrollApproved, + responsivenessPassed, + passingRuns, + requiredPassingRuns, + memoryReturned, +}) { + return ( + directScrollApproved && + responsivenessPassed && + passingRuns >= requiredPassingRuns && + memoryReturned + ); +} diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs new file mode 100644 index 00000000000..c5a54af4b9c --- /dev/null +++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs"; + +const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 }; +const FAST = 49; +const SLOW = 83; + +/** Five runs of 63 steps; `slowAt(run, step)` marks the steps that take five frames. */ +function runs(slowAt) { + return Array.from({ length: 5 }, (_, run) => { + const interactions = Array.from({ length: 63 }, (_, step) => (slowAt(run, step) ? SLOW : FAST)); + return { interactions, frameIntervals: interactions.map(() => 33.3) }; + }); +} + +const slowSteps = (count) => (run, step) => run * 63 + step < count; + +describe("percentile", () => { + it("takes the nearest rank, so the p95 of 315 steps is the 16th-worst", () => { + const values = Array.from({ length: 315 }, (_, index) => index); + expect(percentile(values, 0.95)).toBe(299); + expect(percentile([3, 1, 2], 0.95)).toBe(3); + }); +}); + +describe("judgeResponsiveness", () => { + it("fails 16 slow steps of 315 and passes 15", () => { + expect(judgeResponsiveness(runs(slowSteps(16)), LIMITS)).toMatchObject({ + interactionP95Ms: SLOW, + passed: false, + }); + expect(judgeResponsiveness(runs(slowSteps(15)), LIMITS)).toMatchObject({ + interactionP95Ms: FAST, + passed: true, + }); + }); + + it("passes two slow steps in every run, 10 of 315", () => { + expect( + judgeResponsiveness( + runs((_, step) => step < 2), + LIMITS, + ).passed, + ).toBe(true); + }); + + it("fails one run that is slow throughout", () => { + expect( + judgeResponsiveness( + runs((run) => run === 2), + LIMITS, + ).passed, + ).toBe(false); + }); + + it("fails on the frame interval alone", () => { + const measured = runs(() => false).map((run) => ({ + ...run, + frameIntervals: run.frameIntervals.map(() => 83), + })); + expect(judgeResponsiveness(measured, LIMITS).passed).toBe(false); + }); + + it("throws on a run short of samples instead of reading it as fast", () => { + const measured = runs(() => false); + measured[1] = { interactions: [], frameIntervals: [] }; + expect(() => judgeResponsiveness(measured, LIMITS)).toThrow("Expected 315 scroll samples"); + expect(() => judgeResponsiveness([], LIMITS)).toThrow("Expected 0 scroll samples"); + }); +}); + +describe("gatePassed", () => { + const passing = { + directScrollApproved: true, + responsivenessPassed: true, + passingRuns: 5, + requiredPassingRuns: 4, + memoryReturned: true, + }; + + it("passes only when every check holds", () => { + expect(gatePassed(passing)).toBe(true); + expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false); + expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false); + expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false); + expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false); + }); +}); diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index f8190d30337..b6b794a2a25 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -24,6 +24,7 @@ import { platform, arch } from "node:os"; import puppeteer from "puppeteer-core"; import { resolveChromeExecutable } from "./chrome-executable.mjs"; +import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs"; const STUDIO_URL = process.env.STUDIO_URL; const PROFILE = process.env.TIMELINE_PROFILE || "dense-short"; @@ -61,12 +62,6 @@ if (ROW_VIRTUALIZATION === "off" && ELEMENT_COUNT === 50_000) { process.exit(2); } -function percentile(values, ratio) { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; -} - async function collectHeapBytes(client) { const usage = await client.send("Runtime.getHeapUsage"); return usage.usedSize; @@ -331,12 +326,11 @@ try { : null; run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false; } - // Pooled over every measured step: one run's p95 is only its 4th-worst step, so a brief runner stall failed it. - const pooled = (key) => runs.flatMap((run) => run[key]); - const interactionP95Ms = percentile(pooled("interactions"), 0.95); - const frameIntervalP95Ms = percentile(pooled("frameIntervals"), 0.95); - const responsivenessPassed = - interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs; + const responsiveness = judgeResponsiveness(runs, { + samplesPerRun: budgets.scrollSamplesPerRun, + interactionLimitMs, + frameIntervalLimitMs, + }); await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture()); await page.waitForFunction( @@ -391,9 +385,9 @@ try { directScrollGate, runs, aggregate: { - interactionP95Ms, - frameIntervalP95Ms, - responsivenessPassed, + interactionP95Ms: responsiveness.interactionP95Ms, + frameIntervalP95Ms: responsiveness.frameIntervalP95Ms, + responsivenessPassed: responsiveness.passed, passingRuns, baselineHeapBytes, returnedHeapBytes, @@ -401,13 +395,15 @@ try { }, }; console.log(JSON.stringify(evidence, null, 2)); - exitCode = - directScrollGate.decision === "approved" && - responsivenessPassed && - passingRuns >= budgets.requiredPassingRuns && - memoryReturned - ? 0 - : 1; + exitCode = gatePassed({ + directScrollApproved: directScrollGate.decision === "approved", + responsivenessPassed: responsiveness.passed, + passingRuns, + requiredPassingRuns: budgets.requiredPassingRuns, + memoryReturned, + }) + ? 0 + : 1; } finally { await browser.close(); } From 2aef96468257ac8b6cb695a3ffa5726b13436ccd Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 1 Oct 2026 05:20:38 -0700 Subject: [PATCH 4/4] refactor(studio): share one Chrome launcher across the timeline and fade-handle gates --- packages/studio/tests/e2e/chrome-executable.mjs | 16 ++++++++++++++++ packages/studio/tests/e2e/fade-handles.mjs | 15 ++------------- .../tests/e2e/timeline-viewport-verdict.mjs | 17 ++++++++++------- .../tests/e2e/timeline-virtualization.mjs | 15 ++------------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/packages/studio/tests/e2e/chrome-executable.mjs b/packages/studio/tests/e2e/chrome-executable.mjs index b1a868916e7..8fc89f490ed 100644 --- a/packages/studio/tests/e2e/chrome-executable.mjs +++ b/packages/studio/tests/e2e/chrome-executable.mjs @@ -1,6 +1,7 @@ import { existsSync, readdirSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import puppeteer from "puppeteer-core"; /** Resolve the same Chrome binary for every local Studio browser acceptance test. */ export function resolveChromeExecutable() { @@ -22,3 +23,18 @@ export function resolveChromeExecutable() { ...installed, ].find((candidate) => candidate && existsSync(candidate)); } + +/** Headless Chrome for a Studio browser gate; exits 2 when no Chrome is installed. */ +export async function launchStudioChrome() { + const executablePath = resolveChromeExecutable(); + if (!executablePath) { + console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); + process.exit(2); + } + const browser = await puppeteer.launch({ + executablePath, + headless: true, + args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"], + }); + return { browser, executablePath }; +} diff --git a/packages/studio/tests/e2e/fade-handles.mjs b/packages/studio/tests/e2e/fade-handles.mjs index 18f5cb3c162..a4e4a9a5c22 100644 --- a/packages/studio/tests/e2e/fade-handles.mjs +++ b/packages/studio/tests/e2e/fade-handles.mjs @@ -1,8 +1,7 @@ #!/usr/bin/env node // On a clip narrower than two fade hit boxes, each dot must still lay out at 10 x 10. // Flex shrinking only happens in real layout, so this measures it in Chrome. -import puppeteer from "puppeteer-core"; -import { resolveChromeExecutable } from "./chrome-executable.mjs"; +import { launchStudioChrome } from "./chrome-executable.mjs"; const STUDIO_URL = process.env.STUDIO_URL; const DOT_PX = 10; @@ -12,17 +11,7 @@ if (!STUDIO_URL) { console.error("STUDIO_URL is required and must point at the fade-handles fixture"); process.exit(2); } -const executablePath = resolveChromeExecutable(); -if (!executablePath) { - console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); - process.exit(2); -} - -const browser = await puppeteer.launch({ - executablePath, - headless: true, - args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"], -}); +const { browser } = await launchStudioChrome(); const failures = []; let evidence = {}; try { diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs index 8576dc951f1..d9f0f3f639c 100644 --- a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs +++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs @@ -9,6 +9,15 @@ export function percentile(values, ratio) { return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; } +function assertSampleCount(expected, interactions, frameIntervals) { + if (expected > 0 && interactions.length === expected && frameIntervals.length === expected) + return; + throw new Error( + `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` + + `and ${frameIntervals.length} frame intervals`, + ); +} + /** * p95 over every measured step pooled: one run's p95 is only its 4th-worst step, so a brief runner stall failed it. * Throws when any run is short of samples, so a missing measurement cannot read as a fast one. @@ -19,13 +28,7 @@ export function judgeResponsiveness( ) { const interactions = runs.flatMap((run) => run.interactions); const frameIntervals = runs.flatMap((run) => run.frameIntervals); - const expected = runs.length * samplesPerRun; - if (expected === 0 || interactions.length !== expected || frameIntervals.length !== expected) { - throw new Error( - `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` + - `and ${frameIntervals.length} frame intervals`, - ); - } + assertSampleCount(runs.length * samplesPerRun, interactions, frameIntervals); const interactionP95Ms = percentile(interactions, 0.95); const frameIntervalP95Ms = percentile(frameIntervals, 0.95); return { diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index b6b794a2a25..ff6435755f0 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -22,8 +22,7 @@ * it, and a mismatch would otherwise pass silently against the wrong build. */ import { platform, arch } from "node:os"; -import puppeteer from "puppeteer-core"; -import { resolveChromeExecutable } from "./chrome-executable.mjs"; +import { launchStudioChrome } from "./chrome-executable.mjs"; import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs"; const STUDIO_URL = process.env.STUDIO_URL; @@ -224,17 +223,7 @@ async function measureMaximumReliableScrollWidth(page) { }); } -const executablePath = resolveChromeExecutable(); -if (!executablePath) { - console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); - process.exit(2); -} - -const browser = await puppeteer.launch({ - executablePath, - headless: true, - args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"], -}); +const { browser, executablePath } = await launchStudioChrome(); let exitCode = 1; try { const version = await browser.version();