test(studio): score edits on GSAP-animated size, scale, rotation and keyframes (#4866)

* test(studio): score edits on GSAP-animated size, scale, rotation, keyframes and from/fromTo

* test(studio): settle the edit bench only once no shadow preview is waiting to be swapped in

* test(studio): end a drag at its recorded pointer-up, even when no frame painted while held

* ci(studio): run the keyframed edit accuracy cases in CI across 20 shards

* refactor(studio): split the keyframe render and stray-css checks into small helpers

* test(studio): bank the edit bench's smooth verdict as smooth, judged on the worst frame's work

* style(studio): name the edit gate's smooth count helper

* test(studio): count smooth edit cases only among the accurate ones

* test(studio): only keyframed cases wait for the preview swap, so others keep main's undo timing

* refactor(studio): keep the edit bench's keyframe check apart from the hidden preview scan

* test(studio): check the keyframed cases are in the full grid without pinning its size

* test(studio): quarantine the nudge sequence whose redo waits behind preview videos (#4889)

* test(studio): bank the 255 keyframed edit cases that pass

Author: miguel-heygen

.github/workflows/ci.yml

@@ -1013,15 +1013,15 @@ jobs:
 
   # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json.
   studio-edit-accuracy:
-    name: "Studio: edit accuracy (${{ matrix.shard }}/12)"
+    name: "Studio: edit accuracy (${{ matrix.shard }}/20)"
     needs: [changes, build]
     if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -1046,7 +1046,7 @@ jobs:
         run: |
           set -euo pipefail
           bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; }
-          bench --shard "${{ matrix.shard }}/12" --out /tmp/edit-accuracy/run1
+          bench --shard "${{ matrix.shard }}/20" --out /tmp/edit-accuracy/run1
           mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
             /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
           if (( ${#FLIPPED[@]} > 0 )); then

packages/studio/tests/e2e/edit-accuracy/baseline.json

@@ -242,20 +242,44 @@ const previewFrames = (page) =>
     .filter((u) => u.includes("/preview"))
     .join(" ");
 
-/** Measures once the preview frames and the box have held still for STILL_MS; Studio updates both after a save. */
+/** A hidden preview holding the target is a shadow reload not yet promoted: the visible frame is about to go stale. */
+async function hiddenTarget(page, selector = "#target") {
+  for (const f of page.frames().filter((f) => f.url().includes("/preview"))) {
+    const host = await f.frameElement().catch(() => null);
+    const shown = await host?.evaluate((e) => e.checkVisibility({ visibilityProperty: true }));
+    if (shown === false && (await f.$(selector).catch(() => null))) return true;
+  }
+  return false;
+}
+
+// Keyframed cases only: waiting out the swap lands a later undo in the preview's burst of requests.
+export const swapPending = (ctx) => Boolean(ctx.keys) && hiddenTarget(ctx.page, ctx.selector);
+
+/** Restart the stillness window: a pending swap, a changed set of preview frames, or the box moved. */
+export const unsettledBy = (start, now) =>
+  start.pending ||
+  now.pending ||
+  now.frames !== start.frames ||
+  quadDistance(now.m.visible, start.m.visible) >= 0.01;
+
+/** Measures once the shown preview and the box have held still for STILL_MS; Studio updates both after a save. */
 // fallow-ignore-next-line complexity
 export async function settled(ctx, timeout = 15_000) {
   const deadline = Date.now() + timeout;
-  let start = { m: await measure(ctx), frames: previewFrames(ctx.page) };
+  const read = async () => ({
+    m: await measure(ctx),
+    frames: previewFrames(ctx.page),
+    pending: await swapPending(ctx),
+  });
+  let start = await read();
   let now = start;
   // Compared with the window's first read, so a drift too slow to show read to read still restarts it.
-  for (let since = Date.now(); Date.now() - since < STILL_MS; ) {
+  for (let since = Date.now(); start.pending || Date.now() - since < STILL_MS; ) {
     // A preview that never holds still is a Studio defect: the metrics it feeds fail, the rest still count.
     if (Date.now() > deadline) return { ...now.m, unsettled: true };
     await nextFrame(ctx.page);
-    now = { m: await measure(ctx), frames: previewFrames(ctx.page) };
-    if (now.frames !== start.frames || quadDistance(now.m.visible, start.m.visible) >= 0.01)
-      [start, since] = [now, Date.now()];
+    now = await read();
+    if (unsettledBy(start, now)) [start, since] = [now, Date.now()];
   }
   return now.m;
 }
@@ -268,9 +292,14 @@ export async function openStudio(ctx) {
   let seek = null;
   for (const deadline = Date.now() + 30_000; Date.now() < deadline; await sleep(250)) {
     seek = await ctx.page
-      .evaluate((time) => window.__editBench.call("studio_seek", { time }), PLAYHEAD)
+      .evaluate((time) => window.__editBench.call("studio_seek", { time }), ctx.playhead)
       .catch(String);
-    if (seek?.ok && seek.duration > 0 && seek.playhead === PLAYHEAD && (await findTarget(ctx.page)))
+    if (
+      seek?.ok &&
+      seek.duration > 0 &&
+      seek.playhead === ctx.playhead &&
+      (await findTarget(ctx.page))
+    )
       break;
     seek = null;
   }
@@ -279,6 +308,75 @@ export async function openStudio(ctx) {
   return settled(ctx);
 }
 
+async function seekTo(ctx, time) {
+  const seek = await ctx.page.evaluate(
+    (t) => window.__editBench.call("studio_seek", { time: t }),
+    time,
+  );
+  if (!seek?.ok || seek.playhead !== time)
+    throw new Error(`studio_seek ${time}: ${JSON.stringify(seek)}`);
+  return settled(ctx);
+}
+
+/** Each GSAP-animated property's value at every other keyframe time (and the box there), then back to the playhead. */
+async function readKeyframes(ctx, keys, withBox = false) {
+  const at = {};
+  for (const time of keys.times) {
+    const m = await seekTo(ctx, time);
+    const values = await ctx.handles.target.evaluate((el, props) => {
+      const gsap = el.ownerDocument.defaultView.gsap;
+      return Object.fromEntries(props.map((p) => [p, Number.parseFloat(gsap.getProperty(el, p))]));
+    }, keys.props);
+    at[time] = { values, ...(withBox && { visible: m.visible }) };
+  }
+  await seekTo(ctx, ctx.playhead);
+  return at;
+}
+
+// GSAP's own numbers (px, deg, scale): an untouched keyframe reads back exactly.
+const KEY_TOLERANCE = 0.01;
+
+/** The largest change of an animated value at a keyframe the edit was not on; NaN (unreadable) fails. */
+export function keyframeDrift(before, after) {
+  let worst = { diff: 0, time: null, prop: null };
+  for (const [time, b] of Object.entries(before))
+    for (const [prop, v] of Object.entries(b.values)) {
+      const diff = Math.abs(after[time].values[prop] - v);
+      if (!(diff <= worst.diff)) worst = { diff, time: Number(time), prop };
+    }
+  return { ...worst, pass: worst.diff <= KEY_TOLERANCE };
+}
+
+const declarations = (text = "") =>
+  Object.fromEntries(
+    text
+      .split(";")
+      .map((d) => d.split(":"))
+      .filter((d) => d.length > 1)
+      .map(([k, ...v]) => [k.trim(), v.join(":").trim()]),
+  );
+const capture = (re, text = "") => re.exec(text)?.[1];
+const targetCss = (html) => ({
+  rule: declarations(capture(/#target\s*\{([^}]*)\}/, html)),
+  inline: declarations(capture(/\bstyle="([^"]*)"/, capture(/(<[^>]*\bid="target"[^>]*>)/, html))),
+});
+
+/** Plain CSS the edit wrote for a property GSAP animates: it would override or fight the timeline. */
+export function strayCss(original, saved, props) {
+  const changed = (a, b, file, where) =>
+    props
+      .filter((p) => a[p] !== b[p])
+      .map((p) => `${file} ${where} ${p}: ${a[p] ?? "-"} -> ${b[p] ?? "-"}`);
+  const stray = Object.keys(original).flatMap((file) => {
+    const [a, b] = [targetCss(original[file]), targetCss(saved[file])];
+    return [
+      ...changed(a.rule, b.rule, file, "rule"),
+      ...changed(a.inline, b.inline, file, "inline"),
+    ];
+  });
+  return { pass: stray.length === 0, stray };
+}
+
 /** Puppeteer presses one key at a time: hold the modifiers around the last key. */
 export async function chord(page, keys) {
   const [key, ...mods] = keys.split("+").reverse();
@@ -672,7 +770,14 @@ async function nudgeGesture(ctx, pre) {
 export async function inStudio({ browser, spec, dir, files, url, evidence }, drive) {
   const context = await browser.createBrowserContext();
   const page = await context.newPage();
-  const ctx = { page, dir, files, handles: null };
+  const ctx = {
+    page,
+    dir,
+    files,
+    handles: null,
+    playhead: spec.playhead ?? PLAYHEAD,
+    keys: spec.keys,
+  };
   const consoleErrors = [];
   page.on("pageerror", (e) => consoleErrors.push(e.message));
   evidence.shots = {};
@@ -686,10 +791,12 @@ export async function inStudio({ browser, spec, dir, files, url, evidence }, dri
     let pre = await openStudio(ctx);
     await disableSnap(page);
     const zoom = await setZoom(ctx, spec.zoom, pre.map.toScreen(centre(pre.visible)));
+    // The animated values at the other keyframes, read before anything is selected or edited.
+    const keysBefore = spec.keys && (await readKeyframes(ctx, spec.keys));
     pre = await settled(ctx);
     await selectTarget(ctx, pre);
     pre = await settled(ctx);
-    return await drive({ ctx, page, pre, zoom, shoot, consoleErrors });
+    return await drive({ ctx, page, pre, zoom, shoot, consoleErrors, keysBefore });
   } catch (error) {
     await shoot("error").catch(() => undefined);
     throw error;
@@ -707,7 +814,7 @@ export async function runCase(args) {
 // fallow-ignore-next-line complexity
 async function measureCase(
   { spec, dir, files, evidence },
-  { ctx, page, pre, zoom, shoot, consoleErrors },
+  { ctx, page, pre, zoom, shoot, consoleErrors, keysBefore },
   control,
 ) {
   const original = readFiles(dir, files);
@@ -753,6 +860,8 @@ async function measureCase(
   await page.reload();
   const reloaded = await openStudio(ctx);
   await shoot("reloaded");
+  // From the saved file: the other keyframes keep their values, and no animated property gets plain CSS.
+  const keysAfter = spec.keys && (await readKeyframes(ctx, spec.keys, true));
   const quads = Object.fromEntries(
     Object.entries({ pre, committed, undone, redone, reloaded }).filter(([, m]) => m),
   );
@@ -793,6 +902,11 @@ async function measureCase(
     smooth: { ...drive.smooth, control },
     unsettled: Object.keys(quads).filter((k) => quads[k].unsettled),
     reloaded,
+    ...(spec.keys && {
+      keys: keyframeDrift(keysBefore, keysAfter),
+      css: strayCss(original, readFiles(dir, files), spec.keys.css),
+      keyRender: { time: spec.keys.render, visible: keysAfter[spec.keys.render].visible },
+    }),
     diag: {
       ...drive.diag,
       consoleErrors: consoleErrors.slice(0, 5),

packages/studio/tests/e2e/edit-accuracy/case.mjs

@@ -23,6 +23,76 @@ const AXES = {
   zoom: [50, 100, 200],
 };
 
+// Timelines with keyframes at 0, 2 and 3 s on the property each animates: `css` is what that property is in CSS.
+const T = '"#target"';
+const KEYFRAMED = {
+  size: {
+    lines: [
+      `tl.to(${T}, { width: 300, height: 200, duration: 2, ease: "none" }, 0);`,
+      `tl.to(${T}, { width: 360, height: 240, duration: 1, ease: "none" }, 2);`,
+    ],
+    props: ["width", "height"],
+    css: ["width", "height"],
+  },
+  scale: {
+    lines: [
+      `tl.to(${T}, { scale: 1.25, duration: 2, ease: "none" }, 0);`,
+      `tl.to(${T}, { scale: 1.5, duration: 1, ease: "none" }, 2);`,
+    ],
+    props: ["scaleX", "scaleY"],
+    css: ["scale", "transform"],
+  },
+  spin: {
+    lines: [
+      `tl.to(${T}, { rotation: 20, duration: 2, ease: "none" }, 0);`,
+      `tl.to(${T}, { rotation: 40, duration: 1, ease: "none" }, 2);`,
+    ],
+    props: ["rotation"],
+    css: ["rotate", "transform"],
+  },
+  // A resize meets a keyframes array that also animates width.
+  keys: {
+    lines: [
+      `tl.to(${T}, { keyframes: [{ x: 60, width: 280, duration: 2, ease: "none" }, { x: 120, width: 320, duration: 1, ease: "none" }] }, 0);`,
+    ],
+    props: ["x", "width"],
+    css: ["left", "top", "translate", "transform", "width"],
+  },
+  fromto: {
+    lines: [
+      `tl.from(${T}, { x: -60, duration: 2, ease: "none" }, 0);`,
+      `tl.fromTo(${T}, { x: 0 }, { x: 60, duration: 1, ease: "none" }, 2);`,
+    ],
+    props: ["x"],
+    css: ["left", "top", "translate", "transform"],
+  },
+};
+const KEY_TIMES = [0, 2, 3];
+// On a keyframe the edit changes that keyframe; between two it adds one at the playhead.
+const AT = { on: 2, mid: 1 };
+
+function keyframedCases() {
+  return product({
+    gsap: Object.keys(KEYFRAMED),
+    placement: ["px"],
+    rotation: AXES.rotation,
+    nesting: AXES.nesting,
+    zoom: AXES.zoom,
+    gesture: GESTURES,
+    at: Object.keys(AT),
+  }).map((c) => ({
+    id: `${caseId(c)}-${c.at}`,
+    ...c,
+    playhead: AT[c.at],
+    keys: {
+      times: KEY_TIMES.filter((t) => t !== AT[c.at]),
+      props: KEYFRAMED[c.gsap].props,
+      css: KEYFRAMED[c.gsap].css,
+      render: KEY_TIMES.at(-1),
+    },
+  }));
+}
+
 const product = (axes) =>
   Object.entries(axes).reduce(
     (rows, [key, values]) => rows.flatMap((row) => values.map((v) => ({ ...row, [key]: v }))),
@@ -31,14 +101,15 @@ const product = (axes) =>
 const caseId = (c) =>
   [c.gesture, c.gsap, c.placement, `r${c.rotation}`, c.nesting, `z${c.zoom}`].join("-");
 
-/** `pr` is a smaller slice for CI; its final size is still an open decision. */
+/** `pr` is a smaller slice for CI; `keyframes` is the GSAP-animated set alone, which `full` also runs. */
 export function buildGrid(kind = "full") {
+  if (kind === "keyframes") return keyframedCases();
   return (
     product({ ...AXES, gesture: GESTURES })
       // xPercent only exists through GSAP on the target itself.
       .filter((c) => c.placement !== "xpercent" || !["none", "idle"].includes(c.gsap))
       .map((c) => ({ id: caseId(c), ...c, other: c.gsap === "idle" }))
-      .concat(dragCases())
+      .concat(dragCases(), keyframedCases())
       .filter((c) => kind !== "pr" || (c.zoom === 100 && c.nesting === "root"))
   );
 }
@@ -74,6 +145,7 @@ function targetCss(spec) {
 
 // fallow-ignore-next-line complexity
 function gsapLines(spec) {
+  if (KEYFRAMED[spec.gsap]) return KEYFRAMED[spec.gsap].lines;
   if (spec.gsap === "idle") return [`tl.to("#other", { x: 120, duration: 4, ease: "none" }, 0);`];
   const percent = spec.placement === "xpercent" ? ", xPercent: -50, yPercent: -50" : "";
   if (spec.gsap === "hold") return [`gsap.set("#target", { x: 40, y: 20${percent} });`];