diff --git a/lib/core/engine/collector.js b/lib/core/engine/collector.js index 820f904c5..7699ef48e 100644 --- a/lib/core/engine/collector.js +++ b/lib/core/engine/collector.js @@ -212,9 +212,14 @@ export class Collector { this._logVisualMetrics(data); } for (let key of Object.keys(data.visualMetrics)) { - // Skip VisualProgress/ContentfulProgress etc, and FrameDiffs — - // an array of per-frame objects, nothing to summarize. - if (!key.includes('Progress') && key !== 'FrameDiffs') { + // Skip VisualProgress/ContentfulProgress etc, FrameDiffs (an + // array of per-frame objects) and FrameInstability (a summary + // object) — nothing to summarize. + if ( + !key.includes('Progress') && + key !== 'FrameDiffs' && + key !== 'FrameInstability' + ) { const d = { visualMetrics: {} }; d['visualMetrics'][key] = data.visualMetrics[key]; statistics.addDeep(d); diff --git a/visualmetrics/test_visualmetrics.py b/visualmetrics/test_visualmetrics.py index ba07afd5a..b1d03265a 100644 --- a/visualmetrics/test_visualmetrics.py +++ b/visualmetrics/test_visualmetrics.py @@ -93,10 +93,6 @@ def test_perceptual_speed_index(self): value, _ = vm.calculate_perceptual_speed_index(self.progress, "test_data") self.assertEqual(value, 947) - @unittest.skipUnless( - hasattr(vm, "create_frame_diffs"), - "create_frame_diffs is not in this branch yet", - ) def test_create_frame_diffs(self): frames_dir = Path(self.tmp) / "framediffs" frames_dir.mkdir() @@ -104,7 +100,7 @@ def test_create_frame_diffs(self): for frame in frames: shutil.copy(frame, frames_dir) - diffs = vm.create_frame_diffs(str(frames_dir)) + diffs, instability = vm.create_frame_diffs(str(frames_dir)) self.assertEqual(len(diffs), len(frames) - 1) expected_times = [int(frame.stem[3:]) for frame in frames[1:]] @@ -118,6 +114,16 @@ def test_create_frame_diffs(self): # The first frame has no predecessor, so no diff for it. self.assertFalse((frames_dir / "diff_000000.png").is_file()) + self.assertTrue((frames_dir / "instability.png").is_file()) + self.assertGreater(instability["changedPixels"], 0) + self.assertGreater(instability["changedShare"], 0) + self.assertLessEqual(instability["changedShare"], 100) + self.assertGreaterEqual( + instability["changedPixels"], instability["repeatedlyChangedPixels"] + ) + self.assertGreaterEqual(instability["maxChanges"], 1) + self.assertLessEqual(instability["maxChanges"], len(frames) - 1) + if __name__ == "__main__": unittest.main() diff --git a/visualmetrics/visualmetrics-portable.py b/visualmetrics/visualmetrics-portable.py index af4defaa7..91506642f 100755 --- a/visualmetrics/visualmetrics-portable.py +++ b/visualmetrics/visualmetrics-portable.py @@ -1231,13 +1231,26 @@ def create_frame_diffs(directory): Runs before convert_to_jpeg (which only globs ms_*.png) while the frames are still lossless PNG. + + The same loop also counts how many times each pixel changed across + the whole run and writes instability.png: the final frame ghosted + toward white with every changed pixel tinted from amber (changed + once) to deep red (changed 10 or more times). The color scale is + fixed, not normalized per run, so heatmaps from two runs are + comparable side by side. A page that paints once and stays put is + all amber; text that re-rasterizes over and over shows up as red + speckle. The summary counts are returned next to the per-frame + diffs, with "repeatedly" meaning 5 or more changes. """ diffs = [] + instability = None try: import numpy as np from PIL import Image files = sorted(glob.glob(str(Path(directory) / "ms_*.png"))) + counts = None + final = None for previous_file, current_file in zip(files, files[1:]): with Image.open(previous_file) as previous_image, Image.open( current_file @@ -1248,6 +1261,11 @@ def create_frame_diffs(directory): continue delta = np.abs(previous - current).sum(axis=2) changed = delta > 30 + if counts is None: + counts = np.zeros(changed.shape, dtype=int) + if counts.shape == changed.shape: + counts += changed + final = current out = 255 - (255 - current) * 0.2 out[changed] = [224, 28, 60] stem = Path(current_file).stem @@ -1265,9 +1283,32 @@ def create_frame_diffs(directory): ), } ) + if counts is not None and final is not None: + cap = 10 + heat = np.sqrt(np.minimum(counts, cap) / cap) + amber = np.array([255.0, 196.0, 60.0]) + red = np.array([201.0, 16.0, 46.0]) + color = amber + (red - amber) * heat[:, :, None] + alpha = np.where(counts > 0, 0.35 + 0.65 * heat, 0)[:, :, None] + ghost = 255 - (255 - final) * 0.15 + heatmap = ghost * (1 - alpha) + color * alpha + Image.fromarray(heatmap.astype(np.uint8)).save( + str(Path(directory) / "instability.png") + ) + changed_ever = int((counts > 0).sum()) + changed_repeatedly = int((counts >= 5).sum()) + instability = { + "changedPixels": changed_ever, + "changedShare": round(changed_ever / counts.size * 100, 2), + "repeatedlyChangedPixels": changed_repeatedly, + "repeatedlyChangedShare": round( + changed_repeatedly / counts.size * 100, 2 + ), + "maxChanges": int(counts.max()), + } except BaseException: logging.exception("Error creating frame diffs") - return diffs + return diffs, instability def convert_to_jpeg(directory, quality): @@ -2160,9 +2201,13 @@ def main(): ) if options.framediff and options.dir is not None and metrics is not None: - framediffs = create_frame_diffs(directory) + framediffs, instability = create_frame_diffs(directory) if framediffs: metrics.append({"name": "Frame Diffs", "value": framediffs}) + if instability: + metrics.append( + {"name": "Frame Instability", "value": instability} + ) if options.screenshot is not None: quality = 30