Lossy WebP encoder: skip modes that can't win earlier

The rate-distortion score only grows with each term, so a mode whose
header bits (intra 4x4) or distortion (intra 16x16, chroma) already reach
the best score is dropped before the reconstruction, texture distortion or
rate is computed. Quantized intra 4x4 blocks without levels are the
prediction, without an inverse transform; tTransform keeps its sums in
locals. 23-28% faster at effort 3-6, output unchanged (140 hashes over 5
images x efforts 0-6 x 4 qualities, cwebp tests).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lfv41PqHEGbU7SLg5onpG1

Author: matmen

CHANGELOG.md

@@ -15,6 +15,11 @@
 - AVIF encoding with `effort: 2` also chooses smaller transforms within blocks (transform depth): files about 1% smaller
   (up to 4% for graphics), encoding about 10% slower.
 
+### Performance
+
+- Lossy WebP encoding with `effort` 3 to 6 (the default is 4) is about 25% faster: modes that can no longer beat the
+  best one are dropped before their distortion and rate are computed. The files stay byte for byte what cwebp writes.
+
 ## 3.5.0
 
 ### Features

src/codecs/vp8-enc-dsp.ts

@@ -804,21 +804,28 @@ export function sse16(a: Uint8Array, ao: number, b: Uint8Array, bo: number, h: n
     return count;
 }
 
-/** The weighted sum of the absolute Hadamard transform coefficients of a 4x4 block */
+/** The weighted sum of the absolute Hadamard transform coefficients of a 4x4 block (rows, then columns; unrolled) */
 export function tTransform(input: Uint8Array, i: number, w: ArrayLike<number>): number {
-    let sum = 0;
-    for (let k = 0; k < 4; k++, i += BPS) {
-        const a0 = input[i] + input[i + 2], a1 = input[i + 1] + input[i + 3], a2 = input[i + 1] - input[i + 3], a3 = input[i] - input[i + 2];
-        ftmp[k * 4] = a0 + a1;
-        ftmp[k * 4 + 1] = a3 + a2;
-        ftmp[k * 4 + 2] = a3 - a2;
-        ftmp[k * 4 + 3] = a0 - a1;
-    }
-    for (let k = 0; k < 4; k++) {
-        const a0 = ftmp[k] + ftmp[8 + k], a1 = ftmp[4 + k] + ftmp[12 + k], a2 = ftmp[4 + k] - ftmp[12 + k], a3 = ftmp[k] - ftmp[8 + k];
-        sum += w[k] * Math.abs(a0 + a1) + w[k + 4] * Math.abs(a3 + a2) + w[k + 8] * Math.abs(a3 - a2) + w[k + 12] * Math.abs(a0 - a1);
-    }
-    return sum;
+    const r00 = input[i], r01 = input[i + 1], r02 = input[i + 2], r03 = input[i + 3];
+    const a00 = r00 + r02, a01 = r01 + r03, a02 = r01 - r03, a03 = r00 - r02;
+    const h00 = a00 + a01, h01 = a03 + a02, h02 = a03 - a02, h03 = a00 - a01;
+    const r10 = input[i + 1 * BPS], r11 = input[i + 1 * BPS + 1], r12 = input[i + 1 * BPS + 2], r13 = input[i + 1 * BPS + 3];
+    const a10 = r10 + r12, a11 = r11 + r13, a12 = r11 - r13, a13 = r10 - r12;
+    const h10 = a10 + a11, h11 = a13 + a12, h12 = a13 - a12, h13 = a10 - a11;
+    const r20 = input[i + 2 * BPS], r21 = input[i + 2 * BPS + 1], r22 = input[i + 2 * BPS + 2], r23 = input[i + 2 * BPS + 3];
+    const a20 = r20 + r22, a21 = r21 + r23, a22 = r21 - r23, a23 = r20 - r22;
+    const h20 = a20 + a21, h21 = a23 + a22, h22 = a23 - a22, h23 = a20 - a21;
+    const r30 = input[i + 3 * BPS], r31 = input[i + 3 * BPS + 1], r32 = input[i + 3 * BPS + 2], r33 = input[i + 3 * BPS + 3];
+    const a30 = r30 + r32, a31 = r31 + r33, a32 = r31 - r33, a33 = r30 - r32;
+    const h30 = a30 + a31, h31 = a33 + a32, h32 = a33 - a32, h33 = a30 - a31;
+    const b00 = h00 + h20, b01 = h10 + h30, b02 = h10 - h30, b03 = h00 - h20;
+    const b10 = h01 + h21, b11 = h11 + h31, b12 = h11 - h31, b13 = h01 - h21;
+    const b20 = h02 + h22, b21 = h12 + h32, b22 = h12 - h32, b23 = h02 - h22;
+    const b30 = h03 + h23, b31 = h13 + h33, b32 = h13 - h33, b33 = h03 - h23;
+    return w[0] * Math.abs(b00 + b01) + w[4] * Math.abs(b03 + b02) + w[8] * Math.abs(b03 - b02) + w[12] * Math.abs(b00 - b01) +
+        w[1] * Math.abs(b10 + b11) + w[5] * Math.abs(b13 + b12) + w[9] * Math.abs(b13 - b12) + w[13] * Math.abs(b10 - b11) +
+        w[2] * Math.abs(b20 + b21) + w[6] * Math.abs(b23 + b22) + w[10] * Math.abs(b23 - b22) + w[14] * Math.abs(b20 - b21) +
+        w[3] * Math.abs(b30 + b31) + w[7] * Math.abs(b33 + b32) + w[11] * Math.abs(b33 - b32) + w[15] * Math.abs(b30 - b31);
 }
 
 /**

src/codecs/vp8-enc.ts

@@ -1217,7 +1217,17 @@ function reconstructIntra4(it: Iterator, levels: Int16Array, lo: number, src: Ui
     fTransform(src, s, it.yuvP, r, tmpLevels, 0);
     const nz = it.doTrellis ? trellisQuantize(it.enc, tmpLevels, 0, levels, lo, it.topNz[it.i4 & 3] + it.leftNz[it.i4 >> 2], TYPE_I4_AC, dqm.y1, dqm.lambdaTrellisI4)
         : quantizeBlock(tmpLevels, 0, levels, lo, dqm.y1) ? 1 : 0;
-    iTransform(it.yuvP, r, tmpLevels, 0, out, o, false);
+    // (without levels, the block is the prediction)
+    if (nz) iTransform(it.yuvP, r, tmpLevels, 0, out, o, false);
+    else {
+        const p = it.yuvP;
+        for (let k = 0; k < 4 * BPS; k += BPS) {
+            out[o + k] = p[r + k];
+            out[o + k + 1] = p[r + k + 1];
+            out[o + k + 2] = p[r + k + 2];
+            out[o + k + 3] = p[r + k + 3];
+        }
+    }
     return nz;
 }
 
@@ -1292,17 +1302,17 @@ function pickBestIntra16(it: Iterator, rd: ModeScore): void {
         rdCur.modeI16 = mode;
         rdCur.nz = reconstructIntra16(it, rdCur, dst, Y_OFF, mode);
         rdCur.D = sse16(it.yuvIn, Y_OFF, dst, Y_OFF, 16);
-        rdCur.SD = tlambda ? (tlambda * tDisto16x16(sourceSpectra, dst, Y_OFF, WEIGHT_Y) + 128) >> 8 : 0;
         rdCur.H = FIXED_COSTS_I16[mode];
+        // a very flat block: emphasis on low distortion
+        if (flat) flat = isFlat(rdCur.yAcLevels, 0, 16, FLATNESS_LIMIT_I16);
+        if (flat) rdCur.D *= 2;
+        // (modes that can't beat the best without texture distortion and rate are skipped before them)
+        rdCur.SD = rdCur.R = 0;
+        rdCur.setRDScore(lambda);
+        if (mode > 0 && rdCur.score >= rdBest.score) continue;
+        rdCur.SD = tlambda ? (tlambda * tDisto16x16(sourceSpectra, dst, Y_OFF, WEIGHT_Y) + 128) >> 8 : 0;
+        if (flat) rdCur.SD *= 2;
         rdCur.R = costLuma16(it, rdCur);
-        if (flat) {
-            // a very flat block: emphasis on low distortion
-            flat = isFlat(rdCur.yAcLevels, 0, 16, FLATNESS_LIMIT_I16);
-            if (flat) {
-                rdCur.D *= 2;
-                rdCur.SD *= 2;
-            }
-        }
         rdCur.setRDScore(lambda);
         if (mode === 0 || rdCur.score < rdBest.score) {
             const swap = rdCur;
@@ -1353,12 +1363,18 @@ function pickBestIntra4(it: Iterator, rd: ModeScore): boolean {
         const spectrum = tlambda ? tTransform(it.yuvIn, src, WEIGHT_Y) : 0;
         for (let mode = 0; mode < 10; mode++) {
             const r = rdTmpI4;
+            // (the score only grows from the header bits on: modes that can't beat the best are skipped as early as
+            // possible, which doesn't change the choice)
+            r.H = costs[modeCosts + mode];
+            if (bestMode >= 0 && r.H * lambda >= rdI4.score) continue;
             r.nz = reconstructIntra4(it, tmpLevels2, 0, it.yuvIn, src, tmpDst, td, mode) << it.i4;
             r.D = sse4x4(it.yuvIn, src, tmpDst, td);
-            r.SD = tlambda ? (tlambda * tDisto4x4(spectrum, tmpDst, td, WEIGHT_Y) + 128) >> 8 : 0;
-            r.H = costs[modeCosts + mode];
             // flat areas shouldn't be predicted by complex modes
             r.R = mode > 0 && isFlat(tmpLevels2, 0, 1, FLATNESS_LIMIT_I4) ? FLATNESS_PENALTY : 0;
+            r.SD = 0;
+            r.setRDScore(lambda);
+            if (bestMode >= 0 && r.score >= rdI4.score) continue;
+            r.SD = tlambda ? (tlambda * tDisto4x4(spectrum, tmpDst, td, WEIGHT_Y) + 128) >> 8 : 0;
             r.setRDScore(lambda);
             if (bestMode >= 0 && r.score >= rdI4.score) continue;
             r.R += costLuma4(it, tmpLevels2);
@@ -1412,8 +1428,11 @@ function pickBestUv(it: Iterator, rd: ModeScore): void {
         r.D = sse16(it.yuvIn, U_OFF, tmpDst, U_OFF, 8);
         r.SD = 0;
         r.H = FIXED_COSTS_UV[mode];
-        r.R = costUv(it, r);
-        if (mode > 0 && isFlat(r.uvLevels, 0, 8, FLATNESS_LIMIT_UV)) r.R += FLATNESS_PENALTY * 8;
+        r.R = mode > 0 && isFlat(r.uvLevels, 0, 8, FLATNESS_LIMIT_UV) ? FLATNESS_PENALTY * 8 : 0;
+        // (modes that can't beat the best without their rate are skipped before it)
+        r.setRDScore(lambda);
+        if (mode > 0 && r.score >= rdBestUv.score) continue;
+        r.R += costUv(it, r);
         r.setRDScore(lambda);
         if (mode === 0 || r.score < rdBestUv.score) {
             rdBestUv.copyScore(r);