From f83d6216ecbe94d2def3f3f77645bcb478c37f76 Mon Sep 17 00:00:00 2001 From: lordnn Date: Wed, 12 Aug 2026 01:08:39 +0300 Subject: [PATCH 1/6] New `oapv_dc_removed_had8x8_neon()` implementation. Signed-off-by: lordnn --- src/neon/oapv_sad_neon.c | 479 +++++++++++---------------------------- 1 file changed, 132 insertions(+), 347 deletions(-) diff --git a/src/neon/oapv_sad_neon.c b/src/neon/oapv_sad_neon.c index fe0ff85c..58530a2d 100644 --- a/src/neon/oapv_sad_neon.c +++ b/src/neon/oapv_sad_neon.c @@ -34,6 +34,8 @@ #if ARM_NEON +#define VADDVQ_S32(sum, sads) sum += vaddvq_s32(sads); + /* SAD for 16bit **************************************************************/ /* SSD ***********************************************************************/ static s64 ssd_16b_neon_8x8(int w, int h, void *src1, void *src2, int s_src1, int s_src2) @@ -223,355 +225,138 @@ const oapv_fn_ssd_t oapv_tbl_fn_ssd_16b_neon[2] = NULL}; /* DIFF **********************************************************************/ + int oapv_dc_removed_had8x8_neon(pel* org, int s_org) { int satd = 0; - /* all 128 bit registers are named with a suffix mxnb, where m is the */ - /* number of n bits packed in the register */ - - int16x8_t src0_8x16b, src1_8x16b, src2_8x16b, src3_8x16b; - int16x8_t src4_8x16b, src5_8x16b, src6_8x16b, src7_8x16b; - int16x8_t pred0_8x16b, pred1_8x16b, pred2_8x16b, pred3_8x16b; - int16x8_t pred4_8x16b, pred5_8x16b, pred6_8x16b, pred7_8x16b; - int16x8_t out0_8x16b, out1_8x16b, out2_8x16b, out3_8x16b; - int16x8_t out4_8x16b, out5_8x16b, out6_8x16b, out7_8x16b; - - src0_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src1_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src2_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src3_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src4_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src5_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src6_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - src7_8x16b = (vld1q_s16(&org[0])); - org = org + s_org; - - /**************** 8x8 horizontal transform *******************************/ - /*********************** 8x8 16 bit Transpose ************************/ - - out3_8x16b = vcombine_s16(vget_low_s16(src0_8x16b), vget_low_s16(src1_8x16b)); - out7_8x16b = vcombine_s16(vget_high_s16(src0_8x16b), vget_high_s16(src1_8x16b)); - - pred0_8x16b = vcombine_s16(vget_low_s16(src2_8x16b), vget_low_s16(src3_8x16b)); - src2_8x16b = vcombine_s16(vget_high_s16(src2_8x16b), vget_high_s16(src3_8x16b)); - - out2_8x16b = vcombine_s16(vget_low_s16(src4_8x16b), vget_low_s16(src5_8x16b)); - pred7_8x16b = vcombine_s16(vget_high_s16(src4_8x16b), vget_high_s16(src5_8x16b)); - - pred3_8x16b = vcombine_s16(vget_low_s16(src6_8x16b), vget_low_s16(src7_8x16b)); - src6_8x16b = vcombine_s16(vget_high_s16(src6_8x16b), vget_high_s16(src7_8x16b)); - - - out1_8x16b = vzip1q_s32(out3_8x16b, pred0_8x16b); - out3_8x16b = vzip2q_s32(out3_8x16b, pred0_8x16b); - - pred1_8x16b = vzip1q_s32(out2_8x16b, pred3_8x16b); - pred3_8x16b = vzip2q_s32(out2_8x16b, pred3_8x16b); - - out5_8x16b = vzip1q_s32(out7_8x16b, src2_8x16b); - out7_8x16b = vzip2q_s32(out7_8x16b, src2_8x16b); - - pred5_8x16b = vzip1q_s32(pred7_8x16b, src6_8x16b); - pred7_8x16b = vzip2q_s32(pred7_8x16b, src6_8x16b); - - out0_8x16b = vzip1q_s64(out1_8x16b,pred1_8x16b); - out1_8x16b = vzip2q_s64(out1_8x16b,pred1_8x16b); - out2_8x16b = vzip1q_s64(out3_8x16b,pred3_8x16b); - out3_8x16b = vzip2q_s64(out3_8x16b,pred3_8x16b); - out4_8x16b = vzip1q_s64(out5_8x16b,pred5_8x16b); - out5_8x16b = vzip2q_s64(out5_8x16b,pred5_8x16b); - out6_8x16b = vzip1q_s64(out7_8x16b,pred7_8x16b); - out7_8x16b = vzip2q_s64(out7_8x16b,pred7_8x16b); - - /********************** 8x8 16 bit Transpose End *********************/ - - /* r0 + r1 */ - pred0_8x16b = vaddq_s16(out0_8x16b, out1_8x16b); - /* r2 + r3 */ - pred2_8x16b = vaddq_s16(out2_8x16b, out3_8x16b); - /* r4 + r5 */ - pred4_8x16b = vaddq_s16(out4_8x16b, out5_8x16b); - /* r6 + r7 */ - pred6_8x16b = vaddq_s16(out6_8x16b, out7_8x16b); - - - /* r0 + r1 + r2 + r3 */ - pred1_8x16b = vaddq_s16(pred0_8x16b, pred2_8x16b); - /* r4 + r5 + r6 + r7 */ - pred5_8x16b = vaddq_s16(pred4_8x16b, pred6_8x16b); - /* r0 + r1 + r2 + r3 + r4 + r5 + r6 + r7 */ - src0_8x16b = vaddq_s16(pred1_8x16b, pred5_8x16b); - /* r0 + r1 + r2 + r3 - r4 - r5 - r6 - r7 */ - src4_8x16b = vsubq_s16(pred1_8x16b, pred5_8x16b); - - /* r0 + r1 - r2 - r3 */ - pred1_8x16b = vsubq_s16(pred0_8x16b, pred2_8x16b); - /* r4 + r5 - r6 - r7 */ - pred5_8x16b = vsubq_s16(pred4_8x16b, pred6_8x16b); - /* r0 + r1 - r2 - r3 + r4 + r5 - r6 - r7 */ - src2_8x16b = vaddq_s16(pred1_8x16b, pred5_8x16b); - /* r0 + r1 - r2 - r3 - r4 - r5 + r6 + r7 */ - src6_8x16b = vsubq_s16(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 */ - pred0_8x16b = vsubq_s16(out0_8x16b, out1_8x16b); - /* r2 - r3 */ - pred2_8x16b = vsubq_s16(out2_8x16b, out3_8x16b); - /* r4 - r5 */ - pred4_8x16b = vsubq_s16(out4_8x16b, out5_8x16b); - /* r6 - r7 */ - pred6_8x16b = vsubq_s16(out6_8x16b, out7_8x16b); - - /* r0 - r1 + r2 - r3 */ - pred1_8x16b = vaddq_s16(pred0_8x16b, pred2_8x16b); - /* r4 - r5 + r6 - r7 */ - pred5_8x16b = vaddq_s16(pred4_8x16b, pred6_8x16b); - /* r0 - r1 + r2 - r3 + r4 - r5 + r6 - r7 */ - src1_8x16b = vaddq_s16(pred1_8x16b, pred5_8x16b); - /* r0 - r1 + r2 - r3 - r4 + r5 - r6 + r7 */ - src5_8x16b = vsubq_s16(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 - r2 + r3 */ - pred1_8x16b = vsubq_s16(pred0_8x16b, pred2_8x16b); - /* r4 - r5 - r6 + r7 */ - pred5_8x16b = vsubq_s16(pred4_8x16b, pred6_8x16b); - /* r0 - r1 - r2 + r3 + r4 - r5 - r6 + r7 */ - src3_8x16b = vaddq_s16(pred1_8x16b, pred5_8x16b); - /* r0 - r1 - r2 + r3 - r4 + r5 + r6 - r7 */ - src7_8x16b = vsubq_s16(pred1_8x16b, pred5_8x16b); - - - /*********************** 8x8 16 bit Transpose ************************/ - out3_8x16b = vzip1q_s16(src0_8x16b, src1_8x16b); - pred0_8x16b = vzip1q_s16(src2_8x16b, src3_8x16b); - out2_8x16b = vzip1q_s16(src4_8x16b, src5_8x16b); - pred3_8x16b = vzip1q_s16(src6_8x16b, src7_8x16b); - out7_8x16b = vzip2q_s16(src0_8x16b, src1_8x16b); - src2_8x16b = vzip2q_s16(src2_8x16b, src3_8x16b); - pred7_8x16b = vzip2q_s16(src4_8x16b, src5_8x16b); - src6_8x16b = vzip2q_s16(src6_8x16b, src7_8x16b); - - out1_8x16b = vzip1q_s32(out3_8x16b, pred0_8x16b); - out3_8x16b = vzip2q_s32(out3_8x16b, pred0_8x16b); - - pred1_8x16b = vzip1q_s32(out2_8x16b, pred3_8x16b); - pred3_8x16b = vzip2q_s32(out2_8x16b, pred3_8x16b); - - out5_8x16b = vzip1q_s32(out7_8x16b, src2_8x16b); - out7_8x16b = vzip2q_s32(out7_8x16b, src2_8x16b); - - pred5_8x16b = vzip1q_s32(pred7_8x16b, src6_8x16b); - pred7_8x16b = vzip2q_s32(pred7_8x16b, src6_8x16b); - - src0_8x16b = vzip1q_s64(out1_8x16b,pred1_8x16b); - src1_8x16b = vzip2q_s64(out1_8x16b,pred1_8x16b); - src2_8x16b = vzip1q_s64(out3_8x16b,pred3_8x16b); - src3_8x16b = vzip2q_s64(out3_8x16b,pred3_8x16b); - src4_8x16b = vzip1q_s64(out5_8x16b,pred5_8x16b); - src5_8x16b = vzip2q_s64(out5_8x16b,pred5_8x16b); - src6_8x16b = vzip1q_s64(out7_8x16b,pred7_8x16b); - src7_8x16b = vzip2q_s64(out7_8x16b,pred7_8x16b); - - /********************** 8x8 16 bit Transpose End *********************/ - /**************** 8x8 horizontal transform *******************************/ - { - int16x8_t out0a_8x16b, out1a_8x16b, out2a_8x16b, out3a_8x16b; - int16x8_t out4a_8x16b, out5a_8x16b, out6a_8x16b, out7a_8x16b; - int16x8_t tmp0_8x16b, tmp1_8x16b, tmp2_8x16b, tmp3_8x16b; - int16x8_t tmp4_8x16b, tmp5_8x16b, tmp6_8x16b, tmp7_8x16b; - - /************************* 8x8 Vertical Transform*************************/ - tmp0_8x16b = vcombine_s16(vget_high_s16(src0_8x16b), vcreate_s32(0)); - tmp1_8x16b = vcombine_s16(vget_high_s16(src1_8x16b), vcreate_s32(0)); - tmp2_8x16b = vcombine_s16(vget_high_s16(src2_8x16b), vcreate_s32(0)); - tmp3_8x16b = vcombine_s16(vget_high_s16(src3_8x16b), vcreate_s32(0)); - tmp4_8x16b = vcombine_s16(vget_high_s16(src4_8x16b), vcreate_s32(0)); - tmp5_8x16b = vcombine_s16(vget_high_s16(src5_8x16b), vcreate_s32(0)); - tmp6_8x16b = vcombine_s16(vget_high_s16(src6_8x16b), vcreate_s32(0)); - tmp7_8x16b = vcombine_s16(vget_high_s16(src7_8x16b), vcreate_s32(0)); - - /*************************First 4 pixels ********************************/ - - src0_8x16b = vmovl_s16(vget_low_s16(src0_8x16b)); - src1_8x16b = vmovl_s16(vget_low_s16(src1_8x16b)); - src2_8x16b = vmovl_s16(vget_low_s16(src2_8x16b)); - src3_8x16b = vmovl_s16(vget_low_s16(src3_8x16b)); - src4_8x16b = vmovl_s16(vget_low_s16(src4_8x16b)); - src5_8x16b = vmovl_s16(vget_low_s16(src5_8x16b)); - src6_8x16b = vmovl_s16(vget_low_s16(src6_8x16b)); - src7_8x16b = vmovl_s16(vget_low_s16(src7_8x16b)); - - /* r0 + r1 */ - pred0_8x16b = vaddq_s32(src0_8x16b, src1_8x16b); - /* r2 + r3 */ - pred2_8x16b = vaddq_s32(src2_8x16b, src3_8x16b); - /* r4 + r5 */ - pred4_8x16b = vaddq_s32(src4_8x16b, src5_8x16b); - /* r6 + r7 */ - pred6_8x16b = vaddq_s32(src6_8x16b, src7_8x16b); - - /* r0 + r1 + r2 + r3 */ - pred1_8x16b = vaddq_s32(pred0_8x16b, pred2_8x16b); - /* r4 + r5 + r6 + r7 */ - pred5_8x16b = vaddq_s32(pred4_8x16b, pred6_8x16b); - /* r0 + r1 + r2 + r3 + r4 + r5 + r6 + r7 */ - out0_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 + r1 + r2 + r3 - r4 - r5 - r6 - r7 */ - out4_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 + r1 - r2 - r3 */ - pred1_8x16b = vsubq_s32(pred0_8x16b, pred2_8x16b); - /* r4 + r5 - r6 - r7 */ - pred5_8x16b = vsubq_s32(pred4_8x16b, pred6_8x16b); - /* r0 + r1 - r2 - r3 + r4 + r5 - r6 - r7 */ - out2_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 + r1 - r2 - r3 - r4 - r5 + r6 + r7 */ - out6_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 */ - pred0_8x16b = vsubq_s32(src0_8x16b, src1_8x16b); - /* r2 - r3 */ - pred2_8x16b = vsubq_s32(src2_8x16b, src3_8x16b); - /* r4 - r5 */ - pred4_8x16b = vsubq_s32(src4_8x16b, src5_8x16b); - /* r6 - r7 */ - pred6_8x16b = vsubq_s32(src6_8x16b, src7_8x16b); - - /* r0 - r1 + r2 - r3 */ - pred1_8x16b = vaddq_s32(pred0_8x16b, pred2_8x16b); - /* r4 - r5 + r6 - r7 */ - pred5_8x16b = vaddq_s32(pred4_8x16b, pred6_8x16b); - /* r0 - r1 + r2 - r3 + r4 - r5 + r6 - r7 */ - out1_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 - r1 + r2 - r3 - r4 + r5 - r6 + r7 */ - out5_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 - r2 + r3 */ - pred1_8x16b = vsubq_s32(pred0_8x16b, pred2_8x16b); - /* r4 - r5 - r6 + r7 */ - pred5_8x16b = vsubq_s32(pred4_8x16b, pred6_8x16b); - /* r0 - r1 - r2 + r3 + r4 - r5 - r6 + r7 */ - out3_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 - r1 - r2 + r3 - r4 + r5 + r6 - r7 */ - out7_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /*************************First 4 pixels ********************************/ - - /**************************Next 4 pixels *******************************/ - src0_8x16b = vmovl_s16(vget_low_s16(tmp0_8x16b)); - src1_8x16b = vmovl_s16(vget_low_s16(tmp1_8x16b)); - src2_8x16b = vmovl_s16(vget_low_s16(tmp2_8x16b)); - src3_8x16b = vmovl_s16(vget_low_s16(tmp3_8x16b)); - src4_8x16b = vmovl_s16(vget_low_s16(tmp4_8x16b)); - src5_8x16b = vmovl_s16(vget_low_s16(tmp5_8x16b)); - src6_8x16b = vmovl_s16(vget_low_s16(tmp6_8x16b)); - src7_8x16b = vmovl_s16(vget_low_s16(tmp7_8x16b)); - - /* r0 + r1 */ - pred0_8x16b = vaddq_s32(src0_8x16b, src1_8x16b); - /* r2 + r3 */ - pred2_8x16b = vaddq_s32(src2_8x16b, src3_8x16b); - /* r4 + r5 */ - pred4_8x16b = vaddq_s32(src4_8x16b, src5_8x16b); - /* r6 + r7 */ - pred6_8x16b = vaddq_s32(src6_8x16b, src7_8x16b); - - /* r0 + r1 + r2 + r3 */ - pred1_8x16b = vaddq_s32(pred0_8x16b, pred2_8x16b); - /* r4 + r5 + r6 + r7 */ - pred5_8x16b = vaddq_s32(pred4_8x16b, pred6_8x16b); - /* r0 + r1 + r2 + r3 + r4 + r5 + r6 + r7 */ - out0a_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 + r1 + r2 + r3 - r4 - r5 - r6 - r7 */ - out4a_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 + r1 - r2 - r3 */ - pred1_8x16b = vsubq_s32(pred0_8x16b, pred2_8x16b); - /* r4 + r5 - r6 - r7 */ - pred5_8x16b = vsubq_s32(pred4_8x16b, pred6_8x16b); - /* r0 + r1 - r2 - r3 + r4 + r5 - r6 - r7 */ - out2a_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 + r1 - r2 - r3 - r4 - r5 + r6 + r7 */ - out6a_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 */ - pred0_8x16b = vsubq_s32(src0_8x16b, src1_8x16b); - /* r2 - r3 */ - pred2_8x16b = vsubq_s32(src2_8x16b, src3_8x16b); - /* r4 - r5 */ - pred4_8x16b = vsubq_s32(src4_8x16b, src5_8x16b); - /* r6 - r7 */ - pred6_8x16b = vsubq_s32(src6_8x16b, src7_8x16b); - - /* r0 - r1 + r2 - r3 */ - pred1_8x16b = vaddq_s32(pred0_8x16b, pred2_8x16b); - /* r4 - r5 + r6 - r7 */ - pred5_8x16b = vaddq_s32(pred4_8x16b, pred6_8x16b); - /* r0 - r1 + r2 - r3 + r4 - r5 + r6 - r7 */ - out1a_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 - r1 + r2 - r3 - r4 + r5 - r6 + r7 */ - out5a_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /* r0 - r1 - r2 + r3 */ - pred1_8x16b = vsubq_s32(pred0_8x16b, pred2_8x16b); - /* r4 - r5 - r6 + r7 */ - pred5_8x16b = vsubq_s32(pred4_8x16b, pred6_8x16b); - /* r0 - r1 - r2 + r3 + r4 - r5 - r6 + r7 */ - out3a_8x16b = vaddq_s32(pred1_8x16b, pred5_8x16b); - /* r0 - r1 - r2 + r3 - r4 + r5 + r6 - r7 */ - out7a_8x16b = vsubq_s32(pred1_8x16b, pred5_8x16b); - - /**************************Next 4 pixels *******************************/ - /************************* 8x8 Vertical Transform*************************/ - - /****************************SATD calculation ****************************/ - src0_8x16b = vabsq_s32(out0_8x16b); - src1_8x16b = vabsq_s32(out1_8x16b); - src2_8x16b = vabsq_s32(out2_8x16b); - src3_8x16b = vabsq_s32(out3_8x16b); - src4_8x16b = vabsq_s32(out4_8x16b); - src5_8x16b = vabsq_s32(out5_8x16b); - src6_8x16b = vabsq_s32(out6_8x16b); - src7_8x16b = vabsq_s32(out7_8x16b); - s32* p = (s32*)&src0_8x16b; - p[0] = 0; - - satd = vaddvq_s32(src0_8x16b); - satd += vaddvq_s32(src1_8x16b); - satd += vaddvq_s32(src2_8x16b); - satd += vaddvq_s32(src3_8x16b); - satd += vaddvq_s32(src4_8x16b); - satd += vaddvq_s32(src5_8x16b); - satd += vaddvq_s32(src6_8x16b); - satd += vaddvq_s32(src7_8x16b); - - src0_8x16b = vabsq_s32(out0a_8x16b); - src1_8x16b = vabsq_s32(out1a_8x16b); - src2_8x16b = vabsq_s32(out2a_8x16b); - src3_8x16b = vabsq_s32(out3a_8x16b); - src4_8x16b = vabsq_s32(out4a_8x16b); - src5_8x16b = vabsq_s32(out5a_8x16b); - src6_8x16b = vabsq_s32(out6a_8x16b); - src7_8x16b = vabsq_s32(out7a_8x16b); - - satd += vaddvq_s32(src0_8x16b); - satd += vaddvq_s32(src1_8x16b); - satd += vaddvq_s32(src2_8x16b); - satd += vaddvq_s32(src3_8x16b); - satd += vaddvq_s32(src4_8x16b); - satd += vaddvq_s32(src5_8x16b); - satd += vaddvq_s32(src6_8x16b); - satd += vaddvq_s32(src7_8x16b); - - satd = (satd + 2) >> 2; - return satd; - } + int16x8_t r0, r1, r2, r3, r4, r5, r6, r7, src, t0, t1, t4, t5 ,t6 ,t7; + +// Vert-pass + r0 = vld1q_s16(org); + + src = vld1q_s16(org + s_org); + r1 = vsubq_s16(r0, src); + r0 = vaddq_s16(r0, src); + + t0 = vld1q_s16(org + 2 * s_org); + + src = vld1q_s16(org + 3 * s_org); + t1 = vsubq_s16(t0, src); + t0 = vaddq_s16(t0, src); + + r3 = vsubq_s16(r1, t1); + r2 = vsubq_s16(r0, t0); + r1 = vaddq_s16(r1, t1); + r0 = vaddq_s16(r0, t0); + + t4 = vld1q_s16(org + 4 * s_org); + + src = vld1q_s16(org + 5 * s_org); + t5 = vsubq_s16(t4, src); + t4 = vaddq_s16(t4, src); + + t0 = vld1q_s16(org + 6 * s_org); + + src = vld1q_s16(org + 7 * s_org); + t1 = vsubq_s16(t0, src); + t0 = vaddq_s16(t0, src); + + t7 = vsubq_s16(t5, t1); + t6 = vsubq_s16(t4, t0); + t5 = vaddq_s16(t5, t1); + t4 = vaddq_s16(t4, t0); + + r7 = vsubq_s16(r3, t7); + r6 = vsubq_s16(r2, t6); + r5 = vsubq_s16(r1, t5); + r4 = vsubq_s16(r0, t4); + r3 = vaddq_s16(r3, t7); + r2 = vaddq_s16(r2, t6); + r1 = vaddq_s16(r1, t5); + r0 = vaddq_s16(r0, t4); + +// Transpose and Horz-pass + int16x8x2_t tmp0_8x16bx2, tmp1_8x16bx2; + int32x4x2_t tmp_4x32bx2; + int32x4_t h0, h1, h2, h3, q0, q1, q2, q3, q4, q5, q6, q7; + + tmp0_8x16bx2 = vtrnq_s16(r0, r1); + tmp1_8x16bx2 = vtrnq_s16(r2, r3); + + tmp_4x32bx2 = vtrnq_s32(vreinterpretq_s32_s16(tmp0_8x16bx2.val[0]), vreinterpretq_s32_s16(tmp1_8x16bx2.val[0])); + + h0 = vaddl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h1 = vsubl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + h2 = vaddl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h3 = vsubl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + q0 = vaddq_s32(h0, h2); + q1 = vsubq_s32(h0, h2); + q2 = vaddq_s32(h1, h3); + q3 = vsubq_s32(h1, h3); + + tmp_4x32bx2 = vtrnq_s32(vreinterpretq_s32_s16(tmp0_8x16bx2.val[1]), vreinterpretq_s32_s16(tmp1_8x16bx2.val[1])); + + h0 = vaddl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h1 = vsubl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + h2 = vaddl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h3 = vsubl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + q4 = vaddq_s32(h0, h2); + q5 = vsubq_s32(h0, h2); + q6 = vaddq_s32(h1, h3); + q7 = vsubq_s32(h1, h3); + + VADDVQ_S32(satd, vabsq_s32(vsetq_lane_s32(0, vaddq_s32(q0, q4), 0))); + VADDVQ_S32(satd, vabdq_s32(q0, q4)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q1, q5))); + VADDVQ_S32(satd, vabdq_s32(q1, q5)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q2, q6))); + VADDVQ_S32(satd, vabdq_s32(q2, q6)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q3, q7))); + VADDVQ_S32(satd, vabdq_s32(q3, q7)); + + tmp0_8x16bx2 = vtrnq_s16(r4, r5); + tmp1_8x16bx2 = vtrnq_s16(r6, r7); + + tmp_4x32bx2 = vtrnq_s32(vreinterpretq_s32_s16(tmp0_8x16bx2.val[0]), vreinterpretq_s32_s16(tmp1_8x16bx2.val[0])); + + h0 = vaddl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h1 = vsubl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + h2 = vaddl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h3 = vsubl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + q0 = vaddq_s32(h0, h2); + q1 = vsubq_s32(h0, h2); + q2 = vaddq_s32(h1, h3); + q3 = vsubq_s32(h1, h3); + + tmp_4x32bx2 = vtrnq_s32(vreinterpretq_s32_s16(tmp0_8x16bx2.val[1]), vreinterpretq_s32_s16(tmp1_8x16bx2.val[1])); + + h0 = vaddl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h1 = vsubl_s16(vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_high_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + h2 = vaddl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + h3 = vsubl_s16(vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[0])), vget_low_s16(vreinterpretq_s16_s32(tmp_4x32bx2.val[1]))); + + q4 = vaddq_s32(h0, h2); + q5 = vsubq_s32(h0, h2); + q6 = vaddq_s32(h1, h3); + q7 = vsubq_s32(h1, h3); + + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q0, q4))); + VADDVQ_S32(satd, vabdq_s32(q0, q4)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q1, q5))); + VADDVQ_S32(satd, vabdq_s32(q1, q5)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q2, q6))); + VADDVQ_S32(satd, vabdq_s32(q2, q6)); + VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q3, q7))); + VADDVQ_S32(satd, vabdq_s32(q3, q7)); + + satd = (satd + 2) >> 2; + return satd; } #endif /* ARM_NEON */ From 98fec8eec377f8762532197417b8bdf93cc6347c Mon Sep 17 00:00:00 2001 From: lordnn Date: Wed, 12 Aug 2026 01:18:42 +0300 Subject: [PATCH 2/6] Cleanup, Signed-off-by: lordnn --- src/neon/oapv_sad_neon.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/neon/oapv_sad_neon.c b/src/neon/oapv_sad_neon.c index 58530a2d..3dded662 100644 --- a/src/neon/oapv_sad_neon.c +++ b/src/neon/oapv_sad_neon.c @@ -225,7 +225,6 @@ const oapv_fn_ssd_t oapv_tbl_fn_ssd_16b_neon[2] = NULL}; /* DIFF **********************************************************************/ - int oapv_dc_removed_had8x8_neon(pel* org, int s_org) { int satd = 0; From 64b4fbf7bc6761fcf7afe7049ffef125671917d8 Mon Sep 17 00:00:00 2001 From: lordnn Date: Wed, 12 Aug 2026 22:35:01 +0300 Subject: [PATCH 3/6] Removed duplicate declaration. Signed-off-by: lordnn --- src/neon/oapv_sad_neon.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/neon/oapv_sad_neon.h b/src/neon/oapv_sad_neon.h index d8a8fe52..9e7b0c8b 100644 --- a/src/neon/oapv_sad_neon.h +++ b/src/neon/oapv_sad_neon.h @@ -37,8 +37,6 @@ #if ARM_NEON extern const oapv_fn_ssd_t oapv_tbl_fn_ssd_16b_neon[2]; -int oapv_dc_removed_had8x8_neon(pel* org, int s_org); - int oapv_dc_removed_had8x8_neon(pel* org, int s_org); #endif /* ARM_NEON */ From 88af32f695ef0be360967b58545793f818e814cf Mon Sep 17 00:00:00 2001 From: lordnn Date: Wed, 12 Aug 2026 23:09:34 +0300 Subject: [PATCH 4/6] Added short description message. Signed-off-by: lordnn --- src/neon/oapv_sad_neon.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/neon/oapv_sad_neon.c b/src/neon/oapv_sad_neon.c index 3dded662..a89a521c 100644 --- a/src/neon/oapv_sad_neon.c +++ b/src/neon/oapv_sad_neon.c @@ -227,6 +227,11 @@ const oapv_fn_ssd_t oapv_tbl_fn_ssd_16b_neon[2] = /* DIFF **********************************************************************/ int oapv_dc_removed_had8x8_neon(pel* org, int s_org) { + /* first pass is register-wise on 128-bit s16 row vectors, so its values + reach 8 * 4095 and only just fit in s16; the input is therefore limited + to 12-bit samples. after a transpose the second pass runs register-wise + on 128-bit s32 vectors, since its values can reach 64 * 4095 and do not + fit in s16 */ int satd = 0; int16x8_t r0, r1, r2, r3, r4, r5, r6, r7, src, t0, t1, t4, t5 ,t6 ,t7; From 82cf5cd52ced1d1942df98cf57e2b35f5622a95f Mon Sep 17 00:00:00 2001 From: lordnn Date: Thu, 13 Aug 2026 02:36:43 +0300 Subject: [PATCH 5/6] Added register-wise sum accumulation. Signed-off-by: lordnn --- src/neon/oapv_sad_neon.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/neon/oapv_sad_neon.c b/src/neon/oapv_sad_neon.c index a89a521c..2b6b6d9f 100644 --- a/src/neon/oapv_sad_neon.c +++ b/src/neon/oapv_sad_neon.c @@ -313,14 +313,14 @@ int oapv_dc_removed_had8x8_neon(pel* org, int s_org) q6 = vaddq_s32(h1, h3); q7 = vsubq_s32(h1, h3); - VADDVQ_S32(satd, vabsq_s32(vsetq_lane_s32(0, vaddq_s32(q0, q4), 0))); - VADDVQ_S32(satd, vabdq_s32(q0, q4)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q1, q5))); - VADDVQ_S32(satd, vabdq_s32(q1, q5)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q2, q6))); - VADDVQ_S32(satd, vabdq_s32(q2, q6)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q3, q7))); - VADDVQ_S32(satd, vabdq_s32(q3, q7)); + int32x4_t satv = vabsq_s32(vsetq_lane_s32(0, vaddq_s32(q0, q4), 0)); + satv = vabaq_s32(satv, q0, q4); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q1, q5))); + satv = vabaq_s32(satv, q1, q5); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q2, q6))); + satv = vabaq_s32(satv, q2, q6); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q3, q7))); + satv = vabaq_s32(satv, q3, q7); tmp0_8x16bx2 = vtrnq_s16(r4, r5); tmp1_8x16bx2 = vtrnq_s16(r6, r7); @@ -351,16 +351,16 @@ int oapv_dc_removed_had8x8_neon(pel* org, int s_org) q6 = vaddq_s32(h1, h3); q7 = vsubq_s32(h1, h3); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q0, q4))); - VADDVQ_S32(satd, vabdq_s32(q0, q4)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q1, q5))); - VADDVQ_S32(satd, vabdq_s32(q1, q5)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q2, q6))); - VADDVQ_S32(satd, vabdq_s32(q2, q6)); - VADDVQ_S32(satd, vabsq_s32(vaddq_s32(q3, q7))); - VADDVQ_S32(satd, vabdq_s32(q3, q7)); - - satd = (satd + 2) >> 2; - return satd; + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q0, q4))); + satv = vabaq_s32(satv, q0, q4); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q1, q5))); + satv = vabaq_s32(satv, q1, q5); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q2, q6))); + satv = vabaq_s32(satv, q2, q6); + satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q3, q7))); + satv = vabaq_s32(satv, q3, q7); + + VADDVQ_S32(satd, satv) + return (satd + 2) >> 2; } #endif /* ARM_NEON */ From e9097849dc6622373d017c300c6f7a50470b0898 Mon Sep 17 00:00:00 2001 From: lordnn Date: Fri, 14 Aug 2026 00:06:09 +0300 Subject: [PATCH 6/6] Removed macro. Signed-off-by: lordnn --- src/neon/oapv_sad_neon.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/neon/oapv_sad_neon.c b/src/neon/oapv_sad_neon.c index 2b6b6d9f..b7818b48 100644 --- a/src/neon/oapv_sad_neon.c +++ b/src/neon/oapv_sad_neon.c @@ -34,8 +34,6 @@ #if ARM_NEON -#define VADDVQ_S32(sum, sads) sum += vaddvq_s32(sads); - /* SAD for 16bit **************************************************************/ /* SSD ***********************************************************************/ static s64 ssd_16b_neon_8x8(int w, int h, void *src1, void *src2, int s_src1, int s_src2) @@ -232,7 +230,6 @@ int oapv_dc_removed_had8x8_neon(pel* org, int s_org) to 12-bit samples. after a transpose the second pass runs register-wise on 128-bit s32 vectors, since its values can reach 64 * 4095 and do not fit in s16 */ - int satd = 0; int16x8_t r0, r1, r2, r3, r4, r5, r6, r7, src, t0, t1, t4, t5 ,t6 ,t7; // Vert-pass @@ -360,7 +357,7 @@ int oapv_dc_removed_had8x8_neon(pel* org, int s_org) satv = vaddq_s32(satv, vabsq_s32(vaddq_s32(q3, q7))); satv = vabaq_s32(satv, q3, q7); - VADDVQ_S32(satd, satv) + int satd = vaddvq_s32(satv); return (satd + 2) >> 2; } #endif /* ARM_NEON */