// res is in reverse row order
static INLINE void highbd_idct4x4_1_add_kernel2(uint16_t **dest,
const int stride,
const int16x8_t res,
const int16x8_t max) {
const uint16x4_t a0 = vld1_u16(*dest);
const uint16x4_t a1 = vld1_u16(*dest + stride);
const int16x8_t a = vreinterpretq_s16_u16(vcombine_u16(a1, a0));
// Note: In some profile tests, res is quite close to +/-32767.
// We use saturating addition.
const int16x8_t b = vqaddq_s16(res, a);
const int16x8_t c = vminq_s16(b, max);
const uint16x8_t d = vqshluq_n_s16(c, 0);
vst1_u16(*dest, vget_high_u16(d));
*dest += stride;
vst1_u16(*dest, vget_low_u16(d));
*dest += stride;
}
/* s16x8 saturated add */
void mw_neon_mm_qadd_s16x8(short * A, int Row, int Col, short * B, short * C)
{
int16x8_t neon_a, neon_b, neon_c;
int size = Row * Col;
int i = 0;
int k = 0;
for (i = 8; i <= size ; i+=8)
{
k = i - 8;
neon_a = vld1q_s16(A + k);
neon_b = vld1q_s16(B + k);
neon_c = vqaddq_s16(neon_a, neon_b);
vst1q_s16(C + k, neon_c);
}
k = i - 8;
for (i = 0; i < size % 8; i++)
{
C[k + i] = A[k + i] + B[k + i];
}
}
void vp8_short_idct4x4llm_neon(int16_t *input, unsigned char *pred_ptr,
int pred_stride, unsigned char *dst_ptr,
int dst_stride) {
int i;
uint32x2_t d6u32 = vdup_n_u32(0);
uint8x8_t d1u8;
int16x4_t d2, d3, d4, d5, d10, d11, d12, d13;
uint16x8_t q1u16;
int16x8_t q1s16, q2s16, q3s16, q4s16;
int32x2x2_t v2tmp0, v2tmp1;
int16x4x2_t v2tmp2, v2tmp3;
d2 = vld1_s16(input);
d3 = vld1_s16(input + 4);
d4 = vld1_s16(input + 8);
d5 = vld1_s16(input + 12);
// 1st for loop
q1s16 = vcombine_s16(d2, d4); // Swap d3 d4 here
q2s16 = vcombine_s16(d3, d5);
q3s16 = vqdmulhq_n_s16(q2s16, sinpi8sqrt2);
q4s16 = vqdmulhq_n_s16(q2s16, cospi8sqrt2minus1);
d12 = vqadd_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // a1
d13 = vqsub_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // b1
q3s16 = vshrq_n_s16(q3s16, 1);
q4s16 = vshrq_n_s16(q4s16, 1);
q3s16 = vqaddq_s16(q3s16, q2s16);
q4s16 = vqaddq_s16(q4s16, q2s16);
d10 = vqsub_s16(vget_low_s16(q3s16), vget_high_s16(q4s16)); // c1
d11 = vqadd_s16(vget_high_s16(q3s16), vget_low_s16(q4s16)); // d1
d2 = vqadd_s16(d12, d11);
d3 = vqadd_s16(d13, d10);
d4 = vqsub_s16(d13, d10);
d5 = vqsub_s16(d12, d11);
v2tmp0 = vtrn_s32(vreinterpret_s32_s16(d2), vreinterpret_s32_s16(d4));
v2tmp1 = vtrn_s32(vreinterpret_s32_s16(d3), vreinterpret_s32_s16(d5));
v2tmp2 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[0]),
vreinterpret_s16_s32(v2tmp1.val[0]));
v2tmp3 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[1]),
vreinterpret_s16_s32(v2tmp1.val[1]));
// 2nd for loop
q1s16 = vcombine_s16(v2tmp2.val[0], v2tmp3.val[0]);
q2s16 = vcombine_s16(v2tmp2.val[1], v2tmp3.val[1]);
q3s16 = vqdmulhq_n_s16(q2s16, sinpi8sqrt2);
q4s16 = vqdmulhq_n_s16(q2s16, cospi8sqrt2minus1);
d12 = vqadd_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // a1
d13 = vqsub_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // b1
q3s16 = vshrq_n_s16(q3s16, 1);
q4s16 = vshrq_n_s16(q4s16, 1);
q3s16 = vqaddq_s16(q3s16, q2s16);
q4s16 = vqaddq_s16(q4s16, q2s16);
d10 = vqsub_s16(vget_low_s16(q3s16), vget_high_s16(q4s16)); // c1
d11 = vqadd_s16(vget_high_s16(q3s16), vget_low_s16(q4s16)); // d1
d2 = vqadd_s16(d12, d11);
d3 = vqadd_s16(d13, d10);
d4 = vqsub_s16(d13, d10);
d5 = vqsub_s16(d12, d11);
d2 = vrshr_n_s16(d2, 3);
d3 = vrshr_n_s16(d3, 3);
d4 = vrshr_n_s16(d4, 3);
d5 = vrshr_n_s16(d5, 3);
v2tmp0 = vtrn_s32(vreinterpret_s32_s16(d2), vreinterpret_s32_s16(d4));
v2tmp1 = vtrn_s32(vreinterpret_s32_s16(d3), vreinterpret_s32_s16(d5));
v2tmp2 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[0]),
vreinterpret_s16_s32(v2tmp1.val[0]));
v2tmp3 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[1]),
vreinterpret_s16_s32(v2tmp1.val[1]));
q1s16 = vcombine_s16(v2tmp2.val[0], v2tmp2.val[1]);
q2s16 = vcombine_s16(v2tmp3.val[0], v2tmp3.val[1]);
// dc_only_idct_add
for (i = 0; i < 2; i++, q1s16 = q2s16) {
d6u32 = vld1_lane_u32((const uint32_t *)pred_ptr, d6u32, 0);
pred_ptr += pred_stride;
d6u32 = vld1_lane_u32((const uint32_t *)pred_ptr, d6u32, 1);
pred_ptr += pred_stride;
q1u16 = vaddw_u8(vreinterpretq_u16_s16(q1s16), vreinterpret_u8_u32(d6u32));
d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q1u16));
vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d1u8), 0);
dst_ptr += dst_stride;
vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d1u8), 1);
dst_ptr += dst_stride;
}
return;
}
inline int16x8_t vqaddq(const int16x8_t & v0, const int16x8_t & v1) { return vqaddq_s16(v0, v1); }
void vp8_sixtap_predict16x16_neon(
unsigned char *src_ptr,
int src_pixels_per_line,
int xoffset,
int yoffset,
unsigned char *dst_ptr,
int dst_pitch) {
unsigned char *src, *src_tmp, *dst, *tmpp;
unsigned char tmp[336];
int i, j;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
uint8x8_t d10u8, d11u8, d12u8, d13u8, d14u8, d15u8, d18u8, d19u8;
uint8x8_t d20u8, d21u8, d22u8, d23u8, d24u8, d25u8, d26u8, d27u8;
uint8x8_t d28u8, d29u8, d30u8, d31u8;
int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
uint8x16_t q3u8, q4u8;
uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16, q8u16, q9u16, q10u16;
uint16x8_t q11u16, q12u16, q13u16, q15u16;
int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16, q8s16, q9s16, q10s16;
int16x8_t q11s16, q12s16, q13s16, q15s16;
if (xoffset == 0) { // secondpass_filter8x8_only
// load second_pass filter
dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
// load src data
src_tmp = src_ptr - src_pixels_per_line * 2;
for (i = 0; i < 2; i++) {
src = src_tmp + i * 8;
dst = dst_ptr + i * 8;
d18u8 = vld1_u8(src);
src += src_pixels_per_line;
d19u8 = vld1_u8(src);
src += src_pixels_per_line;
d20u8 = vld1_u8(src);
src += src_pixels_per_line;
d21u8 = vld1_u8(src);
src += src_pixels_per_line;
d22u8 = vld1_u8(src);
src += src_pixels_per_line;
for (j = 0; j < 4; j++) {
d23u8 = vld1_u8(src);
src += src_pixels_per_line;
d24u8 = vld1_u8(src);
src += src_pixels_per_line;
d25u8 = vld1_u8(src);
src += src_pixels_per_line;
d26u8 = vld1_u8(src);
src += src_pixels_per_line;
q3u16 = vmull_u8(d18u8, d0u8);
q4u16 = vmull_u8(d19u8, d0u8);
q5u16 = vmull_u8(d20u8, d0u8);
q6u16 = vmull_u8(d21u8, d0u8);
q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
q7u16 = vmull_u8(d21u8, d3u8);
q8u16 = vmull_u8(d22u8, d3u8);
q9u16 = vmull_u8(d23u8, d3u8);
q10u16 = vmull_u8(d24u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q7s16 = vqaddq_s16(q7s16, q3s16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q9s16 = vqaddq_s16(q9s16, q5s16);
q10s16 = vqaddq_s16(q10s16, q6s16);
d6u8 = vqrshrun_n_s16(q7s16, 7);
d7u8 = vqrshrun_n_s16(q8s16, 7);
d8u8 = vqrshrun_n_s16(q9s16, 7);
d9u8 = vqrshrun_n_s16(q10s16, 7);
d18u8 = d22u8;
d19u8 = d23u8;
d20u8 = d24u8;
d21u8 = d25u8;
d22u8 = d26u8;
vst1_u8(dst, d6u8);
dst += dst_pitch;
vst1_u8(dst, d7u8);
dst += dst_pitch;
vst1_u8(dst, d8u8);
dst += dst_pitch;
vst1_u8(dst, d9u8);
dst += dst_pitch;
}
}
return;
}
// load first_pass filter
dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
// First pass: output_height lines x output_width columns (9x4)
if (yoffset == 0) { // firstpass_filter4x4_only
src = src_ptr - 2;
dst = dst_ptr;
for (i = 0; i < 8; i++) {
d6u8 = vld1_u8(src);
d7u8 = vld1_u8(src + 8);
d8u8 = vld1_u8(src + 16);
src += src_pixels_per_line;
d9u8 = vld1_u8(src);
d10u8 = vld1_u8(src + 8);
d11u8 = vld1_u8(src + 16);
src += src_pixels_per_line;
__builtin_prefetch(src);
__builtin_prefetch(src + src_pixels_per_line);
q6u16 = vmull_u8(d6u8, d0u8);
q7u16 = vmull_u8(d7u8, d0u8);
q8u16 = vmull_u8(d9u8, d0u8);
q9u16 = vmull_u8(d10u8, d0u8);
d20u8 = vext_u8(d6u8, d7u8, 1);
d21u8 = vext_u8(d9u8, d10u8, 1);
d22u8 = vext_u8(d7u8, d8u8, 1);
d23u8 = vext_u8(d10u8, d11u8, 1);
d24u8 = vext_u8(d6u8, d7u8, 4);
d25u8 = vext_u8(d9u8, d10u8, 4);
d26u8 = vext_u8(d7u8, d8u8, 4);
d27u8 = vext_u8(d10u8, d11u8, 4);
d28u8 = vext_u8(d6u8, d7u8, 5);
d29u8 = vext_u8(d9u8, d10u8, 5);
q6u16 = vmlsl_u8(q6u16, d20u8, d1u8);
q8u16 = vmlsl_u8(q8u16, d21u8, d1u8);
q7u16 = vmlsl_u8(q7u16, d22u8, d1u8);
q9u16 = vmlsl_u8(q9u16, d23u8, d1u8);
q6u16 = vmlsl_u8(q6u16, d24u8, d4u8);
q8u16 = vmlsl_u8(q8u16, d25u8, d4u8);
q7u16 = vmlsl_u8(q7u16, d26u8, d4u8);
q9u16 = vmlsl_u8(q9u16, d27u8, d4u8);
q6u16 = vmlal_u8(q6u16, d28u8, d5u8);
q8u16 = vmlal_u8(q8u16, d29u8, d5u8);
d20u8 = vext_u8(d7u8, d8u8, 5);
d21u8 = vext_u8(d10u8, d11u8, 5);
d22u8 = vext_u8(d6u8, d7u8, 2);
d23u8 = vext_u8(d9u8, d10u8, 2);
d24u8 = vext_u8(d7u8, d8u8, 2);
d25u8 = vext_u8(d10u8, d11u8, 2);
d26u8 = vext_u8(d6u8, d7u8, 3);
d27u8 = vext_u8(d9u8, d10u8, 3);
d28u8 = vext_u8(d7u8, d8u8, 3);
d29u8 = vext_u8(d10u8, d11u8, 3);
q7u16 = vmlal_u8(q7u16, d20u8, d5u8);
q9u16 = vmlal_u8(q9u16, d21u8, d5u8);
q6u16 = vmlal_u8(q6u16, d22u8, d2u8);
q8u16 = vmlal_u8(q8u16, d23u8, d2u8);
q7u16 = vmlal_u8(q7u16, d24u8, d2u8);
q9u16 = vmlal_u8(q9u16, d25u8, d2u8);
q10u16 = vmull_u8(d26u8, d3u8);
q11u16 = vmull_u8(d27u8, d3u8);
q12u16 = vmull_u8(d28u8, d3u8);
q15u16 = vmull_u8(d29u8, d3u8);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q11s16 = vreinterpretq_s16_u16(q11u16);
q12s16 = vreinterpretq_s16_u16(q12u16);
q15s16 = vreinterpretq_s16_u16(q15u16);
q6s16 = vqaddq_s16(q6s16, q10s16);
q8s16 = vqaddq_s16(q8s16, q11s16);
q7s16 = vqaddq_s16(q7s16, q12s16);
q9s16 = vqaddq_s16(q9s16, q15s16);
d6u8 = vqrshrun_n_s16(q6s16, 7);
d7u8 = vqrshrun_n_s16(q7s16, 7);
d8u8 = vqrshrun_n_s16(q8s16, 7);
d9u8 = vqrshrun_n_s16(q9s16, 7);
q3u8 = vcombine_u8(d6u8, d7u8);
q4u8 = vcombine_u8(d8u8, d9u8);
vst1q_u8(dst, q3u8);
dst += dst_pitch;
vst1q_u8(dst, q4u8);
dst += dst_pitch;
}
return;
}
src = src_ptr - 2 - src_pixels_per_line * 2;
tmpp = tmp;
for (i = 0; i < 7; i++) {
d6u8 = vld1_u8(src);
d7u8 = vld1_u8(src + 8);
d8u8 = vld1_u8(src + 16);
src += src_pixels_per_line;
d9u8 = vld1_u8(src);
d10u8 = vld1_u8(src + 8);
d11u8 = vld1_u8(src + 16);
src += src_pixels_per_line;
d12u8 = vld1_u8(src);
d13u8 = vld1_u8(src + 8);
d14u8 = vld1_u8(src + 16);
src += src_pixels_per_line;
__builtin_prefetch(src);
__builtin_prefetch(src + src_pixels_per_line);
__builtin_prefetch(src + src_pixels_per_line * 2);
q8u16 = vmull_u8(d6u8, d0u8);
q9u16 = vmull_u8(d7u8, d0u8);
q10u16 = vmull_u8(d9u8, d0u8);
q11u16 = vmull_u8(d10u8, d0u8);
q12u16 = vmull_u8(d12u8, d0u8);
q13u16 = vmull_u8(d13u8, d0u8);
d28u8 = vext_u8(d6u8, d7u8, 1);
d29u8 = vext_u8(d9u8, d10u8, 1);
d30u8 = vext_u8(d12u8, d13u8, 1);
q8u16 = vmlsl_u8(q8u16, d28u8, d1u8);
q10u16 = vmlsl_u8(q10u16, d29u8, d1u8);
q12u16 = vmlsl_u8(q12u16, d30u8, d1u8);
d28u8 = vext_u8(d7u8, d8u8, 1);
d29u8 = vext_u8(d10u8, d11u8, 1);
d30u8 = vext_u8(d13u8, d14u8, 1);
q9u16 = vmlsl_u8(q9u16, d28u8, d1u8);
q11u16 = vmlsl_u8(q11u16, d29u8, d1u8);
q13u16 = vmlsl_u8(q13u16, d30u8, d1u8);
d28u8 = vext_u8(d6u8, d7u8, 4);
d29u8 = vext_u8(d9u8, d10u8, 4);
d30u8 = vext_u8(d12u8, d13u8, 4);
q8u16 = vmlsl_u8(q8u16, d28u8, d4u8);
q10u16 = vmlsl_u8(q10u16, d29u8, d4u8);
q12u16 = vmlsl_u8(q12u16, d30u8, d4u8);
d28u8 = vext_u8(d7u8, d8u8, 4);
d29u8 = vext_u8(d10u8, d11u8, 4);
d30u8 = vext_u8(d13u8, d14u8, 4);
q9u16 = vmlsl_u8(q9u16, d28u8, d4u8);
q11u16 = vmlsl_u8(q11u16, d29u8, d4u8);
q13u16 = vmlsl_u8(q13u16, d30u8, d4u8);
d28u8 = vext_u8(d6u8, d7u8, 5);
d29u8 = vext_u8(d9u8, d10u8, 5);
d30u8 = vext_u8(d12u8, d13u8, 5);
q8u16 = vmlal_u8(q8u16, d28u8, d5u8);
q10u16 = vmlal_u8(q10u16, d29u8, d5u8);
q12u16 = vmlal_u8(q12u16, d30u8, d5u8);
d28u8 = vext_u8(d7u8, d8u8, 5);
d29u8 = vext_u8(d10u8, d11u8, 5);
d30u8 = vext_u8(d13u8, d14u8, 5);
q9u16 = vmlal_u8(q9u16, d28u8, d5u8);
q11u16 = vmlal_u8(q11u16, d29u8, d5u8);
q13u16 = vmlal_u8(q13u16, d30u8, d5u8);
d28u8 = vext_u8(d6u8, d7u8, 2);
d29u8 = vext_u8(d9u8, d10u8, 2);
d30u8 = vext_u8(d12u8, d13u8, 2);
q8u16 = vmlal_u8(q8u16, d28u8, d2u8);
q10u16 = vmlal_u8(q10u16, d29u8, d2u8);
q12u16 = vmlal_u8(q12u16, d30u8, d2u8);
d28u8 = vext_u8(d7u8, d8u8, 2);
d29u8 = vext_u8(d10u8, d11u8, 2);
d30u8 = vext_u8(d13u8, d14u8, 2);
q9u16 = vmlal_u8(q9u16, d28u8, d2u8);
q11u16 = vmlal_u8(q11u16, d29u8, d2u8);
q13u16 = vmlal_u8(q13u16, d30u8, d2u8);
d28u8 = vext_u8(d6u8, d7u8, 3);
d29u8 = vext_u8(d9u8, d10u8, 3);
d30u8 = vext_u8(d12u8, d13u8, 3);
d15u8 = vext_u8(d7u8, d8u8, 3);
d31u8 = vext_u8(d10u8, d11u8, 3);
d6u8 = vext_u8(d13u8, d14u8, 3);
q4u16 = vmull_u8(d28u8, d3u8);
q5u16 = vmull_u8(d29u8, d3u8);
q6u16 = vmull_u8(d30u8, d3u8);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q12s16 = vreinterpretq_s16_u16(q12u16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q10s16 = vqaddq_s16(q10s16, q5s16);
q12s16 = vqaddq_s16(q12s16, q6s16);
q6u16 = vmull_u8(d15u8, d3u8);
q7u16 = vmull_u8(d31u8, d3u8);
q3u16 = vmull_u8(d6u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q11s16 = vreinterpretq_s16_u16(q11u16);
q13s16 = vreinterpretq_s16_u16(q13u16);
q9s16 = vqaddq_s16(q9s16, q6s16);
q11s16 = vqaddq_s16(q11s16, q7s16);
q13s16 = vqaddq_s16(q13s16, q3s16);
d6u8 = vqrshrun_n_s16(q8s16, 7);
d7u8 = vqrshrun_n_s16(q9s16, 7);
d8u8 = vqrshrun_n_s16(q10s16, 7);
d9u8 = vqrshrun_n_s16(q11s16, 7);
d10u8 = vqrshrun_n_s16(q12s16, 7);
d11u8 = vqrshrun_n_s16(q13s16, 7);
vst1_u8(tmpp, d6u8);
tmpp += 8;
vst1_u8(tmpp, d7u8);
tmpp += 8;
vst1_u8(tmpp, d8u8);
tmpp += 8;
vst1_u8(tmpp, d9u8);
tmpp += 8;
vst1_u8(tmpp, d10u8);
tmpp += 8;
vst1_u8(tmpp, d11u8);
tmpp += 8;
}
// Second pass: 16x16
dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
for (i = 0; i < 2; i++) {
dst = dst_ptr + 8 * i;
tmpp = tmp + 8 * i;
d18u8 = vld1_u8(tmpp);
tmpp += 16;
d19u8 = vld1_u8(tmpp);
tmpp += 16;
d20u8 = vld1_u8(tmpp);
tmpp += 16;
d21u8 = vld1_u8(tmpp);
tmpp += 16;
d22u8 = vld1_u8(tmpp);
tmpp += 16;
for (j = 0; j < 4; j++) {
d23u8 = vld1_u8(tmpp);
tmpp += 16;
d24u8 = vld1_u8(tmpp);
tmpp += 16;
d25u8 = vld1_u8(tmpp);
tmpp += 16;
d26u8 = vld1_u8(tmpp);
tmpp += 16;
q3u16 = vmull_u8(d18u8, d0u8);
q4u16 = vmull_u8(d19u8, d0u8);
q5u16 = vmull_u8(d20u8, d0u8);
q6u16 = vmull_u8(d21u8, d0u8);
q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
q7u16 = vmull_u8(d21u8, d3u8);
q8u16 = vmull_u8(d22u8, d3u8);
q9u16 = vmull_u8(d23u8, d3u8);
q10u16 = vmull_u8(d24u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q7s16 = vqaddq_s16(q7s16, q3s16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q9s16 = vqaddq_s16(q9s16, q5s16);
q10s16 = vqaddq_s16(q10s16, q6s16);
d6u8 = vqrshrun_n_s16(q7s16, 7);
d7u8 = vqrshrun_n_s16(q8s16, 7);
d8u8 = vqrshrun_n_s16(q9s16, 7);
d9u8 = vqrshrun_n_s16(q10s16, 7);
d18u8 = d22u8;
d19u8 = d23u8;
d20u8 = d24u8;
d21u8 = d25u8;
d22u8 = d26u8;
vst1_u8(dst, d6u8);
dst += dst_pitch;
vst1_u8(dst, d7u8);
dst += dst_pitch;
vst1_u8(dst, d8u8);
dst += dst_pitch;
vst1_u8(dst, d9u8);
dst += dst_pitch;
}
}
return;
}
void vp8_sixtap_predict8x8_neon(
unsigned char *src_ptr,
int src_pixels_per_line,
int xoffset,
int yoffset,
unsigned char *dst_ptr,
int dst_pitch) {
unsigned char *src, *tmpp;
unsigned char tmp[64];
int i;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
uint8x8_t d18u8, d19u8, d20u8, d21u8, d22u8, d23u8, d24u8, d25u8;
uint8x8_t d26u8, d27u8, d28u8, d29u8, d30u8, d31u8;
int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16;
uint16x8_t q8u16, q9u16, q10u16, q11u16, q12u16;
int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16;
int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16;
uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q9u8, q10u8, q11u8, q12u8;
if (xoffset == 0) { // secondpass_filter8x8_only
// load second_pass filter
dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
// load src data
src = src_ptr - src_pixels_per_line * 2;
d18u8 = vld1_u8(src);
src += src_pixels_per_line;
d19u8 = vld1_u8(src);
src += src_pixels_per_line;
d20u8 = vld1_u8(src);
src += src_pixels_per_line;
d21u8 = vld1_u8(src);
src += src_pixels_per_line;
d22u8 = vld1_u8(src);
src += src_pixels_per_line;
d23u8 = vld1_u8(src);
src += src_pixels_per_line;
d24u8 = vld1_u8(src);
src += src_pixels_per_line;
d25u8 = vld1_u8(src);
src += src_pixels_per_line;
d26u8 = vld1_u8(src);
src += src_pixels_per_line;
d27u8 = vld1_u8(src);
src += src_pixels_per_line;
d28u8 = vld1_u8(src);
src += src_pixels_per_line;
d29u8 = vld1_u8(src);
src += src_pixels_per_line;
d30u8 = vld1_u8(src);
for (i = 2; i > 0; i--) {
q3u16 = vmull_u8(d18u8, d0u8);
q4u16 = vmull_u8(d19u8, d0u8);
q5u16 = vmull_u8(d20u8, d0u8);
q6u16 = vmull_u8(d21u8, d0u8);
q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
q7u16 = vmull_u8(d21u8, d3u8);
q8u16 = vmull_u8(d22u8, d3u8);
q9u16 = vmull_u8(d23u8, d3u8);
q10u16 = vmull_u8(d24u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q7s16 = vqaddq_s16(q7s16, q3s16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q9s16 = vqaddq_s16(q9s16, q5s16);
q10s16 = vqaddq_s16(q10s16, q6s16);
d6u8 = vqrshrun_n_s16(q7s16, 7);
d7u8 = vqrshrun_n_s16(q8s16, 7);
d8u8 = vqrshrun_n_s16(q9s16, 7);
d9u8 = vqrshrun_n_s16(q10s16, 7);
d18u8 = d22u8;
d19u8 = d23u8;
d20u8 = d24u8;
d21u8 = d25u8;
d22u8 = d26u8;
d23u8 = d27u8;
d24u8 = d28u8;
d25u8 = d29u8;
d26u8 = d30u8;
vst1_u8(dst_ptr, d6u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d7u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d8u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d9u8);
dst_ptr += dst_pitch;
}
return;
}
// load first_pass filter
dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
// First pass: output_height lines x output_width columns (9x4)
if (yoffset == 0) // firstpass_filter4x4_only
src = src_ptr - 2;
else
src = src_ptr - 2 - (src_pixels_per_line * 2);
tmpp = tmp;
for (i = 2; i > 0; i--) {
q3u8 = vld1q_u8(src);
src += src_pixels_per_line;
q4u8 = vld1q_u8(src);
src += src_pixels_per_line;
q5u8 = vld1q_u8(src);
src += src_pixels_per_line;
q6u8 = vld1q_u8(src);
src += src_pixels_per_line;
__builtin_prefetch(src);
__builtin_prefetch(src + src_pixels_per_line);
__builtin_prefetch(src + src_pixels_per_line * 2);
q7u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
q8u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
q9u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
q10u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
q7u16 = vmlsl_u8(q7u16, d28u8, d1u8);
q8u16 = vmlsl_u8(q8u16, d29u8, d1u8);
q9u16 = vmlsl_u8(q9u16, d30u8, d1u8);
q10u16 = vmlsl_u8(q10u16, d31u8, d1u8);
d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
q7u16 = vmlsl_u8(q7u16, d28u8, d4u8);
q8u16 = vmlsl_u8(q8u16, d29u8, d4u8);
q9u16 = vmlsl_u8(q9u16, d30u8, d4u8);
q10u16 = vmlsl_u8(q10u16, d31u8, d4u8);
d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
q7u16 = vmlal_u8(q7u16, d28u8, d2u8);
q8u16 = vmlal_u8(q8u16, d29u8, d2u8);
q9u16 = vmlal_u8(q9u16, d30u8, d2u8);
q10u16 = vmlal_u8(q10u16, d31u8, d2u8);
d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
q7u16 = vmlal_u8(q7u16, d28u8, d5u8);
q8u16 = vmlal_u8(q8u16, d29u8, d5u8);
q9u16 = vmlal_u8(q9u16, d30u8, d5u8);
q10u16 = vmlal_u8(q10u16, d31u8, d5u8);
d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
q3u16 = vmull_u8(d28u8, d3u8);
q4u16 = vmull_u8(d29u8, d3u8);
q5u16 = vmull_u8(d30u8, d3u8);
q6u16 = vmull_u8(d31u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q7s16 = vqaddq_s16(q7s16, q3s16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q9s16 = vqaddq_s16(q9s16, q5s16);
q10s16 = vqaddq_s16(q10s16, q6s16);
d22u8 = vqrshrun_n_s16(q7s16, 7);
d23u8 = vqrshrun_n_s16(q8s16, 7);
d24u8 = vqrshrun_n_s16(q9s16, 7);
d25u8 = vqrshrun_n_s16(q10s16, 7);
if (yoffset == 0) { // firstpass_filter8x4_only
vst1_u8(dst_ptr, d22u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d23u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d24u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d25u8);
dst_ptr += dst_pitch;
} else {
vst1_u8(tmpp, d22u8);
tmpp += 8;
vst1_u8(tmpp, d23u8);
tmpp += 8;
vst1_u8(tmpp, d24u8);
tmpp += 8;
vst1_u8(tmpp, d25u8);
tmpp += 8;
}
}
if (yoffset == 0)
return;
// First Pass on rest 5-line data
q3u8 = vld1q_u8(src);
src += src_pixels_per_line;
q4u8 = vld1q_u8(src);
src += src_pixels_per_line;
q5u8 = vld1q_u8(src);
src += src_pixels_per_line;
q6u8 = vld1q_u8(src);
src += src_pixels_per_line;
q7u8 = vld1q_u8(src);
q8u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
q9u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
q10u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
q11u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
q12u16 = vmull_u8(vget_low_u8(q7u8), d0u8);
d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 1);
q8u16 = vmlsl_u8(q8u16, d27u8, d1u8);
q9u16 = vmlsl_u8(q9u16, d28u8, d1u8);
q10u16 = vmlsl_u8(q10u16, d29u8, d1u8);
q11u16 = vmlsl_u8(q11u16, d30u8, d1u8);
q12u16 = vmlsl_u8(q12u16, d31u8, d1u8);
d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 4);
q8u16 = vmlsl_u8(q8u16, d27u8, d4u8);
q9u16 = vmlsl_u8(q9u16, d28u8, d4u8);
q10u16 = vmlsl_u8(q10u16, d29u8, d4u8);
q11u16 = vmlsl_u8(q11u16, d30u8, d4u8);
q12u16 = vmlsl_u8(q12u16, d31u8, d4u8);
d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 2);
q8u16 = vmlal_u8(q8u16, d27u8, d2u8);
q9u16 = vmlal_u8(q9u16, d28u8, d2u8);
q10u16 = vmlal_u8(q10u16, d29u8, d2u8);
q11u16 = vmlal_u8(q11u16, d30u8, d2u8);
q12u16 = vmlal_u8(q12u16, d31u8, d2u8);
d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 5);
q8u16 = vmlal_u8(q8u16, d27u8, d5u8);
q9u16 = vmlal_u8(q9u16, d28u8, d5u8);
q10u16 = vmlal_u8(q10u16, d29u8, d5u8);
q11u16 = vmlal_u8(q11u16, d30u8, d5u8);
q12u16 = vmlal_u8(q12u16, d31u8, d5u8);
d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 3);
q3u16 = vmull_u8(d27u8, d3u8);
q4u16 = vmull_u8(d28u8, d3u8);
q5u16 = vmull_u8(d29u8, d3u8);
q6u16 = vmull_u8(d30u8, d3u8);
q7u16 = vmull_u8(d31u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q11s16 = vreinterpretq_s16_u16(q11u16);
q12s16 = vreinterpretq_s16_u16(q12u16);
q8s16 = vqaddq_s16(q8s16, q3s16);
q9s16 = vqaddq_s16(q9s16, q4s16);
q10s16 = vqaddq_s16(q10s16, q5s16);
q11s16 = vqaddq_s16(q11s16, q6s16);
q12s16 = vqaddq_s16(q12s16, q7s16);
d26u8 = vqrshrun_n_s16(q8s16, 7);
d27u8 = vqrshrun_n_s16(q9s16, 7);
d28u8 = vqrshrun_n_s16(q10s16, 7);
d29u8 = vqrshrun_n_s16(q11s16, 7);
d30u8 = vqrshrun_n_s16(q12s16, 7);
// Second pass: 8x8
dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
d0s8 = vdup_lane_s8(dtmps8, 0);
d1s8 = vdup_lane_s8(dtmps8, 1);
d2s8 = vdup_lane_s8(dtmps8, 2);
d3s8 = vdup_lane_s8(dtmps8, 3);
d4s8 = vdup_lane_s8(dtmps8, 4);
d5s8 = vdup_lane_s8(dtmps8, 5);
d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
tmpp = tmp;
q9u8 = vld1q_u8(tmpp);
tmpp += 16;
q10u8 = vld1q_u8(tmpp);
tmpp += 16;
q11u8 = vld1q_u8(tmpp);
tmpp += 16;
q12u8 = vld1q_u8(tmpp);
d18u8 = vget_low_u8(q9u8);
d19u8 = vget_high_u8(q9u8);
d20u8 = vget_low_u8(q10u8);
d21u8 = vget_high_u8(q10u8);
d22u8 = vget_low_u8(q11u8);
d23u8 = vget_high_u8(q11u8);
d24u8 = vget_low_u8(q12u8);
d25u8 = vget_high_u8(q12u8);
for (i = 2; i > 0; i--) {
q3u16 = vmull_u8(d18u8, d0u8);
q4u16 = vmull_u8(d19u8, d0u8);
q5u16 = vmull_u8(d20u8, d0u8);
q6u16 = vmull_u8(d21u8, d0u8);
q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
q7u16 = vmull_u8(d21u8, d3u8);
q8u16 = vmull_u8(d22u8, d3u8);
q9u16 = vmull_u8(d23u8, d3u8);
q10u16 = vmull_u8(d24u8, d3u8);
q3s16 = vreinterpretq_s16_u16(q3u16);
q4s16 = vreinterpretq_s16_u16(q4u16);
q5s16 = vreinterpretq_s16_u16(q5u16);
q6s16 = vreinterpretq_s16_u16(q6u16);
q7s16 = vreinterpretq_s16_u16(q7u16);
q8s16 = vreinterpretq_s16_u16(q8u16);
q9s16 = vreinterpretq_s16_u16(q9u16);
q10s16 = vreinterpretq_s16_u16(q10u16);
q7s16 = vqaddq_s16(q7s16, q3s16);
q8s16 = vqaddq_s16(q8s16, q4s16);
q9s16 = vqaddq_s16(q9s16, q5s16);
q10s16 = vqaddq_s16(q10s16, q6s16);
d6u8 = vqrshrun_n_s16(q7s16, 7);
d7u8 = vqrshrun_n_s16(q8s16, 7);
d8u8 = vqrshrun_n_s16(q9s16, 7);
d9u8 = vqrshrun_n_s16(q10s16, 7);
d18u8 = d22u8;
d19u8 = d23u8;
d20u8 = d24u8;
d21u8 = d25u8;
d22u8 = d26u8;
d23u8 = d27u8;
d24u8 = d28u8;
d25u8 = d29u8;
d26u8 = d30u8;
vst1_u8(dst_ptr, d6u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d7u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d8u8);
dst_ptr += dst_pitch;
vst1_u8(dst_ptr, d9u8);
dst_ptr += dst_pitch;
}
return;
}
void idct_dequant_full_2x_neon(
int16_t *q,
int16_t *dq,
unsigned char *dst,
int stride) {
unsigned char *dst0, *dst1;
int32x2_t d28, d29, d30, d31;
int16x8_t q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11;
int16x8_t qEmpty = vdupq_n_s16(0);
int32x4x2_t q2tmp0, q2tmp1;
int16x8x2_t q2tmp2, q2tmp3;
int16x4_t dLow0, dLow1, dHigh0, dHigh1;
d28 = d29 = d30 = d31 = vdup_n_s32(0);
// load dq
q0 = vld1q_s16(dq);
dq += 8;
q1 = vld1q_s16(dq);
// load q
q2 = vld1q_s16(q);
vst1q_s16(q, qEmpty);
q += 8;
q3 = vld1q_s16(q);
vst1q_s16(q, qEmpty);
q += 8;
q4 = vld1q_s16(q);
vst1q_s16(q, qEmpty);
q += 8;
q5 = vld1q_s16(q);
vst1q_s16(q, qEmpty);
// load src from dst
dst0 = dst;
dst1 = dst + 4;
d28 = vld1_lane_s32((const int32_t *)dst0, d28, 0);
dst0 += stride;
d28 = vld1_lane_s32((const int32_t *)dst1, d28, 1);
dst1 += stride;
d29 = vld1_lane_s32((const int32_t *)dst0, d29, 0);
dst0 += stride;
d29 = vld1_lane_s32((const int32_t *)dst1, d29, 1);
dst1 += stride;
d30 = vld1_lane_s32((const int32_t *)dst0, d30, 0);
dst0 += stride;
d30 = vld1_lane_s32((const int32_t *)dst1, d30, 1);
dst1 += stride;
d31 = vld1_lane_s32((const int32_t *)dst0, d31, 0);
d31 = vld1_lane_s32((const int32_t *)dst1, d31, 1);
q2 = vmulq_s16(q2, q0);
q3 = vmulq_s16(q3, q1);
q4 = vmulq_s16(q4, q0);
q5 = vmulq_s16(q5, q1);
// vswp
dLow0 = vget_low_s16(q2);
dHigh0 = vget_high_s16(q2);
dLow1 = vget_low_s16(q4);
dHigh1 = vget_high_s16(q4);
q2 = vcombine_s16(dLow0, dLow1);
q4 = vcombine_s16(dHigh0, dHigh1);
dLow0 = vget_low_s16(q3);
dHigh0 = vget_high_s16(q3);
dLow1 = vget_low_s16(q5);
dHigh1 = vget_high_s16(q5);
q3 = vcombine_s16(dLow0, dLow1);
q5 = vcombine_s16(dHigh0, dHigh1);
q6 = vqdmulhq_n_s16(q4, sinpi8sqrt2);
q7 = vqdmulhq_n_s16(q5, sinpi8sqrt2);
q8 = vqdmulhq_n_s16(q4, cospi8sqrt2minus1);
q9 = vqdmulhq_n_s16(q5, cospi8sqrt2minus1);
q10 = vqaddq_s16(q2, q3);
q11 = vqsubq_s16(q2, q3);
q8 = vshrq_n_s16(q8, 1);
q9 = vshrq_n_s16(q9, 1);
q4 = vqaddq_s16(q4, q8);
q5 = vqaddq_s16(q5, q9);
q2 = vqsubq_s16(q6, q5);
q3 = vqaddq_s16(q7, q4);
q4 = vqaddq_s16(q10, q3);
q5 = vqaddq_s16(q11, q2);
q6 = vqsubq_s16(q11, q2);
q7 = vqsubq_s16(q10, q3);
q2tmp0 = vtrnq_s32(vreinterpretq_s32_s16(q4), vreinterpretq_s32_s16(q6));
q2tmp1 = vtrnq_s32(vreinterpretq_s32_s16(q5), vreinterpretq_s32_s16(q7));
q2tmp2 = vtrnq_s16(vreinterpretq_s16_s32(q2tmp0.val[0]),
vreinterpretq_s16_s32(q2tmp1.val[0]));
q2tmp3 = vtrnq_s16(vreinterpretq_s16_s32(q2tmp0.val[1]),
vreinterpretq_s16_s32(q2tmp1.val[1]));
// loop 2
q8 = vqdmulhq_n_s16(q2tmp2.val[1], sinpi8sqrt2);
q9 = vqdmulhq_n_s16(q2tmp3.val[1], sinpi8sqrt2);
q10 = vqdmulhq_n_s16(q2tmp2.val[1], cospi8sqrt2minus1);
q11 = vqdmulhq_n_s16(q2tmp3.val[1], cospi8sqrt2minus1);
q2 = vqaddq_s16(q2tmp2.val[0], q2tmp3.val[0]);
q3 = vqsubq_s16(q2tmp2.val[0], q2tmp3.val[0]);
q10 = vshrq_n_s16(q10, 1);
q11 = vshrq_n_s16(q11, 1);
q10 = vqaddq_s16(q2tmp2.val[1], q10);
q11 = vqaddq_s16(q2tmp3.val[1], q11);
q8 = vqsubq_s16(q8, q11);
q9 = vqaddq_s16(q9, q10);
q4 = vqaddq_s16(q2, q9);
q5 = vqaddq_s16(q3, q8);
q6 = vqsubq_s16(q3, q8);
q7 = vqsubq_s16(q2, q9);
q4 = vrshrq_n_s16(q4, 3);
q5 = vrshrq_n_s16(q5, 3);
q6 = vrshrq_n_s16(q6, 3);
q7 = vrshrq_n_s16(q7, 3);
q2tmp0 = vtrnq_s32(vreinterpretq_s32_s16(q4), vreinterpretq_s32_s16(q6));
q2tmp1 = vtrnq_s32(vreinterpretq_s32_s16(q5), vreinterpretq_s32_s16(q7));
q2tmp2 = vtrnq_s16(vreinterpretq_s16_s32(q2tmp0.val[0]),
vreinterpretq_s16_s32(q2tmp1.val[0]));
q2tmp3 = vtrnq_s16(vreinterpretq_s16_s32(q2tmp0.val[1]),
vreinterpretq_s16_s32(q2tmp1.val[1]));
q4 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q2tmp2.val[0]),
vreinterpret_u8_s32(d28)));
q5 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q2tmp2.val[1]),
vreinterpret_u8_s32(d29)));
q6 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q2tmp3.val[0]),
vreinterpret_u8_s32(d30)));
q7 = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(q2tmp3.val[1]),
vreinterpret_u8_s32(d31)));
d28 = vreinterpret_s32_u8(vqmovun_s16(q4));
d29 = vreinterpret_s32_u8(vqmovun_s16(q5));
d30 = vreinterpret_s32_u8(vqmovun_s16(q6));
d31 = vreinterpret_s32_u8(vqmovun_s16(q7));
dst0 = dst;
dst1 = dst + 4;
vst1_lane_s32((int32_t *)dst0, d28, 0);
dst0 += stride;
vst1_lane_s32((int32_t *)dst1, d28, 1);
dst1 += stride;
vst1_lane_s32((int32_t *)dst0, d29, 0);
dst0 += stride;
vst1_lane_s32((int32_t *)dst1, d29, 1);
dst1 += stride;
vst1_lane_s32((int32_t *)dst0, d30, 0);
dst0 += stride;
vst1_lane_s32((int32_t *)dst1, d30, 1);
dst1 += stride;
vst1_lane_s32((int32_t *)dst0, d31, 0);
vst1_lane_s32((int32_t *)dst1, d31, 1);
return;
}
