static void variance_neon_w8(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
int w, int h, unsigned int *sse, int *sum) {
int i, j;
int16x8_t v_sum = vdupq_n_s16(0);
int32x4_t v_sse_lo = vdupq_n_s32(0);
int32x4_t v_sse_hi = vdupq_n_s32(0);
for (i = 0; i < h; ++i) {
for (j = 0; j < w; j += 8) {
const uint8x8_t v_a = vld1_u8(&a[j]);
const uint8x8_t v_b = vld1_u8(&b[j]);
const uint16x8_t v_diff = vsubl_u8(v_a, v_b);
const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff);
v_sum = vaddq_s16(v_sum, sv_diff);
v_sse_lo = vmlal_s16(v_sse_lo,
vget_low_s16(sv_diff),
vget_low_s16(sv_diff));
v_sse_hi = vmlal_s16(v_sse_hi,
vget_high_s16(sv_diff),
vget_high_s16(sv_diff));
}
a += a_stride;
b += b_stride;
}
*sum = horizontal_add_s16x8(v_sum);
*sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi));
}
static INLINE void IDCT4x4_1D(int16x4_t *d0s16, int16x4_t *d1s16,
int16x4_t *d2s16, int16x8_t *q8s16,
int16x8_t *q9s16) {
int16x4_t d16s16, d17s16, d18s16, d19s16, d23s16, d24s16;
int16x4_t d26s16, d27s16, d28s16, d29s16;
int32x4_t q10s32, q13s32, q14s32, q15s32;
int16x8_t q13s16, q14s16;
d16s16 = vget_low_s16(*q8s16);
d17s16 = vget_high_s16(*q8s16);
d18s16 = vget_low_s16(*q9s16);
d19s16 = vget_high_s16(*q9s16);
d23s16 = vadd_s16(d16s16, d18s16);
d24s16 = vsub_s16(d16s16, d18s16);
q15s32 = vmull_s16(d17s16, *d2s16);
q10s32 = vmull_s16(d17s16, *d0s16);
q13s32 = vmull_s16(d23s16, *d1s16);
q14s32 = vmull_s16(d24s16, *d1s16);
q15s32 = vmlsl_s16(q15s32, d19s16, *d0s16);
q10s32 = vmlal_s16(q10s32, d19s16, *d2s16);
d26s16 = vqrshrn_n_s32(q13s32, 14);
d27s16 = vqrshrn_n_s32(q14s32, 14);
d29s16 = vqrshrn_n_s32(q15s32, 14);
d28s16 = vqrshrn_n_s32(q10s32, 14);
q13s16 = vcombine_s16(d26s16, d27s16);
q14s16 = vcombine_s16(d28s16, d29s16);
*q8s16 = vaddq_s16(q13s16, q14s16);
*q9s16 = vsubq_s16(q13s16, q14s16);
*q9s16 = vcombine_s16(vget_high_s16(*q9s16), vget_low_s16(*q9s16)); // vswp
return;
}
int64_t av1_block_error_fp_neon(const int16_t *coeff, const int16_t *dqcoeff,
int block_size) {
int64x2_t error = vdupq_n_s64(0);
assert(block_size >= 8);
assert((block_size % 8) == 0);
do {
const int16x8_t c = vld1q_s16(coeff);
const int16x8_t d = vld1q_s16(dqcoeff);
const int16x8_t diff = vsubq_s16(c, d);
const int16x4_t diff_lo = vget_low_s16(diff);
const int16x4_t diff_hi = vget_high_s16(diff);
// diff is 15-bits, the squares 30, so we can store 2 in 31-bits before
// accumulating them in 64-bits.
const int32x4_t err0 = vmull_s16(diff_lo, diff_lo);
const int32x4_t err1 = vmlal_s16(err0, diff_hi, diff_hi);
const int64x2_t err2 = vaddl_s32(vget_low_s32(err1), vget_high_s32(err1));
error = vaddq_s64(error, err2);
coeff += 8;
dqcoeff += 8;
block_size -= 8;
} while (block_size != 0);
return vgetq_lane_s64(error, 0) + vgetq_lane_s64(error, 1);
}
void test_vmlals16 (void)
{
int32x4_t out_int32x4_t;
int32x4_t arg0_int32x4_t;
int16x4_t arg1_int16x4_t;
int16x4_t arg2_int16x4_t;
out_int32x4_t = vmlal_s16 (arg0_int32x4_t, arg1_int16x4_t, arg2_int16x4_t);
}
void dotProd_i16_neon(const float *dataf, const float *weightsf, float *vals, const int n, const int len, const float *istd) {
const int16_t *data = (const int16_t *)dataf;
const int16_t *weights = (const int16_t *)weightsf;
weightsf += n * len / 2; // sizeof(float) / sizeof(int16_t)
for (int i = 0; i < n; i += 4) {
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int j = 0; j < len; j += 8) {
int16x4x2_t d0 = vld2_s16(data + j);
int16x4x2_t w0 = vld2_s16(weights);
int16x4x2_t w1 = vld2_s16(weights + 8);
int16x4x2_t w2 = vld2_s16(weights + 16);
int16x4x2_t w3 = vld2_s16(weights + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
weights += 32;
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t val = vcvtq_f32_s32(sum);
val = vmulq_f32(val, vld1q_f32(weightsf + i*2));
val = vmulq_n_f32(val, istd[0]);
val = vaddq_f32(val, vld1q_f32(weightsf + i*2 + 4));
vst1q_f32(vals + i, val);
}
}
// ref, src = [0, 510] - max diff = 16-bits
// bwl = {2, 3, 4}, width = {16, 32, 64}
int vp9_vector_var_neon(int16_t const *ref, int16_t const *src, const int bwl) {
int width = 4 << bwl;
int32x4_t sse = vdupq_n_s32(0);
int16x8_t total = vdupq_n_s16(0);
assert(width >= 8);
assert((width % 8) == 0);
do {
const int16x8_t r = vld1q_s16(ref);
const int16x8_t s = vld1q_s16(src);
const int16x8_t diff = vsubq_s16(r, s); // [-510, 510], 10 bits.
const int16x4_t diff_lo = vget_low_s16(diff);
const int16x4_t diff_hi = vget_high_s16(diff);
sse = vmlal_s16(sse, diff_lo, diff_lo); // dynamic range 26 bits.
sse = vmlal_s16(sse, diff_hi, diff_hi);
total = vaddq_s16(total, diff); // dynamic range 16 bits.
ref += 8;
src += 8;
width -= 8;
} while (width != 0);
{
// Note: 'total''s pairwise addition could be implemented similarly to
// horizontal_add_u16x8(), but one less vpaddl with 'total' when paired
// with the summation of 'sse' performed better on a Cortex-A15.
const int32x4_t t0 = vpaddlq_s16(total); // cascading summation of 'total'
const int32x2_t t1 = vadd_s32(vget_low_s32(t0), vget_high_s32(t0));
const int32x2_t t2 = vpadd_s32(t1, t1);
const int t = vget_lane_s32(t2, 0);
const int64x2_t s0 = vpaddlq_s32(sse); // cascading summation of 'sse'.
const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)),
vreinterpret_s32_s64(vget_high_s64(s0)));
const int s = vget_lane_s32(s1, 0);
const int shift_factor = bwl + 2;
return s - ((t * t) >> shift_factor);
}
}
void aom_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int dest_stride) {
uint8x8_t d26u8, d27u8;
uint32x2_t d26u32, d27u32;
uint16x8_t q8u16, q9u16;
int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16;
int16x4_t d22s16, d23s16, d24s16, d26s16, d27s16, d28s16, d29s16;
int16x8_t q8s16, q9s16, q13s16, q14s16;
int32x4_t q1s32, q13s32, q14s32, q15s32;
int16x4x2_t d0x2s16, d1x2s16;
int32x4x2_t q0x2s32;
uint8_t *d;
d26u32 = d27u32 = vdup_n_u32(0);
q8s16 = vld1q_s16(input);
q9s16 = vld1q_s16(input + 8);
d16s16 = vget_low_s16(q8s16);
d17s16 = vget_high_s16(q8s16);
d18s16 = vget_low_s16(q9s16);
d19s16 = vget_high_s16(q9s16);
d0x2s16 = vtrn_s16(d16s16, d17s16);
d1x2s16 = vtrn_s16(d18s16, d19s16);
q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]);
q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]);
d20s16 = vdup_n_s16((int16_t)cospi_8_64);
d21s16 = vdup_n_s16((int16_t)cospi_16_64);
q0x2s32 =
vtrnq_s32(vreinterpretq_s32_s16(q8s16), vreinterpretq_s32_s16(q9s16));
d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
d22s16 = vdup_n_s16((int16_t)cospi_24_64);
// stage 1
d23s16 = vadd_s16(d16s16, d18s16);
d24s16 = vsub_s16(d16s16, d18s16);
q15s32 = vmull_s16(d17s16, d22s16);
q1s32 = vmull_s16(d17s16, d20s16);
q13s32 = vmull_s16(d23s16, d21s16);
q14s32 = vmull_s16(d24s16, d21s16);
q15s32 = vmlsl_s16(q15s32, d19s16, d20s16);
q1s32 = vmlal_s16(q1s32, d19s16, d22s16);
d26s16 = vqrshrn_n_s32(q13s32, 14);
d27s16 = vqrshrn_n_s32(q14s32, 14);
d29s16 = vqrshrn_n_s32(q15s32, 14);
d28s16 = vqrshrn_n_s32(q1s32, 14);
q13s16 = vcombine_s16(d26s16, d27s16);
q14s16 = vcombine_s16(d28s16, d29s16);
// stage 2
q8s16 = vaddq_s16(q13s16, q14s16);
q9s16 = vsubq_s16(q13s16, q14s16);
d16s16 = vget_low_s16(q8s16);
d17s16 = vget_high_s16(q8s16);
d18s16 = vget_high_s16(q9s16); // vswp d18 d19
d19s16 = vget_low_s16(q9s16);
d0x2s16 = vtrn_s16(d16s16, d17s16);
d1x2s16 = vtrn_s16(d18s16, d19s16);
q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]);
q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]);
q0x2s32 =
vtrnq_s32(vreinterpretq_s32_s16(q8s16), vreinterpretq_s32_s16(q9s16));
d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
// do the transform on columns
// stage 1
d23s16 = vadd_s16(d16s16, d18s16);
d24s16 = vsub_s16(d16s16, d18s16);
q15s32 = vmull_s16(d17s16, d22s16);
q1s32 = vmull_s16(d17s16, d20s16);
q13s32 = vmull_s16(d23s16, d21s16);
q14s32 = vmull_s16(d24s16, d21s16);
q15s32 = vmlsl_s16(q15s32, d19s16, d20s16);
q1s32 = vmlal_s16(q1s32, d19s16, d22s16);
d26s16 = vqrshrn_n_s32(q13s32, 14);
d27s16 = vqrshrn_n_s32(q14s32, 14);
d29s16 = vqrshrn_n_s32(q15s32, 14);
d28s16 = vqrshrn_n_s32(q1s32, 14);
q13s16 = vcombine_s16(d26s16, d27s16);
q14s16 = vcombine_s16(d28s16, d29s16);
// stage 2
q8s16 = vaddq_s16(q13s16, q14s16);
q9s16 = vsubq_s16(q13s16, q14s16);
q8s16 = vrshrq_n_s16(q8s16, 4);
q9s16 = vrshrq_n_s16(q9s16, 4);
d = dest;
d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 0);
d += dest_stride;
d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 1);
d += dest_stride;
d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 1);
d += dest_stride;
d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 0);
q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u32(d26u32));
q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u32(d27u32));
d26u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16));
d27u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16));
d = dest;
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 0);
d += dest_stride;
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 1);
d += dest_stride;
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 1);
d += dest_stride;
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 0);
return;
}
void computeNetwork0new_neon(const float *dataf, const float *weightsf, uint8_t *d) {
const int16_t *data = (const int16_t *)dataf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 128/2; i += 8) {
int16x4x2_t d0 = vld2_s16(data + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 512/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 528/4));
float32x4_t m1, m2, m3, m4;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 544/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + 560/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + 576/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + 592/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + 608/4));
uint32x4_t gte = vcgeq_f32(m1, zeroes_f);
uint16x4_t gte_u16 = vmovn_u32(gte);
uint8x8_t gte_u8 = vmovn_u16(vcombine_u16(gte_u16, vget_low_u16(vreinterpretq_u16_u32(sign_bits_f))));
gte_u8 = vshr_n_u8(gte_u8, 7);
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(gte_u8), 0);
}
void computeNetwork0_i16_neon(const float *inputf, const float *weightsf, uint8_t *d) {
const int16_t *input = (const int16_t *)inputf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 96/2; i += 8) {
int16x4x2_t d0 = vld2_s16(input + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 384/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 400/4));
float32x4_t m1, m2, m3, m4, m5, m6, m7;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 416/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (416+16)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (416+32)/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (416+48)/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + (416+64)/4));
m7 = m1;
m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f));
m1 = vaddq_f32(m1, ones_f);
m7 = vmulq_f32(reciprocal(m1), m7);
m3 = m0;
m0 = vdupq_lane_f32(vget_low_f32(m0), 0);
m1 = vdupq_lane_f32(vget_low_f32(m3), 1);
m2 = vdupq_lane_f32(vget_high_f32(m3), 0);
m3 = vdupq_lane_f32(vget_high_f32(m3), 1);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 496/4));
m1 = vmulq_f32(m1, vld1q_f32(weightsf + (496+16)/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (496+32)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (496+48)/4));
m4 = vdupq_lane_f32(vget_low_f32(m7), 0);
m5 = vdupq_lane_f32(vget_low_f32(m7), 1);
m6 = vdupq_lane_f32(vget_high_f32(m7), 0);
m7 = vdupq_lane_f32(vget_high_f32(m7), 1);
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (496+64)/4));
m5 = vmulq_f32(m5, vld1q_f32(weightsf + (496+80)/4));
m6 = vmulq_f32(m6, vld1q_f32(weightsf + (496+96)/4));
m7 = vmulq_f32(m7, vld1q_f32(weightsf + (496+112)/4));
m0 = vaddq_f32(m0, m1);
m2 = vaddq_f32(m2, m3);
m4 = vaddq_f32(m4, m5);
m6 = vaddq_f32(m6, m7);
m0 = vaddq_f32(m0, m2);
m4 = vaddq_f32(m4, m6);
m0 = vaddq_f32(m0, m4);
m0 = vaddq_f32(m0, vld1q_f32(weightsf + (496+128)/4));
float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0));
d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0));
}
unsigned int vp8_variance16x8_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int32x4_t q8s32, q9s32, q10s32;
int64x2_t q0s64, q1s64, q5s64;
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
for (i = 0; i < 4; i++) { // variance16x8_neon_loop
q0u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q1u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
__builtin_prefetch(src_ptr);
q2u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q3u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
__builtin_prefetch(ref_ptr);
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q10s32 = vaddq_s32(q10s32, q9s32);
q0s64 = vpaddlq_s32(q8s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
void vp8_short_fdct8x4_neon(
int16_t *input,
int16_t *output,
int pitch) {
int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16;
int16x4_t d16s16, d17s16, d26s16, d27s16, d28s16, d29s16;
uint16x4_t d28u16, d29u16;
uint16x8_t q14u16;
int16x8_t q0s16, q1s16, q2s16, q3s16;
int16x8_t q11s16, q12s16, q13s16, q14s16, q15s16, qEmptys16;
int32x4_t q9s32, q10s32, q11s32, q12s32;
int16x8x2_t v2tmp0, v2tmp1;
int32x4x2_t v2tmp2, v2tmp3;
d16s16 = vdup_n_s16(5352);
d17s16 = vdup_n_s16(2217);
q9s32 = vdupq_n_s32(14500);
q10s32 = vdupq_n_s32(7500);
// Part one
pitch >>= 1;
q0s16 = vld1q_s16(input);
input += pitch;
q1s16 = vld1q_s16(input);
input += pitch;
q2s16 = vld1q_s16(input);
input += pitch;
q3s16 = vld1q_s16(input);
v2tmp2 = vtrnq_s32(vreinterpretq_s32_s16(q0s16),
vreinterpretq_s32_s16(q2s16));
v2tmp3 = vtrnq_s32(vreinterpretq_s32_s16(q1s16),
vreinterpretq_s32_s16(q3s16));
v2tmp0 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[0]), // q0
vreinterpretq_s16_s32(v2tmp3.val[0])); // q1
v2tmp1 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[1]), // q2
vreinterpretq_s16_s32(v2tmp3.val[1])); // q3
q11s16 = vaddq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q12s16 = vaddq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q13s16 = vsubq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q14s16 = vsubq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q11s16 = vshlq_n_s16(q11s16, 3);
q12s16 = vshlq_n_s16(q12s16, 3);
q13s16 = vshlq_n_s16(q13s16, 3);
q14s16 = vshlq_n_s16(q14s16, 3);
q0s16 = vaddq_s16(q11s16, q12s16);
q2s16 = vsubq_s16(q11s16, q12s16);
q11s32 = q9s32;
q12s32 = q10s32;
d26s16 = vget_low_s16(q13s16);
d27s16 = vget_high_s16(q13s16);
d28s16 = vget_low_s16(q14s16);
d29s16 = vget_high_s16(q14s16);
q9s32 = vmlal_s16(q9s32, d28s16, d16s16);
q10s32 = vmlal_s16(q10s32, d28s16, d17s16);
q11s32 = vmlal_s16(q11s32, d29s16, d16s16);
q12s32 = vmlal_s16(q12s32, d29s16, d17s16);
q9s32 = vmlal_s16(q9s32, d26s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d26s16, d16s16);
q11s32 = vmlal_s16(q11s32, d27s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d27s16, d16s16);
d2s16 = vshrn_n_s32(q9s32, 12);
d6s16 = vshrn_n_s32(q10s32, 12);
d3s16 = vshrn_n_s32(q11s32, 12);
d7s16 = vshrn_n_s32(q12s32, 12);
q1s16 = vcombine_s16(d2s16, d3s16);
q3s16 = vcombine_s16(d6s16, d7s16);
// Part two
q9s32 = vdupq_n_s32(12000);
q10s32 = vdupq_n_s32(51000);
v2tmp2 = vtrnq_s32(vreinterpretq_s32_s16(q0s16),
vreinterpretq_s32_s16(q2s16));
v2tmp3 = vtrnq_s32(vreinterpretq_s32_s16(q1s16),
vreinterpretq_s32_s16(q3s16));
v2tmp0 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[0]), // q0
vreinterpretq_s16_s32(v2tmp3.val[0])); // q1
v2tmp1 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[1]), // q2
vreinterpretq_s16_s32(v2tmp3.val[1])); // q3
q11s16 = vaddq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q12s16 = vaddq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q13s16 = vsubq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q14s16 = vsubq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q15s16 = vdupq_n_s16(7);
q11s16 = vaddq_s16(q11s16, q15s16);
q0s16 = vaddq_s16(q11s16, q12s16);
q1s16 = vsubq_s16(q11s16, q12s16);
q11s32 = q9s32;
q12s32 = q10s32;
d0s16 = vget_low_s16(q0s16);
d1s16 = vget_high_s16(q0s16);
d2s16 = vget_low_s16(q1s16);
d3s16 = vget_high_s16(q1s16);
d0s16 = vshr_n_s16(d0s16, 4);
d4s16 = vshr_n_s16(d1s16, 4);
d2s16 = vshr_n_s16(d2s16, 4);
d6s16 = vshr_n_s16(d3s16, 4);
d26s16 = vget_low_s16(q13s16);
d27s16 = vget_high_s16(q13s16);
d28s16 = vget_low_s16(q14s16);
d29s16 = vget_high_s16(q14s16);
q9s32 = vmlal_s16(q9s32, d28s16, d16s16);
q10s32 = vmlal_s16(q10s32, d28s16, d17s16);
q11s32 = vmlal_s16(q11s32, d29s16, d16s16);
q12s32 = vmlal_s16(q12s32, d29s16, d17s16);
q9s32 = vmlal_s16(q9s32, d26s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d26s16, d16s16);
q11s32 = vmlal_s16(q11s32, d27s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d27s16, d16s16);
d1s16 = vshrn_n_s32(q9s32, 16);
d3s16 = vshrn_n_s32(q10s32, 16);
d5s16 = vshrn_n_s32(q11s32, 16);
d7s16 = vshrn_n_s32(q12s32, 16);
qEmptys16 = vdupq_n_s16(0);
q14u16 = vceqq_s16(q14s16, qEmptys16);
q14u16 = vmvnq_u16(q14u16);
d28u16 = vget_low_u16(q14u16);
d29u16 = vget_high_u16(q14u16);
d1s16 = vsub_s16(d1s16, vreinterpret_s16_u16(d28u16));
d5s16 = vsub_s16(d5s16, vreinterpret_s16_u16(d29u16));
q0s16 = vcombine_s16(d0s16, d1s16);
q1s16 = vcombine_s16(d2s16, d3s16);
q2s16 = vcombine_s16(d4s16, d5s16);
q3s16 = vcombine_s16(d6s16, d7s16);
vst1q_s16(output, q0s16);
vst1q_s16(output + 8, q1s16);
vst1q_s16(output + 16, q2s16);
vst1q_s16(output + 24, q3s16);
return;
}
unsigned int vp8_variance_halfpixvar16x16_hv_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
int16x4_t d0s16, d1s16, d2s16, d3s16, d10s16, d11s16, d12s16, d13s16;
int16x4_t d18s16, d19s16, d20s16, d21s16, d22s16, d23s16, d24s16, d25s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64, d2s64, d3s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8, q4u8, q5u8, q6u8, q7u8, q8u8, q9u8;
uint16x8_t q0u16, q1u16, q5u16, q6u16, q9u16, q10u16, q11u16, q12u16;
int32x4_t q13s32, q14s32, q15s32;
int64x2_t q0s64, q1s64, q5s64;
q13s32 = vdupq_n_s32(0);
q14s32 = vdupq_n_s32(0);
q15s32 = vdupq_n_s32(0);
q0u8 = vld1q_u8(src_ptr);
q1u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q1u8 = vextq_u8(q0u8, q1u8, 1);
q0u8 = vrhaddq_u8(q0u8, q1u8);
for (i = 0; i < 4; i++) { // vp8_filt_fpo16x16s_4_0_loop_neon
q2u8 = vld1q_u8(src_ptr);
q3u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q4u8 = vld1q_u8(src_ptr);
q5u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q6u8 = vld1q_u8(src_ptr);
q7u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q8u8 = vld1q_u8(src_ptr);
q9u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q3u8 = vextq_u8(q2u8, q3u8, 1);
q5u8 = vextq_u8(q4u8, q5u8, 1);
q7u8 = vextq_u8(q6u8, q7u8, 1);
q9u8 = vextq_u8(q8u8, q9u8, 1);
q1u8 = vrhaddq_u8(q2u8, q3u8);
q2u8 = vrhaddq_u8(q4u8, q5u8);
q3u8 = vrhaddq_u8(q6u8, q7u8);
q4u8 = vrhaddq_u8(q8u8, q9u8);
q0u8 = vrhaddq_u8(q0u8, q1u8);
q1u8 = vrhaddq_u8(q1u8, q2u8);
q2u8 = vrhaddq_u8(q2u8, q3u8);
q3u8 = vrhaddq_u8(q3u8, q4u8);
q5u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q6u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q7u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q8u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
d0u8 = vget_low_u8(q0u8);
d1u8 = vget_high_u8(q0u8);
d2u8 = vget_low_u8(q1u8);
d3u8 = vget_high_u8(q1u8);
d4u8 = vget_low_u8(q2u8);
d5u8 = vget_high_u8(q2u8);
d6u8 = vget_low_u8(q3u8);
d7u8 = vget_high_u8(q3u8);
q9u16 = vsubl_u8(d0u8, vget_low_u8(q5u8));
q10u16 = vsubl_u8(d1u8, vget_high_u8(q5u8));
q11u16 = vsubl_u8(d2u8, vget_low_u8(q6u8));
q12u16 = vsubl_u8(d3u8, vget_high_u8(q6u8));
q0u16 = vsubl_u8(d4u8, vget_low_u8(q7u8));
q1u16 = vsubl_u8(d5u8, vget_high_u8(q7u8));
q5u16 = vsubl_u8(d6u8, vget_low_u8(q8u8));
q6u16 = vsubl_u8(d7u8, vget_high_u8(q8u8));
d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16));
d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q9u16));
q14s32 = vmlal_s16(q14s32, d18s16, d18s16);
q15s32 = vmlal_s16(q15s32, d19s16, d19s16);
d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16));
d21s16 = vreinterpret_s16_u16(vget_high_u16(q10u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q10u16));
q14s32 = vmlal_s16(q14s32, d20s16, d20s16);
q15s32 = vmlal_s16(q15s32, d21s16, d21s16);
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q11u16));
q14s32 = vmlal_s16(q14s32, d22s16, d22s16);
q15s32 = vmlal_s16(q15s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q12u16));
q14s32 = vmlal_s16(q14s32, d24s16, d24s16);
q15s32 = vmlal_s16(q15s32, d25s16, d25s16);
d0s16 = vreinterpret_s16_u16(vget_low_u16(q0u16));
d1s16 = vreinterpret_s16_u16(vget_high_u16(q0u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q0u16));
q14s32 = vmlal_s16(q14s32, d0s16, d0s16);
q15s32 = vmlal_s16(q15s32, d1s16, d1s16);
d2s16 = vreinterpret_s16_u16(vget_low_u16(q1u16));
d3s16 = vreinterpret_s16_u16(vget_high_u16(q1u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q1u16));
q14s32 = vmlal_s16(q14s32, d2s16, d2s16);
q15s32 = vmlal_s16(q15s32, d3s16, d3s16);
d10s16 = vreinterpret_s16_u16(vget_low_u16(q5u16));
d11s16 = vreinterpret_s16_u16(vget_high_u16(q5u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q5u16));
q14s32 = vmlal_s16(q14s32, d10s16, d10s16);
q15s32 = vmlal_s16(q15s32, d11s16, d11s16);
d12s16 = vreinterpret_s16_u16(vget_low_u16(q6u16));
d13s16 = vreinterpret_s16_u16(vget_high_u16(q6u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q6u16));
q14s32 = vmlal_s16(q14s32, d12s16, d12s16);
q15s32 = vmlal_s16(q15s32, d13s16, d13s16);
q0u8 = q4u8;
}
q15s32 = vaddq_s32(q14s32, q15s32);
q0s64 = vpaddlq_s32(q13s32);
q1s64 = vpaddlq_s32(q15s32);
d0s64 = vget_low_s64(q0s64);
d1s64 = vget_high_s64(q0s64);
d2s64 = vget_low_s64(q1s64);
d3s64 = vget_high_s64(q1s64);
d0s64 = vadd_s64(d0s64, d1s64);
d1s64 = vadd_s64(d2s64, d3s64);
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 8);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
unsigned int vpx_mse16x16_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
int64x1_t d0s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8;
int32x4_t q7s32, q8s32, q9s32, q10s32;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int64x2_t q1s64;
q7s32 = vdupq_n_s32(0);
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
for (i = 0; i < 8; i++) { // mse16x16_neon_loop
q0u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q1u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q2u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q3u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q7s32 = vaddq_s32(q7s32, q8s32);
q9s32 = vaddq_s32(q9s32, q10s32);
q10s32 = vaddq_s32(q7s32, q9s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
}
void vpx_idct8x8_12_add_neon(
int16_t *input,
uint8_t *dest,
int dest_stride) {
uint8_t *d1, *d2;
uint8x8_t d0u8, d1u8, d2u8, d3u8;
int16x4_t d10s16, d11s16, d12s16, d13s16, d16s16;
int16x4_t d26s16, d27s16, d28s16, d29s16;
uint64x1_t d0u64, d1u64, d2u64, d3u64;
int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16;
int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16, q13s16, q14s16, q15s16;
uint16x8_t q8u16, q9u16, q10u16, q11u16;
int32x4_t q9s32, q10s32, q11s32, q12s32;
q8s16 = vld1q_s16(input);
q9s16 = vld1q_s16(input + 8);
q10s16 = vld1q_s16(input + 16);
q11s16 = vld1q_s16(input + 24);
q12s16 = vld1q_s16(input + 32);
q13s16 = vld1q_s16(input + 40);
q14s16 = vld1q_s16(input + 48);
q15s16 = vld1q_s16(input + 56);
TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16,
&q12s16, &q13s16, &q14s16, &q15s16);
// First transform rows
// stage 1
q0s16 = vdupq_n_s16(cospi_28_64 * 2);
q1s16 = vdupq_n_s16(cospi_4_64 * 2);
q4s16 = vqrdmulhq_s16(q9s16, q0s16);
q0s16 = vdupq_n_s16(-cospi_20_64 * 2);
q7s16 = vqrdmulhq_s16(q9s16, q1s16);
q1s16 = vdupq_n_s16(cospi_12_64 * 2);
q5s16 = vqrdmulhq_s16(q11s16, q0s16);
q0s16 = vdupq_n_s16(cospi_16_64 * 2);
q6s16 = vqrdmulhq_s16(q11s16, q1s16);
// stage 2 & stage 3 - even half
q1s16 = vdupq_n_s16(cospi_24_64 * 2);
q9s16 = vqrdmulhq_s16(q8s16, q0s16);
q0s16 = vdupq_n_s16(cospi_8_64 * 2);
q13s16 = vqrdmulhq_s16(q10s16, q1s16);
q15s16 = vqrdmulhq_s16(q10s16, q0s16);
// stage 3 -odd half
q0s16 = vaddq_s16(q9s16, q15s16);
q1s16 = vaddq_s16(q9s16, q13s16);
q2s16 = vsubq_s16(q9s16, q13s16);
q3s16 = vsubq_s16(q9s16, q15s16);
// stage 2 - odd half
q13s16 = vsubq_s16(q4s16, q5s16);
q4s16 = vaddq_s16(q4s16, q5s16);
q14s16 = vsubq_s16(q7s16, q6s16);
q7s16 = vaddq_s16(q7s16, q6s16);
d26s16 = vget_low_s16(q13s16);
d27s16 = vget_high_s16(q13s16);
d28s16 = vget_low_s16(q14s16);
d29s16 = vget_high_s16(q14s16);
d16s16 = vdup_n_s16(cospi_16_64);
q9s32 = vmull_s16(d28s16, d16s16);
q10s32 = vmull_s16(d29s16, d16s16);
q11s32 = vmull_s16(d28s16, d16s16);
q12s32 = vmull_s16(d29s16, d16s16);
q9s32 = vmlsl_s16(q9s32, d26s16, d16s16);
q10s32 = vmlsl_s16(q10s32, d27s16, d16s16);
q11s32 = vmlal_s16(q11s32, d26s16, d16s16);
q12s32 = vmlal_s16(q12s32, d27s16, d16s16);
d10s16 = vqrshrn_n_s32(q9s32, 14);
d11s16 = vqrshrn_n_s32(q10s32, 14);
d12s16 = vqrshrn_n_s32(q11s32, 14);
d13s16 = vqrshrn_n_s32(q12s32, 14);
q5s16 = vcombine_s16(d10s16, d11s16);
q6s16 = vcombine_s16(d12s16, d13s16);
// stage 4
q8s16 = vaddq_s16(q0s16, q7s16);
q9s16 = vaddq_s16(q1s16, q6s16);
q10s16 = vaddq_s16(q2s16, q5s16);
q11s16 = vaddq_s16(q3s16, q4s16);
q12s16 = vsubq_s16(q3s16, q4s16);
q13s16 = vsubq_s16(q2s16, q5s16);
q14s16 = vsubq_s16(q1s16, q6s16);
q15s16 = vsubq_s16(q0s16, q7s16);
TRANSPOSE8X8(&q8s16, &q9s16, &q10s16, &q11s16,
&q12s16, &q13s16, &q14s16, &q15s16);
IDCT8x8_1D(&q8s16, &q9s16, &q10s16, &q11s16,
&q12s16, &q13s16, &q14s16, &q15s16);
q8s16 = vrshrq_n_s16(q8s16, 5);
q9s16 = vrshrq_n_s16(q9s16, 5);
q10s16 = vrshrq_n_s16(q10s16, 5);
q11s16 = vrshrq_n_s16(q11s16, 5);
q12s16 = vrshrq_n_s16(q12s16, 5);
q13s16 = vrshrq_n_s16(q13s16, 5);
q14s16 = vrshrq_n_s16(q14s16, 5);
q15s16 = vrshrq_n_s16(q15s16, 5);
d1 = d2 = dest;
d0u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d1u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d2u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d3u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16),
vreinterpret_u8_u64(d0u64));
q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16),
vreinterpret_u8_u64(d1u64));
q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16),
vreinterpret_u8_u64(d2u64));
q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16),
vreinterpret_u8_u64(d3u64));
d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16));
d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16));
d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16));
d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16));
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8));
d2 += dest_stride;
q8s16 = q12s16;
q9s16 = q13s16;
q10s16 = q14s16;
q11s16 = q15s16;
d0u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d1u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d2u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
d3u64 = vld1_u64((uint64_t *)d1);
d1 += dest_stride;
q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16),
vreinterpret_u8_u64(d0u64));
q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16),
vreinterpret_u8_u64(d1u64));
q10u16 = vaddw_u8(vreinterpretq_u16_s16(q10s16),
vreinterpret_u8_u64(d2u64));
q11u16 = vaddw_u8(vreinterpretq_u16_s16(q11s16),
vreinterpret_u8_u64(d3u64));
d0u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16));
d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16));
d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q10u16));
d3u8 = vqmovun_s16(vreinterpretq_s16_u16(q11u16));
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d0u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d1u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d2u8));
d2 += dest_stride;
vst1_u64((uint64_t *)d2, vreinterpret_u64_u8(d3u8));
d2 += dest_stride;
return;
}
inline void MAC4 (int32x4_t *additive_value, int16x4_t *data1, int16x4_t *data2, int32x4_t *mac_output)
{
*mac_output = vmlal_s16(*additive_value,*data1, *data2);
}
static inline int32_t TransformAndFindMaxNeon(int16_t* inre,
int16_t* inim,
int32_t* outre,
int32_t* outim) {
int k;
int16_t* inre1 = inre;
int16_t* inre2 = &inre[FRAMESAMPLES/2 - 4];
int16_t* inim1 = inim;
int16_t* inim2 = &inim[FRAMESAMPLES/2 - 4];
int32_t* outre1 = outre;
int32_t* outre2 = &outre[FRAMESAMPLES/2 - 4];
int32_t* outim1 = outim;
int32_t* outim2 = &outim[FRAMESAMPLES/2 - 4];
const int16_t* kSinTab1 = &WebRtcIsacfix_kSinTab2[0];
const int16_t* kSinTab2 = &WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 - 4];
uint32x4_t max_r = vdupq_n_u32(0);
uint32x4_t max_i = vdupq_n_u32(0);
// Use ">> 5", instead of "<< 9" and then ">> 14" as in the C code.
for (k = 0; k < FRAMESAMPLES/4; k += 4) {
int16x4_t tmpi = vld1_s16(kSinTab1);
kSinTab1 += 4;
int16x4_t tmpr = vld1_s16(kSinTab2);
kSinTab2 -= 4;
int16x4_t inre_0 = vld1_s16(inre1);
inre1 += 4;
int16x4_t inre_1 = vld1_s16(inre2);
inre2 -= 4;
int16x4_t inim_0 = vld1_s16(inim1);
inim1 += 4;
int16x4_t inim_1 = vld1_s16(inim2);
inim2 -= 4;
tmpr = vneg_s16(tmpr);
inre_1 = vrev64_s16(inre_1);
inim_1 = vrev64_s16(inim_1);
tmpr = vrev64_s16(tmpr);
int32x4_t xr = vmull_s16(tmpr, inre_0);
int32x4_t xi = vmull_s16(tmpr, inim_0);
int32x4_t yr = vmull_s16(tmpr, inim_1);
int32x4_t yi = vmull_s16(tmpi, inim_1);
xr = vmlal_s16(xr, tmpi, inim_0);
xi = vmlsl_s16(xi, tmpi, inre_0);
yr = vmlal_s16(yr, tmpi, inre_1);
yi = vmlsl_s16(yi, tmpr, inre_1);
yr = vnegq_s32(yr);
xr = vshrq_n_s32(xr, 5);
xi = vshrq_n_s32(xi, 5);
yr = vshrq_n_s32(yr, 5);
yi = vshrq_n_s32(yi, 5);
int32x4_t outr0 = vsubq_s32(xr, yi);
int32x4_t outr1 = vaddq_s32(xr, yi);
int32x4_t outi0 = vaddq_s32(xi, yr);
int32x4_t outi1 = vsubq_s32(yr, xi);
// Find the absolute maximum in the vectors.
int32x4_t tmp0 = vabsq_s32(outr0);
int32x4_t tmp1 = vabsq_s32(outr1);
int32x4_t tmp2 = vabsq_s32(outi0);
int32x4_t tmp3 = vabsq_s32(outi1);
// vabs doesn't change the value of 0x80000000.
// Use u32 so we don't lose the value 0x80000000.
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp0));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp2));
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp1));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp3));
// Store the vectors.
outr1 = vrev64q_s32(outr1);
outi1 = vrev64q_s32(outi1);
int32x4_t outr_1 = vcombine_s32(vget_high_s32(outr1), vget_low_s32(outr1));
int32x4_t outi_1 = vcombine_s32(vget_high_s32(outi1), vget_low_s32(outi1));
vst1q_s32(outre1, outr0);
outre1 += 4;
vst1q_s32(outim1, outi0);
outim1 += 4;
vst1q_s32(outre2, outr_1);
outre2 -= 4;
vst1q_s32(outim2, outi_1);
outim2 -= 4;
}
max_r = vmaxq_u32(max_r, max_i);
#if defined(WEBRTC_ARCH_ARM64)
uint32_t maximum = vmaxvq_u32(max_r);
#else
uint32x2_t max32x2_r = vmax_u32(vget_low_u32(max_r), vget_high_u32(max_r));
max32x2_r = vpmax_u32(max32x2_r, max32x2_r);
uint32_t maximum = vget_lane_u32(max32x2_r, 0);
#endif
return (int32_t)maximum;
}
static inline int32_t ComplexMulAndFindMaxNeon(int16_t* inre1Q9,
int16_t* inre2Q9,
int32_t* outreQ16,
int32_t* outimQ16) {
int k;
const int16_t* kCosTab = &WebRtcIsacfix_kCosTab1[0];
const int16_t* kSinTab = &WebRtcIsacfix_kSinTab1[0];
// 0.5 / sqrt(240) in Q19 is round((.5 / sqrt(240)) * (2^19)) = 16921.
// Use "16921 << 5" and vqdmulh, instead of ">> 26" as in the C code.
int32_t fact = 16921 << 5;
int32x4_t factq = vdupq_n_s32(fact);
uint32x4_t max_r = vdupq_n_u32(0);
uint32x4_t max_i = vdupq_n_u32(0);
for (k = 0; k < FRAMESAMPLES/2; k += 8) {
int16x8_t tmpr = vld1q_s16(kCosTab);
int16x8_t tmpi = vld1q_s16(kSinTab);
int16x8_t inre1 = vld1q_s16(inre1Q9);
int16x8_t inre2 = vld1q_s16(inre2Q9);
kCosTab += 8;
kSinTab += 8;
inre1Q9 += 8;
inre2Q9 += 8;
// Use ">> 26", instead of ">> 7", ">> 16" and then ">> 3" as in the C code.
int32x4_t tmp0 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre1));
int32x4_t tmp1 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre2));
tmp0 = vmlal_s16(tmp0, vget_low_s16(tmpi), vget_low_s16(inre2));
tmp1 = vmlsl_s16(tmp1, vget_low_s16(tmpi), vget_low_s16(inre1));
#if defined(WEBRTC_ARCH_ARM64)
int32x4_t tmp2 = vmull_high_s16(tmpr, inre1);
int32x4_t tmp3 = vmull_high_s16(tmpr, inre2);
tmp2 = vmlal_high_s16(tmp2, tmpi, inre2);
tmp3 = vmlsl_high_s16(tmp3, tmpi, inre1);
#else
int32x4_t tmp2 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre1));
int32x4_t tmp3 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre2));
tmp2 = vmlal_s16(tmp2, vget_high_s16(tmpi), vget_high_s16(inre2));
tmp3 = vmlsl_s16(tmp3, vget_high_s16(tmpi), vget_high_s16(inre1));
#endif
int32x4_t outr_0 = vqdmulhq_s32(tmp0, factq);
int32x4_t outr_1 = vqdmulhq_s32(tmp2, factq);
int32x4_t outi_0 = vqdmulhq_s32(tmp1, factq);
int32x4_t outi_1 = vqdmulhq_s32(tmp3, factq);
vst1q_s32(outreQ16, outr_0);
outreQ16 += 4;
vst1q_s32(outreQ16, outr_1);
outreQ16 += 4;
vst1q_s32(outimQ16, outi_0);
outimQ16 += 4;
vst1q_s32(outimQ16, outi_1);
outimQ16 += 4;
// Find the absolute maximum in the vectors.
tmp0 = vabsq_s32(outr_0);
tmp1 = vabsq_s32(outr_1);
tmp2 = vabsq_s32(outi_0);
tmp3 = vabsq_s32(outi_1);
// vabs doesn't change the value of 0x80000000.
// Use u32 so we don't lose the value 0x80000000.
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp0));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp2));
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp1));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp3));
}
max_r = vmaxq_u32(max_r, max_i);
#if defined(WEBRTC_ARCH_ARM64)
uint32_t maximum = vmaxvq_u32(max_r);
#else
uint32x2_t max32x2_r = vmax_u32(vget_low_u32(max_r), vget_high_u32(max_r));
max32x2_r = vpmax_u32(max32x2_r, max32x2_r);
uint32_t maximum = vget_lane_u32(max32x2_r, 0);
#endif
return (int32_t)maximum;
}
static inline void PostShiftAndSeparateNeon(int16_t* inre,
int16_t* inim,
int16_t* outre,
int16_t* outim,
int32_t sh) {
int k;
int16_t* inre1 = inre;
int16_t* inre2 = &inre[FRAMESAMPLES/2 - 4];
int16_t* inim1 = inim;
int16_t* inim2 = &inim[FRAMESAMPLES/2 - 4];
int16_t* outre1 = outre;
int16_t* outre2 = &outre[FRAMESAMPLES/2 - 4];
int16_t* outim1 = outim;
int16_t* outim2 = &outim[FRAMESAMPLES/2 - 4];
const int16_t* kSinTab1 = &WebRtcIsacfix_kSinTab2[0];
const int16_t* kSinTab2 = &WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 -4];
// By vshl, we effectively did "<< (-sh - 23)", instead of "<< (-sh)",
// ">> 14" and then ">> 9" as in the C code.
int32x4_t shift = vdupq_n_s32(-sh - 23);
for (k = 0; k < FRAMESAMPLES/4; k += 4) {
int16x4_t tmpi = vld1_s16(kSinTab1);
kSinTab1 += 4;
int16x4_t tmpr = vld1_s16(kSinTab2);
kSinTab2 -= 4;
int16x4_t inre_0 = vld1_s16(inre1);
inre1 += 4;
int16x4_t inre_1 = vld1_s16(inre2);
inre2 -= 4;
int16x4_t inim_0 = vld1_s16(inim1);
inim1 += 4;
int16x4_t inim_1 = vld1_s16(inim2);
inim2 -= 4;
tmpr = vneg_s16(tmpr);
inre_1 = vrev64_s16(inre_1);
inim_1 = vrev64_s16(inim_1);
tmpr = vrev64_s16(tmpr);
int16x4_t xr = vqadd_s16(inre_0, inre_1);
int16x4_t xi = vqsub_s16(inim_0, inim_1);
int16x4_t yr = vqadd_s16(inim_0, inim_1);
int16x4_t yi = vqsub_s16(inre_1, inre_0);
int32x4_t outr0 = vmull_s16(tmpr, xr);
int32x4_t outi0 = vmull_s16(tmpi, xr);
int32x4_t outr1 = vmull_s16(tmpi, yr);
int32x4_t outi1 = vmull_s16(tmpi, yi);
outr0 = vmlsl_s16(outr0, tmpi, xi);
outi0 = vmlal_s16(outi0, tmpr, xi);
outr1 = vmlal_s16(outr1, tmpr, yi);
outi1 = vmlsl_s16(outi1, tmpr, yr);
outr0 = vshlq_s32(outr0, shift);
outi0 = vshlq_s32(outi0, shift);
outr1 = vshlq_s32(outr1, shift);
outi1 = vshlq_s32(outi1, shift);
outr1 = vnegq_s32(outr1);
int16x4_t outre_0 = vmovn_s32(outr0);
int16x4_t outim_0 = vmovn_s32(outi0);
int16x4_t outre_1 = vmovn_s32(outr1);
int16x4_t outim_1 = vmovn_s32(outi1);
outre_1 = vrev64_s16(outre_1);
outim_1 = vrev64_s16(outim_1);
vst1_s16(outre1, outre_0);
outre1 += 4;
vst1_s16(outim1, outim_0);
outim1 += 4;
vst1_s16(outre2, outre_1);
outre2 -= 4;
vst1_s16(outim2, outim_1);
outim2 -= 4;
}
}
void vp8_short_fdct4x4_neon(
int16_t *input,
int16_t *output,
int pitch) {
int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16;
int16x4_t d16s16, d17s16, d26s16, dEmptys16;
uint16x4_t d4u16;
int16x8_t q0s16, q1s16;
int32x4_t q9s32, q10s32, q11s32, q12s32;
int16x4x2_t v2tmp0, v2tmp1;
int32x2x2_t v2tmp2, v2tmp3;
d16s16 = vdup_n_s16(5352);
d17s16 = vdup_n_s16(2217);
q9s32 = vdupq_n_s32(14500);
q10s32 = vdupq_n_s32(7500);
q11s32 = vdupq_n_s32(12000);
q12s32 = vdupq_n_s32(51000);
// Part one
pitch >>= 1;
d0s16 = vld1_s16(input);
input += pitch;
d1s16 = vld1_s16(input);
input += pitch;
d2s16 = vld1_s16(input);
input += pitch;
d3s16 = vld1_s16(input);
v2tmp2 = vtrn_s32(vreinterpret_s32_s16(d0s16),
vreinterpret_s32_s16(d2s16));
v2tmp3 = vtrn_s32(vreinterpret_s32_s16(d1s16),
vreinterpret_s32_s16(d3s16));
v2tmp0 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[0]), // d0
vreinterpret_s16_s32(v2tmp3.val[0])); // d1
v2tmp1 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[1]), // d2
vreinterpret_s16_s32(v2tmp3.val[1])); // d3
d4s16 = vadd_s16(v2tmp0.val[0], v2tmp1.val[1]);
d5s16 = vadd_s16(v2tmp0.val[1], v2tmp1.val[0]);
d6s16 = vsub_s16(v2tmp0.val[1], v2tmp1.val[0]);
d7s16 = vsub_s16(v2tmp0.val[0], v2tmp1.val[1]);
d4s16 = vshl_n_s16(d4s16, 3);
d5s16 = vshl_n_s16(d5s16, 3);
d6s16 = vshl_n_s16(d6s16, 3);
d7s16 = vshl_n_s16(d7s16, 3);
d0s16 = vadd_s16(d4s16, d5s16);
d2s16 = vsub_s16(d4s16, d5s16);
q9s32 = vmlal_s16(q9s32, d7s16, d16s16);
q10s32 = vmlal_s16(q10s32, d7s16, d17s16);
q9s32 = vmlal_s16(q9s32, d6s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d6s16, d16s16);
d1s16 = vshrn_n_s32(q9s32, 12);
d3s16 = vshrn_n_s32(q10s32, 12);
// Part two
v2tmp2 = vtrn_s32(vreinterpret_s32_s16(d0s16),
vreinterpret_s32_s16(d2s16));
v2tmp3 = vtrn_s32(vreinterpret_s32_s16(d1s16),
vreinterpret_s32_s16(d3s16));
v2tmp0 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[0]), // d0
vreinterpret_s16_s32(v2tmp3.val[0])); // d1
v2tmp1 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[1]), // d2
vreinterpret_s16_s32(v2tmp3.val[1])); // d3
d4s16 = vadd_s16(v2tmp0.val[0], v2tmp1.val[1]);
d5s16 = vadd_s16(v2tmp0.val[1], v2tmp1.val[0]);
d6s16 = vsub_s16(v2tmp0.val[1], v2tmp1.val[0]);
d7s16 = vsub_s16(v2tmp0.val[0], v2tmp1.val[1]);
d26s16 = vdup_n_s16(7);
d4s16 = vadd_s16(d4s16, d26s16);
d0s16 = vadd_s16(d4s16, d5s16);
d2s16 = vsub_s16(d4s16, d5s16);
q11s32 = vmlal_s16(q11s32, d7s16, d16s16);
q12s32 = vmlal_s16(q12s32, d7s16, d17s16);
dEmptys16 = vdup_n_s16(0);
d4u16 = vceq_s16(d7s16, dEmptys16);
d0s16 = vshr_n_s16(d0s16, 4);
d2s16 = vshr_n_s16(d2s16, 4);
q11s32 = vmlal_s16(q11s32, d6s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d6s16, d16s16);
d4u16 = vmvn_u16(d4u16);
d1s16 = vshrn_n_s32(q11s32, 16);
d1s16 = vsub_s16(d1s16, vreinterpret_s16_u16(d4u16));
d3s16 = vshrn_n_s32(q12s32, 16);
q0s16 = vcombine_s16(d0s16, d1s16);
q1s16 = vcombine_s16(d2s16, d3s16);
vst1q_s16(output, q0s16);
vst1q_s16(output + 8, q1s16);
return;
}
void ne10_img_hresize_4channels_linear_neon (const unsigned char** src, int** dst, int count,
const int* xofs, const short* alpha,
int swidth, int dwidth, int cn, int xmin, int xmax)
{
int dx, k;
int dx0 = 0;
int16x4x2_t alpha_vec;
uint8x8_t dS0_vec, dS1_vec;
int16x8_t qS0_vec, qS1_vec;
int16x4_t dS0_0123, dS0_4567, dS1_0123, dS1_4567;
int32x4_t qT0_vec, qT1_vec;
int16x4_t dCoeff;
dCoeff = vdup_n_s16 (INTER_RESIZE_COEF_SCALE);
for (k = 0; k <= count - 2; k++)
{
const unsigned char *S0 = src[k], *S1 = src[k + 1];
int *D0 = dst[k], *D1 = dst[k + 1];
for (dx = dx0; dx < xmax; dx += 4)
{
int sx = xofs[dx];
alpha_vec = vld2_s16 (&alpha[dx * 2]);
dS0_vec = vld1_u8 (&S0[sx]);
dS1_vec = vld1_u8 (&S1[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
qS1_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS1_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS0_4567 = vget_high_s16 (qS0_vec);
dS1_0123 = vget_low_s16 (qS1_vec);
dS1_4567 = vget_high_s16 (qS1_vec);
qT0_vec = vmull_s16 (dS0_0123, alpha_vec.val[0]);
qT1_vec = vmull_s16 (dS1_0123, alpha_vec.val[0]);
qT0_vec = vmlal_s16 (qT0_vec, dS0_4567, alpha_vec.val[1]);
qT1_vec = vmlal_s16 (qT1_vec, dS1_4567, alpha_vec.val[1]);
vst1q_s32 (&D0[dx], qT0_vec);
vst1q_s32 (&D1[dx], qT1_vec);
}
for (; dx < dwidth; dx += 4)
{
int sx = xofs[dx];
dS0_vec = vld1_u8 (&S0[sx]);
dS1_vec = vld1_u8 (&S1[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
qS1_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS1_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS1_0123 = vget_low_s16 (qS1_vec);
qT0_vec = vmull_s16 (dS0_0123, dCoeff);
qT1_vec = vmull_s16 (dS1_0123, dCoeff);
vst1q_s32 (&D0[dx], qT0_vec);
vst1q_s32 (&D1[dx], qT1_vec);
}
}
for (; k < count; k++)
{
const unsigned char *S = src[k];
int *D = dst[k];
for (dx = 0; dx < xmax; dx += 4)
{
int sx = xofs[dx];
alpha_vec = vld2_s16 (&alpha[dx * 2]);
dS0_vec = vld1_u8 (&S[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
dS0_4567 = vget_high_s16 (qS0_vec);
qT0_vec = vmull_s16 (dS0_0123, alpha_vec.val[0]);
qT0_vec = vmlal_s16 (qT0_vec, dS0_4567, alpha_vec.val[1]);
vst1q_s32 (&D[dx], qT0_vec);
}
for (; dx < dwidth; dx += 4)
{
int sx = xofs[dx];
dS0_vec = vld1_u8 (&S[sx]);
qS0_vec = vreinterpretq_s16_u16 (vmovl_u8 (dS0_vec));
dS0_0123 = vget_low_s16 (qS0_vec);
qT0_vec = vmull_s16 (dS0_0123, dCoeff);
vst1q_s32 (&D[dx], qT0_vec);
}
}
}
void silk_warped_autocorrelation_FIX_neon(
opus_int32 *corr, /* O Result [order + 1] */
opus_int *scale, /* O Scaling of the correlation vector */
const opus_int16 *input, /* I Input data to correlate */
const opus_int warping_Q16, /* I Warping coefficient */
const opus_int length, /* I Length of input */
const opus_int order /* I Correlation order (even) */
)
{
if( ( MAX_SHAPE_LPC_ORDER > 24 ) || ( order < 6 ) ) {
silk_warped_autocorrelation_FIX_c( corr, scale, input, warping_Q16, length, order );
} else {
opus_int n, i, lsh;
opus_int64 corr_QC[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; /* In reverse order */
opus_int64 corr_QC_orderT;
int64x2_t lsh_s64x2;
const opus_int orderT = ( order + 3 ) & ~3;
opus_int64 *corr_QCT;
opus_int32 *input_QS;
VARDECL( opus_int32, input_QST );
VARDECL( opus_int32, state );
SAVE_STACK;
/* Order must be even */
silk_assert( ( order & 1 ) == 0 );
silk_assert( 2 * QS - QC >= 0 );
ALLOC( input_QST, length + 2 * MAX_SHAPE_LPC_ORDER, opus_int32 );
input_QS = input_QST;
/* input_QS has zero paddings in the beginning and end. */
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
/* Loop over samples */
for( n = 0; n < length - 7; n += 8, input_QS += 8 ) {
const int16x8_t t0_s16x4 = vld1q_s16( input + n );
vst1q_s32( input_QS + 0, vshll_n_s16( vget_low_s16( t0_s16x4 ), QS ) );
vst1q_s32( input_QS + 4, vshll_n_s16( vget_high_s16( t0_s16x4 ), QS ) );
}
for( ; n < length; n++, input_QS++ ) {
input_QS[ 0 ] = silk_LSHIFT32( (opus_int32)input[ n ], QS );
}
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS = input_QST + MAX_SHAPE_LPC_ORDER - orderT;
/* The following loop runs ( length + order ) times, with ( order ) extra epilogues. */
/* The zero paddings in input_QS guarantee corr_QC's correctness even with the extra epilogues. */
/* The values of state_QS will be polluted by the extra epilogues, however they are temporary values. */
/* Keep the C code here to help understand the intrinsics optimization. */
/*
{
opus_int32 state_QS[ 2 ][ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 };
opus_int32 *state_QST[ 3 ];
state_QST[ 0 ] = state_QS[ 0 ];
state_QST[ 1 ] = state_QS[ 1 ];
for( n = 0; n < length + order; n++, input_QS++ ) {
state_QST[ 0 ][ orderT ] = input_QS[ orderT ];
for( i = 0; i < orderT; i++ ) {
corr_QC[ i ] += silk_RSHIFT64( silk_SMULL( state_QST[ 0 ][ i ], input_QS[ i ] ), 2 * QS - QC );
state_QST[ 1 ][ i ] = silk_SMLAWB( state_QST[ 1 ][ i + 1 ], state_QST[ 0 ][ i ] - state_QST[ 0 ][ i + 1 ], warping_Q16 );
}
state_QST[ 2 ] = state_QST[ 0 ];
state_QST[ 0 ] = state_QST[ 1 ];
state_QST[ 1 ] = state_QST[ 2 ];
}
}
*/
{
const int32x4_t warping_Q16_s32x4 = vdupq_n_s32( warping_Q16 << 15 );
const opus_int32 *in = input_QS + orderT;
opus_int o = orderT;
int32x4_t state_QS_s32x4[ 3 ][ 2 ];
ALLOC( state, length + orderT, opus_int32 );
state_QS_s32x4[ 2 ][ 1 ] = vdupq_n_s32( 0 );
/* Calculate 8 taps of all inputs in each loop. */
do {
state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 0 ][ 1 ] =
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 1 ][ 1 ] = vdupq_n_s32( 0 );
n = 0;
do {
calc_corr( input_QS + n, corr_QC, o - 8, state_QS_s32x4[ 0 ][ 0 ] );
calc_corr( input_QS + n, corr_QC, o - 4, state_QS_s32x4[ 0 ][ 1 ] );
state_QS_s32x4[ 2 ][ 1 ] = vld1q_s32( in + n );
vst1q_lane_s32( state + n, state_QS_s32x4[ 0 ][ 0 ], 0 );
state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 0 ][ 1 ], 1 );
state_QS_s32x4[ 2 ][ 1 ] = vextq_s32( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], 1 );
state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 );
state_QS_s32x4[ 0 ][ 1 ] = calc_state( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], state_QS_s32x4[ 1 ][ 1 ], warping_Q16_s32x4 );
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ];
state_QS_s32x4[ 1 ][ 1 ] = state_QS_s32x4[ 2 ][ 1 ];
} while( ++n < ( length + order ) );
in = state;
o -= 8;
} while( o > 4 );
if( o ) {
/* Calculate the last 4 taps of all inputs. */
opus_int32 *stateT = state;
silk_assert( o == 4 );
state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 1 ][ 0 ] = vdupq_n_s32( 0 );
n = length + order;
do {
calc_corr( input_QS, corr_QC, 0, state_QS_s32x4[ 0 ][ 0 ] );
state_QS_s32x4[ 2 ][ 0 ] = vld1q_s32( stateT );
vst1q_lane_s32( stateT, state_QS_s32x4[ 0 ][ 0 ], 0 );
state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], 1 );
state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 );
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ];
input_QS++;
stateT++;
} while( --n );
}
}
{
const opus_int16 *inputT = input;
int32x4_t t_s32x4;
int64x1_t t_s64x1;
int64x2_t t_s64x2 = vdupq_n_s64( 0 );
for( n = 0; n <= length - 8; n += 8 ) {
int16x8_t input_s16x8 = vld1q_s16( inputT );
t_s32x4 = vmull_s16( vget_low_s16( input_s16x8 ), vget_low_s16( input_s16x8 ) );
t_s32x4 = vmlal_s16( t_s32x4, vget_high_s16( input_s16x8 ), vget_high_s16( input_s16x8 ) );
t_s64x2 = vaddw_s32( t_s64x2, vget_low_s32( t_s32x4 ) );
t_s64x2 = vaddw_s32( t_s64x2, vget_high_s32( t_s32x4 ) );
inputT += 8;
}
t_s64x1 = vadd_s64( vget_low_s64( t_s64x2 ), vget_high_s64( t_s64x2 ) );
corr_QC_orderT = vget_lane_s64( t_s64x1, 0 );
for( ; n < length; n++ ) {
corr_QC_orderT += silk_SMULL( input[ n ], input[ n ] );
}
corr_QC_orderT = silk_LSHIFT64( corr_QC_orderT, QC );
corr_QC[ orderT ] = corr_QC_orderT;
}
corr_QCT = corr_QC + orderT - order;
lsh = silk_CLZ64( corr_QC_orderT ) - 35;
lsh = silk_LIMIT( lsh, -12 - QC, 30 - QC );
*scale = -( QC + lsh );
silk_assert( *scale >= -30 && *scale <= 12 );
lsh_s64x2 = vdupq_n_s64( lsh );
for( i = 0; i <= order - 3; i += 4 ) {
int32x4_t corr_s32x4;
int64x2_t corr_QC0_s64x2, corr_QC1_s64x2;
corr_QC0_s64x2 = vld1q_s64( corr_QCT + i );
corr_QC1_s64x2 = vld1q_s64( corr_QCT + i + 2 );
corr_QC0_s64x2 = vshlq_s64( corr_QC0_s64x2, lsh_s64x2 );
corr_QC1_s64x2 = vshlq_s64( corr_QC1_s64x2, lsh_s64x2 );
corr_s32x4 = vcombine_s32( vmovn_s64( corr_QC1_s64x2 ), vmovn_s64( corr_QC0_s64x2 ) );
corr_s32x4 = vrev64q_s32( corr_s32x4 );
vst1q_s32( corr + order - i - 3, corr_s32x4 );
}
if( lsh >= 0 ) {
for( ; i < order + 1; i++ ) {
corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_LSHIFT64( corr_QCT[ i ], lsh ) );
}
} else {
for( ; i < order + 1; i++ ) {
corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_RSHIFT64( corr_QCT[ i ], -lsh ) );
}
}
silk_assert( corr_QCT[ order ] >= 0 ); /* If breaking, decrease QC*/
RESTORE_STACK;
}
#ifdef OPUS_CHECK_ASM
{
opus_int32 corr_c[ MAX_SHAPE_LPC_ORDER + 1 ];
opus_int scale_c;
silk_warped_autocorrelation_FIX_c( corr_c, &scale_c, input, warping_Q16, length, order );
silk_assert( !memcmp( corr_c, corr, sizeof( corr_c[ 0 ] ) * ( order + 1 ) ) );
silk_assert( scale_c == *scale );
}
#endif
}
static INLINE void IDCT8x8_1D(
int16x8_t *q8s16,
int16x8_t *q9s16,
int16x8_t *q10s16,
int16x8_t *q11s16,
int16x8_t *q12s16,
int16x8_t *q13s16,
int16x8_t *q14s16,
int16x8_t *q15s16) {
int16x4_t d0s16, d1s16, d2s16, d3s16;
int16x4_t d8s16, d9s16, d10s16, d11s16, d12s16, d13s16, d14s16, d15s16;
int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16, d22s16, d23s16;
int16x4_t d24s16, d25s16, d26s16, d27s16, d28s16, d29s16, d30s16, d31s16;
int16x8_t q0s16, q1s16, q2s16, q3s16, q4s16, q5s16, q6s16, q7s16;
int32x4_t q2s32, q3s32, q5s32, q6s32, q8s32, q9s32;
int32x4_t q10s32, q11s32, q12s32, q13s32, q15s32;
d0s16 = vdup_n_s16(cospi_28_64);
d1s16 = vdup_n_s16(cospi_4_64);
d2s16 = vdup_n_s16(cospi_12_64);
d3s16 = vdup_n_s16(cospi_20_64);
d16s16 = vget_low_s16(*q8s16);
d17s16 = vget_high_s16(*q8s16);
d18s16 = vget_low_s16(*q9s16);
d19s16 = vget_high_s16(*q9s16);
d20s16 = vget_low_s16(*q10s16);
d21s16 = vget_high_s16(*q10s16);
d22s16 = vget_low_s16(*q11s16);
d23s16 = vget_high_s16(*q11s16);
d24s16 = vget_low_s16(*q12s16);
d25s16 = vget_high_s16(*q12s16);
d26s16 = vget_low_s16(*q13s16);
d27s16 = vget_high_s16(*q13s16);
d28s16 = vget_low_s16(*q14s16);
d29s16 = vget_high_s16(*q14s16);
d30s16 = vget_low_s16(*q15s16);
d31s16 = vget_high_s16(*q15s16);
q2s32 = vmull_s16(d18s16, d0s16);
q3s32 = vmull_s16(d19s16, d0s16);
q5s32 = vmull_s16(d26s16, d2s16);
q6s32 = vmull_s16(d27s16, d2s16);
q2s32 = vmlsl_s16(q2s32, d30s16, d1s16);
q3s32 = vmlsl_s16(q3s32, d31s16, d1s16);
q5s32 = vmlsl_s16(q5s32, d22s16, d3s16);
q6s32 = vmlsl_s16(q6s32, d23s16, d3s16);
d8s16 = vqrshrn_n_s32(q2s32, 14);
d9s16 = vqrshrn_n_s32(q3s32, 14);
d10s16 = vqrshrn_n_s32(q5s32, 14);
d11s16 = vqrshrn_n_s32(q6s32, 14);
q4s16 = vcombine_s16(d8s16, d9s16);
q5s16 = vcombine_s16(d10s16, d11s16);
q2s32 = vmull_s16(d18s16, d1s16);
q3s32 = vmull_s16(d19s16, d1s16);
q9s32 = vmull_s16(d26s16, d3s16);
q13s32 = vmull_s16(d27s16, d3s16);
q2s32 = vmlal_s16(q2s32, d30s16, d0s16);
q3s32 = vmlal_s16(q3s32, d31s16, d0s16);
q9s32 = vmlal_s16(q9s32, d22s16, d2s16);
q13s32 = vmlal_s16(q13s32, d23s16, d2s16);
d14s16 = vqrshrn_n_s32(q2s32, 14);
d15s16 = vqrshrn_n_s32(q3s32, 14);
d12s16 = vqrshrn_n_s32(q9s32, 14);
d13s16 = vqrshrn_n_s32(q13s32, 14);
q6s16 = vcombine_s16(d12s16, d13s16);
q7s16 = vcombine_s16(d14s16, d15s16);
d0s16 = vdup_n_s16(cospi_16_64);
q2s32 = vmull_s16(d16s16, d0s16);
q3s32 = vmull_s16(d17s16, d0s16);
q13s32 = vmull_s16(d16s16, d0s16);
q15s32 = vmull_s16(d17s16, d0s16);
q2s32 = vmlal_s16(q2s32, d24s16, d0s16);
q3s32 = vmlal_s16(q3s32, d25s16, d0s16);
q13s32 = vmlsl_s16(q13s32, d24s16, d0s16);
q15s32 = vmlsl_s16(q15s32, d25s16, d0s16);
d0s16 = vdup_n_s16(cospi_24_64);
d1s16 = vdup_n_s16(cospi_8_64);
d18s16 = vqrshrn_n_s32(q2s32, 14);
d19s16 = vqrshrn_n_s32(q3s32, 14);
d22s16 = vqrshrn_n_s32(q13s32, 14);
d23s16 = vqrshrn_n_s32(q15s32, 14);
*q9s16 = vcombine_s16(d18s16, d19s16);
*q11s16 = vcombine_s16(d22s16, d23s16);
q2s32 = vmull_s16(d20s16, d0s16);
q3s32 = vmull_s16(d21s16, d0s16);
q8s32 = vmull_s16(d20s16, d1s16);
q12s32 = vmull_s16(d21s16, d1s16);
q2s32 = vmlsl_s16(q2s32, d28s16, d1s16);
q3s32 = vmlsl_s16(q3s32, d29s16, d1s16);
q8s32 = vmlal_s16(q8s32, d28s16, d0s16);
q12s32 = vmlal_s16(q12s32, d29s16, d0s16);
d26s16 = vqrshrn_n_s32(q2s32, 14);
d27s16 = vqrshrn_n_s32(q3s32, 14);
d30s16 = vqrshrn_n_s32(q8s32, 14);
d31s16 = vqrshrn_n_s32(q12s32, 14);
*q13s16 = vcombine_s16(d26s16, d27s16);
*q15s16 = vcombine_s16(d30s16, d31s16);
q0s16 = vaddq_s16(*q9s16, *q15s16);
q1s16 = vaddq_s16(*q11s16, *q13s16);
q2s16 = vsubq_s16(*q11s16, *q13s16);
q3s16 = vsubq_s16(*q9s16, *q15s16);
*q13s16 = vsubq_s16(q4s16, q5s16);
q4s16 = vaddq_s16(q4s16, q5s16);
*q14s16 = vsubq_s16(q7s16, q6s16);
q7s16 = vaddq_s16(q7s16, q6s16);
d26s16 = vget_low_s16(*q13s16);
d27s16 = vget_high_s16(*q13s16);
d28s16 = vget_low_s16(*q14s16);
d29s16 = vget_high_s16(*q14s16);
d16s16 = vdup_n_s16(cospi_16_64);
q9s32 = vmull_s16(d28s16, d16s16);
q10s32 = vmull_s16(d29s16, d16s16);
q11s32 = vmull_s16(d28s16, d16s16);
q12s32 = vmull_s16(d29s16, d16s16);
q9s32 = vmlsl_s16(q9s32, d26s16, d16s16);
q10s32 = vmlsl_s16(q10s32, d27s16, d16s16);
q11s32 = vmlal_s16(q11s32, d26s16, d16s16);
q12s32 = vmlal_s16(q12s32, d27s16, d16s16);
d10s16 = vqrshrn_n_s32(q9s32, 14);
d11s16 = vqrshrn_n_s32(q10s32, 14);
d12s16 = vqrshrn_n_s32(q11s32, 14);
d13s16 = vqrshrn_n_s32(q12s32, 14);
q5s16 = vcombine_s16(d10s16, d11s16);
q6s16 = vcombine_s16(d12s16, d13s16);
*q8s16 = vaddq_s16(q0s16, q7s16);
*q9s16 = vaddq_s16(q1s16, q6s16);
*q10s16 = vaddq_s16(q2s16, q5s16);
*q11s16 = vaddq_s16(q3s16, q4s16);
*q12s16 = vsubq_s16(q3s16, q4s16);
*q13s16 = vsubq_s16(q2s16, q5s16);
*q14s16 = vsubq_s16(q1s16, q6s16);
*q15s16 = vsubq_s16(q0s16, q7s16);
return;
}
unsigned int vp8_sub_pixel_variance16x16_neon_func(
const unsigned char *src_ptr,
int src_pixels_per_line,
int xoffset,
int yoffset,
const unsigned char *dst_ptr,
int dst_pixels_per_line,
unsigned int *sse) {
int i;
DECLARE_ALIGNED_ARRAY(16, unsigned char, tmp, 528);
unsigned char *tmpp;
unsigned char *tmpp2;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
uint8x8_t d10u8, d11u8, d12u8, d13u8, d14u8, d15u8, d16u8, d17u8, d18u8;
uint8x8_t d19u8, d20u8, d21u8;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64, d2s64, d3s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8, q4u8, q5u8, q6u8, q7u8, q8u8, q9u8;
uint8x16_t q10u8, q11u8, q12u8, q13u8, q14u8, q15u8;
uint16x8_t q1u16, q2u16, q3u16, q4u16, q5u16, q6u16, q7u16, q8u16;
uint16x8_t q9u16, q10u16, q11u16, q12u16, q13u16, q14u16;
int32x4_t q8s32, q9s32, q10s32;
int64x2_t q0s64, q1s64, q5s64;
tmpp2 = tmp + 272;
tmpp = tmp;
if (xoffset == 0) { // secondpass_bfilter16x16_only
d0u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][1]);
q11u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
for (i = 4; i > 0; i--) {
q12u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q13u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q14u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q15u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
__builtin_prefetch(src_ptr);
__builtin_prefetch(src_ptr + src_pixels_per_line);
__builtin_prefetch(src_ptr + src_pixels_per_line * 2);
q1u16 = vmull_u8(vget_low_u8(q11u8), d0u8);
q2u16 = vmull_u8(vget_high_u8(q11u8), d0u8);
q3u16 = vmull_u8(vget_low_u8(q12u8), d0u8);
q4u16 = vmull_u8(vget_high_u8(q12u8), d0u8);
q5u16 = vmull_u8(vget_low_u8(q13u8), d0u8);
q6u16 = vmull_u8(vget_high_u8(q13u8), d0u8);
q7u16 = vmull_u8(vget_low_u8(q14u8), d0u8);
q8u16 = vmull_u8(vget_high_u8(q14u8), d0u8);
q1u16 = vmlal_u8(q1u16, vget_low_u8(q12u8), d1u8);
q2u16 = vmlal_u8(q2u16, vget_high_u8(q12u8), d1u8);
q3u16 = vmlal_u8(q3u16, vget_low_u8(q13u8), d1u8);
q4u16 = vmlal_u8(q4u16, vget_high_u8(q13u8), d1u8);
q5u16 = vmlal_u8(q5u16, vget_low_u8(q14u8), d1u8);
q6u16 = vmlal_u8(q6u16, vget_high_u8(q14u8), d1u8);
q7u16 = vmlal_u8(q7u16, vget_low_u8(q15u8), d1u8);
q8u16 = vmlal_u8(q8u16, vget_high_u8(q15u8), d1u8);
d2u8 = vqrshrn_n_u16(q1u16, 7);
d3u8 = vqrshrn_n_u16(q2u16, 7);
d4u8 = vqrshrn_n_u16(q3u16, 7);
d5u8 = vqrshrn_n_u16(q4u16, 7);
d6u8 = vqrshrn_n_u16(q5u16, 7);
d7u8 = vqrshrn_n_u16(q6u16, 7);
d8u8 = vqrshrn_n_u16(q7u16, 7);
d9u8 = vqrshrn_n_u16(q8u16, 7);
q1u8 = vcombine_u8(d2u8, d3u8);
q2u8 = vcombine_u8(d4u8, d5u8);
q3u8 = vcombine_u8(d6u8, d7u8);
q4u8 = vcombine_u8(d8u8, d9u8);
q11u8 = q15u8;
vst1q_u8((uint8_t *)tmpp2, q1u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q2u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q3u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q4u8);
tmpp2 += 16;
}
} else if (yoffset == 0) { // firstpass_bfilter16x16_only
d0u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][1]);
for (i = 4; i > 0 ; i--) {
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
__builtin_prefetch(src_ptr);
__builtin_prefetch(src_ptr + src_pixels_per_line);
__builtin_prefetch(src_ptr + src_pixels_per_line * 2);
q7u16 = vmull_u8(d2u8, d0u8);
q8u16 = vmull_u8(d3u8, d0u8);
q9u16 = vmull_u8(d5u8, d0u8);
q10u16 = vmull_u8(d6u8, d0u8);
q11u16 = vmull_u8(d8u8, d0u8);
q12u16 = vmull_u8(d9u8, d0u8);
q13u16 = vmull_u8(d11u8, d0u8);
q14u16 = vmull_u8(d12u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
d11u8 = vext_u8(d11u8, d12u8, 1);
q7u16 = vmlal_u8(q7u16, d2u8, d1u8);
q9u16 = vmlal_u8(q9u16, d5u8, d1u8);
q11u16 = vmlal_u8(q11u16, d8u8, d1u8);
q13u16 = vmlal_u8(q13u16, d11u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
d12u8 = vext_u8(d12u8, d13u8, 1);
q8u16 = vmlal_u8(q8u16, d3u8, d1u8);
q10u16 = vmlal_u8(q10u16, d6u8, d1u8);
q12u16 = vmlal_u8(q12u16, d9u8, d1u8);
q14u16 = vmlal_u8(q14u16, d12u8, d1u8);
d14u8 = vqrshrn_n_u16(q7u16, 7);
d15u8 = vqrshrn_n_u16(q8u16, 7);
d16u8 = vqrshrn_n_u16(q9u16, 7);
d17u8 = vqrshrn_n_u16(q10u16, 7);
d18u8 = vqrshrn_n_u16(q11u16, 7);
d19u8 = vqrshrn_n_u16(q12u16, 7);
d20u8 = vqrshrn_n_u16(q13u16, 7);
d21u8 = vqrshrn_n_u16(q14u16, 7);
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
q10u8 = vcombine_u8(d20u8, d21u8);
vst1q_u8((uint8_t *)tmpp2, q7u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q8u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q9u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q10u8);
tmpp2 += 16;
}
} else {
d0u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][1]);
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
// First Pass: output_height lines x output_width columns (17x16)
for (i = 3; i > 0; i--) {
q7u16 = vmull_u8(d2u8, d0u8);
q8u16 = vmull_u8(d3u8, d0u8);
q9u16 = vmull_u8(d5u8, d0u8);
q10u16 = vmull_u8(d6u8, d0u8);
q11u16 = vmull_u8(d8u8, d0u8);
q12u16 = vmull_u8(d9u8, d0u8);
q13u16 = vmull_u8(d11u8, d0u8);
q14u16 = vmull_u8(d12u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
d11u8 = vext_u8(d11u8, d12u8, 1);
q7u16 = vmlal_u8(q7u16, d2u8, d1u8);
q9u16 = vmlal_u8(q9u16, d5u8, d1u8);
q11u16 = vmlal_u8(q11u16, d8u8, d1u8);
q13u16 = vmlal_u8(q13u16, d11u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
d12u8 = vext_u8(d12u8, d13u8, 1);
q8u16 = vmlal_u8(q8u16, d3u8, d1u8);
q10u16 = vmlal_u8(q10u16, d6u8, d1u8);
q12u16 = vmlal_u8(q12u16, d9u8, d1u8);
q14u16 = vmlal_u8(q14u16, d12u8, d1u8);
d14u8 = vqrshrn_n_u16(q7u16, 7);
d15u8 = vqrshrn_n_u16(q8u16, 7);
d16u8 = vqrshrn_n_u16(q9u16, 7);
d17u8 = vqrshrn_n_u16(q10u16, 7);
d18u8 = vqrshrn_n_u16(q11u16, 7);
d19u8 = vqrshrn_n_u16(q12u16, 7);
d20u8 = vqrshrn_n_u16(q13u16, 7);
d21u8 = vqrshrn_n_u16(q14u16, 7);
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
q10u8 = vcombine_u8(d20u8, d21u8);
vst1q_u8((uint8_t *)tmpp, q7u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q8u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q9u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q10u8);
tmpp += 16;
}
// First-pass filtering for rest 5 lines
d14u8 = vld1_u8(src_ptr);
d15u8 = vld1_u8(src_ptr + 8);
d16u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
q9u16 = vmull_u8(d2u8, d0u8);
q10u16 = vmull_u8(d3u8, d0u8);
q11u16 = vmull_u8(d5u8, d0u8);
q12u16 = vmull_u8(d6u8, d0u8);
q13u16 = vmull_u8(d8u8, d0u8);
q14u16 = vmull_u8(d9u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
q9u16 = vmlal_u8(q9u16, d2u8, d1u8);
q11u16 = vmlal_u8(q11u16, d5u8, d1u8);
q13u16 = vmlal_u8(q13u16, d8u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
q10u16 = vmlal_u8(q10u16, d3u8, d1u8);
q12u16 = vmlal_u8(q12u16, d6u8, d1u8);
q14u16 = vmlal_u8(q14u16, d9u8, d1u8);
q1u16 = vmull_u8(d11u8, d0u8);
q2u16 = vmull_u8(d12u8, d0u8);
q3u16 = vmull_u8(d14u8, d0u8);
q4u16 = vmull_u8(d15u8, d0u8);
d11u8 = vext_u8(d11u8, d12u8, 1);
d14u8 = vext_u8(d14u8, d15u8, 1);
q1u16 = vmlal_u8(q1u16, d11u8, d1u8);
q3u16 = vmlal_u8(q3u16, d14u8, d1u8);
d12u8 = vext_u8(d12u8, d13u8, 1);
d15u8 = vext_u8(d15u8, d16u8, 1);
q2u16 = vmlal_u8(q2u16, d12u8, d1u8);
q4u16 = vmlal_u8(q4u16, d15u8, d1u8);
d10u8 = vqrshrn_n_u16(q9u16, 7);
d11u8 = vqrshrn_n_u16(q10u16, 7);
d12u8 = vqrshrn_n_u16(q11u16, 7);
d13u8 = vqrshrn_n_u16(q12u16, 7);
d14u8 = vqrshrn_n_u16(q13u16, 7);
d15u8 = vqrshrn_n_u16(q14u16, 7);
d16u8 = vqrshrn_n_u16(q1u16, 7);
d17u8 = vqrshrn_n_u16(q2u16, 7);
d18u8 = vqrshrn_n_u16(q3u16, 7);
d19u8 = vqrshrn_n_u16(q4u16, 7);
q5u8 = vcombine_u8(d10u8, d11u8);
q6u8 = vcombine_u8(d12u8, d13u8);
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
vst1q_u8((uint8_t *)tmpp, q5u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q6u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q7u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q8u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q9u8);
// secondpass_filter
d0u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][1]);
tmpp = tmp;
tmpp2 = tmpp + 272;
q11u8 = vld1q_u8(tmpp);
tmpp += 16;
for (i = 4; i > 0; i--) {
q12u8 = vld1q_u8(tmpp);
tmpp += 16;
q13u8 = vld1q_u8(tmpp);
tmpp += 16;
q14u8 = vld1q_u8(tmpp);
tmpp += 16;
q15u8 = vld1q_u8(tmpp);
tmpp += 16;
q1u16 = vmull_u8(vget_low_u8(q11u8), d0u8);
q2u16 = vmull_u8(vget_high_u8(q11u8), d0u8);
q3u16 = vmull_u8(vget_low_u8(q12u8), d0u8);
q4u16 = vmull_u8(vget_high_u8(q12u8), d0u8);
q5u16 = vmull_u8(vget_low_u8(q13u8), d0u8);
q6u16 = vmull_u8(vget_high_u8(q13u8), d0u8);
q7u16 = vmull_u8(vget_low_u8(q14u8), d0u8);
q8u16 = vmull_u8(vget_high_u8(q14u8), d0u8);
q1u16 = vmlal_u8(q1u16, vget_low_u8(q12u8), d1u8);
q2u16 = vmlal_u8(q2u16, vget_high_u8(q12u8), d1u8);
q3u16 = vmlal_u8(q3u16, vget_low_u8(q13u8), d1u8);
q4u16 = vmlal_u8(q4u16, vget_high_u8(q13u8), d1u8);
q5u16 = vmlal_u8(q5u16, vget_low_u8(q14u8), d1u8);
q6u16 = vmlal_u8(q6u16, vget_high_u8(q14u8), d1u8);
q7u16 = vmlal_u8(q7u16, vget_low_u8(q15u8), d1u8);
q8u16 = vmlal_u8(q8u16, vget_high_u8(q15u8), d1u8);
d2u8 = vqrshrn_n_u16(q1u16, 7);
d3u8 = vqrshrn_n_u16(q2u16, 7);
d4u8 = vqrshrn_n_u16(q3u16, 7);
d5u8 = vqrshrn_n_u16(q4u16, 7);
d6u8 = vqrshrn_n_u16(q5u16, 7);
d7u8 = vqrshrn_n_u16(q6u16, 7);
d8u8 = vqrshrn_n_u16(q7u16, 7);
d9u8 = vqrshrn_n_u16(q8u16, 7);
q1u8 = vcombine_u8(d2u8, d3u8);
q2u8 = vcombine_u8(d4u8, d5u8);
q3u8 = vcombine_u8(d6u8, d7u8);
q4u8 = vcombine_u8(d8u8, d9u8);
q11u8 = q15u8;
vst1q_u8((uint8_t *)tmpp2, q1u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q2u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q3u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q4u8);
tmpp2 += 16;
}
}
// sub_pixel_variance16x16_neon
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
tmpp = tmp + 272;
for (i = 0; i < 8; i++) { // sub_pixel_variance16x16_neon_loop
q0u8 = vld1q_u8(tmpp);
tmpp += 16;
q1u8 = vld1q_u8(tmpp);
tmpp += 16;
q2u8 = vld1q_u8(dst_ptr);
dst_ptr += dst_pixels_per_line;
q3u8 = vld1q_u8(dst_ptr);
dst_ptr += dst_pixels_per_line;
d0u8 = vget_low_u8(q0u8);
d1u8 = vget_high_u8(q0u8);
d2u8 = vget_low_u8(q1u8);
d3u8 = vget_high_u8(q1u8);
q11u16 = vsubl_u8(d0u8, vget_low_u8(q2u8));
q12u16 = vsubl_u8(d1u8, vget_high_u8(q2u8));
q13u16 = vsubl_u8(d2u8, vget_low_u8(q3u8));
q14u16 = vsubl_u8(d3u8, vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q10s32 = vaddq_s32(q10s32, q9s32);
q0s64 = vpaddlq_s32(q8s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vget_low_s64(q0s64);
d1s64 = vget_high_s64(q0s64);
d2s64 = vget_low_s64(q1s64);
d3s64 = vget_high_s64(q1s64);
d0s64 = vadd_s64(d0s64, d1s64);
d1s64 = vadd_s64(d2s64, d3s64);
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 8);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
