int32_t dot_product(int16_t *x,
int16_t *y,
uint32_t N, //must be a multiple of 8
uint8_t output_shift)
{
uint32_t n;
#if defined(__x86_64__) || defined(__i386__)
__m128i *x128,*y128,mmtmp1,mmtmp2,mmtmp3,mmcumul,mmcumul_re,mmcumul_im;
__m64 mmtmp7;
__m128i minus_i = _mm_set_epi16(-1,1,-1,1,-1,1,-1,1);
int32_t result;
x128 = (__m128i*) x;
y128 = (__m128i*) y;
mmcumul_re = _mm_setzero_si128();
mmcumul_im = _mm_setzero_si128();
for (n=0; n<(N>>2); n++) {
//printf("n=%d, x128=%p, y128=%p\n",n,x128,y128);
// print_shorts("x",&x128[0]);
// print_shorts("y",&y128[0]);
// this computes Re(z) = Re(x)*Re(y) + Im(x)*Im(y)
mmtmp1 = _mm_madd_epi16(x128[0],y128[0]);
// print_ints("re",&mmtmp1);
// mmtmp1 contains real part of 4 consecutive outputs (32-bit)
// shift and accumulate results
mmtmp1 = _mm_srai_epi32(mmtmp1,output_shift);
mmcumul_re = _mm_add_epi32(mmcumul_re,mmtmp1);
// print_ints("re",&mmcumul_re);
// this computes Im(z) = Re(x)*Im(y) - Re(y)*Im(x)
mmtmp2 = _mm_shufflelo_epi16(y128[0],_MM_SHUFFLE(2,3,0,1));
// print_shorts("y",&mmtmp2);
mmtmp2 = _mm_shufflehi_epi16(mmtmp2,_MM_SHUFFLE(2,3,0,1));
// print_shorts("y",&mmtmp2);
mmtmp2 = _mm_sign_epi16(mmtmp2,minus_i);
// print_shorts("y",&mmtmp2);
mmtmp3 = _mm_madd_epi16(x128[0],mmtmp2);
// print_ints("im",&mmtmp3);
// mmtmp3 contains imag part of 4 consecutive outputs (32-bit)
// shift and accumulate results
mmtmp3 = _mm_srai_epi32(mmtmp3,output_shift);
mmcumul_im = _mm_add_epi32(mmcumul_im,mmtmp3);
// print_ints("im",&mmcumul_im);
x128++;
y128++;
}
// this gives Re Re Im Im
mmcumul = _mm_hadd_epi32(mmcumul_re,mmcumul_im);
// print_ints("cumul1",&mmcumul);
// this gives Re Im Re Im
mmcumul = _mm_hadd_epi32(mmcumul,mmcumul);
// print_ints("cumul2",&mmcumul);
//mmcumul = _mm_srai_epi32(mmcumul,output_shift);
// extract the lower half
mmtmp7 = _mm_movepi64_pi64(mmcumul);
// print_ints("mmtmp7",&mmtmp7);
// pack the result
mmtmp7 = _mm_packs_pi32(mmtmp7,mmtmp7);
// print_shorts("mmtmp7",&mmtmp7);
// convert back to integer
result = _mm_cvtsi64_si32(mmtmp7);
_mm_empty();
_m_empty();
return(result);
#elif defined(__arm__)
int16x4_t *x_128=(int16x4_t*)x;
int16x4_t *y_128=(int16x4_t*)y;
int32x4_t tmp_re,tmp_im;
int32x4_t tmp_re1,tmp_im1;
int32x4_t re_cumul,im_cumul;
int32x2_t re_cumul2,im_cumul2;
int32x4_t shift = vdupq_n_s32(-output_shift);
int32x2x2_t result2;
int16_t conjug[4]__attribute__((aligned(16))) = {-1,1,-1,1} ;
re_cumul = vdupq_n_s32(0);
im_cumul = vdupq_n_s32(0);
for (n=0; n<(N>>2); n++) {
tmp_re = vmull_s16(*x_128++, *y_128++);
//tmp_re = [Re(x[0])Re(y[0]) Im(x[0])Im(y[0]) Re(x[1])Re(y[1]) Im(x[1])Im(y[1])]
tmp_re1 = vmull_s16(*x_128++, *y_128++);
//tmp_re1 = [Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1]) Re(x1[1])Re(x2[2]) Im(x1[1])Im(x2[2])]
tmp_re = vcombine_s32(vpadd_s32(vget_low_s32(tmp_re),vget_high_s32(tmp_re)),
vpadd_s32(vget_low_s32(tmp_re1),vget_high_s32(tmp_re1)));
//tmp_re = [Re(ch[0])Re(rx[0])+Im(ch[0])Im(ch[0]) Re(ch[1])Re(rx[1])+Im(ch[1])Im(ch[1]) Re(ch[2])Re(rx[2])+Im(ch[2]) Im(ch[2]) Re(ch[3])Re(rx[3])+Im(ch[3])Im(ch[3])]
tmp_im = vmull_s16(vrev32_s16(vmul_s16(*x_128++,*(int16x4_t*)conjug)),*y_128++);
//tmp_im = [-Im(ch[0])Re(rx[0]) Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1]) Re(ch[1])Im(rx[1])]
tmp_im1 = vmull_s16(vrev32_s16(vmul_s16(*x_128++,*(int16x4_t*)conjug)),*y_128++);
//tmp_im1 = [-Im(ch[2])Re(rx[2]) Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3]) Re(ch[3])Im(rx[3])]
tmp_im = vcombine_s32(vpadd_s32(vget_low_s32(tmp_im),vget_high_s32(tmp_im)),
vpadd_s32(vget_low_s32(tmp_im1),vget_high_s32(tmp_im1)));
//tmp_im = [-Im(ch[0])Re(rx[0])+Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1])+Re(ch[1])Im(rx[1]) -Im(ch[2])Re(rx[2])+Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3])+Re(ch[3])Im(rx[3])]
re_cumul = vqaddq_s32(re_cumul,vqshlq_s32(tmp_re,shift));
im_cumul = vqaddq_s32(im_cumul,vqshlq_s32(tmp_im,shift));
}
re_cumul2 = vpadd_s32(vget_low_s32(re_cumul),vget_high_s32(re_cumul));
im_cumul2 = vpadd_s32(vget_low_s32(im_cumul),vget_high_s32(im_cumul));
re_cumul2 = vpadd_s32(re_cumul2,re_cumul2);
im_cumul2 = vpadd_s32(im_cumul2,im_cumul2);
result2 = vzip_s32(re_cumul2,im_cumul2);
return(vget_lane_s32(result2.val[0],0));
#endif
}
unsigned int vp8_variance_halfpixvar16x16_hv_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
int16x4_t d0s16, d1s16, d2s16, d3s16, d10s16, d11s16, d12s16, d13s16;
int16x4_t d18s16, d19s16, d20s16, d21s16, d22s16, d23s16, d24s16, d25s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64, d2s64, d3s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8, q4u8, q5u8, q6u8, q7u8, q8u8, q9u8;
uint16x8_t q0u16, q1u16, q5u16, q6u16, q9u16, q10u16, q11u16, q12u16;
int32x4_t q13s32, q14s32, q15s32;
int64x2_t q0s64, q1s64, q5s64;
q13s32 = vdupq_n_s32(0);
q14s32 = vdupq_n_s32(0);
q15s32 = vdupq_n_s32(0);
q0u8 = vld1q_u8(src_ptr);
q1u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q1u8 = vextq_u8(q0u8, q1u8, 1);
q0u8 = vrhaddq_u8(q0u8, q1u8);
for (i = 0; i < 4; i++) { // vp8_filt_fpo16x16s_4_0_loop_neon
q2u8 = vld1q_u8(src_ptr);
q3u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q4u8 = vld1q_u8(src_ptr);
q5u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q6u8 = vld1q_u8(src_ptr);
q7u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q8u8 = vld1q_u8(src_ptr);
q9u8 = vld1q_u8(src_ptr + 16);
src_ptr += source_stride;
q3u8 = vextq_u8(q2u8, q3u8, 1);
q5u8 = vextq_u8(q4u8, q5u8, 1);
q7u8 = vextq_u8(q6u8, q7u8, 1);
q9u8 = vextq_u8(q8u8, q9u8, 1);
q1u8 = vrhaddq_u8(q2u8, q3u8);
q2u8 = vrhaddq_u8(q4u8, q5u8);
q3u8 = vrhaddq_u8(q6u8, q7u8);
q4u8 = vrhaddq_u8(q8u8, q9u8);
q0u8 = vrhaddq_u8(q0u8, q1u8);
q1u8 = vrhaddq_u8(q1u8, q2u8);
q2u8 = vrhaddq_u8(q2u8, q3u8);
q3u8 = vrhaddq_u8(q3u8, q4u8);
q5u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q6u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q7u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q8u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
d0u8 = vget_low_u8(q0u8);
d1u8 = vget_high_u8(q0u8);
d2u8 = vget_low_u8(q1u8);
d3u8 = vget_high_u8(q1u8);
d4u8 = vget_low_u8(q2u8);
d5u8 = vget_high_u8(q2u8);
d6u8 = vget_low_u8(q3u8);
d7u8 = vget_high_u8(q3u8);
q9u16 = vsubl_u8(d0u8, vget_low_u8(q5u8));
q10u16 = vsubl_u8(d1u8, vget_high_u8(q5u8));
q11u16 = vsubl_u8(d2u8, vget_low_u8(q6u8));
q12u16 = vsubl_u8(d3u8, vget_high_u8(q6u8));
q0u16 = vsubl_u8(d4u8, vget_low_u8(q7u8));
q1u16 = vsubl_u8(d5u8, vget_high_u8(q7u8));
q5u16 = vsubl_u8(d6u8, vget_low_u8(q8u8));
q6u16 = vsubl_u8(d7u8, vget_high_u8(q8u8));
d18s16 = vreinterpret_s16_u16(vget_low_u16(q9u16));
d19s16 = vreinterpret_s16_u16(vget_high_u16(q9u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q9u16));
q14s32 = vmlal_s16(q14s32, d18s16, d18s16);
q15s32 = vmlal_s16(q15s32, d19s16, d19s16);
d20s16 = vreinterpret_s16_u16(vget_low_u16(q10u16));
d21s16 = vreinterpret_s16_u16(vget_high_u16(q10u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q10u16));
q14s32 = vmlal_s16(q14s32, d20s16, d20s16);
q15s32 = vmlal_s16(q15s32, d21s16, d21s16);
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q11u16));
q14s32 = vmlal_s16(q14s32, d22s16, d22s16);
q15s32 = vmlal_s16(q15s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q12u16));
q14s32 = vmlal_s16(q14s32, d24s16, d24s16);
q15s32 = vmlal_s16(q15s32, d25s16, d25s16);
d0s16 = vreinterpret_s16_u16(vget_low_u16(q0u16));
d1s16 = vreinterpret_s16_u16(vget_high_u16(q0u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q0u16));
q14s32 = vmlal_s16(q14s32, d0s16, d0s16);
q15s32 = vmlal_s16(q15s32, d1s16, d1s16);
d2s16 = vreinterpret_s16_u16(vget_low_u16(q1u16));
d3s16 = vreinterpret_s16_u16(vget_high_u16(q1u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q1u16));
q14s32 = vmlal_s16(q14s32, d2s16, d2s16);
q15s32 = vmlal_s16(q15s32, d3s16, d3s16);
d10s16 = vreinterpret_s16_u16(vget_low_u16(q5u16));
d11s16 = vreinterpret_s16_u16(vget_high_u16(q5u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q5u16));
q14s32 = vmlal_s16(q14s32, d10s16, d10s16);
q15s32 = vmlal_s16(q15s32, d11s16, d11s16);
d12s16 = vreinterpret_s16_u16(vget_low_u16(q6u16));
d13s16 = vreinterpret_s16_u16(vget_high_u16(q6u16));
q13s32 = vpadalq_s16(q13s32, vreinterpretq_s16_u16(q6u16));
q14s32 = vmlal_s16(q14s32, d12s16, d12s16);
q15s32 = vmlal_s16(q15s32, d13s16, d13s16);
q0u8 = q4u8;
}
q15s32 = vaddq_s32(q14s32, q15s32);
q0s64 = vpaddlq_s32(q13s32);
q1s64 = vpaddlq_s32(q15s32);
d0s64 = vget_low_s64(q0s64);
d1s64 = vget_high_s64(q0s64);
d2s64 = vget_low_s64(q1s64);
d3s64 = vget_high_s64(q1s64);
d0s64 = vadd_s64(d0s64, d1s64);
d1s64 = vadd_s64(d2s64, d3s64);
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 8);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
unsigned int vp8_sub_pixel_variance16x16_neon_func(
const unsigned char *src_ptr,
int src_pixels_per_line,
int xoffset,
int yoffset,
const unsigned char *dst_ptr,
int dst_pixels_per_line,
unsigned int *sse) {
int i;
DECLARE_ALIGNED_ARRAY(16, unsigned char, tmp, 528);
unsigned char *tmpp;
unsigned char *tmpp2;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
uint8x8_t d10u8, d11u8, d12u8, d13u8, d14u8, d15u8, d16u8, d17u8, d18u8;
uint8x8_t d19u8, d20u8, d21u8;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64, d2s64, d3s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8, q4u8, q5u8, q6u8, q7u8, q8u8, q9u8;
uint8x16_t q10u8, q11u8, q12u8, q13u8, q14u8, q15u8;
uint16x8_t q1u16, q2u16, q3u16, q4u16, q5u16, q6u16, q7u16, q8u16;
uint16x8_t q9u16, q10u16, q11u16, q12u16, q13u16, q14u16;
int32x4_t q8s32, q9s32, q10s32;
int64x2_t q0s64, q1s64, q5s64;
tmpp2 = tmp + 272;
tmpp = tmp;
if (xoffset == 0) { // secondpass_bfilter16x16_only
d0u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][1]);
q11u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
for (i = 4; i > 0; i--) {
q12u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q13u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q14u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
q15u8 = vld1q_u8(src_ptr);
src_ptr += src_pixels_per_line;
__builtin_prefetch(src_ptr);
__builtin_prefetch(src_ptr + src_pixels_per_line);
__builtin_prefetch(src_ptr + src_pixels_per_line * 2);
q1u16 = vmull_u8(vget_low_u8(q11u8), d0u8);
q2u16 = vmull_u8(vget_high_u8(q11u8), d0u8);
q3u16 = vmull_u8(vget_low_u8(q12u8), d0u8);
q4u16 = vmull_u8(vget_high_u8(q12u8), d0u8);
q5u16 = vmull_u8(vget_low_u8(q13u8), d0u8);
q6u16 = vmull_u8(vget_high_u8(q13u8), d0u8);
q7u16 = vmull_u8(vget_low_u8(q14u8), d0u8);
q8u16 = vmull_u8(vget_high_u8(q14u8), d0u8);
q1u16 = vmlal_u8(q1u16, vget_low_u8(q12u8), d1u8);
q2u16 = vmlal_u8(q2u16, vget_high_u8(q12u8), d1u8);
q3u16 = vmlal_u8(q3u16, vget_low_u8(q13u8), d1u8);
q4u16 = vmlal_u8(q4u16, vget_high_u8(q13u8), d1u8);
q5u16 = vmlal_u8(q5u16, vget_low_u8(q14u8), d1u8);
q6u16 = vmlal_u8(q6u16, vget_high_u8(q14u8), d1u8);
q7u16 = vmlal_u8(q7u16, vget_low_u8(q15u8), d1u8);
q8u16 = vmlal_u8(q8u16, vget_high_u8(q15u8), d1u8);
d2u8 = vqrshrn_n_u16(q1u16, 7);
d3u8 = vqrshrn_n_u16(q2u16, 7);
d4u8 = vqrshrn_n_u16(q3u16, 7);
d5u8 = vqrshrn_n_u16(q4u16, 7);
d6u8 = vqrshrn_n_u16(q5u16, 7);
d7u8 = vqrshrn_n_u16(q6u16, 7);
d8u8 = vqrshrn_n_u16(q7u16, 7);
d9u8 = vqrshrn_n_u16(q8u16, 7);
q1u8 = vcombine_u8(d2u8, d3u8);
q2u8 = vcombine_u8(d4u8, d5u8);
q3u8 = vcombine_u8(d6u8, d7u8);
q4u8 = vcombine_u8(d8u8, d9u8);
q11u8 = q15u8;
vst1q_u8((uint8_t *)tmpp2, q1u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q2u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q3u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q4u8);
tmpp2 += 16;
}
} else if (yoffset == 0) { // firstpass_bfilter16x16_only
d0u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][1]);
for (i = 4; i > 0 ; i--) {
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
__builtin_prefetch(src_ptr);
__builtin_prefetch(src_ptr + src_pixels_per_line);
__builtin_prefetch(src_ptr + src_pixels_per_line * 2);
q7u16 = vmull_u8(d2u8, d0u8);
q8u16 = vmull_u8(d3u8, d0u8);
q9u16 = vmull_u8(d5u8, d0u8);
q10u16 = vmull_u8(d6u8, d0u8);
q11u16 = vmull_u8(d8u8, d0u8);
q12u16 = vmull_u8(d9u8, d0u8);
q13u16 = vmull_u8(d11u8, d0u8);
q14u16 = vmull_u8(d12u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
d11u8 = vext_u8(d11u8, d12u8, 1);
q7u16 = vmlal_u8(q7u16, d2u8, d1u8);
q9u16 = vmlal_u8(q9u16, d5u8, d1u8);
q11u16 = vmlal_u8(q11u16, d8u8, d1u8);
q13u16 = vmlal_u8(q13u16, d11u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
d12u8 = vext_u8(d12u8, d13u8, 1);
q8u16 = vmlal_u8(q8u16, d3u8, d1u8);
q10u16 = vmlal_u8(q10u16, d6u8, d1u8);
q12u16 = vmlal_u8(q12u16, d9u8, d1u8);
q14u16 = vmlal_u8(q14u16, d12u8, d1u8);
d14u8 = vqrshrn_n_u16(q7u16, 7);
d15u8 = vqrshrn_n_u16(q8u16, 7);
d16u8 = vqrshrn_n_u16(q9u16, 7);
d17u8 = vqrshrn_n_u16(q10u16, 7);
d18u8 = vqrshrn_n_u16(q11u16, 7);
d19u8 = vqrshrn_n_u16(q12u16, 7);
d20u8 = vqrshrn_n_u16(q13u16, 7);
d21u8 = vqrshrn_n_u16(q14u16, 7);
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
q10u8 = vcombine_u8(d20u8, d21u8);
vst1q_u8((uint8_t *)tmpp2, q7u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q8u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q9u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q10u8);
tmpp2 += 16;
}
} else {
d0u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[xoffset][1]);
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
// First Pass: output_height lines x output_width columns (17x16)
for (i = 3; i > 0; i--) {
q7u16 = vmull_u8(d2u8, d0u8);
q8u16 = vmull_u8(d3u8, d0u8);
q9u16 = vmull_u8(d5u8, d0u8);
q10u16 = vmull_u8(d6u8, d0u8);
q11u16 = vmull_u8(d8u8, d0u8);
q12u16 = vmull_u8(d9u8, d0u8);
q13u16 = vmull_u8(d11u8, d0u8);
q14u16 = vmull_u8(d12u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
d11u8 = vext_u8(d11u8, d12u8, 1);
q7u16 = vmlal_u8(q7u16, d2u8, d1u8);
q9u16 = vmlal_u8(q9u16, d5u8, d1u8);
q11u16 = vmlal_u8(q11u16, d8u8, d1u8);
q13u16 = vmlal_u8(q13u16, d11u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
d12u8 = vext_u8(d12u8, d13u8, 1);
q8u16 = vmlal_u8(q8u16, d3u8, d1u8);
q10u16 = vmlal_u8(q10u16, d6u8, d1u8);
q12u16 = vmlal_u8(q12u16, d9u8, d1u8);
q14u16 = vmlal_u8(q14u16, d12u8, d1u8);
d14u8 = vqrshrn_n_u16(q7u16, 7);
d15u8 = vqrshrn_n_u16(q8u16, 7);
d16u8 = vqrshrn_n_u16(q9u16, 7);
d17u8 = vqrshrn_n_u16(q10u16, 7);
d18u8 = vqrshrn_n_u16(q11u16, 7);
d19u8 = vqrshrn_n_u16(q12u16, 7);
d20u8 = vqrshrn_n_u16(q13u16, 7);
d21u8 = vqrshrn_n_u16(q14u16, 7);
d2u8 = vld1_u8(src_ptr);
d3u8 = vld1_u8(src_ptr + 8);
d4u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d5u8 = vld1_u8(src_ptr);
d6u8 = vld1_u8(src_ptr + 8);
d7u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d8u8 = vld1_u8(src_ptr);
d9u8 = vld1_u8(src_ptr + 8);
d10u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
d11u8 = vld1_u8(src_ptr);
d12u8 = vld1_u8(src_ptr + 8);
d13u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
q10u8 = vcombine_u8(d20u8, d21u8);
vst1q_u8((uint8_t *)tmpp, q7u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q8u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q9u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q10u8);
tmpp += 16;
}
// First-pass filtering for rest 5 lines
d14u8 = vld1_u8(src_ptr);
d15u8 = vld1_u8(src_ptr + 8);
d16u8 = vld1_u8(src_ptr + 16);
src_ptr += src_pixels_per_line;
q9u16 = vmull_u8(d2u8, d0u8);
q10u16 = vmull_u8(d3u8, d0u8);
q11u16 = vmull_u8(d5u8, d0u8);
q12u16 = vmull_u8(d6u8, d0u8);
q13u16 = vmull_u8(d8u8, d0u8);
q14u16 = vmull_u8(d9u8, d0u8);
d2u8 = vext_u8(d2u8, d3u8, 1);
d5u8 = vext_u8(d5u8, d6u8, 1);
d8u8 = vext_u8(d8u8, d9u8, 1);
q9u16 = vmlal_u8(q9u16, d2u8, d1u8);
q11u16 = vmlal_u8(q11u16, d5u8, d1u8);
q13u16 = vmlal_u8(q13u16, d8u8, d1u8);
d3u8 = vext_u8(d3u8, d4u8, 1);
d6u8 = vext_u8(d6u8, d7u8, 1);
d9u8 = vext_u8(d9u8, d10u8, 1);
q10u16 = vmlal_u8(q10u16, d3u8, d1u8);
q12u16 = vmlal_u8(q12u16, d6u8, d1u8);
q14u16 = vmlal_u8(q14u16, d9u8, d1u8);
q1u16 = vmull_u8(d11u8, d0u8);
q2u16 = vmull_u8(d12u8, d0u8);
q3u16 = vmull_u8(d14u8, d0u8);
q4u16 = vmull_u8(d15u8, d0u8);
d11u8 = vext_u8(d11u8, d12u8, 1);
d14u8 = vext_u8(d14u8, d15u8, 1);
q1u16 = vmlal_u8(q1u16, d11u8, d1u8);
q3u16 = vmlal_u8(q3u16, d14u8, d1u8);
d12u8 = vext_u8(d12u8, d13u8, 1);
d15u8 = vext_u8(d15u8, d16u8, 1);
q2u16 = vmlal_u8(q2u16, d12u8, d1u8);
q4u16 = vmlal_u8(q4u16, d15u8, d1u8);
d10u8 = vqrshrn_n_u16(q9u16, 7);
d11u8 = vqrshrn_n_u16(q10u16, 7);
d12u8 = vqrshrn_n_u16(q11u16, 7);
d13u8 = vqrshrn_n_u16(q12u16, 7);
d14u8 = vqrshrn_n_u16(q13u16, 7);
d15u8 = vqrshrn_n_u16(q14u16, 7);
d16u8 = vqrshrn_n_u16(q1u16, 7);
d17u8 = vqrshrn_n_u16(q2u16, 7);
d18u8 = vqrshrn_n_u16(q3u16, 7);
d19u8 = vqrshrn_n_u16(q4u16, 7);
q5u8 = vcombine_u8(d10u8, d11u8);
q6u8 = vcombine_u8(d12u8, d13u8);
q7u8 = vcombine_u8(d14u8, d15u8);
q8u8 = vcombine_u8(d16u8, d17u8);
q9u8 = vcombine_u8(d18u8, d19u8);
vst1q_u8((uint8_t *)tmpp, q5u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q6u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q7u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q8u8);
tmpp += 16;
vst1q_u8((uint8_t *)tmpp, q9u8);
// secondpass_filter
d0u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][0]);
d1u8 = vdup_n_u8(bilinear_taps_coeff[yoffset][1]);
tmpp = tmp;
tmpp2 = tmpp + 272;
q11u8 = vld1q_u8(tmpp);
tmpp += 16;
for (i = 4; i > 0; i--) {
q12u8 = vld1q_u8(tmpp);
tmpp += 16;
q13u8 = vld1q_u8(tmpp);
tmpp += 16;
q14u8 = vld1q_u8(tmpp);
tmpp += 16;
q15u8 = vld1q_u8(tmpp);
tmpp += 16;
q1u16 = vmull_u8(vget_low_u8(q11u8), d0u8);
q2u16 = vmull_u8(vget_high_u8(q11u8), d0u8);
q3u16 = vmull_u8(vget_low_u8(q12u8), d0u8);
q4u16 = vmull_u8(vget_high_u8(q12u8), d0u8);
q5u16 = vmull_u8(vget_low_u8(q13u8), d0u8);
q6u16 = vmull_u8(vget_high_u8(q13u8), d0u8);
q7u16 = vmull_u8(vget_low_u8(q14u8), d0u8);
q8u16 = vmull_u8(vget_high_u8(q14u8), d0u8);
q1u16 = vmlal_u8(q1u16, vget_low_u8(q12u8), d1u8);
q2u16 = vmlal_u8(q2u16, vget_high_u8(q12u8), d1u8);
q3u16 = vmlal_u8(q3u16, vget_low_u8(q13u8), d1u8);
q4u16 = vmlal_u8(q4u16, vget_high_u8(q13u8), d1u8);
q5u16 = vmlal_u8(q5u16, vget_low_u8(q14u8), d1u8);
q6u16 = vmlal_u8(q6u16, vget_high_u8(q14u8), d1u8);
q7u16 = vmlal_u8(q7u16, vget_low_u8(q15u8), d1u8);
q8u16 = vmlal_u8(q8u16, vget_high_u8(q15u8), d1u8);
d2u8 = vqrshrn_n_u16(q1u16, 7);
d3u8 = vqrshrn_n_u16(q2u16, 7);
d4u8 = vqrshrn_n_u16(q3u16, 7);
d5u8 = vqrshrn_n_u16(q4u16, 7);
d6u8 = vqrshrn_n_u16(q5u16, 7);
d7u8 = vqrshrn_n_u16(q6u16, 7);
d8u8 = vqrshrn_n_u16(q7u16, 7);
d9u8 = vqrshrn_n_u16(q8u16, 7);
q1u8 = vcombine_u8(d2u8, d3u8);
q2u8 = vcombine_u8(d4u8, d5u8);
q3u8 = vcombine_u8(d6u8, d7u8);
q4u8 = vcombine_u8(d8u8, d9u8);
q11u8 = q15u8;
vst1q_u8((uint8_t *)tmpp2, q1u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q2u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q3u8);
tmpp2 += 16;
vst1q_u8((uint8_t *)tmpp2, q4u8);
tmpp2 += 16;
}
}
// sub_pixel_variance16x16_neon
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
tmpp = tmp + 272;
for (i = 0; i < 8; i++) { // sub_pixel_variance16x16_neon_loop
q0u8 = vld1q_u8(tmpp);
tmpp += 16;
q1u8 = vld1q_u8(tmpp);
tmpp += 16;
q2u8 = vld1q_u8(dst_ptr);
dst_ptr += dst_pixels_per_line;
q3u8 = vld1q_u8(dst_ptr);
dst_ptr += dst_pixels_per_line;
d0u8 = vget_low_u8(q0u8);
d1u8 = vget_high_u8(q0u8);
d2u8 = vget_low_u8(q1u8);
d3u8 = vget_high_u8(q1u8);
q11u16 = vsubl_u8(d0u8, vget_low_u8(q2u8));
q12u16 = vsubl_u8(d1u8, vget_high_u8(q2u8));
q13u16 = vsubl_u8(d2u8, vget_low_u8(q3u8));
q14u16 = vsubl_u8(d3u8, vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q10s32 = vaddq_s32(q10s32, q9s32);
q0s64 = vpaddlq_s32(q8s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vget_low_s64(q0s64);
d1s64 = vget_high_s64(q0s64);
d2s64 = vget_low_s64(q1s64);
d3s64 = vget_high_s64(q1s64);
d0s64 = vadd_s64(d0s64, d1s64);
d1s64 = vadd_s64(d2s64, d3s64);
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 8);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
void silk_warped_autocorrelation_FIX_neon(
opus_int32 *corr, /* O Result [order + 1] */
opus_int *scale, /* O Scaling of the correlation vector */
const opus_int16 *input, /* I Input data to correlate */
const opus_int warping_Q16, /* I Warping coefficient */
const opus_int length, /* I Length of input */
const opus_int order /* I Correlation order (even) */
)
{
if( ( MAX_SHAPE_LPC_ORDER > 24 ) || ( order < 6 ) ) {
silk_warped_autocorrelation_FIX_c( corr, scale, input, warping_Q16, length, order );
} else {
opus_int n, i, lsh;
opus_int64 corr_QC[ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 }; /* In reverse order */
opus_int64 corr_QC_orderT;
int64x2_t lsh_s64x2;
const opus_int orderT = ( order + 3 ) & ~3;
opus_int64 *corr_QCT;
opus_int32 *input_QS;
VARDECL( opus_int32, input_QST );
VARDECL( opus_int32, state );
SAVE_STACK;
/* Order must be even */
silk_assert( ( order & 1 ) == 0 );
silk_assert( 2 * QS - QC >= 0 );
ALLOC( input_QST, length + 2 * MAX_SHAPE_LPC_ORDER, opus_int32 );
input_QS = input_QST;
/* input_QS has zero paddings in the beginning and end. */
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
/* Loop over samples */
for( n = 0; n < length - 7; n += 8, input_QS += 8 ) {
const int16x8_t t0_s16x4 = vld1q_s16( input + n );
vst1q_s32( input_QS + 0, vshll_n_s16( vget_low_s16( t0_s16x4 ), QS ) );
vst1q_s32( input_QS + 4, vshll_n_s16( vget_high_s16( t0_s16x4 ), QS ) );
}
for( ; n < length; n++, input_QS++ ) {
input_QS[ 0 ] = silk_LSHIFT32( (opus_int32)input[ n ], QS );
}
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS += 4;
vst1q_s32( input_QS, vdupq_n_s32( 0 ) );
input_QS = input_QST + MAX_SHAPE_LPC_ORDER - orderT;
/* The following loop runs ( length + order ) times, with ( order ) extra epilogues. */
/* The zero paddings in input_QS guarantee corr_QC's correctness even with the extra epilogues. */
/* The values of state_QS will be polluted by the extra epilogues, however they are temporary values. */
/* Keep the C code here to help understand the intrinsics optimization. */
/*
{
opus_int32 state_QS[ 2 ][ MAX_SHAPE_LPC_ORDER + 1 ] = { 0 };
opus_int32 *state_QST[ 3 ];
state_QST[ 0 ] = state_QS[ 0 ];
state_QST[ 1 ] = state_QS[ 1 ];
for( n = 0; n < length + order; n++, input_QS++ ) {
state_QST[ 0 ][ orderT ] = input_QS[ orderT ];
for( i = 0; i < orderT; i++ ) {
corr_QC[ i ] += silk_RSHIFT64( silk_SMULL( state_QST[ 0 ][ i ], input_QS[ i ] ), 2 * QS - QC );
state_QST[ 1 ][ i ] = silk_SMLAWB( state_QST[ 1 ][ i + 1 ], state_QST[ 0 ][ i ] - state_QST[ 0 ][ i + 1 ], warping_Q16 );
}
state_QST[ 2 ] = state_QST[ 0 ];
state_QST[ 0 ] = state_QST[ 1 ];
state_QST[ 1 ] = state_QST[ 2 ];
}
}
*/
{
const int32x4_t warping_Q16_s32x4 = vdupq_n_s32( warping_Q16 << 15 );
const opus_int32 *in = input_QS + orderT;
opus_int o = orderT;
int32x4_t state_QS_s32x4[ 3 ][ 2 ];
ALLOC( state, length + orderT, opus_int32 );
state_QS_s32x4[ 2 ][ 1 ] = vdupq_n_s32( 0 );
/* Calculate 8 taps of all inputs in each loop. */
do {
state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 0 ][ 1 ] =
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 1 ][ 1 ] = vdupq_n_s32( 0 );
n = 0;
do {
calc_corr( input_QS + n, corr_QC, o - 8, state_QS_s32x4[ 0 ][ 0 ] );
calc_corr( input_QS + n, corr_QC, o - 4, state_QS_s32x4[ 0 ][ 1 ] );
state_QS_s32x4[ 2 ][ 1 ] = vld1q_s32( in + n );
vst1q_lane_s32( state + n, state_QS_s32x4[ 0 ][ 0 ], 0 );
state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 0 ][ 1 ], 1 );
state_QS_s32x4[ 2 ][ 1 ] = vextq_s32( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], 1 );
state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 );
state_QS_s32x4[ 0 ][ 1 ] = calc_state( state_QS_s32x4[ 0 ][ 1 ], state_QS_s32x4[ 2 ][ 1 ], state_QS_s32x4[ 1 ][ 1 ], warping_Q16_s32x4 );
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ];
state_QS_s32x4[ 1 ][ 1 ] = state_QS_s32x4[ 2 ][ 1 ];
} while( ++n < ( length + order ) );
in = state;
o -= 8;
} while( o > 4 );
if( o ) {
/* Calculate the last 4 taps of all inputs. */
opus_int32 *stateT = state;
silk_assert( o == 4 );
state_QS_s32x4[ 0 ][ 0 ] = state_QS_s32x4[ 1 ][ 0 ] = vdupq_n_s32( 0 );
n = length + order;
do {
calc_corr( input_QS, corr_QC, 0, state_QS_s32x4[ 0 ][ 0 ] );
state_QS_s32x4[ 2 ][ 0 ] = vld1q_s32( stateT );
vst1q_lane_s32( stateT, state_QS_s32x4[ 0 ][ 0 ], 0 );
state_QS_s32x4[ 2 ][ 0 ] = vextq_s32( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], 1 );
state_QS_s32x4[ 0 ][ 0 ] = calc_state( state_QS_s32x4[ 0 ][ 0 ], state_QS_s32x4[ 2 ][ 0 ], state_QS_s32x4[ 1 ][ 0 ], warping_Q16_s32x4 );
state_QS_s32x4[ 1 ][ 0 ] = state_QS_s32x4[ 2 ][ 0 ];
input_QS++;
stateT++;
} while( --n );
}
}
{
const opus_int16 *inputT = input;
int32x4_t t_s32x4;
int64x1_t t_s64x1;
int64x2_t t_s64x2 = vdupq_n_s64( 0 );
for( n = 0; n <= length - 8; n += 8 ) {
int16x8_t input_s16x8 = vld1q_s16( inputT );
t_s32x4 = vmull_s16( vget_low_s16( input_s16x8 ), vget_low_s16( input_s16x8 ) );
t_s32x4 = vmlal_s16( t_s32x4, vget_high_s16( input_s16x8 ), vget_high_s16( input_s16x8 ) );
t_s64x2 = vaddw_s32( t_s64x2, vget_low_s32( t_s32x4 ) );
t_s64x2 = vaddw_s32( t_s64x2, vget_high_s32( t_s32x4 ) );
inputT += 8;
}
t_s64x1 = vadd_s64( vget_low_s64( t_s64x2 ), vget_high_s64( t_s64x2 ) );
corr_QC_orderT = vget_lane_s64( t_s64x1, 0 );
for( ; n < length; n++ ) {
corr_QC_orderT += silk_SMULL( input[ n ], input[ n ] );
}
corr_QC_orderT = silk_LSHIFT64( corr_QC_orderT, QC );
corr_QC[ orderT ] = corr_QC_orderT;
}
corr_QCT = corr_QC + orderT - order;
lsh = silk_CLZ64( corr_QC_orderT ) - 35;
lsh = silk_LIMIT( lsh, -12 - QC, 30 - QC );
*scale = -( QC + lsh );
silk_assert( *scale >= -30 && *scale <= 12 );
lsh_s64x2 = vdupq_n_s64( lsh );
for( i = 0; i <= order - 3; i += 4 ) {
int32x4_t corr_s32x4;
int64x2_t corr_QC0_s64x2, corr_QC1_s64x2;
corr_QC0_s64x2 = vld1q_s64( corr_QCT + i );
corr_QC1_s64x2 = vld1q_s64( corr_QCT + i + 2 );
corr_QC0_s64x2 = vshlq_s64( corr_QC0_s64x2, lsh_s64x2 );
corr_QC1_s64x2 = vshlq_s64( corr_QC1_s64x2, lsh_s64x2 );
corr_s32x4 = vcombine_s32( vmovn_s64( corr_QC1_s64x2 ), vmovn_s64( corr_QC0_s64x2 ) );
corr_s32x4 = vrev64q_s32( corr_s32x4 );
vst1q_s32( corr + order - i - 3, corr_s32x4 );
}
if( lsh >= 0 ) {
for( ; i < order + 1; i++ ) {
corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_LSHIFT64( corr_QCT[ i ], lsh ) );
}
} else {
for( ; i < order + 1; i++ ) {
corr[ order - i ] = (opus_int32)silk_CHECK_FIT32( silk_RSHIFT64( corr_QCT[ i ], -lsh ) );
}
}
silk_assert( corr_QCT[ order ] >= 0 ); /* If breaking, decrease QC*/
RESTORE_STACK;
}
#ifdef OPUS_CHECK_ASM
{
opus_int32 corr_c[ MAX_SHAPE_LPC_ORDER + 1 ];
opus_int scale_c;
silk_warped_autocorrelation_FIX_c( corr_c, &scale_c, input, warping_Q16, length, order );
silk_assert( !memcmp( corr_c, corr, sizeof( corr_c[ 0 ] ) * ( order + 1 ) ) );
silk_assert( scale_c == *scale );
}
#endif
}
int multadd_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint8_t zero_flag,
uint32_t N,
int output_shift)
{
// Multiply elementwise the complex conjugate of x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x1 with a dinamic of 15 bit maximum
//
// x2 - input 2 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x2 with a dinamic of 14 bit maximum
///
// y - output in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
//
// zero_flag - Set output (y) to zero prior to disable accumulation
//
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// output_shift - shift to be applied to generate output
uint32_t i; // loop counter
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
#if defined(__x86_64__) || defined(__i386__)
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
#elif defined(__arm__)
int32x4_t tmp_re,tmp_im;
int32x4_t tmp_re1,tmp_im1;
int16x4x2_t tmpy;
int32x4_t shift = vdupq_n_s32(-output_shift);
#endif
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
#if defined(__x86_64__) || defined(__i386__)
tmp_re = _mm_sign_epi16(*x1_128,*(__m128i*)&conjug2[0]);
tmp_re = _mm_madd_epi16(tmp_re,*x2_128);
tmp_im = _mm_shufflelo_epi16(*x1_128,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_shufflehi_epi16(tmp_im,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_madd_epi16(tmp_im,*x2_128);
tmp_re = _mm_srai_epi32(tmp_re,output_shift);
tmp_im = _mm_srai_epi32(tmp_im,output_shift);
tmpy0 = _mm_unpacklo_epi32(tmp_re,tmp_im);
//print_ints("unpack lo:",&tmpy0[i]);
tmpy1 = _mm_unpackhi_epi32(tmp_re,tmp_im);
//print_ints("unpack hi:",&tmpy1[i]);
if (zero_flag == 1)
*y_128 = _mm_packs_epi32(tmpy0,tmpy1);
else
*y_128 = _mm_adds_epi16(*y_128,_mm_packs_epi32(tmpy0,tmpy1));
//print_shorts("*y_128:",&y_128[i]);
#elif defined(__arm__)
msg("mult_cpx_vector not implemented for __arm__");
#endif
x1_128++;
x2_128++;
y_128++;
}
_mm_empty();
_m_empty();
return(0);
}
// Update the noise estimation information.
static void UpdateNoiseEstimateNeon(NoiseSuppressionFixedC* inst, int offset) {
const int16_t kExp2Const = 11819; // Q13
int16_t* ptr_noiseEstLogQuantile = NULL;
int16_t* ptr_noiseEstQuantile = NULL;
int16x4_t kExp2Const16x4 = vdup_n_s16(kExp2Const);
int32x4_t twentyOne32x4 = vdupq_n_s32(21);
int32x4_t constA32x4 = vdupq_n_s32(0x1fffff);
int32x4_t constB32x4 = vdupq_n_s32(0x200000);
int16_t tmp16 = WebRtcSpl_MaxValueW16(inst->noiseEstLogQuantile + offset,
inst->magnLen);
// Guarantee a Q-domain as high as possible and still fit in int16
inst->qNoise = 14 - (int) WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(kExp2Const,
tmp16,
21);
int32x4_t qNoise32x4 = vdupq_n_s32(inst->qNoise);
for (ptr_noiseEstLogQuantile = &inst->noiseEstLogQuantile[offset],
ptr_noiseEstQuantile = &inst->noiseEstQuantile[0];
ptr_noiseEstQuantile < &inst->noiseEstQuantile[inst->magnLen - 3];
ptr_noiseEstQuantile += 4, ptr_noiseEstLogQuantile += 4) {
// tmp32no2 = kExp2Const * inst->noiseEstLogQuantile[offset + i];
int16x4_t v16x4 = vld1_s16(ptr_noiseEstLogQuantile);
int32x4_t v32x4B = vmull_s16(v16x4, kExp2Const16x4);
// tmp32no1 = (0x00200000 | (tmp32no2 & 0x001FFFFF)); // 2^21 + frac
int32x4_t v32x4A = vandq_s32(v32x4B, constA32x4);
v32x4A = vorrq_s32(v32x4A, constB32x4);
// tmp16 = (int16_t)(tmp32no2 >> 21);
v32x4B = vshrq_n_s32(v32x4B, 21);
// tmp16 -= 21;// shift 21 to get result in Q0
v32x4B = vsubq_s32(v32x4B, twentyOne32x4);
// tmp16 += (int16_t) inst->qNoise;
// shift to get result in Q(qNoise)
v32x4B = vaddq_s32(v32x4B, qNoise32x4);
// if (tmp16 < 0) {
// tmp32no1 >>= -tmp16;
// } else {
// tmp32no1 <<= tmp16;
// }
v32x4B = vshlq_s32(v32x4A, v32x4B);
// tmp16 = WebRtcSpl_SatW32ToW16(tmp32no1);
v16x4 = vqmovn_s32(v32x4B);
//inst->noiseEstQuantile[i] = tmp16;
vst1_s16(ptr_noiseEstQuantile, v16x4);
}
// Last iteration:
// inst->quantile[i]=exp(inst->lquantile[offset+i]);
// in Q21
int32_t tmp32no2 = kExp2Const * *ptr_noiseEstLogQuantile;
int32_t tmp32no1 = (0x00200000 | (tmp32no2 & 0x001FFFFF)); // 2^21 + frac
tmp16 = (int16_t)(tmp32no2 >> 21);
tmp16 -= 21;// shift 21 to get result in Q0
tmp16 += (int16_t) inst->qNoise; //shift to get result in Q(qNoise)
if (tmp16 < 0) {
tmp32no1 >>= -tmp16;
} else {
int32x4_t test_vdupq_n_s32(int32_t v1) {
// CHECK-LABEL: test_vdupq_n_s32
return vdupq_n_s32(v1);
// CHECK: dup {{v[0-9]+}}.4s, {{w[0-9]+}}
}
static void PCorr2Q32(const int16_t *in, int32_t *logcorQ8)
{
int16_t scaling,n,k;
int32_t ysum32,csum32, lys, lcs;
int32_t oneQ8;
const int16_t *x, *inptr;
oneQ8 = WEBRTC_SPL_LSHIFT_W32((int32_t)1, 8); // 1.00 in Q8
x = in + PITCH_MAX_LAG/2 + 2;
scaling = WebRtcSpl_GetScalingSquare ((int16_t *) in, PITCH_CORR_LEN2, PITCH_CORR_LEN2);
ysum32 = 1;
csum32 = 0;
x = in + PITCH_MAX_LAG/2 + 2;
for (n = 0; n < PITCH_CORR_LEN2; n++) {
ysum32 += WEBRTC_SPL_MUL_16_16_RSFT( (int16_t) in[n],(int16_t) in[n], scaling); // Q0
csum32 += WEBRTC_SPL_MUL_16_16_RSFT((int16_t) x[n],(int16_t) in[n], scaling); // Q0
}
logcorQ8 += PITCH_LAG_SPAN2 - 1;
lys=Log2Q8((uint32_t) ysum32); // Q8
lys=WEBRTC_SPL_RSHIFT_W32(lys, 1); //sqrt(ysum);
if (csum32>0) {
lcs=Log2Q8((uint32_t) csum32); // 2log(csum) in Q8
if (lcs>(lys + oneQ8) ){ // csum/sqrt(ysum) > 2 in Q8
*logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum))
} else {
*logcorQ8 = oneQ8; // 1.00
}
} else {
*logcorQ8 = 0;
}
for (k = 1; k < PITCH_LAG_SPAN2; k++) {
inptr = &in[k];
ysum32 -= WEBRTC_SPL_MUL_16_16_RSFT( (int16_t) in[k-1],(int16_t) in[k-1], scaling);
ysum32 += WEBRTC_SPL_MUL_16_16_RSFT( (int16_t) in[PITCH_CORR_LEN2 + k - 1],(int16_t) in[PITCH_CORR_LEN2 + k - 1], scaling);
#ifdef WEBRTC_ARCH_ARM_NEON
{
int32_t vbuff[4];
int32x4_t int_32x4_sum = vmovq_n_s32(0);
// Can't shift a Neon register to right with a non-constant shift value.
int32x4_t int_32x4_scale = vdupq_n_s32(-scaling);
// Assert a codition used in loop unrolling at compile-time.
COMPILE_ASSERT(PITCH_CORR_LEN2 %4 == 0);
for (n = 0; n < PITCH_CORR_LEN2; n += 4) {
int16x4_t int_16x4_x = vld1_s16(&x[n]);
int16x4_t int_16x4_in = vld1_s16(&inptr[n]);
int32x4_t int_32x4 = vmull_s16(int_16x4_x, int_16x4_in);
int_32x4 = vshlq_s32(int_32x4, int_32x4_scale);
int_32x4_sum = vaddq_s32(int_32x4_sum, int_32x4);
}
// Use vector store to avoid long stall from data trasferring
// from vector to general register.
vst1q_s32(vbuff, int_32x4_sum);
csum32 = vbuff[0] + vbuff[1];
csum32 += vbuff[2];
csum32 += vbuff[3];
}
#else
csum32 = 0;
if(scaling == 0) {
for (n = 0; n < PITCH_CORR_LEN2; n++) {
csum32 += x[n] * inptr[n];
}
} else {
for (n = 0; n < PITCH_CORR_LEN2; n++) {
csum32 += (x[n] * inptr[n]) >> scaling;
}
}
#endif
logcorQ8--;
lys=Log2Q8((uint32_t)ysum32); // Q8
lys=WEBRTC_SPL_RSHIFT_W32(lys, 1); //sqrt(ysum);
if (csum32>0) {
lcs=Log2Q8((uint32_t) csum32); // 2log(csum) in Q8
if (lcs>(lys + oneQ8) ){ // csum/sqrt(ysum) > 2
*logcorQ8 = lcs - lys; // log2(csum/sqrt(ysum))
} else {
*logcorQ8 = oneQ8; // 1.00
}
} else {
*logcorQ8 = 0;
}
}
}
void vp8_short_fdct8x4_neon(
int16_t *input,
int16_t *output,
int pitch) {
int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16;
int16x4_t d16s16, d17s16, d26s16, d27s16, d28s16, d29s16;
uint16x4_t d28u16, d29u16;
uint16x8_t q14u16;
int16x8_t q0s16, q1s16, q2s16, q3s16;
int16x8_t q11s16, q12s16, q13s16, q14s16, q15s16, qEmptys16;
int32x4_t q9s32, q10s32, q11s32, q12s32;
int16x8x2_t v2tmp0, v2tmp1;
int32x4x2_t v2tmp2, v2tmp3;
d16s16 = vdup_n_s16(5352);
d17s16 = vdup_n_s16(2217);
q9s32 = vdupq_n_s32(14500);
q10s32 = vdupq_n_s32(7500);
// Part one
pitch >>= 1;
q0s16 = vld1q_s16(input);
input += pitch;
q1s16 = vld1q_s16(input);
input += pitch;
q2s16 = vld1q_s16(input);
input += pitch;
q3s16 = vld1q_s16(input);
v2tmp2 = vtrnq_s32(vreinterpretq_s32_s16(q0s16),
vreinterpretq_s32_s16(q2s16));
v2tmp3 = vtrnq_s32(vreinterpretq_s32_s16(q1s16),
vreinterpretq_s32_s16(q3s16));
v2tmp0 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[0]), // q0
vreinterpretq_s16_s32(v2tmp3.val[0])); // q1
v2tmp1 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[1]), // q2
vreinterpretq_s16_s32(v2tmp3.val[1])); // q3
q11s16 = vaddq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q12s16 = vaddq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q13s16 = vsubq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q14s16 = vsubq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q11s16 = vshlq_n_s16(q11s16, 3);
q12s16 = vshlq_n_s16(q12s16, 3);
q13s16 = vshlq_n_s16(q13s16, 3);
q14s16 = vshlq_n_s16(q14s16, 3);
q0s16 = vaddq_s16(q11s16, q12s16);
q2s16 = vsubq_s16(q11s16, q12s16);
q11s32 = q9s32;
q12s32 = q10s32;
d26s16 = vget_low_s16(q13s16);
d27s16 = vget_high_s16(q13s16);
d28s16 = vget_low_s16(q14s16);
d29s16 = vget_high_s16(q14s16);
q9s32 = vmlal_s16(q9s32, d28s16, d16s16);
q10s32 = vmlal_s16(q10s32, d28s16, d17s16);
q11s32 = vmlal_s16(q11s32, d29s16, d16s16);
q12s32 = vmlal_s16(q12s32, d29s16, d17s16);
q9s32 = vmlal_s16(q9s32, d26s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d26s16, d16s16);
q11s32 = vmlal_s16(q11s32, d27s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d27s16, d16s16);
d2s16 = vshrn_n_s32(q9s32, 12);
d6s16 = vshrn_n_s32(q10s32, 12);
d3s16 = vshrn_n_s32(q11s32, 12);
d7s16 = vshrn_n_s32(q12s32, 12);
q1s16 = vcombine_s16(d2s16, d3s16);
q3s16 = vcombine_s16(d6s16, d7s16);
// Part two
q9s32 = vdupq_n_s32(12000);
q10s32 = vdupq_n_s32(51000);
v2tmp2 = vtrnq_s32(vreinterpretq_s32_s16(q0s16),
vreinterpretq_s32_s16(q2s16));
v2tmp3 = vtrnq_s32(vreinterpretq_s32_s16(q1s16),
vreinterpretq_s32_s16(q3s16));
v2tmp0 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[0]), // q0
vreinterpretq_s16_s32(v2tmp3.val[0])); // q1
v2tmp1 = vtrnq_s16(vreinterpretq_s16_s32(v2tmp2.val[1]), // q2
vreinterpretq_s16_s32(v2tmp3.val[1])); // q3
q11s16 = vaddq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q12s16 = vaddq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q13s16 = vsubq_s16(v2tmp0.val[1], v2tmp1.val[0]);
q14s16 = vsubq_s16(v2tmp0.val[0], v2tmp1.val[1]);
q15s16 = vdupq_n_s16(7);
q11s16 = vaddq_s16(q11s16, q15s16);
q0s16 = vaddq_s16(q11s16, q12s16);
q1s16 = vsubq_s16(q11s16, q12s16);
q11s32 = q9s32;
q12s32 = q10s32;
d0s16 = vget_low_s16(q0s16);
d1s16 = vget_high_s16(q0s16);
d2s16 = vget_low_s16(q1s16);
d3s16 = vget_high_s16(q1s16);
d0s16 = vshr_n_s16(d0s16, 4);
d4s16 = vshr_n_s16(d1s16, 4);
d2s16 = vshr_n_s16(d2s16, 4);
d6s16 = vshr_n_s16(d3s16, 4);
d26s16 = vget_low_s16(q13s16);
d27s16 = vget_high_s16(q13s16);
d28s16 = vget_low_s16(q14s16);
d29s16 = vget_high_s16(q14s16);
q9s32 = vmlal_s16(q9s32, d28s16, d16s16);
q10s32 = vmlal_s16(q10s32, d28s16, d17s16);
q11s32 = vmlal_s16(q11s32, d29s16, d16s16);
q12s32 = vmlal_s16(q12s32, d29s16, d17s16);
q9s32 = vmlal_s16(q9s32, d26s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d26s16, d16s16);
q11s32 = vmlal_s16(q11s32, d27s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d27s16, d16s16);
d1s16 = vshrn_n_s32(q9s32, 16);
d3s16 = vshrn_n_s32(q10s32, 16);
d5s16 = vshrn_n_s32(q11s32, 16);
d7s16 = vshrn_n_s32(q12s32, 16);
qEmptys16 = vdupq_n_s16(0);
q14u16 = vceqq_s16(q14s16, qEmptys16);
q14u16 = vmvnq_u16(q14u16);
d28u16 = vget_low_u16(q14u16);
d29u16 = vget_high_u16(q14u16);
d1s16 = vsub_s16(d1s16, vreinterpret_s16_u16(d28u16));
d5s16 = vsub_s16(d5s16, vreinterpret_s16_u16(d29u16));
q0s16 = vcombine_s16(d0s16, d1s16);
q1s16 = vcombine_s16(d2s16, d3s16);
q2s16 = vcombine_s16(d4s16, d5s16);
q3s16 = vcombine_s16(d6s16, d7s16);
vst1q_s16(output, q0s16);
vst1q_s16(output + 8, q1s16);
vst1q_s16(output + 16, q2s16);
vst1q_s16(output + 24, q3s16);
return;
}
unsigned int vp8_variance16x8_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
uint32x2_t d0u32, d10u32;
int64x1_t d0s64, d1s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int32x4_t q8s32, q9s32, q10s32;
int64x2_t q0s64, q1s64, q5s64;
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
for (i = 0; i < 4; i++) { // variance16x8_neon_loop
q0u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q1u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
__builtin_prefetch(src_ptr);
q2u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q3u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
__builtin_prefetch(ref_ptr);
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q10s32 = vaddq_s32(q10s32, q9s32);
q0s64 = vpaddlq_s32(q8s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
vreinterpret_s32_s64(d0s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7);
d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
return vget_lane_u32(d0u32, 0);
}
int rotate_cpx_vector(int16_t *x,
int16_t *alpha,
int16_t *y,
uint32_t N,
uint16_t output_shift)
{
// Multiply elementwise two complex vectors of N elements
// x - input 1 in the format |Re0 Im0 |,......,|Re(N-1) Im(N-1)|
// We assume x1 with a dynamic of 15 bit maximum
//
// alpha - input 2 in the format |Re0 Im0|
// We assume x2 with a dynamic of 15 bit maximum
//
// y - output in the format |Re0 Im0|,......,|Re(N-1) Im(N-1)|
//
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// log2_amp - increase the output amplitude by a factor 2^log2_amp (default is 0)
// WARNING: log2_amp>0 can cause overflow!!
uint32_t i; // loop counter
simd_q15_t *y_128,alpha_128;
int32_t *xd=(int32_t *)x;
#if defined(__x86_64__) || defined(__i386__)
__m128i shift = _mm_cvtsi32_si128(output_shift);
register simd_q15_t m0,m1,m2,m3;
((int16_t *)&alpha_128)[0] = alpha[0];
((int16_t *)&alpha_128)[1] = -alpha[1];
((int16_t *)&alpha_128)[2] = alpha[1];
((int16_t *)&alpha_128)[3] = alpha[0];
((int16_t *)&alpha_128)[4] = alpha[0];
((int16_t *)&alpha_128)[5] = -alpha[1];
((int16_t *)&alpha_128)[6] = alpha[1];
((int16_t *)&alpha_128)[7] = alpha[0];
#elif defined(__arm__)
int32x4_t shift;
int32x4_t ab_re0,ab_re1,ab_im0,ab_im1,re32,im32;
int16_t reflip[8] __attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1};
int32x4x2_t xtmp;
((int16_t *)&alpha_128)[0] = alpha[0];
((int16_t *)&alpha_128)[1] = alpha[1];
((int16_t *)&alpha_128)[2] = alpha[0];
((int16_t *)&alpha_128)[3] = alpha[1];
((int16_t *)&alpha_128)[4] = alpha[0];
((int16_t *)&alpha_128)[5] = alpha[1];
((int16_t *)&alpha_128)[6] = alpha[0];
((int16_t *)&alpha_128)[7] = alpha[1];
int16x8_t bflip = vrev32q_s16(alpha_128);
int16x8_t bconj = vmulq_s16(alpha_128,*(int16x8_t *)reflip);
shift = vdupq_n_s32(-output_shift);
#endif
y_128 = (simd_q15_t *) y;
for(i=0; i<N>>2; i++) {
#if defined(__x86_64__) || defined(__i386__)
m0 = _mm_setr_epi32(xd[0],xd[0],xd[1],xd[1]);
m1 = _mm_setr_epi32(xd[2],xd[2],xd[3],xd[3]);
m2 = _mm_madd_epi16(m0,alpha_128); //complex multiply. result is 32bit [Re Im Re Im]
m3 = _mm_madd_epi16(m1,alpha_128); //complex multiply. result is 32bit [Re Im Re Im]
m2 = _mm_sra_epi32(m2,shift); // shift right by shift in order to compensate for the input amplitude
m3 = _mm_sra_epi32(m3,shift); // shift right by shift in order to compensate for the input amplitude
y_128[0] = _mm_packs_epi32(m2,m3); // pack in 16bit integers with saturation [re im re im re im re im]
#elif defined(__arm__)
ab_re0 = vmull_s16(((int16x4_t*)xd)[0],((int16x4_t*)&bconj)[0]);
ab_re1 = vmull_s16(((int16x4_t*)xd)[1],((int16x4_t*)&bconj)[1]);
ab_im0 = vmull_s16(((int16x4_t*)xd)[0],((int16x4_t*)&bflip)[0]);
ab_im1 = vmull_s16(((int16x4_t*)xd)[1],((int16x4_t*)&bflip)[1]);
re32 = vshlq_s32(vcombine_s32(vpadd_s32(((int32x2_t*)&ab_re0)[0],((int32x2_t*)&ab_re0)[1]),
vpadd_s32(((int32x2_t*)&ab_re1)[0],((int32x2_t*)&ab_re1)[1])),
shift);
im32 = vshlq_s32(vcombine_s32(vpadd_s32(((int32x2_t*)&ab_im0)[0],((int32x2_t*)&ab_im0)[1]),
vpadd_s32(((int32x2_t*)&ab_im1)[0],((int32x2_t*)&ab_im1)[1])),
shift);
xtmp = vzipq_s32(re32,im32);
y_128[0] = vcombine_s16(vmovn_s32(xtmp.val[0]),vmovn_s32(xtmp.val[1]));
#endif
xd+=4;
y_128+=1;
}
_mm_empty();
_m_empty();
return(0);
}
unsigned int vpx_mse16x16_neon(
const unsigned char *src_ptr,
int source_stride,
const unsigned char *ref_ptr,
int recon_stride,
unsigned int *sse) {
int i;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
int64x1_t d0s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8;
int32x4_t q7s32, q8s32, q9s32, q10s32;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int64x2_t q1s64;
q7s32 = vdupq_n_s32(0);
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
for (i = 0; i < 8; i++) { // mse16x16_neon_loop
q0u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q1u8 = vld1q_u8(src_ptr);
src_ptr += source_stride;
q2u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q3u8 = vld1q_u8(ref_ptr);
ref_ptr += recon_stride;
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
}
q7s32 = vaddq_s32(q7s32, q8s32);
q9s32 = vaddq_s32(q9s32, q10s32);
q10s32 = vaddq_s32(q7s32, q9s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
}
//
// box blur a square array of pixels (power of 2, actually)
// if we insist on powers of 2, we don't need to special case some end-of-row/col conditions
// to a specific blur width
//
// also, we're using NEON to vectorize our arithmetic.
// we need to do a division along the way, but NEON doesn't support integer division.
// so rather than divide by, say "w", we multiply by magic(w).
// magic(w) is chosen so that the result of multiplying by it will be the same as
// dividing by w, except that the result will be in the high half of the result.
// yes, dorothy... this is what compilers do, too...
void NEONboxBlur(pixel *src, pixel *dest, unsigned int size, unsigned int blurRad) {
unsigned int wid = 2 * blurRad + 1;
// because NEON doesn't have integer division, we use "magic constants" that will give
// use the result of division by multiplication -- the upper half of the result will be
// (more or less) the result of the division.
// for this, we need to compute the magic numbers corresponding to a given divisor
struct magicu_info minfo = compute_unsigned_magic_info(wid, 16);
int16x8_t preshift = vdupq_n_s16(-minfo.pre_shift); // negative means shift right
int32x4_t postshift = vdupq_n_s32(-(minfo.post_shift+16)); // negative means shift right
uint16x4_t magic = vdup_n_u16(minfo.multiplier);
// fprintf(stderr,"width %5d, preshift %d, postshift %d + 16, increment %d, magic %d\n", wid,
// minfo.pre_shift, minfo.post_shift, minfo.increment, minfo.multiplier);
// if (minfo.pre_shift > 0) fprintf(stderr,"hey, not an odd number!\n");
int i, j, k, ch;
for (i = 0 ; i < size ; i+=8) {
// first, initialize the sum so that we can loop from 0 to size-1
// we'll initialize boxsum for index -1, so that we can move into 0 as part of our loop
uint16x8x4_t boxsum;
uint8x8x4_t firstpixel = vld4_u8((uint8_t *)(src + 0 * size + i));
for (ch = 0 ; ch < 4 ; ch++) {
// boxsum[ch] = blurRad * srcpixel[ch]
boxsum.val[ch] = vmulq_n_u16(vmovl_u8(firstpixel.val[ch]),(blurRad+1)+1);
}
for ( k = 1 ; k < blurRad ; k++) {
uint8x8x4_t srcpixel = vld4_u8((uint8_t *)(src + k * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
boxsum.val[ch] = vaddw_u8(boxsum.val[ch], srcpixel.val[ch]);
}
}
int right = blurRad-1;
int left = -blurRad-1;
if (minfo.increment) {
for ( k = 0 ; k < size ; k++) {
// move to next pixel
unsigned int l = (left < 0)?0:left; // take off the old left
left++;
right++;
unsigned int r = (right < size)?right:(size-1); // but add the new right
uint8x8x4_t addpixel = vld4_u8((uint8_t *)(src + r * size + i));
uint8x8x4_t subpixel = vld4_u8((uint8_t *)(src + l * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
// boxsum[ch] += addpixel[ch] - subpixel[ch];
boxsum.val[ch] = vsubw_u8(vaddw_u8(boxsum.val[ch], addpixel.val[ch]), subpixel.val[ch]);
}
uint8x8x4_t destpixel;
for (ch = 0 ; ch < 4 ; ch++ ) { // compute: destpixel = boxsum / wid
// since 16bit multiplication leads to 32bit results, we need to
// split our task into two chunks, for the hi and low half of our vector
// (because otherwise, it won't all fit into 128 bits)
// this is the meat of the magic division algorithm (see the include file...)
uint16x8_t bsum_preshifted = vshlq_u16(boxsum.val[ch],preshift);
// multiply by the magic number
uint32x4_t res_hi = vmull_u16(vget_high_u16(bsum_preshifted), magic);
res_hi = vaddw_u16(res_hi, magic);
// take the high half and post-shift
uint16x4_t q_hi = vmovn_u32(vshlq_u32(res_hi, postshift));
// pre-shift and multiply by the magic number
uint32x4_t res_lo = vmull_u16(vget_low_u16(bsum_preshifted), magic);
res_lo = vaddw_u16(res_lo, magic);
// take the high half and post-shift
uint16x4_t q_lo = vmovn_u32(vshlq_u32(res_lo, postshift));
destpixel.val[ch] = vqmovn_u16(vcombine_u16(q_lo, q_hi));
}
pixel block[8];
vst4_u8((uint8_t *)&block, destpixel);
for (j = 0 ; j < 8 ; j++ ) {
dest[(i + j)*size + k] = block[j];
}
// vst4_u8((uint8_t *)(dest + k * size + i), destpixel);
}
} else {
for ( k = 0 ; k < size ; k++) {
// move to next pixel
unsigned int l = (left < 0)?0:left; // take off the old left
left++;
right++;
unsigned int r = (right < size)?right:(size-1); // but add the new right
uint8x8x4_t addpixel = vld4_u8((uint8_t *)(src + r * size + i));
uint8x8x4_t subpixel = vld4_u8((uint8_t *)(src + l * size + i));
for (ch = 0 ; ch < 4 ; ch++ ) {
// boxsum[ch] += addpixel[ch] - subpixel[ch];
boxsum.val[ch] = vsubw_u8(vaddw_u8(boxsum.val[ch], addpixel.val[ch]), subpixel.val[ch]);
}
uint8x8x4_t destpixel;
for (ch = 0 ; ch < 4 ; ch++ ) { // compute: destpixel = boxsum / wid
// since 16bit multiplication leads to 32bit results, we need to
// split our task into two chunks, for the hi and low half of our vector
// (because otherwise, it won't all fit into 128 bits)
// this is the meat of the magic division algorithm (see the include file...)
uint16x8_t bsum_preshifted = vshlq_u16(boxsum.val[ch],preshift);
// multiply by the magic number
// take the high half and post-shift
uint32x4_t res_hi = vmull_u16(vget_high_u16(bsum_preshifted), magic);
uint16x4_t q_hi = vmovn_u32(vshlq_u32(res_hi, postshift));
// multiply by the magic number
// take the high half and post-shift
uint32x4_t res_lo = vmull_u16(vget_low_u16(bsum_preshifted), magic);
uint16x4_t q_lo = vmovn_u32(vshlq_u32(res_lo, postshift));
destpixel.val[ch] = vqmovn_u16(vcombine_u16(q_lo, q_hi));
}
pixel block[8];
vst4_u8((uint8_t *)&block, destpixel);
for (j = 0 ; j < 8 ; j++ ) {
dest[(i + j)*size + k] = block[j];
}
// vst4_u8((uint8_t *)(dest + k * size + i), destpixel);
}
}
}
}
int mult_cpx_conj_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
int output_shift,
int madd)
{
// Multiply elementwise the complex conjugate of x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x1 with a dinamic of 15 bit maximum
//
// x2 - input 2 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x2 with a dinamic of 14 bit maximum
///
// y - output in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
//
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// output_shift - shift to be applied to generate output
//
// madd - add the output to y
uint32_t i; // loop counter
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
#if defined(__x86_64__) || defined(__i386__)
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
#elif defined(__arm__)
int32x4_t tmp_re,tmp_im;
int32x4_t tmp_re1,tmp_im1;
int16x4x2_t tmpy;
int32x4_t shift = vdupq_n_s32(-output_shift);
#endif
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
#if defined(__x86_64__) || defined(__i386__)
tmp_re = _mm_madd_epi16(*x1_128,*x2_128);
tmp_im = _mm_shufflelo_epi16(*x1_128,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_shufflehi_epi16(tmp_im,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_sign_epi16(tmp_im,*(__m128i*)&conjug[0]);
tmp_im = _mm_madd_epi16(tmp_im,*x2_128);
tmp_re = _mm_srai_epi32(tmp_re,output_shift);
tmp_im = _mm_srai_epi32(tmp_im,output_shift);
tmpy0 = _mm_unpacklo_epi32(tmp_re,tmp_im);
tmpy1 = _mm_unpackhi_epi32(tmp_re,tmp_im);
if (madd==0)
*y_128 = _mm_packs_epi32(tmpy0,tmpy1);
else
*y_128 += _mm_packs_epi32(tmpy0,tmpy1);
#elif defined(__arm__)
tmp_re = vmull_s16(((simdshort_q15_t *)x1_128)[0], ((simdshort_q15_t*)x2_128)[0]);
//tmp_re = [Re(x1[0])Re(x2[0]) Im(x1[0])Im(x2[0]) Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1])]
tmp_re1 = vmull_s16(((simdshort_q15_t *)x1_128)[1], ((simdshort_q15_t*)x2_128)[1]);
//tmp_re1 = [Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1]) Re(x1[1])Re(x2[2]) Im(x1[1])Im(x2[2])]
tmp_re = vcombine_s32(vpadd_s32(vget_low_s32(tmp_re),vget_high_s32(tmp_re)),
vpadd_s32(vget_low_s32(tmp_re1),vget_high_s32(tmp_re1)));
//tmp_re = [Re(ch[0])Re(rx[0])+Im(ch[0])Im(ch[0]) Re(ch[1])Re(rx[1])+Im(ch[1])Im(ch[1]) Re(ch[2])Re(rx[2])+Im(ch[2]) Im(ch[2]) Re(ch[3])Re(rx[3])+Im(ch[3])Im(ch[3])]
tmp_im = vmull_s16(vrev32_s16(vmul_s16(((simdshort_q15_t*)x2_128)[0],*(simdshort_q15_t*)conjug)), ((simdshort_q15_t*)x1_128)[0]);
//tmp_im = [-Im(ch[0])Re(rx[0]) Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1]) Re(ch[1])Im(rx[1])]
tmp_im1 = vmull_s16(vrev32_s16(vmul_s16(((simdshort_q15_t*)x2_128)[1],*(simdshort_q15_t*)conjug)), ((simdshort_q15_t*)x1_128)[1]);
//tmp_im1 = [-Im(ch[2])Re(rx[2]) Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3]) Re(ch[3])Im(rx[3])]
tmp_im = vcombine_s32(vpadd_s32(vget_low_s32(tmp_im),vget_high_s32(tmp_im)),
vpadd_s32(vget_low_s32(tmp_im1),vget_high_s32(tmp_im1)));
//tmp_im = [-Im(ch[0])Re(rx[0])+Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1])+Re(ch[1])Im(rx[1]) -Im(ch[2])Re(rx[2])+Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3])+Re(ch[3])Im(rx[3])]
tmp_re = vqshlq_s32(tmp_re,shift);
tmp_im = vqshlq_s32(tmp_im,shift);
tmpy = vzip_s16(vmovn_s32(tmp_re),vmovn_s32(tmp_im));
if (madd==0)
*y_128 = vcombine_s16(tmpy.val[0],tmpy.val[1]);
else
*y_128 += vcombine_s16(tmpy.val[0],tmpy.val[1]);
#endif
x1_128++;
x2_128++;
y_128++;
}
_mm_empty();
_m_empty();
return(0);
}
void vp8_short_fdct4x4_neon(
int16_t *input,
int16_t *output,
int pitch) {
int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16, d6s16, d7s16;
int16x4_t d16s16, d17s16, d26s16, dEmptys16;
uint16x4_t d4u16;
int16x8_t q0s16, q1s16;
int32x4_t q9s32, q10s32, q11s32, q12s32;
int16x4x2_t v2tmp0, v2tmp1;
int32x2x2_t v2tmp2, v2tmp3;
d16s16 = vdup_n_s16(5352);
d17s16 = vdup_n_s16(2217);
q9s32 = vdupq_n_s32(14500);
q10s32 = vdupq_n_s32(7500);
q11s32 = vdupq_n_s32(12000);
q12s32 = vdupq_n_s32(51000);
// Part one
pitch >>= 1;
d0s16 = vld1_s16(input);
input += pitch;
d1s16 = vld1_s16(input);
input += pitch;
d2s16 = vld1_s16(input);
input += pitch;
d3s16 = vld1_s16(input);
v2tmp2 = vtrn_s32(vreinterpret_s32_s16(d0s16),
vreinterpret_s32_s16(d2s16));
v2tmp3 = vtrn_s32(vreinterpret_s32_s16(d1s16),
vreinterpret_s32_s16(d3s16));
v2tmp0 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[0]), // d0
vreinterpret_s16_s32(v2tmp3.val[0])); // d1
v2tmp1 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[1]), // d2
vreinterpret_s16_s32(v2tmp3.val[1])); // d3
d4s16 = vadd_s16(v2tmp0.val[0], v2tmp1.val[1]);
d5s16 = vadd_s16(v2tmp0.val[1], v2tmp1.val[0]);
d6s16 = vsub_s16(v2tmp0.val[1], v2tmp1.val[0]);
d7s16 = vsub_s16(v2tmp0.val[0], v2tmp1.val[1]);
d4s16 = vshl_n_s16(d4s16, 3);
d5s16 = vshl_n_s16(d5s16, 3);
d6s16 = vshl_n_s16(d6s16, 3);
d7s16 = vshl_n_s16(d7s16, 3);
d0s16 = vadd_s16(d4s16, d5s16);
d2s16 = vsub_s16(d4s16, d5s16);
q9s32 = vmlal_s16(q9s32, d7s16, d16s16);
q10s32 = vmlal_s16(q10s32, d7s16, d17s16);
q9s32 = vmlal_s16(q9s32, d6s16, d17s16);
q10s32 = vmlsl_s16(q10s32, d6s16, d16s16);
d1s16 = vshrn_n_s32(q9s32, 12);
d3s16 = vshrn_n_s32(q10s32, 12);
// Part two
v2tmp2 = vtrn_s32(vreinterpret_s32_s16(d0s16),
vreinterpret_s32_s16(d2s16));
v2tmp3 = vtrn_s32(vreinterpret_s32_s16(d1s16),
vreinterpret_s32_s16(d3s16));
v2tmp0 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[0]), // d0
vreinterpret_s16_s32(v2tmp3.val[0])); // d1
v2tmp1 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[1]), // d2
vreinterpret_s16_s32(v2tmp3.val[1])); // d3
d4s16 = vadd_s16(v2tmp0.val[0], v2tmp1.val[1]);
d5s16 = vadd_s16(v2tmp0.val[1], v2tmp1.val[0]);
d6s16 = vsub_s16(v2tmp0.val[1], v2tmp1.val[0]);
d7s16 = vsub_s16(v2tmp0.val[0], v2tmp1.val[1]);
d26s16 = vdup_n_s16(7);
d4s16 = vadd_s16(d4s16, d26s16);
d0s16 = vadd_s16(d4s16, d5s16);
d2s16 = vsub_s16(d4s16, d5s16);
q11s32 = vmlal_s16(q11s32, d7s16, d16s16);
q12s32 = vmlal_s16(q12s32, d7s16, d17s16);
dEmptys16 = vdup_n_s16(0);
d4u16 = vceq_s16(d7s16, dEmptys16);
d0s16 = vshr_n_s16(d0s16, 4);
d2s16 = vshr_n_s16(d2s16, 4);
q11s32 = vmlal_s16(q11s32, d6s16, d17s16);
q12s32 = vmlsl_s16(q12s32, d6s16, d16s16);
d4u16 = vmvn_u16(d4u16);
d1s16 = vshrn_n_s32(q11s32, 16);
d1s16 = vsub_s16(d1s16, vreinterpret_s16_u16(d4u16));
d3s16 = vshrn_n_s32(q12s32, 16);
q0s16 = vcombine_s16(d0s16, d1s16);
q1s16 = vcombine_s16(d2s16, d3s16);
vst1q_s16(output, q0s16);
vst1q_s16(output + 8, q1s16);
return;
}
static float32x4_t vpowq_f32(float32x4_t a, float32x4_t b) {
// a^b = exp2(b * log2(a))
// exp2(x) and log2(x) are calculated using polynomial approximations.
float32x4_t log2_a, b_log2_a, a_exp_b;
// Calculate log2(x), x = a.
{
// To calculate log2(x), we decompose x like this:
// x = y * 2^n
// n is an integer
// y is in the [1.0, 2.0) range
//
// log2(x) = log2(y) + n
// n can be evaluated by playing with float representation.
// log2(y) in a small range can be approximated, this code uses an order
// five polynomial approximation. The coefficients have been
// estimated with the Remez algorithm and the resulting
// polynomial has a maximum relative error of 0.00086%.
// Compute n.
// This is done by masking the exponent, shifting it into the top bit of
// the mantissa, putting eight into the biased exponent (to shift/
// compensate the fact that the exponent has been shifted in the top/
// fractional part and finally getting rid of the implicit leading one
// from the mantissa by substracting it out.
const uint32x4_t vec_float_exponent_mask = vdupq_n_u32(0x7F800000);
const uint32x4_t vec_eight_biased_exponent = vdupq_n_u32(0x43800000);
const uint32x4_t vec_implicit_leading_one = vdupq_n_u32(0x43BF8000);
const uint32x4_t two_n = vandq_u32(vreinterpretq_u32_f32(a),
vec_float_exponent_mask);
const uint32x4_t n_1 = vshrq_n_u32(two_n, kShiftExponentIntoTopMantissa);
const uint32x4_t n_0 = vorrq_u32(n_1, vec_eight_biased_exponent);
const float32x4_t n =
vsubq_f32(vreinterpretq_f32_u32(n_0),
vreinterpretq_f32_u32(vec_implicit_leading_one));
// Compute y.
const uint32x4_t vec_mantissa_mask = vdupq_n_u32(0x007FFFFF);
const uint32x4_t vec_zero_biased_exponent_is_one = vdupq_n_u32(0x3F800000);
const uint32x4_t mantissa = vandq_u32(vreinterpretq_u32_f32(a),
vec_mantissa_mask);
const float32x4_t y =
vreinterpretq_f32_u32(vorrq_u32(mantissa,
vec_zero_biased_exponent_is_one));
// Approximate log2(y) ~= (y - 1) * pol5(y).
// pol5(y) = C5 * y^5 + C4 * y^4 + C3 * y^3 + C2 * y^2 + C1 * y + C0
const float32x4_t C5 = vdupq_n_f32(-3.4436006e-2f);
const float32x4_t C4 = vdupq_n_f32(3.1821337e-1f);
const float32x4_t C3 = vdupq_n_f32(-1.2315303f);
const float32x4_t C2 = vdupq_n_f32(2.5988452f);
const float32x4_t C1 = vdupq_n_f32(-3.3241990f);
const float32x4_t C0 = vdupq_n_f32(3.1157899f);
float32x4_t pol5_y = C5;
pol5_y = vmlaq_f32(C4, y, pol5_y);
pol5_y = vmlaq_f32(C3, y, pol5_y);
pol5_y = vmlaq_f32(C2, y, pol5_y);
pol5_y = vmlaq_f32(C1, y, pol5_y);
pol5_y = vmlaq_f32(C0, y, pol5_y);
const float32x4_t y_minus_one =
vsubq_f32(y, vreinterpretq_f32_u32(vec_zero_biased_exponent_is_one));
const float32x4_t log2_y = vmulq_f32(y_minus_one, pol5_y);
// Combine parts.
log2_a = vaddq_f32(n, log2_y);
}
// b * log2(a)
b_log2_a = vmulq_f32(b, log2_a);
// Calculate exp2(x), x = b * log2(a).
{
// To calculate 2^x, we decompose x like this:
// x = n + y
// n is an integer, the value of x - 0.5 rounded down, therefore
// y is in the [0.5, 1.5) range
//
// 2^x = 2^n * 2^y
// 2^n can be evaluated by playing with float representation.
// 2^y in a small range can be approximated, this code uses an order two
// polynomial approximation. The coefficients have been estimated
// with the Remez algorithm and the resulting polynomial has a
// maximum relative error of 0.17%.
// To avoid over/underflow, we reduce the range of input to ]-127, 129].
const float32x4_t max_input = vdupq_n_f32(129.f);
const float32x4_t min_input = vdupq_n_f32(-126.99999f);
const float32x4_t x_min = vminq_f32(b_log2_a, max_input);
const float32x4_t x_max = vmaxq_f32(x_min, min_input);
// Compute n.
const float32x4_t half = vdupq_n_f32(0.5f);
const float32x4_t x_minus_half = vsubq_f32(x_max, half);
const int32x4_t x_minus_half_floor = vcvtq_s32_f32(x_minus_half);
// Compute 2^n.
const int32x4_t float_exponent_bias = vdupq_n_s32(127);
const int32x4_t two_n_exponent =
vaddq_s32(x_minus_half_floor, float_exponent_bias);
const float32x4_t two_n =
vreinterpretq_f32_s32(vshlq_n_s32(two_n_exponent, kFloatExponentShift));
// Compute y.
const float32x4_t y = vsubq_f32(x_max, vcvtq_f32_s32(x_minus_half_floor));
// Approximate 2^y ~= C2 * y^2 + C1 * y + C0.
const float32x4_t C2 = vdupq_n_f32(3.3718944e-1f);
const float32x4_t C1 = vdupq_n_f32(6.5763628e-1f);
const float32x4_t C0 = vdupq_n_f32(1.0017247f);
float32x4_t exp2_y = C2;
exp2_y = vmlaq_f32(C1, y, exp2_y);
exp2_y = vmlaq_f32(C0, y, exp2_y);
// Combine parts.
a_exp_b = vmulq_f32(exp2_y, two_n);
}
return a_exp_b;
}
static inline void PostShiftAndSeparateNeon(int16_t* inre,
int16_t* inim,
int16_t* outre,
int16_t* outim,
int32_t sh) {
int k;
int16_t* inre1 = inre;
int16_t* inre2 = &inre[FRAMESAMPLES/2 - 4];
int16_t* inim1 = inim;
int16_t* inim2 = &inim[FRAMESAMPLES/2 - 4];
int16_t* outre1 = outre;
int16_t* outre2 = &outre[FRAMESAMPLES/2 - 4];
int16_t* outim1 = outim;
int16_t* outim2 = &outim[FRAMESAMPLES/2 - 4];
const int16_t* kSinTab1 = &WebRtcIsacfix_kSinTab2[0];
const int16_t* kSinTab2 = &WebRtcIsacfix_kSinTab2[FRAMESAMPLES/4 -4];
// By vshl, we effectively did "<< (-sh - 23)", instead of "<< (-sh)",
// ">> 14" and then ">> 9" as in the C code.
int32x4_t shift = vdupq_n_s32(-sh - 23);
for (k = 0; k < FRAMESAMPLES/4; k += 4) {
int16x4_t tmpi = vld1_s16(kSinTab1);
kSinTab1 += 4;
int16x4_t tmpr = vld1_s16(kSinTab2);
kSinTab2 -= 4;
int16x4_t inre_0 = vld1_s16(inre1);
inre1 += 4;
int16x4_t inre_1 = vld1_s16(inre2);
inre2 -= 4;
int16x4_t inim_0 = vld1_s16(inim1);
inim1 += 4;
int16x4_t inim_1 = vld1_s16(inim2);
inim2 -= 4;
tmpr = vneg_s16(tmpr);
inre_1 = vrev64_s16(inre_1);
inim_1 = vrev64_s16(inim_1);
tmpr = vrev64_s16(tmpr);
int16x4_t xr = vqadd_s16(inre_0, inre_1);
int16x4_t xi = vqsub_s16(inim_0, inim_1);
int16x4_t yr = vqadd_s16(inim_0, inim_1);
int16x4_t yi = vqsub_s16(inre_1, inre_0);
int32x4_t outr0 = vmull_s16(tmpr, xr);
int32x4_t outi0 = vmull_s16(tmpi, xr);
int32x4_t outr1 = vmull_s16(tmpi, yr);
int32x4_t outi1 = vmull_s16(tmpi, yi);
outr0 = vmlsl_s16(outr0, tmpi, xi);
outi0 = vmlal_s16(outi0, tmpr, xi);
outr1 = vmlal_s16(outr1, tmpr, yi);
outi1 = vmlsl_s16(outi1, tmpr, yr);
outr0 = vshlq_s32(outr0, shift);
outi0 = vshlq_s32(outi0, shift);
outr1 = vshlq_s32(outr1, shift);
outi1 = vshlq_s32(outi1, shift);
outr1 = vnegq_s32(outr1);
int16x4_t outre_0 = vmovn_s32(outr0);
int16x4_t outim_0 = vmovn_s32(outi0);
int16x4_t outre_1 = vmovn_s32(outr1);
int16x4_t outim_1 = vmovn_s32(outi1);
outre_1 = vrev64_s16(outre_1);
outim_1 = vrev64_s16(outim_1);
vst1_s16(outre1, outre_0);
outre1 += 4;
vst1_s16(outim1, outim_0);
outim1 += 4;
vst1_s16(outre2, outre_1);
outre2 -= 4;
vst1_s16(outim2, outim_1);
outim2 -= 4;
}
}
inline int32x4_t vdupq_n(const s32 & val) { return vdupq_n_s32(val); }
static inline int32_t ComplexMulAndFindMaxNeon(int16_t* inre1Q9,
int16_t* inre2Q9,
int32_t* outreQ16,
int32_t* outimQ16) {
int k;
const int16_t* kCosTab = &WebRtcIsacfix_kCosTab1[0];
const int16_t* kSinTab = &WebRtcIsacfix_kSinTab1[0];
// 0.5 / sqrt(240) in Q19 is round((.5 / sqrt(240)) * (2^19)) = 16921.
// Use "16921 << 5" and vqdmulh, instead of ">> 26" as in the C code.
int32_t fact = 16921 << 5;
int32x4_t factq = vdupq_n_s32(fact);
uint32x4_t max_r = vdupq_n_u32(0);
uint32x4_t max_i = vdupq_n_u32(0);
for (k = 0; k < FRAMESAMPLES/2; k += 8) {
int16x8_t tmpr = vld1q_s16(kCosTab);
int16x8_t tmpi = vld1q_s16(kSinTab);
int16x8_t inre1 = vld1q_s16(inre1Q9);
int16x8_t inre2 = vld1q_s16(inre2Q9);
kCosTab += 8;
kSinTab += 8;
inre1Q9 += 8;
inre2Q9 += 8;
// Use ">> 26", instead of ">> 7", ">> 16" and then ">> 3" as in the C code.
int32x4_t tmp0 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre1));
int32x4_t tmp1 = vmull_s16(vget_low_s16(tmpr), vget_low_s16(inre2));
tmp0 = vmlal_s16(tmp0, vget_low_s16(tmpi), vget_low_s16(inre2));
tmp1 = vmlsl_s16(tmp1, vget_low_s16(tmpi), vget_low_s16(inre1));
#if defined(WEBRTC_ARCH_ARM64)
int32x4_t tmp2 = vmull_high_s16(tmpr, inre1);
int32x4_t tmp3 = vmull_high_s16(tmpr, inre2);
tmp2 = vmlal_high_s16(tmp2, tmpi, inre2);
tmp3 = vmlsl_high_s16(tmp3, tmpi, inre1);
#else
int32x4_t tmp2 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre1));
int32x4_t tmp3 = vmull_s16(vget_high_s16(tmpr), vget_high_s16(inre2));
tmp2 = vmlal_s16(tmp2, vget_high_s16(tmpi), vget_high_s16(inre2));
tmp3 = vmlsl_s16(tmp3, vget_high_s16(tmpi), vget_high_s16(inre1));
#endif
int32x4_t outr_0 = vqdmulhq_s32(tmp0, factq);
int32x4_t outr_1 = vqdmulhq_s32(tmp2, factq);
int32x4_t outi_0 = vqdmulhq_s32(tmp1, factq);
int32x4_t outi_1 = vqdmulhq_s32(tmp3, factq);
vst1q_s32(outreQ16, outr_0);
outreQ16 += 4;
vst1q_s32(outreQ16, outr_1);
outreQ16 += 4;
vst1q_s32(outimQ16, outi_0);
outimQ16 += 4;
vst1q_s32(outimQ16, outi_1);
outimQ16 += 4;
// Find the absolute maximum in the vectors.
tmp0 = vabsq_s32(outr_0);
tmp1 = vabsq_s32(outr_1);
tmp2 = vabsq_s32(outi_0);
tmp3 = vabsq_s32(outi_1);
// vabs doesn't change the value of 0x80000000.
// Use u32 so we don't lose the value 0x80000000.
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp0));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp2));
max_r = vmaxq_u32(max_r, vreinterpretq_u32_s32(tmp1));
max_i = vmaxq_u32(max_i, vreinterpretq_u32_s32(tmp3));
}
max_r = vmaxq_u32(max_r, max_i);
#if defined(WEBRTC_ARCH_ARM64)
uint32_t maximum = vmaxvq_u32(max_r);
#else
uint32x2_t max32x2_r = vmax_u32(vget_low_u32(max_r), vget_high_u32(max_r));
max32x2_r = vpmax_u32(max32x2_r, max32x2_r);
uint32_t maximum = vget_lane_u32(max32x2_r, 0);
#endif
return (int32_t)maximum;
}
bool decode_yuv_neon(unsigned char* out, unsigned char const* y, unsigned char const* uv, int width, int height, unsigned char fill_alpha=0xff)
{
// pre-condition : width, height must be even
if (0!=(width&1) || width<2 || 0!=(height&1) || height<2 || !out || !y || !uv)
return false;
// in & out pointers
unsigned char* dst = out;
// constants
int const stride = width*trait::bytes_per_pixel;
int const itHeight = height>>1;
int const itWidth = width>>3;
uint8x8_t const Yshift = vdup_n_u8(16);
int16x8_t const half = vdupq_n_u16(128);
int32x4_t const rounding = vdupq_n_s32(128);
// tmp variable
uint16x8_t t;
// pixel block to temporary store 8 pixels
typename trait::PixelBlock pblock = trait::init_pixelblock(fill_alpha);
for (int j=0; j<itHeight; ++j, y+=width, dst+=stride) {
for (int i=0; i<itWidth; ++i, y+=8, uv+=8, dst+=(8*trait::bytes_per_pixel)) {
t = vmovl_u8(vqsub_u8(vld1_u8(y), Yshift));
int32x4_t const Y00 = vmulq_n_u32(vmovl_u16(vget_low_u16(t)), 298);
int32x4_t const Y01 = vmulq_n_u32(vmovl_u16(vget_high_u16(t)), 298);
t = vmovl_u8(vqsub_u8(vld1_u8(y+width), Yshift));
int32x4_t const Y10 = vmulq_n_u32(vmovl_u16(vget_low_u16(t)), 298);
int32x4_t const Y11 = vmulq_n_u32(vmovl_u16(vget_high_u16(t)), 298);
// trait::loadvu pack 4 sets of uv into a uint8x8_t, layout : { v0,u0, v1,u1, v2,u2, v3,u3 }
t = vsubq_s16((int16x8_t)vmovl_u8(trait::loadvu(uv)), half);
// UV.val[0] : v0, v1, v2, v3
// UV.val[1] : u0, u1, u2, u3
int16x4x2_t const UV = vuzp_s16(vget_low_s16(t), vget_high_s16(t));
// tR : 128+409V
// tG : 128-100U-208V
// tB : 128+516U
int32x4_t const tR = vmlal_n_s16(rounding, UV.val[0], 409);
int32x4_t const tG = vmlal_n_s16(vmlal_n_s16(rounding, UV.val[0], -208), UV.val[1], -100);
int32x4_t const tB = vmlal_n_s16(rounding, UV.val[1], 516);
int32x4x2_t const R = vzipq_s32(tR, tR); // [tR0, tR0, tR1, tR1] [ tR2, tR2, tR3, tR3]
int32x4x2_t const G = vzipq_s32(tG, tG); // [tG0, tG0, tG1, tG1] [ tG2, tG2, tG3, tG3]
int32x4x2_t const B = vzipq_s32(tB, tB); // [tB0, tB0, tB1, tB1] [ tB2, tB2, tB3, tB3]
// upper 8 pixels
trait::store_pixel_block(dst, pblock,
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(R.val[0], Y00)), vqmovun_s32(vaddq_s32(R.val[1], Y01))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(G.val[0], Y00)), vqmovun_s32(vaddq_s32(G.val[1], Y01))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(B.val[0], Y00)), vqmovun_s32(vaddq_s32(B.val[1], Y01))), 8));
// lower 8 pixels
trait::store_pixel_block(dst+stride, pblock,
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(R.val[0], Y10)), vqmovun_s32(vaddq_s32(R.val[1], Y11))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(G.val[0], Y10)), vqmovun_s32(vaddq_s32(G.val[1], Y11))), 8),
vshrn_n_u16(vcombine_u16(vqmovun_s32(vaddq_s32(B.val[0], Y10)), vqmovun_s32(vaddq_s32(B.val[1], Y11))), 8));
}
}
return true;
}
static inline void PostShiftAndDivideAndDemodulateNeon(int16_t* inre,
int16_t* inim,
int32_t* outre1,
int32_t* outre2,
int32_t sh) {
int k;
int16_t* p_inre = inre;
int16_t* p_inim = inim;
int32_t* p_outre1 = outre1;
int32_t* p_outre2 = outre2;
const int16_t* kCosTab = &WebRtcIsacfix_kCosTab1[0];
const int16_t* kSinTab = &WebRtcIsacfix_kSinTab1[0];
int32x4_t shift = vdupq_n_s32(-sh - 16);
// Divide through by the normalizing constant:
// scale all values with 1/240, i.e. with 273 in Q16.
// 273/65536 ~= 0.0041656
// 1/240 ~= 0.0041666
int16x8_t scale = vdupq_n_s16(273);
// Sqrt(240) in Q11 is round(15.49193338482967 * 2048) = 31727.
int factQ19 = 31727 << 16;
int32x4_t fact = vdupq_n_s32(factQ19);
for (k = 0; k < FRAMESAMPLES/2; k += 8) {
int16x8_t inre16x8 = vld1q_s16(p_inre);
int16x8_t inim16x8 = vld1q_s16(p_inim);
p_inre += 8;
p_inim += 8;
int16x8_t tmpr = vld1q_s16(kCosTab);
int16x8_t tmpi = vld1q_s16(kSinTab);
kCosTab += 8;
kSinTab += 8;
// By vshl and vmull, we effectively did "<< (-sh - 16)",
// instead of "<< (-sh)" and ">> 16" as in the C code.
int32x4_t outre1_0 = vmull_s16(vget_low_s16(inre16x8), vget_low_s16(scale));
int32x4_t outre2_0 = vmull_s16(vget_low_s16(inim16x8), vget_low_s16(scale));
#if defined(WEBRTC_ARCH_ARM64)
int32x4_t outre1_1 = vmull_high_s16(inre16x8, scale);
int32x4_t outre2_1 = vmull_high_s16(inim16x8, scale);
#else
int32x4_t outre1_1 = vmull_s16(vget_high_s16(inre16x8),
vget_high_s16(scale));
int32x4_t outre2_1 = vmull_s16(vget_high_s16(inim16x8),
vget_high_s16(scale));
#endif
outre1_0 = vshlq_s32(outre1_0, shift);
outre1_1 = vshlq_s32(outre1_1, shift);
outre2_0 = vshlq_s32(outre2_0, shift);
outre2_1 = vshlq_s32(outre2_1, shift);
// Demodulate and separate.
int32x4_t tmpr_0 = vmovl_s16(vget_low_s16(tmpr));
int32x4_t tmpi_0 = vmovl_s16(vget_low_s16(tmpi));
#if defined(WEBRTC_ARCH_ARM64)
int32x4_t tmpr_1 = vmovl_high_s16(tmpr);
int32x4_t tmpi_1 = vmovl_high_s16(tmpi);
#else
int32x4_t tmpr_1 = vmovl_s16(vget_high_s16(tmpr));
int32x4_t tmpi_1 = vmovl_s16(vget_high_s16(tmpi));
#endif
int64x2_t xr0 = vmull_s32(vget_low_s32(tmpr_0), vget_low_s32(outre1_0));
int64x2_t xi0 = vmull_s32(vget_low_s32(tmpr_0), vget_low_s32(outre2_0));
int64x2_t xr2 = vmull_s32(vget_low_s32(tmpr_1), vget_low_s32(outre1_1));
int64x2_t xi2 = vmull_s32(vget_low_s32(tmpr_1), vget_low_s32(outre2_1));
xr0 = vmlsl_s32(xr0, vget_low_s32(tmpi_0), vget_low_s32(outre2_0));
xi0 = vmlal_s32(xi0, vget_low_s32(tmpi_0), vget_low_s32(outre1_0));
xr2 = vmlsl_s32(xr2, vget_low_s32(tmpi_1), vget_low_s32(outre2_1));
xi2 = vmlal_s32(xi2, vget_low_s32(tmpi_1), vget_low_s32(outre1_1));
#if defined(WEBRTC_ARCH_ARM64)
int64x2_t xr1 = vmull_high_s32(tmpr_0, outre1_0);
int64x2_t xi1 = vmull_high_s32(tmpr_0, outre2_0);
int64x2_t xr3 = vmull_high_s32(tmpr_1, outre1_1);
int64x2_t xi3 = vmull_high_s32(tmpr_1, outre2_1);
xr1 = vmlsl_high_s32(xr1, tmpi_0, outre2_0);
xi1 = vmlal_high_s32(xi1, tmpi_0, outre1_0);
xr3 = vmlsl_high_s32(xr3, tmpi_1, outre2_1);
xi3 = vmlal_high_s32(xi3, tmpi_1, outre1_1);
#else
int64x2_t xr1 = vmull_s32(vget_high_s32(tmpr_0), vget_high_s32(outre1_0));
int64x2_t xi1 = vmull_s32(vget_high_s32(tmpr_0), vget_high_s32(outre2_0));
int64x2_t xr3 = vmull_s32(vget_high_s32(tmpr_1), vget_high_s32(outre1_1));
int64x2_t xi3 = vmull_s32(vget_high_s32(tmpr_1), vget_high_s32(outre2_1));
xr1 = vmlsl_s32(xr1, vget_high_s32(tmpi_0), vget_high_s32(outre2_0));
xi1 = vmlal_s32(xi1, vget_high_s32(tmpi_0), vget_high_s32(outre1_0));
xr3 = vmlsl_s32(xr3, vget_high_s32(tmpi_1), vget_high_s32(outre2_1));
xi3 = vmlal_s32(xi3, vget_high_s32(tmpi_1), vget_high_s32(outre1_1));
#endif
outre1_0 = vcombine_s32(vshrn_n_s64(xr0, 10), vshrn_n_s64(xr1, 10));
outre2_0 = vcombine_s32(vshrn_n_s64(xi0, 10), vshrn_n_s64(xi1, 10));
outre1_1 = vcombine_s32(vshrn_n_s64(xr2, 10), vshrn_n_s64(xr3, 10));
outre2_1 = vcombine_s32(vshrn_n_s64(xi2, 10), vshrn_n_s64(xi3, 10));
outre1_0 = vqdmulhq_s32(outre1_0, fact);
outre2_0 = vqdmulhq_s32(outre2_0, fact);
outre1_1 = vqdmulhq_s32(outre1_1, fact);
outre2_1 = vqdmulhq_s32(outre2_1, fact);
vst1q_s32(p_outre1, outre1_0);
p_outre1 += 4;
vst1q_s32(p_outre1, outre1_1);
p_outre1 += 4;
vst1q_s32(p_outre2, outre2_0);
p_outre2 += 4;
vst1q_s32(p_outre2, outre2_1);
p_outre2 += 4;
}
}
void sLine_onMessage(HvBase *_c, SignalLine *o, int letIn,
const HvMessage * const m, void *sendMessage) {
if (msg_isFloat(m,0)) {
if (msg_isFloat(m,1)) {
// new ramp
int n = ctx_millisecondsToSamples(_c, msg_getFloat(m,1));
#if HV_SIMD_AVX
float x = (o->n[1] > 0) ? (o->x[7] + (o->m[7]/8.0f)) : o->t[7]; // current output value
float s = (msg_getFloat(m,0) - x) / ((float) n); // slope per sample
o->n = _mm_set_epi32(n-3, n-2, n-1, n);
o->x = _mm256_set_ps(x+7.0f*s, x+6.0f*s, x+5.0f*s, x+4.0f*s, x+3.0f*s, x+2.0f*s, x+s, x);
o->m = _mm256_set1_ps(8.0f*s);
o->t = _mm256_set1_ps(msg_getFloat(m,0));
#elif HV_SIMD_SSE
float x = (o->n[1] > 0) ? (o->x[3] + (o->m[3]/4.0f)) : o->t[3];
float s = (msg_getFloat(m,0) - x) / ((float) n); // slope per sample
o->n = _mm_set_epi32(n-3, n-2, n-1, n);
o->x = _mm_set_ps(x+3.0f*s, x+2.0f*s, x+s, x);
o->m = _mm_set1_ps(4.0f*s);
o->t = _mm_set1_ps(msg_getFloat(m,0));
#elif HV_SIMD_NEON
float x = (o->n[3] > 0) ? (o->x[3] + (o->m[3]/4.0f)) : o->t[3];
float s = (msg_getFloat(m,0) - x) / ((float) n);
o->n = (int32x4_t) {n, n-1, n-2, n-3};
o->x = (float32x4_t) {x, x+s, x+2.0f*s, x+3.0f*s};
o->m = vdupq_n_f32(4.0f*s);
o->t = vdupq_n_f32(msg_getFloat(m,0));
#else // HV_SIMD_NONE
o->x = (o->n > 0) ? (o->x + o->m) : o->t; // new current value
o->n = n; // new distance to target
o->m = (msg_getFloat(m,0) - o->x) / ((float) n); // slope per sample
o->t = msg_getFloat(m,0);
#endif
} else {
// Jump to value
#if HV_SIMD_AVX
o->n = _mm_setzero_si128();
o->x = _mm256_set1_ps(msg_getFloat(m,0));
o->m = _mm256_setzero_ps();
o->t = _mm256_set1_ps(msg_getFloat(m,0));
#elif HV_SIMD_SSE
o->n = _mm_setzero_si128();
o->x = _mm_set1_ps(msg_getFloat(m,0));
o->m = _mm_setzero_ps();
o->t = _mm_set1_ps(msg_getFloat(m,0));
#elif HV_SIMD_NEON
o->n = vdupq_n_s32(0);
o->x = vdupq_n_f32(0.0f);
o->m = vdupq_n_f32(0.0f);
o->t = vdupq_n_f32(0.0f);
#else // HV_SIMD_NONE
o->n = 0;
o->x = msg_getFloat(m,0);
o->m = 0.0f;
o->t = msg_getFloat(m,0);
#endif
}
} else if (msg_compareSymbol(m,0,"stop")) {
// Stop line at current position
#if HV_SIMD_AVX
// note o->n[1] is a 64-bit integer; two packed 32-bit ints. We only want to know if the high int is positive,
// which can be done simply by testing the long int for positiveness.
float x = (o->n[1] > 0) ? (o->x[7] + (o->m[7]/8.0f)) : o->t[7];
o->n = _mm_setzero_si128();
o->x = _mm256_set1_ps(x);
o->m = _mm256_setzero_ps();
o->t = _mm256_set1_ps(x);
#elif HV_SIMD_SSE
float x = (o->n[1] > 0) ? (o->x[3] + (o->m[3]/4.0f)) : o->t[3];
o->n = _mm_setzero_si128();
o->x = _mm_set1_ps(x);
o->m = _mm_setzero_ps();
o->t = _mm_set1_ps(x);
#elif HV_SIMD_NEON
float x = (o->n[3] > 0) ? (o->x[3] + (o->m[3]/4.0f)) : o->t[3];
o->n = vdupq_n_s32(0);
o->x = vdupq_n_f32(x);
o->m = vdupq_n_f32(0.0f);
o->t = vdupq_n_f32(x);
#else // HV_SIMD_NONE
o->n = 0;
o->x += o->m;
o->m = 0.0f;
o->t = o->x;
#endif
}
}
// Contains a function for the core loop in the normalized lattice MA
// filter routine for iSAC codec, optimized for ARM Neon platform.
// It does:
// for 0 <= n < HALF_SUBFRAMELEN - 1:
// *ptr2 = input2 * (*ptr2) + input0 * (*ptr0));
// *ptr1 = input1 * (*ptr0) + input0 * (*ptr2);
// Output is not bit-exact with the reference C code, due to the replacement
// of WEBRTC_SPL_MUL_16_32_RSFT15 and LATTICE_MUL_32_32_RSFT16 with Neon
// instructions. The difference should not be bigger than 1.
void WebRtcIsacfix_FilterMaLoopNeon(int16_t input0, // Filter coefficient
int16_t input1, // Filter coefficient
int32_t input2, // Inverse coefficient
int32_t* ptr0, // Sample buffer
int32_t* ptr1, // Sample buffer
int32_t* ptr2) // Sample buffer
{
int n = 0;
int loop = (HALF_SUBFRAMELEN - 1) >> 3;
int loop_tail = (HALF_SUBFRAMELEN - 1) & 0x7;
int32x4_t input0_v = vdupq_n_s32((int32_t)input0 << 16);
int32x4_t input1_v = vdupq_n_s32((int32_t)input1 << 16);
int32x4_t input2_v = vdupq_n_s32(input2);
int32x4_t tmp0a, tmp1a, tmp2a, tmp3a;
int32x4_t tmp0b, tmp1b, tmp2b, tmp3b;
int32x4_t ptr0va, ptr1va, ptr2va;
int32x4_t ptr0vb, ptr1vb, ptr2vb;
// Unroll to process 8 samples at once.
for (n = 0; n < loop; n++) {
ptr0va = vld1q_s32(ptr0);
ptr0vb = vld1q_s32(ptr0 + 4);
ptr0 += 8;
ptr2va = vld1q_s32(ptr2);
ptr2vb = vld1q_s32(ptr2 + 4);
// Calculate tmp0 = (*ptr0) * input0.
tmp0a = vqrdmulhq_s32(ptr0va, input0_v);
tmp0b = vqrdmulhq_s32(ptr0vb, input0_v);
// Calculate tmp1 = (*ptr0) * input1.
tmp1a = vqrdmulhq_s32(ptr0va, input1_v);
tmp1b = vqrdmulhq_s32(ptr0vb, input1_v);
// Calculate tmp2 = tmp0 + *(ptr2).
tmp2a = vaddq_s32(tmp0a, ptr2va);
tmp2b = vaddq_s32(tmp0b, ptr2vb);
tmp2a = vshlq_n_s32(tmp2a, 15);
tmp2b = vshlq_n_s32(tmp2b, 15);
// Calculate *ptr2 = input2 * tmp2.
ptr2va = vqrdmulhq_s32(tmp2a, input2_v);
ptr2vb = vqrdmulhq_s32(tmp2b, input2_v);
vst1q_s32(ptr2, ptr2va);
vst1q_s32(ptr2 + 4, ptr2vb);
ptr2 += 8;
// Calculate tmp3 = ptr2v * input0.
tmp3a = vqrdmulhq_s32(ptr2va, input0_v);
tmp3b = vqrdmulhq_s32(ptr2vb, input0_v);
// Calculate *ptr1 = tmp1 + tmp3.
ptr1va = vaddq_s32(tmp1a, tmp3a);
ptr1vb = vaddq_s32(tmp1b, tmp3b);
vst1q_s32(ptr1, ptr1va);
vst1q_s32(ptr1 + 4, ptr1vb);
ptr1 += 8;
}
// Process four more samples.
if (loop_tail & 0x4) {
ptr0va = vld1q_s32(ptr0);
ptr2va = vld1q_s32(ptr2);
ptr0 += 4;
// Calculate tmp0 = (*ptr0) * input0.
tmp0a = vqrdmulhq_s32(ptr0va, input0_v);
// Calculate tmp1 = (*ptr0) * input1.
tmp1a = vqrdmulhq_s32(ptr0va, input1_v);
// Calculate tmp2 = tmp0 + *(ptr2).
tmp2a = vaddq_s32(tmp0a, ptr2va);
tmp2a = vshlq_n_s32(tmp2a, 15);
// Calculate *ptr2 = input2 * tmp2.
ptr2va = vqrdmulhq_s32(tmp2a, input2_v);
vst1q_s32(ptr2, ptr2va);
ptr2 += 4;
// Calculate tmp3 = *(ptr2) * input0.
tmp3a = vqrdmulhq_s32(ptr2va, input0_v);
// Calculate *ptr1 = tmp1 + tmp3.
ptr1va = vaddq_s32(tmp1a, tmp3a);
vst1q_s32(ptr1, ptr1va);
ptr1 += 4;
}
// Process two more samples.
if (loop_tail & 0x2) {
int32x2_t ptr0v_tail, ptr2v_tail, ptr1v_tail;
int32x2_t tmp0_tail, tmp1_tail, tmp2_tail, tmp3_tail;
ptr0v_tail = vld1_s32(ptr0);
ptr2v_tail = vld1_s32(ptr2);
ptr0 += 2;
// Calculate tmp0 = (*ptr0) * input0.
tmp0_tail = vqrdmulh_s32(ptr0v_tail, vget_low_s32(input0_v));
// Calculate tmp1 = (*ptr0) * input1.
tmp1_tail = vqrdmulh_s32(ptr0v_tail, vget_low_s32(input1_v));
// Calculate tmp2 = tmp0 + *(ptr2).
tmp2_tail = vadd_s32(tmp0_tail, ptr2v_tail);
tmp2_tail = vshl_n_s32(tmp2_tail, 15);
// Calculate *ptr2 = input2 * tmp2.
ptr2v_tail = vqrdmulh_s32(tmp2_tail, vget_low_s32(input2_v));
vst1_s32(ptr2, ptr2v_tail);
ptr2 += 2;
// Calculate tmp3 = *(ptr2) * input0.
tmp3_tail = vqrdmulh_s32(ptr2v_tail, vget_low_s32(input0_v));
// Calculate *ptr1 = tmp1 + tmp3.
ptr1v_tail = vadd_s32(tmp1_tail, tmp3_tail);
vst1_s32(ptr1, ptr1v_tail);
ptr1 += 2;
}
// Process one more sample.
if (loop_tail & 0x1) {
int16_t t16a = (int16_t)(input2 >> 16);
int16_t t16b = (int16_t)input2;
if (t16b < 0) t16a++;
int32_t tmp32a;
int32_t tmp32b;
// Calculate *ptr2 = input2 * (*ptr2 + input0 * (*ptr0)).
tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr0);
tmp32b = *ptr2 + tmp32a;
*ptr2 = (int32_t)(WEBRTC_SPL_MUL(t16a, tmp32b) +
(WEBRTC_SPL_MUL_16_32_RSFT16(t16b, tmp32b)));
// Calculate *ptr1 = input1 * (*ptr0) + input0 * (*ptr2).
tmp32a = WEBRTC_SPL_MUL_16_32_RSFT15(input1, *ptr0);
tmp32b = WEBRTC_SPL_MUL_16_32_RSFT15(input0, *ptr2);
*ptr1 = tmp32a + tmp32b;
}
