int32_t dot_product(int16_t *x,
int16_t *y,
uint32_t N, //must be a multiple of 8
uint8_t output_shift)
{
uint32_t n;
#if defined(__x86_64__) || defined(__i386__)
__m128i *x128,*y128,mmtmp1,mmtmp2,mmtmp3,mmcumul,mmcumul_re,mmcumul_im;
__m64 mmtmp7;
__m128i minus_i = _mm_set_epi16(-1,1,-1,1,-1,1,-1,1);
int32_t result;
x128 = (__m128i*) x;
y128 = (__m128i*) y;
mmcumul_re = _mm_setzero_si128();
mmcumul_im = _mm_setzero_si128();
for (n=0; n<(N>>2); n++) {
//printf("n=%d, x128=%p, y128=%p\n",n,x128,y128);
// print_shorts("x",&x128[0]);
// print_shorts("y",&y128[0]);
// this computes Re(z) = Re(x)*Re(y) + Im(x)*Im(y)
mmtmp1 = _mm_madd_epi16(x128[0],y128[0]);
// print_ints("re",&mmtmp1);
// mmtmp1 contains real part of 4 consecutive outputs (32-bit)
// shift and accumulate results
mmtmp1 = _mm_srai_epi32(mmtmp1,output_shift);
mmcumul_re = _mm_add_epi32(mmcumul_re,mmtmp1);
// print_ints("re",&mmcumul_re);
// this computes Im(z) = Re(x)*Im(y) - Re(y)*Im(x)
mmtmp2 = _mm_shufflelo_epi16(y128[0],_MM_SHUFFLE(2,3,0,1));
// print_shorts("y",&mmtmp2);
mmtmp2 = _mm_shufflehi_epi16(mmtmp2,_MM_SHUFFLE(2,3,0,1));
// print_shorts("y",&mmtmp2);
mmtmp2 = _mm_sign_epi16(mmtmp2,minus_i);
// print_shorts("y",&mmtmp2);
mmtmp3 = _mm_madd_epi16(x128[0],mmtmp2);
// print_ints("im",&mmtmp3);
// mmtmp3 contains imag part of 4 consecutive outputs (32-bit)
// shift and accumulate results
mmtmp3 = _mm_srai_epi32(mmtmp3,output_shift);
mmcumul_im = _mm_add_epi32(mmcumul_im,mmtmp3);
// print_ints("im",&mmcumul_im);
x128++;
y128++;
}
// this gives Re Re Im Im
mmcumul = _mm_hadd_epi32(mmcumul_re,mmcumul_im);
// print_ints("cumul1",&mmcumul);
// this gives Re Im Re Im
mmcumul = _mm_hadd_epi32(mmcumul,mmcumul);
// print_ints("cumul2",&mmcumul);
//mmcumul = _mm_srai_epi32(mmcumul,output_shift);
// extract the lower half
mmtmp7 = _mm_movepi64_pi64(mmcumul);
// print_ints("mmtmp7",&mmtmp7);
// pack the result
mmtmp7 = _mm_packs_pi32(mmtmp7,mmtmp7);
// print_shorts("mmtmp7",&mmtmp7);
// convert back to integer
result = _mm_cvtsi64_si32(mmtmp7);
_mm_empty();
_m_empty();
return(result);
#elif defined(__arm__)
int16x4_t *x_128=(int16x4_t*)x;
int16x4_t *y_128=(int16x4_t*)y;
int32x4_t tmp_re,tmp_im;
int32x4_t tmp_re1,tmp_im1;
int32x4_t re_cumul,im_cumul;
int32x2_t re_cumul2,im_cumul2;
int32x4_t shift = vdupq_n_s32(-output_shift);
int32x2x2_t result2;
int16_t conjug[4]__attribute__((aligned(16))) = {-1,1,-1,1} ;
re_cumul = vdupq_n_s32(0);
im_cumul = vdupq_n_s32(0);
for (n=0; n<(N>>2); n++) {
tmp_re = vmull_s16(*x_128++, *y_128++);
//tmp_re = [Re(x[0])Re(y[0]) Im(x[0])Im(y[0]) Re(x[1])Re(y[1]) Im(x[1])Im(y[1])]
tmp_re1 = vmull_s16(*x_128++, *y_128++);
//tmp_re1 = [Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1]) Re(x1[1])Re(x2[2]) Im(x1[1])Im(x2[2])]
tmp_re = vcombine_s32(vpadd_s32(vget_low_s32(tmp_re),vget_high_s32(tmp_re)),
vpadd_s32(vget_low_s32(tmp_re1),vget_high_s32(tmp_re1)));
//tmp_re = [Re(ch[0])Re(rx[0])+Im(ch[0])Im(ch[0]) Re(ch[1])Re(rx[1])+Im(ch[1])Im(ch[1]) Re(ch[2])Re(rx[2])+Im(ch[2]) Im(ch[2]) Re(ch[3])Re(rx[3])+Im(ch[3])Im(ch[3])]
tmp_im = vmull_s16(vrev32_s16(vmul_s16(*x_128++,*(int16x4_t*)conjug)),*y_128++);
//tmp_im = [-Im(ch[0])Re(rx[0]) Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1]) Re(ch[1])Im(rx[1])]
tmp_im1 = vmull_s16(vrev32_s16(vmul_s16(*x_128++,*(int16x4_t*)conjug)),*y_128++);
//tmp_im1 = [-Im(ch[2])Re(rx[2]) Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3]) Re(ch[3])Im(rx[3])]
tmp_im = vcombine_s32(vpadd_s32(vget_low_s32(tmp_im),vget_high_s32(tmp_im)),
vpadd_s32(vget_low_s32(tmp_im1),vget_high_s32(tmp_im1)));
//tmp_im = [-Im(ch[0])Re(rx[0])+Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1])+Re(ch[1])Im(rx[1]) -Im(ch[2])Re(rx[2])+Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3])+Re(ch[3])Im(rx[3])]
re_cumul = vqaddq_s32(re_cumul,vqshlq_s32(tmp_re,shift));
im_cumul = vqaddq_s32(im_cumul,vqshlq_s32(tmp_im,shift));
}
re_cumul2 = vpadd_s32(vget_low_s32(re_cumul),vget_high_s32(re_cumul));
im_cumul2 = vpadd_s32(vget_low_s32(im_cumul),vget_high_s32(im_cumul));
re_cumul2 = vpadd_s32(re_cumul2,re_cumul2);
im_cumul2 = vpadd_s32(im_cumul2,im_cumul2);
result2 = vzip_s32(re_cumul2,im_cumul2);
return(vget_lane_s32(result2.val[0],0));
#endif
}
void test_vpadds32 (void)
{
int32x2_t out_int32x2_t;
int32x2_t arg0_int32x2_t;
int32x2_t arg1_int32x2_t;
out_int32x2_t = vpadd_s32 (arg0_int32x2_t, arg1_int32x2_t);
}
void dotProd_i16_neon(const float *dataf, const float *weightsf, float *vals, const int n, const int len, const float *istd) {
const int16_t *data = (const int16_t *)dataf;
const int16_t *weights = (const int16_t *)weightsf;
weightsf += n * len / 2; // sizeof(float) / sizeof(int16_t)
for (int i = 0; i < n; i += 4) {
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int j = 0; j < len; j += 8) {
int16x4x2_t d0 = vld2_s16(data + j);
int16x4x2_t w0 = vld2_s16(weights);
int16x4x2_t w1 = vld2_s16(weights + 8);
int16x4x2_t w2 = vld2_s16(weights + 16);
int16x4x2_t w3 = vld2_s16(weights + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
weights += 32;
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t val = vcvtq_f32_s32(sum);
val = vmulq_f32(val, vld1q_f32(weightsf + i*2));
val = vmulq_n_f32(val, istd[0]);
val = vaddq_f32(val, vld1q_f32(weightsf + i*2 + 4));
vst1q_f32(vals + i, val);
}
}
// ref, src = [0, 510] - max diff = 16-bits
// bwl = {2, 3, 4}, width = {16, 32, 64}
int vp9_vector_var_neon(int16_t const *ref, int16_t const *src, const int bwl) {
int width = 4 << bwl;
int32x4_t sse = vdupq_n_s32(0);
int16x8_t total = vdupq_n_s16(0);
assert(width >= 8);
assert((width % 8) == 0);
do {
const int16x8_t r = vld1q_s16(ref);
const int16x8_t s = vld1q_s16(src);
const int16x8_t diff = vsubq_s16(r, s); // [-510, 510], 10 bits.
const int16x4_t diff_lo = vget_low_s16(diff);
const int16x4_t diff_hi = vget_high_s16(diff);
sse = vmlal_s16(sse, diff_lo, diff_lo); // dynamic range 26 bits.
sse = vmlal_s16(sse, diff_hi, diff_hi);
total = vaddq_s16(total, diff); // dynamic range 16 bits.
ref += 8;
src += 8;
width -= 8;
} while (width != 0);
{
// Note: 'total''s pairwise addition could be implemented similarly to
// horizontal_add_u16x8(), but one less vpaddl with 'total' when paired
// with the summation of 'sse' performed better on a Cortex-A15.
const int32x4_t t0 = vpaddlq_s16(total); // cascading summation of 'total'
const int32x2_t t1 = vadd_s32(vget_low_s32(t0), vget_high_s32(t0));
const int32x2_t t2 = vpadd_s32(t1, t1);
const int t = vget_lane_s32(t2, 0);
const int64x2_t s0 = vpaddlq_s32(sse); // cascading summation of 'sse'.
const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)),
vreinterpret_s32_s64(vget_high_s64(s0)));
const int s = vget_lane_s32(s1, 0);
const int shift_factor = bwl + 2;
return s - ((t * t) >> shift_factor);
}
}
void computeNetwork0new_neon(const float *dataf, const float *weightsf, uint8_t *d) {
const int16_t *data = (const int16_t *)dataf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 128/2; i += 8) {
int16x4x2_t d0 = vld2_s16(data + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 512/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 528/4));
float32x4_t m1, m2, m3, m4;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 544/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + 560/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + 576/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + 592/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + 608/4));
uint32x4_t gte = vcgeq_f32(m1, zeroes_f);
uint16x4_t gte_u16 = vmovn_u32(gte);
uint8x8_t gte_u8 = vmovn_u16(vcombine_u16(gte_u16, vget_low_u16(vreinterpretq_u16_u32(sign_bits_f))));
gte_u8 = vshr_n_u8(gte_u8, 7);
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(gte_u8), 0);
}
void computeNetwork0_i16_neon(const float *inputf, const float *weightsf, uint8_t *d) {
const int16_t *input = (const int16_t *)inputf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 96/2; i += 8) {
int16x4x2_t d0 = vld2_s16(input + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 384/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 400/4));
float32x4_t m1, m2, m3, m4, m5, m6, m7;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 416/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (416+16)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (416+32)/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (416+48)/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + (416+64)/4));
m7 = m1;
m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f));
m1 = vaddq_f32(m1, ones_f);
m7 = vmulq_f32(reciprocal(m1), m7);
m3 = m0;
m0 = vdupq_lane_f32(vget_low_f32(m0), 0);
m1 = vdupq_lane_f32(vget_low_f32(m3), 1);
m2 = vdupq_lane_f32(vget_high_f32(m3), 0);
m3 = vdupq_lane_f32(vget_high_f32(m3), 1);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 496/4));
m1 = vmulq_f32(m1, vld1q_f32(weightsf + (496+16)/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (496+32)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (496+48)/4));
m4 = vdupq_lane_f32(vget_low_f32(m7), 0);
m5 = vdupq_lane_f32(vget_low_f32(m7), 1);
m6 = vdupq_lane_f32(vget_high_f32(m7), 0);
m7 = vdupq_lane_f32(vget_high_f32(m7), 1);
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (496+64)/4));
m5 = vmulq_f32(m5, vld1q_f32(weightsf + (496+80)/4));
m6 = vmulq_f32(m6, vld1q_f32(weightsf + (496+96)/4));
m7 = vmulq_f32(m7, vld1q_f32(weightsf + (496+112)/4));
m0 = vaddq_f32(m0, m1);
m2 = vaddq_f32(m2, m3);
m4 = vaddq_f32(m4, m5);
m6 = vaddq_f32(m6, m7);
m0 = vaddq_f32(m0, m2);
m4 = vaddq_f32(m4, m6);
m0 = vaddq_f32(m0, m4);
m0 = vaddq_f32(m0, vld1q_f32(weightsf + (496+128)/4));
float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0));
d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0));
}
int32x2_t
test_vpadd_s32 (int32x2_t __a, int32x2_t __b)
{
return vpadd_s32(__a, __b);
}
int rotate_cpx_vector(int16_t *x,
int16_t *alpha,
int16_t *y,
uint32_t N,
uint16_t output_shift)
{
// Multiply elementwise two complex vectors of N elements
// x - input 1 in the format |Re0 Im0 |,......,|Re(N-1) Im(N-1)|
// We assume x1 with a dynamic of 15 bit maximum
//
// alpha - input 2 in the format |Re0 Im0|
// We assume x2 with a dynamic of 15 bit maximum
//
// y - output in the format |Re0 Im0|,......,|Re(N-1) Im(N-1)|
//
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// log2_amp - increase the output amplitude by a factor 2^log2_amp (default is 0)
// WARNING: log2_amp>0 can cause overflow!!
uint32_t i; // loop counter
simd_q15_t *y_128,alpha_128;
int32_t *xd=(int32_t *)x;
#if defined(__x86_64__) || defined(__i386__)
__m128i shift = _mm_cvtsi32_si128(output_shift);
register simd_q15_t m0,m1,m2,m3;
((int16_t *)&alpha_128)[0] = alpha[0];
((int16_t *)&alpha_128)[1] = -alpha[1];
((int16_t *)&alpha_128)[2] = alpha[1];
((int16_t *)&alpha_128)[3] = alpha[0];
((int16_t *)&alpha_128)[4] = alpha[0];
((int16_t *)&alpha_128)[5] = -alpha[1];
((int16_t *)&alpha_128)[6] = alpha[1];
((int16_t *)&alpha_128)[7] = alpha[0];
#elif defined(__arm__)
int32x4_t shift;
int32x4_t ab_re0,ab_re1,ab_im0,ab_im1,re32,im32;
int16_t reflip[8] __attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1};
int32x4x2_t xtmp;
((int16_t *)&alpha_128)[0] = alpha[0];
((int16_t *)&alpha_128)[1] = alpha[1];
((int16_t *)&alpha_128)[2] = alpha[0];
((int16_t *)&alpha_128)[3] = alpha[1];
((int16_t *)&alpha_128)[4] = alpha[0];
((int16_t *)&alpha_128)[5] = alpha[1];
((int16_t *)&alpha_128)[6] = alpha[0];
((int16_t *)&alpha_128)[7] = alpha[1];
int16x8_t bflip = vrev32q_s16(alpha_128);
int16x8_t bconj = vmulq_s16(alpha_128,*(int16x8_t *)reflip);
shift = vdupq_n_s32(-output_shift);
#endif
y_128 = (simd_q15_t *) y;
for(i=0; i<N>>2; i++) {
#if defined(__x86_64__) || defined(__i386__)
m0 = _mm_setr_epi32(xd[0],xd[0],xd[1],xd[1]);
m1 = _mm_setr_epi32(xd[2],xd[2],xd[3],xd[3]);
m2 = _mm_madd_epi16(m0,alpha_128); //complex multiply. result is 32bit [Re Im Re Im]
m3 = _mm_madd_epi16(m1,alpha_128); //complex multiply. result is 32bit [Re Im Re Im]
m2 = _mm_sra_epi32(m2,shift); // shift right by shift in order to compensate for the input amplitude
m3 = _mm_sra_epi32(m3,shift); // shift right by shift in order to compensate for the input amplitude
y_128[0] = _mm_packs_epi32(m2,m3); // pack in 16bit integers with saturation [re im re im re im re im]
#elif defined(__arm__)
ab_re0 = vmull_s16(((int16x4_t*)xd)[0],((int16x4_t*)&bconj)[0]);
ab_re1 = vmull_s16(((int16x4_t*)xd)[1],((int16x4_t*)&bconj)[1]);
ab_im0 = vmull_s16(((int16x4_t*)xd)[0],((int16x4_t*)&bflip)[0]);
ab_im1 = vmull_s16(((int16x4_t*)xd)[1],((int16x4_t*)&bflip)[1]);
re32 = vshlq_s32(vcombine_s32(vpadd_s32(((int32x2_t*)&ab_re0)[0],((int32x2_t*)&ab_re0)[1]),
vpadd_s32(((int32x2_t*)&ab_re1)[0],((int32x2_t*)&ab_re1)[1])),
shift);
im32 = vshlq_s32(vcombine_s32(vpadd_s32(((int32x2_t*)&ab_im0)[0],((int32x2_t*)&ab_im0)[1]),
vpadd_s32(((int32x2_t*)&ab_im1)[0],((int32x2_t*)&ab_im1)[1])),
shift);
xtmp = vzipq_s32(re32,im32);
y_128[0] = vcombine_s16(vmovn_s32(xtmp.val[0]),vmovn_s32(xtmp.val[1]));
#endif
xd+=4;
y_128+=1;
}
_mm_empty();
_m_empty();
return(0);
}
int mult_cpx_conj_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
int output_shift,
int madd)
{
// Multiply elementwise the complex conjugate of x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x1 with a dinamic of 15 bit maximum
//
// x2 - input 2 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
// We assume x2 with a dinamic of 14 bit maximum
///
// y - output in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
//
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// output_shift - shift to be applied to generate output
//
// madd - add the output to y
uint32_t i; // loop counter
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
#if defined(__x86_64__) || defined(__i386__)
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
#elif defined(__arm__)
int32x4_t tmp_re,tmp_im;
int32x4_t tmp_re1,tmp_im1;
int16x4x2_t tmpy;
int32x4_t shift = vdupq_n_s32(-output_shift);
#endif
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
#if defined(__x86_64__) || defined(__i386__)
tmp_re = _mm_madd_epi16(*x1_128,*x2_128);
tmp_im = _mm_shufflelo_epi16(*x1_128,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_shufflehi_epi16(tmp_im,_MM_SHUFFLE(2,3,0,1));
tmp_im = _mm_sign_epi16(tmp_im,*(__m128i*)&conjug[0]);
tmp_im = _mm_madd_epi16(tmp_im,*x2_128);
tmp_re = _mm_srai_epi32(tmp_re,output_shift);
tmp_im = _mm_srai_epi32(tmp_im,output_shift);
tmpy0 = _mm_unpacklo_epi32(tmp_re,tmp_im);
tmpy1 = _mm_unpackhi_epi32(tmp_re,tmp_im);
if (madd==0)
*y_128 = _mm_packs_epi32(tmpy0,tmpy1);
else
*y_128 += _mm_packs_epi32(tmpy0,tmpy1);
#elif defined(__arm__)
tmp_re = vmull_s16(((simdshort_q15_t *)x1_128)[0], ((simdshort_q15_t*)x2_128)[0]);
//tmp_re = [Re(x1[0])Re(x2[0]) Im(x1[0])Im(x2[0]) Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1])]
tmp_re1 = vmull_s16(((simdshort_q15_t *)x1_128)[1], ((simdshort_q15_t*)x2_128)[1]);
//tmp_re1 = [Re(x1[1])Re(x2[1]) Im(x1[1])Im(x2[1]) Re(x1[1])Re(x2[2]) Im(x1[1])Im(x2[2])]
tmp_re = vcombine_s32(vpadd_s32(vget_low_s32(tmp_re),vget_high_s32(tmp_re)),
vpadd_s32(vget_low_s32(tmp_re1),vget_high_s32(tmp_re1)));
//tmp_re = [Re(ch[0])Re(rx[0])+Im(ch[0])Im(ch[0]) Re(ch[1])Re(rx[1])+Im(ch[1])Im(ch[1]) Re(ch[2])Re(rx[2])+Im(ch[2]) Im(ch[2]) Re(ch[3])Re(rx[3])+Im(ch[3])Im(ch[3])]
tmp_im = vmull_s16(vrev32_s16(vmul_s16(((simdshort_q15_t*)x2_128)[0],*(simdshort_q15_t*)conjug)), ((simdshort_q15_t*)x1_128)[0]);
//tmp_im = [-Im(ch[0])Re(rx[0]) Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1]) Re(ch[1])Im(rx[1])]
tmp_im1 = vmull_s16(vrev32_s16(vmul_s16(((simdshort_q15_t*)x2_128)[1],*(simdshort_q15_t*)conjug)), ((simdshort_q15_t*)x1_128)[1]);
//tmp_im1 = [-Im(ch[2])Re(rx[2]) Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3]) Re(ch[3])Im(rx[3])]
tmp_im = vcombine_s32(vpadd_s32(vget_low_s32(tmp_im),vget_high_s32(tmp_im)),
vpadd_s32(vget_low_s32(tmp_im1),vget_high_s32(tmp_im1)));
//tmp_im = [-Im(ch[0])Re(rx[0])+Re(ch[0])Im(rx[0]) -Im(ch[1])Re(rx[1])+Re(ch[1])Im(rx[1]) -Im(ch[2])Re(rx[2])+Re(ch[2])Im(rx[2]) -Im(ch[3])Re(rx[3])+Re(ch[3])Im(rx[3])]
tmp_re = vqshlq_s32(tmp_re,shift);
tmp_im = vqshlq_s32(tmp_im,shift);
tmpy = vzip_s16(vmovn_s32(tmp_re),vmovn_s32(tmp_im));
if (madd==0)
*y_128 = vcombine_s16(tmpy.val[0],tmpy.val[1]);
else
*y_128 += vcombine_s16(tmpy.val[0],tmpy.val[1]);
#endif
x1_128++;
x2_128++;
y_128++;
}
_mm_empty();
_m_empty();
return(0);
}
