void test_vcombinef32 (void) { float32x4_t out_float32x4_t; float32x2_t arg0_float32x2_t; float32x2_t arg1_float32x2_t; out_float32x4_t = vcombine_f32 (arg0_float32x2_t, arg1_float32x2_t); }
//__attribute__ ((noinline)) inline vector<float, 4> cmin(vector<float, 4> const& v) noexcept { using vector_type = typename vector_traits<float, 4>::vector_type; auto tmp(vpmin_f32(vget_low_f32(float32x4_t(v.data_)), vget_high_f32(float32x4_t(v.data_)))); tmp = vpmin_f32(tmp, tmp); return { vector_type(vcombine_f32(tmp, tmp)) }; }
static void cft1st_128_neon(float* a) { const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign); int j, k2; for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) { float32x4_t a00v = vld1q_f32(&a[j + 0]); float32x4_t a04v = vld1q_f32(&a[j + 4]); float32x4_t a08v = vld1q_f32(&a[j + 8]); float32x4_t a12v = vld1q_f32(&a[j + 12]); float32x4_t a01v = vcombine_f32(vget_low_f32(a00v), vget_low_f32(a08v)); float32x4_t a23v = vcombine_f32(vget_high_f32(a00v), vget_high_f32(a08v)); float32x4_t a45v = vcombine_f32(vget_low_f32(a04v), vget_low_f32(a12v)); float32x4_t a67v = vcombine_f32(vget_high_f32(a04v), vget_high_f32(a12v)); const float32x4_t wk1rv = vld1q_f32(&rdft_wk1r[k2]); const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2]); const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2]); const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2]); const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2]); const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2]); float32x4_t x0v = vaddq_f32(a01v, a23v); const float32x4_t x1v = vsubq_f32(a01v, a23v); const float32x4_t x2v = vaddq_f32(a45v, a67v); const float32x4_t x3v = vsubq_f32(a45v, a67v); const float32x4_t x3w = vrev64q_f32(x3v); float32x4_t x0w; a01v = vaddq_f32(x0v, x2v); x0v = vsubq_f32(x0v, x2v); x0w = vrev64q_f32(x0v); a45v = vmulq_f32(wk2rv, x0v); a45v = vmlaq_f32(a45v, wk2iv, x0w); x0v = vmlaq_f32(x1v, x3w, vec_swap_sign); x0w = vrev64q_f32(x0v); a23v = vmulq_f32(wk1rv, x0v); a23v = vmlaq_f32(a23v, wk1iv, x0w); x0v = vmlsq_f32(x1v, x3w, vec_swap_sign); x0w = vrev64q_f32(x0v); a67v = vmulq_f32(wk3rv, x0v); a67v = vmlaq_f32(a67v, wk3iv, x0w); a00v = vcombine_f32(vget_low_f32(a01v), vget_low_f32(a23v)); a04v = vcombine_f32(vget_low_f32(a45v), vget_low_f32(a67v)); a08v = vcombine_f32(vget_high_f32(a01v), vget_high_f32(a23v)); a12v = vcombine_f32(vget_high_f32(a45v), vget_high_f32(a67v)); vst1q_f32(&a[j + 0], a00v); vst1q_f32(&a[j + 4], a04v); vst1q_f32(&a[j + 8], a08v); vst1q_f32(&a[j + 12], a12v); } }
// Window time domain data to be used by the fft. static void WindowDataNEON(float* x_windowed, const float* x) { int i; for (i = 0; i < PART_LEN; i += 4) { const float32x4_t vec_Buf1 = vld1q_f32(&x[i]); const float32x4_t vec_Buf2 = vld1q_f32(&x[PART_LEN + i]); const float32x4_t vec_sqrtHanning = vld1q_f32(&WebRtcAec_sqrtHanning[i]); // A B C D float32x4_t vec_sqrtHanning_rev = vld1q_f32(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]); // B A D C vec_sqrtHanning_rev = vrev64q_f32(vec_sqrtHanning_rev); // D C B A vec_sqrtHanning_rev = vcombine_f32(vget_high_f32(vec_sqrtHanning_rev), vget_low_f32(vec_sqrtHanning_rev)); vst1q_f32(&x_windowed[i], vmulq_f32(vec_Buf1, vec_sqrtHanning)); vst1q_f32(&x_windowed[PART_LEN + i], vmulq_f32(vec_Buf2, vec_sqrtHanning_rev)); } }
void dotProd_neon(const float *data, const float *weights, float *vals, const int n, const int len, const float *istd) { for (int i = 0; i < n; i += 4) { float32x4_t accum0 = { 0.0f, 0.0f, 0.0f, 0.0f }; float32x4_t accum1 = accum0; float32x4_t accum2 = accum0; float32x4_t accum3 = accum0; for (int j = 0; j < len; j += 4) { float32x4_t d0 = vld1q_f32(data + j); float32x4_t d1 = d0; float32x4_t d2 = d0; float32x4_t d3 = d0; float32x4_t w0 = vld1q_f32(weights); float32x4_t w1 = vld1q_f32(weights + 4); float32x4_t w2 = vld1q_f32(weights + 8); float32x4_t w3 = vld1q_f32(weights + 12); accum0 = vaddq_f32(accum0, vmulq_f32(d0, w0)); accum1 = vaddq_f32(accum1, vmulq_f32(d1, w1)); accum2 = vaddq_f32(accum2, vmulq_f32(d2, w2)); accum3 = vaddq_f32(accum3, vmulq_f32(d3, w3)); weights += 16; } float32x2_t sum0 = vpadd_f32(vget_low_f32(accum0), vget_high_f32(accum0)); float32x2_t sum1 = vpadd_f32(vget_low_f32(accum1), vget_high_f32(accum1)); float32x2_t sum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2)); float32x2_t sum3 = vpadd_f32(vget_low_f32(accum3), vget_high_f32(accum3)); sum0 = vpadd_f32(sum0, sum1); sum1 = vpadd_f32(sum2, sum3); float32x4_t sum = vcombine_f32(sum0, sum1); sum = vmulq_n_f32(sum, istd[0]); sum = vaddq_f32(sum, vld1q_f32(weights + n*len + i)); vst1q_f32(vals + i, sum); } }
__inline static float32x4_t reverse_order_f32x4(float32x4_t in) { // A B C D -> C D A B const float32x4_t rev = vcombine_f32(vget_high_f32(in), vget_low_f32(in)); // C D A B -> D C B A return vrev64q_f32(rev); }
// CHECK-LABEL: define <4 x float> @test_vcombine_f32(<2 x float> %low, <2 x float> %high) #0 { // CHECK: [[SHUFFLE_I:%.*]] = shufflevector <2 x float> %low, <2 x float> %high, <4 x i32> <i32 0, i32 1, i32 2, i32 3> // CHECK: ret <4 x float> [[SHUFFLE_I]] float32x4_t test_vcombine_f32(float32x2_t low, float32x2_t high) { return vcombine_f32(low, high); }
void fft_real_neon( CkFftContext* context, const float* input, CkFftComplex* output, int count) { int countDiv2 = count/2; fft_neon(context, (const CkFftComplex*) input, output, countDiv2, false, 1, context->fwdExpTable, context->maxCount / countDiv2); output[countDiv2] = output[0]; int expTableStride = context->maxCount/count; const CkFftComplex* exp0 = context->fwdExpTable; const CkFftComplex* exp1 = context->fwdExpTable + countDiv2 * expTableStride; CkFftComplex* p0 = output; CkFftComplex* p1 = output + countDiv2 - 3; const CkFftComplex* pEnd = p0 + count/4; while (p0 < pEnd) { float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0); float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1); float32x2_t hi, lo; // reverse z1 real z1_v.val[0] = vrev64q_f32(z1_v.val[0]); hi = vget_high_f32(z1_v.val[0]); lo = vget_low_f32(z1_v.val[0]); z1_v.val[0] = vcombine_f32(hi, lo); // reverse z1 imaginary z1_v.val[1] = vrev64q_f32(z1_v.val[1]); hi = vget_high_f32(z1_v.val[1]); lo = vget_low_f32(z1_v.val[1]); z1_v.val[1] = vcombine_f32(hi, lo); float32x4x2_t sum_v; sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]); sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]); float32x4x2_t diff_v; diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]); diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]); float32x4x2_t exp_v; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3); exp0 += expTableStride; float32x4x2_t f_v; f_v.val[0] = vnegq_f32(exp_v.val[1]); f_v.val[1] = exp_v.val[0]; float32x4x2_t c_v; multiply(f_v, diff_v, c_v); subtract(sum_v, c_v, z0_v); vst2q_f32((float32_t*) p0, z0_v); diff_v.val[0] = vnegq_f32(diff_v.val[0]); sum_v.val[1] = vnegq_f32(sum_v.val[1]); exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3); exp1 -= expTableStride; f_v.val[0] = vnegq_f32(exp_v.val[1]); f_v.val[1] = exp_v.val[0]; multiply(f_v, diff_v, c_v); subtract(sum_v, c_v, z1_v); // reverse z1 real z1_v.val[0] = vrev64q_f32(z1_v.val[0]); hi = vget_high_f32(z1_v.val[0]); lo = vget_low_f32(z1_v.val[0]); z1_v.val[0] = vcombine_f32(hi, lo); // reverse z1 imaginary z1_v.val[1] = vrev64q_f32(z1_v.val[1]); hi = vget_high_f32(z1_v.val[1]); lo = vget_low_f32(z1_v.val[1]); z1_v.val[1] = vcombine_f32(hi, lo); vst2q_f32((float32_t*) p1, z1_v); p0 += 4; p1 -= 4; } if (count > 8) { // middle: p0->real = p0->real * 2.0f; p0->imag = -p0->imag * 2.0f; } }
void fft_real_inverse_neon( CkFftContext* context, const CkFftComplex* input, float* output, int count, CkFftComplex* tmpBuf) { int countDiv2 = count/2; int expTableStride = context->maxCount/count; const CkFftComplex* exp0 = context->invExpTable; const CkFftComplex* exp1 = context->invExpTable + countDiv2 * expTableStride; const CkFftComplex* p0 = input; const CkFftComplex* p1 = input + countDiv2 - 3; CkFftComplex* tmp0 = tmpBuf; CkFftComplex* tmp1 = tmpBuf + countDiv2 - 3; const CkFftComplex* pEnd = p0 + count/4; while (p0 < pEnd) { float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0); float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1); float32x2_t hi, lo; // reverse z1 real z1_v.val[0] = vrev64q_f32(z1_v.val[0]); hi = vget_high_f32(z1_v.val[0]); lo = vget_low_f32(z1_v.val[0]); z1_v.val[0] = vcombine_f32(hi, lo); // reverse z1 imaginary z1_v.val[1] = vrev64q_f32(z1_v.val[1]); hi = vget_high_f32(z1_v.val[1]); lo = vget_low_f32(z1_v.val[1]); z1_v.val[1] = vcombine_f32(hi, lo); float32x4x2_t sum_v; sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]); sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]); float32x4x2_t diff_v; diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]); diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]); float32x4x2_t exp_v; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2); exp0 += expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3); exp0 += expTableStride; float32x4x2_t f_v; f_v.val[0] = vnegq_f32(exp_v.val[1]); f_v.val[1] = exp_v.val[0]; float32x4x2_t c_v; multiply(f_v, diff_v, c_v); add(sum_v, c_v, z0_v); vst2q_f32((float32_t*) tmp0, z0_v); diff_v.val[0] = vnegq_f32(diff_v.val[0]); sum_v.val[1] = vnegq_f32(sum_v.val[1]); exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2); exp1 -= expTableStride; exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3); exp1 -= expTableStride; f_v.val[0] = vnegq_f32(exp_v.val[1]); f_v.val[1] = exp_v.val[0]; multiply(f_v, diff_v, c_v); add(sum_v, c_v, z1_v); // reverse z1 real z1_v.val[0] = vrev64q_f32(z1_v.val[0]); hi = vget_high_f32(z1_v.val[0]); lo = vget_low_f32(z1_v.val[0]); z1_v.val[0] = vcombine_f32(hi, lo); // reverse z1 imaginary z1_v.val[1] = vrev64q_f32(z1_v.val[1]); hi = vget_high_f32(z1_v.val[1]); lo = vget_low_f32(z1_v.val[1]); z1_v.val[1] = vcombine_f32(hi, lo); vst2q_f32((float32_t*) tmp1, z1_v); p0 += 4; tmp0 += 4; p1 -= 4; tmp1 -= 4; } // middle: tmp0->real = p0->real * 2.0f; tmp0->imag = -p0->imag * 2.0f; fft_neon(context, tmpBuf, (CkFftComplex*) output, countDiv2, true, 1, context->invExpTable, context->maxCount / countDiv2); }
void computeNetwork0_neon(const float *input, const float *weights, uint8_t *d) { float32x4_t m0 = { 0.0f, 0.0f, 0.0f, 0.0f }; float32x4_t m1 = m0; float32x4_t m2 = m0; float32x4_t m3 = m0; float32x4_t m4, m5, m6, m7; for (int i = 0; i < 192/4; i += 4) { m4 = vld1q_f32(input + i); m5 = m4; m6 = m4; m7 = m4; m4 = vmulq_f32(m4, vld1q_f32(weights + i * 4)); m5 = vmulq_f32(m5, vld1q_f32(weights + i * 4 + 4)); m6 = vmulq_f32(m6, vld1q_f32(weights + i * 4 + 8)); m7 = vmulq_f32(m7, vld1q_f32(weights + i * 4 + 12)); m0 = vaddq_f32(m0, m4); m1 = vaddq_f32(m1, m5); m2 = vaddq_f32(m2, m6); m3 = vaddq_f32(m3, m7); } float32x2_t sum0 = vpadd_f32(vget_low_f32(m0), vget_high_f32(m0)); float32x2_t sum1 = vpadd_f32(vget_low_f32(m1), vget_high_f32(m1)); float32x2_t sum2 = vpadd_f32(vget_low_f32(m2), vget_high_f32(m2)); float32x2_t sum3 = vpadd_f32(vget_low_f32(m3), vget_high_f32(m3)); sum0 = vpadd_f32(sum0, sum1); sum1 = vpadd_f32(sum2, sum3); m0 = vcombine_f32(sum0, sum1); m0 = vaddq_f32(m0, vld1q_f32(weights + 768/4)); m1 = m0; m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l)); m0 = vaddq_f32(m0, ones_f); m0 = vmulq_f32(reciprocal(m0), m1); m1 = vdupq_lane_f32(vget_low_f32(m0), 0); m2 = vdupq_lane_f32(vget_low_f32(m0), 1); m3 = vdupq_lane_f32(vget_high_f32(m0), 0); m4 = vdupq_lane_f32(vget_high_f32(m0), 1); m1 = vmulq_f32(m1, vld1q_f32(weights + 784/4)); m2 = vmulq_f32(m2, vld1q_f32(weights + (784+16)/4)); m3 = vmulq_f32(m3, vld1q_f32(weights + (784+32)/4)); m4 = vmulq_f32(m4, vld1q_f32(weights + (784+48)/4)); m1 = vaddq_f32(m1, m2); m3 = vaddq_f32(m3, m4); m1 = vaddq_f32(m1, m3); m1 = vaddq_f32(m1, vld1q_f32(weights + (784+64)/4)); m7 = m1; m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f)); m1 = vaddq_f32(m1, ones_f); m7 = vmulq_f32(reciprocal(m1), m7); m3 = m0; m0 = vdupq_lane_f32(vget_low_f32(m0), 0); m1 = vdupq_lane_f32(vget_low_f32(m3), 1); m2 = vdupq_lane_f32(vget_high_f32(m3), 0); m3 = vdupq_lane_f32(vget_high_f32(m3), 1); m0 = vmulq_f32(m0, vld1q_f32(weights + 864/4)); m1 = vmulq_f32(m1, vld1q_f32(weights + (864+16)/4)); m2 = vmulq_f32(m2, vld1q_f32(weights + (864+32)/4)); m3 = vmulq_f32(m3, vld1q_f32(weights + (864+48)/4)); m4 = vdupq_lane_f32(vget_low_f32(m7), 0); m5 = vdupq_lane_f32(vget_low_f32(m7), 1); m6 = vdupq_lane_f32(vget_high_f32(m7), 0); m7 = vdupq_lane_f32(vget_high_f32(m7), 1); m4 = vmulq_f32(m4, vld1q_f32(weights + (864+64)/4)); m5 = vmulq_f32(m5, vld1q_f32(weights + (864+80)/4)); m6 = vmulq_f32(m6, vld1q_f32(weights + (864+96)/4)); m7 = vmulq_f32(m7, vld1q_f32(weights + (864+112)/4)); m0 = vaddq_f32(m0, m1); m2 = vaddq_f32(m2, m3); m4 = vaddq_f32(m4, m5); m6 = vaddq_f32(m6, m7); m0 = vaddq_f32(m0, m2); m4 = vaddq_f32(m4, m6); m0 = vaddq_f32(m0, m4); m0 = vaddq_f32(m0, vld1q_f32(weights + (864+128)/4)); float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0)); d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0)); }
/** * @brief vector_mul_matrix. * * @param src1[in] the input vector(1*k) * src2[in] the input matrix(k*n) * dst[out] the output vector(1*n) * kn[in] DIM_K & DIM_N * * @return void */ void neon_vectormulmatrix_float(float * dst, const float * src1, const float * src2, int *kn) { int j ,l; int k = kn[0]; int n = kn[1]; const float * src1_p = src1; const float * src2_p = src2; float * dst_p = dst; for (j = 0; j <= n - 4; j += 4) { float32x2_t d16 = {0}; float32x2_t d17 = {0}; float32x2_t d18 = {0}; float32x2_t d19 = {0}; float32x2_t d20; float32x2_t d21; float32x4_t q0; src1_p = src1; src2_p = src2 + j * k; for (l = 0; l <= k - 4; l += 4) { // Matrix A float32x4_t q8 = vld1q_f32(src1_p); float32x2_t d0 = vget_low_f32(q8); float32x2_t d1 = vget_high_f32(q8); // Matrix B float32x4_t q12 = vld1q_f32(src2_p); float32x4_t q13 = vld1q_f32(src2_p + k); float32x4_t q14 = vld1q_f32(src2_p + k * 2); float32x4_t q15 = vld1q_f32(src2_p + k * 3); float32x2_t d8 = vget_low_f32(q12); float32x2_t d9 = vget_high_f32(q12); float32x2_t d10 = vget_low_f32(q13); float32x2_t d11 = vget_high_f32(q13); float32x2_t d12 = vget_low_f32(q14); float32x2_t d13 = vget_high_f32(q14); float32x2_t d14 = vget_low_f32(q15); float32x2_t d15 = vget_high_f32(q15); d16 = vmla_f32(d16, d0, d8); d17 = vmla_f32(d17, d0, d10); d18 = vmla_f32(d18, d0, d12); d19 = vmla_f32(d19, d0, d14); d16 = vmla_f32(d16, d1, d9); d17 = vmla_f32(d17, d1, d11); d18 = vmla_f32(d18, d1, d13); d19 = vmla_f32(d19, d1, d15); src1_p += 4; src2_p += 4; }// end for l d16 = vpadd_f32(d16, d17); d18 = vpadd_f32(d18, d19); float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; for(; l < k; l ++) { float src1_d; src1_d = *src1_p; sum0 += src1_d * *src2_p; sum1 += src1_d * *(src2_p + k); sum2 += src1_d * *(src2_p + 2 * k); sum3 += src1_d * *(src2_p + 3 * k); src1_p++; src2_p++; } d20 = vset_lane_f32(sum0, d20, 0); d20 = vset_lane_f32(sum1, d20, 1); d21 = vset_lane_f32(sum2, d21, 0); d21 = vset_lane_f32(sum3, d21, 1); q0 = vaddq_f32(vcombine_f32(d16, d18), vcombine_f32(d20, d21)); vst1q_f32(dst_p, q0); dst_p += 4; }// end for j }
static void ne10_fft_split_c2r_1d_float32_neon (ne10_fft_cpx_float32_t *dst, const ne10_fft_cpx_float32_t *src, ne10_fft_cpx_float32_t *twiddles, ne10_int32_t ncfft) { ne10_int32_t k; ne10_int32_t count = ncfft / 2; ne10_fft_cpx_float32_t fk, fnkc, fek, fok, tmp; float32x4x2_t q2_fk, q2_fnkc, q2_tw, q2_dst, q2_dst2; float32x4_t q_fnkc_r, q_fnkc_i; float32x4_t q_fek_r, q_fek_i, q_fok_r, q_fok_i; float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val; float32x4_t q_dst2_r, q_dst2_i; float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles; dst[0].r = (src[0].r + src[ncfft].r) * 0.5f; dst[0].i = (src[0].r - src[ncfft].r) * 0.5f; if (count >= 4) { for (k = 1; k <= count ; k += 4) { p_src = (float32_t*) (& (src[k])); p_src2 = (float32_t*) (& (src[ncfft - k - 3])); p_twiddles = (float32_t*) (& (twiddles[k - 1])); p_dst = (float32_t*) (& (dst[k])); p_dst2 = (float32_t*) (& (dst[ncfft - k - 3])); q2_fk = vld2q_f32 (p_src); q2_fnkc = vld2q_f32 (p_src2); q2_tw = vld2q_f32 (p_twiddles); q2_fnkc.val[0] = vrev64q_f32 (q2_fnkc.val[0]); q2_fnkc.val[1] = vrev64q_f32 (q2_fnkc.val[1]); q_fnkc_r = vcombine_f32 (vget_high_f32 (q2_fnkc.val[0]), vget_low_f32 (q2_fnkc.val[0])); q_fnkc_i = vcombine_f32 (vget_high_f32 (q2_fnkc.val[1]), vget_low_f32 (q2_fnkc.val[1])); q_fnkc_i = vnegq_f32 (q_fnkc_i); q_fek_r = vaddq_f32 (q2_fk.val[0], q_fnkc_r); q_fek_i = vaddq_f32 (q2_fk.val[1], q_fnkc_i); q_tmp0 = vsubq_f32 (q2_fk.val[0], q_fnkc_r); q_tmp1 = vsubq_f32 (q2_fk.val[1], q_fnkc_i); q_fok_r = vmulq_f32 (q_tmp0, q2_tw.val[0]); q_fok_i = vmulq_f32 (q_tmp1, q2_tw.val[0]); q_tmp2 = vmulq_f32 (q_tmp1, q2_tw.val[1]); q_tmp3 = vmulq_f32 (q_tmp0, q2_tw.val[1]); q_fok_r = vaddq_f32 (q_fok_r, q_tmp2); q_fok_i = vsubq_f32 (q_fok_i, q_tmp3); q_val = vdupq_n_f32 (0.5f); q_dst2_r = vsubq_f32 (q_fek_r, q_fok_r); q_dst2_i = vsubq_f32 (q_fok_i, q_fek_i); q2_dst.val[0] = vaddq_f32 (q_fek_r, q_fok_r); q2_dst.val[1] = vaddq_f32 (q_fek_i, q_fok_i); q_dst2_r = vmulq_f32 (q_dst2_r, q_val); q_dst2_i = vmulq_f32 (q_dst2_i, q_val); q2_dst.val[0] = vmulq_f32 (q2_dst.val[0], q_val); q2_dst.val[1] = vmulq_f32 (q2_dst.val[1], q_val); q_dst2_r = vrev64q_f32 (q_dst2_r); q_dst2_i = vrev64q_f32 (q_dst2_i); q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r)); q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i)); vst2q_f32 (p_dst, q2_dst); vst2q_f32 (p_dst2, q2_dst2); } } else { for (k = 1; k <= count ; k++) { fk = src[k]; fnkc.r = src[ncfft - k].r; fnkc.i = -src[ncfft - k].i; fek.r = fk.r + fnkc.r; fek.i = fk.i + fnkc.i; tmp.r = fk.r - fnkc.r; tmp.i = fk.i - fnkc.i; fok.r = tmp.r * twiddles[k - 1].r + tmp.i * twiddles[k - 1].i; fok.i = tmp.i * twiddles[k - 1].r - tmp.r * twiddles[k - 1].i; dst[k].r = (fek.r + fok.r) * 0.5f; dst[k].i = (fek.i + fok.i) * 0.5f; dst[ncfft - k].r = (fek.r - fok.r) * 0.5f; dst[ncfft - k].i = (fok.i - fek.i) * 0.5f; } } }
static void ne10_fft_split_r2c_1d_float32_neon (ne10_fft_cpx_float32_t *dst, const ne10_fft_cpx_float32_t *src, ne10_fft_cpx_float32_t *twiddles, ne10_int32_t ncfft) { ne10_int32_t k; ne10_int32_t count = ncfft / 2; ne10_fft_cpx_float32_t fpnk, fpk, f1k, f2k, tw, tdc; float32x4x2_t q2_fpk, q2_fpnk, q2_tw, q2_dst, q2_dst2; float32x4_t q_fpnk_r, q_fpnk_i; float32x4_t q_f1k_r, q_f1k_i, q_f2k_r, q_f2k_i; float32x4_t q_tw_r, q_tw_i; float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val; float32x4_t q_dst_r, q_dst_i, q_dst2_r, q_dst2_i; float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles; tdc.r = src[0].r; tdc.i = src[0].i; dst[0].r = tdc.r + tdc.i; dst[ncfft].r = tdc.r - tdc.i; dst[ncfft].i = dst[0].i = 0; if (count >= 4) { for (k = 1; k <= count ; k += 4) { p_src = (float32_t*) (& (src[k])); p_src2 = (float32_t*) (& (src[ncfft - k - 3])); p_twiddles = (float32_t*) (& (twiddles[k - 1])); p_dst = (float32_t*) (& (dst[k])); p_dst2 = (float32_t*) (& (dst[ncfft - k - 3])); q2_fpk = vld2q_f32 (p_src); q2_fpnk = vld2q_f32 (p_src2); q2_tw = vld2q_f32 (p_twiddles); q2_fpnk.val[0] = vrev64q_f32 (q2_fpnk.val[0]); q2_fpnk.val[1] = vrev64q_f32 (q2_fpnk.val[1]); q_fpnk_r = vcombine_f32 (vget_high_f32 (q2_fpnk.val[0]), vget_low_f32 (q2_fpnk.val[0])); q_fpnk_i = vcombine_f32 (vget_high_f32 (q2_fpnk.val[1]), vget_low_f32 (q2_fpnk.val[1])); q_fpnk_i = vnegq_f32 (q_fpnk_i); q_f1k_r = vaddq_f32 (q2_fpk.val[0], q_fpnk_r); q_f1k_i = vaddq_f32 (q2_fpk.val[1], q_fpnk_i); q_f2k_r = vsubq_f32 (q2_fpk.val[0], q_fpnk_r); q_f2k_i = vsubq_f32 (q2_fpk.val[1], q_fpnk_i); q_tmp0 = vmulq_f32 (q_f2k_r, q2_tw.val[0]); q_tmp1 = vmulq_f32 (q_f2k_i, q2_tw.val[1]); q_tmp2 = vmulq_f32 (q_f2k_r, q2_tw.val[1]); q_tmp3 = vmulq_f32 (q_f2k_i, q2_tw.val[0]); q_tw_r = vsubq_f32 (q_tmp0, q_tmp1); q_tw_i = vaddq_f32 (q_tmp2, q_tmp3); q_val = vdupq_n_f32 (0.5f); q_dst2_r = vsubq_f32 (q_f1k_r, q_tw_r); q_dst2_i = vsubq_f32 (q_tw_i, q_f1k_i); q_dst_r = vaddq_f32 (q_f1k_r, q_tw_r); q_dst_i = vaddq_f32 (q_f1k_i, q_tw_i); q_dst2_r = vmulq_f32 (q_dst2_r, q_val); q_dst2_i = vmulq_f32 (q_dst2_i, q_val); q2_dst.val[0] = vmulq_f32 (q_dst_r, q_val); q2_dst.val[1] = vmulq_f32 (q_dst_i, q_val); q_dst2_r = vrev64q_f32 (q_dst2_r); q_dst2_i = vrev64q_f32 (q_dst2_i); q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r)); q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i)); vst2q_f32 (p_dst, q2_dst); vst2q_f32 (p_dst2, q2_dst2); } } else { for (k = 1; k <= count ; k++) { fpk = src[k]; fpnk.r = src[ncfft - k].r; fpnk.i = - src[ncfft - k].i; f1k.r = fpk.r + fpnk.r; f1k.i = fpk.i + fpnk.i; f2k.r = fpk.r - fpnk.r; f2k.i = fpk.i - fpnk.i; tw.r = f2k.r * (twiddles[k - 1]).r - f2k.i * (twiddles[k - 1]).i; tw.i = f2k.r * (twiddles[k - 1]).i + f2k.i * (twiddles[k - 1]).r; dst[k].r = (f1k.r + tw.r) * 0.5f; dst[k].i = (f1k.i + tw.i) * 0.5f; dst[ncfft - k].r = (f1k.r - tw.r) * 0.5f; dst[ncfft - k].i = (tw.i - f1k.i) * 0.5f; } } }
static void ne10_fft16_backward_float32_neon (ne10_fft_cpx_float32_t * Fout, ne10_fft_cpx_float32_t * Fin, ne10_fft_cpx_float32_t * twiddles) { ne10_fft_cpx_float32_t *tw1, *tw2, *tw3; // the first stage float32_t *p_src0, *p_src4, *p_src8, *p_src12; float32x4x2_t q2_in_0123, q2_in_4567, q2_in_89ab, q2_in_cdef; float32x4_t q_t0_r, q_t0_i, q_t1_r, q_t1_i, q_t2_r, q_t2_i, q_t3_r, q_t3_i; float32x4_t q_out_r048c, q_out_i048c, q_out_r159d, q_out_i159d; float32x4_t q_out_r26ae, q_out_i26ae, q_out_r37bf, q_out_i37bf; p_src0 = (float32_t*) (& (Fin[0])); p_src4 = (float32_t*) (& (Fin[4])); p_src8 = (float32_t*) (& (Fin[8])); p_src12 = (float32_t*) (& (Fin[12])); q2_in_0123 = vld2q_f32 (p_src0); q2_in_4567 = vld2q_f32 (p_src4); q2_in_89ab = vld2q_f32 (p_src8); q2_in_cdef = vld2q_f32 (p_src12); q_t2_r = vsubq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]); q_t2_i = vsubq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]); q_t3_r = vaddq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]); q_t3_i = vaddq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]); q_t0_r = vaddq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]); q_t0_i = vaddq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]); q_t1_r = vsubq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]); q_t1_i = vsubq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]); q_out_r26ae = vsubq_f32 (q_t3_r, q_t0_r); q_out_i26ae = vsubq_f32 (q_t3_i, q_t0_i); q_out_r048c = vaddq_f32 (q_t3_r, q_t0_r); q_out_i048c = vaddq_f32 (q_t3_i, q_t0_i); q_out_r159d = vsubq_f32 (q_t2_r, q_t1_i); q_out_i159d = vaddq_f32 (q_t2_i, q_t1_r); q_out_r37bf = vaddq_f32 (q_t2_r, q_t1_i); q_out_i37bf = vsubq_f32 (q_t2_i, q_t1_r); // second stages float32_t *p_dst0, *p_dst1, *p_dst2, *p_dst3; float32_t *p_tw1, *p_tw2, *p_tw3; float32x4_t q_s0_r, q_s0_i, q_s1_r, q_s1_i, q_s2_r, q_s2_i; float32x4_t q_s3_r, q_s3_i, q_s4_r, q_s4_i, q_s5_r, q_s5_i; float32x4x2_t q2_tmp_0, q2_tmp_1, q2_tmp_2, q2_tmp_3; float32x4_t q_in_r0123, q_in_r4567, q_in_r89ab, q_in_rcdef; float32x4_t q_in_i0123, q_in_i4567, q_in_i89ab, q_in_icdef; float32x4x2_t q2_tw1, q2_tw2, q2_tw3; float32x4x2_t q2_out_0123, q2_out_4567, q2_out_89ab, q2_out_cdef; float32x4_t q_one_by_nfft; tw1 = twiddles; tw2 = twiddles + 4; tw3 = twiddles + 8; p_dst0 = (float32_t*) (&Fout[0]); p_dst1 = (float32_t*) (&Fout[4]); p_dst2 = (float32_t*) (&Fout[8]); p_dst3 = (float32_t*) (&Fout[12]); p_tw1 = (float32_t*) tw1; p_tw2 = (float32_t*) tw2; p_tw3 = (float32_t*) tw3; q2_tmp_0 = vzipq_f32 (q_out_r048c, q_out_r159d); q2_tmp_1 = vzipq_f32 (q_out_i048c, q_out_i159d); q2_tmp_2 = vzipq_f32 (q_out_r26ae, q_out_r37bf); q2_tmp_3 = vzipq_f32 (q_out_i26ae, q_out_i37bf); q_in_r0123 = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[0]), vget_low_f32 (q2_tmp_2.val[0])); q_in_i0123 = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[0]), vget_low_f32 (q2_tmp_3.val[0])); q_in_r4567 = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[0]), vget_high_f32 (q2_tmp_2.val[0])); q_in_i4567 = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[0]), vget_high_f32 (q2_tmp_3.val[0])); q_in_r89ab = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[1]), vget_low_f32 (q2_tmp_2.val[1])); q_in_i89ab = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[1]), vget_low_f32 (q2_tmp_3.val[1])); q_in_rcdef = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[1]), vget_high_f32 (q2_tmp_2.val[1])); q_in_icdef = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[1]), vget_high_f32 (q2_tmp_3.val[1])); q2_tw1 = vld2q_f32 (p_tw1); q2_tw2 = vld2q_f32 (p_tw2); q2_tw3 = vld2q_f32 (p_tw3); q_s0_r = vmulq_f32 (q_in_r4567, q2_tw1.val[0]); q_s0_i = vmulq_f32 (q_in_i4567, q2_tw1.val[0]); q_s1_r = vmulq_f32 (q_in_r89ab, q2_tw2.val[0]); q_s1_i = vmulq_f32 (q_in_i89ab, q2_tw2.val[0]); q_s2_r = vmulq_f32 (q_in_rcdef, q2_tw3.val[0]); q_s2_i = vmulq_f32 (q_in_icdef, q2_tw3.val[0]); q_s0_r = vmlaq_f32 (q_s0_r, q_in_i4567, q2_tw1.val[1]); q_s0_i = vmlsq_f32 (q_s0_i, q_in_r4567, q2_tw1.val[1]); q_s1_r = vmlaq_f32 (q_s1_r, q_in_i89ab, q2_tw2.val[1]); q_s1_i = vmlsq_f32 (q_s1_i, q_in_r89ab, q2_tw2.val[1]); q_s2_r = vmlaq_f32 (q_s2_r, q_in_icdef, q2_tw3.val[1]); q_s2_i = vmlsq_f32 (q_s2_i, q_in_rcdef, q2_tw3.val[1]); q_s5_r = vsubq_f32 (q_in_r0123, q_s1_r); q_s5_i = vsubq_f32 (q_in_i0123, q_s1_i); q2_out_0123.val[0] = vaddq_f32 (q_in_r0123, q_s1_r); q2_out_0123.val[1] = vaddq_f32 (q_in_i0123, q_s1_i); q_s3_r = vaddq_f32 (q_s0_r, q_s2_r); q_s3_i = vaddq_f32 (q_s0_i, q_s2_i); q_s4_r = vsubq_f32 (q_s0_r, q_s2_r); q_s4_i = vsubq_f32 (q_s0_i, q_s2_i); q_one_by_nfft = vdupq_n_f32 (0.0625f); q2_out_89ab.val[0] = vsubq_f32 (q2_out_0123.val[0], q_s3_r); q2_out_89ab.val[1] = vsubq_f32 (q2_out_0123.val[1], q_s3_i); q2_out_0123.val[0] = vaddq_f32 (q2_out_0123.val[0], q_s3_r); q2_out_0123.val[1] = vaddq_f32 (q2_out_0123.val[1], q_s3_i); q2_out_4567.val[0] = vsubq_f32 (q_s5_r, q_s4_i); q2_out_4567.val[1] = vaddq_f32 (q_s5_i, q_s4_r); q2_out_cdef.val[0] = vaddq_f32 (q_s5_r, q_s4_i); q2_out_cdef.val[1] = vsubq_f32 (q_s5_i, q_s4_r); q2_out_89ab.val[0] = vmulq_f32 (q2_out_89ab.val[0], q_one_by_nfft); q2_out_89ab.val[1] = vmulq_f32 (q2_out_89ab.val[1], q_one_by_nfft); q2_out_0123.val[0] = vmulq_f32 (q2_out_0123.val[0], q_one_by_nfft); q2_out_0123.val[1] = vmulq_f32 (q2_out_0123.val[1], q_one_by_nfft); q2_out_4567.val[0] = vmulq_f32 (q2_out_4567.val[0], q_one_by_nfft); q2_out_4567.val[1] = vmulq_f32 (q2_out_4567.val[1], q_one_by_nfft); q2_out_cdef.val[0] = vmulq_f32 (q2_out_cdef.val[0], q_one_by_nfft); q2_out_cdef.val[1] = vmulq_f32 (q2_out_cdef.val[1], q_one_by_nfft); vst2q_f32 (p_dst0, q2_out_0123); vst2q_f32 (p_dst1, q2_out_4567); vst2q_f32 (p_dst2, q2_out_89ab); vst2q_f32 (p_dst3, q2_out_cdef); }
inline float32x4_t vcombine(const float32x2_t & v0, const float32x2_t & v1) { return vcombine_f32(v0, v1); }
static void cftmdl_128_neon(float* a) { int j; const int l = 8; const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign); float32x4_t wk1rv = vld1q_f32(cftmdl_wk1r); for (j = 0; j < l; j += 2) { const float32x2_t a_00 = vld1_f32(&a[j + 0]); const float32x2_t a_08 = vld1_f32(&a[j + 8]); const float32x2_t a_32 = vld1_f32(&a[j + 32]); const float32x2_t a_40 = vld1_f32(&a[j + 40]); const float32x4_t a_00_32 = vcombine_f32(a_00, a_32); const float32x4_t a_08_40 = vcombine_f32(a_08, a_40); const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40); const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40); const float32x2_t a_16 = vld1_f32(&a[j + 16]); const float32x2_t a_24 = vld1_f32(&a[j + 24]); const float32x2_t a_48 = vld1_f32(&a[j + 48]); const float32x2_t a_56 = vld1_f32(&a[j + 56]); const float32x4_t a_16_48 = vcombine_f32(a_16, a_48); const float32x4_t a_24_56 = vcombine_f32(a_24, a_56); const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56); const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56); const float32x4_t xx0 = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1); const float32x4_t x1_x3_add = vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); const float32x4_t x1_x3_sub = vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); const float32x2_t yy0_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 0); const float32x2_t yy0_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 0); const float32x4_t yy0_as = vcombine_f32(yy0_a, yy0_s); const float32x2_t yy1_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 1); const float32x2_t yy1_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 1); const float32x4_t yy1_as = vcombine_f32(yy1_a, yy1_s); const float32x4_t yy0 = vmlaq_f32(yy0_as, vec_swap_sign, yy1_as); const float32x4_t yy4 = vmulq_f32(wk1rv, yy0); const float32x4_t xx1_rev = vrev64q_f32(xx1); const float32x4_t yy4_rev = vrev64q_f32(yy4); vst1_f32(&a[j + 0], vget_low_f32(xx0)); vst1_f32(&a[j + 32], vget_high_f32(xx0)); vst1_f32(&a[j + 16], vget_low_f32(xx1)); vst1_f32(&a[j + 48], vget_high_f32(xx1_rev)); a[j + 48] = -a[j + 48]; vst1_f32(&a[j + 8], vget_low_f32(x1_x3_add)); vst1_f32(&a[j + 24], vget_low_f32(x1_x3_sub)); vst1_f32(&a[j + 40], vget_low_f32(yy4)); vst1_f32(&a[j + 56], vget_high_f32(yy4_rev)); } { const int k = 64; const int k1 = 2; const int k2 = 2 * k1; const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2 + 0]); const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2 + 0]); const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2 + 0]); const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2 + 0]); const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2 + 0]); wk1rv = vld1q_f32(&rdft_wk1r[k2 + 0]); for (j = k; j < l + k; j += 2) { const float32x2_t a_00 = vld1_f32(&a[j + 0]); const float32x2_t a_08 = vld1_f32(&a[j + 8]); const float32x2_t a_32 = vld1_f32(&a[j + 32]); const float32x2_t a_40 = vld1_f32(&a[j + 40]); const float32x4_t a_00_32 = vcombine_f32(a_00, a_32); const float32x4_t a_08_40 = vcombine_f32(a_08, a_40); const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40); const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40); const float32x2_t a_16 = vld1_f32(&a[j + 16]); const float32x2_t a_24 = vld1_f32(&a[j + 24]); const float32x2_t a_48 = vld1_f32(&a[j + 48]); const float32x2_t a_56 = vld1_f32(&a[j + 56]); const float32x4_t a_16_48 = vcombine_f32(a_16, a_48); const float32x4_t a_24_56 = vcombine_f32(a_24, a_56); const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56); const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56); const float32x4_t xx = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1); const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1); const float32x4_t x1_x3_add = vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); const float32x4_t x1_x3_sub = vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1); float32x4_t xx4 = vmulq_f32(wk2rv, xx1); float32x4_t xx12 = vmulq_f32(wk1rv, x1_x3_add); float32x4_t xx22 = vmulq_f32(wk3rv, x1_x3_sub); xx4 = vmlaq_f32(xx4, wk2iv, vrev64q_f32(xx1)); xx12 = vmlaq_f32(xx12, wk1iv, vrev64q_f32(x1_x3_add)); xx22 = vmlaq_f32(xx22, wk3iv, vrev64q_f32(x1_x3_sub)); vst1_f32(&a[j + 0], vget_low_f32(xx)); vst1_f32(&a[j + 32], vget_high_f32(xx)); vst1_f32(&a[j + 16], vget_low_f32(xx4)); vst1_f32(&a[j + 48], vget_high_f32(xx4)); vst1_f32(&a[j + 8], vget_low_f32(xx12)); vst1_f32(&a[j + 40], vget_high_f32(xx12)); vst1_f32(&a[j + 24], vget_low_f32(xx22)); vst1_f32(&a[j + 56], vget_high_f32(xx22)); } } }
/** * @brief Elem_t¿‡–Õæÿ’ÛA”ÎElem_t¿‡–Õæÿ’ÛBœ‡≥À. * * @param dst[out] ‰≥ˆæÿ’ÛC. * src1[in] ‰»Îæÿ’ÛA. * src2[in] ‰»Îæÿ’ÛB. * mkn[in] æÿ’Ûµƒ∏˜∏ˆŒ¨ ˝. * * @return void */ void neon_matrixmul_4x4float(Elem_t * dst, Elem_t * src1, Elem_t * src2, int *mkn) { int m = mkn[0]; int k = mkn[1]; int n = mkn[2]; for (int i = 0; i < m; i += 4) { for (int j = 0; j < n; j += 4) { float32x2_t d16 = {0}; float32x2_t d17 = {0}; float32x2_t d18 = {0}; float32x2_t d19 = {0}; float32x2_t d20 = {0}; float32x2_t d21 = {0}; float32x2_t d22 = {0}; float32x2_t d23 = {0}; float32x2_t d24 = {0}; float32x2_t d25 = {0}; float32x2_t d26 = {0}; float32x2_t d27 = {0}; float32x2_t d28 = {0}; float32x2_t d29 = {0}; float32x2_t d30 = {0}; float32x2_t d31 = {0}; for (int l = 0; l < k; l += 4) { // Matrix A float32x4_t q8 = vld1q_f32(src1 ); float32x4_t q9 = vld1q_f32(src1 + k ); float32x4_t q10 = vld1q_f32(src1 + k*2); float32x4_t q11 = vld1q_f32(src1 + k*3); float32x2_t d0 = vget_low_f32(q8); float32x2_t d1 = vget_high_f32(q8); float32x2_t d2 = vget_low_f32(q9); float32x2_t d3 = vget_high_f32(q9); float32x2_t d4 = vget_low_f32(q10); float32x2_t d5 = vget_high_f32(q10); float32x2_t d6 = vget_low_f32(q11); float32x2_t d7 = vget_high_f32(q11); // Matrix B float32x4_t q12 = vld1q_f32(src2 ); float32x4_t q13 = vld1q_f32(src2 + k ); float32x4_t q14 = vld1q_f32(src2 + k*2); float32x4_t q15 = vld1q_f32(src2 + k*3); float32x2_t d8 = vget_low_f32(q12); float32x2_t d9 = vget_high_f32(q12); float32x2_t d10 = vget_low_f32(q13); float32x2_t d11 = vget_high_f32(q13); float32x2_t d12 = vget_low_f32(q14); float32x2_t d13 = vget_high_f32(q14); float32x2_t d14 = vget_low_f32(q15); float32x2_t d15 = vget_high_f32(q15); d16 = vmla_f32(d16, d0, d8); d17 = vmla_f32(d17, d0, d10); d18 = vmla_f32(d18, d0, d12); d19 = vmla_f32(d19, d0, d14); d16 = vmla_f32(d16, d1, d9); d17 = vmla_f32(d17, d1, d11); d18 = vmla_f32(d18, d1, d13); d19 = vmla_f32(d19, d1, d15); d20 = vmla_f32(d20, d2, d8); d21 = vmla_f32(d21, d2, d10); d22 = vmla_f32(d22, d2, d12); d23 = vmla_f32(d23, d2, d14); d20 = vmla_f32(d20, d3, d9); d21 = vmla_f32(d21, d3, d11); d22 = vmla_f32(d22, d3, d13); d23 = vmla_f32(d23, d3, d15); d24 = vmla_f32(d24, d4, d8); d25 = vmla_f32(d25, d4, d10); d26 = vmla_f32(d26, d4, d12); d27 = vmla_f32(d27, d4, d14); d24 = vmla_f32(d24, d5, d9); d25 = vmla_f32(d25, d5, d11); d26 = vmla_f32(d26, d5, d13); d27 = vmla_f32(d27, d5, d15); d28 = vmla_f32(d28, d6, d8); d29 = vmla_f32(d29, d6, d10); d30 = vmla_f32(d30, d6, d12); d31 = vmla_f32(d31, d6, d14); d28 = vmla_f32(d28, d7, d9); d29 = vmla_f32(d29, d7, d11); d30 = vmla_f32(d30, d7, d13); d31 = vmla_f32(d31, d7, d15); src1 += 4; src2 += 4; }// end for l d16 = vpadd_f32(d16, d17); d18 = vpadd_f32(d18, d19); d20 = vpadd_f32(d20, d21); d22 = vpadd_f32(d22, d23); d24 = vpadd_f32(d24, d25); d26 = vpadd_f32(d26, d27); d28 = vpadd_f32(d28, d29); d30 = vpadd_f32(d30, d31); vst1q_f32(dst , vcombine_f32(d16, d18)); vst1q_f32(dst + n , vcombine_f32(d20, d22)); vst1q_f32(dst + n*2, vcombine_f32(d24, d26)); vst1q_f32(dst + n*3, vcombine_f32(d28, d30)); src1 -= k; src2 += k*3; dst += 4; }// end for j src1 += k*4; src2 -= k*n; dst += n*3; }// end for i }