void test_vcombinef32 (void)
{
float32x4_t out_float32x4_t;
float32x2_t arg0_float32x2_t;
float32x2_t arg1_float32x2_t;
out_float32x4_t = vcombine_f32 (arg0_float32x2_t, arg1_float32x2_t);
}
//__attribute__ ((noinline))
inline vector<float, 4> cmin(vector<float, 4> const& v) noexcept
{
using vector_type = typename vector_traits<float, 4>::vector_type;
auto tmp(vpmin_f32(vget_low_f32(float32x4_t(v.data_)),
vget_high_f32(float32x4_t(v.data_))));
tmp = vpmin_f32(tmp, tmp);
return {
vector_type(vcombine_f32(tmp, tmp))
};
}
static void cft1st_128_neon(float* a) {
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
int j, k2;
for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) {
float32x4_t a00v = vld1q_f32(&a[j + 0]);
float32x4_t a04v = vld1q_f32(&a[j + 4]);
float32x4_t a08v = vld1q_f32(&a[j + 8]);
float32x4_t a12v = vld1q_f32(&a[j + 12]);
float32x4_t a01v = vcombine_f32(vget_low_f32(a00v), vget_low_f32(a08v));
float32x4_t a23v = vcombine_f32(vget_high_f32(a00v), vget_high_f32(a08v));
float32x4_t a45v = vcombine_f32(vget_low_f32(a04v), vget_low_f32(a12v));
float32x4_t a67v = vcombine_f32(vget_high_f32(a04v), vget_high_f32(a12v));
const float32x4_t wk1rv = vld1q_f32(&rdft_wk1r[k2]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2]);
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2]);
float32x4_t x0v = vaddq_f32(a01v, a23v);
const float32x4_t x1v = vsubq_f32(a01v, a23v);
const float32x4_t x2v = vaddq_f32(a45v, a67v);
const float32x4_t x3v = vsubq_f32(a45v, a67v);
const float32x4_t x3w = vrev64q_f32(x3v);
float32x4_t x0w;
a01v = vaddq_f32(x0v, x2v);
x0v = vsubq_f32(x0v, x2v);
x0w = vrev64q_f32(x0v);
a45v = vmulq_f32(wk2rv, x0v);
a45v = vmlaq_f32(a45v, wk2iv, x0w);
x0v = vmlaq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a23v = vmulq_f32(wk1rv, x0v);
a23v = vmlaq_f32(a23v, wk1iv, x0w);
x0v = vmlsq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a67v = vmulq_f32(wk3rv, x0v);
a67v = vmlaq_f32(a67v, wk3iv, x0w);
a00v = vcombine_f32(vget_low_f32(a01v), vget_low_f32(a23v));
a04v = vcombine_f32(vget_low_f32(a45v), vget_low_f32(a67v));
a08v = vcombine_f32(vget_high_f32(a01v), vget_high_f32(a23v));
a12v = vcombine_f32(vget_high_f32(a45v), vget_high_f32(a67v));
vst1q_f32(&a[j + 0], a00v);
vst1q_f32(&a[j + 4], a04v);
vst1q_f32(&a[j + 8], a08v);
vst1q_f32(&a[j + 12], a12v);
}
}
// Window time domain data to be used by the fft.
static void WindowDataNEON(float* x_windowed, const float* x) {
int i;
for (i = 0; i < PART_LEN; i += 4) {
const float32x4_t vec_Buf1 = vld1q_f32(&x[i]);
const float32x4_t vec_Buf2 = vld1q_f32(&x[PART_LEN + i]);
const float32x4_t vec_sqrtHanning = vld1q_f32(&WebRtcAec_sqrtHanning[i]);
// A B C D
float32x4_t vec_sqrtHanning_rev =
vld1q_f32(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]);
// B A D C
vec_sqrtHanning_rev = vrev64q_f32(vec_sqrtHanning_rev);
// D C B A
vec_sqrtHanning_rev = vcombine_f32(vget_high_f32(vec_sqrtHanning_rev),
vget_low_f32(vec_sqrtHanning_rev));
vst1q_f32(&x_windowed[i], vmulq_f32(vec_Buf1, vec_sqrtHanning));
vst1q_f32(&x_windowed[PART_LEN + i],
vmulq_f32(vec_Buf2, vec_sqrtHanning_rev));
}
}
void dotProd_neon(const float *data, const float *weights, float *vals, const int n, const int len, const float *istd) {
for (int i = 0; i < n; i += 4) {
float32x4_t accum0 = { 0.0f, 0.0f, 0.0f, 0.0f };
float32x4_t accum1 = accum0;
float32x4_t accum2 = accum0;
float32x4_t accum3 = accum0;
for (int j = 0; j < len; j += 4) {
float32x4_t d0 = vld1q_f32(data + j);
float32x4_t d1 = d0;
float32x4_t d2 = d0;
float32x4_t d3 = d0;
float32x4_t w0 = vld1q_f32(weights);
float32x4_t w1 = vld1q_f32(weights + 4);
float32x4_t w2 = vld1q_f32(weights + 8);
float32x4_t w3 = vld1q_f32(weights + 12);
accum0 = vaddq_f32(accum0, vmulq_f32(d0, w0));
accum1 = vaddq_f32(accum1, vmulq_f32(d1, w1));
accum2 = vaddq_f32(accum2, vmulq_f32(d2, w2));
accum3 = vaddq_f32(accum3, vmulq_f32(d3, w3));
weights += 16;
}
float32x2_t sum0 = vpadd_f32(vget_low_f32(accum0), vget_high_f32(accum0));
float32x2_t sum1 = vpadd_f32(vget_low_f32(accum1), vget_high_f32(accum1));
float32x2_t sum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2));
float32x2_t sum3 = vpadd_f32(vget_low_f32(accum3), vget_high_f32(accum3));
sum0 = vpadd_f32(sum0, sum1);
sum1 = vpadd_f32(sum2, sum3);
float32x4_t sum = vcombine_f32(sum0, sum1);
sum = vmulq_n_f32(sum, istd[0]);
sum = vaddq_f32(sum, vld1q_f32(weights + n*len + i));
vst1q_f32(vals + i, sum);
}
}
__inline static float32x4_t reverse_order_f32x4(float32x4_t in) {
// A B C D -> C D A B
const float32x4_t rev = vcombine_f32(vget_high_f32(in), vget_low_f32(in));
// C D A B -> D C B A
return vrev64q_f32(rev);
}
// CHECK-LABEL: define <4 x float> @test_vcombine_f32(<2 x float> %low, <2 x float> %high) #0 {
// CHECK: [[SHUFFLE_I:%.*]] = shufflevector <2 x float> %low, <2 x float> %high, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
// CHECK: ret <4 x float> [[SHUFFLE_I]]
float32x4_t test_vcombine_f32(float32x2_t low, float32x2_t high) {
return vcombine_f32(low, high);
}
void fft_real_neon(
CkFftContext* context,
const float* input,
CkFftComplex* output,
int count)
{
int countDiv2 = count/2;
fft_neon(context, (const CkFftComplex*) input, output, countDiv2, false, 1, context->fwdExpTable, context->maxCount / countDiv2);
output[countDiv2] = output[0];
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->fwdExpTable;
const CkFftComplex* exp1 = context->fwdExpTable + countDiv2 * expTableStride;
CkFftComplex* p0 = output;
CkFftComplex* p1 = output + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) p0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) p1, z1_v);
p0 += 4;
p1 -= 4;
}
if (count > 8)
{
// middle:
p0->real = p0->real * 2.0f;
p0->imag = -p0->imag * 2.0f;
}
}
void fft_real_inverse_neon(
CkFftContext* context,
const CkFftComplex* input,
float* output,
int count,
CkFftComplex* tmpBuf)
{
int countDiv2 = count/2;
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->invExpTable;
const CkFftComplex* exp1 = context->invExpTable + countDiv2 * expTableStride;
const CkFftComplex* p0 = input;
const CkFftComplex* p1 = input + countDiv2 - 3;
CkFftComplex* tmp0 = tmpBuf;
CkFftComplex* tmp1 = tmpBuf + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) tmp0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) tmp1, z1_v);
p0 += 4;
tmp0 += 4;
p1 -= 4;
tmp1 -= 4;
}
// middle:
tmp0->real = p0->real * 2.0f;
tmp0->imag = -p0->imag * 2.0f;
fft_neon(context, tmpBuf, (CkFftComplex*) output, countDiv2, true, 1, context->invExpTable, context->maxCount / countDiv2);
}
void computeNetwork0_neon(const float *input, const float *weights, uint8_t *d) {
float32x4_t m0 = { 0.0f, 0.0f, 0.0f, 0.0f };
float32x4_t m1 = m0;
float32x4_t m2 = m0;
float32x4_t m3 = m0;
float32x4_t m4, m5, m6, m7;
for (int i = 0; i < 192/4; i += 4) {
m4 = vld1q_f32(input + i);
m5 = m4;
m6 = m4;
m7 = m4;
m4 = vmulq_f32(m4, vld1q_f32(weights + i * 4));
m5 = vmulq_f32(m5, vld1q_f32(weights + i * 4 + 4));
m6 = vmulq_f32(m6, vld1q_f32(weights + i * 4 + 8));
m7 = vmulq_f32(m7, vld1q_f32(weights + i * 4 + 12));
m0 = vaddq_f32(m0, m4);
m1 = vaddq_f32(m1, m5);
m2 = vaddq_f32(m2, m6);
m3 = vaddq_f32(m3, m7);
}
float32x2_t sum0 = vpadd_f32(vget_low_f32(m0), vget_high_f32(m0));
float32x2_t sum1 = vpadd_f32(vget_low_f32(m1), vget_high_f32(m1));
float32x2_t sum2 = vpadd_f32(vget_low_f32(m2), vget_high_f32(m2));
float32x2_t sum3 = vpadd_f32(vget_low_f32(m3), vget_high_f32(m3));
sum0 = vpadd_f32(sum0, sum1);
sum1 = vpadd_f32(sum2, sum3);
m0 = vcombine_f32(sum0, sum1);
m0 = vaddq_f32(m0, vld1q_f32(weights + 768/4));
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weights + 784/4));
m2 = vmulq_f32(m2, vld1q_f32(weights + (784+16)/4));
m3 = vmulq_f32(m3, vld1q_f32(weights + (784+32)/4));
m4 = vmulq_f32(m4, vld1q_f32(weights + (784+48)/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weights + (784+64)/4));
m7 = m1;
m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f));
m1 = vaddq_f32(m1, ones_f);
m7 = vmulq_f32(reciprocal(m1), m7);
m3 = m0;
m0 = vdupq_lane_f32(vget_low_f32(m0), 0);
m1 = vdupq_lane_f32(vget_low_f32(m3), 1);
m2 = vdupq_lane_f32(vget_high_f32(m3), 0);
m3 = vdupq_lane_f32(vget_high_f32(m3), 1);
m0 = vmulq_f32(m0, vld1q_f32(weights + 864/4));
m1 = vmulq_f32(m1, vld1q_f32(weights + (864+16)/4));
m2 = vmulq_f32(m2, vld1q_f32(weights + (864+32)/4));
m3 = vmulq_f32(m3, vld1q_f32(weights + (864+48)/4));
m4 = vdupq_lane_f32(vget_low_f32(m7), 0);
m5 = vdupq_lane_f32(vget_low_f32(m7), 1);
m6 = vdupq_lane_f32(vget_high_f32(m7), 0);
m7 = vdupq_lane_f32(vget_high_f32(m7), 1);
m4 = vmulq_f32(m4, vld1q_f32(weights + (864+64)/4));
m5 = vmulq_f32(m5, vld1q_f32(weights + (864+80)/4));
m6 = vmulq_f32(m6, vld1q_f32(weights + (864+96)/4));
m7 = vmulq_f32(m7, vld1q_f32(weights + (864+112)/4));
m0 = vaddq_f32(m0, m1);
m2 = vaddq_f32(m2, m3);
m4 = vaddq_f32(m4, m5);
m6 = vaddq_f32(m6, m7);
m0 = vaddq_f32(m0, m2);
m4 = vaddq_f32(m4, m6);
m0 = vaddq_f32(m0, m4);
m0 = vaddq_f32(m0, vld1q_f32(weights + (864+128)/4));
float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0));
d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0));
}
/**
* @brief vector_mul_matrix.
*
* @param src1[in] the input vector(1*k)
* src2[in] the input matrix(k*n)
* dst[out] the output vector(1*n)
* kn[in] DIM_K & DIM_N
*
* @return void
*/
void neon_vectormulmatrix_float(float * dst,
const float * src1,
const float * src2,
int *kn)
{
int j ,l;
int k = kn[0];
int n = kn[1];
const float * src1_p = src1;
const float * src2_p = src2;
float * dst_p = dst;
for (j = 0; j <= n - 4; j += 4) {
float32x2_t d16 = {0};
float32x2_t d17 = {0};
float32x2_t d18 = {0};
float32x2_t d19 = {0};
float32x2_t d20;
float32x2_t d21;
float32x4_t q0;
src1_p = src1;
src2_p = src2 + j * k;
for (l = 0; l <= k - 4; l += 4) {
// Matrix A
float32x4_t q8 = vld1q_f32(src1_p);
float32x2_t d0 = vget_low_f32(q8);
float32x2_t d1 = vget_high_f32(q8);
// Matrix B
float32x4_t q12 = vld1q_f32(src2_p);
float32x4_t q13 = vld1q_f32(src2_p + k);
float32x4_t q14 = vld1q_f32(src2_p + k * 2);
float32x4_t q15 = vld1q_f32(src2_p + k * 3);
float32x2_t d8 = vget_low_f32(q12);
float32x2_t d9 = vget_high_f32(q12);
float32x2_t d10 = vget_low_f32(q13);
float32x2_t d11 = vget_high_f32(q13);
float32x2_t d12 = vget_low_f32(q14);
float32x2_t d13 = vget_high_f32(q14);
float32x2_t d14 = vget_low_f32(q15);
float32x2_t d15 = vget_high_f32(q15);
d16 = vmla_f32(d16, d0, d8);
d17 = vmla_f32(d17, d0, d10);
d18 = vmla_f32(d18, d0, d12);
d19 = vmla_f32(d19, d0, d14);
d16 = vmla_f32(d16, d1, d9);
d17 = vmla_f32(d17, d1, d11);
d18 = vmla_f32(d18, d1, d13);
d19 = vmla_f32(d19, d1, d15);
src1_p += 4;
src2_p += 4;
}// end for l
d16 = vpadd_f32(d16, d17);
d18 = vpadd_f32(d18, d19);
float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
for(; l < k; l ++) {
float src1_d;
src1_d = *src1_p;
sum0 += src1_d * *src2_p;
sum1 += src1_d * *(src2_p + k);
sum2 += src1_d * *(src2_p + 2 * k);
sum3 += src1_d * *(src2_p + 3 * k);
src1_p++;
src2_p++;
}
d20 = vset_lane_f32(sum0, d20, 0);
d20 = vset_lane_f32(sum1, d20, 1);
d21 = vset_lane_f32(sum2, d21, 0);
d21 = vset_lane_f32(sum3, d21, 1);
q0 = vaddq_f32(vcombine_f32(d16, d18), vcombine_f32(d20, d21));
vst1q_f32(dst_p, q0);
dst_p += 4;
}// end for j
}
static void ne10_fft_split_c2r_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fk, fnkc, fek, fok, tmp;
float32x4x2_t q2_fk, q2_fnkc, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fnkc_r, q_fnkc_i;
float32x4_t q_fek_r, q_fek_i, q_fok_r, q_fok_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
dst[0].r = (src[0].r + src[ncfft].r) * 0.5f;
dst[0].i = (src[0].r - src[ncfft].r) * 0.5f;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fk = vld2q_f32 (p_src);
q2_fnkc = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fnkc.val[0] = vrev64q_f32 (q2_fnkc.val[0]);
q2_fnkc.val[1] = vrev64q_f32 (q2_fnkc.val[1]);
q_fnkc_r = vcombine_f32 (vget_high_f32 (q2_fnkc.val[0]), vget_low_f32 (q2_fnkc.val[0]));
q_fnkc_i = vcombine_f32 (vget_high_f32 (q2_fnkc.val[1]), vget_low_f32 (q2_fnkc.val[1]));
q_fnkc_i = vnegq_f32 (q_fnkc_i);
q_fek_r = vaddq_f32 (q2_fk.val[0], q_fnkc_r);
q_fek_i = vaddq_f32 (q2_fk.val[1], q_fnkc_i);
q_tmp0 = vsubq_f32 (q2_fk.val[0], q_fnkc_r);
q_tmp1 = vsubq_f32 (q2_fk.val[1], q_fnkc_i);
q_fok_r = vmulq_f32 (q_tmp0, q2_tw.val[0]);
q_fok_i = vmulq_f32 (q_tmp1, q2_tw.val[0]);
q_tmp2 = vmulq_f32 (q_tmp1, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_tmp0, q2_tw.val[1]);
q_fok_r = vaddq_f32 (q_fok_r, q_tmp2);
q_fok_i = vsubq_f32 (q_fok_i, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_fek_r, q_fok_r);
q_dst2_i = vsubq_f32 (q_fok_i, q_fek_i);
q2_dst.val[0] = vaddq_f32 (q_fek_r, q_fok_r);
q2_dst.val[1] = vaddq_f32 (q_fek_i, q_fok_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q2_dst.val[0], q_val);
q2_dst.val[1] = vmulq_f32 (q2_dst.val[1], q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fk = src[k];
fnkc.r = src[ncfft - k].r;
fnkc.i = -src[ncfft - k].i;
fek.r = fk.r + fnkc.r;
fek.i = fk.i + fnkc.i;
tmp.r = fk.r - fnkc.r;
tmp.i = fk.i - fnkc.i;
fok.r = tmp.r * twiddles[k - 1].r + tmp.i * twiddles[k - 1].i;
fok.i = tmp.i * twiddles[k - 1].r - tmp.r * twiddles[k - 1].i;
dst[k].r = (fek.r + fok.r) * 0.5f;
dst[k].i = (fek.i + fok.i) * 0.5f;
dst[ncfft - k].r = (fek.r - fok.r) * 0.5f;
dst[ncfft - k].i = (fok.i - fek.i) * 0.5f;
}
}
}
static void ne10_fft_split_r2c_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fpnk, fpk, f1k, f2k, tw, tdc;
float32x4x2_t q2_fpk, q2_fpnk, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fpnk_r, q_fpnk_i;
float32x4_t q_f1k_r, q_f1k_i, q_f2k_r, q_f2k_i;
float32x4_t q_tw_r, q_tw_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst_r, q_dst_i, q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
tdc.r = src[0].r;
tdc.i = src[0].i;
dst[0].r = tdc.r + tdc.i;
dst[ncfft].r = tdc.r - tdc.i;
dst[ncfft].i = dst[0].i = 0;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fpk = vld2q_f32 (p_src);
q2_fpnk = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fpnk.val[0] = vrev64q_f32 (q2_fpnk.val[0]);
q2_fpnk.val[1] = vrev64q_f32 (q2_fpnk.val[1]);
q_fpnk_r = vcombine_f32 (vget_high_f32 (q2_fpnk.val[0]), vget_low_f32 (q2_fpnk.val[0]));
q_fpnk_i = vcombine_f32 (vget_high_f32 (q2_fpnk.val[1]), vget_low_f32 (q2_fpnk.val[1]));
q_fpnk_i = vnegq_f32 (q_fpnk_i);
q_f1k_r = vaddq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f1k_i = vaddq_f32 (q2_fpk.val[1], q_fpnk_i);
q_f2k_r = vsubq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f2k_i = vsubq_f32 (q2_fpk.val[1], q_fpnk_i);
q_tmp0 = vmulq_f32 (q_f2k_r, q2_tw.val[0]);
q_tmp1 = vmulq_f32 (q_f2k_i, q2_tw.val[1]);
q_tmp2 = vmulq_f32 (q_f2k_r, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_f2k_i, q2_tw.val[0]);
q_tw_r = vsubq_f32 (q_tmp0, q_tmp1);
q_tw_i = vaddq_f32 (q_tmp2, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_f1k_r, q_tw_r);
q_dst2_i = vsubq_f32 (q_tw_i, q_f1k_i);
q_dst_r = vaddq_f32 (q_f1k_r, q_tw_r);
q_dst_i = vaddq_f32 (q_f1k_i, q_tw_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q_dst_r, q_val);
q2_dst.val[1] = vmulq_f32 (q_dst_i, q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fpk = src[k];
fpnk.r = src[ncfft - k].r;
fpnk.i = - src[ncfft - k].i;
f1k.r = fpk.r + fpnk.r;
f1k.i = fpk.i + fpnk.i;
f2k.r = fpk.r - fpnk.r;
f2k.i = fpk.i - fpnk.i;
tw.r = f2k.r * (twiddles[k - 1]).r - f2k.i * (twiddles[k - 1]).i;
tw.i = f2k.r * (twiddles[k - 1]).i + f2k.i * (twiddles[k - 1]).r;
dst[k].r = (f1k.r + tw.r) * 0.5f;
dst[k].i = (f1k.i + tw.i) * 0.5f;
dst[ncfft - k].r = (f1k.r - tw.r) * 0.5f;
dst[ncfft - k].i = (tw.i - f1k.i) * 0.5f;
}
}
}
static void ne10_fft16_backward_float32_neon (ne10_fft_cpx_float32_t * Fout,
ne10_fft_cpx_float32_t * Fin,
ne10_fft_cpx_float32_t * twiddles)
{
ne10_fft_cpx_float32_t *tw1, *tw2, *tw3;
// the first stage
float32_t *p_src0, *p_src4, *p_src8, *p_src12;
float32x4x2_t q2_in_0123, q2_in_4567, q2_in_89ab, q2_in_cdef;
float32x4_t q_t0_r, q_t0_i, q_t1_r, q_t1_i, q_t2_r, q_t2_i, q_t3_r, q_t3_i;
float32x4_t q_out_r048c, q_out_i048c, q_out_r159d, q_out_i159d;
float32x4_t q_out_r26ae, q_out_i26ae, q_out_r37bf, q_out_i37bf;
p_src0 = (float32_t*) (& (Fin[0]));
p_src4 = (float32_t*) (& (Fin[4]));
p_src8 = (float32_t*) (& (Fin[8]));
p_src12 = (float32_t*) (& (Fin[12]));
q2_in_0123 = vld2q_f32 (p_src0);
q2_in_4567 = vld2q_f32 (p_src4);
q2_in_89ab = vld2q_f32 (p_src8);
q2_in_cdef = vld2q_f32 (p_src12);
q_t2_r = vsubq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]);
q_t2_i = vsubq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]);
q_t3_r = vaddq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]);
q_t3_i = vaddq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]);
q_t0_r = vaddq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]);
q_t0_i = vaddq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]);
q_t1_r = vsubq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]);
q_t1_i = vsubq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]);
q_out_r26ae = vsubq_f32 (q_t3_r, q_t0_r);
q_out_i26ae = vsubq_f32 (q_t3_i, q_t0_i);
q_out_r048c = vaddq_f32 (q_t3_r, q_t0_r);
q_out_i048c = vaddq_f32 (q_t3_i, q_t0_i);
q_out_r159d = vsubq_f32 (q_t2_r, q_t1_i);
q_out_i159d = vaddq_f32 (q_t2_i, q_t1_r);
q_out_r37bf = vaddq_f32 (q_t2_r, q_t1_i);
q_out_i37bf = vsubq_f32 (q_t2_i, q_t1_r);
// second stages
float32_t *p_dst0, *p_dst1, *p_dst2, *p_dst3;
float32_t *p_tw1, *p_tw2, *p_tw3;
float32x4_t q_s0_r, q_s0_i, q_s1_r, q_s1_i, q_s2_r, q_s2_i;
float32x4_t q_s3_r, q_s3_i, q_s4_r, q_s4_i, q_s5_r, q_s5_i;
float32x4x2_t q2_tmp_0, q2_tmp_1, q2_tmp_2, q2_tmp_3;
float32x4_t q_in_r0123, q_in_r4567, q_in_r89ab, q_in_rcdef;
float32x4_t q_in_i0123, q_in_i4567, q_in_i89ab, q_in_icdef;
float32x4x2_t q2_tw1, q2_tw2, q2_tw3;
float32x4x2_t q2_out_0123, q2_out_4567, q2_out_89ab, q2_out_cdef;
float32x4_t q_one_by_nfft;
tw1 = twiddles;
tw2 = twiddles + 4;
tw3 = twiddles + 8;
p_dst0 = (float32_t*) (&Fout[0]);
p_dst1 = (float32_t*) (&Fout[4]);
p_dst2 = (float32_t*) (&Fout[8]);
p_dst3 = (float32_t*) (&Fout[12]);
p_tw1 = (float32_t*) tw1;
p_tw2 = (float32_t*) tw2;
p_tw3 = (float32_t*) tw3;
q2_tmp_0 = vzipq_f32 (q_out_r048c, q_out_r159d);
q2_tmp_1 = vzipq_f32 (q_out_i048c, q_out_i159d);
q2_tmp_2 = vzipq_f32 (q_out_r26ae, q_out_r37bf);
q2_tmp_3 = vzipq_f32 (q_out_i26ae, q_out_i37bf);
q_in_r0123 = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[0]), vget_low_f32 (q2_tmp_2.val[0]));
q_in_i0123 = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[0]), vget_low_f32 (q2_tmp_3.val[0]));
q_in_r4567 = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[0]), vget_high_f32 (q2_tmp_2.val[0]));
q_in_i4567 = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[0]), vget_high_f32 (q2_tmp_3.val[0]));
q_in_r89ab = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[1]), vget_low_f32 (q2_tmp_2.val[1]));
q_in_i89ab = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[1]), vget_low_f32 (q2_tmp_3.val[1]));
q_in_rcdef = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[1]), vget_high_f32 (q2_tmp_2.val[1]));
q_in_icdef = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[1]), vget_high_f32 (q2_tmp_3.val[1]));
q2_tw1 = vld2q_f32 (p_tw1);
q2_tw2 = vld2q_f32 (p_tw2);
q2_tw3 = vld2q_f32 (p_tw3);
q_s0_r = vmulq_f32 (q_in_r4567, q2_tw1.val[0]);
q_s0_i = vmulq_f32 (q_in_i4567, q2_tw1.val[0]);
q_s1_r = vmulq_f32 (q_in_r89ab, q2_tw2.val[0]);
q_s1_i = vmulq_f32 (q_in_i89ab, q2_tw2.val[0]);
q_s2_r = vmulq_f32 (q_in_rcdef, q2_tw3.val[0]);
q_s2_i = vmulq_f32 (q_in_icdef, q2_tw3.val[0]);
q_s0_r = vmlaq_f32 (q_s0_r, q_in_i4567, q2_tw1.val[1]);
q_s0_i = vmlsq_f32 (q_s0_i, q_in_r4567, q2_tw1.val[1]);
q_s1_r = vmlaq_f32 (q_s1_r, q_in_i89ab, q2_tw2.val[1]);
q_s1_i = vmlsq_f32 (q_s1_i, q_in_r89ab, q2_tw2.val[1]);
q_s2_r = vmlaq_f32 (q_s2_r, q_in_icdef, q2_tw3.val[1]);
q_s2_i = vmlsq_f32 (q_s2_i, q_in_rcdef, q2_tw3.val[1]);
q_s5_r = vsubq_f32 (q_in_r0123, q_s1_r);
q_s5_i = vsubq_f32 (q_in_i0123, q_s1_i);
q2_out_0123.val[0] = vaddq_f32 (q_in_r0123, q_s1_r);
q2_out_0123.val[1] = vaddq_f32 (q_in_i0123, q_s1_i);
q_s3_r = vaddq_f32 (q_s0_r, q_s2_r);
q_s3_i = vaddq_f32 (q_s0_i, q_s2_i);
q_s4_r = vsubq_f32 (q_s0_r, q_s2_r);
q_s4_i = vsubq_f32 (q_s0_i, q_s2_i);
q_one_by_nfft = vdupq_n_f32 (0.0625f);
q2_out_89ab.val[0] = vsubq_f32 (q2_out_0123.val[0], q_s3_r);
q2_out_89ab.val[1] = vsubq_f32 (q2_out_0123.val[1], q_s3_i);
q2_out_0123.val[0] = vaddq_f32 (q2_out_0123.val[0], q_s3_r);
q2_out_0123.val[1] = vaddq_f32 (q2_out_0123.val[1], q_s3_i);
q2_out_4567.val[0] = vsubq_f32 (q_s5_r, q_s4_i);
q2_out_4567.val[1] = vaddq_f32 (q_s5_i, q_s4_r);
q2_out_cdef.val[0] = vaddq_f32 (q_s5_r, q_s4_i);
q2_out_cdef.val[1] = vsubq_f32 (q_s5_i, q_s4_r);
q2_out_89ab.val[0] = vmulq_f32 (q2_out_89ab.val[0], q_one_by_nfft);
q2_out_89ab.val[1] = vmulq_f32 (q2_out_89ab.val[1], q_one_by_nfft);
q2_out_0123.val[0] = vmulq_f32 (q2_out_0123.val[0], q_one_by_nfft);
q2_out_0123.val[1] = vmulq_f32 (q2_out_0123.val[1], q_one_by_nfft);
q2_out_4567.val[0] = vmulq_f32 (q2_out_4567.val[0], q_one_by_nfft);
q2_out_4567.val[1] = vmulq_f32 (q2_out_4567.val[1], q_one_by_nfft);
q2_out_cdef.val[0] = vmulq_f32 (q2_out_cdef.val[0], q_one_by_nfft);
q2_out_cdef.val[1] = vmulq_f32 (q2_out_cdef.val[1], q_one_by_nfft);
vst2q_f32 (p_dst0, q2_out_0123);
vst2q_f32 (p_dst1, q2_out_4567);
vst2q_f32 (p_dst2, q2_out_89ab);
vst2q_f32 (p_dst3, q2_out_cdef);
}
inline float32x4_t vcombine(const float32x2_t & v0, const float32x2_t & v1) { return vcombine_f32(v0, v1); }
static void cftmdl_128_neon(float* a) {
int j;
const int l = 8;
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
float32x4_t wk1rv = vld1q_f32(cftmdl_wk1r);
for (j = 0; j < l; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx0 = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x2_t yy0_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 0);
const float32x2_t yy0_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 0);
const float32x4_t yy0_as = vcombine_f32(yy0_a, yy0_s);
const float32x2_t yy1_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 1);
const float32x2_t yy1_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 1);
const float32x4_t yy1_as = vcombine_f32(yy1_a, yy1_s);
const float32x4_t yy0 = vmlaq_f32(yy0_as, vec_swap_sign, yy1_as);
const float32x4_t yy4 = vmulq_f32(wk1rv, yy0);
const float32x4_t xx1_rev = vrev64q_f32(xx1);
const float32x4_t yy4_rev = vrev64q_f32(yy4);
vst1_f32(&a[j + 0], vget_low_f32(xx0));
vst1_f32(&a[j + 32], vget_high_f32(xx0));
vst1_f32(&a[j + 16], vget_low_f32(xx1));
vst1_f32(&a[j + 48], vget_high_f32(xx1_rev));
a[j + 48] = -a[j + 48];
vst1_f32(&a[j + 8], vget_low_f32(x1_x3_add));
vst1_f32(&a[j + 24], vget_low_f32(x1_x3_sub));
vst1_f32(&a[j + 40], vget_low_f32(yy4));
vst1_f32(&a[j + 56], vget_high_f32(yy4_rev));
}
{
const int k = 64;
const int k1 = 2;
const int k2 = 2 * k1;
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2 + 0]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2 + 0]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2 + 0]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2 + 0]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2 + 0]);
wk1rv = vld1q_f32(&rdft_wk1r[k2 + 0]);
for (j = k; j < l + k; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
float32x4_t xx4 = vmulq_f32(wk2rv, xx1);
float32x4_t xx12 = vmulq_f32(wk1rv, x1_x3_add);
float32x4_t xx22 = vmulq_f32(wk3rv, x1_x3_sub);
xx4 = vmlaq_f32(xx4, wk2iv, vrev64q_f32(xx1));
xx12 = vmlaq_f32(xx12, wk1iv, vrev64q_f32(x1_x3_add));
xx22 = vmlaq_f32(xx22, wk3iv, vrev64q_f32(x1_x3_sub));
vst1_f32(&a[j + 0], vget_low_f32(xx));
vst1_f32(&a[j + 32], vget_high_f32(xx));
vst1_f32(&a[j + 16], vget_low_f32(xx4));
vst1_f32(&a[j + 48], vget_high_f32(xx4));
vst1_f32(&a[j + 8], vget_low_f32(xx12));
vst1_f32(&a[j + 40], vget_high_f32(xx12));
vst1_f32(&a[j + 24], vget_low_f32(xx22));
vst1_f32(&a[j + 56], vget_high_f32(xx22));
}
}
}
/**
* @brief Elem_t¿‡–Õæÿ’ÛA”ÎElem_t¿‡–Õæÿ’ÛBœ‡≥À.
*
* @param dst[out] ‰≥ˆæÿ’ÛC.
* src1[in] ‰»Îæÿ’ÛA.
* src2[in] ‰»Îæÿ’ÛB.
* mkn[in] æÿ’Ûµƒ∏˜∏ˆŒ¨ ˝.
*
* @return void
*/
void neon_matrixmul_4x4float(Elem_t * dst,
Elem_t * src1,
Elem_t * src2,
int *mkn)
{
int m = mkn[0];
int k = mkn[1];
int n = mkn[2];
for (int i = 0; i < m; i += 4)
{
for (int j = 0; j < n; j += 4)
{
float32x2_t d16 = {0};
float32x2_t d17 = {0};
float32x2_t d18 = {0};
float32x2_t d19 = {0};
float32x2_t d20 = {0};
float32x2_t d21 = {0};
float32x2_t d22 = {0};
float32x2_t d23 = {0};
float32x2_t d24 = {0};
float32x2_t d25 = {0};
float32x2_t d26 = {0};
float32x2_t d27 = {0};
float32x2_t d28 = {0};
float32x2_t d29 = {0};
float32x2_t d30 = {0};
float32x2_t d31 = {0};
for (int l = 0; l < k; l += 4)
{
// Matrix A
float32x4_t q8 = vld1q_f32(src1 );
float32x4_t q9 = vld1q_f32(src1 + k );
float32x4_t q10 = vld1q_f32(src1 + k*2);
float32x4_t q11 = vld1q_f32(src1 + k*3);
float32x2_t d0 = vget_low_f32(q8);
float32x2_t d1 = vget_high_f32(q8);
float32x2_t d2 = vget_low_f32(q9);
float32x2_t d3 = vget_high_f32(q9);
float32x2_t d4 = vget_low_f32(q10);
float32x2_t d5 = vget_high_f32(q10);
float32x2_t d6 = vget_low_f32(q11);
float32x2_t d7 = vget_high_f32(q11);
// Matrix B
float32x4_t q12 = vld1q_f32(src2 );
float32x4_t q13 = vld1q_f32(src2 + k );
float32x4_t q14 = vld1q_f32(src2 + k*2);
float32x4_t q15 = vld1q_f32(src2 + k*3);
float32x2_t d8 = vget_low_f32(q12);
float32x2_t d9 = vget_high_f32(q12);
float32x2_t d10 = vget_low_f32(q13);
float32x2_t d11 = vget_high_f32(q13);
float32x2_t d12 = vget_low_f32(q14);
float32x2_t d13 = vget_high_f32(q14);
float32x2_t d14 = vget_low_f32(q15);
float32x2_t d15 = vget_high_f32(q15);
d16 = vmla_f32(d16, d0, d8);
d17 = vmla_f32(d17, d0, d10);
d18 = vmla_f32(d18, d0, d12);
d19 = vmla_f32(d19, d0, d14);
d16 = vmla_f32(d16, d1, d9);
d17 = vmla_f32(d17, d1, d11);
d18 = vmla_f32(d18, d1, d13);
d19 = vmla_f32(d19, d1, d15);
d20 = vmla_f32(d20, d2, d8);
d21 = vmla_f32(d21, d2, d10);
d22 = vmla_f32(d22, d2, d12);
d23 = vmla_f32(d23, d2, d14);
d20 = vmla_f32(d20, d3, d9);
d21 = vmla_f32(d21, d3, d11);
d22 = vmla_f32(d22, d3, d13);
d23 = vmla_f32(d23, d3, d15);
d24 = vmla_f32(d24, d4, d8);
d25 = vmla_f32(d25, d4, d10);
d26 = vmla_f32(d26, d4, d12);
d27 = vmla_f32(d27, d4, d14);
d24 = vmla_f32(d24, d5, d9);
d25 = vmla_f32(d25, d5, d11);
d26 = vmla_f32(d26, d5, d13);
d27 = vmla_f32(d27, d5, d15);
d28 = vmla_f32(d28, d6, d8);
d29 = vmla_f32(d29, d6, d10);
d30 = vmla_f32(d30, d6, d12);
d31 = vmla_f32(d31, d6, d14);
d28 = vmla_f32(d28, d7, d9);
d29 = vmla_f32(d29, d7, d11);
d30 = vmla_f32(d30, d7, d13);
d31 = vmla_f32(d31, d7, d15);
src1 += 4;
src2 += 4;
}// end for l
d16 = vpadd_f32(d16, d17);
d18 = vpadd_f32(d18, d19);
d20 = vpadd_f32(d20, d21);
d22 = vpadd_f32(d22, d23);
d24 = vpadd_f32(d24, d25);
d26 = vpadd_f32(d26, d27);
d28 = vpadd_f32(d28, d29);
d30 = vpadd_f32(d30, d31);
vst1q_f32(dst , vcombine_f32(d16, d18));
vst1q_f32(dst + n , vcombine_f32(d20, d22));
vst1q_f32(dst + n*2, vcombine_f32(d24, d26));
vst1q_f32(dst + n*3, vcombine_f32(d28, d30));
src1 -= k;
src2 += k*3;
dst += 4;
}// end for j
src1 += k*4;
src2 -= k*n;
dst += n*3;
}// end for i
}
