static void cft1st_128_neon(float* a) {
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
int j, k2;
for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) {
float32x4_t a00v = vld1q_f32(&a[j + 0]);
float32x4_t a04v = vld1q_f32(&a[j + 4]);
float32x4_t a08v = vld1q_f32(&a[j + 8]);
float32x4_t a12v = vld1q_f32(&a[j + 12]);
float32x4_t a01v = vcombine_f32(vget_low_f32(a00v), vget_low_f32(a08v));
float32x4_t a23v = vcombine_f32(vget_high_f32(a00v), vget_high_f32(a08v));
float32x4_t a45v = vcombine_f32(vget_low_f32(a04v), vget_low_f32(a12v));
float32x4_t a67v = vcombine_f32(vget_high_f32(a04v), vget_high_f32(a12v));
const float32x4_t wk1rv = vld1q_f32(&rdft_wk1r[k2]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2]);
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2]);
float32x4_t x0v = vaddq_f32(a01v, a23v);
const float32x4_t x1v = vsubq_f32(a01v, a23v);
const float32x4_t x2v = vaddq_f32(a45v, a67v);
const float32x4_t x3v = vsubq_f32(a45v, a67v);
const float32x4_t x3w = vrev64q_f32(x3v);
float32x4_t x0w;
a01v = vaddq_f32(x0v, x2v);
x0v = vsubq_f32(x0v, x2v);
x0w = vrev64q_f32(x0v);
a45v = vmulq_f32(wk2rv, x0v);
a45v = vmlaq_f32(a45v, wk2iv, x0w);
x0v = vmlaq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a23v = vmulq_f32(wk1rv, x0v);
a23v = vmlaq_f32(a23v, wk1iv, x0w);
x0v = vmlsq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a67v = vmulq_f32(wk3rv, x0v);
a67v = vmlaq_f32(a67v, wk3iv, x0w);
a00v = vcombine_f32(vget_low_f32(a01v), vget_low_f32(a23v));
a04v = vcombine_f32(vget_low_f32(a45v), vget_low_f32(a67v));
a08v = vcombine_f32(vget_high_f32(a01v), vget_high_f32(a23v));
a12v = vcombine_f32(vget_high_f32(a45v), vget_high_f32(a67v));
vst1q_f32(&a[j + 0], a00v);
vst1q_f32(&a[j + 4], a04v);
vst1q_f32(&a[j + 8], a08v);
vst1q_f32(&a[j + 12], a12v);
}
}
void test_vrev64Qf32 (void)
{
float32x4_t out_float32x4_t;
float32x4_t arg0_float32x4_t;
out_float32x4_t = vrev64q_f32 (arg0_float32x4_t);
}
// Window time domain data to be used by the fft.
static void WindowDataNEON(float* x_windowed, const float* x) {
int i;
for (i = 0; i < PART_LEN; i += 4) {
const float32x4_t vec_Buf1 = vld1q_f32(&x[i]);
const float32x4_t vec_Buf2 = vld1q_f32(&x[PART_LEN + i]);
const float32x4_t vec_sqrtHanning = vld1q_f32(&WebRtcAec_sqrtHanning[i]);
// A B C D
float32x4_t vec_sqrtHanning_rev =
vld1q_f32(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]);
// B A D C
vec_sqrtHanning_rev = vrev64q_f32(vec_sqrtHanning_rev);
// D C B A
vec_sqrtHanning_rev = vcombine_f32(vget_high_f32(vec_sqrtHanning_rev),
vget_low_f32(vec_sqrtHanning_rev));
vst1q_f32(&x_windowed[i], vmulq_f32(vec_Buf1, vec_sqrtHanning));
vst1q_f32(&x_windowed[PART_LEN + i],
vmulq_f32(vec_Buf2, vec_sqrtHanning_rev));
}
}
static void cftmdl_128_neon(float* a) {
int j;
const int l = 8;
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
float32x4_t wk1rv = vld1q_f32(cftmdl_wk1r);
for (j = 0; j < l; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx0 = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x2_t yy0_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 0);
const float32x2_t yy0_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 0);
const float32x4_t yy0_as = vcombine_f32(yy0_a, yy0_s);
const float32x2_t yy1_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 1);
const float32x2_t yy1_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 1);
const float32x4_t yy1_as = vcombine_f32(yy1_a, yy1_s);
const float32x4_t yy0 = vmlaq_f32(yy0_as, vec_swap_sign, yy1_as);
const float32x4_t yy4 = vmulq_f32(wk1rv, yy0);
const float32x4_t xx1_rev = vrev64q_f32(xx1);
const float32x4_t yy4_rev = vrev64q_f32(yy4);
vst1_f32(&a[j + 0], vget_low_f32(xx0));
vst1_f32(&a[j + 32], vget_high_f32(xx0));
vst1_f32(&a[j + 16], vget_low_f32(xx1));
vst1_f32(&a[j + 48], vget_high_f32(xx1_rev));
a[j + 48] = -a[j + 48];
vst1_f32(&a[j + 8], vget_low_f32(x1_x3_add));
vst1_f32(&a[j + 24], vget_low_f32(x1_x3_sub));
vst1_f32(&a[j + 40], vget_low_f32(yy4));
vst1_f32(&a[j + 56], vget_high_f32(yy4_rev));
}
{
const int k = 64;
const int k1 = 2;
const int k2 = 2 * k1;
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2 + 0]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2 + 0]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2 + 0]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2 + 0]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2 + 0]);
wk1rv = vld1q_f32(&rdft_wk1r[k2 + 0]);
for (j = k; j < l + k; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
float32x4_t xx4 = vmulq_f32(wk2rv, xx1);
float32x4_t xx12 = vmulq_f32(wk1rv, x1_x3_add);
float32x4_t xx22 = vmulq_f32(wk3rv, x1_x3_sub);
xx4 = vmlaq_f32(xx4, wk2iv, vrev64q_f32(xx1));
xx12 = vmlaq_f32(xx12, wk1iv, vrev64q_f32(x1_x3_add));
xx22 = vmlaq_f32(xx22, wk3iv, vrev64q_f32(x1_x3_sub));
vst1_f32(&a[j + 0], vget_low_f32(xx));
vst1_f32(&a[j + 32], vget_high_f32(xx));
vst1_f32(&a[j + 16], vget_low_f32(xx4));
vst1_f32(&a[j + 48], vget_high_f32(xx4));
vst1_f32(&a[j + 8], vget_low_f32(xx12));
vst1_f32(&a[j + 40], vget_high_f32(xx12));
vst1_f32(&a[j + 24], vget_low_f32(xx22));
vst1_f32(&a[j + 56], vget_high_f32(xx22));
}
}
}
static void rftbsub_128_neon(float* a) {
const float* c = rdft_w + 32;
int j1, j2;
const float32x4_t mm_half = vdupq_n_f32(0.5f);
a[1] = -a[1];
// Vectorized code (four at once).
// Note: commented number are indexes for the first iteration of the loop.
for (j1 = 1, j2 = 2; j2 + 7 < 64; j1 += 4, j2 += 8) {
// Load 'wk'.
const float32x4_t c_j1 = vld1q_f32(&c[j1]); // 1, 2, 3, 4,
const float32x4_t c_k1 = vld1q_f32(&c[29 - j1]); // 28, 29, 30, 31,
const float32x4_t wkrt = vsubq_f32(mm_half, c_k1); // 28, 29, 30, 31,
const float32x4_t wkr_ = reverse_order_f32x4(wkrt); // 31, 30, 29, 28,
const float32x4_t wki_ = c_j1; // 1, 2, 3, 4,
// Load and shuffle 'a'.
// 2, 4, 6, 8, 3, 5, 7, 9
float32x4x2_t a_j2_p = vld2q_f32(&a[0 + j2]);
// 120, 122, 124, 126, 121, 123, 125, 127,
const float32x4x2_t k2_0_4 = vld2q_f32(&a[122 - j2]);
// 126, 124, 122, 120
const float32x4_t a_k2_p0 = reverse_order_f32x4(k2_0_4.val[0]);
// 127, 125, 123, 121
const float32x4_t a_k2_p1 = reverse_order_f32x4(k2_0_4.val[1]);
// Calculate 'x'.
const float32x4_t xr_ = vsubq_f32(a_j2_p.val[0], a_k2_p0);
// 2-126, 4-124, 6-122, 8-120,
const float32x4_t xi_ = vaddq_f32(a_j2_p.val[1], a_k2_p1);
// 3-127, 5-125, 7-123, 9-121,
// Calculate product into 'y'.
// yr = wkr * xr - wki * xi;
// yi = wkr * xi + wki * xr;
const float32x4_t a_ = vmulq_f32(wkr_, xr_);
const float32x4_t b_ = vmulq_f32(wki_, xi_);
const float32x4_t c_ = vmulq_f32(wkr_, xi_);
const float32x4_t d_ = vmulq_f32(wki_, xr_);
const float32x4_t yr_ = vaddq_f32(a_, b_); // 2-126, 4-124, 6-122, 8-120,
const float32x4_t yi_ = vsubq_f32(c_, d_); // 3-127, 5-125, 7-123, 9-121,
// Update 'a'.
// a[j2 + 0] -= yr;
// a[j2 + 1] -= yi;
// a[k2 + 0] += yr;
// a[k2 + 1] -= yi;
// 126, 124, 122, 120,
const float32x4_t a_k2_p0n = vaddq_f32(a_k2_p0, yr_);
// 127, 125, 123, 121,
const float32x4_t a_k2_p1n = vsubq_f32(yi_, a_k2_p1);
// Shuffle in right order and store.
// 2, 3, 4, 5, 6, 7, 8, 9,
const float32x4_t a_k2_p0nr = vrev64q_f32(a_k2_p0n);
const float32x4_t a_k2_p1nr = vrev64q_f32(a_k2_p1n);
// 124, 125, 126, 127, 120, 121, 122, 123
const float32x4x2_t a_k2_n = vzipq_f32(a_k2_p0nr, a_k2_p1nr);
// 2, 4, 6, 8,
a_j2_p.val[0] = vsubq_f32(a_j2_p.val[0], yr_);
// 3, 5, 7, 9,
a_j2_p.val[1] = vsubq_f32(yi_, a_j2_p.val[1]);
// 2, 3, 4, 5, 6, 7, 8, 9,
vst2q_f32(&a[0 + j2], a_j2_p);
vst1q_f32(&a[122 - j2], a_k2_n.val[1]);
vst1q_f32(&a[126 - j2], a_k2_n.val[0]);
}
// Scalar code for the remaining items.
for (; j2 < 64; j1 += 1, j2 += 2) {
const int k2 = 128 - j2;
const int k1 = 32 - j1;
const float wkr = 0.5f - c[k1];
const float wki = c[j1];
const float xr = a[j2 + 0] - a[k2 + 0];
const float xi = a[j2 + 1] + a[k2 + 1];
const float yr = wkr * xr + wki * xi;
const float yi = wkr * xi - wki * xr;
a[j2 + 0] = a[j2 + 0] - yr;
a[j2 + 1] = yi - a[j2 + 1];
a[k2 + 0] = yr + a[k2 + 0];
a[k2 + 1] = yi - a[k2 + 1];
}
a[65] = -a[65];
}
__inline static float32x4_t reverse_order_f32x4(float32x4_t in) {
// A B C D -> C D A B
const float32x4_t rev = vcombine_f32(vget_high_f32(in), vget_low_f32(in));
// C D A B -> D C B A
return vrev64q_f32(rev);
}
void fft_real_neon(
CkFftContext* context,
const float* input,
CkFftComplex* output,
int count)
{
int countDiv2 = count/2;
fft_neon(context, (const CkFftComplex*) input, output, countDiv2, false, 1, context->fwdExpTable, context->maxCount / countDiv2);
output[countDiv2] = output[0];
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->fwdExpTable;
const CkFftComplex* exp1 = context->fwdExpTable + countDiv2 * expTableStride;
CkFftComplex* p0 = output;
CkFftComplex* p1 = output + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) p0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) p1, z1_v);
p0 += 4;
p1 -= 4;
}
if (count > 8)
{
// middle:
p0->real = p0->real * 2.0f;
p0->imag = -p0->imag * 2.0f;
}
}
void fft_real_inverse_neon(
CkFftContext* context,
const CkFftComplex* input,
float* output,
int count,
CkFftComplex* tmpBuf)
{
int countDiv2 = count/2;
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->invExpTable;
const CkFftComplex* exp1 = context->invExpTable + countDiv2 * expTableStride;
const CkFftComplex* p0 = input;
const CkFftComplex* p1 = input + countDiv2 - 3;
CkFftComplex* tmp0 = tmpBuf;
CkFftComplex* tmp1 = tmpBuf + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) tmp0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) tmp1, z1_v);
p0 += 4;
tmp0 += 4;
p1 -= 4;
tmp1 -= 4;
}
// middle:
tmp0->real = p0->real * 2.0f;
tmp0->imag = -p0->imag * 2.0f;
fft_neon(context, tmpBuf, (CkFftComplex*) output, countDiv2, true, 1, context->invExpTable, context->maxCount / countDiv2);
}
inline float32x4_t vrev64q(const float32x4_t & v) { return vrev64q_f32(v); }
static void ne10_fft_split_c2r_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fk, fnkc, fek, fok, tmp;
float32x4x2_t q2_fk, q2_fnkc, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fnkc_r, q_fnkc_i;
float32x4_t q_fek_r, q_fek_i, q_fok_r, q_fok_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
dst[0].r = (src[0].r + src[ncfft].r) * 0.5f;
dst[0].i = (src[0].r - src[ncfft].r) * 0.5f;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fk = vld2q_f32 (p_src);
q2_fnkc = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fnkc.val[0] = vrev64q_f32 (q2_fnkc.val[0]);
q2_fnkc.val[1] = vrev64q_f32 (q2_fnkc.val[1]);
q_fnkc_r = vcombine_f32 (vget_high_f32 (q2_fnkc.val[0]), vget_low_f32 (q2_fnkc.val[0]));
q_fnkc_i = vcombine_f32 (vget_high_f32 (q2_fnkc.val[1]), vget_low_f32 (q2_fnkc.val[1]));
q_fnkc_i = vnegq_f32 (q_fnkc_i);
q_fek_r = vaddq_f32 (q2_fk.val[0], q_fnkc_r);
q_fek_i = vaddq_f32 (q2_fk.val[1], q_fnkc_i);
q_tmp0 = vsubq_f32 (q2_fk.val[0], q_fnkc_r);
q_tmp1 = vsubq_f32 (q2_fk.val[1], q_fnkc_i);
q_fok_r = vmulq_f32 (q_tmp0, q2_tw.val[0]);
q_fok_i = vmulq_f32 (q_tmp1, q2_tw.val[0]);
q_tmp2 = vmulq_f32 (q_tmp1, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_tmp0, q2_tw.val[1]);
q_fok_r = vaddq_f32 (q_fok_r, q_tmp2);
q_fok_i = vsubq_f32 (q_fok_i, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_fek_r, q_fok_r);
q_dst2_i = vsubq_f32 (q_fok_i, q_fek_i);
q2_dst.val[0] = vaddq_f32 (q_fek_r, q_fok_r);
q2_dst.val[1] = vaddq_f32 (q_fek_i, q_fok_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q2_dst.val[0], q_val);
q2_dst.val[1] = vmulq_f32 (q2_dst.val[1], q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fk = src[k];
fnkc.r = src[ncfft - k].r;
fnkc.i = -src[ncfft - k].i;
fek.r = fk.r + fnkc.r;
fek.i = fk.i + fnkc.i;
tmp.r = fk.r - fnkc.r;
tmp.i = fk.i - fnkc.i;
fok.r = tmp.r * twiddles[k - 1].r + tmp.i * twiddles[k - 1].i;
fok.i = tmp.i * twiddles[k - 1].r - tmp.r * twiddles[k - 1].i;
dst[k].r = (fek.r + fok.r) * 0.5f;
dst[k].i = (fek.i + fok.i) * 0.5f;
dst[ncfft - k].r = (fek.r - fok.r) * 0.5f;
dst[ncfft - k].i = (fok.i - fek.i) * 0.5f;
}
}
}
static void ne10_fft_split_r2c_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fpnk, fpk, f1k, f2k, tw, tdc;
float32x4x2_t q2_fpk, q2_fpnk, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fpnk_r, q_fpnk_i;
float32x4_t q_f1k_r, q_f1k_i, q_f2k_r, q_f2k_i;
float32x4_t q_tw_r, q_tw_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst_r, q_dst_i, q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
tdc.r = src[0].r;
tdc.i = src[0].i;
dst[0].r = tdc.r + tdc.i;
dst[ncfft].r = tdc.r - tdc.i;
dst[ncfft].i = dst[0].i = 0;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fpk = vld2q_f32 (p_src);
q2_fpnk = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fpnk.val[0] = vrev64q_f32 (q2_fpnk.val[0]);
q2_fpnk.val[1] = vrev64q_f32 (q2_fpnk.val[1]);
q_fpnk_r = vcombine_f32 (vget_high_f32 (q2_fpnk.val[0]), vget_low_f32 (q2_fpnk.val[0]));
q_fpnk_i = vcombine_f32 (vget_high_f32 (q2_fpnk.val[1]), vget_low_f32 (q2_fpnk.val[1]));
q_fpnk_i = vnegq_f32 (q_fpnk_i);
q_f1k_r = vaddq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f1k_i = vaddq_f32 (q2_fpk.val[1], q_fpnk_i);
q_f2k_r = vsubq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f2k_i = vsubq_f32 (q2_fpk.val[1], q_fpnk_i);
q_tmp0 = vmulq_f32 (q_f2k_r, q2_tw.val[0]);
q_tmp1 = vmulq_f32 (q_f2k_i, q2_tw.val[1]);
q_tmp2 = vmulq_f32 (q_f2k_r, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_f2k_i, q2_tw.val[0]);
q_tw_r = vsubq_f32 (q_tmp0, q_tmp1);
q_tw_i = vaddq_f32 (q_tmp2, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_f1k_r, q_tw_r);
q_dst2_i = vsubq_f32 (q_tw_i, q_f1k_i);
q_dst_r = vaddq_f32 (q_f1k_r, q_tw_r);
q_dst_i = vaddq_f32 (q_f1k_i, q_tw_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q_dst_r, q_val);
q2_dst.val[1] = vmulq_f32 (q_dst_i, q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fpk = src[k];
fpnk.r = src[ncfft - k].r;
fpnk.i = - src[ncfft - k].i;
f1k.r = fpk.r + fpnk.r;
f1k.i = fpk.i + fpnk.i;
f2k.r = fpk.r - fpnk.r;
f2k.i = fpk.i - fpnk.i;
tw.r = f2k.r * (twiddles[k - 1]).r - f2k.i * (twiddles[k - 1]).i;
tw.i = f2k.r * (twiddles[k - 1]).i + f2k.i * (twiddles[k - 1]).r;
dst[k].r = (f1k.r + tw.r) * 0.5f;
dst[k].i = (f1k.i + tw.i) * 0.5f;
dst[ncfft - k].r = (f1k.r - tw.r) * 0.5f;
dst[ncfft - k].i = (tw.i - f1k.i) * 0.5f;
}
}
}