void test_vget_highf32 (void)
{
float32x2_t out_float32x2_t;
float32x4_t arg0_float32x4_t;
out_float32x2_t = vget_high_f32 (arg0_float32x4_t);
}
f64 dotProduct(const Size2D &_size,
const f32 * src0Base, ptrdiff_t src0Stride,
const f32 * src1Base, ptrdiff_t src1Stride)
{
internal::assertSupportedConfiguration();
#ifdef CAROTENE_NEON
Size2D size(_size);
if (src0Stride == src1Stride &&
src0Stride == (ptrdiff_t)(size.width * sizeof(f32)))
{
size.width *= size.height;
size.height = 1;
}
#define DOT_FLOAT_BLOCKSIZE (1 << 13)
f64 result = 0.0;
for (size_t row = 0; row < size.height; ++row)
{
const f32 * src0 = internal::getRowPtr(src0Base, src0Stride, row);
const f32 * src1 = internal::getRowPtr(src1Base, src1Stride, row);
size_t i = 0;
while(i + 4 <= size.width)
{
size_t lim = std::min(i + DOT_FLOAT_BLOCKSIZE, size.width) - 4;
float32x4_t v_sum = vdupq_n_f32(0.0f);
for( ; i <= lim; i += 4 )
{
internal::prefetch(src0 + i);
internal::prefetch(src1 + i);
v_sum = vmlaq_f32(v_sum, vld1q_f32(src0 + i), vld1q_f32(src1 + i));
}
float32x2_t vres = vpadd_f32(vget_low_f32(v_sum),vget_high_f32(v_sum));
result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1);
}
if(i + 2 <= size.width)
{
float32x2_t vres = vmul_f32(vld1_f32(src0 + i), vld1_f32(src1 + i));
result += vget_lane_f32(vres, 0) + vget_lane_f32(vres, 1);
i += 2;
}
for (; i < size.width; ++i)
result += src0[i] * src1[i];
}
return result;
#else
(void)_size;
(void)src0Base;
(void)src0Stride;
(void)src1Base;
(void)src1Stride;
return 0;
#endif
}
static inline void neon_make_rgb(float32x4_t macropixel, float32x4_t *rgba0p,
float32x4_t *rgba1p)
{
const float32x4_t u_coeff = {0.0, -0.34455, 1.7790, 0.0 };
const float32x4_t v_coeff = {1.4075, -0.7169, 0.0, 0.0 };
float32x4_t y0_vec, y1_vec, u_vec, v_vec, uv_vec;
float32x2_t y0_u, y1_v;
const float32_t alpha = 255.0;
/* macropixel is [Y0, U, Y1, V]. */
/* since vdupq_lane_f32 will only take two element vectors we */
/* need to pick macropixel apart to build vectors of the components. */
/* so make y0_u be the first half of macropixel [Y0, U] and */
/* y1_v be the second half [Y1, V]. */
y0_u = vget_low_f32(macropixel);
y1_v = vget_high_f32(macropixel);
/* now copy Y0 to all elements of y0_vec, then overwrite element 3 */
/* with alpha. */
y0_vec = vdupq_lane_f32(y0_u, 0);
y0_vec = vsetq_lane_f32(alpha, y0_vec, 3);
/* make u_vec be [U, U, U, U]. we'll do that using */
/* vdupq_lane_f32 and selecting U (element 1) from y0_u */
u_vec = vdupq_lane_f32(y0_u, 1);
/* now copy Y1 to all elements of y1_vec, then overwrite element 3 */
/* with alpha. */
y1_vec = vdupq_lane_f32(y1_v, 0);
y1_vec = vsetq_lane_f32(alpha, y1_vec, 3);
/* make v_vec be [V, V, V, V]. we'll do that using */
/* vdupq_lane_f32 and selecting V (element 1) from y1_v */
v_vec = vdupq_lane_f32(y1_v, 1);
/* now multiply u_vec * u_coeff and v_vec by v_coeff. */
u_vec = vmulq_f32(u_vec, u_coeff);
v_vec = vmulq_f32(v_vec, v_coeff);
/* add u_vec and v_vec to form uv_vec. use that to build */
/* rgba0 and rgba1 by adding y0_vec, y1_vec*/
uv_vec = vaddq_f32(u_vec, v_vec);
*rgba0p = vaddq_f32(y0_vec, uv_vec);
*rgba1p = vaddq_f32(y1_vec, uv_vec);
}
void dotProd_neon(const float *data, const float *weights, float *vals, const int n, const int len, const float *istd) {
for (int i = 0; i < n; i += 4) {
float32x4_t accum0 = { 0.0f, 0.0f, 0.0f, 0.0f };
float32x4_t accum1 = accum0;
float32x4_t accum2 = accum0;
float32x4_t accum3 = accum0;
for (int j = 0; j < len; j += 4) {
float32x4_t d0 = vld1q_f32(data + j);
float32x4_t d1 = d0;
float32x4_t d2 = d0;
float32x4_t d3 = d0;
float32x4_t w0 = vld1q_f32(weights);
float32x4_t w1 = vld1q_f32(weights + 4);
float32x4_t w2 = vld1q_f32(weights + 8);
float32x4_t w3 = vld1q_f32(weights + 12);
accum0 = vaddq_f32(accum0, vmulq_f32(d0, w0));
accum1 = vaddq_f32(accum1, vmulq_f32(d1, w1));
accum2 = vaddq_f32(accum2, vmulq_f32(d2, w2));
accum3 = vaddq_f32(accum3, vmulq_f32(d3, w3));
weights += 16;
}
float32x2_t sum0 = vpadd_f32(vget_low_f32(accum0), vget_high_f32(accum0));
float32x2_t sum1 = vpadd_f32(vget_low_f32(accum1), vget_high_f32(accum1));
float32x2_t sum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2));
float32x2_t sum3 = vpadd_f32(vget_low_f32(accum3), vget_high_f32(accum3));
sum0 = vpadd_f32(sum0, sum1);
sum1 = vpadd_f32(sum2, sum3);
float32x4_t sum = vcombine_f32(sum0, sum1);
sum = vmulq_n_f32(sum, istd[0]);
sum = vaddq_f32(sum, vld1q_f32(weights + n*len + i));
vst1q_f32(vals + i, sum);
}
}
//__attribute__ ((noinline))
inline vector<float, 4> cmin(vector<float, 4> const& v) noexcept
{
using vector_type = typename vector_traits<float, 4>::vector_type;
auto tmp(vpmin_f32(vget_low_f32(float32x4_t(v.data_)),
vget_high_f32(float32x4_t(v.data_))));
tmp = vpmin_f32(tmp, tmp);
return {
vector_type(vcombine_f32(tmp, tmp))
};
}
static void cft1st_128_neon(float* a) {
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
int j, k2;
for (k2 = 0, j = 0; j < 128; j += 16, k2 += 4) {
float32x4_t a00v = vld1q_f32(&a[j + 0]);
float32x4_t a04v = vld1q_f32(&a[j + 4]);
float32x4_t a08v = vld1q_f32(&a[j + 8]);
float32x4_t a12v = vld1q_f32(&a[j + 12]);
float32x4_t a01v = vcombine_f32(vget_low_f32(a00v), vget_low_f32(a08v));
float32x4_t a23v = vcombine_f32(vget_high_f32(a00v), vget_high_f32(a08v));
float32x4_t a45v = vcombine_f32(vget_low_f32(a04v), vget_low_f32(a12v));
float32x4_t a67v = vcombine_f32(vget_high_f32(a04v), vget_high_f32(a12v));
const float32x4_t wk1rv = vld1q_f32(&rdft_wk1r[k2]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2]);
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2]);
float32x4_t x0v = vaddq_f32(a01v, a23v);
const float32x4_t x1v = vsubq_f32(a01v, a23v);
const float32x4_t x2v = vaddq_f32(a45v, a67v);
const float32x4_t x3v = vsubq_f32(a45v, a67v);
const float32x4_t x3w = vrev64q_f32(x3v);
float32x4_t x0w;
a01v = vaddq_f32(x0v, x2v);
x0v = vsubq_f32(x0v, x2v);
x0w = vrev64q_f32(x0v);
a45v = vmulq_f32(wk2rv, x0v);
a45v = vmlaq_f32(a45v, wk2iv, x0w);
x0v = vmlaq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a23v = vmulq_f32(wk1rv, x0v);
a23v = vmlaq_f32(a23v, wk1iv, x0w);
x0v = vmlsq_f32(x1v, x3w, vec_swap_sign);
x0w = vrev64q_f32(x0v);
a67v = vmulq_f32(wk3rv, x0v);
a67v = vmlaq_f32(a67v, wk3iv, x0w);
a00v = vcombine_f32(vget_low_f32(a01v), vget_low_f32(a23v));
a04v = vcombine_f32(vget_low_f32(a45v), vget_low_f32(a67v));
a08v = vcombine_f32(vget_high_f32(a01v), vget_high_f32(a23v));
a12v = vcombine_f32(vget_high_f32(a45v), vget_high_f32(a67v));
vst1q_f32(&a[j + 0], a00v);
vst1q_f32(&a[j + 4], a04v);
vst1q_f32(&a[j + 8], a08v);
vst1q_f32(&a[j + 12], a12v);
}
}
static int PartitionDelayNEON(const AecCore* aec) {
// Measures the energy in each filter partition and returns the partition with
// highest energy.
// TODO(bjornv): Spread computational cost by computing one partition per
// block?
float wfEnMax = 0;
int i;
int delay = 0;
for (i = 0; i < aec->num_partitions; i++) {
int j;
int pos = i * PART_LEN1;
float wfEn = 0;
float32x4_t vec_wfEn = vdupq_n_f32(0.0f);
// vectorized code (four at once)
for (j = 0; j + 3 < PART_LEN1; j += 4) {
const float32x4_t vec_wfBuf0 = vld1q_f32(&aec->wfBuf[0][pos + j]);
const float32x4_t vec_wfBuf1 = vld1q_f32(&aec->wfBuf[1][pos + j]);
vec_wfEn = vmlaq_f32(vec_wfEn, vec_wfBuf0, vec_wfBuf0);
vec_wfEn = vmlaq_f32(vec_wfEn, vec_wfBuf1, vec_wfBuf1);
}
{
float32x2_t vec_total;
// A B C D
vec_total = vpadd_f32(vget_low_f32(vec_wfEn), vget_high_f32(vec_wfEn));
// A+B C+D
vec_total = vpadd_f32(vec_total, vec_total);
// A+B+C+D A+B+C+D
wfEn = vget_lane_f32(vec_total, 0);
}
// scalar code for the remaining items.
for (; j < PART_LEN1; j++) {
wfEn += aec->wfBuf[0][pos + j] * aec->wfBuf[0][pos + j] +
aec->wfBuf[1][pos + j] * aec->wfBuf[1][pos + j];
}
if (wfEn > wfEnMax) {
wfEnMax = wfEn;
delay = i;
}
}
return delay;
}
// Window time domain data to be used by the fft.
static void WindowDataNEON(float* x_windowed, const float* x) {
int i;
for (i = 0; i < PART_LEN; i += 4) {
const float32x4_t vec_Buf1 = vld1q_f32(&x[i]);
const float32x4_t vec_Buf2 = vld1q_f32(&x[PART_LEN + i]);
const float32x4_t vec_sqrtHanning = vld1q_f32(&WebRtcAec_sqrtHanning[i]);
// A B C D
float32x4_t vec_sqrtHanning_rev =
vld1q_f32(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]);
// B A D C
vec_sqrtHanning_rev = vrev64q_f32(vec_sqrtHanning_rev);
// D C B A
vec_sqrtHanning_rev = vcombine_f32(vget_high_f32(vec_sqrtHanning_rev),
vget_low_f32(vec_sqrtHanning_rev));
vst1q_f32(&x_windowed[i], vmulq_f32(vec_Buf1, vec_sqrtHanning));
vst1q_f32(&x_windowed[PART_LEN + i],
vmulq_f32(vec_Buf2, vec_sqrtHanning_rev));
}
}
/**
* @brief vector_dot_vector.
*
* @param dst[out] the output element(1*1)
* @param src1[in] the input vector(1*n)
* src2[in] the input vector(1*n)
* dimN[in] size of vector
*
* @return void
*/
void neon_VecdotVec(float *dst,
const float *src1,
const float *src2,
const int dimN)
{
float *mat0 = (float *)src1;
float *mat1 = (float *)src2;
float32x4_t q0 = vld1q_f32(mat0);
float32x4_t q1 = vld1q_f32(mat1);
q0 = vmulq_f32(q0, q1);
int j = 4;
for (; j <= dimN - 4; j += 4)
{
float32x4_t q2 = vld1q_f32(mat0 + j);
float32x4_t q3 = vld1q_f32(mat1 + j);
q0 = vmlaq_f32(q0, q2, q3);
}
float32x2_t d0 = vpadd_f32(vget_low_f32(q0), vget_high_f32(q0));
d0 = vpadd_f32(d0, d0);
*dst = *((float *)&d0);
for (; j < dimN; j++) {
*dst += src1[j] * src2[j];
}
}
__inline static float32x4_t reverse_order_f32x4(float32x4_t in) {
// A B C D -> C D A B
const float32x4_t rev = vcombine_f32(vget_high_f32(in), vget_low_f32(in));
// C D A B -> D C B A
return vrev64q_f32(rev);
}
/**
* @brief vector_mul_matrix.
*
* @param src1[in] the input vector(1*k)
* src2[in] the input matrix(k*n)
* dst[out] the output vector(1*n)
* kn[in] DIM_K & DIM_N
*
* @return void
*/
void neon_vectormulmatrix_float(float * dst,
const float * src1,
const float * src2,
int *kn)
{
int j ,l;
int k = kn[0];
int n = kn[1];
const float * src1_p = src1;
const float * src2_p = src2;
float * dst_p = dst;
for (j = 0; j <= n - 4; j += 4) {
float32x2_t d16 = {0};
float32x2_t d17 = {0};
float32x2_t d18 = {0};
float32x2_t d19 = {0};
float32x2_t d20;
float32x2_t d21;
float32x4_t q0;
src1_p = src1;
src2_p = src2 + j * k;
for (l = 0; l <= k - 4; l += 4) {
// Matrix A
float32x4_t q8 = vld1q_f32(src1_p);
float32x2_t d0 = vget_low_f32(q8);
float32x2_t d1 = vget_high_f32(q8);
// Matrix B
float32x4_t q12 = vld1q_f32(src2_p);
float32x4_t q13 = vld1q_f32(src2_p + k);
float32x4_t q14 = vld1q_f32(src2_p + k * 2);
float32x4_t q15 = vld1q_f32(src2_p + k * 3);
float32x2_t d8 = vget_low_f32(q12);
float32x2_t d9 = vget_high_f32(q12);
float32x2_t d10 = vget_low_f32(q13);
float32x2_t d11 = vget_high_f32(q13);
float32x2_t d12 = vget_low_f32(q14);
float32x2_t d13 = vget_high_f32(q14);
float32x2_t d14 = vget_low_f32(q15);
float32x2_t d15 = vget_high_f32(q15);
d16 = vmla_f32(d16, d0, d8);
d17 = vmla_f32(d17, d0, d10);
d18 = vmla_f32(d18, d0, d12);
d19 = vmla_f32(d19, d0, d14);
d16 = vmla_f32(d16, d1, d9);
d17 = vmla_f32(d17, d1, d11);
d18 = vmla_f32(d18, d1, d13);
d19 = vmla_f32(d19, d1, d15);
src1_p += 4;
src2_p += 4;
}// end for l
d16 = vpadd_f32(d16, d17);
d18 = vpadd_f32(d18, d19);
float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
for(; l < k; l ++) {
float src1_d;
src1_d = *src1_p;
sum0 += src1_d * *src2_p;
sum1 += src1_d * *(src2_p + k);
sum2 += src1_d * *(src2_p + 2 * k);
sum3 += src1_d * *(src2_p + 3 * k);
src1_p++;
src2_p++;
}
d20 = vset_lane_f32(sum0, d20, 0);
d20 = vset_lane_f32(sum1, d20, 1);
d21 = vset_lane_f32(sum2, d21, 0);
d21 = vset_lane_f32(sum3, d21, 1);
q0 = vaddq_f32(vcombine_f32(d16, d18), vcombine_f32(d20, d21));
vst1q_f32(dst_p, q0);
dst_p += 4;
}// end for j
}
void fft_real_neon(
CkFftContext* context,
const float* input,
CkFftComplex* output,
int count)
{
int countDiv2 = count/2;
fft_neon(context, (const CkFftComplex*) input, output, countDiv2, false, 1, context->fwdExpTable, context->maxCount / countDiv2);
output[countDiv2] = output[0];
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->fwdExpTable;
const CkFftComplex* exp1 = context->fwdExpTable + countDiv2 * expTableStride;
CkFftComplex* p0 = output;
CkFftComplex* p1 = output + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) p0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
subtract(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) p1, z1_v);
p0 += 4;
p1 -= 4;
}
if (count > 8)
{
// middle:
p0->real = p0->real * 2.0f;
p0->imag = -p0->imag * 2.0f;
}
}
void fft_real_inverse_neon(
CkFftContext* context,
const CkFftComplex* input,
float* output,
int count,
CkFftComplex* tmpBuf)
{
int countDiv2 = count/2;
int expTableStride = context->maxCount/count;
const CkFftComplex* exp0 = context->invExpTable;
const CkFftComplex* exp1 = context->invExpTable + countDiv2 * expTableStride;
const CkFftComplex* p0 = input;
const CkFftComplex* p1 = input + countDiv2 - 3;
CkFftComplex* tmp0 = tmpBuf;
CkFftComplex* tmp1 = tmpBuf + countDiv2 - 3;
const CkFftComplex* pEnd = p0 + count/4;
while (p0 < pEnd)
{
float32x4x2_t z0_v = vld2q_f32((const float32_t*) p0);
float32x4x2_t z1_v = vld2q_f32((const float32_t*) p1);
float32x2_t hi, lo;
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
float32x4x2_t sum_v;
sum_v.val[0] = vaddq_f32(z0_v.val[0], z1_v.val[0]);
sum_v.val[1] = vsubq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t diff_v;
diff_v.val[0] = vsubq_f32(z0_v.val[0], z1_v.val[0]);
diff_v.val[1] = vaddq_f32(z0_v.val[1], z1_v.val[1]);
float32x4x2_t exp_v;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 0);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 1);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 2);
exp0 += expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp0, exp_v, 3);
exp0 += expTableStride;
float32x4x2_t f_v;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
float32x4x2_t c_v;
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z0_v);
vst2q_f32((float32_t*) tmp0, z0_v);
diff_v.val[0] = vnegq_f32(diff_v.val[0]);
sum_v.val[1] = vnegq_f32(sum_v.val[1]);
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 0);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 1);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 2);
exp1 -= expTableStride;
exp_v = vld2q_lane_f32((const float32_t*) exp1, exp_v, 3);
exp1 -= expTableStride;
f_v.val[0] = vnegq_f32(exp_v.val[1]);
f_v.val[1] = exp_v.val[0];
multiply(f_v, diff_v, c_v);
add(sum_v, c_v, z1_v);
// reverse z1 real
z1_v.val[0] = vrev64q_f32(z1_v.val[0]);
hi = vget_high_f32(z1_v.val[0]);
lo = vget_low_f32(z1_v.val[0]);
z1_v.val[0] = vcombine_f32(hi, lo);
// reverse z1 imaginary
z1_v.val[1] = vrev64q_f32(z1_v.val[1]);
hi = vget_high_f32(z1_v.val[1]);
lo = vget_low_f32(z1_v.val[1]);
z1_v.val[1] = vcombine_f32(hi, lo);
vst2q_f32((float32_t*) tmp1, z1_v);
p0 += 4;
tmp0 += 4;
p1 -= 4;
tmp1 -= 4;
}
// middle:
tmp0->real = p0->real * 2.0f;
tmp0->imag = -p0->imag * 2.0f;
fft_neon(context, tmpBuf, (CkFftComplex*) output, countDiv2, true, 1, context->invExpTable, context->maxCount / countDiv2);
}
void computeNetwork0new_neon(const float *dataf, const float *weightsf, uint8_t *d) {
const int16_t *data = (const int16_t *)dataf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 128/2; i += 8) {
int16x4x2_t d0 = vld2_s16(data + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 512/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 528/4));
float32x4_t m1, m2, m3, m4;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 544/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + 560/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + 576/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + 592/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + 608/4));
uint32x4_t gte = vcgeq_f32(m1, zeroes_f);
uint16x4_t gte_u16 = vmovn_u32(gte);
uint8x8_t gte_u8 = vmovn_u16(vcombine_u16(gte_u16, vget_low_u16(vreinterpretq_u16_u32(sign_bits_f))));
gte_u8 = vshr_n_u8(gte_u8, 7);
vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(gte_u8), 0);
}
/**
* @brief Elem_t¿‡–Õæÿ’ÛA”ÎElem_t¿‡–Õæÿ’ÛBœ‡≥À.
*
* @param dst[out] ‰≥ˆæÿ’ÛC.
* src1[in] ‰»Îæÿ’ÛA.
* src2[in] ‰»Îæÿ’ÛB.
* mkn[in] æÿ’Ûµƒ∏˜∏ˆŒ¨ ˝.
*
* @return void
*/
void neon_matrixmul_4x4float(Elem_t * dst,
Elem_t * src1,
Elem_t * src2,
int *mkn)
{
int m = mkn[0];
int k = mkn[1];
int n = mkn[2];
for (int i = 0; i < m; i += 4)
{
for (int j = 0; j < n; j += 4)
{
float32x2_t d16 = {0};
float32x2_t d17 = {0};
float32x2_t d18 = {0};
float32x2_t d19 = {0};
float32x2_t d20 = {0};
float32x2_t d21 = {0};
float32x2_t d22 = {0};
float32x2_t d23 = {0};
float32x2_t d24 = {0};
float32x2_t d25 = {0};
float32x2_t d26 = {0};
float32x2_t d27 = {0};
float32x2_t d28 = {0};
float32x2_t d29 = {0};
float32x2_t d30 = {0};
float32x2_t d31 = {0};
for (int l = 0; l < k; l += 4)
{
// Matrix A
float32x4_t q8 = vld1q_f32(src1 );
float32x4_t q9 = vld1q_f32(src1 + k );
float32x4_t q10 = vld1q_f32(src1 + k*2);
float32x4_t q11 = vld1q_f32(src1 + k*3);
float32x2_t d0 = vget_low_f32(q8);
float32x2_t d1 = vget_high_f32(q8);
float32x2_t d2 = vget_low_f32(q9);
float32x2_t d3 = vget_high_f32(q9);
float32x2_t d4 = vget_low_f32(q10);
float32x2_t d5 = vget_high_f32(q10);
float32x2_t d6 = vget_low_f32(q11);
float32x2_t d7 = vget_high_f32(q11);
// Matrix B
float32x4_t q12 = vld1q_f32(src2 );
float32x4_t q13 = vld1q_f32(src2 + k );
float32x4_t q14 = vld1q_f32(src2 + k*2);
float32x4_t q15 = vld1q_f32(src2 + k*3);
float32x2_t d8 = vget_low_f32(q12);
float32x2_t d9 = vget_high_f32(q12);
float32x2_t d10 = vget_low_f32(q13);
float32x2_t d11 = vget_high_f32(q13);
float32x2_t d12 = vget_low_f32(q14);
float32x2_t d13 = vget_high_f32(q14);
float32x2_t d14 = vget_low_f32(q15);
float32x2_t d15 = vget_high_f32(q15);
d16 = vmla_f32(d16, d0, d8);
d17 = vmla_f32(d17, d0, d10);
d18 = vmla_f32(d18, d0, d12);
d19 = vmla_f32(d19, d0, d14);
d16 = vmla_f32(d16, d1, d9);
d17 = vmla_f32(d17, d1, d11);
d18 = vmla_f32(d18, d1, d13);
d19 = vmla_f32(d19, d1, d15);
d20 = vmla_f32(d20, d2, d8);
d21 = vmla_f32(d21, d2, d10);
d22 = vmla_f32(d22, d2, d12);
d23 = vmla_f32(d23, d2, d14);
d20 = vmla_f32(d20, d3, d9);
d21 = vmla_f32(d21, d3, d11);
d22 = vmla_f32(d22, d3, d13);
d23 = vmla_f32(d23, d3, d15);
d24 = vmla_f32(d24, d4, d8);
d25 = vmla_f32(d25, d4, d10);
d26 = vmla_f32(d26, d4, d12);
d27 = vmla_f32(d27, d4, d14);
d24 = vmla_f32(d24, d5, d9);
d25 = vmla_f32(d25, d5, d11);
d26 = vmla_f32(d26, d5, d13);
d27 = vmla_f32(d27, d5, d15);
d28 = vmla_f32(d28, d6, d8);
d29 = vmla_f32(d29, d6, d10);
d30 = vmla_f32(d30, d6, d12);
d31 = vmla_f32(d31, d6, d14);
d28 = vmla_f32(d28, d7, d9);
d29 = vmla_f32(d29, d7, d11);
d30 = vmla_f32(d30, d7, d13);
d31 = vmla_f32(d31, d7, d15);
src1 += 4;
src2 += 4;
}// end for l
d16 = vpadd_f32(d16, d17);
d18 = vpadd_f32(d18, d19);
d20 = vpadd_f32(d20, d21);
d22 = vpadd_f32(d22, d23);
d24 = vpadd_f32(d24, d25);
d26 = vpadd_f32(d26, d27);
d28 = vpadd_f32(d28, d29);
d30 = vpadd_f32(d30, d31);
vst1q_f32(dst , vcombine_f32(d16, d18));
vst1q_f32(dst + n , vcombine_f32(d20, d22));
vst1q_f32(dst + n*2, vcombine_f32(d24, d26));
vst1q_f32(dst + n*3, vcombine_f32(d28, d30));
src1 -= k;
src2 += k*3;
dst += 4;
}// end for j
src1 += k*4;
src2 -= k*n;
dst += n*3;
}// end for i
}
void nnp_conv1x1_upto_4x4__neon(
uint32_t input_channels_subblock_size,
uint32_t output_channels_subblock_size,
size_t input_channels,
size_t image_size,
const float* input,
const float* kernel,
float* output)
{
const float*restrict input0 = input;
const float*restrict input1 = input_channels_subblock_size > 1 ? input0 + image_size : input0;
const float*restrict input2 = input_channels_subblock_size > 2 ? input1 + image_size : input1;
const float*restrict input3 = input_channels_subblock_size > 3 ? input2 + image_size : input2;
const float*restrict kernel0 = kernel;
const float*restrict kernel1 = output_channels_subblock_size > 1 ? kernel0 + input_channels : kernel0;
const float*restrict kernel2 = output_channels_subblock_size > 2 ? kernel1 + input_channels : kernel1;
const float*restrict kernel3 = output_channels_subblock_size > 3 ? kernel2 + input_channels : kernel2;
float32x4_t vkernel0x = vld1q_dup_f32(kernel0);
float32x4_t vkernel1x = vld1q_dup_f32(kernel1);
float32x4_t vkernel2x = vld1q_dup_f32(kernel2);
float32x4_t vkernel3x = vld1q_dup_f32(kernel3);
if (input_channels_subblock_size > 1) {
vkernel0x = vld1q_lane_f32(kernel0 + 1, vkernel0x, 1);
vkernel1x = vld1q_lane_f32(kernel1 + 1, vkernel1x, 1);
vkernel2x = vld1q_lane_f32(kernel2 + 1, vkernel2x, 1);
vkernel3x = vld1q_lane_f32(kernel3 + 1, vkernel3x, 1);
if (input_channels_subblock_size > 2) {
vkernel0x = vld1q_lane_f32(kernel0 + 2, vkernel0x, 2);
vkernel1x = vld1q_lane_f32(kernel1 + 2, vkernel1x, 2);
vkernel2x = vld1q_lane_f32(kernel2 + 2, vkernel2x, 2);
vkernel3x = vld1q_lane_f32(kernel3 + 2, vkernel3x, 2);
if (input_channels_subblock_size > 3) {
vkernel0x = vld1q_lane_f32(kernel0 + 3, vkernel0x, 3);
vkernel1x = vld1q_lane_f32(kernel1 + 3, vkernel1x, 3);
vkernel2x = vld1q_lane_f32(kernel2 + 3, vkernel2x, 3);
vkernel3x = vld1q_lane_f32(kernel3 + 3, vkernel3x, 3);
}
}
}
float*restrict output0 = output;
float*restrict output1 = output_channels_subblock_size > 1 ? output0 + image_size : output0;
float*restrict output2 = output_channels_subblock_size > 2 ? output1 + image_size : output1;
float*restrict output3 = output_channels_subblock_size > 3 ? output2 + image_size : output2;
while (image_size >= 4) {
float32x4_t voutput0 = vld1q_f32(output0);
float32x4_t voutput1 = vld1q_f32(output1);
float32x4_t voutput2 = vld1q_f32(output2);
float32x4_t voutput3 = vld1q_f32(output3);
const float32x4_t vinput0 = vld1q_f32(input0); input0 += 4;
voutput0 = vmuladdq_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladdq_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladdq_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladdq_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 1) {
const float32x4_t vinput1 = vld1q_f32(input1); input1 += 4;
voutput0 = vmuladdq_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladdq_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladdq_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladdq_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 2) {
const float32x4_t vinput2 = vld1q_f32(input2); input2 += 4;
voutput0 = vmuladdq_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladdq_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladdq_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladdq_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
if (input_channels_subblock_size > 3) {
const float32x4_t vinput3 = vld1q_f32(input3); input3 += 4;
voutput0 = vmuladdq_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladdq_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladdq_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladdq_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
}
}
}
vst1q_f32(output0, voutput0); output0 += 4;
if (output_channels_subblock_size > 1) {
vst1q_f32(output1, voutput1); output1 += 4;
if (output_channels_subblock_size > 2) {
vst1q_f32(output2, voutput2); output2 += 4;
if (output_channels_subblock_size > 3) {
vst1q_f32(output3, voutput3); output3 += 4;
}
}
}
image_size -= 4;
}
if (image_size >= 2) {
float32x2_t voutput0 = vld1_f32(output0);
float32x2_t voutput1 = vld1_f32(output1);
float32x2_t voutput2 = vld1_f32(output2);
float32x2_t voutput3 = vld1_f32(output3);
const float32x2_t vinput0 = vld1_f32(input0); input0 += 2;
voutput0 = vmuladd_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 1) {
const float32x2_t vinput1 = vld1_f32(input1); input1 += 2;
voutput0 = vmuladd_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 2) {
const float32x2_t vinput2 = vld1_f32(input2); input2 += 2;
voutput0 = vmuladd_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
if (input_channels_subblock_size > 3) {
const float32x2_t vinput3 = vld1_f32(input3); input3 += 2;
voutput0 = vmuladd_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
}
}
}
vst1_f32(output0, voutput0); output0 += 2;
if (output_channels_subblock_size > 1) {
vst1_f32(output1, voutput1); output1 += 2;
if (output_channels_subblock_size > 2) {
vst1_f32(output2, voutput2); output2 += 2;
if (output_channels_subblock_size > 3) {
vst1_f32(output3, voutput3); output3 += 2;
}
}
}
image_size -= 2;
}
if (image_size != 0) {
float32x2_t voutput0 = vld1_dup_f32(output0);
float32x2_t voutput1 = vld1_dup_f32(output1);
float32x2_t voutput2 = vld1_dup_f32(output2);
float32x2_t voutput3 = vld1_dup_f32(output3);
const float32x2_t vinput0 = vld1_dup_f32(input0);
voutput0 = vmuladd_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 1) {
const float32x2_t vinput1 = vld1_dup_f32(input1);
voutput0 = vmuladd_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
if (input_channels_subblock_size > 2) {
const float32x2_t vinput2 = vld1_dup_f32(input2);
voutput0 = vmuladd_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
if (input_channels_subblock_size > 3) {
const float32x2_t vinput3 = vld1_dup_f32(input3);
voutput0 = vmuladd_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
}
}
}
vst1_lane_f32(output0, voutput0, 0);
if (output_channels_subblock_size > 1) {
vst1_lane_f32(output1, voutput1, 0);
if (output_channels_subblock_size > 2) {
vst1_lane_f32(output2, voutput2, 0);
if (output_channels_subblock_size > 3) {
vst1_lane_f32(output3, voutput3, 0);
}
}
}
}
}
float32x2_t test_vget_high_f32(float32x4_t a) {
// CHECK-LABEL: test_vget_high_f32:
return vget_high_f32(a);
// CHECK: dup d0, {{v[0-9]+}}.d[1]
}
static void cftmdl_128_neon(float* a) {
int j;
const int l = 8;
const float32x4_t vec_swap_sign = vld1q_f32((float32_t*)k_swap_sign);
float32x4_t wk1rv = vld1q_f32(cftmdl_wk1r);
for (j = 0; j < l; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx0 = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x2_t yy0_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 0);
const float32x2_t yy0_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 0);
const float32x4_t yy0_as = vcombine_f32(yy0_a, yy0_s);
const float32x2_t yy1_a = vdup_lane_f32(vget_high_f32(x1_x3_add), 1);
const float32x2_t yy1_s = vdup_lane_f32(vget_high_f32(x1_x3_sub), 1);
const float32x4_t yy1_as = vcombine_f32(yy1_a, yy1_s);
const float32x4_t yy0 = vmlaq_f32(yy0_as, vec_swap_sign, yy1_as);
const float32x4_t yy4 = vmulq_f32(wk1rv, yy0);
const float32x4_t xx1_rev = vrev64q_f32(xx1);
const float32x4_t yy4_rev = vrev64q_f32(yy4);
vst1_f32(&a[j + 0], vget_low_f32(xx0));
vst1_f32(&a[j + 32], vget_high_f32(xx0));
vst1_f32(&a[j + 16], vget_low_f32(xx1));
vst1_f32(&a[j + 48], vget_high_f32(xx1_rev));
a[j + 48] = -a[j + 48];
vst1_f32(&a[j + 8], vget_low_f32(x1_x3_add));
vst1_f32(&a[j + 24], vget_low_f32(x1_x3_sub));
vst1_f32(&a[j + 40], vget_low_f32(yy4));
vst1_f32(&a[j + 56], vget_high_f32(yy4_rev));
}
{
const int k = 64;
const int k1 = 2;
const int k2 = 2 * k1;
const float32x4_t wk2rv = vld1q_f32(&rdft_wk2r[k2 + 0]);
const float32x4_t wk2iv = vld1q_f32(&rdft_wk2i[k2 + 0]);
const float32x4_t wk1iv = vld1q_f32(&rdft_wk1i[k2 + 0]);
const float32x4_t wk3rv = vld1q_f32(&rdft_wk3r[k2 + 0]);
const float32x4_t wk3iv = vld1q_f32(&rdft_wk3i[k2 + 0]);
wk1rv = vld1q_f32(&rdft_wk1r[k2 + 0]);
for (j = k; j < l + k; j += 2) {
const float32x2_t a_00 = vld1_f32(&a[j + 0]);
const float32x2_t a_08 = vld1_f32(&a[j + 8]);
const float32x2_t a_32 = vld1_f32(&a[j + 32]);
const float32x2_t a_40 = vld1_f32(&a[j + 40]);
const float32x4_t a_00_32 = vcombine_f32(a_00, a_32);
const float32x4_t a_08_40 = vcombine_f32(a_08, a_40);
const float32x4_t x0r0_0i0_0r1_x0i1 = vaddq_f32(a_00_32, a_08_40);
const float32x4_t x1r0_1i0_1r1_x1i1 = vsubq_f32(a_00_32, a_08_40);
const float32x2_t a_16 = vld1_f32(&a[j + 16]);
const float32x2_t a_24 = vld1_f32(&a[j + 24]);
const float32x2_t a_48 = vld1_f32(&a[j + 48]);
const float32x2_t a_56 = vld1_f32(&a[j + 56]);
const float32x4_t a_16_48 = vcombine_f32(a_16, a_48);
const float32x4_t a_24_56 = vcombine_f32(a_24, a_56);
const float32x4_t x2r0_2i0_2r1_x2i1 = vaddq_f32(a_16_48, a_24_56);
const float32x4_t x3r0_3i0_3r1_x3i1 = vsubq_f32(a_16_48, a_24_56);
const float32x4_t xx = vaddq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t xx1 = vsubq_f32(x0r0_0i0_0r1_x0i1, x2r0_2i0_2r1_x2i1);
const float32x4_t x3i0_3r0_3i1_x3r1 = vrev64q_f32(x3r0_3i0_3r1_x3i1);
const float32x4_t x1_x3_add =
vmlaq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
const float32x4_t x1_x3_sub =
vmlsq_f32(x1r0_1i0_1r1_x1i1, vec_swap_sign, x3i0_3r0_3i1_x3r1);
float32x4_t xx4 = vmulq_f32(wk2rv, xx1);
float32x4_t xx12 = vmulq_f32(wk1rv, x1_x3_add);
float32x4_t xx22 = vmulq_f32(wk3rv, x1_x3_sub);
xx4 = vmlaq_f32(xx4, wk2iv, vrev64q_f32(xx1));
xx12 = vmlaq_f32(xx12, wk1iv, vrev64q_f32(x1_x3_add));
xx22 = vmlaq_f32(xx22, wk3iv, vrev64q_f32(x1_x3_sub));
vst1_f32(&a[j + 0], vget_low_f32(xx));
vst1_f32(&a[j + 32], vget_high_f32(xx));
vst1_f32(&a[j + 16], vget_low_f32(xx4));
vst1_f32(&a[j + 48], vget_high_f32(xx4));
vst1_f32(&a[j + 8], vget_low_f32(xx12));
vst1_f32(&a[j + 40], vget_high_f32(xx12));
vst1_f32(&a[j + 24], vget_low_f32(xx22));
vst1_f32(&a[j + 56], vget_high_f32(xx22));
}
}
}
// Updates the following smoothed Power Spectral Densities (PSD):
// - sd : near-end
// - se : residual echo
// - sx : far-end
// - sde : cross-PSD of near-end and residual echo
// - sxd : cross-PSD of near-end and far-end
//
// In addition to updating the PSDs, also the filter diverge state is determined
// upon actions are taken.
static void SmoothedPSD(AecCore* aec,
float efw[2][PART_LEN1],
float dfw[2][PART_LEN1],
float xfw[2][PART_LEN1],
int* extreme_filter_divergence) {
// Power estimate smoothing coefficients.
const float* ptrGCoh = aec->extended_filter_enabled
? WebRtcAec_kExtendedSmoothingCoefficients[aec->mult - 1]
: WebRtcAec_kNormalSmoothingCoefficients[aec->mult - 1];
int i;
float sdSum = 0, seSum = 0;
const float32x4_t vec_15 = vdupq_n_f32(WebRtcAec_kMinFarendPSD);
float32x4_t vec_sdSum = vdupq_n_f32(0.0f);
float32x4_t vec_seSum = vdupq_n_f32(0.0f);
for (i = 0; i + 3 < PART_LEN1; i += 4) {
const float32x4_t vec_dfw0 = vld1q_f32(&dfw[0][i]);
const float32x4_t vec_dfw1 = vld1q_f32(&dfw[1][i]);
const float32x4_t vec_efw0 = vld1q_f32(&efw[0][i]);
const float32x4_t vec_efw1 = vld1q_f32(&efw[1][i]);
const float32x4_t vec_xfw0 = vld1q_f32(&xfw[0][i]);
const float32x4_t vec_xfw1 = vld1q_f32(&xfw[1][i]);
float32x4_t vec_sd = vmulq_n_f32(vld1q_f32(&aec->sd[i]), ptrGCoh[0]);
float32x4_t vec_se = vmulq_n_f32(vld1q_f32(&aec->se[i]), ptrGCoh[0]);
float32x4_t vec_sx = vmulq_n_f32(vld1q_f32(&aec->sx[i]), ptrGCoh[0]);
float32x4_t vec_dfw_sumsq = vmulq_f32(vec_dfw0, vec_dfw0);
float32x4_t vec_efw_sumsq = vmulq_f32(vec_efw0, vec_efw0);
float32x4_t vec_xfw_sumsq = vmulq_f32(vec_xfw0, vec_xfw0);
vec_dfw_sumsq = vmlaq_f32(vec_dfw_sumsq, vec_dfw1, vec_dfw1);
vec_efw_sumsq = vmlaq_f32(vec_efw_sumsq, vec_efw1, vec_efw1);
vec_xfw_sumsq = vmlaq_f32(vec_xfw_sumsq, vec_xfw1, vec_xfw1);
vec_xfw_sumsq = vmaxq_f32(vec_xfw_sumsq, vec_15);
vec_sd = vmlaq_n_f32(vec_sd, vec_dfw_sumsq, ptrGCoh[1]);
vec_se = vmlaq_n_f32(vec_se, vec_efw_sumsq, ptrGCoh[1]);
vec_sx = vmlaq_n_f32(vec_sx, vec_xfw_sumsq, ptrGCoh[1]);
vst1q_f32(&aec->sd[i], vec_sd);
vst1q_f32(&aec->se[i], vec_se);
vst1q_f32(&aec->sx[i], vec_sx);
{
float32x4x2_t vec_sde = vld2q_f32(&aec->sde[i][0]);
float32x4_t vec_dfwefw0011 = vmulq_f32(vec_dfw0, vec_efw0);
float32x4_t vec_dfwefw0110 = vmulq_f32(vec_dfw0, vec_efw1);
vec_sde.val[0] = vmulq_n_f32(vec_sde.val[0], ptrGCoh[0]);
vec_sde.val[1] = vmulq_n_f32(vec_sde.val[1], ptrGCoh[0]);
vec_dfwefw0011 = vmlaq_f32(vec_dfwefw0011, vec_dfw1, vec_efw1);
vec_dfwefw0110 = vmlsq_f32(vec_dfwefw0110, vec_dfw1, vec_efw0);
vec_sde.val[0] = vmlaq_n_f32(vec_sde.val[0], vec_dfwefw0011, ptrGCoh[1]);
vec_sde.val[1] = vmlaq_n_f32(vec_sde.val[1], vec_dfwefw0110, ptrGCoh[1]);
vst2q_f32(&aec->sde[i][0], vec_sde);
}
{
float32x4x2_t vec_sxd = vld2q_f32(&aec->sxd[i][0]);
float32x4_t vec_dfwxfw0011 = vmulq_f32(vec_dfw0, vec_xfw0);
float32x4_t vec_dfwxfw0110 = vmulq_f32(vec_dfw0, vec_xfw1);
vec_sxd.val[0] = vmulq_n_f32(vec_sxd.val[0], ptrGCoh[0]);
vec_sxd.val[1] = vmulq_n_f32(vec_sxd.val[1], ptrGCoh[0]);
vec_dfwxfw0011 = vmlaq_f32(vec_dfwxfw0011, vec_dfw1, vec_xfw1);
vec_dfwxfw0110 = vmlsq_f32(vec_dfwxfw0110, vec_dfw1, vec_xfw0);
vec_sxd.val[0] = vmlaq_n_f32(vec_sxd.val[0], vec_dfwxfw0011, ptrGCoh[1]);
vec_sxd.val[1] = vmlaq_n_f32(vec_sxd.val[1], vec_dfwxfw0110, ptrGCoh[1]);
vst2q_f32(&aec->sxd[i][0], vec_sxd);
}
vec_sdSum = vaddq_f32(vec_sdSum, vec_sd);
vec_seSum = vaddq_f32(vec_seSum, vec_se);
}
{
float32x2_t vec_sdSum_total;
float32x2_t vec_seSum_total;
// A B C D
vec_sdSum_total = vpadd_f32(vget_low_f32(vec_sdSum),
vget_high_f32(vec_sdSum));
vec_seSum_total = vpadd_f32(vget_low_f32(vec_seSum),
vget_high_f32(vec_seSum));
// A+B C+D
vec_sdSum_total = vpadd_f32(vec_sdSum_total, vec_sdSum_total);
vec_seSum_total = vpadd_f32(vec_seSum_total, vec_seSum_total);
// A+B+C+D A+B+C+D
sdSum = vget_lane_f32(vec_sdSum_total, 0);
seSum = vget_lane_f32(vec_seSum_total, 0);
}
// scalar code for the remaining items.
for (; i < PART_LEN1; i++) {
aec->sd[i] = ptrGCoh[0] * aec->sd[i] +
ptrGCoh[1] * (dfw[0][i] * dfw[0][i] + dfw[1][i] * dfw[1][i]);
aec->se[i] = ptrGCoh[0] * aec->se[i] +
ptrGCoh[1] * (efw[0][i] * efw[0][i] + efw[1][i] * efw[1][i]);
// We threshold here to protect against the ill-effects of a zero farend.
// The threshold is not arbitrarily chosen, but balances protection and
// adverse interaction with the algorithm's tuning.
// TODO(bjornv): investigate further why this is so sensitive.
aec->sx[i] =
ptrGCoh[0] * aec->sx[i] +
ptrGCoh[1] * WEBRTC_SPL_MAX(
xfw[0][i] * xfw[0][i] + xfw[1][i] * xfw[1][i],
WebRtcAec_kMinFarendPSD);
aec->sde[i][0] =
ptrGCoh[0] * aec->sde[i][0] +
ptrGCoh[1] * (dfw[0][i] * efw[0][i] + dfw[1][i] * efw[1][i]);
aec->sde[i][1] =
ptrGCoh[0] * aec->sde[i][1] +
ptrGCoh[1] * (dfw[0][i] * efw[1][i] - dfw[1][i] * efw[0][i]);
aec->sxd[i][0] =
ptrGCoh[0] * aec->sxd[i][0] +
ptrGCoh[1] * (dfw[0][i] * xfw[0][i] + dfw[1][i] * xfw[1][i]);
aec->sxd[i][1] =
ptrGCoh[0] * aec->sxd[i][1] +
ptrGCoh[1] * (dfw[0][i] * xfw[1][i] - dfw[1][i] * xfw[0][i]);
sdSum += aec->sd[i];
seSum += aec->se[i];
}
// Divergent filter safeguard update.
aec->divergeState = (aec->divergeState ? 1.05f : 1.0f) * seSum > sdSum;
// Signal extreme filter divergence if the error is significantly larger
// than the nearend (13 dB).
*extreme_filter_divergence = (seSum > (19.95f * sdSum));
}
static void ne10_fft16_backward_float32_neon (ne10_fft_cpx_float32_t * Fout,
ne10_fft_cpx_float32_t * Fin,
ne10_fft_cpx_float32_t * twiddles)
{
ne10_fft_cpx_float32_t *tw1, *tw2, *tw3;
// the first stage
float32_t *p_src0, *p_src4, *p_src8, *p_src12;
float32x4x2_t q2_in_0123, q2_in_4567, q2_in_89ab, q2_in_cdef;
float32x4_t q_t0_r, q_t0_i, q_t1_r, q_t1_i, q_t2_r, q_t2_i, q_t3_r, q_t3_i;
float32x4_t q_out_r048c, q_out_i048c, q_out_r159d, q_out_i159d;
float32x4_t q_out_r26ae, q_out_i26ae, q_out_r37bf, q_out_i37bf;
p_src0 = (float32_t*) (& (Fin[0]));
p_src4 = (float32_t*) (& (Fin[4]));
p_src8 = (float32_t*) (& (Fin[8]));
p_src12 = (float32_t*) (& (Fin[12]));
q2_in_0123 = vld2q_f32 (p_src0);
q2_in_4567 = vld2q_f32 (p_src4);
q2_in_89ab = vld2q_f32 (p_src8);
q2_in_cdef = vld2q_f32 (p_src12);
q_t2_r = vsubq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]);
q_t2_i = vsubq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]);
q_t3_r = vaddq_f32 (q2_in_0123.val[0], q2_in_89ab.val[0]);
q_t3_i = vaddq_f32 (q2_in_0123.val[1], q2_in_89ab.val[1]);
q_t0_r = vaddq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]);
q_t0_i = vaddq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]);
q_t1_r = vsubq_f32 (q2_in_4567.val[0], q2_in_cdef.val[0]);
q_t1_i = vsubq_f32 (q2_in_4567.val[1], q2_in_cdef.val[1]);
q_out_r26ae = vsubq_f32 (q_t3_r, q_t0_r);
q_out_i26ae = vsubq_f32 (q_t3_i, q_t0_i);
q_out_r048c = vaddq_f32 (q_t3_r, q_t0_r);
q_out_i048c = vaddq_f32 (q_t3_i, q_t0_i);
q_out_r159d = vsubq_f32 (q_t2_r, q_t1_i);
q_out_i159d = vaddq_f32 (q_t2_i, q_t1_r);
q_out_r37bf = vaddq_f32 (q_t2_r, q_t1_i);
q_out_i37bf = vsubq_f32 (q_t2_i, q_t1_r);
// second stages
float32_t *p_dst0, *p_dst1, *p_dst2, *p_dst3;
float32_t *p_tw1, *p_tw2, *p_tw3;
float32x4_t q_s0_r, q_s0_i, q_s1_r, q_s1_i, q_s2_r, q_s2_i;
float32x4_t q_s3_r, q_s3_i, q_s4_r, q_s4_i, q_s5_r, q_s5_i;
float32x4x2_t q2_tmp_0, q2_tmp_1, q2_tmp_2, q2_tmp_3;
float32x4_t q_in_r0123, q_in_r4567, q_in_r89ab, q_in_rcdef;
float32x4_t q_in_i0123, q_in_i4567, q_in_i89ab, q_in_icdef;
float32x4x2_t q2_tw1, q2_tw2, q2_tw3;
float32x4x2_t q2_out_0123, q2_out_4567, q2_out_89ab, q2_out_cdef;
float32x4_t q_one_by_nfft;
tw1 = twiddles;
tw2 = twiddles + 4;
tw3 = twiddles + 8;
p_dst0 = (float32_t*) (&Fout[0]);
p_dst1 = (float32_t*) (&Fout[4]);
p_dst2 = (float32_t*) (&Fout[8]);
p_dst3 = (float32_t*) (&Fout[12]);
p_tw1 = (float32_t*) tw1;
p_tw2 = (float32_t*) tw2;
p_tw3 = (float32_t*) tw3;
q2_tmp_0 = vzipq_f32 (q_out_r048c, q_out_r159d);
q2_tmp_1 = vzipq_f32 (q_out_i048c, q_out_i159d);
q2_tmp_2 = vzipq_f32 (q_out_r26ae, q_out_r37bf);
q2_tmp_3 = vzipq_f32 (q_out_i26ae, q_out_i37bf);
q_in_r0123 = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[0]), vget_low_f32 (q2_tmp_2.val[0]));
q_in_i0123 = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[0]), vget_low_f32 (q2_tmp_3.val[0]));
q_in_r4567 = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[0]), vget_high_f32 (q2_tmp_2.val[0]));
q_in_i4567 = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[0]), vget_high_f32 (q2_tmp_3.val[0]));
q_in_r89ab = vcombine_f32 (vget_low_f32 (q2_tmp_0.val[1]), vget_low_f32 (q2_tmp_2.val[1]));
q_in_i89ab = vcombine_f32 (vget_low_f32 (q2_tmp_1.val[1]), vget_low_f32 (q2_tmp_3.val[1]));
q_in_rcdef = vcombine_f32 (vget_high_f32 (q2_tmp_0.val[1]), vget_high_f32 (q2_tmp_2.val[1]));
q_in_icdef = vcombine_f32 (vget_high_f32 (q2_tmp_1.val[1]), vget_high_f32 (q2_tmp_3.val[1]));
q2_tw1 = vld2q_f32 (p_tw1);
q2_tw2 = vld2q_f32 (p_tw2);
q2_tw3 = vld2q_f32 (p_tw3);
q_s0_r = vmulq_f32 (q_in_r4567, q2_tw1.val[0]);
q_s0_i = vmulq_f32 (q_in_i4567, q2_tw1.val[0]);
q_s1_r = vmulq_f32 (q_in_r89ab, q2_tw2.val[0]);
q_s1_i = vmulq_f32 (q_in_i89ab, q2_tw2.val[0]);
q_s2_r = vmulq_f32 (q_in_rcdef, q2_tw3.val[0]);
q_s2_i = vmulq_f32 (q_in_icdef, q2_tw3.val[0]);
q_s0_r = vmlaq_f32 (q_s0_r, q_in_i4567, q2_tw1.val[1]);
q_s0_i = vmlsq_f32 (q_s0_i, q_in_r4567, q2_tw1.val[1]);
q_s1_r = vmlaq_f32 (q_s1_r, q_in_i89ab, q2_tw2.val[1]);
q_s1_i = vmlsq_f32 (q_s1_i, q_in_r89ab, q2_tw2.val[1]);
q_s2_r = vmlaq_f32 (q_s2_r, q_in_icdef, q2_tw3.val[1]);
q_s2_i = vmlsq_f32 (q_s2_i, q_in_rcdef, q2_tw3.val[1]);
q_s5_r = vsubq_f32 (q_in_r0123, q_s1_r);
q_s5_i = vsubq_f32 (q_in_i0123, q_s1_i);
q2_out_0123.val[0] = vaddq_f32 (q_in_r0123, q_s1_r);
q2_out_0123.val[1] = vaddq_f32 (q_in_i0123, q_s1_i);
q_s3_r = vaddq_f32 (q_s0_r, q_s2_r);
q_s3_i = vaddq_f32 (q_s0_i, q_s2_i);
q_s4_r = vsubq_f32 (q_s0_r, q_s2_r);
q_s4_i = vsubq_f32 (q_s0_i, q_s2_i);
q_one_by_nfft = vdupq_n_f32 (0.0625f);
q2_out_89ab.val[0] = vsubq_f32 (q2_out_0123.val[0], q_s3_r);
q2_out_89ab.val[1] = vsubq_f32 (q2_out_0123.val[1], q_s3_i);
q2_out_0123.val[0] = vaddq_f32 (q2_out_0123.val[0], q_s3_r);
q2_out_0123.val[1] = vaddq_f32 (q2_out_0123.val[1], q_s3_i);
q2_out_4567.val[0] = vsubq_f32 (q_s5_r, q_s4_i);
q2_out_4567.val[1] = vaddq_f32 (q_s5_i, q_s4_r);
q2_out_cdef.val[0] = vaddq_f32 (q_s5_r, q_s4_i);
q2_out_cdef.val[1] = vsubq_f32 (q_s5_i, q_s4_r);
q2_out_89ab.val[0] = vmulq_f32 (q2_out_89ab.val[0], q_one_by_nfft);
q2_out_89ab.val[1] = vmulq_f32 (q2_out_89ab.val[1], q_one_by_nfft);
q2_out_0123.val[0] = vmulq_f32 (q2_out_0123.val[0], q_one_by_nfft);
q2_out_0123.val[1] = vmulq_f32 (q2_out_0123.val[1], q_one_by_nfft);
q2_out_4567.val[0] = vmulq_f32 (q2_out_4567.val[0], q_one_by_nfft);
q2_out_4567.val[1] = vmulq_f32 (q2_out_4567.val[1], q_one_by_nfft);
q2_out_cdef.val[0] = vmulq_f32 (q2_out_cdef.val[0], q_one_by_nfft);
q2_out_cdef.val[1] = vmulq_f32 (q2_out_cdef.val[1], q_one_by_nfft);
vst2q_f32 (p_dst0, q2_out_0123);
vst2q_f32 (p_dst1, q2_out_4567);
vst2q_f32 (p_dst2, q2_out_89ab);
vst2q_f32 (p_dst3, q2_out_cdef);
}
static void ne10_fft_split_c2r_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fk, fnkc, fek, fok, tmp;
float32x4x2_t q2_fk, q2_fnkc, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fnkc_r, q_fnkc_i;
float32x4_t q_fek_r, q_fek_i, q_fok_r, q_fok_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
dst[0].r = (src[0].r + src[ncfft].r) * 0.5f;
dst[0].i = (src[0].r - src[ncfft].r) * 0.5f;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fk = vld2q_f32 (p_src);
q2_fnkc = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fnkc.val[0] = vrev64q_f32 (q2_fnkc.val[0]);
q2_fnkc.val[1] = vrev64q_f32 (q2_fnkc.val[1]);
q_fnkc_r = vcombine_f32 (vget_high_f32 (q2_fnkc.val[0]), vget_low_f32 (q2_fnkc.val[0]));
q_fnkc_i = vcombine_f32 (vget_high_f32 (q2_fnkc.val[1]), vget_low_f32 (q2_fnkc.val[1]));
q_fnkc_i = vnegq_f32 (q_fnkc_i);
q_fek_r = vaddq_f32 (q2_fk.val[0], q_fnkc_r);
q_fek_i = vaddq_f32 (q2_fk.val[1], q_fnkc_i);
q_tmp0 = vsubq_f32 (q2_fk.val[0], q_fnkc_r);
q_tmp1 = vsubq_f32 (q2_fk.val[1], q_fnkc_i);
q_fok_r = vmulq_f32 (q_tmp0, q2_tw.val[0]);
q_fok_i = vmulq_f32 (q_tmp1, q2_tw.val[0]);
q_tmp2 = vmulq_f32 (q_tmp1, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_tmp0, q2_tw.val[1]);
q_fok_r = vaddq_f32 (q_fok_r, q_tmp2);
q_fok_i = vsubq_f32 (q_fok_i, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_fek_r, q_fok_r);
q_dst2_i = vsubq_f32 (q_fok_i, q_fek_i);
q2_dst.val[0] = vaddq_f32 (q_fek_r, q_fok_r);
q2_dst.val[1] = vaddq_f32 (q_fek_i, q_fok_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q2_dst.val[0], q_val);
q2_dst.val[1] = vmulq_f32 (q2_dst.val[1], q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fk = src[k];
fnkc.r = src[ncfft - k].r;
fnkc.i = -src[ncfft - k].i;
fek.r = fk.r + fnkc.r;
fek.i = fk.i + fnkc.i;
tmp.r = fk.r - fnkc.r;
tmp.i = fk.i - fnkc.i;
fok.r = tmp.r * twiddles[k - 1].r + tmp.i * twiddles[k - 1].i;
fok.i = tmp.i * twiddles[k - 1].r - tmp.r * twiddles[k - 1].i;
dst[k].r = (fek.r + fok.r) * 0.5f;
dst[k].i = (fek.i + fok.i) * 0.5f;
dst[ncfft - k].r = (fek.r - fok.r) * 0.5f;
dst[ncfft - k].i = (fok.i - fek.i) * 0.5f;
}
}
}
static void ne10_fft_split_r2c_1d_float32_neon (ne10_fft_cpx_float32_t *dst,
const ne10_fft_cpx_float32_t *src,
ne10_fft_cpx_float32_t *twiddles,
ne10_int32_t ncfft)
{
ne10_int32_t k;
ne10_int32_t count = ncfft / 2;
ne10_fft_cpx_float32_t fpnk, fpk, f1k, f2k, tw, tdc;
float32x4x2_t q2_fpk, q2_fpnk, q2_tw, q2_dst, q2_dst2;
float32x4_t q_fpnk_r, q_fpnk_i;
float32x4_t q_f1k_r, q_f1k_i, q_f2k_r, q_f2k_i;
float32x4_t q_tw_r, q_tw_i;
float32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3, q_val;
float32x4_t q_dst_r, q_dst_i, q_dst2_r, q_dst2_i;
float32_t *p_src, *p_src2, *p_dst, *p_dst2, *p_twiddles;
tdc.r = src[0].r;
tdc.i = src[0].i;
dst[0].r = tdc.r + tdc.i;
dst[ncfft].r = tdc.r - tdc.i;
dst[ncfft].i = dst[0].i = 0;
if (count >= 4)
{
for (k = 1; k <= count ; k += 4)
{
p_src = (float32_t*) (& (src[k]));
p_src2 = (float32_t*) (& (src[ncfft - k - 3]));
p_twiddles = (float32_t*) (& (twiddles[k - 1]));
p_dst = (float32_t*) (& (dst[k]));
p_dst2 = (float32_t*) (& (dst[ncfft - k - 3]));
q2_fpk = vld2q_f32 (p_src);
q2_fpnk = vld2q_f32 (p_src2);
q2_tw = vld2q_f32 (p_twiddles);
q2_fpnk.val[0] = vrev64q_f32 (q2_fpnk.val[0]);
q2_fpnk.val[1] = vrev64q_f32 (q2_fpnk.val[1]);
q_fpnk_r = vcombine_f32 (vget_high_f32 (q2_fpnk.val[0]), vget_low_f32 (q2_fpnk.val[0]));
q_fpnk_i = vcombine_f32 (vget_high_f32 (q2_fpnk.val[1]), vget_low_f32 (q2_fpnk.val[1]));
q_fpnk_i = vnegq_f32 (q_fpnk_i);
q_f1k_r = vaddq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f1k_i = vaddq_f32 (q2_fpk.val[1], q_fpnk_i);
q_f2k_r = vsubq_f32 (q2_fpk.val[0], q_fpnk_r);
q_f2k_i = vsubq_f32 (q2_fpk.val[1], q_fpnk_i);
q_tmp0 = vmulq_f32 (q_f2k_r, q2_tw.val[0]);
q_tmp1 = vmulq_f32 (q_f2k_i, q2_tw.val[1]);
q_tmp2 = vmulq_f32 (q_f2k_r, q2_tw.val[1]);
q_tmp3 = vmulq_f32 (q_f2k_i, q2_tw.val[0]);
q_tw_r = vsubq_f32 (q_tmp0, q_tmp1);
q_tw_i = vaddq_f32 (q_tmp2, q_tmp3);
q_val = vdupq_n_f32 (0.5f);
q_dst2_r = vsubq_f32 (q_f1k_r, q_tw_r);
q_dst2_i = vsubq_f32 (q_tw_i, q_f1k_i);
q_dst_r = vaddq_f32 (q_f1k_r, q_tw_r);
q_dst_i = vaddq_f32 (q_f1k_i, q_tw_i);
q_dst2_r = vmulq_f32 (q_dst2_r, q_val);
q_dst2_i = vmulq_f32 (q_dst2_i, q_val);
q2_dst.val[0] = vmulq_f32 (q_dst_r, q_val);
q2_dst.val[1] = vmulq_f32 (q_dst_i, q_val);
q_dst2_r = vrev64q_f32 (q_dst2_r);
q_dst2_i = vrev64q_f32 (q_dst2_i);
q2_dst2.val[0] = vcombine_f32 (vget_high_f32 (q_dst2_r), vget_low_f32 (q_dst2_r));
q2_dst2.val[1] = vcombine_f32 (vget_high_f32 (q_dst2_i), vget_low_f32 (q_dst2_i));
vst2q_f32 (p_dst, q2_dst);
vst2q_f32 (p_dst2, q2_dst2);
}
}
else
{
for (k = 1; k <= count ; k++)
{
fpk = src[k];
fpnk.r = src[ncfft - k].r;
fpnk.i = - src[ncfft - k].i;
f1k.r = fpk.r + fpnk.r;
f1k.i = fpk.i + fpnk.i;
f2k.r = fpk.r - fpnk.r;
f2k.i = fpk.i - fpnk.i;
tw.r = f2k.r * (twiddles[k - 1]).r - f2k.i * (twiddles[k - 1]).i;
tw.i = f2k.r * (twiddles[k - 1]).i + f2k.i * (twiddles[k - 1]).r;
dst[k].r = (f1k.r + tw.r) * 0.5f;
dst[k].i = (f1k.i + tw.i) * 0.5f;
dst[ncfft - k].r = (f1k.r - tw.r) * 0.5f;
dst[ncfft - k].i = (tw.i - f1k.i) * 0.5f;
}
}
}
float32x2_t test_vget_high_f32(float32x4_t a) {
// CHECK-COMMON-LABEL: test_vget_high_f32:
return vget_high_f32(a);
// CHECK-AARCH64: dup d0, {{v[0-9]+}}.d[1]
// CHECK-ARM64: ext v0.16b, v0.16b, v0.16b, #8
}
inline float32x2_t vget_high(const float32x4_t & v) { return vget_high_f32(v); }
void computeNetwork0_neon(const float *input, const float *weights, uint8_t *d) {
float32x4_t m0 = { 0.0f, 0.0f, 0.0f, 0.0f };
float32x4_t m1 = m0;
float32x4_t m2 = m0;
float32x4_t m3 = m0;
float32x4_t m4, m5, m6, m7;
for (int i = 0; i < 192/4; i += 4) {
m4 = vld1q_f32(input + i);
m5 = m4;
m6 = m4;
m7 = m4;
m4 = vmulq_f32(m4, vld1q_f32(weights + i * 4));
m5 = vmulq_f32(m5, vld1q_f32(weights + i * 4 + 4));
m6 = vmulq_f32(m6, vld1q_f32(weights + i * 4 + 8));
m7 = vmulq_f32(m7, vld1q_f32(weights + i * 4 + 12));
m0 = vaddq_f32(m0, m4);
m1 = vaddq_f32(m1, m5);
m2 = vaddq_f32(m2, m6);
m3 = vaddq_f32(m3, m7);
}
float32x2_t sum0 = vpadd_f32(vget_low_f32(m0), vget_high_f32(m0));
float32x2_t sum1 = vpadd_f32(vget_low_f32(m1), vget_high_f32(m1));
float32x2_t sum2 = vpadd_f32(vget_low_f32(m2), vget_high_f32(m2));
float32x2_t sum3 = vpadd_f32(vget_low_f32(m3), vget_high_f32(m3));
sum0 = vpadd_f32(sum0, sum1);
sum1 = vpadd_f32(sum2, sum3);
m0 = vcombine_f32(sum0, sum1);
m0 = vaddq_f32(m0, vld1q_f32(weights + 768/4));
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weights + 784/4));
m2 = vmulq_f32(m2, vld1q_f32(weights + (784+16)/4));
m3 = vmulq_f32(m3, vld1q_f32(weights + (784+32)/4));
m4 = vmulq_f32(m4, vld1q_f32(weights + (784+48)/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weights + (784+64)/4));
m7 = m1;
m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f));
m1 = vaddq_f32(m1, ones_f);
m7 = vmulq_f32(reciprocal(m1), m7);
m3 = m0;
m0 = vdupq_lane_f32(vget_low_f32(m0), 0);
m1 = vdupq_lane_f32(vget_low_f32(m3), 1);
m2 = vdupq_lane_f32(vget_high_f32(m3), 0);
m3 = vdupq_lane_f32(vget_high_f32(m3), 1);
m0 = vmulq_f32(m0, vld1q_f32(weights + 864/4));
m1 = vmulq_f32(m1, vld1q_f32(weights + (864+16)/4));
m2 = vmulq_f32(m2, vld1q_f32(weights + (864+32)/4));
m3 = vmulq_f32(m3, vld1q_f32(weights + (864+48)/4));
m4 = vdupq_lane_f32(vget_low_f32(m7), 0);
m5 = vdupq_lane_f32(vget_low_f32(m7), 1);
m6 = vdupq_lane_f32(vget_high_f32(m7), 0);
m7 = vdupq_lane_f32(vget_high_f32(m7), 1);
m4 = vmulq_f32(m4, vld1q_f32(weights + (864+64)/4));
m5 = vmulq_f32(m5, vld1q_f32(weights + (864+80)/4));
m6 = vmulq_f32(m6, vld1q_f32(weights + (864+96)/4));
m7 = vmulq_f32(m7, vld1q_f32(weights + (864+112)/4));
m0 = vaddq_f32(m0, m1);
m2 = vaddq_f32(m2, m3);
m4 = vaddq_f32(m4, m5);
m6 = vaddq_f32(m6, m7);
m0 = vaddq_f32(m0, m2);
m4 = vaddq_f32(m4, m6);
m0 = vaddq_f32(m0, m4);
m0 = vaddq_f32(m0, vld1q_f32(weights + (864+128)/4));
float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0));
d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0));
}
void computeNetwork0_i16_neon(const float *inputf, const float *weightsf, uint8_t *d) {
const int16_t *input = (const int16_t *)inputf;
const int16_t *weights = (const int16_t *)weightsf;
int32x4_t accum0 = { 0, 0, 0, 0 };
int32x4_t accum1 = accum0;
int32x4_t accum2 = accum0;
int32x4_t accum3 = accum0;
for (int i = 0; i < 96/2; i += 8) {
int16x4x2_t d0 = vld2_s16(input + i);
int16x4x2_t w0 = vld2_s16(weights + i * 4);
int16x4x2_t w1 = vld2_s16(weights + i * 4 + 8);
int16x4x2_t w2 = vld2_s16(weights + i * 4 + 16);
int16x4x2_t w3 = vld2_s16(weights + i * 4 + 24);
accum0 = vmlal_s16(accum0, d0.val[0], w0.val[0]);
accum0 = vmlal_s16(accum0, d0.val[1], w0.val[1]);
accum1 = vmlal_s16(accum1, d0.val[0], w1.val[0]);
accum1 = vmlal_s16(accum1, d0.val[1], w1.val[1]);
accum2 = vmlal_s16(accum2, d0.val[0], w2.val[0]);
accum2 = vmlal_s16(accum2, d0.val[1], w2.val[1]);
accum3 = vmlal_s16(accum3, d0.val[0], w3.val[0]);
accum3 = vmlal_s16(accum3, d0.val[1], w3.val[1]);
}
int32x2_t sum0 = vpadd_s32(vget_low_s32(accum0), vget_high_s32(accum0));
int32x2_t sum1 = vpadd_s32(vget_low_s32(accum1), vget_high_s32(accum1));
int32x2_t sum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
int32x2_t sum3 = vpadd_s32(vget_low_s32(accum3), vget_high_s32(accum3));
sum0 = vpadd_s32(sum0, sum1);
sum1 = vpadd_s32(sum2, sum3);
int32x4_t sum = vcombine_s32(sum0, sum1);
float32x4_t m0 = vcvtq_f32_s32(sum);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 384/4));
m0 = vaddq_f32(m0, vld1q_f32(weightsf + 400/4));
float32x4_t m1, m2, m3, m4, m5, m6, m7;
m1 = m0;
m0 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m0), sign_bits_f_zero_l));
m0 = vaddq_f32(m0, ones_f);
m0 = vmulq_f32(reciprocal(m0), m1);
m1 = vdupq_lane_f32(vget_low_f32(m0), 0);
m2 = vdupq_lane_f32(vget_low_f32(m0), 1);
m3 = vdupq_lane_f32(vget_high_f32(m0), 0);
m4 = vdupq_lane_f32(vget_high_f32(m0), 1);
m1 = vmulq_f32(m1, vld1q_f32(weightsf + 416/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (416+16)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (416+32)/4));
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (416+48)/4));
m1 = vaddq_f32(m1, m2);
m3 = vaddq_f32(m3, m4);
m1 = vaddq_f32(m1, m3);
m1 = vaddq_f32(m1, vld1q_f32(weightsf + (416+64)/4));
m7 = m1;
m1 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(m1), sign_bits_f));
m1 = vaddq_f32(m1, ones_f);
m7 = vmulq_f32(reciprocal(m1), m7);
m3 = m0;
m0 = vdupq_lane_f32(vget_low_f32(m0), 0);
m1 = vdupq_lane_f32(vget_low_f32(m3), 1);
m2 = vdupq_lane_f32(vget_high_f32(m3), 0);
m3 = vdupq_lane_f32(vget_high_f32(m3), 1);
m0 = vmulq_f32(m0, vld1q_f32(weightsf + 496/4));
m1 = vmulq_f32(m1, vld1q_f32(weightsf + (496+16)/4));
m2 = vmulq_f32(m2, vld1q_f32(weightsf + (496+32)/4));
m3 = vmulq_f32(m3, vld1q_f32(weightsf + (496+48)/4));
m4 = vdupq_lane_f32(vget_low_f32(m7), 0);
m5 = vdupq_lane_f32(vget_low_f32(m7), 1);
m6 = vdupq_lane_f32(vget_high_f32(m7), 0);
m7 = vdupq_lane_f32(vget_high_f32(m7), 1);
m4 = vmulq_f32(m4, vld1q_f32(weightsf + (496+64)/4));
m5 = vmulq_f32(m5, vld1q_f32(weightsf + (496+80)/4));
m6 = vmulq_f32(m6, vld1q_f32(weightsf + (496+96)/4));
m7 = vmulq_f32(m7, vld1q_f32(weightsf + (496+112)/4));
m0 = vaddq_f32(m0, m1);
m2 = vaddq_f32(m2, m3);
m4 = vaddq_f32(m4, m5);
m6 = vaddq_f32(m6, m7);
m0 = vaddq_f32(m0, m2);
m4 = vaddq_f32(m4, m6);
m0 = vaddq_f32(m0, m4);
m0 = vaddq_f32(m0, vld1q_f32(weightsf + (496+128)/4));
float32x2_t maximum = vmax_f32(vget_low_f32(m0), vget_high_f32(m0));
d[0] = (vget_lane_f32(maximum, 1) <= vget_lane_f32(maximum, 0));
}
void nnp_conv1x1_only_4x4__neon(
size_t input_channels,
size_t image_size,
const float* input,
const float* kernel,
float* output)
{
const float* input0 = input;
const float* input1 = input0 + image_size;
const float* input2 = input1 + image_size;
const float* input3 = input2 + image_size;
const float32x4_t vkernel0x = vld1q_f32(kernel);
kernel += input_channels;
const float32x4_t vkernel1x = vld1q_f32(kernel);
kernel += input_channels;
const float32x4_t vkernel2x = vld1q_f32(kernel);
kernel += input_channels;
const float32x4_t vkernel3x = vld1q_f32(kernel);
float* output0 = output;
float* output1 = output0 + image_size;
float* output2 = output1 + image_size;
float* output3 = output2 + image_size;
while (image_size >= 4) {
float32x4_t voutput0 = vld1q_f32(output0);
float32x4_t voutput1 = vld1q_f32(output1);
float32x4_t voutput2 = vld1q_f32(output2);
float32x4_t voutput3 = vld1q_f32(output3);
const float32x4_t vinput0 = vld1q_f32(input0); input0 += 4;
voutput0 = vmuladdq_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladdq_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladdq_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladdq_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
const float32x4_t vinput1 = vld1q_f32(input1); input1 += 4;
voutput0 = vmuladdq_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladdq_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladdq_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladdq_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
const float32x4_t vinput2 = vld1q_f32(input2); input2 += 4;
voutput0 = vmuladdq_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladdq_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladdq_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladdq_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
const float32x4_t vinput3 = vld1q_f32(input3); input3 += 4;
voutput0 = vmuladdq_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladdq_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladdq_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladdq_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
vst1q_f32(output0, voutput0); output0 += 4;
vst1q_f32(output1, voutput1); output1 += 4;
vst1q_f32(output2, voutput2); output2 += 4;
vst1q_f32(output3, voutput3); output3 += 4;
image_size -= 4;
}
if (image_size >= 2) {
float32x2_t voutput0 = vld1_f32(output0);
float32x2_t voutput1 = vld1_f32(output1);
float32x2_t voutput2 = vld1_f32(output2);
float32x2_t voutput3 = vld1_f32(output3);
const float32x2_t vinput0 = vld1_f32(input0); input0 += 2;
voutput0 = vmuladd_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
const float32x2_t vinput1 = vld1_f32(input1); input1 += 2;
voutput0 = vmuladd_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
const float32x2_t vinput2 = vld1_f32(input2); input2 += 2;
voutput0 = vmuladd_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
const float32x2_t vinput3 = vld1_f32(input3); input3 += 2;
voutput0 = vmuladd_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
vst1_f32(output0, voutput0); output0 += 2;
vst1_f32(output1, voutput1); output1 += 2;
vst1_f32(output2, voutput2); output2 += 2;
vst1_f32(output3, voutput3); output3 += 2;
image_size -= 2;
}
if (image_size != 0) {
float32x2_t voutput0 = vld1_dup_f32(output0);
float32x2_t voutput1 = vld1_dup_f32(output1);
float32x2_t voutput2 = vld1_dup_f32(output2);
float32x2_t voutput3 = vld1_dup_f32(output3);
const float32x2_t vinput0 = vld1_dup_f32(input0);
voutput0 = vmuladd_lane0_f32(voutput0, vinput0, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput0, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput0, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput0, vget_low_f32(vkernel3x));
const float32x2_t vinput1 = vld1_dup_f32(input1);
voutput0 = vmuladd_lane1_f32(voutput0, vinput1, vget_low_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput1, vget_low_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput1, vget_low_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput1, vget_low_f32(vkernel3x));
const float32x2_t vinput2 = vld1_dup_f32(input2);
voutput0 = vmuladd_lane0_f32(voutput0, vinput2, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane0_f32(voutput1, vinput2, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane0_f32(voutput2, vinput2, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane0_f32(voutput3, vinput2, vget_high_f32(vkernel3x));
const float32x2_t vinput3 = vld1_dup_f32(input3);
voutput0 = vmuladd_lane1_f32(voutput0, vinput3, vget_high_f32(vkernel0x));
voutput1 = vmuladd_lane1_f32(voutput1, vinput3, vget_high_f32(vkernel1x));
voutput2 = vmuladd_lane1_f32(voutput2, vinput3, vget_high_f32(vkernel2x));
voutput3 = vmuladd_lane1_f32(voutput3, vinput3, vget_high_f32(vkernel3x));
vst1_lane_f32(output0, voutput0, 0);
vst1_lane_f32(output1, voutput1, 0);
vst1_lane_f32(output2, voutput2, 0);
vst1_lane_f32(output3, voutput3, 0);
}
}
// CHECK-LABEL: define <2 x float> @test_vget_high_f32(<4 x float> %a) #0 {
// CHECK: [[SHUFFLE_I:%.*]] = shufflevector <4 x float> %a, <4 x float> %a, <2 x i32> <i32 2, i32 3>
// CHECK: ret <2 x float> [[SHUFFLE_I]]
float32x2_t test_vget_high_f32(float32x4_t a) {
return vget_high_f32(a);
}
