/* ---------------------------------------------------------------------------- */
COMPLEX
Cadd(COMPLEX x, COMPLEX y) {
COMPLEX z;
c_re(z) = c_re(x) + c_re(y);
c_im(z) = c_im(x) + c_im(y);
return z;
}
/* ---------------------------------------------------------------------------- */
COMPLEX
Cscl(COMPLEX x, REAL a) {
COMPLEX z;
c_re(z) = c_re(x) * a;
c_im(z) = c_im(x) * a;
return z;
}
/* ---------------------------------------------------------------------------- */
COMPLEX
Csub(COMPLEX x, COMPLEX y) {
COMPLEX z;
c_re(z) = c_re(x) - c_re(y);
c_im(z) = c_im(x) - c_im(y);
return z;
}
/* ---------------------------------------------------------------------------- */
COMPLEX
Cmul(COMPLEX x, COMPLEX y) {
COMPLEX z;
c_re(z) = c_re(x) * c_re(y) - c_im(x) * c_im(y);
c_im(z) = c_im(x) * c_re(y) + c_re(x) * c_im(y);
return z;
}
void testnd_out_of_place(int rank, int *n, fftw_direction dir,
fftwnd_plan validated_plan)
{
int istride, ostride;
int N, dim, i;
fftw_complex *in1, *in2, *out1, *out2;
fftwnd_plan p;
int flags = measure_flag | wisdom_flag;
if (coinflip())
flags |= FFTW_THREADSAFE;
N = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
in1 = (fftw_complex *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_complex));
out1 = (fftw_complex *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_complex));
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
out2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
p = fftwnd_create_plan(rank, n, dir, flags);
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/* generate random inputs */
for (i = 0; i < N; ++i) {
int j;
c_re(in2[i]) = DRAND();
c_im(in2[i]) = DRAND();
for (j = 0; j < istride; ++j) {
c_re(in1[i * istride + j]) = c_re(in2[i]);
c_im(in1[i * istride + j]) = c_im(in2[i]);
}
}
for (ostride = 1; ostride <= MAX_STRIDE; ++ostride) {
int howmany = (istride < ostride) ? istride : ostride;
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
fftwnd_threads(nthreads, p, howmany, in1, istride, 1,
out1, ostride, 1);
else
fftwnd_threads_one(nthreads, p, in1, out1);
fftwnd(validated_plan, 1, in2, 1, 1, out2, 1, 1);
for (i = 0; i < howmany; ++i)
CHECK(compute_error_complex(out1 + i, ostride, out2, 1, N)
< TOLERANCE,
"testnd_out_of_place: wrong answer");
}
}
fftwnd_destroy_plan(p);
fftw_free(out2);
fftw_free(in2);
fftw_free(out1);
fftw_free(in1);
}
double linear(dofft_closure *k, int realp,
int n, C *inA, C *inB, C *inC, C *outA,
C *outB, C *outC, C *tmp, int rounds, double tol)
{
int j;
double e = 0.0;
for (j = 0; j < rounds; ++j) {
C alpha, beta;
c_re(alpha) = mydrand();
c_im(alpha) = realp ? 0.0 : mydrand();
c_re(beta) = mydrand();
c_im(beta) = realp ? 0.0 : mydrand();
arand(inA, n);
arand(inB, n);
k->apply(k, inA, outA);
k->apply(k, inB, outB);
ascale(outA, alpha, n);
ascale(outB, beta, n);
aadd(tmp, outA, outB, n);
ascale(inA, alpha, n);
ascale(inB, beta, n);
aadd(inC, inA, inB, n);
k->apply(k, inC, outC);
e = dmax(e, acmp(outC, tmp, n, "linear", tol));
}
return e;
}
void realrft (
double *in,
unsigned n,
double *out
)
{
COMPLEX *c_in, *c_out;
unsigned i;
if (n == 0 ||
(c_in = (COMPLEX *) malloc (n * sizeof (COMPLEX))) == 0 ||
(c_out = (COMPLEX *) malloc (n * sizeof (COMPLEX))) == 0)
return;
c_re (c_in [0]) = in [0]; /* dc */
c_im (c_in [0]) = 0;
for (i = 1; i < (n + 1) / 2; i++) { /* geconj. symm. harmonischen */
c_re (c_in [i]) = in [2 * i - 1] / 2;
c_im (c_in [i]) = in [2 * i] / -2;
c_re (c_in [n - i]) = in [2 * i - 1] / 2;
c_im (c_in [n - i]) = in [2 * i] / 2;
}
if (n % 2 == 0) { /* Nyquist */
c_re (c_in [n / 2]) = in [n - 1];
c_im (c_in [n / 2]) = 0;
}
rft (c_in, n, c_out);
for (i = 0; i < n; i++)
out [i] = c_re (c_out [i]);
free ((char *) c_in);
free ((char *) c_out);
}
void fftwi_twiddle_2(fftw_complex *A, const fftw_complex *W, int iostride,
int m, int dist)
{
int i;
fftw_complex *inout;
inout = A;
for (i = m; i > 0; i = i - 1, inout = inout + dist, W = W + 1) {
fftw_real tmp1;
fftw_real tmp8;
fftw_real tmp6;
fftw_real tmp7;
ASSERT_ALIGNED_DOUBLE;
tmp1 = c_re(inout[0]);
tmp8 = c_im(inout[0]);
{
fftw_real tmp3;
fftw_real tmp5;
fftw_real tmp2;
fftw_real tmp4;
ASSERT_ALIGNED_DOUBLE;
tmp3 = c_re(inout[iostride]);
tmp5 = c_im(inout[iostride]);
tmp2 = c_re(W[0]);
tmp4 = c_im(W[0]);
tmp6 = (tmp2 * tmp3) + (tmp4 * tmp5);
tmp7 = (tmp2 * tmp5) - (tmp4 * tmp3);
}
c_re(inout[iostride]) = tmp1 - tmp6;
c_re(inout[0]) = tmp1 + tmp6;
c_im(inout[0]) = tmp7 + tmp8;
c_im(inout[iostride]) = tmp8 - tmp7;
}
}
void realfft (
double *in,
unsigned n,
double *out
)
{
COMPLEX *c_in, *c_out;
unsigned i;
if (n == 0 ||
(c_in = (COMPLEX *) malloc (n * sizeof (COMPLEX))) == 0 ||
(c_out = (COMPLEX *) malloc (n * sizeof (COMPLEX))) == 0)
return;
for (i = 0; i < n; i++) {
c_re (c_in [i]) = in [i];
c_im (c_in [i]) = 0;
}
fft (c_in, n, c_out);
out [0] = c_re (c_out [0]); /* cos van dc */
for (i = 1; i < (n + 1) / 2; i++) { /* cos/sin i-de harmonische */
out [2 * i - 1] = c_re (c_out [i]) * 2;
out [2 * i] = c_im (c_out [i]) * -2;
}
if (n % 2 == 0) /* cos van Nyquist */
out [n - 1] = c_re (c_out [n / 2]);
free ((char *) c_in);
free ((char *) c_out);
}
static void mkre01(C *a, int n) /* n should be be multiple of 4 */
{
R a0;
a0 = c_re(a[0]);
mko00(a, n/2, 0);
c_re(a[n/2]) = -(c_re(a[0]) = a0);
mkre00(a, n);
}
/* C = A - B */
void asub(C *c, C *a, C *b, int n)
{
int i;
for (i = 0; i < n; ++i) {
c_re(c[i]) = c_re(a[i]) - c_re(b[i]);
c_im(c[i]) = c_im(a[i]) - c_im(b[i]);
}
}
/* C = A */
void acopy(C *c, C *a, int n)
{
int i;
for (i = 0; i < n; ++i) {
c_re(c[i]) = c_re(a[i]);
c_im(c[i]) = c_im(a[i]);
}
}
/* C = A - B */
void array_sub(fftw_complex *C, fftw_complex *A, fftw_complex *B, int n)
{
int i;
for (i = 0; i < n; ++i) {
c_re(C[i]) = c_re(A[i]) - c_re(B[i]);
c_im(C[i]) = c_im(A[i]) - c_im(B[i]);
}
}
void accuracy_test(dofft_closure *k, aconstrain constrain,
int sign, int n, C *a, C *b, int rounds, int impulse_rounds,
double t[6])
{
int r, i;
int ntests = 0;
bench_complex czero = {0, 0};
for (i = 0; i < 6; ++i) t[i] = 0.0;
for (r = 0; r < rounds; ++r) {
arand(a, n);
if (one_accuracy_test(k, constrain, sign, n, a, b, t))
++ntests;
}
/* impulses at beginning of array */
for (r = 0; r < impulse_rounds; ++r) {
if (r > n - r - 1)
continue;
caset(a, n, czero);
c_re(a[r]) = c_im(a[r]) = 1.0;
if (one_accuracy_test(k, constrain, sign, n, a, b, t))
++ntests;
}
/* impulses at end of array */
for (r = 0; r < impulse_rounds; ++r) {
if (r <= n - r - 1)
continue;
caset(a, n, czero);
c_re(a[n - r - 1]) = c_im(a[n - r - 1]) = 1.0;
if (one_accuracy_test(k, constrain, sign, n, a, b, t))
++ntests;
}
/* randomly-located impulses */
for (r = 0; r < impulse_rounds; ++r) {
caset(a, n, czero);
i = rand() % n;
c_re(a[i]) = c_im(a[i]) = 1.0;
if (one_accuracy_test(k, constrain, sign, n, a, b, t))
++ntests;
}
t[0] /= ntests;
t[1] = sqrt(t[1] / ntests);
t[3] /= ntests;
t[4] = sqrt(t[4] / ntests);
fftaccuracy_done();
}
void fftw_no_twiddle_1(const fftw_complex *input, fftw_complex *output, int istride, int ostride)
{
fftw_real tmp1;
fftw_real tmp2;
ASSERT_ALIGNED_DOUBLE;
tmp1 = c_re(input[0]);
c_re(output[0]) = tmp1;
tmp2 = c_im(input[0]);
c_im(output[0]) = tmp2;
}
/* A = alpha * A (complex, in place) */
void ascale(C *a, C alpha, int n)
{
int i;
for (i = 0; i < n; ++i) {
R xr = c_re(a[i]), xi = c_im(a[i]);
c_re(a[i]) = xr * c_re(alpha) - xi * c_im(alpha);
c_im(a[i]) = xr * c_im(alpha) + xi * c_re(alpha);
}
}
/* A = alpha * A (in place) */
void array_scale(fftw_complex *A, fftw_complex alpha, int n)
{
int i;
for (i = 0; i < n; ++i) {
fftw_complex a = A[i];
c_re(A[i]) = c_re(a) * c_re(alpha) - c_im(a) * c_im(alpha);
c_im(A[i]) = c_re(a) * c_im(alpha) + c_im(a) * c_re(alpha);
}
}
static int
settbls(fftw_complex *w1, fftw_complex *w2, fftw_complex *w3, fftw_complex *w4,
int n1, int n2, int m1, int m2) {
int j, k, is, ir;
int ldw1, ldw2, ldw3, ldw4;
double pi2, px;
pi2 = 8.0 * atan(1.0);
px = -pi2 / n1 / n2;
ldw1 = m1;
ldw2 = m1;
ldw3 = m2;
ldw4 = n1/m1;
#ifdef _OPENMP
#pragma omp parallel
{
#pragma omp for private(j, ir)
#endif
for (k = 0; k < m2; ++k) {
for (j = 0; j < m1; ++j) {
c_re(ARR2D(w1, j, k, ldw1)) = cos(px * j * k);
c_im(ARR2D(w1, j, k, ldw1)) = sin(px * j * k);
}
for (ir = 0; ir < n1/m1; ++ir) {
c_re(ARR2D(w3, k, ir, ldw3)) = cos(px * k * ir * m1);
c_im(ARR2D(w3, k, ir, ldw3)) = sin(px * k * ir * m1);
}
}
#ifdef _OPENMP
#pragma omp for private(j, ir)
#endif
for (is = 0; is < n2/m2; ++is) {
for (j = 0; j < m1; ++j) {
c_re(ARR2D(w2, j, is, ldw2)) = cos(px * j * is * m2);
c_im(ARR2D(w2, j, is, ldw2)) = sin(px * j * is * m2);
}
for (ir = 0; ir < n1/m1; ++ir) {
c_re(ARR2D(w4, ir, is, ldw4)) = cos(px * ir * m1 * is * m2);
c_im(ARR2D(w4, ir, is, ldw4)) = sin(px * ir * m1 * is * m2);
}
}
#ifdef _OPENMP
}
#endif
return 0;
} /* settbls */
/*
* This function is called in other files, so we cannot declare
* it static.
*/
void fftw_strided_copy(int n, fftw_complex *in, int ostride,
fftw_complex *out)
{
int i;
fftw_real r0, r1, i0, i1;
fftw_real r2, r3, i2, i3;
i = 0;
for (; i < (n & 3); ++i) {
out[i * ostride] = in[i];
}
for (; i < n; i += 4) {
r0 = c_re(in[i]);
i0 = c_im(in[i]);
r1 = c_re(in[i + 1]);
i1 = c_im(in[i + 1]);
r2 = c_re(in[i + 2]);
i2 = c_im(in[i + 2]);
r3 = c_re(in[i + 3]);
i3 = c_im(in[i + 3]);
c_re(out[i * ostride]) = r0;
c_im(out[i * ostride]) = i0;
c_re(out[(i + 1) * ostride]) = r1;
c_im(out[(i + 1) * ostride]) = i1;
c_re(out[(i + 2) * ostride]) = r2;
c_im(out[(i + 2) * ostride]) = i2;
c_re(out[(i + 3) * ostride]) = r3;
c_im(out[(i + 3) * ostride]) = i3;
}
}
static void assign_conj(C *Ac, C *A, int rank, const bench_iodim *dim, int stride)
{
if (rank == 0) {
c_re(*Ac) = c_re(*A);
c_im(*Ac) = -c_im(*A);
}
else {
int i, n0 = dim[rank - 1].n, s = stride;
rank -= 1;
stride *= n0;
assign_conj(Ac, A, rank, dim, stride);
for (i = 1; i < n0; ++i)
assign_conj(Ac + (n0 - i) * s, A + i * s, rank, dim, stride);
}
}
/*
* in: array of n/2 + 1 complex numbers (* howmany).
* out: array of n real numbers (* howmany).
* work: array of n real numbers (stride 1)
*
* We must have out != in if dist < stride.
*/
void rfftw_c2real_aux(fftw_plan plan, int howmany,
fftw_complex *in, int istride, int idist,
fftw_real *out, int ostride, int odist,
fftw_real *work)
{
fftw_plan_node *p = plan->root;
switch (p->type) {
case FFTW_HC2REAL:
{
fftw_hc2real_codelet *codelet = p->nodeu.hc2real.codelet;
int j;
HACK_ALIGN_STACK_ODD();
for (j = 0; j < howmany; ++j)
codelet(&c_re(*(in + j * idist)),
&c_im(*(in + j * idist)),
out + j * odist,
istride * 2, istride * 2, ostride);
break;
}
default:
{
int j, n = plan->n;
for (j = 0; j < howmany; ++j, in += idist, out += odist) {
rfftw_c2hc(n, in, istride, work);
rfftw_executor_simple(n, work, out, p, 1, ostride);
}
break;
}
}
}
int feld_rxprocess(struct trx *trx, float *buf, int len)
{
struct feld *s = (struct feld *) trx->modem;
complex z, *zp;
int i, n;
if (trx->bandwidth != trx->hell_bandwidth) {
float lp = trx->hell_bandwidth / 2.0 / SampleRate;
fftfilt_set_freqs(s->fftfilt, 0, lp);
trx->bandwidth = trx->hell_bandwidth;
}
while (len-- > 0) {
/* create analytic signal... */
c_re(z) = c_im(z) = *buf++;
filter_run(s->hilbert, z, &z);
/* ...so it can be shifted in frequency */
z = mixer(trx, z);
n = fftfilt_run(s->fftfilt, z, &zp);
for (i = 0; i < n; i++)
feld_rx(trx, zp[i]);
}
return 0;
}
void fftw_no_twiddle_2(const fftw_complex *input, fftw_complex *output, int istride, int ostride)
{
fftw_real tmp1;
fftw_real tmp2;
fftw_real tmp3;
fftw_real tmp4;
ASSERT_ALIGNED_DOUBLE;
tmp1 = c_re(input[0]);
tmp2 = c_re(input[istride]);
c_re(output[ostride]) = tmp1 - tmp2;
c_re(output[0]) = tmp1 + tmp2;
tmp3 = c_im(input[0]);
tmp4 = c_im(input[istride]);
c_im(output[ostride]) = tmp3 - tmp4;
c_im(output[0]) = tmp3 + tmp4;
}
static void dft_apply(dofft_closure *k_, bench_complex *in, bench_complex *out)
{
dofft_dft_closure *k = (dofft_dft_closure *)k_;
bench_problem *p = k->p;
bench_tensor *totalsz, *pckdsz;
bench_tensor *totalsz_swap, *pckdsz_swap;
bench_real *ri, *ii, *ro, *io;
int totalscale;
totalsz = tensor_append(p->vecsz, p->sz);
pckdsz = verify_pack(totalsz, 2);
ri = (bench_real *) p->in;
ro = (bench_real *) p->out;
totalsz_swap = tensor_copy_swapio(totalsz);
pckdsz_swap = tensor_copy_swapio(pckdsz);
/* confusion: the stride is the distance between complex elements
when using interleaved format, but it is the distance between
real elements when using split format */
if (p->split) {
ii = p->ini ? (bench_real *) p->ini : ri + p->iphyssz;
io = p->outi ? (bench_real *) p->outi : ro + p->ophyssz;
totalscale = 1;
} else {
ii = p->ini ? (bench_real *) p->ini : ri + 1;
io = p->outi ? (bench_real *) p->outi : ro + 1;
totalscale = 2;
}
cpy(&c_re(in[0]), &c_im(in[0]), pckdsz, 1,
ri, ii, totalsz, totalscale);
after_problem_ccopy_from(p, ri, ii);
doit(1, p);
after_problem_ccopy_to(p, ro, io);
if (k->k.recopy_input)
cpy(ri, ii, totalsz_swap, totalscale,
&c_re(in[0]), &c_im(in[0]), pckdsz_swap, 1);
cpy(ro, io, totalsz, totalscale,
&c_re(out[0]), &c_im(out[0]), pckdsz, 1);
tensor_destroy(totalsz);
tensor_destroy(pckdsz);
tensor_destroy(totalsz_swap);
tensor_destroy(pckdsz_swap);
}
static int all_zero(C *a, int n)
{
int i;
for (i = 0; i < n; ++i)
if (c_re(a[i]) != 0.0 || c_im(a[i]) != 0.0)
return 0;
return 1;
}
/* reverse of rfftw_hc2c */
void rfftw_c2hc(int n, fftw_complex *in, int istride, fftw_real *out)
{
int n2 = (n + 1) / 2;
int i = 1;
out[0] = c_re(in[0]);
for (; i < ((n2 - 1) & 3) + 1; ++i) {
out[i] = c_re(in[i * istride]);
out[n - i] = c_im(in[i * istride]);
}
for (; i < n2; i += 4) {
fftw_real r0, r1, r2, r3;
fftw_real i0, i1, i2, i3;
r0 = c_re(in[i * istride]);
i0 = c_im(in[i * istride]);
r1 = c_re(in[(i + 1) * istride]);
i1 = c_im(in[(i + 1) * istride]);
r2 = c_re(in[(i + 2) * istride]);
i2 = c_im(in[(i + 2) * istride]);
r3 = c_re(in[(i + 3) * istride]);
i3 = c_im(in[(i + 3) * istride]);
out[i] = r0;
out[i + 1] = r1;
out[i + 2] = r2;
out[i + 3] = r3;
out[n - (i + 3)] = i3;
out[n - (i + 2)] = i2;
out[n - (i + 1)] = i1;
out[n - i] = i0;
}
if ((n & 1) == 0) /* store the Nyquist frequency */
out[n2] = c_re(in[n2 * istride]);
}
void aphase_shift(C *b, C *a, int n, int nb, int na, double sign)
{
int j, jb, ja;
trigreal twopin;
twopin = K2PI / n;
for (jb = 0; jb < nb; ++jb)
for (j = 0; j < n; ++j) {
trigreal s = sign * SIN(j * twopin);
trigreal c = COS(j * twopin);
for (ja = 0; ja < na; ++ja) {
int k = (jb * n + j) * na + ja;
c_re(b[k]) = c_re(a[k]) * c - c_im(a[k]) * s;
c_im(b[k]) = c_re(a[k]) * s + c_im(a[k]) * c;
}
}
}
/*
* convert real/imag arrays into one complex array
*/
void copy_ri2c(bench_real *rin, bench_real *iin, bench_complex *out,
unsigned int n)
{
unsigned int i;
for (i = 0; i < n; ++i) {
c_re(out[i]) = rin[i];
c_im(out[i]) = iin[i];
}
}
static void init_test_array(fftw_complex *arr, int stride, int n)
{
int j;
for (j = 0; j < n; ++j) {
c_re(arr[stride * j]) = 0.0;
c_im(arr[stride * j]) = 0.0;
}
}
void arand(C *a, int n)
{
int i;
/* generate random inputs */
for (i = 0; i < n; ++i) {
c_re(a[i]) = mydrand();
c_im(a[i]) = mydrand();
}
}