void testnd_out_of_place(int rank, int *n, fftw_direction dir,
fftwnd_plan validated_plan)
{
int istride, ostride;
int N, dim, i;
fftw_complex *in1, *in2, *out1, *out2;
fftwnd_plan p;
int flags = measure_flag | wisdom_flag;
if (coinflip())
flags |= FFTW_THREADSAFE;
N = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
in1 = (fftw_complex *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_complex));
out1 = (fftw_complex *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_complex));
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
out2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
p = fftwnd_create_plan(rank, n, dir, flags);
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/* generate random inputs */
for (i = 0; i < N; ++i) {
int j;
c_re(in2[i]) = DRAND();
c_im(in2[i]) = DRAND();
for (j = 0; j < istride; ++j) {
c_re(in1[i * istride + j]) = c_re(in2[i]);
c_im(in1[i * istride + j]) = c_im(in2[i]);
}
}
for (ostride = 1; ostride <= MAX_STRIDE; ++ostride) {
int howmany = (istride < ostride) ? istride : ostride;
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
fftwnd_threads(nthreads, p, howmany, in1, istride, 1,
out1, ostride, 1);
else
fftwnd_threads_one(nthreads, p, in1, out1);
fftwnd(validated_plan, 1, in2, 1, 1, out2, 1, 1);
for (i = 0; i < howmany; ++i)
CHECK(compute_error_complex(out1 + i, ostride, out2, 1, N)
< TOLERANCE,
"testnd_out_of_place: wrong answer");
}
}
fftwnd_destroy_plan(p);
fftw_free(out2);
fftw_free(in2);
fftw_free(out1);
fftw_free(in1);
}
void array_compare(fftw_complex *A, fftw_complex *B, int n)
{
double d = compute_error_complex(A, 1, B, 1, n);
if (d > TOLERANCE) {
fflush(stdout);
fprintf(stderr, "Found relative error %e\n", d);
fftw_die("failure in Ergun's verification procedure\n");
}
}
void testnd_in_place(int rank, int *n, fftw_direction dir,
fftwnd_plan validated_plan,
int alternate_api, int specific, int force_buffered)
{
int local_nx, local_x_start, local_ny_after_transpose,
local_y_start_after_transpose, total_local_size;
int istride;
int N, dim, i;
fftw_complex *in1, *work = 0, *in2;
fftwnd_mpi_plan p = 0;
int flags = measure_flag | wisdom_flag | FFTW_IN_PLACE;
if (specific || rank < 2)
return;
if (coinflip())
flags |= FFTW_THREADSAFE;
if (force_buffered)
flags |= FFTWND_FORCE_BUFFERED;
N = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
if (alternate_api && (rank == 2 || rank == 3)) {
if (rank == 2)
p = fftw2d_mpi_create_plan(MPI_COMM_WORLD,
n[0], n[1], dir, flags);
else
p = fftw3d_mpi_create_plan(MPI_COMM_WORLD,
n[0], n[1], n[2], dir, flags);
}
else /* standard api */
p = fftwnd_mpi_create_plan(MPI_COMM_WORLD, rank, n, dir, flags);
fftwnd_mpi_local_sizes(p, &local_nx, &local_x_start,
&local_ny_after_transpose,
&local_y_start_after_transpose,
&total_local_size);
in1 = (fftw_complex *) fftw_malloc(total_local_size * MAX_STRIDE
* sizeof(fftw_complex));
if (coinflip()) {
WHEN_VERBOSE(1, my_printf("w/work..."));
work = (fftw_complex *) fftw_malloc(total_local_size * MAX_STRIDE
* sizeof(fftw_complex));
}
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/* generate random inputs */
for (i = 0; i < N; ++i) {
c_re(in2[i]) = DRAND();
c_im(in2[i]) = DRAND();
}
for (i = 0; i < local_nx * (N/n[0]); ++i) {
int j;
for (j = 0; j < istride; ++j) {
c_re(in1[i * istride + j]) = c_re((in2 + local_x_start
* (N/n[0])) [i]);
c_im(in1[i * istride + j]) = c_im((in2 + local_x_start
* (N/n[0])) [i]);
}
}
fftwnd_mpi(p, istride, in1, work, FFTW_NORMAL_ORDER);
fftwnd(validated_plan, 1, in2, 1, 1, NULL, 0, 0);
for (i = 0; i < istride; ++i)
CHECK(compute_error_complex(in1 + i, istride,
in2 + local_x_start * (N/n[0]),
1, local_nx * (N/n[0])) < TOLERANCE,
"testnd_in_place: wrong answer");
}
fftwnd_mpi_destroy_plan(p);
fftw_free(in2);
fftw_free(work);
fftw_free(in1);
}
void test_in_place(int n, int istride, int howmany, fftw_direction dir,
fftw_plan validated_plan, int specific)
{
int local_n, local_start, local_n_after_transform,
local_start_after_transform, total_local_size;
fftw_complex *in1, *work = NULL, *in2, *out2;
fftw_mpi_plan plan;
int i;
int flags = measure_flag | wisdom_flag | FFTW_IN_PLACE;
if (specific) {
WHEN_VERBOSE(2, my_printf("N/A\n"));
return;
}
if (coinflip())
flags |= FFTW_THREADSAFE;
plan = fftw_mpi_create_plan(MPI_COMM_WORLD, n, dir, flags);
fftw_mpi_local_sizes(plan, &local_n, &local_start,
&local_n_after_transform,
&local_start_after_transform,
&total_local_size);
in1 = (fftw_complex *) fftw_malloc(total_local_size
* sizeof(fftw_complex) * howmany);
if (coinflip()) {
WHEN_VERBOSE(2, my_printf("w/work..."));
work = (fftw_complex *) fftw_malloc(total_local_size
* sizeof(fftw_complex) * howmany);
}
in2 = (fftw_complex *) fftw_malloc(n * sizeof(fftw_complex) * howmany);
out2 = (fftw_complex *) fftw_malloc(n * sizeof(fftw_complex) * howmany);
/* generate random inputs */
for (i = 0; i < n * howmany; ++i) {
c_re(in2[i]) = DRAND();
c_im(in2[i]) = DRAND();
}
for (i = 0; i < local_n * howmany; ++i) {
c_re(in1[i]) = c_re(in2[i + local_start*howmany]);
c_im(in1[i]) = c_im(in2[i + local_start*howmany]);
}
/* fft-ize */
fftw_mpi(plan, howmany, in1, work);
fftw_mpi_destroy_plan(plan);
fftw(validated_plan, howmany, in2, howmany, 1, out2, howmany, 1);
CHECK(compute_error_complex(in1, 1,
out2 + local_start_after_transform*howmany, 1,
howmany*local_n_after_transform) < TOLERANCE,
"test_in_place: wrong answer");
WHEN_VERBOSE(2, my_printf("OK\n"));
fftw_free(in1);
fftw_free(work);
fftw_free(in2);
fftw_free(out2);
}
void testnd_in_place(int rank, int *n, fftw_direction dir,
fftwnd_plan validated_plan,
int alternate_api, int specific, int force_buffered)
{
int istride;
int N, dim, i;
fftw_complex *in1, *in2, *out2;
fftwnd_plan p;
int flags = measure_flag | wisdom_flag | FFTW_IN_PLACE;
if (coinflip())
flags |= FFTW_THREADSAFE;
if (force_buffered)
flags |= FFTWND_FORCE_BUFFERED;
N = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
in1 = (fftw_complex *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_complex));
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
out2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
if (!specific) {
if (alternate_api && (rank == 2 || rank == 3)) {
if (rank == 2)
p = fftw2d_create_plan(n[0], n[1], dir, flags);
else
p = fftw3d_create_plan(n[0], n[1], n[2], dir, flags);
} else /* standard api */
p = fftwnd_create_plan(rank, n, dir, flags);
} else { /* specific plan creation */
if (alternate_api && (rank == 2 || rank == 3)) {
if (rank == 2)
p = fftw2d_create_plan_specific(n[0], n[1], dir, flags,
in1, 1,
(fftw_complex *) NULL, 1);
else
p = fftw3d_create_plan_specific(n[0], n[1], n[2], dir, flags,
in1, 1,
(fftw_complex *) NULL, 1);
} else /* standard api */
p = fftwnd_create_plan_specific(rank, n, dir, flags,
in1, 1,
(fftw_complex *) NULL, 1);
}
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/*
* generate random inputs */
for (i = 0; i < N; ++i) {
int j;
c_re(in2[i]) = DRAND();
c_im(in2[i]) = DRAND();
for (j = 0; j < istride; ++j) {
c_re(in1[i * istride + j]) = c_re(in2[i]);
c_im(in1[i * istride + j]) = c_im(in2[i]);
}
}
if (istride != 1 || istride != 1 || coinflip())
fftwnd(p, istride, in1, istride, 1, (fftw_complex *) NULL, 1, 1);
else
fftwnd_one(p, in1, NULL);
fftwnd(validated_plan, 1, in2, 1, 1, out2, 1, 1);
for (i = 0; i < istride; ++i)
CHECK(compute_error_complex(in1 + i, istride, out2, 1, N) < TOLERANCE,
"testnd_in_place: wrong answer");
}
fftwnd_destroy_plan(p);
fftw_free(out2);
fftw_free(in2);
fftw_free(in1);
}
void test_in_place(int n, int istride, int howmany, fftw_direction dir,
fftw_plan validated_plan, int specific)
{
fftw_complex *in1, *in2, *out2;
fftw_plan plan;
int i, j;
int flags = measure_flag | wisdom_flag | FFTW_IN_PLACE;
if (coinflip())
flags |= FFTW_THREADSAFE;
in1 = (fftw_complex *) fftw_malloc(istride * n * sizeof(fftw_complex) * howmany);
in2 = (fftw_complex *) fftw_malloc(n * sizeof(fftw_complex) * howmany);
out2 = (fftw_complex *) fftw_malloc(n * sizeof(fftw_complex) * howmany);
if (!specific)
plan = fftw_create_plan(n, dir, flags);
else
plan = fftw_create_plan_specific(n, dir, flags,
in1, istride,
(fftw_complex *) NULL, 0);
/* generate random inputs */
for (i = 0; i < n * howmany; ++i) {
c_re(in1[i * istride]) = c_re(in2[i]) = DRAND();
c_im(in1[i * istride]) = c_im(in2[i]) = DRAND();
}
/*
* fill in other positions of the array, to make sure that
* fftw doesn't overwrite them
*/
for (j = 1; j < istride; ++j)
for (i = 0; i < n * howmany; ++i) {
c_re(in1[i * istride + j]) = i * istride + j;
c_im(in1[i * istride + j]) = i * istride - j;
}
CHECK(plan != NULL, "can't create plan");
WHEN_VERBOSE(2, fftw_print_plan(plan));
/* fft-ize */
if (howmany != 1 || istride != 1 || coinflip())
fftw(plan, howmany, in1, istride, n * istride,
(fftw_complex *) NULL, 0, 0);
else
fftw_one(plan, in1, NULL);
fftw_destroy_plan(plan);
/* check for overwriting */
for (j = 1; j < istride; ++j)
for (i = 0; i < n * howmany; ++i)
CHECK(c_re(in1[i * istride + j]) == i * istride + j &&
c_im(in1[i * istride + j]) == i * istride - j,
"input has been overwritten");
for (i = 0; i < howmany; ++i) {
fftw(validated_plan, 1, in2 + n * i, 1, n, out2 + n * i, 1, n);
}
CHECK(compute_error_complex(in1, istride, out2, 1, n * howmany) < TOLERANCE,
"test_in_place: wrong answer");
WHEN_VERBOSE(2, printf("OK\n"));
fftw_free(in1);
fftw_free(in2);
fftw_free(out2);
}
void testnd_in_place(int rank, int *n, fftwnd_plan validated_plan,
int alternate_api, int specific)
{
int istride, ostride, howmany;
int N, dim, i, j, k;
int nc, nhc, nr;
fftw_real *in1, *out3;
fftw_complex *in2, *out1, *out2;
fftwnd_plan p, ip;
int flags = measure_flag | wisdom_flag | FFTW_IN_PLACE;
if (coinflip())
flags |= FFTW_THREADSAFE;
N = nc = nr = nhc = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
if (rank > 0) {
nr = n[rank - 1];
nc = N / nr;
nhc = nr / 2 + 1;
}
in1 = (fftw_real *) fftw_malloc(2 * nhc * nc * MAX_STRIDE * sizeof(fftw_real));
out3 = in1;
out1 = (fftw_complex *) in1;
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
out2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
if (alternate_api && specific && (rank == 2 || rank == 3)) {
if (rank == 2) {
p = rfftw2d_create_plan_specific(n[0], n[1],
FFTW_REAL_TO_COMPLEX, flags,
in1, MAX_STRIDE, 0, 0);
ip = rfftw2d_create_plan_specific(n[0], n[1],
FFTW_COMPLEX_TO_REAL, flags,
in1, MAX_STRIDE, 0, 0);
} else {
p = rfftw3d_create_plan_specific(n[0], n[1], n[2],
FFTW_REAL_TO_COMPLEX, flags,
in1, MAX_STRIDE, 0, 0);
ip = rfftw3d_create_plan_specific(n[0], n[1], n[2],
FFTW_COMPLEX_TO_REAL, flags,
in1, MAX_STRIDE, 0, 0);
}
} else if (specific) {
p = rfftwnd_create_plan_specific(rank, n, FFTW_REAL_TO_COMPLEX,
flags,
in1, MAX_STRIDE, in1, MAX_STRIDE);
ip = rfftwnd_create_plan_specific(rank, n, FFTW_COMPLEX_TO_REAL,
flags,
in1, MAX_STRIDE, in1, MAX_STRIDE);
} else if (alternate_api && (rank == 2 || rank == 3)) {
if (rank == 2) {
p = rfftw2d_create_plan(n[0], n[1], FFTW_REAL_TO_COMPLEX,
flags);
ip = rfftw2d_create_plan(n[0], n[1], FFTW_COMPLEX_TO_REAL,
flags);
} else {
p = rfftw3d_create_plan(n[0], n[1], n[2], FFTW_REAL_TO_COMPLEX,
flags);
ip = rfftw3d_create_plan(n[0], n[1], n[2], FFTW_COMPLEX_TO_REAL,
flags);
}
} else {
p = rfftwnd_create_plan(rank, n, FFTW_REAL_TO_COMPLEX, flags);
ip = rfftwnd_create_plan(rank, n, FFTW_COMPLEX_TO_REAL, flags);
}
CHECK(p != NULL && ip != NULL, "can't create plan");
for (i = 0; i < nc * nhc * 2 * MAX_STRIDE; ++i)
out3[i] = 0;
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/* generate random inputs */
for (i = 0; i < nc; ++i)
for (j = 0; j < nr; ++j) {
c_re(in2[i * nr + j]) = DRAND();
c_im(in2[i * nr + j]) = 0.0;
for (k = 0; k < istride; ++k)
in1[(i * nhc * 2 + j) * istride + k]
= c_re(in2[i * nr + j]);
}
fftwnd(validated_plan, 1, in2, 1, 1, out2, 1, 1);
howmany = ostride = istride;
WHEN_VERBOSE(2, printf("\n testing in-place stride %d...",
istride));
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
rfftwnd_real_to_complex(p, howmany, in1, istride, 1,
out1, ostride, 1);
else
rfftwnd_one_real_to_complex(p, in1, NULL);
for (i = 0; i < nc; ++i)
for (k = 0; k < howmany; ++k)
CHECK(compute_error_complex(out1 + i * nhc * ostride + k,
ostride,
out2 + i * nr, 1,
nhc) < TOLERANCE,
"in-place (r2c): wrong answer");
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
rfftwnd_complex_to_real(ip, howmany, out1, ostride, 1,
out3, istride, 1);
else
rfftwnd_one_complex_to_real(ip, out1, NULL);
for (i = 0; i < nc * nhc * 2 * istride; ++i)
out3[i] *= 1.0 / N;
for (i = 0; i < nc; ++i)
for (k = 0; k < howmany; ++k)
CHECK(compute_error(out3 + i * nhc * 2 * istride + k,
istride,
(fftw_real *) (in2 + i * nr), 2,
nr) < TOLERANCE,
"in-place (c2r): wrong answer (check 2)");
}
rfftwnd_destroy_plan(p);
rfftwnd_destroy_plan(ip);
fftw_free(out2);
fftw_free(in2);
fftw_free(in1);
}
void testnd_out_of_place(int rank, int *n, fftwnd_plan validated_plan)
{
int istride, ostride;
int N, dim, i, j, k;
int nc, nhc, nr;
fftw_real *in1, *out3;
fftw_complex *in2, *out1, *out2;
fftwnd_plan p, ip;
int flags = measure_flag | wisdom_flag;
if (coinflip())
flags |= FFTW_THREADSAFE;
N = nc = nr = nhc = 1;
for (dim = 0; dim < rank; ++dim)
N *= n[dim];
if (rank > 0) {
nr = n[rank - 1];
nc = N / nr;
nhc = nr / 2 + 1;
}
in1 = (fftw_real *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_real));
out3 = (fftw_real *) fftw_malloc(N * MAX_STRIDE * sizeof(fftw_real));
out1 = (fftw_complex *) fftw_malloc(nhc * nc * MAX_STRIDE
* sizeof(fftw_complex));
in2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
out2 = (fftw_complex *) fftw_malloc(N * sizeof(fftw_complex));
p = rfftwnd_create_plan(rank, n, FFTW_REAL_TO_COMPLEX, flags);
ip = rfftwnd_create_plan(rank, n, FFTW_COMPLEX_TO_REAL, flags);
CHECK(p != NULL && ip != NULL, "can't create plan");
for (istride = 1; istride <= MAX_STRIDE; ++istride) {
/* generate random inputs */
for (i = 0; i < nc; ++i)
for (j = 0; j < nr; ++j) {
c_re(in2[i * nr + j]) = DRAND();
c_im(in2[i * nr + j]) = 0.0;
for (k = 0; k < istride; ++k)
in1[(i * nr + j) * istride + k]
= c_re(in2[i * nr + j]);
}
for (i = 0; i < N * istride; ++i)
out3[i] = 0.0;
fftwnd(validated_plan, 1, in2, 1, 1, out2, 1, 1);
for (ostride = 1; ostride <= MAX_STRIDE; ++ostride) {
int howmany = (istride < ostride) ? istride : ostride;
WHEN_VERBOSE(2, printf("\n testing stride %d/%d...",
istride, ostride));
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
rfftwnd_real_to_complex(p, howmany, in1, istride, 1,
out1, ostride, 1);
else
rfftwnd_one_real_to_complex(p, in1, out1);
for (i = 0; i < nc; ++i)
for (k = 0; k < howmany; ++k)
CHECK(compute_error_complex(out1 + i * nhc * ostride + k,
ostride,
out2 + i * nr, 1,
nhc) < TOLERANCE,
"out-of-place (r2c): wrong answer");
if (howmany != 1 || istride != 1 || ostride != 1 || coinflip())
rfftwnd_complex_to_real(ip, howmany, out1, ostride, 1,
out3, istride, 1);
else
rfftwnd_one_complex_to_real(ip, out1, out3);
for (i = 0; i < N * istride; ++i)
out3[i] *= 1.0 / N;
if (istride == howmany)
CHECK(compute_error(out3, 1, in1, 1, N * istride)
< TOLERANCE, "out-of-place (c2r): wrong answer");
for (i = 0; i < nc; ++i)
for (k = 0; k < howmany; ++k)
CHECK(compute_error(out3 + i * nr * istride + k,
istride,
(fftw_real *) (in2 + i * nr), 2,
nr) < TOLERANCE,
"out-of-place (c2r): wrong answer (check 2)");
}
}
rfftwnd_destroy_plan(p);
rfftwnd_destroy_plan(ip);
fftw_free(out3);
fftw_free(out2);
fftw_free(in2);
fftw_free(out1);
fftw_free(in1);
}
