void fftwnd(fftwnd_plan p, int howmany,
fftw_complex *in, int istride, int idist,
fftw_complex *out, int ostride, int odist)
{
fftw_complex *work;
#ifdef FFTW_DEBUG
if (p->rank > 0 && (p->plans[0]->flags & FFTW_THREADSAFE)
&& p->nwork && p->work)
fftw_die("bug with FFTW_THREADSAFE flag\n");
#endif
if (p->nwork && !p->work)
work = (fftw_complex *) fftw_malloc(p->nwork * sizeof(fftw_complex));
else
work = p->work;
switch (p->rank) {
case 0:
break;
case 1:
if (p->is_in_place) /* fft is in-place */
fftw(p->plans[0], howmany, in, istride, idist,
work, 1, 0);
else
fftw(p->plans[0], howmany, in, istride, idist,
out, ostride, odist);
break;
default: /* rank >= 2 */
{
if (p->is_in_place) {
out = in;
ostride = istride;
odist = idist;
}
if (howmany > 1 && odist < ostride)
fftwnd_aux_howmany(p, 0, howmany,
in, istride, idist,
out, ostride, odist,
work);
else {
int i;
for (i = 0; i < howmany; ++i)
fftwnd_aux(p, 0,
in + i * idist, istride,
out + i * odist, ostride,
work);
}
}
}
if (p->nwork && !p->work)
fftw_free(work);
}
static void *fftwnd_aux_many_thread(fftw_loop_data *loop_data)
{
int min = loop_data->min, max = loop_data->max;
fftwnd_aux_many_data *d = (fftwnd_aux_many_data *) loop_data->data;
int distance = d->distance, cur_dim = d->cur_dim;
fftwnd_plan plan = d->plan;
fftw_complex *in = d->in, *out = d->out;
int istride = d->istride, ostride = d->ostride;
fftw_complex *work = d->work + loop_data->thread_num * plan->nwork;
for (; min < max; ++min)
fftwnd_aux(plan,cur_dim,
in + min*istride*distance,istride,
out + min*ostride*distance,ostride,
work);
return 0;
}
void fftwnd_aux(fftwnd_plan p, int cur_dim,
fftw_complex *in, int istride,
fftw_complex *out, int ostride,
fftw_complex *work)
{
int n_after = p->n_after[cur_dim], n = p->n[cur_dim];
if (cur_dim == p->rank - 2) {
/* just do the last dimension directly: */
if (p->is_in_place)
fftw(p->plans[p->rank - 1], n,
in, istride, n_after * istride,
work, 1, 0);
else
fftw(p->plans[p->rank - 1], n,
in, istride, n_after * istride,
out, ostride, n_after * ostride);
} else { /* we have at least two dimensions to go */
int i;
/*
* process the subsequent dimensions recursively, in hyperslabs,
* to get maximum locality:
*/
for (i = 0; i < n; ++i)
fftwnd_aux(p, cur_dim + 1,
in + i * n_after * istride, istride,
out + i * n_after * ostride, ostride, work);
}
/* do the current dimension (in-place): */
if (p->nbuffers == 0) {
fftw(p->plans[cur_dim], n_after,
out, n_after * ostride, ostride,
work, 1, 0);
} else /* using contiguous copy buffers: */
fftw_buffered(p->plans[cur_dim], n_after,
out, n_after * ostride, ostride,
work, p->nbuffers, work + n);
}
