static void fftwnd_aux_many_threads(int nthreads, int n, int n_after, fftwnd_plan plan, int cur_dim, fftw_complex *in, int istride, fftw_complex *out, int ostride) { fftw_complex *tmp; fftwnd_aux_many_data d; if (nthreads > n) nthreads = n; tmp = (fftw_complex *) fftw_malloc(nthreads * plan->nwork * sizeof(fftw_complex)); d.plan = plan; d.cur_dim = cur_dim; d.distance = n_after; d.in = in; d.out = out; d.istride = istride; d.ostride = ostride; d.work = tmp; fftw_thread_spawn_loop(n, nthreads, fftwnd_aux_many_thread, &d); fftw_free(tmp); }
void rfftw_c2real_overlap_threads_aux(fftw_plan plan, int howmany, fftw_complex *in, int istride, int idist, fftw_real *out, int ostride, int odist, fftw_real *work, int nthreads) { rexec2_thread_data d; d.plan = plan; d.in = in; d.istride = istride; d.idist = idist; d.out = out; d.ostride = ostride; d.odist = odist; d.work = work; fftw_thread_spawn_loop(howmany, nthreads, c2real_overlap_aux_thread1, &d); fftw_thread_spawn_loop(howmany, nthreads, c2real_overlap_aux_thread2, &d); }
void rfftwnd_real2c_threads_aux(fftwnd_plan p, int cur_dim, fftw_real *in, int istride, fftw_complex *out, int ostride, fftw_complex *work, int nthreads) { int n_after = p->n_after[cur_dim], n = p->n[cur_dim]; if (cur_dim == p->rank - 2) { /* just do the last dimension directly: */ if (p->is_in_place) rfftw_real2c_threads_aux(p->plans[p->rank - 1], n, in, istride, (n_after * istride) * 2, out, istride, n_after * istride, (fftw_real *) work, nthreads); else rfftw_real2c_threads_aux(p->plans[p->rank - 1], n, in, istride, p->plans[p->rank - 1]->n * istride, out, ostride, n_after * ostride, (fftw_real *) work, nthreads); } else { /* we have at least two dimensions to go */ int nr = p->plans[p->rank - 1]->n; aux_data d; d.p = p; d.cur_dim = cur_dim + 1; d.in = in; d.istride = istride; d.idist = istride * (p->is_in_place ? n_after * 2 : nr * (n_after / (nr/2 + 1))); d.out = out; d.ostride = ostride; d.odist = ostride * n_after; d.work = work; fftw_thread_spawn_loop(n, nthreads, real2c_aux_thread, &d); } /* do the current dimension (in-place): */ /* (Use internal function instead of fftw_threads so that we can pass our workspace array.) */ fftw_executor_many_inplace_threads(p->plans[cur_dim]->n, out, work, p->plans[cur_dim]->root, n_after * ostride, n_after, ostride, nthreads); }
void rfftwnd_c2real_aux_howmany_threads(fftwnd_plan p, int cur_dim, int howmany, fftw_complex *in, int istride, int idist, fftw_real *out, int ostride, int odist, fftw_complex *work, int nwork, int nthreads) { int n_after = p->n_after[cur_dim], n = p->n[cur_dim]; /* do the current dimension (in-place): */ { fftw_howmany_data d; d.p = p->plans[cur_dim]; d.howmany = howmany; d.io_data = in; d.iostride = n_after * istride; d.iodist = idist; d.iodist0 = istride; d.work = work; d.wdist = nwork; fftw_thread_spawn_loop(n_after, nthreads, fftw_howmany_thread, &d); } if (cur_dim == p->rank - 2) { howmany_aux_data d; d.p = p->plans[p->rank - 1]; d.howmany = howmany; d.in = in; d.istride = istride; d.idist = idist; d.out = out; d.ostride = ostride; d.odist = odist; d.work = (fftw_real *) work; d.wdist = nwork * 2; /* just do the last dimension directly: */ if (p->is_in_place) { d.idist0 = n_after * istride; d.odist0 = n_after * ostride * 2; fftw_thread_spawn_loop(n, nthreads, c2r_overlap_howmany_thread, &d); } else { d.odist0 = p->plans[p->rank - 1]->n * ostride; d.idist0 = n_after * istride; fftw_thread_spawn_loop(n, nthreads, c2r_howmany_thread, &d); } } else { /* we have at least two dimensions to go */ /* * process the subsequent dimensions recursively, in hyperslabs, * to get maximum locality: */ int nr = p->plans[p->rank - 1]->n; int n_after_r = p->is_in_place ? n_after * 2 : nr * (n_after / (nr/2 + 1)); howmany_hyperslab_aux_data d; d.p = p; d.cur_dim = cur_dim + 1; d.howmany = howmany; d.in = in; d.istride = istride; d.idist = idist; d.idist0 = n_after * istride; d.out = out; d.ostride = ostride; d.odist = odist; d.odist0 = n_after_r * ostride; d.work = work; d.wdist = nwork; fftw_thread_spawn_loop(n, nthreads, c2r_hyperslab_howmany_thread, &d); } }