int main_fft(int argc, char* argv[]) { bool unitary = false; bool inv = false; const struct opt_s opts[] = { OPT_SET('u', &unitary, "unitary"), OPT_SET('i', &inv, "inverse"), }; cmdline(&argc, argv, 3, 3, usage_str, help_str, ARRAY_SIZE(opts), opts); long dims[DIMS]; complex float* idata = load_cfl(argv[2], DIMS, dims); complex float* data = create_cfl(argv[3], DIMS, dims); unsigned long flags = labs(atol(argv[1])); md_copy(DIMS, dims, data, idata, sizeof(complex float)); unmap_cfl(DIMS, dims, idata); if (unitary) fftscale(DIMS, dims, flags, data, data); (inv ? ifftc : fftc)(DIMS, dims, flags, data, data); unmap_cfl(DIMS, dims, data); exit(0); }
static struct linop_s* linop_fft_create_priv(int N, const long dims[N], unsigned int flags, bool forward, bool center) { const struct operator_s* plan = fft_measure_create(N, dims, flags, true, false); const struct operator_s* iplan = fft_measure_create(N, dims, flags, true, true); PTR_ALLOC(struct fft_linop_s, data); SET_TYPEID(fft_linop_s, data); data->frw = plan; data->adj = iplan; data->N = N; data->center = center; data->dims = *TYPE_ALLOC(long[N]); md_copy_dims(N, data->dims, dims); data->strs = *TYPE_ALLOC(long[N]); md_calc_strides(N, data->strs, data->dims, CFL_SIZE); long fft_dims[N]; md_select_dims(N, flags, fft_dims, dims); data->nscale = (float)md_calc_size(N, fft_dims); lop_fun_t apply = forward ? fft_linop_apply : fft_linop_adjoint; lop_fun_t adjoint = forward ? fft_linop_adjoint : fft_linop_apply; struct linop_s* lop = linop_create(N, dims, N, dims, CAST_UP(PTR_PASS(data)), apply, adjoint, fft_linop_normal, NULL, fft_linop_free); if (center) { // FIXME: should only allocate flagged dims complex float* fftmod_mat = md_alloc(N, dims, CFL_SIZE); complex float* fftmodk_mat = md_alloc(N, dims, CFL_SIZE); // we need fftmodk only because we want to apply scaling only once complex float one[1] = { 1. }; md_fill(N, dims, fftmod_mat, one, CFL_SIZE); fftmod(N, dims, flags, fftmodk_mat, fftmod_mat); fftscale(N, dims, flags, fftmod_mat, fftmodk_mat); struct linop_s* mod = linop_cdiag_create(N, dims, ~0u, fftmod_mat); struct linop_s* modk = linop_cdiag_create(N, dims, ~0u, fftmodk_mat); struct linop_s* tmp = linop_chain(mod, lop); tmp = linop_chain(tmp, modk); linop_free(lop); linop_free(mod); linop_free(modk); lop = tmp; } return lop; }
/** * Create maps operator, m = S x * * @param max_dims maximal dimensions across all data structures * @param sens_flags active map dimensions * @param sens sensitivities * @param gpu TRUE if using gpu */ struct linop_s* maps_create(const long max_dims[DIMS], unsigned int sens_flags, const complex float* sens, bool gpu) { struct maps_data* data = maps_create_data(max_dims, sens_flags, sens, gpu); // scale the sensitivity maps by the FFT scale factor fftscale(DIMS, data->mps_dims, FFT_FLAGS, data->sens, data->sens); return linop_create(DIMS, data->ksp_dims, DIMS, data->img_dims, data, maps_apply, maps_apply_adjoint, maps_apply_normal, maps_apply_pinverse, maps_free_data); }
static void sense_adjoint(const void* _data, complex float* imgs, const complex float* out) { const struct sense_data* data = _data; md_zmulc2(DIMS, data->data_dims, data->data_strs, data->tmp, data->data_strs, out, data->mask_strs, data->pattern); ifftc(DIMS, data->data_dims, FFT_FLAGS, data->tmp, data->tmp); fftscale(DIMS, data->data_dims, FFT_FLAGS, data->tmp, data->tmp); md_clear(DIMS, data->imgs_dims, imgs, CFL_SIZE); md_zfmacc2(DIMS, data->sens_dims, data->imgs_strs, imgs, data->data_strs, data->tmp, data->sens_strs, data->sens); }
static void sense_forward(const void* _data, complex float* out, const complex float* imgs) { const struct sense_data* data = _data; md_clear(DIMS, data->data_dims, out, CFL_SIZE); md_zfmac2(DIMS, data->sens_dims, data->data_strs, out, data->sens_strs, data->sens, data->imgs_strs, imgs); fftc(DIMS, data->data_dims, FFT_FLAGS, out, out); fftscale(DIMS, data->data_dims, FFT_FLAGS, out, out); md_zmul2(DIMS, data->data_dims, data->data_strs, out, data->data_strs, out, data->mask_strs, data->pattern); }
/** * * NUFFT operator initialization * * @param N - number of dimensions * @param ksp_dims - kspace dimension * @param cim_dims - coil images dimension * @param traj - trajectory * @param conf - configuration options * @param use_gpu - use gpu boolean * */ struct linop_s* nufft_create(unsigned int N, const long ksp_dims[N], const long cim_dims[N], const long traj_dims[N], const complex float* traj, const complex float* weights, struct nufft_conf_s conf, bool use_gpu) { struct nufft_data* data = (struct nufft_data*)xmalloc(sizeof(struct nufft_data)); data->N = N; data->use_gpu = use_gpu; data->traj = traj; data->conf = conf; data->width = 3.; data->beta = calc_beta(2., data->width); // get dims assert(md_check_compat(N - 3, 0, ksp_dims + 3, cim_dims + 3)); unsigned int ND = N + 3; data->ksp_dims = xmalloc(ND * sizeof(long)); data->cim_dims = xmalloc(ND * sizeof(long)); data->cml_dims = xmalloc(ND * sizeof(long)); data->img_dims = xmalloc(ND * sizeof(long)); data->trj_dims = xmalloc(ND * sizeof(long)); data->lph_dims = xmalloc(ND * sizeof(long)); data->psf_dims = xmalloc(ND * sizeof(long)); data->wgh_dims = xmalloc(ND * sizeof(long)); data->ksp_strs = xmalloc(ND * sizeof(long)); data->cim_strs = xmalloc(ND * sizeof(long)); data->cml_strs = xmalloc(ND * sizeof(long)); data->img_strs = xmalloc(ND * sizeof(long)); data->trj_strs = xmalloc(ND * sizeof(long)); data->lph_strs = xmalloc(ND * sizeof(long)); data->psf_strs = xmalloc(ND * sizeof(long)); data->wgh_strs = xmalloc(ND * sizeof(long)); md_singleton_dims(ND, data->cim_dims); md_singleton_dims(ND, data->ksp_dims); md_copy_dims(N, data->cim_dims, cim_dims); md_copy_dims(N, data->ksp_dims, ksp_dims); md_select_dims(ND, FFT_FLAGS, data->img_dims, data->cim_dims); assert(3 == traj_dims[0]); assert(traj_dims[1] == ksp_dims[1]); assert(traj_dims[2] == ksp_dims[2]); assert(md_check_compat(N - 3, ~0, traj_dims + 3, ksp_dims + 3)); assert(md_check_bounds(N - 3, ~0, traj_dims + 3, ksp_dims + 3)); md_singleton_dims(ND, data->trj_dims); md_copy_dims(N, data->trj_dims, traj_dims); // get strides md_calc_strides(ND, data->cim_strs, data->cim_dims, CFL_SIZE); md_calc_strides(ND, data->img_strs, data->img_dims, CFL_SIZE); md_calc_strides(ND, data->trj_strs, data->trj_dims, CFL_SIZE); md_calc_strides(ND, data->ksp_strs, data->ksp_dims, CFL_SIZE); data->weights = NULL; if (NULL != weights) { md_singleton_dims(ND, data->wgh_dims); md_select_dims(N, ~MD_BIT(0), data->wgh_dims, data->trj_dims); md_calc_strides(ND, data->wgh_strs, data->wgh_dims, CFL_SIZE); complex float* tmp = md_alloc(ND, data->wgh_dims, CFL_SIZE); md_copy(ND, data->wgh_dims, tmp, weights, CFL_SIZE); data->weights = tmp; } complex float* roll = md_alloc(ND, data->img_dims, CFL_SIZE); rolloff_correction(2., data->width, data->beta, data->img_dims, roll); data->roll = roll; complex float* linphase = compute_linphases(N, data->lph_dims, data->img_dims); md_calc_strides(ND, data->lph_strs, data->lph_dims, CFL_SIZE); if (!conf.toeplitz) md_zmul2(ND, data->lph_dims, data->lph_strs, linphase, data->lph_strs, linphase, data->img_strs, data->roll); fftmod(ND, data->lph_dims, FFT_FLAGS, linphase, linphase); fftscale(ND, data->lph_dims, FFT_FLAGS, linphase, linphase); // md_zsmul(ND, data->lph_dims, linphase, linphase, 1. / (float)(data->trj_dims[1] * data->trj_dims[2])); complex float* fftm = md_alloc(ND, data->img_dims, CFL_SIZE); md_zfill(ND, data->img_dims, fftm, 1.); fftmod(ND, data->img_dims, FFT_FLAGS, fftm, fftm); data->fftmod = fftm; data->linphase = linphase; data->psf = NULL; if (conf.toeplitz) { #if 0 md_copy_dims(ND, data->psf_dims, data->lph_dims); #else md_copy_dims(3, data->psf_dims, data->lph_dims); md_copy_dims(ND - 3, data->psf_dims + 3, data->trj_dims + 3); data->psf_dims[N] = data->lph_dims[N]; #endif md_calc_strides(ND, data->psf_strs, data->psf_dims, CFL_SIZE); data->psf = compute_psf2(N, data->psf_dims, data->trj_dims, data->traj, data->weights); } md_copy_dims(ND, data->cml_dims, data->cim_dims); data->cml_dims[N + 0] = data->lph_dims[N + 0]; md_calc_strides(ND, data->cml_strs, data->cml_dims, CFL_SIZE); data->cm2_dims = xmalloc(ND * sizeof(long)); // ! md_copy_dims(ND, data->cm2_dims, data->cim_dims); for (int i = 0; i < 3; i++) data->cm2_dims[i] = (1 == cim_dims[i]) ? 1 : (2 * cim_dims[i]); data->grid = md_alloc(ND, data->cml_dims, CFL_SIZE); data->fft_op = linop_fft_create(ND, data->cml_dims, FFT_FLAGS, use_gpu); return linop_create(N, ksp_dims, N, cim_dims, data, nufft_apply, nufft_apply_adjoint, nufft_apply_normal, NULL, nufft_free_data); }
struct noir_data* noir_init(const long dims[DIMS], const complex float* mask, const complex float* psf, bool rvc, bool use_gpu) { #ifdef USE_CUDA md_alloc_fun_t my_alloc = use_gpu ? md_alloc_gpu : md_alloc; #else assert(!use_gpu); md_alloc_fun_t my_alloc = md_alloc; #endif PTR_ALLOC(struct noir_data, data); data->rvc = rvc; md_copy_dims(DIMS, data->dims, dims); md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG|CSHIFT_FLAG, data->sign_dims, dims); md_calc_strides(DIMS, data->sign_strs, data->sign_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG|MAPS_FLAG, data->coil_dims, dims); md_calc_strides(DIMS, data->coil_strs, data->coil_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS|MAPS_FLAG|CSHIFT_FLAG, data->imgs_dims, dims); md_calc_strides(DIMS, data->imgs_strs, data->imgs_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG, data->data_dims, dims); md_calc_strides(DIMS, data->data_strs, data->data_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS, data->mask_dims, dims); md_calc_strides(DIMS, data->mask_strs, data->mask_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS, data->wght_dims, dims); md_calc_strides(DIMS, data->wght_strs, data->wght_dims, CFL_SIZE); md_select_dims(DIMS, FFT_FLAGS|CSHIFT_FLAG, data->ptrn_dims, dims); md_calc_strides(DIMS, data->ptrn_strs, data->ptrn_dims, CFL_SIZE); complex float* weights = md_alloc(DIMS, data->wght_dims, CFL_SIZE); noir_calc_weights(dims, weights); fftmod(DIMS, data->wght_dims, FFT_FLAGS, weights, weights); fftscale(DIMS, data->wght_dims, FFT_FLAGS, weights, weights); data->weights = weights; #ifdef USE_CUDA if (use_gpu) { data->weights = md_gpu_move(DIMS, data->wght_dims, weights, CFL_SIZE); } #endif complex float* ptr = my_alloc(DIMS, data->ptrn_dims, CFL_SIZE); md_copy(DIMS, data->ptrn_dims, ptr, psf, CFL_SIZE); fftmod(DIMS, data->ptrn_dims, FFT_FLAGS, ptr, ptr); data->pattern = ptr; complex float* msk = my_alloc(DIMS, data->mask_dims, CFL_SIZE); if (NULL == mask) { assert(!use_gpu); md_zfill(DIMS, data->mask_dims, msk, 1.); } else { md_copy(DIMS, data->mask_dims, msk, mask, CFL_SIZE); } // fftmod(DIMS, data->mask_dims, 7, msk, msk); fftscale(DIMS, data->mask_dims, FFT_FLAGS, msk, msk); data->mask = msk; data->sens = my_alloc(DIMS, data->coil_dims, CFL_SIZE); data->xn = my_alloc(DIMS, data->imgs_dims, CFL_SIZE); data->tmp = my_alloc(DIMS, data->sign_dims, CFL_SIZE); return data; }
void ifftuc(unsigned int D, const long dimensions[__VLA(D)], unsigned long flags, complex float* dst, const complex float* src) { ifftc(D, dimensions, flags, dst, src); fftscale(D, dimensions, flags, dst, dst); }