int main_fft(int argc, char* argv[])
{
bool unitary = false;
bool inv = false;
const struct opt_s opts[] = {
OPT_SET('u', &unitary, "unitary"),
OPT_SET('i', &inv, "inverse"),
};
cmdline(&argc, argv, 3, 3, usage_str, help_str, ARRAY_SIZE(opts), opts);
long dims[DIMS];
complex float* idata = load_cfl(argv[2], DIMS, dims);
complex float* data = create_cfl(argv[3], DIMS, dims);
unsigned long flags = labs(atol(argv[1]));
md_copy(DIMS, dims, data, idata, sizeof(complex float));
unmap_cfl(DIMS, dims, idata);
if (unitary)
fftscale(DIMS, dims, flags, data, data);
(inv ? ifftc : fftc)(DIMS, dims, flags, data, data);
unmap_cfl(DIMS, dims, data);
exit(0);
}
static struct linop_s* linop_fft_create_priv(int N, const long dims[N], unsigned int flags, bool forward, bool center)
{
const struct operator_s* plan = fft_measure_create(N, dims, flags, true, false);
const struct operator_s* iplan = fft_measure_create(N, dims, flags, true, true);
PTR_ALLOC(struct fft_linop_s, data);
SET_TYPEID(fft_linop_s, data);
data->frw = plan;
data->adj = iplan;
data->N = N;
data->center = center;
data->dims = *TYPE_ALLOC(long[N]);
md_copy_dims(N, data->dims, dims);
data->strs = *TYPE_ALLOC(long[N]);
md_calc_strides(N, data->strs, data->dims, CFL_SIZE);
long fft_dims[N];
md_select_dims(N, flags, fft_dims, dims);
data->nscale = (float)md_calc_size(N, fft_dims);
lop_fun_t apply = forward ? fft_linop_apply : fft_linop_adjoint;
lop_fun_t adjoint = forward ? fft_linop_adjoint : fft_linop_apply;
struct linop_s* lop = linop_create(N, dims, N, dims, CAST_UP(PTR_PASS(data)), apply, adjoint, fft_linop_normal, NULL, fft_linop_free);
if (center) {
// FIXME: should only allocate flagged dims
complex float* fftmod_mat = md_alloc(N, dims, CFL_SIZE);
complex float* fftmodk_mat = md_alloc(N, dims, CFL_SIZE);
// we need fftmodk only because we want to apply scaling only once
complex float one[1] = { 1. };
md_fill(N, dims, fftmod_mat, one, CFL_SIZE);
fftmod(N, dims, flags, fftmodk_mat, fftmod_mat);
fftscale(N, dims, flags, fftmod_mat, fftmodk_mat);
struct linop_s* mod = linop_cdiag_create(N, dims, ~0u, fftmod_mat);
struct linop_s* modk = linop_cdiag_create(N, dims, ~0u, fftmodk_mat);
struct linop_s* tmp = linop_chain(mod, lop);
tmp = linop_chain(tmp, modk);
linop_free(lop);
linop_free(mod);
linop_free(modk);
lop = tmp;
}
return lop;
}
/**
* Create maps operator, m = S x
*
* @param max_dims maximal dimensions across all data structures
* @param sens_flags active map dimensions
* @param sens sensitivities
* @param gpu TRUE if using gpu
*/
struct linop_s* maps_create(const long max_dims[DIMS],
unsigned int sens_flags, const complex float* sens, bool gpu)
{
struct maps_data* data = maps_create_data(max_dims, sens_flags, sens, gpu);
// scale the sensitivity maps by the FFT scale factor
fftscale(DIMS, data->mps_dims, FFT_FLAGS, data->sens, data->sens);
return linop_create(DIMS, data->ksp_dims, DIMS, data->img_dims, data,
maps_apply, maps_apply_adjoint, maps_apply_normal, maps_apply_pinverse, maps_free_data);
}
static void sense_adjoint(const void* _data, complex float* imgs, const complex float* out)
{
const struct sense_data* data = _data;
md_zmulc2(DIMS, data->data_dims, data->data_strs, data->tmp, data->data_strs, out, data->mask_strs, data->pattern);
ifftc(DIMS, data->data_dims, FFT_FLAGS, data->tmp, data->tmp);
fftscale(DIMS, data->data_dims, FFT_FLAGS, data->tmp, data->tmp);
md_clear(DIMS, data->imgs_dims, imgs, CFL_SIZE);
md_zfmacc2(DIMS, data->sens_dims, data->imgs_strs, imgs, data->data_strs, data->tmp, data->sens_strs, data->sens);
}
static void sense_forward(const void* _data, complex float* out, const complex float* imgs)
{
const struct sense_data* data = _data;
md_clear(DIMS, data->data_dims, out, CFL_SIZE);
md_zfmac2(DIMS, data->sens_dims, data->data_strs, out, data->sens_strs, data->sens, data->imgs_strs, imgs);
fftc(DIMS, data->data_dims, FFT_FLAGS, out, out);
fftscale(DIMS, data->data_dims, FFT_FLAGS, out, out);
md_zmul2(DIMS, data->data_dims, data->data_strs, out, data->data_strs, out, data->mask_strs, data->pattern);
}
/**
*
* NUFFT operator initialization
*
* @param N - number of dimensions
* @param ksp_dims - kspace dimension
* @param cim_dims - coil images dimension
* @param traj - trajectory
* @param conf - configuration options
* @param use_gpu - use gpu boolean
*
*/
struct linop_s* nufft_create(unsigned int N, const long ksp_dims[N], const long cim_dims[N], const long traj_dims[N], const complex float* traj, const complex float* weights, struct nufft_conf_s conf, bool use_gpu)
{
struct nufft_data* data = (struct nufft_data*)xmalloc(sizeof(struct nufft_data));
data->N = N;
data->use_gpu = use_gpu;
data->traj = traj;
data->conf = conf;
data->width = 3.;
data->beta = calc_beta(2., data->width);
// get dims
assert(md_check_compat(N - 3, 0, ksp_dims + 3, cim_dims + 3));
unsigned int ND = N + 3;
data->ksp_dims = xmalloc(ND * sizeof(long));
data->cim_dims = xmalloc(ND * sizeof(long));
data->cml_dims = xmalloc(ND * sizeof(long));
data->img_dims = xmalloc(ND * sizeof(long));
data->trj_dims = xmalloc(ND * sizeof(long));
data->lph_dims = xmalloc(ND * sizeof(long));
data->psf_dims = xmalloc(ND * sizeof(long));
data->wgh_dims = xmalloc(ND * sizeof(long));
data->ksp_strs = xmalloc(ND * sizeof(long));
data->cim_strs = xmalloc(ND * sizeof(long));
data->cml_strs = xmalloc(ND * sizeof(long));
data->img_strs = xmalloc(ND * sizeof(long));
data->trj_strs = xmalloc(ND * sizeof(long));
data->lph_strs = xmalloc(ND * sizeof(long));
data->psf_strs = xmalloc(ND * sizeof(long));
data->wgh_strs = xmalloc(ND * sizeof(long));
md_singleton_dims(ND, data->cim_dims);
md_singleton_dims(ND, data->ksp_dims);
md_copy_dims(N, data->cim_dims, cim_dims);
md_copy_dims(N, data->ksp_dims, ksp_dims);
md_select_dims(ND, FFT_FLAGS, data->img_dims, data->cim_dims);
assert(3 == traj_dims[0]);
assert(traj_dims[1] == ksp_dims[1]);
assert(traj_dims[2] == ksp_dims[2]);
assert(md_check_compat(N - 3, ~0, traj_dims + 3, ksp_dims + 3));
assert(md_check_bounds(N - 3, ~0, traj_dims + 3, ksp_dims + 3));
md_singleton_dims(ND, data->trj_dims);
md_copy_dims(N, data->trj_dims, traj_dims);
// get strides
md_calc_strides(ND, data->cim_strs, data->cim_dims, CFL_SIZE);
md_calc_strides(ND, data->img_strs, data->img_dims, CFL_SIZE);
md_calc_strides(ND, data->trj_strs, data->trj_dims, CFL_SIZE);
md_calc_strides(ND, data->ksp_strs, data->ksp_dims, CFL_SIZE);
data->weights = NULL;
if (NULL != weights) {
md_singleton_dims(ND, data->wgh_dims);
md_select_dims(N, ~MD_BIT(0), data->wgh_dims, data->trj_dims);
md_calc_strides(ND, data->wgh_strs, data->wgh_dims, CFL_SIZE);
complex float* tmp = md_alloc(ND, data->wgh_dims, CFL_SIZE);
md_copy(ND, data->wgh_dims, tmp, weights, CFL_SIZE);
data->weights = tmp;
}
complex float* roll = md_alloc(ND, data->img_dims, CFL_SIZE);
rolloff_correction(2., data->width, data->beta, data->img_dims, roll);
data->roll = roll;
complex float* linphase = compute_linphases(N, data->lph_dims, data->img_dims);
md_calc_strides(ND, data->lph_strs, data->lph_dims, CFL_SIZE);
if (!conf.toeplitz)
md_zmul2(ND, data->lph_dims, data->lph_strs, linphase, data->lph_strs, linphase, data->img_strs, data->roll);
fftmod(ND, data->lph_dims, FFT_FLAGS, linphase, linphase);
fftscale(ND, data->lph_dims, FFT_FLAGS, linphase, linphase);
// md_zsmul(ND, data->lph_dims, linphase, linphase, 1. / (float)(data->trj_dims[1] * data->trj_dims[2]));
complex float* fftm = md_alloc(ND, data->img_dims, CFL_SIZE);
md_zfill(ND, data->img_dims, fftm, 1.);
fftmod(ND, data->img_dims, FFT_FLAGS, fftm, fftm);
data->fftmod = fftm;
data->linphase = linphase;
data->psf = NULL;
if (conf.toeplitz) {
#if 0
md_copy_dims(ND, data->psf_dims, data->lph_dims);
#else
md_copy_dims(3, data->psf_dims, data->lph_dims);
md_copy_dims(ND - 3, data->psf_dims + 3, data->trj_dims + 3);
data->psf_dims[N] = data->lph_dims[N];
#endif
md_calc_strides(ND, data->psf_strs, data->psf_dims, CFL_SIZE);
data->psf = compute_psf2(N, data->psf_dims, data->trj_dims, data->traj, data->weights);
}
md_copy_dims(ND, data->cml_dims, data->cim_dims);
data->cml_dims[N + 0] = data->lph_dims[N + 0];
md_calc_strides(ND, data->cml_strs, data->cml_dims, CFL_SIZE);
data->cm2_dims = xmalloc(ND * sizeof(long));
// !
md_copy_dims(ND, data->cm2_dims, data->cim_dims);
for (int i = 0; i < 3; i++)
data->cm2_dims[i] = (1 == cim_dims[i]) ? 1 : (2 * cim_dims[i]);
data->grid = md_alloc(ND, data->cml_dims, CFL_SIZE);
data->fft_op = linop_fft_create(ND, data->cml_dims, FFT_FLAGS, use_gpu);
return linop_create(N, ksp_dims, N, cim_dims,
data, nufft_apply, nufft_apply_adjoint, nufft_apply_normal, NULL, nufft_free_data);
}
struct noir_data* noir_init(const long dims[DIMS], const complex float* mask, const complex float* psf, bool rvc, bool use_gpu)
{
#ifdef USE_CUDA
md_alloc_fun_t my_alloc = use_gpu ? md_alloc_gpu : md_alloc;
#else
assert(!use_gpu);
md_alloc_fun_t my_alloc = md_alloc;
#endif
PTR_ALLOC(struct noir_data, data);
data->rvc = rvc;
md_copy_dims(DIMS, data->dims, dims);
md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG|CSHIFT_FLAG, data->sign_dims, dims);
md_calc_strides(DIMS, data->sign_strs, data->sign_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG|MAPS_FLAG, data->coil_dims, dims);
md_calc_strides(DIMS, data->coil_strs, data->coil_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS|MAPS_FLAG|CSHIFT_FLAG, data->imgs_dims, dims);
md_calc_strides(DIMS, data->imgs_strs, data->imgs_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG, data->data_dims, dims);
md_calc_strides(DIMS, data->data_strs, data->data_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS, data->mask_dims, dims);
md_calc_strides(DIMS, data->mask_strs, data->mask_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS, data->wght_dims, dims);
md_calc_strides(DIMS, data->wght_strs, data->wght_dims, CFL_SIZE);
md_select_dims(DIMS, FFT_FLAGS|CSHIFT_FLAG, data->ptrn_dims, dims);
md_calc_strides(DIMS, data->ptrn_strs, data->ptrn_dims, CFL_SIZE);
complex float* weights = md_alloc(DIMS, data->wght_dims, CFL_SIZE);
noir_calc_weights(dims, weights);
fftmod(DIMS, data->wght_dims, FFT_FLAGS, weights, weights);
fftscale(DIMS, data->wght_dims, FFT_FLAGS, weights, weights);
data->weights = weights;
#ifdef USE_CUDA
if (use_gpu) {
data->weights = md_gpu_move(DIMS, data->wght_dims, weights, CFL_SIZE);
}
#endif
complex float* ptr = my_alloc(DIMS, data->ptrn_dims, CFL_SIZE);
md_copy(DIMS, data->ptrn_dims, ptr, psf, CFL_SIZE);
fftmod(DIMS, data->ptrn_dims, FFT_FLAGS, ptr, ptr);
data->pattern = ptr;
complex float* msk = my_alloc(DIMS, data->mask_dims, CFL_SIZE);
if (NULL == mask) {
assert(!use_gpu);
md_zfill(DIMS, data->mask_dims, msk, 1.);
} else {
md_copy(DIMS, data->mask_dims, msk, mask, CFL_SIZE);
}
// fftmod(DIMS, data->mask_dims, 7, msk, msk);
fftscale(DIMS, data->mask_dims, FFT_FLAGS, msk, msk);
data->mask = msk;
data->sens = my_alloc(DIMS, data->coil_dims, CFL_SIZE);
data->xn = my_alloc(DIMS, data->imgs_dims, CFL_SIZE);
data->tmp = my_alloc(DIMS, data->sign_dims, CFL_SIZE);
return data;
}
void ifftuc(unsigned int D, const long dimensions[__VLA(D)], unsigned long flags, complex float* dst, const complex float* src)
{
ifftc(D, dimensions, flags, dst, src);
fftscale(D, dimensions, flags, dst, dst);
}
