/*
* Adjoint of finite difference operator along specified dimensions.
* Equivalent to finite difference in reverse order
*
* @param snip if false: keeps the original value for the last entry;
* if true: implements the adjoint of the difference matrix with all zero first row
*
* optr = [-diff(iptr); iptr(end)] = flip(fdiff_apply(flip(iptr)))
*/
static void fdiff_apply_adjoint(const linop_data_t* _data, complex float* optr, const complex float* iptr)
{
const auto data = CAST_DOWN(fdiff_s, _data);
md_copy2(data->D, data->dims, data->str, optr, data->str, iptr, CFL_SIZE);
for (unsigned int i=0; i < data->D; i++) {
unsigned int single_flag = data->flags & MD_BIT(i);
if (single_flag) {
complex float* tmp = md_alloc_sameplace(data->D, data->dims, CFL_SIZE, optr);
complex float* tmp2 = md_alloc_sameplace(data->D, data->dims, CFL_SIZE, optr);
md_flip2(data->D, data->dims, single_flag, data->str, tmp2, data->str, optr, CFL_SIZE);
md_zfinitediff_core2(data->D, data->dims, single_flag, false, tmp, data->str, tmp2, data->str, tmp2);
md_flip2(data->D, data->dims, single_flag, data->str, optr, data->str, tmp2, CFL_SIZE);
md_free(tmp2);
md_free(tmp);
if (data->snip) {
long zdims[data->D];
md_select_dims(data->D, ~0, zdims, data->dims);
zdims[i] = 1;
md_zsub2(data->D, zdims, data->str, optr, data->str, optr, data->str, iptr);
}
}
}
}
static void homodyne(struct wdata wdata, unsigned int flags, unsigned int N, const long dims[N],
const long strs[N], complex float* data, const complex float* idata)
{
long cdims[N];
md_copy_dims(N, cdims, dims);
// cdims[0] = cdims[1] = cdims[2] = 24;
cdims[wdata.pfdim] = (wdata.frac - 0.5) * dims[wdata.pfdim];
complex float* center = md_alloc(N, cdims, CFL_SIZE);
complex float* phase = md_alloc(N, dims, CFL_SIZE);
md_resize_center(N, cdims, center, dims, idata, CFL_SIZE);
md_resize_center(N, dims, phase, cdims, center, CFL_SIZE);
md_free(center);
ifftuc(N, dims, flags, phase, phase);
md_zphsr(N, dims, phase, phase);
md_zmul2(N, dims, strs, data, strs, idata, wdata.wstrs, wdata.weights);
ifftuc(N, dims, flags, data, data);
md_zmulc(N, dims, data, data, phase);
md_zreal(N, dims, data, data);
md_free(phase);
}
void calc_geo_phantom(const long dims[DIMS], complex float* out, bool kspace, int phtype)
{
complex float* round = md_alloc(DIMS, dims, CFL_SIZE);
complex float* angular = md_alloc(DIMS, dims, CFL_SIZE);
switch (phtype) {
case 1:
sample(DIMS, dims, round, ARRAY_SIZE(phantom_geo1), phantom_geo1, kspace, true);
sample(DIMS, dims, angular, ARRAY_SIZE(phantom_geo2), phantom_geo2, kspace, false);
md_zadd(DIMS, dims, out, round, angular);
break;
case 2:
sample(DIMS, dims, round, ARRAY_SIZE(phantom_geo4), phantom_geo4, kspace, true);
sample(DIMS, dims, angular, ARRAY_SIZE(phantom_geo3), phantom_geo3, kspace, false);
md_zadd(DIMS, dims, out, round, angular);
break;
default:
assert(0);
}
md_free(round);
md_free(angular);
}
static void nufft_free_data(const void* _data)
{
struct nufft_data* data = (struct nufft_data*)_data;
free(data->ksp_dims);
free(data->cim_dims);
free(data->cml_dims);
free(data->img_dims);
free(data->trj_dims);
free(data->lph_dims);
free(data->psf_dims);
free(data->wgh_dims);
free(data->ksp_strs);
free(data->cim_strs);
free(data->cml_strs);
free(data->img_strs);
free(data->trj_strs);
free(data->lph_strs);
free(data->psf_strs);
free(data->wgh_strs);
md_free(data->grid);
md_free((void*)data->linphase);
md_free((void*)data->psf);
md_free((void*)data->fftmod);
md_free((void*)data->weights);
linop_free(data->fft_op);
free(data);
}
static void unisoftthresh_apply(const operator_data_t* _data, float mu, complex float* dst, const complex float* src)
{
const auto data = CAST_DOWN(thresh_s, _data);
if (0. == mu) {
md_copy(data->D, data->dim, dst, src, CFL_SIZE);
} else {
const long* transform_dims = linop_codomain(data->unitary_op)->dims;
const long* transform_strs = linop_codomain(data->unitary_op)->strs;
complex float* tmp = md_alloc_sameplace(data->D, transform_dims, CFL_SIZE, dst);
linop_forward(data->unitary_op, data->D, transform_dims, tmp, data->D, data->dim, src);
complex float* tmp_norm = md_alloc_sameplace(data->D, data->norm_dim, CFL_SIZE, dst);
md_zsoftthresh_core2(data->D, transform_dims, data->lambda * mu, data->flags, tmp_norm, transform_strs, tmp, transform_strs, tmp);
md_free(tmp_norm);
linop_adjoint(data->unitary_op, data->D, data->dim, dst, data->D, transform_dims, tmp);
md_free(tmp);
}
}
float nuclearnorm(long M, long N, const complex float* d) { // FIXME: destroys input
long minMN = MIN(M,N);
long dimsU[3] = {M, minMN, 1};
long dimsVT[3] = {minMN, N, 1};
long dimsS[3] = {minMN, 1, 1};
complex float* U = md_alloc_sameplace(3, dimsU, CFL_SIZE, d);
complex float* VT = md_alloc_sameplace(3, dimsVT, CFL_SIZE, d );
float* S = md_alloc_sameplace(3, dimsS, FL_SIZE, d);
// SVD
lapack_svd_econ(M, N, (complex float (*) []) U, (complex float (*) []) VT, S, (complex float (*) [N])d);
float nnorm = 0.;
for (int i = 0; i < minMN; i++)
nnorm += S[i];
md_free(U);
md_free(VT);
md_free(S);
return nnorm;
}
static bool test_md_zmatmul(void)
{
int A = 10;
int B = 20;
int C = 30;
long odims[3] = { C, 1, A };
long idims1[3] = { 1, B, A };
long idims2[3] = { C, B, 1 };
complex float* dst1 = md_alloc(3, odims, CFL_SIZE);
complex float* dst2 = md_alloc(3, odims, CFL_SIZE);
complex float* src1 = md_alloc(3, idims1, CFL_SIZE);
complex float* src2 = md_alloc(3, idims2, CFL_SIZE);
md_gaussian_rand(3, odims, dst1);
md_gaussian_rand(3, odims, dst2);
md_gaussian_rand(3, idims1, src1);
md_gaussian_rand(3, idims2, src2);
md_zmatmul(3, odims, dst1, idims1, src1, idims2, src2);
matrix_mult(A, B, C, &MD_CAST_ARRAY2(complex float, 3, odims, dst2, 0, 2),
&MD_CAST_ARRAY2(const complex float, 3, idims1, src1, 1, 2),
&MD_CAST_ARRAY2(const complex float, 3, idims2, src2, 0, 1));
double err = md_znrmse(3, odims, dst2, dst1);
md_free(src1);
md_free(src2);
md_free(dst1);
md_free(dst2);
return (err < UT_TOL);
}
static void prox_lineq_del(const operator_data_t* _data)
{
struct prox_lineq_data* pdata = CAST_DOWN(prox_lineq_data, _data);
md_free(pdata->adj);
md_free(pdata->tmp);
free(pdata);
}
int
g_parse_opts(char *input, _p_opt_cb _proc, void *arg, int dl_o, int dl_v)
{
mda m_dl_o =
{ 0 };
md_init(&m_dl_o, 32);
int ret = 0;
int c = split_string(input, dl_o, &m_dl_o);
if (c == 0)
{
ret = 1;
goto end_1;
}
p_md_obj ptr_o = m_dl_o.objects;
while (ptr_o)
{
char *opt = (char*) ptr_o->ptr;
mda m_dl_v =
{ 0 };
md_init(&m_dl_v, 8);
if (split_string(opt, dl_v, &m_dl_v) == 0)
{
md_free(&m_dl_v);
ret = 2;
goto end_1;
}
if (m_dl_v.offset < 1)
{
ret = 3;
goto end_1;
}
if (0 != _proc(&m_dl_v, arg))
{
ret = 4;
goto end_1;
}
md_free(&m_dl_v);
ptr_o = ptr_o->next;
}
end_1: ;
md_free(&m_dl_o);
return ret;
}
/*
* Cumulative sum along dimensions specified by flags (with strides)
*
* optr = cumsum(iptr)
*/
void md_zcumsum2(unsigned int D, const long dims[D], unsigned int flags, const long ostrs[D], complex float* optr, const long istrs[D], const complex float* iptr)
{
complex float* tmp = md_alloc_sameplace(D, dims, CFL_SIZE, optr);
complex float* tmp2 = md_alloc_sameplace(D, dims, CFL_SIZE, optr);
md_zcumsum_core2(D, dims, flags, tmp, tmp2, ostrs, optr, istrs, iptr);
md_free(tmp);
md_free(tmp2);
}
static void prox_dfwavelet_del(const operator_data_t* _data)
{
struct prox_dfwavelet_data* data = CONTAINER_OF(_data, struct prox_dfwavelet_data, base);
md_free(data->vx);
md_free(data->vy);
md_free(data->vz);
dfwavelet_free(data->plan);
free(data);
}
void fwtN(unsigned int N, unsigned int flags, const long shifts[N], const long dims[N], const long ostr[2 * N], complex float* out, const long istr[N], const complex float* in, const long flen, const float filter[2][2][flen])
{
long odims[2 * N];
wavelet_dims(N, flags, odims, dims, flen);
assert(md_calc_size(2 * N, odims) >= md_calc_size(N, dims));
// FIXME one of these is unnecessary if we use the output
complex float* tmpA = md_alloc_sameplace(2 * N, odims, CFL_SIZE, out);
complex float* tmpB = md_alloc_sameplace(2 * N, odims, CFL_SIZE, out);
long tidims[2 * N];
md_copy_dims(N, tidims, dims);
md_singleton_dims(N, tidims + N);
long tistrs[2 * N];
md_calc_strides(2 * N, tistrs, tidims, CFL_SIZE);
long todims[2 * N];
md_copy_dims(2 * N, todims, tidims);
long tostrs[2 * N];
// maybe we should push the randshift into lower levels
//md_copy2(N, dims, tistrs, tmpA, istr, in, CFL_SIZE);
md_circ_shift2(N, dims, shifts, tistrs, tmpA, istr, in, CFL_SIZE);
for (unsigned int i = 0; i < N; i++) {
if (MD_IS_SET(flags, i)) {
todims[0 + i] = odims[0 + i];
todims[N + i] = odims[N + i];
md_calc_strides(2 * N, tostrs, todims, CFL_SIZE);
fwt1(2 * N, i, tidims, tostrs, tmpB, (void*)tmpB + tostrs[N + i], tistrs, tmpA, flen, filter);
md_copy_dims(2 * N, tidims, todims);
md_copy_dims(2 * N, tistrs, tostrs);
complex float* swap = tmpA;
tmpA = tmpB;
tmpB = swap;
}
}
md_copy2(2 * N, todims, ostr, out, tostrs, tmpA, CFL_SIZE);
md_free(tmpA);
md_free(tmpB);
}
void fwt1(unsigned int N, unsigned int d, const long dims[N], const long ostr[N], complex float* low, complex float* hgh, const long istr[N], const complex float* in, const long flen, const float filter[2][2][flen])
{
debug_printf(DP_DEBUG4, "fwt1: %d/%d\n", d, N);
debug_print_dims(DP_DEBUG4, N, dims);
assert(dims[d] >= 2);
long odims[N];
md_copy_dims(N, odims, dims);
odims[d] = bandsize(dims[d], flen);
debug_print_dims(DP_DEBUG4, N, odims);
long o = d + 1;
long u = N - o;
// 0 1 2 3 4 5 6|7
// --d-- * --u--|N
// ---o---
assert(d == md_calc_blockdim(d, dims + 0, istr + 0, CFL_SIZE));
assert(u == md_calc_blockdim(u, dims + o, istr + o, CFL_SIZE * md_calc_size(o, dims)));
assert(d == md_calc_blockdim(d, odims + 0, ostr + 0, CFL_SIZE));
assert(u == md_calc_blockdim(u, odims + o, ostr + o, CFL_SIZE * md_calc_size(o, odims)));
// merge dims
long wdims[3] = { md_calc_size(d, dims), dims[d], md_calc_size(u, dims + o) };
long wistr[3] = { CFL_SIZE, istr[d], CFL_SIZE * md_calc_size(o, dims) };
long wostr[3] = { CFL_SIZE, ostr[d], CFL_SIZE * md_calc_size(o, odims) };
#ifdef USE_CUDA
if (cuda_ondevice(in)) {
assert(cuda_ondevice(low));
assert(cuda_ondevice(hgh));
float* flow = md_gpu_move(1, MD_DIMS(flen), filter[0][0], FL_SIZE);
float* fhgh = md_gpu_move(1, MD_DIMS(flen), filter[0][1], FL_SIZE);
wl3_cuda_down3(wdims, wostr, low, wistr, in, flen, flow);
wl3_cuda_down3(wdims, wostr, hgh, wistr, in, flen, fhgh);
md_free(flow);
md_free(fhgh);
return;
}
#endif
// no clear needed
wavelet_down3(wdims, wostr, low, wistr, in, flen, filter[0][0]);
wavelet_down3(wdims, wostr, hgh, wistr, in, flen, filter[0][1]);
}
void iwtN(unsigned int N, unsigned int flags, const long shifts[N], const long dims[N], const long ostr[N], complex float* out, const long istr[2 * N], const complex float* in, const long flen, const float filter[2][2][flen])
{
long idims[2 * N];
wavelet_dims(N, flags, idims, dims, flen);
assert(md_calc_size(2 * N, idims) >= md_calc_size(N, dims));
complex float* tmpA = md_alloc_sameplace(2 * N, idims, CFL_SIZE, out);
complex float* tmpB = md_alloc_sameplace(2 * N, idims, CFL_SIZE, out);
long tidims[2 * N];
md_copy_dims(2 * N, tidims, idims);
long tistrs[2 * N];
md_calc_strides(2 * N, tistrs, tidims, CFL_SIZE);
long todims[2 * N];
md_copy_dims(2 * N, todims, tidims);
long tostrs[2 * N];
long ishifts[N];
for (unsigned int i = 0; i < N; i++)
ishifts[i] = -shifts[i];
md_copy2(2 * N, tidims, tistrs, tmpA, istr, in, CFL_SIZE);
for (int i = N - 1; i >= 0; i--) { // run backwards to maintain contigous blocks
if (MD_IS_SET(flags, i)) {
todims[0 + i] = dims[0 + i];
todims[N + i] = 1;
md_calc_strides(2 * N, tostrs, todims, CFL_SIZE);
iwt1(2 * N, i, todims, tostrs, tmpB, tistrs, tmpA, (void*)tmpA + tistrs[N + i], flen, filter);
md_copy_dims(2 * N, tidims, todims);
md_copy_dims(2 * N, tistrs, tostrs);
complex float* swap = tmpA;
tmpA = tmpB;
tmpB = swap;
}
}
//md_copy2(N, dims, ostr, out, tostrs, tmpA, CFL_SIZE);
md_circ_shift2(N, dims, ishifts, ostr, out, tostrs, tmpA, CFL_SIZE);
md_free(tmpA);
md_free(tmpB);
}
// [AC-Adaptive]
static void radial_self_delays(unsigned int N, float shifts[N], const float phi[N], const long dims[DIMS], const complex float* in)
{
unsigned int d = 2;
unsigned int flags = (1 << d);
assert(N == dims[d]);
long dims1[DIMS];
md_select_dims(DIMS, ~flags, dims1, dims);
complex float* tmp1 = md_alloc(DIMS, dims1, CFL_SIZE);
complex float* tmp2 = md_alloc(DIMS, dims1, CFL_SIZE);
long pos[DIMS] = { 0 };
for (unsigned int i = 0; i < dims[d]; i++) {
pos[d] = i;
md_copy_block(DIMS, pos, dims1, tmp1, dims, in, CFL_SIZE);
// find opposing spoke
float mdelta = 0.;
int mindex = 0;
for (unsigned int j = 0; j < dims[d]; j++) {
float delta = cabsf(cexpf(1.i * phi[j]) - cexpf(1.i * phi[i]));
if (mdelta <= delta) {
mdelta = delta;
mindex = j;
}
}
pos[d] = mindex;
md_copy_block(DIMS, pos, dims1, tmp2, dims, in, CFL_SIZE);
unsigned int d2 = 1;
float rshifts[DIMS];
md_flip(DIMS, dims1, MD_BIT(d2), tmp2, tmp2, CFL_SIZE); // could be done by iFFT in est_subpixel_shift
est_subpixel_shift(DIMS, rshifts, dims1, MD_BIT(d2), tmp2, tmp1);
float mshift = rshifts[d2] / 2.; // mdelta
shifts[i] = mshift;
}
md_free(tmp1);
md_free(tmp2);
}
void iwt1(unsigned int N, unsigned int d, const long dims[N], const long ostr[N], complex float* out, const long istr[N], const complex float* low, const complex float* hgh, const long flen, const float filter[2][2][flen])
{
debug_printf(DP_DEBUG4, "ifwt1: %d/%d\n", d, N);
debug_print_dims(DP_DEBUG4, N, dims);
assert(dims[d] >= 2);
long idims[N];
md_copy_dims(N, idims, dims);
idims[d] = bandsize(dims[d], flen);
debug_print_dims(DP_DEBUG4, N, idims);
long o = d + 1;
long u = N - o;
// 0 1 2 3 4 5 6|7
// --d-- * --u--|N
// ---o---
assert(d == md_calc_blockdim(d, dims + 0, ostr + 0, CFL_SIZE));
assert(u == md_calc_blockdim(u, dims + o, ostr + o, CFL_SIZE * md_calc_size(o, dims)));
assert(d == md_calc_blockdim(d, idims + 0, istr + 0, CFL_SIZE));
assert(u == md_calc_blockdim(u, idims + o, istr + o, CFL_SIZE * md_calc_size(o, idims)));
long wdims[3] = { md_calc_size(d, dims), dims[d], md_calc_size(u, dims + o) };
long wistr[3] = { CFL_SIZE, istr[d], CFL_SIZE * md_calc_size(o, idims) };
long wostr[3] = { CFL_SIZE, ostr[d], CFL_SIZE * md_calc_size(o, dims) };
md_clear(3, wdims, out, CFL_SIZE); // we cannot clear because we merge outputs
#ifdef USE_CUDA
if (cuda_ondevice(out)) {
assert(cuda_ondevice(low));
assert(cuda_ondevice(hgh));
float* flow = md_gpu_move(1, MD_DIMS(flen), filter[1][0], FL_SIZE);
float* fhgh = md_gpu_move(1, MD_DIMS(flen), filter[1][1], FL_SIZE);
wl3_cuda_up3(wdims, wostr, out, wistr, low, flen, flow);
wl3_cuda_up3(wdims, wostr, out, wistr, hgh, flen, fhgh);
md_free(flow);
md_free(fhgh);
return;
}
#endif
wavelet_up3(wdims, wostr, out, wistr, low, flen, filter[1][0]);
wavelet_up3(wdims, wostr, out, wistr, hgh, flen, filter[1][1]);
}
void noir_recon(const struct noir_conf_s* conf, const long dims[DIMS], complex float* outbuf, complex float* sensout, const complex float* psf, const complex float* mask, const complex float* kspace)
{
long imgs_dims[DIMS];
long coil_dims[DIMS];
long data_dims[DIMS];
long img1_dims[DIMS];
md_select_dims(DIMS, FFT_FLAGS|MAPS_FLAG|CSHIFT_FLAG, imgs_dims, dims);
md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG|MAPS_FLAG, coil_dims, dims);
md_select_dims(DIMS, FFT_FLAGS|COIL_FLAG, data_dims, dims);
md_select_dims(DIMS, FFT_FLAGS, img1_dims, dims);
long skip = md_calc_size(DIMS, imgs_dims);
long size = skip + md_calc_size(DIMS, coil_dims);
long data_size = md_calc_size(DIMS, data_dims);
long d1[1] = { size };
complex float* img = md_alloc_sameplace(1, d1, CFL_SIZE, kspace);
complex float* imgH = md_alloc_sameplace(1, d1, CFL_SIZE, kspace);
md_clear(DIMS, imgs_dims, img, CFL_SIZE);
md_zfill(DIMS, img1_dims, outbuf, 1.); // initial only first image
md_copy(DIMS, img1_dims, img, outbuf, CFL_SIZE);
md_clear(DIMS, coil_dims, img + skip, CFL_SIZE);
md_clear(DIMS, imgs_dims, imgH, CFL_SIZE);
md_clear(DIMS, coil_dims, imgH + skip, CFL_SIZE);
struct noir_data* ndata = noir_init(dims, mask, psf, conf->rvc, conf->usegpu);
struct data data = { ndata };
struct iter3_irgnm_conf irgnm_conf = { .iter = conf->iter, .alpha = conf->alpha, .redu = conf->redu };
iter3_irgnm(&irgnm_conf.base, frw, der, adj, &data, size * 2, (float*)img, data_size * 2, (const float*)kspace);
md_copy(DIMS, imgs_dims, outbuf, img, CFL_SIZE);
if (NULL != sensout) {
assert(!conf->usegpu);
noir_forw_coils(ndata, sensout, img + skip);
}
noir_free(ndata);
md_free(img);
md_free(imgH);
}
/*
* Cumulative sum - inverse of finite difference operator
*
* optr = cumsum(iptr);
*/
static void cumsum_apply(const linop_data_t* _data, float lambda, complex float* optr, const complex float* iptr)
{
const auto data = CAST_DOWN(fdiff_s, _data);
assert(0. == lambda);
complex float* tmp = md_alloc_sameplace(data->D, data->dims, CFL_SIZE, optr);
complex float* tmp2 = md_alloc_sameplace(data->D, data->dims, CFL_SIZE, optr);
md_zcumsum_core2(data->D, data->dims, data->flags, tmp, tmp2, data->str, optr, data->str, iptr);
md_free(tmp2);
md_free(tmp);
}
static complex float* compute_psf2(unsigned int N, const long psf_dims[N + 3], const long trj_dims[N + 3], const complex float* traj, const complex float* weights)
{
unsigned int ND = N + 3;
long img_dims[ND];
long img_strs[ND];
md_select_dims(ND, ~MD_BIT(N + 0), img_dims, psf_dims);
md_calc_strides(ND, img_strs, img_dims, CFL_SIZE);
// PSF 2x size
long img2_dims[ND];
long img2_strs[ND];
md_copy_dims(ND, img2_dims, img_dims);
for (int i = 0; i < 3; i++)
img2_dims[i] = (1 == img_dims[i]) ? 1 : (2 * img_dims[i]);
md_calc_strides(ND, img2_strs, img2_dims, CFL_SIZE);
complex float* traj2 = md_alloc(ND, trj_dims, CFL_SIZE);
md_zsmul(ND, trj_dims, traj2, traj, 2.);
complex float* psft = compute_psf(ND, img2_dims, trj_dims, traj2, weights);
md_free(traj2);
fftuc(ND, img2_dims, FFT_FLAGS, psft, psft);
// reformat
long sub2_strs[ND];
md_copy_strides(ND, sub2_strs, img2_strs);
for(int i = 0; i < 3; i++)
sub2_strs[i] *= 2;;
complex float* psf = md_alloc(ND, psf_dims, CFL_SIZE);
long factors[N];
for (unsigned int i = 0; i < N; i++)
factors[i] = ((img_dims[i] > 1) && (i < 3)) ? 2 : 1;
md_decompose(N + 0, factors, psf_dims, psf, img2_dims, psft, CFL_SIZE);
md_free(psft);
return psf;
}
static double bench_generic_add(long dims[DIMS], unsigned int flags, bool forloop)
{
long dimsX[DIMS];
long dimsY[DIMS];
long dimsC[DIMS];
md_select_dims(DIMS, flags, dimsX, dims);
md_select_dims(DIMS, ~flags, dimsC, dims);
md_select_dims(DIMS, ~0u, dimsY, dims);
long strsX[DIMS];
long strsY[DIMS];
md_calc_strides(DIMS, strsX, dimsX, CFL_SIZE);
md_calc_strides(DIMS, strsY, dimsY, CFL_SIZE);
complex float* x = md_alloc(DIMS, dimsX, CFL_SIZE);
complex float* y = md_alloc(DIMS, dimsY, CFL_SIZE);
md_gaussian_rand(DIMS, dimsX, x);
md_gaussian_rand(DIMS, dimsY, y);
long L = md_calc_size(DIMS, dimsC);
long T = md_calc_size(DIMS, dimsX);
double tic = timestamp();
if (forloop) {
for (long i = 0; i < L; i++) {
for (long j = 0; j < T; j++)
y[i + j * L] += x[j];
}
} else {
md_zaxpy2(DIMS, dims, strsY, y, 1., strsX, x);
}
double toc = timestamp();
md_free(x);
md_free(y);
return toc - tic;
}
void tv_adjoint(unsigned int D, const long dims[D], unsigned int flags, complex float* out, const complex float* in)
{
unsigned int N = bitcount(flags);
assert(N == dims[D - 1]); // we use the highest dim to store our different partial derivatives
unsigned int flags2 = flags;
complex float* tmp = md_alloc_sameplace(D - 1, dims, CFL_SIZE, out);
md_clear(D - 1, dims, out, CFL_SIZE);
md_clear(D - 1, dims, tmp, CFL_SIZE);
for (unsigned int i = 0; i < N; i++) {
unsigned int lsb = ffs(flags2) - 1;
flags2 = MD_CLEAR(flags2, lsb);
md_zfdiff_backwards(D - 1, dims, lsb, tmp, in + i * md_calc_size(D - 1, dims));
md_zadd(D - 1, dims, out, out, tmp);
}
md_free(tmp);
assert(0 == flags2);
}
/**
* Compute Strang's circulant preconditioner
*
* Strang's reconditioner is simply the cropped psf in the image domain
*
*/
static complex float* compute_precond(unsigned int N, const long* pre_dims, const long* pre_strs, const long* psf_dims, const long* psf_strs, const complex float* psf, const complex float* linphase)
{
int ND = N + 1;
unsigned long flags = FFT_FLAGS;
complex float* pre = md_alloc(ND, pre_dims, CFL_SIZE);
complex float* psft = md_alloc(ND, psf_dims, CFL_SIZE);
// Transform psf to image domain
ifftuc(ND, psf_dims, flags, psft, psf);
// Compensate for linear phase to get cropped psf
md_clear(ND, pre_dims, pre, CFL_SIZE);
md_zfmacc2(ND, psf_dims, pre_strs, pre, psf_strs, psft, psf_strs, linphase);
md_free(psft);
// Transform to Fourier domain
fftuc(N, pre_dims, flags, pre, pre);
md_zabs(N, pre_dims, pre, pre);
md_zsadd(N, pre_dims, pre, pre, 1e-3);
return pre;
}
void calone(const struct ecalib_conf* conf, const long cov_dims[4], complex float* imgcov, unsigned int SN, float svals[SN], const long calreg_dims[DIMS], const complex float* data)
{
assert(1 == md_calc_size(DIMS - 5, calreg_dims + 5));
#if 1
long nskerns_dims[5];
complex float* nskerns;
compute_kernels(conf, nskerns_dims, &nskerns, SN, svals, calreg_dims, data);
#else
long channels = calreg_dims[3];
long kx = conf->kdims[0];
long ky = conf->kdims[1];
long kz = conf->kdims[2];
long nskerns_dims[5] = { kx, ky, kz, channels, 0 };
long N = md_calc_size(4, nskerns_dims);
assert(N > 0);
nskerns_dims[4] = N;
complex float* nskerns = md_alloc(5, nskerns_dims, CFL_SIZE);
long nr_kernels = channels;
nskerns_dims[4] = channels;
spirit_kernel(nskerns_dims, nskerns, calreg_dims, data);
#endif
compute_imgcov(cov_dims, imgcov, nskerns_dims, nskerns);
md_free(nskerns);
}
/**
* Proximal function for f(z) = 0.5 || y - A z ||_2^2.
* Solution is (A^H A + (1/mu) I)z = A^H y + (1/mu)(x_plus_u)
*
* @param prox_data should be of type prox_normaleq_data
* @param mu proximal penalty
* @param z output
* @param x_plus_u input
*/
static void prox_normaleq_fun(const operator_data_t* prox_data, float mu, float* z, const float* x_plus_u)
{
struct prox_normaleq_data* pdata = CAST_DOWN(prox_normaleq_data, prox_data);
if (0 == mu) {
md_copy(1, MD_DIMS(pdata->size), z, x_plus_u, FL_SIZE);
} else {
float rho = 1. / mu;
float* b = md_alloc_sameplace(1, MD_DIMS(pdata->size), FL_SIZE, x_plus_u);
md_copy(1, MD_DIMS(pdata->size), b, pdata->adj, FL_SIZE);
md_axpy(1, MD_DIMS(pdata->size), b, rho, x_plus_u);
if (NULL == pdata->op->norm_inv) {
struct iter_conjgrad_conf* cg_conf = pdata->cgconf;
cg_conf->l2lambda = rho;
iter_conjgrad(CAST_UP(cg_conf), pdata->op->normal, NULL, pdata->size, z, (float*)b, NULL);
} else {
linop_norm_inv_iter((struct linop_s*)pdata->op, rho, z, b);
}
md_free(b);
}
}
// Forward: from image to kspace
static void nufft_apply(const void* _data, complex float* dst, const complex float* src)
{
struct nufft_data* data = (struct nufft_data*)_data;
assert(!data->conf.toeplitz); // if toeplitz linphase has no roll, so would need to be added
unsigned int ND = data->N + 3;
md_zmul2(ND, data->cml_dims, data->cml_strs, data->grid, data->cim_strs, src, data->lph_strs, data->linphase);
linop_forward(data->fft_op, ND, data->cml_dims, data->grid, ND, data->cml_dims, data->grid);
md_zmul2(ND, data->cml_dims, data->cml_strs, data->grid, data->cml_strs, data->grid, data->img_strs, data->fftmod);
md_clear(ND, data->ksp_dims, dst, CFL_SIZE);
complex float* gridX = md_alloc(data->N, data->cm2_dims, CFL_SIZE);
long factors[data->N];
for (unsigned int i = 0; i < data->N; i++)
factors[i] = ((data->img_dims[i] > 1) && (i < 3)) ? 2 : 1;
md_recompose(data->N, factors, data->cm2_dims, gridX, data->cml_dims, data->grid, CFL_SIZE);
grid2H(2., data->width, data->beta, ND, data->trj_dims, data->traj, data->ksp_dims, dst, data->cm2_dims, gridX);
md_free(gridX);
if (NULL != data->weights)
md_zmul2(data->N, data->ksp_dims, data->ksp_strs, dst, data->ksp_strs, dst, data->wgh_strs, data->weights);
}
complex float* compute_psf(unsigned int N, const long img2_dims[N], const long trj_dims[N], const complex float* traj, const complex float* weights)
{
long ksp_dims1[N];
md_select_dims(N, ~MD_BIT(0), ksp_dims1, trj_dims);
struct linop_s* op2 = nufft_create(N, ksp_dims1, img2_dims, trj_dims, traj, NULL,
nufft_conf_defaults, false);
complex float* ones = md_alloc(N, ksp_dims1, CFL_SIZE);
md_zfill(N, ksp_dims1, ones, 1.);
if (NULL != weights) {
md_zmul(N, ksp_dims1, ones, ones, weights);
md_zmulc(N, ksp_dims1, ones, ones, weights);
}
complex float* psft = md_alloc(N, img2_dims, CFL_SIZE);
linop_adjoint_unchecked(op2, psft, ones);
md_free(ones);
linop_free(op2);
return psft;
}
SSL_ROM_TEXT_SECTION
int md_free_ctx( md_context_t *ctx )
{
md_free( ctx );
return( 0 );
}
static void mmap_free(mmap_t* m)
{
if(!m) return;
munmap(m->data,m->sz);
fclose(m->f);
md_free(m);
}
static double bench_wavelet_thresh(int version, long scale)
{
long dims[DIMS] = { 1, 256 * scale, 256 * scale, 1, 16, 1, 1, 1 };
long minsize[DIMS] = { [0 ... DIMS - 1] = 1 };
minsize[0] = MIN(dims[0], 16);
minsize[1] = MIN(dims[1], 16);
minsize[2] = MIN(dims[2], 16);
const struct operator_p_s* p;
switch (version) {
case 2:
p = prox_wavethresh_create(DIMS, dims, 7, minsize, 1.1, true, false);
break;
case 3:
p = prox_wavelet3_thresh_create(DIMS, dims, 6, minsize, 1.1, true);
break;
default:
assert(0);
}
complex float* x = md_alloc(DIMS, dims, CFL_SIZE);
md_gaussian_rand(DIMS, dims, x);
double tic = timestamp();
operator_p_apply(p, 0.98, DIMS, dims, x, DIMS, dims, x);
double toc = timestamp();
md_free(x);
operator_p_free(p);
return toc - tic;
}
static void linop_matrix_apply_normal(const linop_data_t* _data, complex float* dst, const complex float* src)
{
struct operator_matrix_s* data = CAST_DOWN(operator_matrix_s, _data);
if (NULL == data->mat_gram) {
complex float* tmp = md_alloc_sameplace(data->N, data->out_dims, CFL_SIZE, src);
linop_matrix_apply(_data, tmp, src);
linop_matrix_apply_adjoint(_data, dst, tmp);
md_free(tmp);
} else {
const complex float* mat_gram = data->mat_gram;
#ifdef USE_CUDA
if (cuda_ondevice(src)) {
if (NULL == data->mat_gram_gpu)
data->mat_gram_gpu = md_gpu_move(2 * data->N, data->grm_dims, data->mat_gram, CFL_SIZE);
mat_gram = data->mat_gram_gpu;
}
#endif
md_ztenmul(2 * data->N, data->gout_dims, dst, data->gin_dims, src, data->grm_dims, mat_gram);
}
}
