/**
* Iterative Soft Thresholding
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
* @param monitor compute objective value, errors, etc.
*/
void ist(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild, long N,
const struct vec_iter_s* vops,
struct iter_op_s op,
struct iter_op_p_s thresh,
float* x, const float* b,
struct iter_monitor_s* monitor)
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
iter_monitor(monitor, vops, x);
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
iter_op_p_call(thresh, tau, x, x);
iter_op_call(op, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau * lambda_scale, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
vops->del(r);
}
/**
* Iterative Soft Thresholding/FISTA to solve min || b - Ax ||_2 + lambda || T x ||_1
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
*/
void fista(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild,
long N,
const struct vec_iter_s* vops,
struct iter_op_s op,
struct iter_op_p_s thresh,
float* x, const float* b,
struct iter_monitor_s* monitor)
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
float* o = vops->allocate(N);
float ra = 1.;
vops->copy(N, o, x);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
iter_monitor(monitor, vops, x);
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
iter_op_p_call(thresh, lambda_scale * tau, x, x);
ravine(vops, N, &ra, x, o); // FISTA
iter_op_call(op, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
debug_printf(DP_DEBUG2, "\t\tFISTA iterations: %u\n", itrdata.iter);
vops->del(o);
vops->del(r);
}
/**
* Landweber L. An iteration formula for Fredholm integral equations of the
* first kind. Amer. J. Math. 1951; 73, 615-624.
*/
void landweber(unsigned int maxiter, float epsilon, float alpha, long N, long M,
const struct vec_iter_s* vops,
struct iter_op_s op,
struct iter_op_s adj,
float* x, const float* b,
struct iter_monitor_s* monitor)
{
float* r = vops->allocate(M);
float* p = vops->allocate(N);
double rsnot = vops->norm(M, b);
for (unsigned int i = 0; i < maxiter; i++) {
iter_monitor(monitor, vops, x);
iter_op_call(op, r, x); // r = A x
vops->xpay(M, -1., r, b); // r = b - r = b - A x
double rsnew = vops->norm(M, r);
debug_printf(DP_DEBUG3, "#%d: %f\n", i, rsnew / rsnot);
if (rsnew < epsilon)
break;
iter_op_call(adj, p, r);
vops->axpy(N, x, alpha, p);
}
vops->del(r);
vops->del(p);
}
/**
* Conjugate Gradient Descent to solve Ax = b for symmetric A
*
* @param maxiter maximum number of iterations
* @param regularization parameter
* @param epsilon stop criterion
* @param N size of input, x
* @param vops vector ops definition
* @param linop linear operator, i.e. A
* @param x initial estimate
* @param b observations
*/
float conjgrad(unsigned int maxiter, float l2lambda, float epsilon,
long N,
const struct vec_iter_s* vops,
struct iter_op_s linop,
float* x, const float* b,
struct iter_monitor_s* monitor)
{
float* r = vops->allocate(N);
float* p = vops->allocate(N);
float* Ap = vops->allocate(N);
// The first calculation of the residual might not
// be necessary in some cases...
iter_op_call(linop, r, x); // r = A x
vops->axpy(N, r, l2lambda, x);
vops->xpay(N, -1., r, b); // r = b - r = b - A x
vops->copy(N, p, r); // p = r
float rsnot = (float)pow(vops->norm(N, r), 2.);
float rsold = rsnot;
float rsnew = rsnot;
float eps_squared = pow(epsilon, 2.);
unsigned int i = 0;
if (0. == rsold) {
debug_printf(DP_DEBUG3, "CG: early out\n");
goto cleanup;
}
for (i = 0; i < maxiter; i++) {
iter_monitor(monitor, vops, x);
debug_printf(DP_DEBUG3, "#%d: %f\n", i, (double)sqrtf(rsnew));
iter_op_call(linop, Ap, p); // Ap = A p
vops->axpy(N, Ap, l2lambda, p);
float pAp = (float)vops->dot(N, p, Ap);
if (0. == pAp)
break;
float alpha = rsold / pAp;
vops->axpy(N, x, +alpha, p);
vops->axpy(N, r, -alpha, Ap);
rsnew = (float)pow(vops->norm(N, r), 2.);
float beta = rsnew / rsold;
rsold = rsnew;
if (rsnew <= eps_squared) {
//debug_printf(DP_DEBUG3, "%d ", i);
break;
}
vops->xpay(N, beta, p, r); // p = beta * p + r
}
cleanup:
vops->del(Ap);
vops->del(p);
vops->del(r);
debug_printf(DP_DEBUG2, "\t cg: %3d\n", i);
return sqrtf(rsnew);
}
/**
* Iterative Soft Thresholding
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
*/
void ist(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild, long N, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*thresh)(void* data, float lambda, float* dst, const float* src),
void* tdata,
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if (NULL != obj_eval) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#%d OBJVAL= %f\n", itrdata.iter, objval);
}
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
thresh(tdata, tau, x, x);
op(data, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau * lambda_scale, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
if (NULL != x_truth)
vops->del(x_err);
vops->del(r);
}
/**
* Iterative Soft Thresholding/FISTA to solve min || b - Ax ||_2 + lambda || T x ||_1
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
*/
void fista(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild,
long N, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*thresh)(void* data, float lambda, float* dst, const float* src),
void* tdata,
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
float* o = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
float ra = 1.;
vops->copy(N, o, x);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if (NULL != obj_eval) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#%d OBJVAL= %f\n", itrdata.iter, objval);
}
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
thresh(tdata, lambda_scale * tau, x, x);
ravine(vops, N, &ra, x, o); // FISTA
op(data, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
vops->del(o);
vops->del(r);
if (NULL != x_truth)
vops->del(x_err);
}
/**
* Landweber L. An iteration formula for Fredholm integral equations of the
* first kind. Amer. J. Math. 1951; 73, 615-624.
*/
void landweber(unsigned int maxiter, float epsilon, float alpha, long N, long M, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*adj)(void* data, float* dst, const float* src),
float* x, const float* b,
float (*obj_eval)(const void*, const float*))
{
float* r = vops->allocate(M);
float* p = vops->allocate(N);
double rsnot = vops->norm(M, b);
UNUSED(obj_eval);
for (unsigned int i = 0; i < maxiter; i++) {
op(data, r, x); // r = A x
vops->xpay(M, -1., r, b); // r = b - r = b - A x
double rsnew = vops->norm(M, r);
debug_printf(DP_DEBUG3, "#%d: %f\n", i, rsnew / rsnot);
if (rsnew < epsilon)
break;
adj(data, p, r);
vops->axpy(N, x, alpha, p);
}
vops->del(r);
vops->del(p);
}
/**
* Conjugate Gradient Descent to solve Ax = b for symmetric A
*
* @param maxiter maximum number of iterations
* @param regularization parameter
* @param epsilon stop criterion
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param linop linear operator, i.e. A
* @param x initial estimate
* @param b observations
*/
float conjgrad(unsigned int maxiter, float l2lambda, float epsilon,
long N, void* data,
const struct vec_iter_s* vops,
void (*linop)(void* data, float* dst, const float* src),
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
float* r = vops->allocate(N);
float* p = vops->allocate(N);
float* Ap = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
// The first calculation of the residual might not
// be necessary in some cases...
linop(data, r, x); // r = A x
vops->axpy(N, r, l2lambda, x);
vops->xpay(N, -1., r, b); // r = b - r = b - A x
vops->copy(N, p, r); // p = r
float rsnot = (float)pow(vops->norm(N, r), 2.);
float rsold = rsnot;
float rsnew = rsnot;
float eps_squared = pow(epsilon, 2.);
if (0. == rsold) {
debug_printf(DP_DEBUG3, "CG: early out\n");
return 0.;
}
for (unsigned int i = 0; i < maxiter; i++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if ((NULL != obj_eval) && (NULL != obj_eval_data)) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#CG%d OBJVAL= %f\n", i, objval);
}
debug_printf(DP_DEBUG3, "#%d: %f\n", i, (double)sqrtf(rsnew));
linop(data, Ap, p); // Ap = A p
vops->axpy(N, Ap, l2lambda, p);
float pAp = (float)vops->dot(N, p, Ap);
if (0. == pAp)
break;
float alpha = rsold / pAp;
vops->axpy(N, x, +alpha, p);
vops->axpy(N, r, -alpha, Ap);
rsnew = (float)pow(vops->norm(N, r), 2.);
float beta = rsnew / rsold;
rsold = rsnew;
if (rsnew <= eps_squared) {
//debug_printf(DP_DEBUG3, "%d ", i);
break;
}
vops->xpay(N, beta, p, r); // p = beta * p + r
}
vops->del(Ap);
vops->del(p);
vops->del(r);
if (NULL != x_truth)
vops->del(x_err);
return sqrtf(rsnew);
}