void CSNGAInd::rnd_init() //variable initival
{
for(int n=0;n<nvar;n++)
{
x_var[n] = lowBound[n] + rnd_uni(&rnd_uni_init)*(uppBound[n] - lowBound[n]); //initival the variable
}
obj_eval();
}
/*------------------------------------------------------------------------- * Function: ptr_apply * * Purpose: Applying a pointer type to an argument list consisting of * a single SILO database object (SDO) causes the object to * be cast to that type. * * Return: Success: Ptr to a new SDO object with the appropriate * type. * * Failure: NIL * * Programmer: Robb Matzke * [email protected] * Dec 6 1996 * * Modifications: * *------------------------------------------------------------------------- */ static obj_t ptr_apply (obj_t _self, obj_t args) { obj_t sdo=NIL, retval=NIL; if (1!=F_length(args)) { out_errorn ("typecast: wrong number of arguments"); return NIL; } sdo = obj_eval (cons_head (args)); retval = sdo_cast (sdo, _self); obj_dest (sdo); return retval; }
/***************************************************************
Load
***************************************************************/
static obj_ptr _load_imp(cptr file, obj_ptr env)
{
port_ptr p;
obj_ptr o = NIL;
p = port_open_input_file(file);
while (!port_eof(p))
{
o = obj_read(p);
if (ERRORP(o)) break;
o = obj_eval(o, env);
if (ERRORP(o)) break;
port_skip_while(p, isspace);
}
if (ERRORP(o))
error_add_file_info(o, port_name(p), port_line(p));
port_close(p);
return o;
}
/*
* ADMM (ADMM-2 from Afonso et al.)
*
* Solves min_x 0.5 || y - Ax ||_2^2 + sum_i f_i(G_i x), where the f_i are
* arbitrary convex functions. If Aop is NULL, solves min_x sum_i f_i(G_i x)
*
* Each iteration requires solving the proximal of f_i, as well as applying
* G_i, G_i^H, and G_i^H G_i, all which must be provided in admm_plan_s.
*/
void admm(struct admm_history_s* history, const struct admm_plan_s* plan,
unsigned int D, const long z_dims[D],
long N, float* x, const float* x_adj,
const struct vec_iter_s* vops,
void (*Aop)(void* _data, float* _dst, const float* _src),
void* Aop_data,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
bool fast = plan->fast;
double ABSTOL = plan->ABSTOL;
double RELTOL = plan->RELTOL;
float tau = plan->tau;
float mu = plan->mu;
unsigned int num_funs = D;
long pos = 0;
long fake_strs[num_funs];
md_singleton_dims(num_funs, fake_strs);
long M = md_calc_offset(num_funs, fake_strs, z_dims);
// allocate memory for history
history->r_norm = *TYPE_ALLOC(double[plan->maxiter]);
history->s_norm = *TYPE_ALLOC(double[plan->maxiter]);
history->eps_pri = *TYPE_ALLOC(double[plan->maxiter]);
history->eps_dual = *TYPE_ALLOC(double[plan->maxiter]);
history->objective = *TYPE_ALLOC(double[plan->maxiter]);
history->rho = *TYPE_ALLOC(float[plan->maxiter]);
history->relMSE = *TYPE_ALLOC(double[plan->maxiter]);
long Mjmax = 0;
for(unsigned int i = 0; i < num_funs; i++)
Mjmax = MAX(Mjmax, z_dims[i]);
struct iter_history_s cghistory;
cghistory.numiter = 0;
cghistory.relMSE = *TYPE_ALLOC(double[plan->maxitercg]);
cghistory.objective = *TYPE_ALLOC(double[plan->maxitercg]);
cghistory.resid = *TYPE_ALLOC(double[plan->maxitercg]);
// allocate memory for all of our auxiliary variables
float* z = vops->allocate(M);
float* u = vops->allocate(M);
float* rhs = vops->allocate(N);
float* r = vops->allocate(M);
float* s = vops->allocate(N);
float* Gjx_plus_uj = vops->allocate(Mjmax);
float* GH_usum = NULL;
float* zj_old = NULL;
if (!fast) {
GH_usum = vops->allocate(N);
zj_old = vops->allocate(Mjmax);
}
float* x_err = NULL;
if (NULL != plan->image_truth)
x_err = vops->allocate(N);
if (!fast) {
if (NULL != plan->image_truth)
debug_printf(DP_DEBUG3, "%3s\t%3s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\n", "iter", "cgiter", "rho", "r norm", "eps pri", "s norm", "eps dual", "obj", "relMSE");
else
debug_printf(DP_DEBUG3, "%3s\t%3s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\n", "iter", "cgiter", "rho", "r norm", "eps pri", "s norm", "eps dual", "obj");
}
float rho = plan->rho;
struct admm_normaleq_data ndata;
ndata.N = N;
ndata.num_funs = num_funs;
ndata.ops = plan->ops;
ndata.Aop = Aop;
ndata.Aop_data = Aop_data;
ndata.rho = 1.;
ndata.tmp = vops->allocate(N);
struct cg_data_s* cgdata = (struct cg_data_s*) cg_data_init(N, vops);
// hogwild
int hw_K = 1;
int hw_k = 0;
unsigned int grad_iter = 0; // keep track of number of gradient evaluations
if (plan->do_warmstart) {
for (unsigned int j = 0; j < num_funs; j++) {
// initialize for j'th function update
pos = md_calc_offset(j, fake_strs, z_dims);
long Mj = z_dims[j];
plan->ops[j].forward(plan->ops[j].data, Gjx_plus_uj, x); // Gj(x)
if (0 == rho)
vops->copy(Mj, z + pos, Gjx_plus_uj);
else
plan->prox_ops[j].prox_fun(plan->prox_ops[j].data, 1. / rho, z + pos, Gjx_plus_uj);
vops->sub(Mj, u + pos, Gjx_plus_uj, z + pos);
}
} else {
vops->clear(M, z);
vops->clear(M, u);
}
for (unsigned int i = 0; i < plan->maxiter; i++) {
// update x
vops->clear(N, rhs);
vops->sub(M, r, z, u);
for (unsigned int j = 0; j < num_funs; j++) {
pos = md_calc_offset(j, fake_strs, z_dims);
plan->ops[j].adjoint(plan->ops[j].data, s, r + pos);
vops->add(N, rhs, rhs, s);
}
if ((NULL != Aop) && (NULL != Aop_data)) {
vops->xpay(N, rho, rhs, x_adj);
ndata.rho = rho;
}
// x update: use plan->xupdate_fun if specified. use conjgrad otherwise
if ((NULL != plan->xupdate_fun) && (NULL != plan->xupdate_data)) {
plan->xupdate_fun(plan->xupdate_data, rho, x, rhs);
grad_iter++;
} else {
float eps = vops->norm(N, rhs);
if (eps > 0.) {
conjgrad_hist_prealloc(&cghistory, plan->maxitercg, 0., 1.E-3 * eps, N, &ndata, cgdata, vops, admm_normaleq, x, rhs, plan->image_truth, obj_eval_data, obj_eval);
//conjgrad_hist(&cghistory, plan->maxitercg, 0., 1.E-3 * eps, N, &ndata, vops, admm_normaleq, x, rhs, plan->image_truth, obj_eval_data, obj_eval);
} else {
cghistory.numiter = 0;
cghistory.relMSE[0] = 0.;
cghistory.objective[0] = 0.;
cghistory.resid[0] = 0.;
}
grad_iter += cghistory.numiter;
}
if ((NULL != obj_eval) && (NULL != obj_eval_data))
history->objective[i] = obj_eval(obj_eval_data, x);
else
history->objective[i] = 0.;
double n1 = 0.;
if (!fast) {
vops->clear(N, GH_usum);
vops->clear(N, s);
vops->clear(M, r);
}
// z_j prox
for (unsigned int j = 0; j < num_funs; j++) {
// initialize for j'th function update
pos = md_calc_offset(j, fake_strs, z_dims);
long Mj = z_dims[j];
plan->ops[j].forward(plan->ops[j].data, Gjx_plus_uj, x); // Gj(x)
// over-relaxation: Gjx_hat = alpha * Gj(x) + (1 - alpha) * zj_old
if (!fast) {
vops->copy(Mj, zj_old, z + pos);
vops->copy(Mj, r + pos, Gjx_plus_uj); // rj = Gj(x)
n1 = n1 + vops->dot(Mj, r + pos, r + pos);
vops->smul(Mj, plan->alpha, Gjx_plus_uj, Gjx_plus_uj);
vops->axpy(Mj, Gjx_plus_uj, (1. - plan->alpha), z + pos);
}
vops->add(Mj, Gjx_plus_uj, Gjx_plus_uj, u + pos); // Gj(x) + uj
if (0 == rho)
vops->copy(Mj, z + pos, Gjx_plus_uj);
else
plan->prox_ops[j].prox_fun(plan->prox_ops[j].data, 1. / rho, z + pos, Gjx_plus_uj);
vops->sub(Mj, u + pos, Gjx_plus_uj, z + pos);
if (!fast) {
// rj = rj - zj = Gj(x) - zj
vops->sub(Mj, r + pos, r + pos, z + pos);
// add next term to s: s = s + Gj^H (zj - zj_old)
vops->sub(Mj, zj_old, z + pos, zj_old);
plan->ops[j].adjoint(plan->ops[j].data, rhs, zj_old);
vops->add(N, s, s, rhs);
// GH_usum += G_j^H uj (for updating eps_dual)
plan->ops[j].adjoint(plan->ops[j].data, rhs, u + pos);
vops->add(N, GH_usum, GH_usum, rhs);
}
}
history->rho[i] = rho;
if (!fast) {
history->s_norm[i] = rho * vops->norm(N, s);
history->r_norm[i] = vops->norm(M, r);
n1 = sqrt(n1);
double n2 = vops->norm(M, z);
history->eps_pri[i] = ABSTOL * sqrt(M) + RELTOL * (n1 > n2 ? n1 : n2);
history->eps_dual[i] = ABSTOL * sqrt(N) + RELTOL * rho * vops->norm(N, GH_usum);
if (NULL != plan->image_truth) {
vops->sub(N, x_err, x, plan->image_truth);
history->relMSE[i] = vops->norm(N, x_err) / vops->norm(N, plan->image_truth);
}
if (NULL != plan->image_truth)
debug_printf(DP_DEBUG3, "%3d\t%3d\t%10.4f\t%10.4f\t%10.4f\t%10.4f\t%10.4f\t%10.4f\t%10.4f\n", i, grad_iter, history->rho[i], history->r_norm[i], history->eps_pri[i], history->s_norm[i], history->eps_dual[i], history->objective[i], history->relMSE[i]);
else
debug_printf(DP_DEBUG3, "%3d\t%3d\t%10.4f\t%10.4f\t%10.4f\t%10.4f\t%10.5f\t%10.4f\n", i, grad_iter, history->rho[i], history->r_norm[i], history->eps_pri[i], history->s_norm[i], history->eps_dual[i], history->objective[i]);
if ((grad_iter > plan->maxiter)
|| ((history->r_norm[i] < history->eps_pri[i])
&& (history->s_norm[i] < history->eps_dual[i]))) {
history->numiter = i;
break;
}
if (plan->dynamic_rho) {
if (history->r_norm[i] > mu * history->s_norm[i]) {
rho = rho * tau;
vops->smul(M, 1. / tau, u, u);
} else if (history->s_norm[i] > mu * history->r_norm[i]) {
rho = rho / tau;
vops->smul(M, tau, u, u);
}
}
} else {
debug_printf(DP_DEBUG3, "### ITER: %d (%d)\n", i, grad_iter);
if (grad_iter > plan->maxiter)
break;
}
if (plan->hogwild) {
hw_k++;
if (hw_k == hw_K) {
hw_k = 0;
rho *= 2.;
hw_K *= 2;
vops->smul(M, 0.5, u, u);
}
}
}
// cleanup
vops->del(z);
vops->del(u);
vops->del(rhs);
vops->del(Gjx_plus_uj);
vops->del(r);
vops->del(s);
if (!fast) {
vops->del(GH_usum);
vops->del(zj_old);
}
if (NULL != plan->image_truth)
vops->del(x_err);
vops->del(ndata.tmp);
cg_data_free(cgdata, vops);
free(cghistory.resid);
free(cghistory.objective);
free(cghistory.relMSE);
free(history->r_norm);
free(history->s_norm);
free(history->eps_pri);
free(history->eps_dual);
free(history->objective);
free(history->rho);
}
/**
* Conjugate Gradient Descent with history saving and pre-allocated data.
* The history and cgdata should be preallocated.
*
* Preallocating the data is useful if the conjgrad is called within
* an iteration loop (i.e. admm).
*/
float conjgrad_hist_prealloc(struct iter_history_s* history, unsigned int maxiter, float l2lambda, float epsilon,
long N, void* data, const struct cg_data_s* cgdata,
const struct vec_iter_s* vops,
void (*linop)(void* data, float* dst, const float* src),
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
float* r = cgdata->r;
float* p = cgdata->p;
float* Ap = cgdata->Ap;
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
// The first calculation of the residual might not
// be necessary in some cases...
linop(data, r, x); // r = A x
vops->axpy(N, r, l2lambda, x);
vops->xpay(N, -1., r, b); // r = b - r = b - A x
vops->copy(N, p, r); // p = r
float rsnot = (float)pow(vops->norm(N, r), 2.);
float rsold = rsnot;
float rsnew = rsnot;
float eps_squared = pow(epsilon, 2.);
for (unsigned int i = 0; i < maxiter; i++) {
history->numiter = i + 1;
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
float relMSE = vops->norm(N, x_err) / vops->norm(N, x_truth);
history->relMSE[i] = relMSE;
debug_printf(DP_DEBUG3, "relMSE = %f\n", relMSE);
}
if ((NULL != obj_eval) && (NULL != obj_eval_data)) {
float objval = obj_eval(obj_eval_data, x);
history->objective[i] = objval;
debug_printf(DP_DEBUG3, "#CG%d OBJVAL= %f\n", i, objval);
}
//debug_printf(DP_DEBUG3, "#%d: %f\n", i, (double)sqrtf(rsnew));
linop(data, Ap, p); // Ap = A p
vops->axpy(N, Ap, l2lambda, p);
float alpha = rsold / (float)vops->dot(N, p, Ap);
vops->axpy(N, x, +alpha, p);
vops->axpy(N, r, -alpha, Ap);
rsnew = (float)pow(vops->norm(N, r), 2.);
float beta = rsnew / rsold;
rsold = rsnew;
if (rsnew <= eps_squared) {
//debug_printf(DP_DEBUG3, "%d ", i);
break;
}
vops->xpay(N, beta, p, r); // p = beta * p + r
history->resid[i] = sqrtf(rsnew);
}
if (NULL != x_truth)
vops->del(x_err);
return sqrtf(rsnew);
}
/**
* Iterative Soft Thresholding
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
*/
void ist(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild, long N, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*thresh)(void* data, float lambda, float* dst, const float* src),
void* tdata,
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if (NULL != obj_eval) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#%d OBJVAL= %f\n", itrdata.iter, objval);
}
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
thresh(tdata, tau, x, x);
op(data, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau * lambda_scale, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
if (NULL != x_truth)
vops->del(x_err);
vops->del(r);
}
/**
* Iterative Soft Thresholding/FISTA to solve min || b - Ax ||_2 + lambda || T x ||_1
*
* @param maxiter maximum number of iterations
* @param epsilon stop criterion
* @param tau (step size) weighting on the residual term, A^H (b - Ax)
* @param lambda_start initial regularization weighting
* @param lambda_end final regularization weighting (for continuation)
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param op linear operator, e.g. A
* @param thresh threshold function, e.g. complex soft threshold
* @param x initial estimate
* @param b observations
*/
void fista(unsigned int maxiter, float epsilon, float tau,
float continuation, bool hogwild,
long N, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*thresh)(void* data, float lambda, float* dst, const float* src),
void* tdata,
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
struct iter_data itrdata = {
.rsnew = 1.,
.rsnot = 1.,
.iter = 0,
.maxiter = maxiter,
};
float* r = vops->allocate(N);
float* o = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
float ra = 1.;
vops->copy(N, o, x);
itrdata.rsnot = vops->norm(N, b);
float ls_old = 1.;
float lambda_scale = 1.;
int hogwild_k = 0;
int hogwild_K = 10;
for (itrdata.iter = 0; itrdata.iter < maxiter; itrdata.iter++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if (NULL != obj_eval) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#%d OBJVAL= %f\n", itrdata.iter, objval);
}
ls_old = lambda_scale;
lambda_scale = ist_continuation(&itrdata, continuation);
if (lambda_scale != ls_old)
debug_printf(DP_DEBUG3, "##lambda_scale = %f\n", lambda_scale);
thresh(tdata, lambda_scale * tau, x, x);
ravine(vops, N, &ra, x, o); // FISTA
op(data, r, x); // r = A x
vops->xpay(N, -1., r, b); // r = b - r = b - A x
itrdata.rsnew = vops->norm(N, r);
debug_printf(DP_DEBUG3, "#It %03d: %f \n", itrdata.iter, itrdata.rsnew / itrdata.rsnot);
if (itrdata.rsnew < epsilon)
break;
vops->axpy(N, x, tau, r);
if (hogwild)
hogwild_k++;
if (hogwild_k == hogwild_K) {
hogwild_K *= 2;
hogwild_k = 0;
tau /= 2;
}
}
debug_printf(DP_DEBUG3, "\n");
vops->del(o);
vops->del(r);
if (NULL != x_truth)
vops->del(x_err);
}
/**
* Landweber L. An iteration formula for Fredholm integral equations of the
* first kind. Amer. J. Math. 1951; 73, 615-624.
*/
void landweber(unsigned int maxiter, float epsilon, float alpha, long N, long M, void* data,
const struct vec_iter_s* vops,
void (*op)(void* data, float* dst, const float* src),
void (*adj)(void* data, float* dst, const float* src),
float* x, const float* b,
float (*obj_eval)(const void*, const float*))
{
float* r = vops->allocate(M);
float* p = vops->allocate(N);
double rsnot = vops->norm(M, b);
UNUSED(obj_eval);
for (unsigned int i = 0; i < maxiter; i++) {
op(data, r, x); // r = A x
vops->xpay(M, -1., r, b); // r = b - r = b - A x
double rsnew = vops->norm(M, r);
debug_printf(DP_DEBUG3, "#%d: %f\n", i, rsnew / rsnot);
if (rsnew < epsilon)
break;
adj(data, p, r);
vops->axpy(N, x, alpha, p);
}
vops->del(r);
vops->del(p);
}
/**
* Conjugate Gradient Descent to solve Ax = b for symmetric A
*
* @param maxiter maximum number of iterations
* @param regularization parameter
* @param epsilon stop criterion
* @param N size of input, x
* @param data structure, e.g. sense_data
* @param vops vector ops definition
* @param linop linear operator, i.e. A
* @param x initial estimate
* @param b observations
*/
float conjgrad(unsigned int maxiter, float l2lambda, float epsilon,
long N, void* data,
const struct vec_iter_s* vops,
void (*linop)(void* data, float* dst, const float* src),
float* x, const float* b, const float* x_truth,
void* obj_eval_data,
float (*obj_eval)(const void*, const float*))
{
float* r = vops->allocate(N);
float* p = vops->allocate(N);
float* Ap = vops->allocate(N);
float* x_err = NULL;
if (NULL != x_truth)
x_err = vops->allocate(N);
// The first calculation of the residual might not
// be necessary in some cases...
linop(data, r, x); // r = A x
vops->axpy(N, r, l2lambda, x);
vops->xpay(N, -1., r, b); // r = b - r = b - A x
vops->copy(N, p, r); // p = r
float rsnot = (float)pow(vops->norm(N, r), 2.);
float rsold = rsnot;
float rsnew = rsnot;
float eps_squared = pow(epsilon, 2.);
if (0. == rsold) {
debug_printf(DP_DEBUG3, "CG: early out\n");
return 0.;
}
for (unsigned int i = 0; i < maxiter; i++) {
if (NULL != x_truth) {
vops->sub(N, x_err, x, x_truth);
debug_printf(DP_DEBUG3, "relMSE = %f\n", vops->norm(N, x_err) / vops->norm(N, x_truth));
}
if ((NULL != obj_eval) && (NULL != obj_eval_data)) {
float objval = obj_eval(obj_eval_data, x);
debug_printf(DP_DEBUG3, "#CG%d OBJVAL= %f\n", i, objval);
}
debug_printf(DP_DEBUG3, "#%d: %f\n", i, (double)sqrtf(rsnew));
linop(data, Ap, p); // Ap = A p
vops->axpy(N, Ap, l2lambda, p);
float pAp = (float)vops->dot(N, p, Ap);
if (0. == pAp)
break;
float alpha = rsold / pAp;
vops->axpy(N, x, +alpha, p);
vops->axpy(N, r, -alpha, Ap);
rsnew = (float)pow(vops->norm(N, r), 2.);
float beta = rsnew / rsold;
rsold = rsnew;
if (rsnew <= eps_squared) {
//debug_printf(DP_DEBUG3, "%d ", i);
break;
}
vops->xpay(N, beta, p, r); // p = beta * p + r
}
vops->del(Ap);
vops->del(p);
vops->del(r);
if (NULL != x_truth)
vops->del(x_err);
return sqrtf(rsnew);
}
