bool MatrixEquation::Solve_Cholesky(Vector& x) const
{
if(!IsValid() || !A.isSquare()) {
cout<<"Invalid dimensions in Solve_Cholesky"<<endl;
return false;
}
LDLDecomposition<Real> chol;
chol.set(A);
return chol.backSub(b,x);
}
ConvergenceResult MinimizationProblem::SolveLM(int& iters,Real lambda0,Real lambdaScale)
{
Real lambda=lambda0;
grad.resize(x.n);
H.resize(x.n,x.n);
dx.resize(x.n);
Vector x0,diagH;
LDLDecomposition<Real> ldl;
Real fx0;
fx = (*f)(x);
bool tookStep=true;
int maxIters=iters;
for(iters=0;iters<maxIters;iters++) {
if(tookStep) {
f->Gradient(x,grad);
f->Hessian(x,H);
H.getDiagCopy(0,diagH);
}
for(int i=0;i<x.n;i++) H(i,i) += Max((Real)1e-4,diagH(i))*lambda;
ldl.set(H);
Vector d; ldl.LDL.getDiagRef(0,d);
for(int i=0;i<x.n;i++) {
if(d(i) <= 1e-3) { //non-PD or nearly non-PD (adjust lambda?)
d(i) = 1e-3;
}
}
fx0 = fx;
x0 = x;
ldl.backSub(grad,dx);
dx.inplaceNegative();
x += dx;
//check if the step works or not
fx = (*f)(x);
if(Abs(fx0 - fx) < tolf) return ConvergenceF;
if(grad.maxAbsElement() < tolgrad) return ConvergenceF;
if(dx.maxAbsElement() < tolx) return ConvergenceX;
if(fx < fx0) {
tookStep=true;
lambda /= lambdaScale;
}
else {
tookStep=false;
x = x0;
fx = fx0;
H.copyDiag(0,diagH);
lambda *= lambdaScale;
}
}
return MaxItersReached;
}
ConvergenceResult ConstrainedMinimizationProblem::SolveNewton(int& iters)
{
if(!CheckPoint(x)) return ConvergenceError;
Vector dx;
fx=(*f)(x);
grad.resize(x.n);
H.resize(x.n,x.n);
LDLDecomposition<Real> ldl;
int maxIters=iters;
for(iters=0;iters<maxIters;iters++) {
f->Gradient(x,grad);
f->Hessian(x,H);
ldl.set(H);
Vector d;
ldl.LDL.getDiagRef(0,d);
if(d.minElement() < 0) {
if(verbose>=1) cout<<"Warning, hessian is not positive definite"<<endl;
for(int i=0;i<d.n;i++)
if(d(i) < 1e-4) d(i) = 1e-4;
//return ConvergenceError;
}
ldl.backSub(grad,dx);
if(dot(dx,grad) < Zero) {
if(verbose>=1) cout<<"ConstrainedMinimizationProblem::SolveNewton(): Warning, hessian direction opposes gradient on step "<<iters<<endl;
//dx.setNegative(grad);
}
dx.inplaceNegative();
Real dxnorm0 = dx.maxAbsElement();
NullspaceProjection(x,dx);
Real dxnorm = dx.maxAbsElement();
Real alpha0 = Min(dxnorm0/dxnorm,(Real)10);
ConvergenceResult res=LineMinimizationStep(dx,alpha0);
if(res != MaxItersReached) return res;
}
if(verbose>=1) cout<<"ConstrainedMinimzationProblem::Solve(): Max iters reached "<<iters<<"."<<endl;
return MaxItersReached;
}
bool LeastSquares(const Matrix& data,const Vector& outcome,
Vector& coeffs,Real& offset)
{
assert(outcome.n == data.m);
//solve for vector (coeffs,offset) using normal equations
Matrix A(data.n+1,data.n+1);
Vector b(data.n+1),c(data.n+1);
Vector xi,xj;
for(int i=0;i<data.n;i++) {
data.getColRef(i,xi);
for(int j=i;j<data.n;j++) {
data.getColRef(j,xj);
A(j,i) = A(i,j) = xi.dot(xj);
}
b(i) = xi.dot(outcome);
}
for(int i=0;i<data.n;i++) {
data.getColRef(i,xi);
A(data.n,i) = A(i,data.n) = Sum(xi);
}
A(data.n,data.n) = (Real)data.m;
b(data.n) = Sum(outcome);
//Solve Ac=b for coefficients c;
LDLDecomposition<Real> ldl;
ldl.set(A);
ldl.backSub(b,c);
/* {
cout<<"Cholesky didn't succeed!"<<endl;
cout<<"Matrix: "<<endl<<MatrixPrinter(A)<<endl;
return false;
}
*/
coeffs.resize(data.n);
c.getSubVectorCopy(0,coeffs);
offset = c(data.n);
return true;
}
ConvergenceResult BCMinimizationProblem::SolveNewton(int& iters)
{
grad.resize(x.n);
H.resize(x.n,x.n);
Vector dx(x.n);
LDLDecomposition<Real> ldl;
fx = (*f)(x);
int maxIters=iters;
for(iters=0;iters<maxIters;iters++) {
f->Gradient(x,grad);
f->Hessian(x,H);
//f(dx + x0) ~= f(x0) + grad_f*dx + 1/2 dx'*H_f*dx
//grad(f)(x+dx) ~= grad_f + 1/2*H_f*dx + O(dx^2)
//setting = grad_f(x+dx) = 0, we see dx ~= -H_f^-1 grad_f
//use that as the backtracking search direction (if it's in same direction as grad)
ldl.set(H);
Vector d;
ldl.LDL.getDiagRef(0,d);
if(d.minElement() < 0) {
if(verbose>=1) cout<<"Warning, hessian is not positive definite"<<endl;
for(int i=0;i<d.n;i++)
if(d(i) < 1e-4) d(i) = 1e-4;
//return ConvergenceError;
}
ldl.backSub(grad,dx);
if(dot(dx,grad) < Zero) {
if(verbose>=1) cout<<"BCMinimizationProblem::SolveNewton(): Warning, hessian direction opposes gradient on step "<<iters<<endl;
//dx.setNegative(grad);
}
dx.inplaceNegative();
Real alpha0 = 1;
ConvergenceResult res = LineMinimizationStep(dx,alpha0);
if(res != MaxItersReached) return res;
}
return MaxItersReached;
}
ConvergenceResult BCMinimizationProblem::SolveQuasiNewton(int& iters)
{
Assert(H.m == x.n && H.n == x.n);
grad.resize(x.n);
Vector x0,grad0,s,q,upd,temp,dx(x.n);
LDLDecomposition<Real> ldl;
ldl.set(H);
fx = (*f)(x);
f->Gradient(x,grad);
int maxIters=iters;
for(iters=0;iters<maxIters;iters++) {
ldl.backSub(grad,dx);
if(dot(dx,grad) < Zero) {
if(verbose>=1) cout<<"BCMinimizationProblem::SolveQuasiNewton(): Warning, hessian direction opposes gradient on step "<<iters<<endl;
//dx.setNegative(grad);
}
grad.inplaceNegative();
x0 = x;
grad0 = grad;
Real alpha0=1.0;
ConvergenceResult res = LineMinimizationStep(dx,alpha0);
if(res != MaxItersReached) return res;
//update the gradient
f->Gradient(x,grad);
//Hessian update step
//BFGS has H' = H + q*qt / qt*s - Ht*s*st*H/st*H*s
//where s = x - x0
//q = grad - grad0
s.sub(x,x0);
q.sub(grad,grad0);
//first part
Real qdots = q.dot(s);
if(qdots <= 0) {
if(verbose >= 2) printf("BCMinimizationProblem: Warning, gradient update is in wrong direction\n");
//revert to identity
ldl.LDL.setIdentity();
continue;
}
else {
upd.div(q,Sqrt(qdots));
ldl.update(upd);
}
//second part
//multiply s by H => upd
ldl.mulLT(s,temp);
ldl.mulD(temp,temp);
ldl.mulL(temp,upd);
Real sHs = upd.dot(s);
Assert(sHs > 0);
upd /= Sqrt(sHs);
if(!ldl.downdate(upd)) {
if(verbose>=1) cout<<"Unable to maintain strict positive definiteness of hessian!"<<endl;
//revert to identity
ldl.LDL.setIdentity();
continue;
//TODO: fix the downdate
//return ConvergenceError;
}
Vector d;
ldl.LDL.getDiagRef(0,d);
if(d.minElement() <= 0) {
if(verbose>=1) cout<<"Unable to maintain positive definiteness of hessian!"<<endl;
return ConvergenceError;
}
}
return MaxItersReached;
}