void SteepestDescent (Vector & x, const MinFunction & fun, const OptiParameters & par) { int it, n = x.Size(); Vector xnew(n), p(n), g(n), g2(n); double val, alphahat; int fail; val = fun.FuncGrad(x, g); alphahat = 1; // testout << "f = "; for (it = 0; it < 10; it++) { // testout << val << " "; // p = -g; p.Set (-1, g); lines (x, xnew, p, val, g, fun, par, alphahat, -1e5, 0.1, 0.1, 1, 10, 0.1, 0.1, 0.6, fail); x = xnew; } // testout << endl; }
double BFGS ( Vector & x, // i: Startwert // o: Loesung, falls IFAIL = 0 const MinFunction & fun, const OptiParameters & par, double eps ) { int i, j, n = x.Size(); long it; char a1crit, a3acrit; Vector d(n), g(n), p(n), temp(n), bs(n), xneu(n), y(n), s(n), x0(n); DenseMatrix l(n); DenseMatrix hesse(n); double /* normg, */ alphahat, hd, fold; double a1, a2; const double mu1 = 0.1, sigma = 0.1, xi1 = 1, xi2 = 10; const double tau = 0.1, tau1 = 0.1, tau2 = 0.6; Vector typx(x.Size()); // i: typische Groessenordnung der Komponenten double f, f0; double typf; // i: typische Groessenordnung der Loesung double fmin = -1e5; // i: untere Schranke fuer Funktionswert // double eps = 1e-8; // i: Abbruchschranke fuer relativen Gradienten double tauf = 0.1; // i: Abbruchschranke fuer die relative Aenderung der // Funktionswerte int ifail; // o: 0 .. Erfolg // -1 .. Unterschreitung von fmin // 1 .. kein Erfolg bei Liniensuche // 2 .. Überschreitung von itmax typx = par.typx; typf = par.typf; l = 0; for (i = 1; i <= n; i++) l.Elem(i, i) = 1; f = fun.FuncGrad (x, g); f0 = f; x0 = x; it = 0; do { // Restart if (it % (5 * n) == 0) { for (i = 1; i <= n; i++) d.Elem(i) = typf/ sqr (typx.Get(i)); // 1; for (i = 2; i <= n; i++) for (j = 1; j < i; j++) l.Elem(i, j) = 0; /* hesse = 0; for (i = 1; i <= n; i++) hesse.Elem(i, i) = typf / sqr (typx.Get(i)); fun.ApproximateHesse (x, hesse); Cholesky (hesse, l, d); */ } it++; if (it > par.maxit_bfgs) { ifail = 2; break; } // Solve with factorized B SolveLDLt (l, d, g, p); // (*testout) << "l " << l << endl // << "d " << d << endl // << "g " << g << endl // << "p " << p << endl; p *= -1; y = g; fold = f; // line search alphahat = 1; lines (x, xneu, p, f, g, fun, par, alphahat, fmin, mu1, sigma, xi1, xi2, tau, tau1, tau2, ifail); if(ifail == 1) (*testout) << "no success with linesearch" << endl; /* // if (it > par.maxit_bfgs/2) { (*testout) << "x = " << x << endl; (*testout) << "xneu = " << xneu << endl; (*testout) << "f = " << f << endl; (*testout) << "g = " << g << endl; } */ // (*testout) << "it = " << it << " f = " << f << endl; // if (ifail != 0) break; s.Set2 (1, xneu, -1, x); y *= -1; y.Add (1,g); // y += g; x = xneu; // BFGS Update MultLDLt (l, d, s, bs); a1 = y * s; a2 = s * bs; if (a1 > 0 && a2 > 0) { if (LDLtUpdate (l, d, 1 / a1, y) != 0) { // [JW] according to [JS] we can safely ignore this error //cerr << "BFGS update error1" << endl; //(*testout) << "BFGS update error1" << endl; //(*testout) << "l " << endl << l << endl //<< "d " << d << endl; ifail = 1; break; } if (LDLtUpdate (l, d, -1 / a2, bs) != 0) { // [JW] according to [JS] we can safely ignore this error //cerr << "BFGS update error2" << endl; //(*testout) << "BFGS update error2" << endl; //(*testout) << "l " << endl << l << endl // << "d " << d << endl; ifail = 1; break; } } // Calculate stop conditions hd = eps * max2 (typf, fabs (f)); a1crit = 1; for (i = 1; i <= n; i++) if ( fabs (g.Elem(i)) * max2 (typx.Elem(i), fabs (x.Elem(i))) > hd) a1crit = 0; a3acrit = (fold - f <= tauf * max2 (typf, fabs (f))); // testout << "g = " << g << endl; // testout << "a1crit, a3crit = " << int(a1crit) << ", " << int(a3acrit) << endl; /* // Output for tests normg = sqrt (g * g); testout << "it =" << setw (5) << it << " f =" << setw (12) << setprecision (5) << f << " |g| =" << setw (12) << setprecision (5) << normg; testout << " x = (" << setw (12) << setprecision (5) << x.Elem(1); for (i = 2; i <= n; i++) testout << "," << setw (12) << setprecision (5) << x.Elem(i); testout << ")" << endl; */ //(*testout) << "it = " << it << " f = " << f << " x = " << x << endl // << " g = " << g << " p = " << p << endl << endl; // (*testout) << "|g| = " << g.L2Norm() << endl; if (g.L2Norm() < fun.GradStopping (x)) break; } while (!a1crit || !a3acrit); /* (*testout) << "it = " << it << " g = " << g << " f = " << f << " fail = " << ifail << endl; */ if (f0 < f || (ifail == 1)) { (*testout) << "fail, f = " << f << " f0 = " << f0 << endl; f = f0; x = x0; } // (*testout) << "x = " << x << ", x0 = " << x0 << endl; return f; }
/// Line search, modified Mangasarien conditions void lines (Vector & x, // i: initial point of line-search Vector & xneu, // o: solution, if successful Vector & p, // i: search direction double & f, // i: function-value at x // o: function-value at xneu, iff ifail = 0 Vector & g, // i: gradient at x // o: gradient at xneu, iff ifail = 0 const MinFunction & fun, // function to minimize const OptiParameters & par, double & alphahat, // i: initial value for alpha_hat // o: solution alpha iff ifail = 0 double fmin, // i: lower bound for f double mu1, // i: Parameter mu_1 of Alg.2.1 double sigma, // i: Parameter sigma of Alg.2.1 double xi1, // i: Parameter xi_1 of Alg.2.1 double xi2, // i: Parameter xi_1 of Alg.2.1 double tau, // i: Parameter tau of Alg.2.1 double tau1, // i: Parameter tau_1 of Alg.2.1 double tau2, // i: Parameter tau_2 of Alg.2.1 int & ifail) // o: 0 on success // -1 bei termination because lower limit fmin // 1 bei illegal termination due to different reasons { double phi0, phi0prime, phi1, phi1prime, phihatprime; double alpha1, alpha2, alphaincr, c; char flag = 1; long it; alpha1 = 0; alpha2 = 1e50; phi0 = phi1 = f; phi0prime = g * p; if (phi0prime > 0) { ifail = 1; return; } ifail = 1; phi1prime = phi0prime; // (*testout) << "phi0prime = " << phi0prime << endl; // it = 100000l; it = 0; while (it++ <= par.maxit_linsearch) { xneu.Set2 (1, x, alphahat, p); // f = fun.FuncGrad (xneu, g); // f = fun.Func (xneu); f = fun.FuncDeriv (xneu, p, phihatprime); // (*testout) << "lines, f = " << f << " phip = " << phihatprime << endl; if (f < fmin) { ifail = -1; break; } if (alpha2 - alpha1 < eps0 * alpha2) { ifail = 0; break; } // (*testout) << "i = " << it << " al = " << alphahat << " f = " << f << " fprime " << phihatprime << endl;; if (f - phi0 > mu1 * alphahat * phi1prime + eps0 * fabs (phi0)) { flag = 0; alpha2 = alphahat; c = (f - phi1 - phi1prime * (alphahat-alpha1)) / sqr (alphahat-alpha1); alphahat = alpha1 - 0.5 * phi1prime / c; if (alphahat > alpha2) alphahat = alpha1 + 1/(4*c) * ( (sigma+mu1) * phi0prime - 2*phi1prime + sqrt (sqr(phi1prime - mu1 * phi0prime) - 4 * (phi1 - phi0 - mu1 * alpha1 * phi0prime) * c)); alphahat = max2 (alphahat, alpha1 + tau * (alpha2 - alpha1)); alphahat = min2 (alphahat, alpha2 - tau * (alpha2 - alpha1)); // (*testout) << " if-branch" << endl; } else { /* f = fun.FuncGrad (xneu, g); phihatprime = g * p; */ f = fun.FuncDeriv (xneu, p, phihatprime); if (phihatprime < sigma * phi0prime * (1 + eps0)) { if (phi1prime < phihatprime) // Approximationsfunktion ist konvex alphaincr = (alphahat - alpha1) * phihatprime / (phi1prime - phihatprime); else alphaincr = 1e99; // MAXDOUBLE; if (flag) { alphaincr = max2 (alphaincr, xi1 * (alphahat-alpha1)); alphaincr = min2 (alphaincr, xi2 * (alphahat-alpha1)); } else { alphaincr = max2 (alphaincr, tau1 * (alpha2 - alphahat)); alphaincr = min2 (alphaincr, tau2 * (alpha2 - alphahat)); } alpha1 = alphahat; alphahat += alphaincr; phi1 = f; phi1prime = phihatprime; } else { ifail = 0; // Erfolg !! break; } // (*testout) << " else, " << endl; } } // (*testout) << "linsearch: it = " << it << " ifail = " << ifail << endl; fun.FuncGrad (xneu, g); if (it < 0) ifail = 1; // (*testout) << "fail = " << ifail << endl; }
double BFGS ( Vector & x, // i: Startwert // o: Loesung, falls IFAIL = 0 const MinFunction & fun, const OptiParameters & par, double eps ) { int n = x.Size(); long it; char a1crit, a3acrit; Vector d(n), g(n), p(n), temp(n), bs(n), xneu(n), y(n), s(n), x0(n); DenseMatrix l(n); DenseMatrix hesse(n); double /* normg, */ alphahat, hd, fold; double a1, a2; const double mu1 = 0.1, sigma = 0.1, xi1 = 1, xi2 = 10; const double tau = 0.1, tau1 = 0.1, tau2 = 0.6; Vector typx(x.Size()); // i: typische Groessenordnung der Komponenten double f, f0; double typf; // i: typische Groessenordnung der Loesung double fmin = -1e5; // i: untere Schranke fuer Funktionswert // double eps = 1e-8; // i: Abbruchschranke fuer relativen Gradienten double tauf = 0.1; // i: Abbruchschranke fuer die relative Aenderung der // Funktionswerte int ifail; // o: 0 .. Erfolg // -1 .. Unterschreitung von fmin // 1 .. kein Erfolg bei Liniensuche // 2 .. Überschreitung von itmax typx = par.typx; typf = par.typf; l = 0; for (int i = 1; i <= n; i++) l.Elem(i, i) = 1; f = fun.FuncGrad (x, g); f0 = f; x0 = x; it = 0; do { // Restart if (it % (5 * n) == 0) { for (int i = 1; i <= n; i++) d(i-1) = typf/ sqr (typx(i-1)); // 1; for (int i = 2; i <= n; i++) for (int j = 1; j < i; j++) l.Elem(i, j) = 0; } it++; if (it > par.maxit_bfgs) { ifail = 2; break; } // Solve with factorized B SolveLDLt (l, d, g, p); p *= -1; y = g; fold = f; // line search alphahat = 1; lines (x, xneu, p, f, g, fun, par, alphahat, fmin, mu1, sigma, xi1, xi2, tau, tau1, tau2, ifail); if(ifail == 1) cout << "no success with linesearch" << endl; s.Set2 (1, xneu, -1, x); y *= -1; y.Add (1,g); // y += g; x = xneu; // BFGS Update MultLDLt (l, d, s, bs); a1 = y * s; a2 = s * bs; if (a1 > 0 && a2 > 0) { if (LDLtUpdate (l, d, 1 / a1, y) != 0) { cerr << "BFGS update error1" << endl; cout << "BFGS update error1" << endl; cout << "l " << endl << l << endl << "d " << d << endl; ifail = 1; break; } if (LDLtUpdate (l, d, -1 / a2, bs) != 0) { cerr << "BFGS update error2" << endl; cout << "BFGS update error2" << endl; cout << "l " << endl << l << endl << "d " << d << endl; ifail = 1; break; } } // Calculate stop conditions hd = eps * max2 (typf, fabs (f)); a1crit = 1; for (int i = 1; i <= n; i++) if ( fabs (g(i-1)) * max2 (typx(i-1), fabs (x(i-1))) > hd) a1crit = 0; a3acrit = (fold - f <= tauf * max2 (typf, fabs (f))); if (g.L2Norm() < fun.GradStopping (x)) break; } while (!a1crit || !a3acrit); if (f0 < f || (ifail == 1)) { cout << "fail, f = " << f << " f0 = " << f0 << endl; f = f0; x = x0; } return f; }