// Perform exact (and slower) NR test using the exact fisher information matrix.
vector<double> LogisticRegression::newtonRaphson(const vector<vector<double> > &data, const vector<double> &response, vector<vector<double> > &invInfMatrix, double startVal)
{
vector<double> betas;
// Variables used in the computation:
alglib::real_1d_array tempBetas;
alglib::real_2d_array tempData;
alglib::real_2d_array tempDataTrans;
alglib::real_1d_array oldExpY;
alglib::real_2d_array hessian; // holds data' * w * data. Returned in last input param.
alglib::real_1d_array expY;
alglib::real_1d_array W; // holds diagonal of the w matrix above.
alglib::real_1d_array adjy;
alglib::real_1d_array work;
double stop_var = 1e-10;
int iter = 0;
int maxIter = 200;
int numVars = data.size();
if(numVars < 1){
throw NewtonRaphsonFailureEx();
}
int numSamples = data.at(0).size();
if(numSamples < 1){
throw NewtonRaphsonFailureEx();
}
tempBetas.setlength(numVars);
tempData.setlength(numSamples,numVars);
tempDataTrans.setlength(numVars, numSamples);
oldExpY.setlength(numSamples);
expY.setlength(numSamples);
hessian.setlength(numVars, numVars);
adjy.setlength(numSamples);
W.setlength(numSamples);
work.setlength(numVars);
for(int i=0;i < numVars; i++){
for(int j=0;j < numSamples; j++){
tempData(j,i) = data.at(i).at(j);
}
tempBetas(i) = startVal;
}
for(int i=0;i < numSamples;i++){
oldExpY(i) = -1;
adjy(i) = 0; // makes valgrind happier.
}
// End initial setup.
// In each iteration, create a hessian and a first derivative.
while(iter < maxIter){
//adjy <- data * tempBetas (get new y guess)
matrixvectormultiply(tempData, 0, numSamples-1,0,numVars-1,false,
tempBetas, 0, numVars-1, 1.0,
adjy, 0, numSamples-1, 0.0);
// adjy = 1 / (1 + exp(-adjy))
for(int i=0;i < numSamples; i++){
expY(i) = 1 / (1 + exp(-adjy(i)));
}
// build deriv.
for(int i=0;i < numSamples; i++){
W(i) = expY(i) * (1 - expY(i));
}
// adjy = adjy + (y-expy) ./ deriv
for(int i=0;i<numSamples;i++){
adjy(i) = adjy(i) + (response.at(i) - expY(i)) / W(i);
}
// build data' * w * data
// set to hessian.
// Also doing secondary computation (see inside)
for(int i=0; i < numVars; i++){
for(int j=0; j < numVars; j++)
hessian(i,j) = 0.0;
}
for(int indiv=0; indiv < numSamples; ++indiv){
for(int i = 0; i < numVars; i++){
for(int j = 0; j < numVars; j++){
hessian(i,j) += W(indiv) * tempData(indiv, j) * tempData(indiv, i);
}
// NOTE: as a speedup, I'm also computing X' * W
tempDataTrans(i, indiv) = W(indiv) * tempData(indiv, i);
}
}
alglib::matinvreport report;
alglib::ae_int_t reportInfo;
rmatrixinverse(hessian, reportInfo, report);
if(reportInfo != 1 ){
throw SingularMatrixEx();
}
// Check condition number.
if (report.r1 < condition_number_limit){
throw ConditionNumberEx(1.0/report.r1);
}
// work <- X'W * adjy
matrixvectormultiply(tempDataTrans,0,numVars-1,0,numSamples-1,false,
adjy,0,numSamples-1,1,
work,0,numVars-1,0.0);
// tempBetas <= invHessian * work
matrixvectormultiply(hessian,0,numVars-1,0,numVars-1,false,
work,0,numVars-1,1.0,
tempBetas,0,numVars-1,0.0);
#if DEBUG_NR
cout << "Betas ";
for(int i=0;i < numVars;i++) cout << tempBetas(i) << " " ;
cout << endl;
#endif
double stop = 0.0;
// Could be computed as a 1-norm.
// This should be done as a sum of abs diff.
for(int i=0;i < numSamples;i++){
stop += abs(expY(i) - oldExpY(i));
}
if (stop < numSamples*stop_var){
break;
}
oldExpY = expY;
iter++;
}
if(iter == maxIter){
throw NewtonRaphsonIterationEx();
}
betas.clear();
for(int i=0;i<numVars;i++){
betas.push_back(tempBetas(i));
}
for(int i=0;i<numVars;i++){
for(int j=0;j<numVars;j++){
invInfMatrix.at(i).at(j) = hessian(i,j);
}
}
//dumpMatrix(invInfMatrix);
return betas;
}
/* If we're about to run off the matrix, adjust accordingly (if possible) */
adjust()
{
adjx();
adjy();
}
/*
* @depricated
*
* NR implementation by RTG.
*
* Note on matrix vector multiplication:
* void matrixmatrixmultiply(const alglib::real_2d_array& a,
* int ai1,
* int ai2,
* int aj1,
* int aj2,
* bool transa,
* const alglib::real_2d_array& b,
* int bi1,
* int bi2,
* int bj1,
* int bj2,
* bool transb,
* double alpha,
* alglib::real_2d_array& c,
* int ci1,
* int ci2,
* int cj1,
* int cj2,
* double beta,
* alglib::real_1d_array& work);
*
* transa is true if transposed.
* Operation is: c = A * alpha * b + beta * C
*
*
* @input const vector<vector<double>> data holds explantory variables. each inner vector is a variable.
* @input const vector<double> response holds response variable.
* @input vector<double> Place to return the information matrix inverse. Column major order.
* Expects correct size.
*/
vector<double> LogisticRegression::newtonRaphsonFast(const vector<vector<double> > &data, const vector<double> &response
, vector<vector<double> > &invInfMatrix, double startVal)
{
vector<double> betas;
// Variables used in the computation:
alglib::real_1d_array tempBetas;
alglib::real_2d_array tempData;
alglib::real_1d_array oldExpY;
alglib::real_2d_array tempDeriv; // holds data' * w * data. Returned in last input param.
alglib::real_1d_array expY;
alglib::real_1d_array adjy;
double stop_var = 1e-10;
int iter = 0;
int maxIter = 200;
int numVars = data.size();
if(numVars < 1){
throw NewtonRaphsonFailureEx();
}
int numSamples = data.at(0).size();
if(numSamples < 1){
throw NewtonRaphsonFailureEx();
}
tempBetas.setlength(numVars);
tempData.setlength(numSamples,numVars);
oldExpY.setlength(numSamples);
expY.setlength(numSamples);
tempDeriv.setlength(numVars,numVars);
adjy.setlength(numSamples);
for(int i=0;i < numVars; i++){
for(int j=0;j < numSamples; j++){
tempData(j,i) = data.at(i).at(j);
}
tempBetas(i) = startVal;
}
for(int i=0;i < numSamples;i++){
oldExpY(i) = -1;
adjy(i) = 0; // makes valgrind happier.
}
while(iter < maxIter){
//data * tempBetas
matrixvectormultiply(tempData, 0, numSamples-1,0,numVars-1,false,
tempBetas, 0, numVars-1, 1.0,
adjy, 0, numSamples-1, 0.0);
for(int i=0;i < numSamples; i++){
expY(i) = 1 / (1 + exp(-adjy(i)));
}
// build deriv.
double deriv = -100000;
for(int i=0;i < numSamples; i++){
if(expY(i) * (1 - expY(i)) > deriv)
deriv = expY(i) * (1 - expY(i));
}
if(stop_var * 0.001 > deriv) deriv = stop_var * 0.001;
// adjy = adjy + (y-expy) ./ deriv
for(int i=0;i<numSamples;i++){
adjy(i) = adjy(i) + (response.at(i) - expY(i)) / deriv;
}
// build data' * w * data
alglib::real_1d_array work;
work.setlength(numSamples); // This temporary workspace must be one larger than the longest
// row or column that will be seen. Otherwise, the program
// crashes or - worse! - corrupts data.
matrixmatrixmultiply(tempData,0,numSamples-1,0,numVars-1,true,
tempData,0,numSamples-1,0,numVars-1,false,deriv,
tempDeriv,0,numVars-1,0,numVars-1,0.0,work);
#if DEBUG_NR
cout << "A' * w * A " << endl;
cout << tempDeriv(0,0) << " " << tempDeriv(0,1) << endl;
cout << tempDeriv(1,0) << " " << tempDeriv(1,1) << endl;
cout << endl;
#endif
alglib::matinvreport report;
alglib::ae_int_t reportInfo;
rmatrixinverse(tempDeriv, reportInfo, report);
if( reportInfo != 1 ){
throw SingularMatrixEx();
}
#if DEBUG_NR
cout << "inv(A' * w * A) " << endl;
cout << tempDeriv(0,0) << " " << tempDeriv(0,1) << endl;
cout << tempDeriv(1,0) << " " << tempDeriv(1,1) << endl;
cout << endl;
#endif
matrixvectormultiply(tempData,0,numSamples-1,0,numVars-1,true,
adjy,0,numSamples-1,deriv,
work,0,numVars-1,0.0);
matrixvectormultiply(tempDeriv,0,numVars-1,0,numVars-1,false,
work,0,numVars-1,1.0,
tempBetas,0,numVars-1,0.0);
#if DEBUG_NR
cout << "Betas ";
for(int i=0;i < numVars;i++) cout << tempBetas(i) << " " ;
cout << endl;
#endif
double stop = 0.0;
for(int i=0;i < numSamples;i++){
stop += abs(expY(i) - oldExpY(i));
}
if (stop < numSamples*stop_var){
break;
}
oldExpY = expY;
iter++;
}
if(iter == maxIter){
throw NewtonRaphsonIterationEx();
}
betas.clear();
for(int i=0;i<numVars;i++){
betas.push_back(tempBetas(i));
}
for(int i=0;i<numVars;i++){
for(int j=0;j<numVars;j++){
invInfMatrix.at(i).at(j) = tempDeriv(i,j);
}
}
return betas;
}
