/************************************************************************* N-dimensional multiclass Fisher LDA Subroutine finds coefficients of linear combinations which optimally separates training set on classes. It returns N-dimensional basis whose vector are sorted by quality of training set separation (in descending order). INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars]. First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=0 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -4, if internal EVD subroutine hasn't converged * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, NVars<1, NClasses<2) * 1, if task has been solved * 2, if there was a multicollinearity in training set, but task has been solved. W - basis, array[0..NVars-1,0..NVars-1] columns of matrix stores basis vectors, sorted by quality of training set separation (in descending order) -- ALGLIB -- Copyright 31.05.2008 by Bochkanov Sergey *************************************************************************/ void fisherldan(const ap::real_2d_array& xy, int npoints, int nvars, int nclasses, int& info, ap::real_2d_array& w) { int i; int j; int k; int m; double v; ap::integer_1d_array c; ap::real_1d_array mu; ap::real_2d_array muc; ap::integer_1d_array nc; ap::real_2d_array sw; ap::real_2d_array st; ap::real_2d_array z; ap::real_2d_array z2; ap::real_2d_array tm; ap::real_2d_array sbroot; ap::real_2d_array a; ap::real_2d_array xyproj; ap::real_2d_array wproj; ap::real_1d_array tf; ap::real_1d_array d; ap::real_1d_array d2; ap::real_1d_array work; // // Test data // if( npoints<0||nvars<1||nclasses<2 ) { info = -1; return; } for(i = 0; i <= npoints-1; i++) { if( ap::round(xy(i,nvars))<0||ap::round(xy(i,nvars))>=nclasses ) { info = -2; return; } } info = 1; // // Special case: NPoints<=1 // Degenerate task. // if( npoints<=1 ) { info = 2; w.setbounds(0, nvars-1, 0, nvars-1); for(i = 0; i <= nvars-1; i++) { for(j = 0; j <= nvars-1; j++) { if( i==j ) { w(i,j) = 1; } else { w(i,j) = 0; } } } return; } // // Prepare temporaries // tf.setbounds(0, nvars-1); work.setbounds(1, ap::maxint(nvars, npoints)); // // Convert class labels from reals to integers (just for convenience) // c.setbounds(0, npoints-1); for(i = 0; i <= npoints-1; i++) { c(i) = ap::round(xy(i,nvars)); } // // Calculate class sizes and means // mu.setbounds(0, nvars-1); muc.setbounds(0, nclasses-1, 0, nvars-1); nc.setbounds(0, nclasses-1); for(j = 0; j <= nvars-1; j++) { mu(j) = 0; } for(i = 0; i <= nclasses-1; i++) { nc(i) = 0; for(j = 0; j <= nvars-1; j++) { muc(i,j) = 0; } } for(i = 0; i <= npoints-1; i++) { ap::vadd(&mu(0), 1, &xy(i, 0), 1, ap::vlen(0,nvars-1)); ap::vadd(&muc(c(i), 0), 1, &xy(i, 0), 1, ap::vlen(0,nvars-1)); nc(c(i)) = nc(c(i))+1; } for(i = 0; i <= nclasses-1; i++) { v = double(1)/double(nc(i)); ap::vmul(&muc(i, 0), 1, ap::vlen(0,nvars-1), v); } v = double(1)/double(npoints); ap::vmul(&mu(0), 1, ap::vlen(0,nvars-1), v); // // Create ST matrix // st.setbounds(0, nvars-1, 0, nvars-1); for(i = 0; i <= nvars-1; i++) { for(j = 0; j <= nvars-1; j++) { st(i,j) = 0; } } for(k = 0; k <= npoints-1; k++) { ap::vmove(&tf(0), 1, &xy(k, 0), 1, ap::vlen(0,nvars-1)); ap::vsub(&tf(0), 1, &mu(0), 1, ap::vlen(0,nvars-1)); for(i = 0; i <= nvars-1; i++) { v = tf(i); ap::vadd(&st(i, 0), 1, &tf(0), 1, ap::vlen(0,nvars-1), v); } } // // Create SW matrix // sw.setbounds(0, nvars-1, 0, nvars-1); for(i = 0; i <= nvars-1; i++) { for(j = 0; j <= nvars-1; j++) { sw(i,j) = 0; } } for(k = 0; k <= npoints-1; k++) { ap::vmove(&tf(0), 1, &xy(k, 0), 1, ap::vlen(0,nvars-1)); ap::vsub(&tf(0), 1, &muc(c(k), 0), 1, ap::vlen(0,nvars-1)); for(i = 0; i <= nvars-1; i++) { v = tf(i); ap::vadd(&sw(i, 0), 1, &tf(0), 1, ap::vlen(0,nvars-1), v); } } // // Maximize ratio J=(w'*ST*w)/(w'*SW*w). // // First, make transition from w to v such that w'*ST*w becomes v'*v: // v = root(ST)*w = R*w // R = root(D)*Z' // w = (root(ST)^-1)*v = RI*v // RI = Z*inv(root(D)) // J = (v'*v)/(v'*(RI'*SW*RI)*v) // ST = Z*D*Z' // // so we have // // J = (v'*v) / (v'*(inv(root(D))*Z'*SW*Z*inv(root(D)))*v) = // = (v'*v) / (v'*A*v) // if( !smatrixevd(st, nvars, 1, true, d, z) ) { info = -4; return; } w.setbounds(0, nvars-1, 0, nvars-1); if( ap::fp_less_eq(d(nvars-1),0)||ap::fp_less_eq(d(0),1000*ap::machineepsilon*d(nvars-1)) ) { // // Special case: D[NVars-1]<=0 // Degenerate task (all variables takes the same value). // if( ap::fp_less_eq(d(nvars-1),0) ) { info = 2; for(i = 0; i <= nvars-1; i++) { for(j = 0; j <= nvars-1; j++) { if( i==j ) { w(i,j) = 1; } else { w(i,j) = 0; } } } return; } // // Special case: degenerate ST matrix, multicollinearity found. // Since we know ST eigenvalues/vectors we can translate task to // non-degenerate form. // // Let WG is orthogonal basis of the non zero variance subspace // of the ST and let WZ is orthogonal basis of the zero variance // subspace. // // Projection on WG allows us to use LDA on reduced M-dimensional // subspace, N-M vectors of WZ allows us to update reduced LDA // factors to full N-dimensional subspace. // m = 0; for(k = 0; k <= nvars-1; k++) { if( ap::fp_less_eq(d(k),1000*ap::machineepsilon*d(nvars-1)) ) { m = k+1; } } ap::ap_error::make_assertion(m!=0, "FisherLDAN: internal error #1"); xyproj.setbounds(0, npoints-1, 0, nvars-m); matrixmatrixmultiply(xy, 0, npoints-1, 0, nvars-1, false, z, 0, nvars-1, m, nvars-1, false, 1.0, xyproj, 0, npoints-1, 0, nvars-m-1, 0.0, work); for(i = 0; i <= npoints-1; i++) { xyproj(i,nvars-m) = xy(i,nvars); } fisherldan(xyproj, npoints, nvars-m, nclasses, info, wproj); if( info<0 ) { return; } matrixmatrixmultiply(z, 0, nvars-1, m, nvars-1, false, wproj, 0, nvars-m-1, 0, nvars-m-1, false, 1.0, w, 0, nvars-1, 0, nvars-m-1, 0.0, work); for(k = nvars-m; k <= nvars-1; k++) { ap::vmove(&w(0, k), w.getstride(), &z(0, k-(nvars-m)), z.getstride(), ap::vlen(0,nvars-1)); } info = 2; } else { // // General case: no multicollinearity // tm.setbounds(0, nvars-1, 0, nvars-1); a.setbounds(0, nvars-1, 0, nvars-1); matrixmatrixmultiply(sw, 0, nvars-1, 0, nvars-1, false, z, 0, nvars-1, 0, nvars-1, false, 1.0, tm, 0, nvars-1, 0, nvars-1, 0.0, work); matrixmatrixmultiply(z, 0, nvars-1, 0, nvars-1, true, tm, 0, nvars-1, 0, nvars-1, false, 1.0, a, 0, nvars-1, 0, nvars-1, 0.0, work); for(i = 0; i <= nvars-1; i++) { for(j = 0; j <= nvars-1; j++) { a(i,j) = a(i,j)/sqrt(d(i)*d(j)); } } if( !smatrixevd(a, nvars, 1, true, d2, z2) ) { info = -4; return; } for(k = 0; k <= nvars-1; k++) { for(i = 0; i <= nvars-1; i++) { tf(i) = z2(i,k)/sqrt(d(i)); } for(i = 0; i <= nvars-1; i++) { v = ap::vdotproduct(&z(i, 0), 1, &tf(0), 1, ap::vlen(0,nvars-1)); w(i,k) = v; } } } // // Post-processing: // * normalization // * converting to non-negative form, if possible // for(k = 0; k <= nvars-1; k++) { v = ap::vdotproduct(&w(0, k), w.getstride(), &w(0, k), w.getstride(), ap::vlen(0,nvars-1)); v = 1/sqrt(v); ap::vmul(&w(0, k), w.getstride(), ap::vlen(0,nvars-1), v); v = 0; for(i = 0; i <= nvars-1; i++) { v = v+w(i,k); } if( ap::fp_less(v,0) ) { ap::vmul(&w(0, k), w.getstride(), ap::vlen(0,nvars-1), -1); } } }
/************************************************************************* Algorithm for solving the following generalized symmetric positive-definite eigenproblem: A*x = lambda*B*x (1) or A*B*x = lambda*x (2) or B*A*x = lambda*x (3). where A is a symmetric matrix, B - symmetric positive-definite matrix. The problem is solved by reducing it to an ordinary symmetric eigenvalue problem. Input parameters: A - symmetric matrix which is given by its upper or lower triangular part. Array whose indexes range within [0..N-1, 0..N-1]. N - size of matrices A and B. IsUpperA - storage format of matrix A. B - symmetric positive-definite matrix which is given by its upper or lower triangular part. Array whose indexes range within [0..N-1, 0..N-1]. IsUpperB - storage format of matrix B. ZNeeded - if ZNeeded is equal to: * 0, the eigenvectors are not returned; * 1, the eigenvectors are returned. ProblemType - if ProblemType is equal to: * 1, the following problem is solved: A*x = lambda*B*x; * 2, the following problem is solved: A*B*x = lambda*x; * 3, the following problem is solved: B*A*x = lambda*x. Output parameters: D - eigenvalues in ascending order. Array whose index ranges within [0..N-1]. Z - if ZNeeded is equal to: * 0, Z hasn’t changed; * 1, Z contains eigenvectors. Array whose indexes range within [0..N-1, 0..N-1]. The eigenvectors are stored in matrix columns. It should be noted that the eigenvectors in such problems do not form an orthogonal system. Result: True, if the problem was solved successfully. False, if the error occurred during the Cholesky decomposition of matrix B (the matrix isn’t positive-definite) or during the work of the iterative algorithm for solving the symmetric eigenproblem. See also the GeneralizedSymmetricDefiniteEVDReduce subroutine. -- ALGLIB -- Copyright 1.28.2006 by Bochkanov Sergey *************************************************************************/ bool smatrixgevd(ap::real_2d_array a, int n, bool isuppera, const ap::real_2d_array& b, bool isupperb, int zneeded, int problemtype, ap::real_1d_array& d, ap::real_2d_array& z) { bool result; ap::real_2d_array r; ap::real_2d_array t; bool isupperr; int j1; int j2; int j1inc; int j2inc; int i; int j; double v; // // Reduce and solve // result = smatrixgevdreduce(a, n, isuppera, b, isupperb, problemtype, r, isupperr); if( !result ) { return result; } result = smatrixevd(a, n, zneeded, isuppera, d, t); if( !result ) { return result; } // // Transform eigenvectors if needed // if( zneeded!=0 ) { // // fill Z with zeros // z.setbounds(0, n-1, 0, n-1); for(j = 0; j <= n-1; j++) { z(0,j) = 0.0; } for(i = 1; i <= n-1; i++) { ap::vmove(&z(i, 0), &z(0, 0), ap::vlen(0,n-1)); } // // Setup R properties // if( isupperr ) { j1 = 0; j2 = n-1; j1inc = +1; j2inc = 0; } else { j1 = 0; j2 = 0; j1inc = 0; j2inc = +1; } // // Calculate R*Z // for(i = 0; i <= n-1; i++) { for(j = j1; j <= j2; j++) { v = r(i,j); ap::vadd(&z(i, 0), &t(j, 0), ap::vlen(0,n-1), v); } j1 = j1+j1inc; j2 = j2+j2inc; } } return result; }
/************************************************************************* Tests EVD problem *************************************************************************/ static void testevdproblem(const ap::real_2d_array& a, const ap::real_2d_array& al, const ap::real_2d_array& au, int n, double& materr, double& valerr, double& orterr, bool& wnsorted, int& failc) { ap::real_1d_array lambda; ap::real_1d_array lambdaref; ap::real_2d_array z; bool wsucc; int i; int j; double v; // // Test simple EVD: values and full vectors, lower A // unset1d(lambdaref); unset2d(z); wsucc = smatrixevd(al, n, 1, false, lambdaref, z); if( !wsucc ) { failc = failc+1; return; } materr = ap::maxreal(materr, testproduct(a, n, z, lambdaref)); orterr = ap::maxreal(orterr, testort(z, n)); for(i = 0; i <= n-2; i++) { if( lambdaref(i+1)<lambdaref(i) ) { wnsorted = true; } } // // Test simple EVD: values and full vectors, upper A // unset1d(lambda); unset2d(z); wsucc = smatrixevd(au, n, 1, true, lambda, z); if( !wsucc ) { failc = failc+1; return; } materr = ap::maxreal(materr, testproduct(a, n, z, lambda)); orterr = ap::maxreal(orterr, testort(z, n)); for(i = 0; i <= n-2; i++) { if( lambda(i+1)<lambda(i) ) { wnsorted = true; } } // // Test simple EVD: values only, lower A // unset1d(lambda); unset2d(z); wsucc = smatrixevd(al, n, 0, false, lambda, z); if( !wsucc ) { failc = failc+1; return; } for(i = 0; i <= n-1; i++) { valerr = ap::maxreal(valerr, fabs(lambda(i)-lambdaref(i))); } // // Test simple EVD: values only, upper A // unset1d(lambda); unset2d(z); wsucc = smatrixevd(au, n, 0, true, lambda, z); if( !wsucc ) { failc = failc+1; return; } for(i = 0; i <= n-1; i++) { valerr = ap::maxreal(valerr, fabs(lambda(i)-lambdaref(i))); } }