int KinshipHolder::loadDecomposed() {
LineReader lr(this->eigenFileName);
int lineNo = 0;
int fieldLen = 0;
std::vector<std::string> fd;
std::vector<int> columnToExtract;
std::vector<std::string> header; // header line of the kinship eigen file
Eigen::MatrixXf& matK = this->matK->mat;
Eigen::MatrixXf& matS = this->matS->mat;
Eigen::MatrixXf& matU = this->matU->mat;
const std::vector<std::string>& names = *this->pSample;
const int NumSample = (int)names.size();
std::map<std::string, int> nameMap;
makeMap(names, &nameMap);
std::map<std::string, int> headerMap;
while (lr.readLineBySep(&fd, "\t ")) {
++lineNo;
if (lineNo == 1) { // check header
header = fd;
fieldLen = fd.size();
if (fieldLen < 3) { // at least three columns: IID, Lambda, U1
logger->error(
"Insufficient column number (<3) in the first line of kinsihp "
"file!");
return -1;
};
for (size_t i = 0; i != fd.size(); ++i) {
fd[i] = tolower(fd[i]);
}
makeMap(fd, &headerMap);
if (fd.size() != headerMap.size()) {
logger->error("Kinship file have duplicated headers!");
return -1;
}
// check IID, Lambda, U1, U2, ... U(N) where (N) is the sample size
if (headerMap.count("iid") == 0) {
logger->error("Missing 'IID' column!");
return -1;
}
columnToExtract.push_back(headerMap["iid"]);
if (headerMap.count("lambda") == 0) {
logger->error("Missing 'Lambda' column!");
return -1;
}
columnToExtract.push_back(headerMap["lambda"]);
std::string s;
for (int i = 0; i < NumSample; ++i) {
s = "u";
s += toString(i + 1);
if (headerMap.count(s) == 0) {
logger->error("Missing '%s' column!", s.c_str());
return -1;
}
columnToExtract.push_back(headerMap[s]);
}
s = "u";
s += toString(NumSample + 1);
if (headerMap.count(s) != 0) {
logger->error("Unexpected column '%s'!", s.c_str());
return -1;
}
matS.resize(NumSample, 1);
matU.resize(NumSample, NumSample);
continue;
}
// body lines
if ((int)fd.size() != fieldLen) {
logger->error(
"Inconsistent column number [ %zu ] (used to be [ %d ])in kinship "
"file line [ %d ] - skip this file!",
fd.size(), fieldLen, lineNo);
return -1;
}
const int iidColumn = columnToExtract[0];
const std::string& iid = fd[iidColumn];
if (nameMap.count(iid) == 0) {
logger->error("Unexpected sample [ %s ]!", iid.c_str());
return -1;
}
const int row = nameMap[iid];
const int lambdaColumn = columnToExtract[1];
double temp = 0.0;
if (!str2double(fd[lambdaColumn], &temp)) {
logger->warn("Invalid numeric value [ %s ] treated as zero!",
fd[lambdaColumn].c_str());
}
matS(lineNo - 2, 0) = temp;
for (int i = 0; i < NumSample; ++i) {
int uColumn = columnToExtract[i + 2];
if (!str2double(fd[uColumn], &temp)) {
logger->warn("Invalid numeric value [ %s ] treated as zero!",
fd[lambdaColumn].c_str());
}
matU(row, i) = temp;
}
}
// verify eigen decomposition results make senses
// check largest eigen vector and eigen value
Eigen::MatrixXf v1 = matK * matU.col(0);
Eigen::MatrixXf v2 = matS(0, 0) * matU.col(0);
if (matS(0, 0) > 0.5 && v1.col(0).norm() > .5 && v2.col(0).norm() > 0.5 &&
corr(v1, v2) < 0.8) {
logger->warn("Cannot verify spectral decompose results!");
return -1;
}
// check the min(10, NumSample) random eigen vector and eigen value
int randomCol = 10;
if (randomCol > NumSample - 1) {
randomCol = NumSample - 1;
}
v1 = matK * matU.col(randomCol);
v2 = matS(randomCol, 0) * matU.col(randomCol);
if (matS(randomCol, 0) > 0.5 && v1.col(0).norm() > 0.5 &&
v2.col(0).norm() > 0.5 && corr(v1, v2) < 0.8) {
logger->warn("Cannot verify spectral decompose results!");
return -1;
}
#ifdef DEBUG
std::string tmp = fn;
tmp += ".tmp";
std::ofstream ofs(tmp.c_str(), std::ofstream::out);
ofs << mat;
ofs.close();
#endif
// fprintf(stderr, "Kinship matrix [ %d x %d ] loaded", (int)mat.rows(),
// (int)mat.cols());
if (this->matK) {
delete this->matK;
this->matK = NULL;
}
return 0;
}
void iterativeSolvers_unitTest() {
// small matrix test
{
Vector4f test = makeVector4f(1.0f, 3.14f, 2.0f, 2.71f);
Matrix4f mat = makeRotX4f(0.32f) * makeRotY4f(-0.39f) * makeRotZ4f(0.76f);
mat += mat.transpose();
mat += IDENTITY4F*5;
Vector4f rhs = mat * test;
Matrix4f invMat = invertMatrix(mat);
Vector4f directFloat = invMat * rhs;
DVectorD testD(4); for (card32 i=0; i<4; i++) testD[i] = test[i];
DVectorD rhsD(4); for (card32 i=0; i<4; i++) rhsD[i] = rhs[i];
SparseMatrixD matS(4);
for (card32 r=0; r<4; r++) {
for (card32 c=0; c<4; c++) {
double v = mat[c][r];
if (v != 0) {
matS[r].setEntry(c, v);
}
}
}
output << "Matrix:\n" << mat.toString() << "\n";
output << "true solution: " << test.toString() << "\n";
output << "right hand side: " << rhs.toString() << "\n";
output << "direct float prec. solution: " << directFloat.toString() << "\n\n";
output << "Matrix:\n" << matS.toString() << "\n";
output << "true solution: " << testD.toString() << "\n";
output << "right hand side: " << rhsD.toString() << "\n\n";
DVectorD x = nullDVector<double>(4);
card32 numIterations = 1000;
gaussSeidelSolve(matS, x, rhsD, numIterations, 1E-20, 1E-5, false);
output << "Gauss-Seidel solution: " << x.toString() << "\n\n";
numIterations = 1000;
x = nullDVector<double>(4);
steepestDescentSolve(matS, x, rhsD, numIterations, 1E-5, false);
output << "Steepest-Descent solution: " << x.toString() << "\n\n";
numIterations = 1000;
x = nullDVector<double>(4);
conjugateGradientsSolve(matS, x, rhsD, numIterations, 1E-5, true);
output << "CG solution: " << x.toString() << "\n\n";
}
// large matrix test
{
SparseMatrixD mat(MDIM*MDIM);
for (int y=0; y<MDIM; y++) {
for (int x=0; x<MDIM; x++) {
addEntry(mat, x,y, x,y, -4, MDIM);
addEntry(mat, x,y, x+1,y, 1, MDIM);
addEntry(mat, x,y, x-1,y, 1, MDIM);
addEntry(mat, x,y, x,y+1, 1, MDIM);
addEntry(mat, x,y, x,y-1, 1, MDIM);
}
}
Timer T; card32 vt;
DVectorD x = nullDVector<double>(MDIM*MDIM);
DVectorD b = scalarDVector<double>(MDIM*MDIM, 1);
DVectorD gs, sd, cg;
x = nullDVector<double>(MDIM*MDIM);
T.getDeltaValue();
card32 numIterations = 100000;
conjugateGradientsSolve(mat, x, b, numIterations, 1E-7, false);
vt = T.getDeltaValue();
output << "CG time: " << vt << "\n\n";
output << "Large sytem CG solution (LAPL(f) == 1):\n";
for (int y=0; y<MDIM; y++) {
for (int xp=0; xp<MDIM; xp++) {
output << x[xp +y*MDIM] << "\t";
}
output.cr();
}
output.cr();
cg = x;
x = nullDVector<double>(MDIM*MDIM);
T.getDeltaValue();
numIterations = 100000;
gaussSeidelSolve(mat, x, b, numIterations, 1E-20, 1E-7, false);
vt = T.getDeltaValue();
output << "GS time: " << vt << "\n\n";
output << "Large sytem Gauss-Seidel solution (LAPL(f) == 1):\n";
for (int y=0; y<MDIM; y++) {
for (int xp=0; xp<MDIM; xp++) {
output << x[xp +y*MDIM] << "\t";
}
output.cr();
}
output.cr();
gs = x;
x = nullDVector<double>(MDIM*MDIM);
T.getDeltaValue();
numIterations = 100000;
steepestDescentSolve(mat, x, b, numIterations, 1E-7, false);
vt = T.getDeltaValue();
output << "SD time: " << vt << "\n\n";
output << "Large sytem steepest descent solution (LAPL(f) == 1):\n";
for (int y=0; y<MDIM; y++) {
for (int xp=0; xp<MDIM; xp++) {
output << (long double)x[xp +y*MDIM] << "\t";
}
output.cr();
}
output.cr();
sd = x;
x = nullDVector<double>(MDIM*MDIM);
output << "|gs-sd|_2 " << (long double)norm(gs-sd) << "\n";
output << "|gs-cg|_2 " << (long double)norm(gs-cg) << "\n";
vector< DVectorD > eigenVects;
vector< double > eigenvalues;
DVectorD eigenvector;
double eigenvalue;
for (int i=0; i<10; i++) {
powerIteration(mat, eigenVects, eigenvector, eigenvalue, 1.0001, 1000, false);
eigenVects.push_back(eigenvector);
eigenvalues.push_back(eigenvalue);
}
for (int i=0; i<10; i++) {
output << i << " largest of large sytem eigenvector (LAPL(f) == 1):\n";
for (int y=0; y<MDIM; y++) {
for (int xp=0; xp<MDIM; xp++) {
output << eigenVects[i][xp +y*MDIM] << "\t";
}
output.cr();
}
output.cr();
}
output << "eigenvalue:\n";
for (int i=0; i<10; i++) {
output << eigenvalues[i] << "\n";
}
T.getDeltaValue();
eigenVects.clear();
for (int i=0; i<10; i++) {
GaussSeidelSolver<double> linSolve;
inversePowerIteration(mat, eigenVects, eigenvector, eigenvalue, &linSolve, 1.0001, 1000, false);
eigenvalues.push_back(eigenvalue);
eigenVects.push_back(eigenvector);
}
output << "GS-PowerIt time: " << T.getDeltaValue() << "\n\n";
eigenVects.clear();
for (int i=0; i<10; i++) {
CGSolver<double> linSolve;
inversePowerIteration(mat, eigenVects, eigenvector, eigenvalue, &linSolve, 1.0001, 1000, false);
eigenvalues.push_back(eigenvalue);
eigenVects.push_back(eigenvector);
}
output << "CG-PowerIt time: " << T.getDeltaValue() << "\n\n";
eigenVects.clear();
for (int i=0; i<10; i++) {
SteepestDescentSolver<double> linSolve;
inversePowerIteration(mat, eigenVects, eigenvector, eigenvalue, &linSolve, 1.0001, 1000, false);
eigenvalues.push_back(eigenvalue);
eigenVects.push_back(eigenvector);
}
output << "Steepest descent-PowerIt time: " << T.getDeltaValue() << "\n\n";
for (int i=0; i<10; i++) {
output << i << " smallest of large sytem eigenvector (LAPL(f) == 1):\n";
for (int y=0; y<MDIM; y++) {
for (int xp=0; xp<MDIM; xp++) {
output << eigenVects[i][xp +y*MDIM] << "\t";
}
output.cr();
}
output.cr();
}
output << "eigenvalues:\n";
for (int i=0; i<10; i++) {
output << eigenvalues[i] << "\n";
}
}
{
Timer T;
output << "cg scaling test\n";
for (unsigned i=1; i<21; i++) {
const int size = i * 10;
SparseMatrixD mat(size*size);
for (int y=0; y<size; y++) {
for (int x=0; x<size; x++) {
addEntry(mat, x,y, x,y, -4, size);
addEntry(mat, x,y, x+1,y, 1, size);
addEntry(mat, x,y, x-1,y, 1, size);
addEntry(mat, x,y, x,y+1, 1, size);
addEntry(mat, x,y, x,y-1, 1, size);
}
}
Timer T; card32 vt;
DVectorD x = nullDVector<double>(size*size);
DVectorD b = scalarDVector<double>(size*size, 1);
T.getDeltaValue();
card32 numIterations = 100000;
conjugateGradientsSolve(mat, x, b, numIterations, 1E-7, false);
vt = T.getDeltaValue();
output << i << "\t" << vt << "\n";
}
}
{
Timer T;
output << "gs scaling test\n";
for (unsigned i=1; i<21; i++) {
const int size = i * 10;
SparseMatrixD mat(size*size);
for (int y=0; y<size; y++) {
for (int x=0; x<size; x++) {
addEntry(mat, x,y, x,y, -4, size);
addEntry(mat, x,y, x+1,y, 1, size);
addEntry(mat, x,y, x-1,y, 1, size);
addEntry(mat, x,y, x,y+1, 1, size);
addEntry(mat, x,y, x,y-1, 1, size);
}
}
Timer T; card32 vt;
DVectorD x = nullDVector<double>(size*size);
DVectorD b = scalarDVector<double>(size*size, 1);
T.getDeltaValue();
card32 numIterations = 100000;
gaussSeidelSolve(mat, x, b, numIterations, 1E-20, 1E-7, false);
vt = T.getDeltaValue();
output << i << "\t" << vt << "\n";
}
}
{
Timer T;
output << "sd scaling test\n";
for (unsigned i=1; i<21; i++) {
const int size = i * 10;
SparseMatrixD mat(size*size);
for (int y=0; y<size; y++) {
for (int x=0; x<size; x++) {
addEntry(mat, x,y, x,y, -4, size);
addEntry(mat, x,y, x+1,y, 1, size);
addEntry(mat, x,y, x-1,y, 1, size);
addEntry(mat, x,y, x,y+1, 1, size);
addEntry(mat, x,y, x,y-1, 1, size);
}
}
Timer T; card32 vt;
DVectorD x = nullDVector<double>(size*size);
DVectorD b = scalarDVector<double>(size*size, 1);
T.getDeltaValue();
card32 numIterations = 100000;
steepestDescentSolve(mat, x, b, numIterations, 1E-7, false);
vt = T.getDeltaValue();
output << i << "\t" << vt << "\n";
}
}
}
