bool FirthRegression::FitFirthModel(Matrix & X, Vector & y, int nrrounds)
{
this-> Reset(X);
G_to_Eigen(X, &this->w->X);
G_to_Eigen(y, &this->w->y);
int rounds = 0;
// double lastDeviance, currentDeviance;
Eigen::MatrixXf xw; // W^(1/2) * X
// Newton-Raphson
while (rounds < nrrounds) {
// std::cout << "beta = " << this->w->beta << "\n";
this->w->eta = this->w->X * this->w->beta;
this->w->p = (1.0 + (-this->w->eta.array()).exp()).inverse();
this->w->V = this->w->p.array() * (1.0 - this->w->p.array());
xw = (this->w->V.array().sqrt().matrix().asDiagonal() * this->w->X).eval(); // W^(1/2) * X
this->w->D = xw.transpose() * xw; // X' V X
this->w->covB = this->w->D.eval().ldlt().solve(Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows()));
// double rel = ((this->w->D * this->w->covB).array() - Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows()).array()).matrix().norm() / this->w->D.rows() / this->w->D.rows();
// // printf("norm = %g\n", rel);
// if (rel > 1e-6) { // use relative accuracy to evalute convergence
if ((this->w->D * this->w->covB - Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows())).norm() > 1e-3) {
// cannot inverse
// printToFile(this->w->D, "matD", "D");
// printToFile(this->w->covB, "matCovB", "B");
return false;
}
this->w->h = (xw * this->w->covB * xw.transpose()).diagonal();
this->w->r = this->w->X.transpose() * (this->w->y - this->w->p + (this->w->h.array() * (0.5 - this->w->p.array())).matrix()); // X' (y-mu)
this->w->delta_beta = this->w->covB * this->w->r;
this->w->beta += this->w->delta_beta;
// printf("norm = %g\n", this->w->delta_beta.norm());
// use relative accuracy to evalute convergence
if (rounds > 1 && (this->w->beta.norm() > 0 && this->w->delta_beta.norm() / this->w->beta.norm() < 1e-6)) {
rounds = 0;
break;
}
rounds ++;
}
if (rounds == nrrounds)
{
printf("Not enough iterations!");
return false;
}
// this->w->covB = this->w->D.eval().llt().solve(Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows()));
Eigen_to_G(this->w->beta, &B);
Eigen_to_G(this->w->covB, &covB);
Eigen_to_G(this->w->p, &p);
Eigen_to_G(this->w->V, &V);
return true;
}
bool FirthRegression::FitFirthModel(Matrix & X, Vector & succ, Vector& total, int nrrounds) {
this-> Reset(X);
G_to_Eigen(X, &this->w->X);
G_to_Eigen(succ, &this->w->succ);
G_to_Eigen(total, &this->w->total);
int rounds = 0;
// double lastDeviance, currentDeviance;
Eigen::MatrixXf xw; // W^(1/2) * X
// Newton-Raphson
while (rounds < nrrounds) {
// beta = beta + solve( t(X)%*%diag(p*(1-p)) %*%X) %*% t(X) %*% (Y-p);
this->w->eta = this->w->X * this->w->beta;
this->w->p = (-this->w->eta.array().exp() + 1.0).inverse();
this->w->V = this->w->p.array() * (1.0 - this->w->p.array()) * this->w->total.array();
xw = (this->w->V.array().sqrt().matrix().asDiagonal() * this->w->X).eval();
this->w->D = xw.transpose() * xw; // this->w->X.transpose() * this->w->V.asDiagonal() * this->w->X; // X' V X
this->w->covB = this->w->D.eval().llt().solve(Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows()));
// double rel = ((this->w->D * this->w->covB).array() - Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows()).array()).matrix().norm() / this->w->D.rows() / this->w->D.rows();
// if (rel > 1e-6) { // use relative accuracy to evalute convergence
if ((this->w->D * this->w->covB - Eigen::MatrixXf::Identity(this->w->D.rows(), this->w->D.rows())).norm() > 1e-3) {
// cannot inverse
return false;
}
this->w->h = (xw.transpose() * this->w->covB * xw).diagonal();
this->w->r = this->w->X.transpose() * (this->w->succ.array() - this->w->total.array() * this->w->p.array()
+ this->w->total.array() * this->w->h.array() * (0.5 - this->w->p.array())).matrix();
this->w->delta_beta = this->w->covB * this->w->r;
this->w->beta += this->w->delta_beta;
if (rounds > 1 && (this->w->beta.norm() > 0 && this->w->delta_beta.norm() / this->w->beta.norm() < 1e-6)) {
rounds = 0;
break;
}
rounds ++;
}
if (rounds == nrrounds)
{
printf("Not enough iterations!");
return false;
}
Eigen_to_G(this->w->beta, &B);
Eigen_to_G(this->w->covB, &covB);
Eigen_to_G(this->w->p, &p);
Eigen_to_G(this->w->V, &V);
return true;
}
void CholeskyInverseMatrix(Matrix& in, Matrix* out) {
if (in.rows != in.cols) return;
const int n = in.rows;
Eigen::MatrixXf x, res;
G_to_Eigen(in, &x);
res = x.ldlt().solve(Eigen::MatrixXf::Identity(n, n));
Eigen_to_G(res, out);
}
double GetQFromNewResidual(
Vector& res_G) // e.g. permuted residual under NULL
{
// calculate Q
Eigen::VectorXf res;
G_to_Eigen(res_G, &res);
double Q = (this->K_sqrt * res).squaredNorm();
return Q;
}
int TestCovariate(Matrix& Xnull, Matrix& y, Matrix& Xcol) {
G_to_Eigen(Xcol, &this->g);
U = (resid.transpose() * sigmaMatInv * g).sum();
V_GG = (g.transpose() * sigmaMatInv * g);
V_XG = (x.transpose() * sigmaMatInv * g);
V_XX = (x.transpose() * sigmaMatInv * x);
const int m = V_XX.rows();
V = (V_GG - V_XG.transpose() * V_XX.ldlt().solve(Eigen::MatrixXf::Identity(m, m)) * V_XG).sum();
return 0;
}
int FitNullModel(Matrix& Xnull, Matrix& Y) {
G_to_Eigen(Xnull, &this->x);
G_to_Eigen(Y, &this->y);
// append identity matrix
AppendIdentity();
// initialize beta and sigma2
sigma2.resize(kinship.size());
for (size_t i = 0; i < sigma2.size(); ++i) {
sigma2[i] = 1.0;
}
calculateSigmaMat();
calculateBeta();
// fit null model
calculateLLK();
double oldLLK = llk;
int time = 1;
double diff;
while (true) {
calculateSigma2();
diff = llk - oldLLK;
if (diff < 1e-6) {
#ifdef DEBUG
fprintf(stderr, "Model converges or llk cannot be improved\n");
#endif
break;
}
oldLLK = llk;
time ++;
if (time > 100) {
#ifdef DEBUG
fprintf(stderr, "Model probably do not converge at 100 times, llk = %g", llk);
#endif
break;
}
}
return 0;
}
int TestCovariate(Matrix& Xnull, Matrix& Y, Matrix& Xcol,
const EigenMatrix& kinshipU, const EigenMatrix& kinshipS){
Eigen::MatrixXf g;
G_to_Eigen(Xcol, &g);
// store U'*G for computing AF later.
const Eigen::MatrixXf& U = kinshipU.mat;
this->ug = U.transpose() * g;
Eigen::RowVectorXf g_mean = g.colwise().mean();
g = g.rowwise() - g_mean;
double gTg = g.array().square().sum();
double t_new = (g.array() * this->transformedY.array()).sum();
t_new = t_new * t_new / gTg;
double t_score = t_new / this->gamma;
this->betaG = (g.transpose() * this->transformedY).sum() / gTg / this->gamma;
this->betaGVar = this->ySigmaY / gTg / this->gamma;
this->pvalue = gsl_cdf_chisq_Q(t_score, 1.0);
return 0;
}
int Fit(Vector& res_G, // residual under NULL -- may change when permuting
Vector& v_G, // variance under NULL -- may change when permuting
Matrix& X_G, // covariance
Matrix& G_G, // genotype
Vector& w_G) // weight
{
this->nPeople = X_G.rows;
this->nMarker = G_G.cols;
this->nCovariate = X_G.cols;
// calculation w_sqrt
G_to_Eigen(w_G, &this->w_sqrt);
w_sqrt = w_sqrt.cwiseSqrt();
// calculate K = G * W * G'
Eigen::MatrixXf G;
G_to_Eigen(G_G, &G);
this->K_sqrt.noalias() = w_sqrt.asDiagonal() * G.transpose();
// calculate Q = ||res * K||
Eigen::VectorXf res;
G_to_Eigen(res_G, &res);
this->Q = (this->K_sqrt * res).squaredNorm();
// calculate P0 = V - V X (X' V X)^(-1) X' V
Eigen::VectorXf v;
G_to_Eigen(v_G, &v);
if (this->nCovariate == 1) {
P0 = -v * v.transpose() / v.sum();
// printf("dim(P0) = %d, %d\n", P0.rows(), P0.cols());
// printf("dim(v) = %d\n", v.size());
P0.diagonal() += v;
// printf("dim(v) = %d\n", v.size());
} else {
Eigen::MatrixXf X;
G_to_Eigen(X_G, &X);
Eigen::MatrixXf XtV; // X^t V
XtV.noalias() = X.transpose() * v.asDiagonal();
P0 = -XtV.transpose() * ((XtV * X).inverse()) * XtV;
P0.diagonal() += v;
}
// dump();
// Eigen::MatrixXf tmp = K_sqrt * P0 * K_sqrt.transpose();
// dumpToFile(tmp, "out.tmp");
// eigen decomposition
Eigen::SelfAdjointEigenSolver<Eigen::MatrixXf> es;
es.compute(K_sqrt * P0 * K_sqrt.transpose());
#ifdef DEBUG
std::ofstream k("K");
k << K_sqrt;
k.close();
#endif
// std::ofstream p("P0");
// p << P0;
// p.close();
this->mixChiSq.reset();
int r_ub = std::min(nPeople, nMarker);
int r = 0; // es.eigenvalues().size();
int eigen_len = es.eigenvalues().size();
for (int i = eigen_len - 1; i >= 0; i--) {
if (es.eigenvalues()[i] > ZBOUND && r < r_ub) {
this->mixChiSq.addLambda(es.eigenvalues()[i]);
r++;
} else
break;
}
// calculate p-value
this->pValue = this->mixChiSq.getPvalue(this->Q);
if (this->pValue == 0.0 || this->pValue == 1.0) {
this->pValue = this->mixChiSq.getLiuPvalue(this->Q);
}
return 0;
};
int AppendKinship(Matrix& k) {
Eigen::MatrixXf mat;
G_to_Eigen(k, &mat);
return AppendKinship(mat);
}
int FitNullModel(Matrix& mat_Xnull, Matrix& mat_y,
const EigenMatrix& kinshipU, const EigenMatrix& kinshipS){
// type conversion
Eigen::MatrixXf x;
Eigen::MatrixXf y;
G_to_Eigen(mat_Xnull, &x);
G_to_Eigen(mat_y, &y);
this->lambda = kinshipS.mat;
const Eigen::MatrixXf& U = kinshipU.mat;
// rotate
this->ux = U.transpose() * x;
this->uy = U.transpose() * y;
// get beta, sigma2_g and delta
// where delta = sigma2_e / sigma2_g
double loglik[101];
int maxIndex = -1;
double maxLogLik = 0;
for (int i = 0; i <= 100; ++i ){
double d = exp(-10 + i * 0.2);
getBetaSigma2(d);
loglik[i] = getLogLikelihood(d);
// fprintf(stderr, "%d\tdelta=%g\tll=%lf\n", i, delta, loglik[i]);
if (std::isnan(loglik[i])) {
continue;
}
if (maxIndex < 0 || loglik[i] > maxLogLik) {
maxIndex = i;
maxLogLik = loglik[i];
}
}
if (maxIndex < -1) {
fprintf(stderr, "Cannot optimize\n");
return -1;
}
if (maxIndex == 0 || maxIndex == 100) {
// on the boundary
// do not try maximize it.
} else {
gsl_function F;
F.function = goalFunction;
F.params = this;
Minimizer minimizer;
double lb = exp(-10 + (maxIndex-1) * 0.2);
double ub = exp(-10 + (maxIndex+1) * 0.2);
double start = exp(-10 + maxIndex * 0.2);
if (minimizer.minimize(F, start, lb, ub)) {
// fprintf(stderr, "Minimization failed, fall back to initial guess.\n");
this->delta = start;
} else {
this->delta = minimizer.getX();
// fprintf(stderr, "minimization succeed when delta = %g, sigma2_g = %g\n", this->delta, this->sigma2_g);
}
}
// store some intermediate results
// fprintf(stderr, "maxIndex = %d, delta = %g, Try brent\n", maxIndex, delta);
// fprintf(stderr, "beta[%d][%d] = %g\n", (int)beta.rows(), (int)beta.cols(), beta(0,0));
this->h2 = 1.0 /(1.0 + this->delta);
this->sigma2 = this->sigma2_g * this->h2;
// we derive different formular to replace original eqn (7)
this->gamma = (this->lambda.array() / (this->lambda.array() + this->delta)).sum() / this->sigma2_g / (this->ux.rows() - 1 ) ;
// fprintf(stderr, "gamma = %g\n", this->gamma);
// transformedY = \Sigma^{-1} * (y_tilda) and y_tilda = y - X * \beta
// since \Sigma = (\sigma^2_g * h^2 ) * (U * (\lambda + delta) * U')
// transformedY = 1 / (\sigma^2_g * h^2 ) * (U * (\lambda+delta)^{-1} * U' * (y_tilda))
// = 1 / (\sigma^2_g * h^2 ) * (U * \lambda^{-1} * (uResid))
// since h^2 = 1 / (1+delta)
// transformedY = (1 + delta/ (\sigma^2_g ) * (U * \lambda^{-1} * (uResid))
Eigen::MatrixXf resid = y - x * (x.transpose() * x).eval().ldlt().solve(x.transpose() * y); // this is y_tilda
this->transformedY.noalias() = U.transpose() * resid;
this->transformedY = (this->lambda.array() + this->delta).inverse().matrix().asDiagonal() * this->transformedY;
this->transformedY = U * this->transformedY;
this->transformedY /= this->sigma2_g;
// fprintf(stderr, "transformedY(0,0) = %g\n", transformedY(0,0));
this->ySigmaY= (resid.array() * transformedY.array()).sum();
return 0;
}
int FitNullModel(Matrix& mat_Xnull, Matrix& mat_y,
const EigenMatrix& kinshipU, const EigenMatrix& kinshipS){
// sanity check
if (mat_Xnull.rows != mat_y.rows) return -1;
if (mat_Xnull.rows != kinshipU.mat.rows()) return -1;
if (mat_Xnull.rows != kinshipS.mat.rows()) return -1;
// type conversion
G_to_Eigen(mat_Xnull, &this->ux);
G_to_Eigen(mat_y, &this->uy);
this->lambda = kinshipS.mat;
const Eigen::MatrixXf& U = kinshipU.mat;
// rotate
this->ux = U.transpose() * this->ux;
this->uy = U.transpose() * this->uy;
// get beta, sigma and delta
// where delta = sigma2_e / sigma2_g
double loglik[101];
int maxIndex = -1;
double maxLogLik = 0;
for (int i = 0; i <= 100; ++i ){
delta = exp(-10 + i * 0.2);
getBetaSigma2(delta);
loglik[i] = getLogLikelihood(delta);
#ifdef DEBUG
fprintf(stderr, "%d\tdelta=%g\tll=%lf\n", i, delta, loglik[i]);
fprintf(stderr, "beta(0)=%lf\tsigma2=%lf\n",
beta(0), sigma2);
#endif
if (std::isnan(loglik[i])) {
continue;
}
if (maxIndex < 0 || loglik[i] > maxLogLik) {
maxIndex = i;
maxLogLik = loglik[i];
}
}
if (maxIndex < -1) {
fprintf(stderr, "Cannot optimize\n");
return -1;
}
#if 0
fprintf(stderr, "maxIndex = %d\tll=%lf\t\tbeta(0)=%lf\tsigma2=%lf\n",
maxIndex, maxLogLik, beta(0), sigma2);
#endif
if (maxIndex == 0 || maxIndex == 100) {
// on the boundary
// do not try maximize it.
} else {
gsl_function F;
F.function = goalFunction;
F.params = this;
Minimizer minimizer;
double lb = exp(-10 + (maxIndex-1) * 0.2);
double ub = exp(-10 + (maxIndex+1) * 0.2);
double start = exp(-10 + maxIndex * 0.2);
if (minimizer.minimize(F, start, lb, ub)) {
fprintf(stderr, "Minimization failed, fall back to initial guess.\n");
this->delta = start;
} else {
this->delta = minimizer.getX();
#ifdef DEBUG
fprintf(stderr, "minimization succeed when delta = %g, sigma2 = %g\n", this->delta, this->sigma2);
#endif
}
}
// store some intermediate results
#ifdef DEBUG
fprintf(stderr, "delta = sigma2_e/sigma2_g, and sigma2 is sigma2_g\n");
fprintf(stderr, "maxIndex = %d, delta = %g, Try brent\n", maxIndex, delta);
fprintf(stderr, "beta[0][0] = %g\t sigma2_g = %g\tsigma2_e = %g\n", beta(0,0), this->sigma2, delta * sigma2);
#endif
// if (this->test == MetaCov::LRT) {
// this->nullLikelihood = getLogLikelihood(this->delta);
// } else if (this->test == MetaCov::SCORE) {
// this->uResid = this->uy - this->ux * this->beta;
// }
return 0;
}
