int PoissonGlm::EstIRLS(gsl_matrix *Y, gsl_matrix *X, gsl_matrix *O, gsl_matrix *B, double *a)
{
initialGlm(Y, X, O, B);
gsl_set_error_handler_off();
gsl_rng *rnd=gsl_rng_alloc(gsl_rng_mt19937);
unsigned int i, j;
int status;
double yij, mij, vij, wij, tol, hii, uij, wei;
gsl_vector_view Xwi, Xi, vj, hj, dj;
gsl_matrix *WX = gsl_matrix_alloc(nRows, nParams);
gsl_matrix *TMP = gsl_matrix_alloc(nRows, nParams);
gsl_matrix *XwX = gsl_matrix_alloc(nParams, nParams);
for (j=0; j<nVars; j++) {
if ( a!=NULL ) theta[j]=a[j];
// estimate mu and beta
iterconv[j] = betaEst(j, maxiter, &tol, theta[j]);
if ((mmRef->warning==TRUE)&(iterconv[j]==maxiter))
printf("Warning: EstIRLS reached max iterations, may not converge in the %d-th variable (dev=%.4f, err=%.4f)!\n", j, dev[j], tol);
gsl_matrix_memcpy (WX, X);
for (i=0; i<nRows; i++) {
mij = gsl_matrix_get(Mu, i, j);
// get variance
vij = varfunc( mij, theta[j] );
gsl_matrix_set(Var, i, j, vij);
// get weight
wij = sqrt(weifunc(mij, theta[j]));
gsl_matrix_set(wHalf, i, j, wij);
// get (Pearson) residuals
yij = gsl_matrix_get(Y, i, j);
gsl_matrix_set(Res, i, j, (yij-mij)/sqrt(vij));
// get PIT residuals for discrete data
wei = gsl_rng_uniform_pos (rnd); // wei ~ U(0, 1)
uij = wei*cdf(yij, mij, theta[j]);
if (yij>0) uij=uij+(1-wei)*cdf((yij-1),mij,theta[j]);
gsl_matrix_set(PitRes, i, j, uij);
// get elementry log-likelihood
ll[j] = ll[j] + llfunc( yij, mij, theta[j]);
// W^1/2 X
Xwi = gsl_matrix_row (WX, i);
gsl_vector_scale(&Xwi.vector, wij);
}
aic[j]=-ll[j]+2*(nParams);
// X^T * W * X
gsl_matrix_set_identity(XwX);
gsl_blas_dsyrk (CblasLower, CblasTrans, 1.0, WX, 0.0, XwX);
status=gsl_linalg_cholesky_decomp (XwX);
if (status==GSL_EDOM) {
if (mmRef->warning==TRUE)
printf("Warning: singular matrix in calculating pit-residuals. An eps*I is added to the singular matrix.\n");
gsl_matrix_set_identity(XwX);
gsl_blas_dsyrk (CblasLower, CblasTrans, 1.0, WX, mintol, XwX);
gsl_linalg_cholesky_decomp (XwX);
}
gsl_linalg_cholesky_invert (XwX);
// Calc varBeta
dj = gsl_matrix_diagonal (XwX);
vj = gsl_matrix_column (varBeta, j);
gsl_vector_memcpy (&vj.vector, &dj.vector);
// hii is diagonal element of H=X*(X'WX)^-1*X'*W
hj = gsl_matrix_column (sqrt1_Hii, j);
gsl_blas_dsymm(CblasRight,CblasLower,1.0,XwX,Xref,0.0,TMP); // X*(X'WX)^-1
for (i=0; i<nRows; i++) {
Xwi=gsl_matrix_row(TMP, i);
Xi=gsl_matrix_row(Xref, i);
wij=gsl_matrix_get(wHalf, i, j);
gsl_blas_ddot(&Xwi.vector, &Xi.vector, &hii);
gsl_vector_set(&hj.vector, i, MAX(mintol, sqrt(MAX(0, 1-wij*wij*hii))));
}
}
// standardize perason residuals by rp/sqrt(1-hii)
// gsl_matrix_div_elements (Res, sqrt1_Hii);
// subtractMean(Res); // have mean subtracted
gsl_matrix_free(XwX);
gsl_matrix_free(WX);
gsl_matrix_free(TMP);
gsl_rng_free(rnd);
return SUCCESS;
}
int NBinGlm::nbinfit(gsl_matrix *Y, gsl_matrix *X, gsl_matrix *O, gsl_matrix *B)
{
gsl_set_error_handler_off();
initialGlm(Y, X, O, B);
gsl_rng *rnd=gsl_rng_alloc(gsl_rng_mt19937);
unsigned int i, j; //, isConv;
double yij, mij, vij, hii, uij, wij, wei;
double th, tol, dev_th_b_old;
int status;
// gsl_vector_view b0j, m0j, e0j, v0j;
gsl_matrix *WX = gsl_matrix_alloc(nRows, nParams);
gsl_matrix *TMP = gsl_matrix_alloc(nRows, nParams);
gsl_matrix *XwX = gsl_matrix_alloc(nParams, nParams);
gsl_vector_view Xwi, Xi, vj, dj, hj;
for (j=0; j<nVars; j++) {
betaEst(j, maxiter, &tol, maxtol); //poisson
// Get initial theta estimates
iterconv[j]=0.0;
if (mmRef->estiMethod==CHI2) {
th = getDisper(j, 1.0);
while ( iterconv[j]<maxiter ) {
//printf("th=%.2f, iterconv[%d]=%d\n", th, j, iterconv[j]);
iterconv[j]++;
dev_th_b_old = dev[j];
betaEst(j, 1.0, &tol, th); // 1-step beta
th = getDisper(j, th)/th;
tol = ABS((dev[j]-dev_th_b_old)/(ABS(dev[j])+0.1));
if (tol<eps) break;
} }
else if (mmRef->estiMethod==NEWTON) {
th = thetaML(0.0, j, maxiter);
while ( iterconv[j]<maxiter ) {
iterconv[j]++;
dev_th_b_old = dev[j];
th = thetaML(th, j, maxiter2);
betaEst(j, maxiter2, &tol, th);
tol=ABS((dev[j]-dev_th_b_old)/(ABS(dev[j])+0.1));
if (tol<eps) break;
} }
else {
th = getfAfAdash(0.0, j, maxiter);
/* lm=0;
for (i=0; i<nRows; i++) {
yij = gsl_matrix_get(Y, i, j);
mij = gsl_matrix_get(Mu, i, j);
lm = lm + llfunc( yij, mij, th);
} */
while ( iterconv[j]<maxiter ) {
iterconv[j]++;
dev_th_b_old = dev[j];
betaEst(j, maxiter2, &tol, th);
th = getfAfAdash(th, j, 1.0);
tol=ABS((dev[j]-dev_th_b_old)/(ABS(dev[j])+0.1));
if (tol<eps) break;
}
}
if ((iterconv[j]==maxiter)&(mmRef->warning==TRUE))
printf("Warning: reached maximum itrations - negative binomial may NOT converge in the %d-th variable (dev=%.4f, err=%.4f, theta=%.4f)!\n", j, dev[j], tol, th);
// other properties based on mu and phi
theta[j] = th;
gsl_matrix_memcpy(WX, Xref);
ll[j]=0;
for (i=0; i<nRows; i++) {
yij = gsl_matrix_get(Y, i, j);
mij = gsl_matrix_get(Mu, i, j);
vij = varfunc( mij, th);
gsl_matrix_set(Var, i, j, vij);
wij = sqrt(weifunc(mij, th));
gsl_matrix_set(wHalf, i, j, wij);
gsl_matrix_set(Res, i, j, (yij-mij)/sqrt(vij));
ll[j] = ll[j] + llfunc( yij, mij, th);
// get PIT residuals for discrete data
wei = gsl_rng_uniform_pos (rnd); // wei ~ U(0, 1)
uij=wei*cdf(yij, mij, th);
if (yij>0) uij=uij+(1-wei)*cdf((yij-1),mij,th);
gsl_matrix_set(PitRes, i, j, uij);
// W^1/2 X
Xwi = gsl_matrix_row (WX, i);
gsl_vector_scale(&Xwi.vector, wij);
}
aic[j]=-ll[j]+2*(nParams+1);
// X^T * W * X
gsl_matrix_set_identity (XwX);
gsl_blas_dsyrk (CblasLower, CblasTrans, 1.0, WX, 0.0, XwX);
status=gsl_linalg_cholesky_decomp (XwX);
if (status==GSL_EDOM) {
if (mmRef->warning==TRUE)
printf("Warning: singular matrix in calculating pit-residuals. An eps*I is added to the singular matrix.\n");
gsl_matrix_set_identity (XwX);
gsl_blas_dsyrk (CblasLower, CblasTrans, 1.0, WX, mintol, XwX);
gsl_linalg_cholesky_decomp (XwX);
}
gsl_linalg_cholesky_invert (XwX); // (X'WX)^-1
// Calc varBeta
vj = gsl_matrix_column (varBeta, j);
dj = gsl_matrix_diagonal (XwX);
gsl_vector_memcpy (&vj.vector, &dj.vector);
// hii is diagonal element of H=X*(X'WX)^-1*X'*W
hj = gsl_matrix_column (sqrt1_Hii, j);
gsl_blas_dsymm(CblasRight,CblasLower,1.0,XwX,Xref,0.0,TMP); // X*(X'WX)^-1
for (i=0; i<nRows; i++) {
Xwi=gsl_matrix_row(TMP, i);
Xi=gsl_matrix_row(Xref, i);
wij=gsl_matrix_get(wHalf, i, j);
gsl_blas_ddot(&Xwi.vector, &Xi.vector, &hii);
gsl_vector_set(&hj.vector, i, MAX(mintol, sqrt(MAX(0, 1-wij*wij*hii))));
//printf("hii=%.4f, wij=%.4f, sqrt(1-wij*wij*hii)=%.4f\n", hii, wij, sqrt(1-wij*wij*hii));
}
} // end nVar for j loop
// gsl_matrix_div_elements (Res, sqrt1_Hii);
// subtractMean(Res);
gsl_matrix_free(XwX);
gsl_matrix_free(WX);
gsl_matrix_free(TMP);
gsl_rng_free(rnd);
return SUCCESS;
}
void BayesUnfoldingExample641()
{
#ifdef __CINT__ // Avoid CINT badness
Printf("Please compile this script (root BayesUnfoldingExample641.C+) "
"or use ROOT 6.");
gSystem->Exit(0);
#endif
if (!gROOT->IsBatch())
{
Printf("Several canvases coming...adding -b flag.");
gROOT->SetBatch();
}
gStyle->SetOptStat(0);
gStyle->SetPaintTextFormat(".2f");
if (gSystem->Getenv("TMPDIR"))
gSystem->SetBuildDir(gSystem->Getenv("TMPDIR"));
TRandom3 ran;
TStopwatch watch; // Watch starts here. A call to Start() would reset it.
TObjArray *cList = new TObjArray(); // List of drawn canvases --> PDF file
// Set up the problem
double bins[Nt+1] = {0};
for (int j=0; j<=Nt; j++)
bins[j] = 500*TMath::Exp(0.15*j);
TestProblem testprob = AtlasDiJetMass(Nt, Nr, bins, bins,
apar, bpar, nevts, evtWeight);
TH2D *hM = testprob.Response;
TH1D *hT = testprob.xTruth;
TH1D *hTmc = testprob.xTruthEst;
TH1D *hMt = testprob.xIni;
TH1D *hD = testprob.bNoisy;
TH1D *heff = testprob.eff;
SetHistProps(hT,kRed+2,kNone,kRed+2);
SetHistProps(hTmc,kRed,kNone,kRed);
SetHistProps(hD,kBlack,kNone,kBlack,kFullCircle,1.5);
TMatrixD M = MatrixUtils::Hist2Matrix(hM);
TVectorD T = MatrixUtils::Hist2Vec(hT); // \hat{T}
TVectorD Tmc = MatrixUtils::Hist2Vec(hTmc); // \tilde{T}
TVectorD D = MatrixUtils::Hist2Vec(hD);
TVectorD eff = MatrixUtils::Hist2Vec(heff);
TVectorD Pt = MatrixUtils::ElemDiv(MatrixUtils::Hist2Vec(hMt), eff); // P(t)
TMatrixD Prt = MatrixUtils::DivRowsByVector(M, Pt); // P(r|t)
// Compute initial sampling volume and do MCMC sampling
TGraphAsymmErrors *box = HyperBox(hTmc);
SetGraphProps(box, kGreen+2, kNone, kSpring, kFullSquare, 1.0);
// Likelihood functor
LogPoissonLikeFn llfunc(Prt, D);
// Curvature regularization.
// Note that this is not directly penalizing curvature of the solution.
// Instead it smooths the solution divided by the trial spectrum.
std::vector<double> regpars;
regpars.push_back(alpha); // Regularization strength
for (int i=0; i<box->GetN(); i++)
regpars.push_back(box->GetY()[i]);
CurvatureRegFn regfunc(regpars);
TTree *tmcmc = SampleMH(nMcmcSamples, 1e4, 0.01, box, llfunc, regfunc);
// Create marginal prob. distributions from MCMC
std::cout << Form("Marginalizing parameters from Markov chain...")
<< std::flush;
TH1D *hMCMC[Nt];
for (int t=0; t<Nt; t++)
{
double tlo = box->GetY()[t] - box->GetEYlow()[t];
double thi = box->GetY()[t] + box->GetEYhigh()[t];
hMCMC[t] = new TH1D(Form("hMCMC%d",t),"",nMcmcBins, tlo, thi);
hMCMC[t]->SetTitle(Form("MCMC - point %d;"
"entries;"
"Marginal posterior probability",t));
// Marginalize with unit weight when using MCMC, weight by
// likelihood if sampling was uniform.
tmcmc->Draw(Form("T%d >> hMCMC%d",t,t), "", "goff");
hMCMC[t]->Scale(1./hMCMC[t]->Integral(1, nMcmcBins));
SetHistProps(hMCMC[t], kBlack, kYellow, kBlack, kFullCircle, 1.0);
hMCMC[t]->GetYaxis()->SetTitleOffset(1.5);
}
Printf("Done marginalizing MCMC.");
// Now compute reduced sampling volume, and do uniform sampling
TGraphAsymmErrors *rbox = ReducedSamplingVolume(hMCMC, box);
SetGraphProps(rbox, kBlack, kNone, kNone, kFullSquare, 1.0);
TH1D *hFlat[Nt];
if (doUniformSampling)
{
TTree *tflat = SampleUniform(nFlatSamples, D, Prt, rbox);
std::cout << Form("Marginalizing parameters from uniform volume...")
<< std::flush;
for (int t=0; t<Nt; t++)
{
double tlo = rbox->GetY()[t] - rbox->GetEYlow()[t];
double thi = rbox->GetY()[t] + rbox->GetEYhigh()[t];
hFlat[t] = new TH1D(Form("hFlat%d",t),"",nFlatBins, tlo, thi);
hFlat[t]->SetTitle(Form("Uniform sampling - point %d;"
"dijet mass (GeV/c^{2});"
"Marginal posterior probability",t));
tflat->Draw(Form("T%d >> hFlat%d",t,t), "L", "goff");
hFlat[t]->Scale(1./hFlat[t]->Integral(1,nFlatBins));
SetHistProps(hFlat[t], kBlack, kOrange, kBlack, kFullCircle, 1.0);
}
Printf("Done marginalizing uniform volume.");
}
// Unfolded spectrum from MCMC
TGraphErrors *unf1 = new TGraphErrors();
SetGraphProps(unf1, kBlue, kNone, kBlue, kOpenSquare, 1.5);
unf1->SetLineWidth(2);
for (int t=0; t<Nt; t++)
{
MaxDensityInterval mdi = GetMDI(hMCMC[t], 0.68);
unf1->SetPoint(t, hD->GetBinCenter(t+1), mdi.u);
unf1->SetPointError(t, 0.48*hD->GetBinWidth(t+1), mdi.du);
}
// Unfolded spectrum from uniform sampling after volume reduction
TGraphErrors *unf2 = 0;
if (doUniformSampling)
{
unf2 = new TGraphErrors();
SetGraphProps(unf2, kRed, kNone, kRed, kOpenSquare, 1.5);
unf2->SetLineWidth(2);
for (int t=0; t<Nt; t++)
{
MaxDensityInterval mdi = GetMDI(hFlat[t], 0.68);
unf2->SetPoint(t, hD->GetBinCenter(t+1), mdi.u);
unf2->SetPointError(t, 0.47*hD->GetBinWidth(t+1), mdi.du);
}
}
Printf("Drawing results...");
DrawObject(hM, "colz", "matrix", cList, 550, 500);
gPad->SetLogx(); gPad->SetLogy(); gPad->SetLogz();
gPad->SetRightMargin(0.15);
DrawObject(heff, "", "efficiency", cList);
// Draw marginal dists. from MCMC
for (int t=0; t<Nt; t++)
{
DrawObject(hMCMC[t], "", Form("post_%d", t), cList);
gPad->SetLeftMargin(0.15);
if (doUniformSampling)
{
hFlat[t]->Scale(1./hFlat[t]->Integral(1, nFlatBins,"width"));
hFlat[t]->Draw("same");
}
double ymin = hMCMC[t]->GetMinimum();
double ymax = hMCMC[t]->GetMaximum();
double yDraw = 0.25*(ymax-ymin);
DataPoint(hD, hMCMC[t], t, 0.75*yDraw)->Draw("ep same");
TruePoint(hT, hMCMC[t], t, yDraw)->Draw("p same");
MD68Point(hMCMC[t], yDraw)->Draw("ep same");
}
// Result!
hT->GetYaxis()->SetRangeUser(0.002, 101*nevts*evtWeight);
DrawObject(hT, "ep", "result", cList);
gPad->SetLogy();
box->Draw("e5 same");
if (doUniformSampling || drawReducedVolume)
rbox->Draw("e5 same");
hT->Draw("same");
hD->Draw("ep same");
unf1->Draw("ep same");
if (unf2)
unf2->Draw("ep same");
if (printPDFs)
{
PrintPDFs(cList, "pdfs"); // Print individuals into ./pdfs dir
PrintPDF(cList, "pdfs/mcmc_unfold_example"); // Multipage PDF
}
Printf("All done.");
watch.Stop();
watch.Print();
return;
}
