MCMCInterval * Tprime::GetMcmcInterval(ModelConfig mc,
double conf_level,
int n_iter,
int n_burn,
double left_side_tail_fraction,
int n_bins) {
//
// Bayesian MCMC calculation using arbitrary ModelConfig
// Want an efficient proposal function, so derive it from covariance
// matrix of fit
//
RooAbsData * _data = data;
//RooAbsData * _data = pWs->data("obsData");
//RooStats::ModelConfig * _mc = (RooStats::ModelConfig *)pWs->genobj("ModelConfig");
RooStats::ModelConfig * _mc = GetModelConfig();
_mc->Print();
//RooFitResult * fit = pWs->pdf("model_tprime")->fitTo(*_data,Save());
RooFitResult * fit = _mc->GetPdf()->fitTo(*_data,Save());
ProposalHelper ph;
ph.SetVariables((RooArgSet&)fit->floatParsFinal());
ph.SetCovMatrix(fit->covarianceMatrix());
ph.SetUpdateProposalParameters(kTRUE); // auto-create mean vars and add mappings
ph.SetCacheSize(100);
ProposalFunction * pf = ph.GetProposalFunction();
//delete pf;
//pf = new SequentialProposal();
MCMCCalculator mcmc( *_data, mc );
mcmc.SetConfidenceLevel(conf_level);
mcmc.SetNumIters(n_iter); // Metropolis-Hastings algorithm iterations
mcmc.SetProposalFunction(*pf);
mcmc.SetNumBurnInSteps(n_burn); // first N steps to be ignored as burn-in
mcmc.SetLeftSideTailFraction(left_side_tail_fraction);
mcmc.SetNumBins(n_bins);
//mcInt = mcmc.GetInterval();
try {
mcInt = mcmc.GetInterval();
} catch ( std::length_error &ex) {
mcInt = 0;
}
//std::cout << "!!!!!!!!!!!!!! interval" << std::endl;
if (mcInt == 0) std::cout << "No interval found!" << std::endl;
delete fit;
delete pf;
return mcInt;
}
MCMCInterval * TwoBody::GetMcmcInterval_OldWay(ModelConfig mc,
double conf_level,
int n_iter,
int n_burn,
double left_side_tail_fraction,
int n_bins){
// use MCMCCalculator (takes about 1 min)
// Want an efficient proposal function, so derive it from covariance
// matrix of fit
RooFitResult* fit = ws->pdf("model")->fitTo(*data,Save());
ProposalHelper ph;
ph.SetVariables((RooArgSet&)fit->floatParsFinal());
ph.SetCovMatrix(fit->covarianceMatrix());
ph.SetUpdateProposalParameters(kTRUE); // auto-create mean vars and add mappings
ph.SetCacheSize(100);
ProposalFunction* pf = ph.GetProposalFunction();
MCMCCalculator mcmc( *data, mc );
mcmc.SetConfidenceLevel(conf_level);
mcmc.SetNumIters(n_iter); // Metropolis-Hastings algorithm iterations
mcmc.SetProposalFunction(*pf);
mcmc.SetNumBurnInSteps(n_burn); // first N steps to be ignored as burn-in
mcmc.SetLeftSideTailFraction(left_side_tail_fraction);
mcmc.SetNumBins(n_bins);
//mcInt = mcmc.GetInterval();
try {
mcInt = mcmc.GetInterval();
} catch ( std::length_error &ex) {
mcInt = 0;
}
//std::cout << "!!!!!!!!!!!!!! interval" << std::endl;
if (mcInt == 0) std::cout << "No interval found!" << std::endl;
return mcInt;
}
//
// calculation of the limit: assumes that wspace is set up and observations
// contained in data
//
MyLimit computeLimit (RooWorkspace* wspace, RooDataSet* data, StatMethod method, bool draw) {
// let's time this challenging example
TStopwatch t;
//
// get nominal signal
//
RooRealVar exp_sig(*wspace->var("s"));
double exp_sig_val = exp_sig.getVal();
std::cout << "exp_sig = " << exp_sig_val << std::endl;
/////////////////////////////////////////////////////
// Now the statistical tests
// model config
std::cout << wspace->pdf("model") << " "
<< wspace->pdf("prior") << " "
<< wspace->set("poi") << " "
<< wspace->set("nuis") << std::endl;
ModelConfig modelConfig("RA4abcd");
modelConfig.SetWorkspace(*wspace);
modelConfig.SetPdf(*wspace->pdf("model"));
modelConfig.SetPriorPdf(*wspace->pdf("prior"));
modelConfig.SetParametersOfInterest(*wspace->set("poi"));
modelConfig.SetNuisanceParameters(*wspace->set("nuis"));
//////////////////////////////////////////////////
// If you want to see the covariance matrix uncomment
// wspace->pdf("model")->fitTo(*data);
// use ProfileLikelihood
if ( method == ProfileLikelihoodMethod ) {
ProfileLikelihoodCalculator plc(*data, modelConfig);
plc.SetConfidenceLevel(0.95);
RooFit::MsgLevel msglevel = RooMsgService::instance().globalKillBelow();
RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL);
LikelihoodInterval* plInt = plc.GetInterval();
RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL);
plInt->LowerLimit( *wspace->var("s") ); // get ugly print out of the way. Fix.
// RooMsgService::instance().setGlobalKillBelow(RooFit::DEBUG);
if ( draw ) {
TCanvas* c = new TCanvas("ProfileLikelihood");
LikelihoodIntervalPlot* lrplot = new LikelihoodIntervalPlot(plInt);
lrplot->Draw();
}
// RooMsgService::instance().setGlobalKillBelow(msglevel);
double lowLim = plInt->LowerLimit(*wspace->var("s"));
double uppLim = plInt->UpperLimit(*wspace->var("s"));
// double exp_sig_val = wspace->var("s")->getVal();
// double exp_sig_val = exp_sig.getVal();
cout << "Profile Likelihood interval on s = [" <<
lowLim << ", " <<
uppLim << "]" << " " << exp_sig_val << endl;
// MyLimit result(plInt->IsInInterval(exp_sig),
MyLimit result(exp_sig_val>lowLim&&exp_sig_val<uppLim,lowLim,uppLim);
// std::cout << "isIn " << result << std::endl;
delete plInt;
// delete modelConfig;
return result;
}
// use FeldmaCousins (takes ~20 min)
if ( method == FeldmanCousinsMethod ) {
FeldmanCousins fc(*data, modelConfig);
fc.SetConfidenceLevel(0.95);
//number counting: dataset always has 1 entry with N events observed
fc.FluctuateNumDataEntries(false);
fc.UseAdaptiveSampling(true);
fc.SetNBins(100);
PointSetInterval* fcInt = NULL;
fcInt = (PointSetInterval*) fc.GetInterval(); // fix cast
double lowLim = fcInt->LowerLimit(*wspace->var("s"));
double uppLim = fcInt->UpperLimit(*wspace->var("s"));
// double exp_sig_val = wspace->var("s")->getVal();
cout << "Feldman Cousins interval on s = [" << lowLim << " " << uppLim << endl;
// std::cout << "isIn " << result << std::endl;
MyLimit result(exp_sig_val>lowLim&&exp_sig_val<uppLim,
fcInt->LowerLimit(*wspace->var("s")),fcInt->UpperLimit(*wspace->var("s")));
delete fcInt;
return result;
}
// use BayesianCalculator (only 1-d parameter of interest, slow for this problem)
if ( method == BayesianMethod ) {
BayesianCalculator bc(*data, modelConfig);
bc.SetConfidenceLevel(0.95);
bc.SetLeftSideTailFraction(0.5);
SimpleInterval* bInt = NULL;
if( wspace->set("poi")->getSize() == 1) {
bInt = bc.GetInterval();
if ( draw ) {
TCanvas* c = new TCanvas("Bayesian");
// the plot takes a long time and print lots of error
// using a scan it is better
bc.SetScanOfPosterior(50);
RooPlot* bplot = bc.GetPosteriorPlot();
bplot->Draw();
}
cout << "Bayesian interval on s = [" <<
bInt->LowerLimit( ) << ", " <<
bInt->UpperLimit( ) << "]" << endl;
// std::cout << "isIn " << result << std::endl;
MyLimit result(bInt->IsInInterval(exp_sig),
bInt->LowerLimit(),bInt->UpperLimit());
delete bInt;
return result;
} else {
cout << "Bayesian Calc. only supports on parameter of interest" << endl;
return MyLimit();
}
}
// use MCMCCalculator (takes about 1 min)
// Want an efficient proposal function, so derive it from covariance
// matrix of fit
if ( method == MCMCMethod ) {
RooFitResult* fit = wspace->pdf("model")->fitTo(*data,Save());
ProposalHelper ph;
ph.SetVariables((RooArgSet&)fit->floatParsFinal());
ph.SetCovMatrix(fit->covarianceMatrix());
ph.SetUpdateProposalParameters(kTRUE); // auto-create mean vars and add mappings
ph.SetCacheSize(100);
ProposalFunction* pf = ph.GetProposalFunction();
MCMCCalculator mc(*data, modelConfig);
mc.SetConfidenceLevel(0.95);
mc.SetProposalFunction(*pf);
mc.SetNumBurnInSteps(100); // first N steps to be ignored as burn-in
mc.SetNumIters(100000);
mc.SetLeftSideTailFraction(0.5); // make a central interval
MCMCInterval* mcInt = NULL;
mcInt = mc.GetInterval();
MCMCIntervalPlot mcPlot(*mcInt);
mcPlot.Draw();
cout << "MCMC interval on s = [" <<
mcInt->LowerLimit(*wspace->var("s") ) << ", " <<
mcInt->UpperLimit(*wspace->var("s") ) << "]" << endl;
// std::cout << "isIn " << result << std::endl;
MyLimit result(mcInt->IsInInterval(exp_sig),
mcInt->LowerLimit(*wspace->var("s")),mcInt->UpperLimit(*wspace->var("s")));
delete mcInt;
return result;
}
t.Print();
// delete modelConfig;
return MyLimit();
}
void rs101_limitexample()
{
// --------------------------------------
// An example of setting a limit in a number counting experiment with uncertainty on background and signal
// to time the macro
TStopwatch t;
t.Start();
// --------------------------------------
// The Model building stage
// --------------------------------------
RooWorkspace* wspace = new RooWorkspace();
wspace->factory("Poisson::countingModel(obs[150,0,300], sum(s[50,0,120]*ratioSigEff[1.,0,3.],b[100]*ratioBkgEff[1.,0.,3.]))"); // counting model
// wspace->factory("Gaussian::sigConstraint(ratioSigEff,1,0.05)"); // 5% signal efficiency uncertainty
// wspace->factory("Gaussian::bkgConstraint(ratioBkgEff,1,0.1)"); // 10% background efficiency uncertainty
wspace->factory("Gaussian::sigConstraint(gSigEff[1,0,3],ratioSigEff,0.05)"); // 5% signal efficiency uncertainty
wspace->factory("Gaussian::bkgConstraint(gSigBkg[1,0,3],ratioBkgEff,0.2)"); // 10% background efficiency uncertainty
wspace->factory("PROD::modelWithConstraints(countingModel,sigConstraint,bkgConstraint)"); // product of terms
wspace->Print();
RooAbsPdf* modelWithConstraints = wspace->pdf("modelWithConstraints"); // get the model
RooRealVar* obs = wspace->var("obs"); // get the observable
RooRealVar* s = wspace->var("s"); // get the signal we care about
RooRealVar* b = wspace->var("b"); // get the background and set it to a constant. Uncertainty included in ratioBkgEff
b->setConstant();
RooRealVar* ratioSigEff = wspace->var("ratioSigEff"); // get uncertain parameter to constrain
RooRealVar* ratioBkgEff = wspace->var("ratioBkgEff"); // get uncertain parameter to constrain
RooArgSet constrainedParams(*ratioSigEff, *ratioBkgEff); // need to constrain these in the fit (should change default behavior)
RooRealVar * gSigEff = wspace->var("gSigEff"); // global observables for signal efficiency
RooRealVar * gSigBkg = wspace->var("gSigBkg"); // global obs for background efficiency
gSigEff->setConstant();
gSigBkg->setConstant();
// Create an example dataset with 160 observed events
obs->setVal(160.);
RooDataSet* data = new RooDataSet("exampleData", "exampleData", RooArgSet(*obs));
data->add(*obs);
RooArgSet all(*s, *ratioBkgEff, *ratioSigEff);
// not necessary
modelWithConstraints->fitTo(*data, RooFit::Constrain(RooArgSet(*ratioSigEff, *ratioBkgEff)));
// Now let's make some confidence intervals for s, our parameter of interest
RooArgSet paramOfInterest(*s);
ModelConfig modelConfig(wspace);
modelConfig.SetPdf(*modelWithConstraints);
modelConfig.SetParametersOfInterest(paramOfInterest);
modelConfig.SetNuisanceParameters(constrainedParams);
modelConfig.SetObservables(*obs);
modelConfig.SetGlobalObservables( RooArgSet(*gSigEff,*gSigBkg));
modelConfig.SetName("ModelConfig");
wspace->import(modelConfig);
wspace->import(*data);
wspace->SetName("w");
wspace->writeToFile("rs101_ws.root");
// First, let's use a Calculator based on the Profile Likelihood Ratio
//ProfileLikelihoodCalculator plc(*data, *modelWithConstraints, paramOfInterest);
ProfileLikelihoodCalculator plc(*data, modelConfig);
plc.SetTestSize(.05);
ConfInterval* lrinterval = plc.GetInterval(); // that was easy.
// Let's make a plot
TCanvas* dataCanvas = new TCanvas("dataCanvas");
dataCanvas->Divide(2,1);
dataCanvas->cd(1);
LikelihoodIntervalPlot plotInt((LikelihoodInterval*)lrinterval);
plotInt.SetTitle("Profile Likelihood Ratio and Posterior for S");
plotInt.Draw();
// Second, use a Calculator based on the Feldman Cousins technique
FeldmanCousins fc(*data, modelConfig);
fc.UseAdaptiveSampling(true);
fc.FluctuateNumDataEntries(false); // number counting analysis: dataset always has 1 entry with N events observed
fc.SetNBins(100); // number of points to test per parameter
fc.SetTestSize(.05);
// fc.SaveBeltToFile(true); // optional
ConfInterval* fcint = NULL;
fcint = fc.GetInterval(); // that was easy.
RooFitResult* fit = modelWithConstraints->fitTo(*data, Save(true));
// Third, use a Calculator based on Markov Chain monte carlo
// Before configuring the calculator, let's make a ProposalFunction
// that will achieve a high acceptance rate
ProposalHelper ph;
ph.SetVariables((RooArgSet&)fit->floatParsFinal());
ph.SetCovMatrix(fit->covarianceMatrix());
ph.SetUpdateProposalParameters(true);
ph.SetCacheSize(100);
ProposalFunction* pdfProp = ph.GetProposalFunction(); // that was easy
MCMCCalculator mc(*data, modelConfig);
mc.SetNumIters(20000); // steps to propose in the chain
mc.SetTestSize(.05); // 95% CL
mc.SetNumBurnInSteps(40); // ignore first N steps in chain as "burn in"
mc.SetProposalFunction(*pdfProp);
mc.SetLeftSideTailFraction(0.5); // find a "central" interval
MCMCInterval* mcInt = (MCMCInterval*)mc.GetInterval(); // that was easy
// Get Lower and Upper limits from Profile Calculator
cout << "Profile lower limit on s = " << ((LikelihoodInterval*) lrinterval)->LowerLimit(*s) << endl;
cout << "Profile upper limit on s = " << ((LikelihoodInterval*) lrinterval)->UpperLimit(*s) << endl;
// Get Lower and Upper limits from FeldmanCousins with profile construction
if (fcint != NULL) {
double fcul = ((PointSetInterval*) fcint)->UpperLimit(*s);
double fcll = ((PointSetInterval*) fcint)->LowerLimit(*s);
cout << "FC lower limit on s = " << fcll << endl;
cout << "FC upper limit on s = " << fcul << endl;
TLine* fcllLine = new TLine(fcll, 0, fcll, 1);
TLine* fculLine = new TLine(fcul, 0, fcul, 1);
fcllLine->SetLineColor(kRed);
fculLine->SetLineColor(kRed);
fcllLine->Draw("same");
fculLine->Draw("same");
dataCanvas->Update();
}
// Plot MCMC interval and print some statistics
MCMCIntervalPlot mcPlot(*mcInt);
mcPlot.SetLineColor(kMagenta);
mcPlot.SetLineWidth(2);
mcPlot.Draw("same");
double mcul = mcInt->UpperLimit(*s);
double mcll = mcInt->LowerLimit(*s);
cout << "MCMC lower limit on s = " << mcll << endl;
cout << "MCMC upper limit on s = " << mcul << endl;
cout << "MCMC Actual confidence level: "
<< mcInt->GetActualConfidenceLevel() << endl;
// 3-d plot of the parameter points
dataCanvas->cd(2);
// also plot the points in the markov chain
RooDataSet * chainData = mcInt->GetChainAsDataSet();
assert(chainData);
std::cout << "plotting the chain data - nentries = " << chainData->numEntries() << std::endl;
TTree* chain = RooStats::GetAsTTree("chainTreeData","chainTreeData",*chainData);
assert(chain);
chain->SetMarkerStyle(6);
chain->SetMarkerColor(kRed);
chain->Draw("s:ratioSigEff:ratioBkgEff","nll_MarkovChain_local_","box"); // 3-d box proportional to posterior
// the points used in the profile construction
RooDataSet * parScanData = (RooDataSet*) fc.GetPointsToScan();
assert(parScanData);
std::cout << "plotting the scanned points used in the frequentist construction - npoints = " << parScanData->numEntries() << std::endl;
// getting the tree and drawing it -crashes (very strange....);
// TTree* parameterScan = RooStats::GetAsTTree("parScanTreeData","parScanTreeData",*parScanData);
// assert(parameterScan);
// parameterScan->Draw("s:ratioSigEff:ratioBkgEff","","goff");
TGraph2D *gr = new TGraph2D(parScanData->numEntries());
for (int ievt = 0; ievt < parScanData->numEntries(); ++ievt) {
const RooArgSet * evt = parScanData->get(ievt);
double x = evt->getRealValue("ratioBkgEff");
double y = evt->getRealValue("ratioSigEff");
double z = evt->getRealValue("s");
gr->SetPoint(ievt, x,y,z);
// std::cout << ievt << " " << x << " " << y << " " << z << std::endl;
}
gr->SetMarkerStyle(24);
gr->Draw("P SAME");
delete wspace;
delete lrinterval;
delete mcInt;
delete fcint;
delete data;
// print timing info
t.Stop();
t.Print();
}
