void OneSidedFrequentistUpperLimitWithBands_intermediate(const char* infile = "",
const char* workspaceName = "combined",
const char* modelConfigName = "ModelConfig",
const char* dataName = "obsData"){
double confidenceLevel=0.95;
// degrade/improve number of pseudo-experiments used to define the confidence belt.
// value of 1 corresponds to default number of toys in the tail, which is 50/(1-confidenceLevel)
double additionalToysFac = 1.;
int nPointsToScan = 30; // number of steps in the parameter of interest
int nToyMC = 100; // number of toys used to define the expected limit and band
TStopwatch t;
t.Start();
/////////////////////////////////////////////////////////////
// First part is just to access a user-defined file
// or create the standard example file if it doesn't exist
////////////////////////////////////////////////////////////
const char* filename = "";
if (!strcmp(infile,""))
filename = "results/example_combined_GaussExample_model.root";
else
filename = infile;
// Check if example input file exists
TFile *file = TFile::Open(filename);
// if input file was specified byt not found, quit
if(!file && strcmp(infile,"")){
cout <<"file not found" << endl;
return;
}
// if default file not found, try to create it
if(!file ){
// Normally this would be run on the command line
cout <<"will run standard hist2workspace example"<<endl;
gROOT->ProcessLine(".! prepareHistFactory .");
gROOT->ProcessLine(".! hist2workspace config/example.xml");
cout <<"\n\n---------------------"<<endl;
cout <<"Done creating example input"<<endl;
cout <<"---------------------\n\n"<<endl;
}
// now try to access the file again
file = TFile::Open(filename);
if(!file){
// if it is still not there, then we can't continue
cout << "Not able to run hist2workspace to create example input" <<endl;
return;
}
/////////////////////////////////////////////////////////////
// Now get the data and workspace
////////////////////////////////////////////////////////////
// get the workspace out of the file
RooWorkspace* w = (RooWorkspace*) file->Get(workspaceName);
if(!w){
cout <<"workspace not found" << endl;
return;
}
// get the modelConfig out of the file
ModelConfig* mc = (ModelConfig*) w->obj(modelConfigName);
// get the modelConfig out of the file
RooAbsData* data = w->data(dataName);
// make sure ingredients are found
if(!data || !mc){
w->Print();
cout << "data or ModelConfig was not found" <<endl;
return;
}
cout << "Found data and ModelConfig:" <<endl;
mc->Print();
/////////////////////////////////////////////////////////////
// Now get the POI for convenience
// you may want to adjust the range of your POI
////////////////////////////////////////////////////////////
RooRealVar* firstPOI = (RooRealVar*) mc->GetParametersOfInterest()->first();
// firstPOI->setMin(0);
// firstPOI->setMax(10);
/////////////////////////////////////////////
// create and use the FeldmanCousins tool
// to find and plot the 95% confidence interval
// on the parameter of interest as specified
// in the model config
// REMEMBER, we will change the test statistic
// so this is NOT a Feldman-Cousins interval
FeldmanCousins fc(*data,*mc);
fc.SetConfidenceLevel(confidenceLevel);
fc.AdditionalNToysFactor(additionalToysFac); // improve sampling that defines confidence belt
// fc.UseAdaptiveSampling(true); // speed it up a bit, but don't use for expectd limits
fc.SetNBins(nPointsToScan); // set how many points per parameter of interest to scan
fc.CreateConfBelt(true); // save the information in the belt for plotting
/////////////////////////////////////////////
// Feldman-Cousins is a unified limit by definition
// but the tool takes care of a few things for us like which values
// of the nuisance parameters should be used to generate toys.
// so let's just change the test statistic and realize this is
// no longer "Feldman-Cousins" but is a fully frequentist Neyman-Construction.
// ProfileLikelihoodTestStatModified onesided(*mc->GetPdf());
// fc.GetTestStatSampler()->SetTestStatistic(&onesided);
// ((ToyMCSampler*) fc.GetTestStatSampler())->SetGenerateBinned(true);
ToyMCSampler* toymcsampler = (ToyMCSampler*) fc.GetTestStatSampler();
ProfileLikelihoodTestStat* testStat = dynamic_cast<ProfileLikelihoodTestStat*>(toymcsampler->GetTestStatistic());
testStat->SetOneSided(true);
// test speedups:
testStat->SetReuseNLL(true);
// toymcsampler->setUseMultiGen(true); // not fully validated
// Since this tool needs to throw toy MC the PDF needs to be
// extended or the tool needs to know how many entries in a dataset
// per pseudo experiment.
// In the 'number counting form' where the entries in the dataset
// are counts, and not values of discriminating variables, the
// datasets typically only have one entry and the PDF is not
// extended.
if(!mc->GetPdf()->canBeExtended()){
if(data->numEntries()==1)
fc.FluctuateNumDataEntries(false);
else
cout <<"Not sure what to do about this model" <<endl;
}
// We can use PROOF to speed things along in parallel
ProofConfig pc(*w, 4, "",false);
if(mc->GetGlobalObservables()){
cout << "will use global observables for unconditional ensemble"<<endl;
mc->GetGlobalObservables()->Print();
toymcsampler->SetGlobalObservables(*mc->GetGlobalObservables());
}
toymcsampler->SetProofConfig(&pc); // enable proof
// Now get the interval
PointSetInterval* interval = fc.GetInterval();
ConfidenceBelt* belt = fc.GetConfidenceBelt();
// print out the iterval on the first Parameter of Interest
cout << "\n95% interval on " <<firstPOI->GetName()<<" is : ["<<
interval->LowerLimit(*firstPOI) << ", "<<
interval->UpperLimit(*firstPOI) <<"] "<<endl;
// get observed UL and value of test statistic evaluated there
RooArgSet tmpPOI(*firstPOI);
double observedUL = interval->UpperLimit(*firstPOI);
firstPOI->setVal(observedUL);
double obsTSatObsUL = fc.GetTestStatSampler()->EvaluateTestStatistic(*data,tmpPOI);
// Ask the calculator which points were scanned
RooDataSet* parameterScan = (RooDataSet*) fc.GetPointsToScan();
RooArgSet* tmpPoint;
// make a histogram of parameter vs. threshold
TH1F* histOfThresholds = new TH1F("histOfThresholds","",
parameterScan->numEntries(),
firstPOI->getMin(),
firstPOI->getMax());
histOfThresholds->GetXaxis()->SetTitle(firstPOI->GetName());
histOfThresholds->GetYaxis()->SetTitle("Threshold");
// loop through the points that were tested and ask confidence belt
// what the upper/lower thresholds were.
// For FeldmanCousins, the lower cut off is always 0
for(Int_t i=0; i<parameterScan->numEntries(); ++i){
tmpPoint = (RooArgSet*) parameterScan->get(i)->clone("temp");
double arMax = belt->GetAcceptanceRegionMax(*tmpPoint);
double poiVal = tmpPoint->getRealValue(firstPOI->GetName()) ;
histOfThresholds->Fill(poiVal,arMax);
}
TCanvas* c1 = new TCanvas();
c1->Divide(2);
c1->cd(1);
histOfThresholds->SetMinimum(0);
histOfThresholds->Draw();
c1->cd(2);
/////////////////////////////////////////////////////////////
// Now we generate the expected bands and power-constriant
////////////////////////////////////////////////////////////
// First: find parameter point for mu=0, with conditional MLEs for nuisance parameters
RooAbsReal* nll = mc->GetPdf()->createNLL(*data);
RooAbsReal* profile = nll->createProfile(*mc->GetParametersOfInterest());
firstPOI->setVal(0.);
profile->getVal(); // this will do fit and set nuisance parameters to profiled values
RooArgSet* poiAndNuisance = new RooArgSet();
if(mc->GetNuisanceParameters())
poiAndNuisance->add(*mc->GetNuisanceParameters());
poiAndNuisance->add(*mc->GetParametersOfInterest());
w->saveSnapshot("paramsToGenerateData",*poiAndNuisance);
RooArgSet* paramsToGenerateData = (RooArgSet*) poiAndNuisance->snapshot();
cout << "\nWill use these parameter points to generate pseudo data for bkg only" << endl;
paramsToGenerateData->Print("v");
double CLb=0;
double CLbinclusive=0;
// Now we generate background only and find distribution of upper limits
TH1F* histOfUL = new TH1F("histOfUL","",100,0,firstPOI->getMax());
histOfUL->GetXaxis()->SetTitle("Upper Limit (background only)");
histOfUL->GetYaxis()->SetTitle("Entries");
for(int imc=0; imc<nToyMC; ++imc){
// set parameters back to values for generating pseudo data
w->loadSnapshot("paramsToGenerateData");
// in 5.30 there is a nicer way to generate toy data & randomize global obs
RooAbsData* toyData = toymcsampler->GenerateToyData(*paramsToGenerateData);
// get test stat at observed UL in observed data
firstPOI->setVal(observedUL);
double toyTSatObsUL = fc.GetTestStatSampler()->EvaluateTestStatistic(*toyData,tmpPOI);
// toyData->get()->Print("v");
// cout <<"obsTSatObsUL " <<obsTSatObsUL << "toyTS " << toyTSatObsUL << endl;
if(obsTSatObsUL < toyTSatObsUL) // (should be checked)
CLb+= (1.)/nToyMC;
if(obsTSatObsUL <= toyTSatObsUL) // (should be checked)
CLbinclusive+= (1.)/nToyMC;
// loop over points in belt to find upper limit for this toy data
double thisUL = 0;
for(Int_t i=0; i<parameterScan->numEntries(); ++i){
tmpPoint = (RooArgSet*) parameterScan->get(i)->clone("temp");
double arMax = belt->GetAcceptanceRegionMax(*tmpPoint);
firstPOI->setVal( tmpPoint->getRealValue(firstPOI->GetName()) );
double thisTS = fc.GetTestStatSampler()->EvaluateTestStatistic(*toyData,tmpPOI);
if(thisTS<=arMax){
thisUL = firstPOI->getVal();
} else{
break;
}
}
histOfUL->Fill(thisUL);
delete toyData;
}
histOfUL->Draw();
c1->SaveAs("one-sided_upper_limit_output.pdf");
// if you want to see a plot of the sampling distribution for a particular scan point:
// Now find bands and power constraint
Double_t* bins = histOfUL->GetIntegral();
TH1F* cumulative = (TH1F*) histOfUL->Clone("cumulative");
cumulative->SetContent(bins);
double band2sigDown=0, band1sigDown=0, bandMedian=0, band1sigUp=0,band2sigUp=0;
for(int i=1; i<=cumulative->GetNbinsX(); ++i){
if(bins[i]<RooStats::SignificanceToPValue(2))
band2sigDown=cumulative->GetBinCenter(i);
if(bins[i]<RooStats::SignificanceToPValue(1))
band1sigDown=cumulative->GetBinCenter(i);
if(bins[i]<0.5)
bandMedian=cumulative->GetBinCenter(i);
if(bins[i]<RooStats::SignificanceToPValue(-1))
band1sigUp=cumulative->GetBinCenter(i);
if(bins[i]<RooStats::SignificanceToPValue(-2))
band2sigUp=cumulative->GetBinCenter(i);
}
t.Stop();
t.Print();
cout << "-2 sigma band " << band2sigDown << endl;
cout << "-1 sigma band " << band1sigDown << endl;
cout << "median of band " << bandMedian << " [Power Constriant)]" << endl;
cout << "+1 sigma band " << band1sigUp << endl;
cout << "+2 sigma band " << band2sigUp << endl;
// print out the iterval on the first Parameter of Interest
cout << "\nobserved 95% upper-limit "<< interval->UpperLimit(*firstPOI) <<endl;
cout << "CLb strict [P(toy>obs|0)] for observed 95% upper-limit "<< CLb <<endl;
cout << "CLb inclusive [P(toy>=obs|0)] for observed 95% upper-limit "<< CLbinclusive <<endl;
delete profile;
delete nll;
}
result fit_toy(RooWorkspace* wspace, int n, const RooArgSet* globals) {
RooRandom::randomGenerator()->SetSeed(0);
// TFile f(filename);
// RooWorkspace *wspace = (RooWorkspace*)f.Get("combined");
ModelConfig* model = (ModelConfig*)wspace->obj("ModelConfig");
RooAbsPdf* pdf;
pdf = model->GetPdf();
RooAbsPdf* top_constraint = (RooAbsPdf*)wspace->obj("top_ratio_constraint");
RooAbsPdf* vv_constraint = (RooAbsPdf*)wspace->obj("vv_ratio_constraint");
RooAbsPdf* top_vv_constraint_sf = (RooAbsPdf*)wspace->obj("top_vv_ratio_sf_constraint");
RooAbsPdf* top_vv_constraint_of = (RooAbsPdf*)wspace->obj("top_vv_ratio_of_constraint");
// generate constraint global observables
RooRealVar *nom_top_ratio = (RooRealVar*)wspace->obj("nom_top_ratio");
nom_top_ratio->setRange(0, 100);
RooRealVar *nom_vv_ratio = (RooRealVar*)wspace->obj("nom_vv_ratio");
nom_vv_ratio->setRange(0,100);
RooRealVar *nom_top_vv_ratio_sf = (RooRealVar*)wspace->obj("nom_top_vv_ratio_sf");
nom_top_vv_ratio_sf->setRange(0,100);
RooRealVar *nom_top_vv_ratio_of = (RooRealVar*)wspace->obj("nom_top_vv_ratio_of");
nom_top_vv_ratio_of->setRange(0,100);
RooDataSet *nom_top_generated = top_constraint->generateSimGlobal(RooArgSet(*nom_top_ratio), 1);
nom_top_ratio->setVal(((RooRealVar*)nom_top_generated->get(0)->find("nom_top_ratio"))->getVal());
RooDataSet *nom_vv_generated = vv_constraint->generateSimGlobal(RooArgSet(*nom_vv_ratio), 1);
nom_vv_ratio->setVal(((RooRealVar*)nom_vv_generated->get(0)->find("nom_vv_ratio"))->getVal());
RooDataSet *nom_top_vv_sf_generated = top_vv_constraint_sf->generateSimGlobal(RooArgSet(*nom_top_vv_ratio_sf), 1);
nom_top_vv_ratio_sf->setVal(((RooRealVar*)nom_top_vv_sf_generated->get(0)->find("nom_top_vv_ratio_sf"))->getVal());
RooDataSet *nom_top_vv_of_generated = top_vv_constraint_of->generateSimGlobal(RooArgSet(*nom_top_vv_ratio_of), 1);
nom_top_vv_ratio_of->setVal(((RooRealVar*)nom_top_vv_of_generated->get(0)->find("nom_top_vv_ratio_of"))->getVal());
NumEventsTestStat* dummy = new NumEventsTestStat(*pdf);
ToyMCSampler* mc = new ToyMCSampler(*dummy, 1);
mc->SetPdf(*pdf);
mc->SetObservables(*model->GetObservables());
mc->SetGlobalObservables(*globals);
mc->SetNuisanceParameters(*model->GetNuisanceParameters());
mc->SetParametersForTestStat(*model->GetParametersOfInterest());
mc->SetNEventsPerToy(n);
RooArgSet constr;
constr.add(*(model->GetNuisanceParameters()));
RemoveConstantParameters(&constr);
RooDataSet* toy_data = (RooDataSet*)mc->GenerateToyData(*const_cast<RooArgSet*>(model->GetSnapshot()));
RooFitResult *res = pdf->fitTo(*toy_data, Constrain(constr), PrintLevel(0), Save(),
Range("fitRange"), InitialHesse(),
ExternalConstraints(RooArgSet(*top_constraint, *vv_constraint, *top_vv_constraint_sf, *top_vv_constraint_of)));
result yield = get_results(wspace, res);
yield.of.generated_sum.val = toy_data->sumEntries("(channelCat==channelCat::of) & (obs_x_of>120)");
yield.sf.generated_sum.val = toy_data->sumEntries("(channelCat==channelCat::sf) & (obs_x_sf>120)");
delete mc;
delete dummy;
// f.Close();
return yield;
}
