int main( int argc, char** argv ){
boost::program_options::options_description desc("options");
boost::program_options::options_description visible("Usage: cmscond_list_iov [options] \n");
visible.add_options()
("connect,c",boost::program_options::value<std::string>(),"connection string(required)")
("user,u",boost::program_options::value<std::string>(),"user name (default \"\")")
("pass,p",boost::program_options::value<std::string>(),"password (default \"\")")
("authPath,P",boost::program_options::value<std::string>(),"path to authentication.xml")
("debug,d","switch on debug mode")
("help,h", "help message")
;
desc.add(visible);
boost::program_options::variables_map vm;
try{
boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(desc).run(), vm);
boost::program_options::notify(vm);
}catch(const boost::program_options::error& er) {
std::cerr << er.what()<<std::endl;
return 1;
}
if (vm.count("help")) {
std::cout << visible <<std::endl;;
return 0;
}
std::string connect;
std::string authPath("");
std::string user("");
std::string pass("");
bool debug=false;
if(!vm.count("connect")){
std::cerr <<"[Error] no connect[c] option given \n";
std::cerr<<" please do "<<argv[0]<<" --help \n";
return 1;
}else{
connect=vm["connect"].as<std::string>();
}
if(vm.count("user")){
user=vm["user"].as<std::string>();
}
if(vm.count("pass")){
pass=vm["pass"].as<std::string>();
}
if( vm.count("authPath") ){
authPath=vm["authPath"].as<std::string>();
}
if(vm.count("debug")){
debug=true;
}
cond::DbConnection connection;
if( !authPath.empty() ){
connection.configuration().setAuthenticationPath( authPath );
}else{
std::string userenv(std::string("CORAL_AUTH_USER="******"CORAL_AUTH_PASSWORD=")+pass);
::putenv(const_cast<char*>(userenv.c_str()));
::putenv(const_cast<char*>(passenv.c_str()));
}
if(debug){
connection.configuration().setMessageLevel( coral::Debug );
}else{
connection.configuration().setMessageLevel( coral::Error );
}
connection.configure();
cond::DbSession session = connection.createSession();
session.open( connect );
try{
cond::Logger mylogger(session);
mylogger.getWriteLock();
mylogger.createLogDBIfNonExist();
mylogger.releaseWriteLock();
}catch(std::exception& er){
std::cout<<er.what()<<std::endl;
}
return 0;
}
int cond::TestGTPerf::execute(){
std::string gtag = getOptionValue<std::string>("globaltag");
bool debug = hasDebug();
std::string connect = getOptionValue<std::string>("connect");
bool verbose = hasOptionValue("verbose");
int nThrF = getOptionValue<int>("n_fetch");
int nThrD = getOptionValue<int>("n_deser");
std::cout << "\n++> going to use " << nThrF << " threads for loading, " << nThrD << " threads for deserialization. \n" << std::endl;
std::string serType = "unknown";
if ( connect.find("CMS_CONDITIONS") != -1 ) {
serType = "ROOT-5";
} else if (connect.find("CMS_TEST_CONDITIONS") != -1 ) {
serType = "boost";
}
Time_t startRun= 150005;
if(hasOptionValue("start_run")) startRun = getOptionValue<Time_t>("start_run");
Time_t startTs= 5800013687234232320;
if(hasOptionValue("start_ts")) startTs = getOptionValue<Time_t>("start_ts");
Time_t startLumi= 908900979179966;
if(hasOptionValue("start_lumi")) startLumi = getOptionValue<Time_t>("start_lumi");
std::string authPath("");
if( hasOptionValue("authPath")) authPath = getOptionValue<std::string>("authPath");
initializePluginManager();
Timer timex(serType);
ConnectionPoolWrapper connPool( 1, authPath, hasDebug() );
Session session = connPool.createSession( connect );
session.transaction().start();
std::cout <<"Loading Global Tag "<<gtag<<std::endl;
GTProxy gt = session.readGlobalTag( gtag );
session.transaction().commit();
std::cout <<"Loading "<<gt.size()<<" tags..."<<std::endl;
std::vector<UntypedPayloadProxy *> proxies;
std::map<std::string,size_t> requests;
size_t nt = 0;
for( auto t: gt ){
nt++;
UntypedPayloadProxy * p = new UntypedPayloadProxy;
p->init( session );
try{
p->load( t.tagName() );
if (nThrF == 1) { // detailed info only needed in single-threaded mode to get the types/names
p->setRecordInfo( t.recordName(), t.recordLabel() );
}
proxies.push_back( p );
requests.insert( std::make_pair( t.tagName(), 0 ) );
} catch ( const cond::Exception& e ){
std::cout <<"ERROR: "<<e.what()<<std::endl;
}
}
std::cout << proxies.size() << " tags successfully loaded." << std::endl;
timex.interval("loading iovs");
Time_t run = startRun;
Time_t lumi = startLumi;
Time_t ts = startTs;
if (nThrF > 1) session.transaction().commit();
tbb::task_scheduler_init init( nThrF );
std::vector<std::shared_ptr<FetchWorker> > tasks;
std::string payloadTypeName;
for( auto p: proxies ){
payloadTypeName = p->payloadType();
// ignore problematic ones for now
if ( (payloadTypeName == "SiPixelGainCalibrationOffline") // 2 * 133 MB !!!
) {
std::cout << "WARNING: Ignoring problematic payload of type " << payloadTypeName << std::endl;
continue;
}
if (nThrF > 1) {
auto fw = std::make_shared<FetchWorker>(connPool, connect, p, (std::map<std::string,size_t> *) &requests,
run, lumi, ts);
tasks.push_back(fw);
} else {
bool loaded = false;
time::TimeType ttype = p->timeType();
auto r = requests.find( p->tag() );
try{
if( ttype==runnumber ){
p->get( run, hasDebug() );
r->second++;
} else if( ttype==lumiid ){
p->get( lumi, hasDebug() );
r->second++;
} else if( ttype==timestamp){
p->get( ts, hasDebug() );
r->second++;
} else {
std::cout <<"WARNING: iov request on tag "<<p->tag()<<" (timeType="<<time::timeTypeName(p->timeType())<<") has been skipped."<<std::endl;
}
timex.fetchInt(p->getBufferSize()); // keep track of time vs. size
} catch ( const cond::Exception& e ){
std::cout <<"ERROR:"<<e.what()<<std::endl;
}
} // end else (single thread)
}
tbb::parallel_for_each(tasks.begin(),tasks.end(),invoker<std::shared_ptr<FetchWorker> >() );
std::cout << "global counter : " << fooGlobal << std::endl;
if (nThrF == 1) session.transaction().commit();
// session.transaction().commit();
timex.interval("loading payloads");
size_t totBufSize = 0;
for( auto p: proxies ){
totBufSize += p->getBufferSize();
}
std::cout << "++> total buffer size used : " << totBufSize << std::endl;
std::vector<std::shared_ptr<void> > payloads;
payloads.resize(400); //-todo: check we don't have more payloads than that !!
std::shared_ptr<void> payloadPtr;
tbb::task_scheduler_init initD( nThrD );
std::vector<std::shared_ptr<DeserialWorker> > tasksD;
timex.interval("setup deserialization");
int nEmpty = 0;
int nBig = 0;
int index = 0;
for( auto p: proxies ){
/// if ( p->getBufferSize() == 0 ) { // nothing to do for these ...
/// std::cout << "empty buffer found for " << p->payloadType() << std::endl;
/// nEmpty++;
/// continue;
/// }
payloadTypeName = p->payloadType();
// ignore problematic ones for now
if ( (payloadTypeName == "SiPixelGainCalibrationForHLT")
or (payloadTypeName == "SiPixelGainCalibrationOffline") // 2 * 133 MB !!!
or (payloadTypeName == "DTKeyedConfig")
or (payloadTypeName == "std::vector<unsigned long long>")
or (payloadTypeName == " AlignmentSurfaceDeformations")
// only in root for now:
or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer")
or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer")
or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer")
or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer")
) {
std::cout << "INFO: Ignoring payload of type " << payloadTypeName << std::endl;
continue;
}
if (nThrD > 1) {
auto dw = std::make_shared<DeserialWorker>(p, payloads[index]);
tasksD.push_back(dw);
} else { // single tread only
try {
std::pair<std::string, std::shared_ptr<void> > result = fetchOne( payloadTypeName, p->getBuffer(), p->getStreamerInfo(), payloadPtr);
payloads.push_back(result.second);
} catch ( const cond::Exception& e ){
std::cout << "\nERROR (cond): " << e.what() << std::endl;
std::cout << "for payload type name: " << payloadTypeName << std::endl;
} catch ( const std::exception& e ){
std::cout << "\nERROR (boost/std): " << e.what() << std::endl;
std::cout << "for payload type name: " << payloadTypeName << std::endl;
}
timex.deserInt(p->getBufferSize()); // keep track of time vs. size
} // single-thread
index++; // increment index into payloads
}
std::cout << std::endl;
tbb::parallel_for_each(tasksD.begin(),tasksD.end(),invoker<std::shared_ptr<DeserialWorker> >() );
timex.interval("deserializing payloads");
std::cout << "global counter : " << fooGlobal << std::endl;
std::cout << "found " << nEmpty << " empty payloads while deserialising " << std::endl;
std::cout <<std::endl;
std::cout <<"*** End of job."<<std::endl;
std::cout <<"*** GT: "<<gtag<<" Tags:"<<gt.size()<<" Loaded:"<<proxies.size()<<std::endl;
std::cout<<std::endl;
for( auto p: proxies ){
auto r = requests.find( p->tag() );
if( verbose ){
std::cout <<"*** Tag: "<<p->tag()<<" Requests processed:"<<r->second<<" Queries:"<< p->numberOfQueries() <<std::endl;
const std::vector<std::string>& hist = p->history();
for( auto e: p->history() ) std::cout <<" "<<e<<std::endl;
}
}
// only for igprof checking of live mem:
// ::exit(0);
timex.interval("postprocessing ... ");
timex.showIntervals();
if ( nThrF == 1) {
std::ofstream ofs("fetchInfo.txt");
timex.showFetchInfo(ofs);
std::ofstream ofs2("sizeInfo.txt");
for ( auto p: proxies ) {
ofs2 << p->payloadType() << "[" << p->recName() << ":" << p->recLabel() << "]" << " : " << p->getBufferSize() << std::endl;
}
}
if ( nThrD == 1) {
std::ofstream ofs1("deserializeInfo.txt");
timex.showDeserInfo(ofs1);
}
return 0;
}
