int cond::ListIOVUtilities::execute(){ initializePluginManager(); bool listAll = hasOptionValue("all"); cond::DbSession session = openDbSession( "connect", true ); if( listAll ){ cond::MetaData metadata_svc(session); std::vector<std::string> alltags; cond::DbScopedTransaction transaction(session); transaction.start(true); metadata_svc.listAllTags(alltags); transaction.commit(); std::copy (alltags.begin(), alltags.end(), std::ostream_iterator<std::string>(std::cout,"\n") ); }else{ std::string tag = getOptionValue<std::string>("tag"); cond::MetaData metadata_svc(session); std::string token; cond::DbScopedTransaction transaction(session); transaction.start(true); token=metadata_svc.getToken(tag); transaction.commit(); { bool verbose = hasOptionValue("verbose"); bool details = hasOptionValue("summary"); TFile * xml=0; if (details) { xml = TFile::Open(std::string(tag+".xml").c_str(),"recreate"); } cond::IOVProxy iov( session, token, !details, details); unsigned int counter=0; std::string payloadContainer=iov.payloadContainerName(); std::cout<<"Tag "<<tag; if (verbose) std::cout << "\nStamp: " << iov.iov().comment() << "; time " << cond::time::to_boost(iov.iov().timestamp()) << "; revision " << iov.iov().revision(); std::cout <<"\nTimeType " << cond::timeTypeSpecs[iov.timetype()].name <<"\nPayloadContainerName "<<payloadContainer<<"\n" <<"since \t till \t payloadToken"<<std::endl; for (cond::IOVProxy::const_iterator ioviterator=iov.begin(); ioviterator!=iov.end(); ioviterator++) { std::cout<<ioviterator->since() << " \t "<<ioviterator->till() <<" \t "<<ioviterator->wrapperToken(); if (details) { ora::Object obj = session.getObject(ioviterator->wrapperToken()); std::ostringstream ss; ss << tag << '_' << ioviterator->since(); xml->WriteObjectAny(obj.address(),obj.typeName().c_str(), ss.str().c_str()); obj.destruct(); } std::cout<<std::endl; ++counter; } if (xml) xml->Close(); std::cout<<"Total # of payload objects: "<<counter<<std::endl; } } return 0; }
int cond::TestGTPerf::execute(){ std::string gtag = getOptionValue<std::string>("globaltag"); bool debug = hasDebug(); std::string connect = getOptionValue<std::string>("connect"); bool verbose = hasOptionValue("verbose"); int nThrF = getOptionValue<int>("n_fetch"); int nThrD = getOptionValue<int>("n_deser"); std::cout << "\n++> going to use " << nThrF << " threads for loading, " << nThrD << " threads for deserialization. \n" << std::endl; std::string serType = "unknown"; if ( connect.find("CMS_CONDITIONS") != -1 ) { serType = "ROOT-5"; } else if (connect.find("CMS_TEST_CONDITIONS") != -1 ) { serType = "boost"; } Time_t startRun= 150005; if(hasOptionValue("start_run")) startRun = getOptionValue<Time_t>("start_run"); Time_t startTs= 5800013687234232320; if(hasOptionValue("start_ts")) startTs = getOptionValue<Time_t>("start_ts"); Time_t startLumi= 908900979179966; if(hasOptionValue("start_lumi")) startLumi = getOptionValue<Time_t>("start_lumi"); std::string authPath(""); if( hasOptionValue("authPath")) authPath = getOptionValue<std::string>("authPath"); initializePluginManager(); Timer timex(serType); ConnectionPoolWrapper connPool( 1, authPath, hasDebug() ); Session session = connPool.createSession( connect ); session.transaction().start(); std::cout <<"Loading Global Tag "<<gtag<<std::endl; GTProxy gt = session.readGlobalTag( gtag ); session.transaction().commit(); std::cout <<"Loading "<<gt.size()<<" tags..."<<std::endl; std::vector<UntypedPayloadProxy *> proxies; std::map<std::string,size_t> requests; size_t nt = 0; for( auto t: gt ){ nt++; UntypedPayloadProxy * p = new UntypedPayloadProxy; p->init( session ); try{ p->load( t.tagName() ); if (nThrF == 1) { // detailed info only needed in single-threaded mode to get the types/names p->setRecordInfo( t.recordName(), t.recordLabel() ); } proxies.push_back( p ); requests.insert( std::make_pair( t.tagName(), 0 ) ); } catch ( const cond::Exception& e ){ std::cout <<"ERROR: "<<e.what()<<std::endl; } } std::cout << proxies.size() << " tags successfully loaded." << std::endl; timex.interval("loading iovs"); Time_t run = startRun; Time_t lumi = startLumi; Time_t ts = startTs; if (nThrF > 1) session.transaction().commit(); tbb::task_scheduler_init init( nThrF ); std::vector<std::shared_ptr<FetchWorker> > tasks; std::string payloadTypeName; for( auto p: proxies ){ payloadTypeName = p->payloadType(); // ignore problematic ones for now if ( (payloadTypeName == "SiPixelGainCalibrationOffline") // 2 * 133 MB !!! ) { std::cout << "WARNING: Ignoring problematic payload of type " << payloadTypeName << std::endl; continue; } if (nThrF > 1) { auto fw = std::make_shared<FetchWorker>(connPool, connect, p, (std::map<std::string,size_t> *) &requests, run, lumi, ts); tasks.push_back(fw); } else { bool loaded = false; time::TimeType ttype = p->timeType(); auto r = requests.find( p->tag() ); try{ if( ttype==runnumber ){ p->get( run, hasDebug() ); r->second++; } else if( ttype==lumiid ){ p->get( lumi, hasDebug() ); r->second++; } else if( ttype==timestamp){ p->get( ts, hasDebug() ); r->second++; } else { std::cout <<"WARNING: iov request on tag "<<p->tag()<<" (timeType="<<time::timeTypeName(p->timeType())<<") has been skipped."<<std::endl; } timex.fetchInt(p->getBufferSize()); // keep track of time vs. size } catch ( const cond::Exception& e ){ std::cout <<"ERROR:"<<e.what()<<std::endl; } } // end else (single thread) } tbb::parallel_for_each(tasks.begin(),tasks.end(),invoker<std::shared_ptr<FetchWorker> >() ); std::cout << "global counter : " << fooGlobal << std::endl; if (nThrF == 1) session.transaction().commit(); // session.transaction().commit(); timex.interval("loading payloads"); size_t totBufSize = 0; for( auto p: proxies ){ totBufSize += p->getBufferSize(); } std::cout << "++> total buffer size used : " << totBufSize << std::endl; std::vector<std::shared_ptr<void> > payloads; payloads.resize(400); //-todo: check we don't have more payloads than that !! std::shared_ptr<void> payloadPtr; tbb::task_scheduler_init initD( nThrD ); std::vector<std::shared_ptr<DeserialWorker> > tasksD; timex.interval("setup deserialization"); int nEmpty = 0; int nBig = 0; int index = 0; for( auto p: proxies ){ /// if ( p->getBufferSize() == 0 ) { // nothing to do for these ... /// std::cout << "empty buffer found for " << p->payloadType() << std::endl; /// nEmpty++; /// continue; /// } payloadTypeName = p->payloadType(); // ignore problematic ones for now if ( (payloadTypeName == "SiPixelGainCalibrationForHLT") or (payloadTypeName == "SiPixelGainCalibrationOffline") // 2 * 133 MB !!! or (payloadTypeName == "DTKeyedConfig") or (payloadTypeName == "std::vector<unsigned long long>") or (payloadTypeName == " AlignmentSurfaceDeformations") // only in root for now: or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer") or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer") or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer") or (payloadTypeName == "PhysicsTools::Calibration::MVAComputerContainer") ) { std::cout << "INFO: Ignoring payload of type " << payloadTypeName << std::endl; continue; } if (nThrD > 1) { auto dw = std::make_shared<DeserialWorker>(p, payloads[index]); tasksD.push_back(dw); } else { // single tread only try { std::pair<std::string, std::shared_ptr<void> > result = fetchOne( payloadTypeName, p->getBuffer(), p->getStreamerInfo(), payloadPtr); payloads.push_back(result.second); } catch ( const cond::Exception& e ){ std::cout << "\nERROR (cond): " << e.what() << std::endl; std::cout << "for payload type name: " << payloadTypeName << std::endl; } catch ( const std::exception& e ){ std::cout << "\nERROR (boost/std): " << e.what() << std::endl; std::cout << "for payload type name: " << payloadTypeName << std::endl; } timex.deserInt(p->getBufferSize()); // keep track of time vs. size } // single-thread index++; // increment index into payloads } std::cout << std::endl; tbb::parallel_for_each(tasksD.begin(),tasksD.end(),invoker<std::shared_ptr<DeserialWorker> >() ); timex.interval("deserializing payloads"); std::cout << "global counter : " << fooGlobal << std::endl; std::cout << "found " << nEmpty << " empty payloads while deserialising " << std::endl; std::cout <<std::endl; std::cout <<"*** End of job."<<std::endl; std::cout <<"*** GT: "<<gtag<<" Tags:"<<gt.size()<<" Loaded:"<<proxies.size()<<std::endl; std::cout<<std::endl; for( auto p: proxies ){ auto r = requests.find( p->tag() ); if( verbose ){ std::cout <<"*** Tag: "<<p->tag()<<" Requests processed:"<<r->second<<" Queries:"<< p->numberOfQueries() <<std::endl; const std::vector<std::string>& hist = p->history(); for( auto e: p->history() ) std::cout <<" "<<e<<std::endl; } } // only for igprof checking of live mem: // ::exit(0); timex.interval("postprocessing ... "); timex.showIntervals(); if ( nThrF == 1) { std::ofstream ofs("fetchInfo.txt"); timex.showFetchInfo(ofs); std::ofstream ofs2("sizeInfo.txt"); for ( auto p: proxies ) { ofs2 << p->payloadType() << "[" << p->recName() << ":" << p->recLabel() << "]" << " : " << p->getBufferSize() << std::endl; } } if ( nThrD == 1) { std::ofstream ofs1("deserializeInfo.txt"); timex.showDeserInfo(ofs1); } return 0; }