int main(int argc, char** argv) { vflg = false; uint32_t tlvl = 0; bool dflg = false; int c; int32_t sid = -1; bool Bflg = false; opterr = 0; while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF) switch (c) { case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break; case 'v': vflg = true; break; case 'd': dflg = true; break; case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break; case 'B': Bflg = true; break; case 'h': case '?': default: usage(); return (c == 'h' ? 0 : 1); break; } if (dflg) vflg = true; if ((argc - optind) < 1) { usage(); return 1; } ifstream inputf; ByteStream bs; ByteStream dbs; ByteStream eoq; ByteStream tbs; ByteStream statsStream; ByteStream::quadbyte q = 0; eoq << q; uint32_t sessionid; time_t t; SJLP jl; DeliveredTableMap tm; DeliveredTableMap::iterator iter; DeliveredTableMap::iterator end; CalpontSelectExecutionPlan csep; struct timeval start_time; struct timeval end_time; MessageQueueClient* mqc = 0; if (!dflg) mqc = new MessageQueueClient("ExeMgr1"); if (sid == -1) { time(&t); sessionid = static_cast<uint32_t>(t); } else { sessionid = static_cast<uint32_t>(sid); } sessionid &= 0x7fffffff; logging::ErrorCodes errorCodes; for ( ; optind < argc; optind++) { inputf.open(argv[optind]); if (!inputf.good()) { cerr << "error opening plan stream " << argv[optind] << endl; return 1; } bs.reset(); inputf >> bs; inputf.close(); csep.unserialize(bs); csep.sessionID(sessionid); SessionManager sm; csep.verID(sm.verID()); csep.traceFlags(0); ResourceManager rm; jl = JobListFactory::makeJobList(&csep, rm); csep.traceFlags(tlvl); if (vflg) { if (dflg) cout << endl << "Query:" << endl; else { cout << endl << "Session: " << sessionid << ", Sending Query"; if (Bflg) cout << " (" << argv[optind] << ')'; cout << ':' << endl; } if (!Bflg) cout << csep.data() << endl << endl; } if (dflg) continue; try { dbs.reset(); csep.serialize(dbs); gettimeofday(&start_time, 0); //try tuples first, but expect the worst... bool expectTuples = false; ByteStream tbs; ByteStream::quadbyte tqb = 4; tbs << tqb; mqc->write(tbs); //send the CSEP mqc->write(dbs); //read the response to the tuple request tbs = mqc->read(); idbassert(tbs.length() == 4); tbs >> tqb; if (tqb == 4) expectTuples = true; if (!expectTuples) cout << "Using TableBand I/F" << endl; else cout << "Using tuple I/F" << endl; tm = jl->deliveredTables(); iter = tm.begin(); end = tm.end(); OID toid; uint64_t rowTot; bool reported = false; bool needRGCtor = true; while (iter != end) { toid = iter->first; q = static_cast<ByteStream::quadbyte>(toid); tbs.reset(); tbs << q; mqc->write(tbs); ByteStream tbbs; TableBand tb; RowGroup rg; rowTot = 0; uint16_t status = 0; TableBand::VBA::size_type rc; ofstream out; for (;;) { tbbs = mqc->read(); #if 0 cout << tbbs.length() << endl; out.open("bs1.dat"); idbassert(out.good()); out << tbbs; out.close(); tbbs = mqc->read(); cout << tbbs.length() << endl; out.open("bs2.dat"); idbassert(out.good()); out << tbbs; out.close(); tbbs = mqc->read(); cout << tbbs.length() << endl; out.open("bs3.dat"); idbassert(out.good()); out << tbbs; out.close(); #endif if(tbbs.length()) { if (!expectTuples) tb.unserialize(tbbs); else { if (needRGCtor) { rg.deserialize(tbbs); needRGCtor = false; tbbs = mqc->read(); } rg.setData((uint8_t*)tbbs.buf()); } } else { //@bug 1346 if (!status) status = logging::makeJobListErr; break; } if (!expectTuples) { rc = tb.getRowCount(); status = tb.getStatus(); } else { rc = rg.getRowCount(); status = rg.getStatus(); if (rc == 0) status = 0; } if (rc == 0) break; rowTot += rc; } BatchPrimitive* step = dynamic_cast<BatchPrimitive*>( iter->second.get() ); if (vflg && step) { cout << "For table " << step->tableName(); if (!Bflg) cout << " " << toid; cout << ": read " << rowTot << " rows" << endl; } if (status && !reported) { cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n"; reported = true; } if (!step && !reported) { cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n"; reported = true; } ++iter; } if (vflg) { gettimeofday(&end_time, 0); cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs" << endl; //...Ask for query stats through special table id of 3 const OID TABLE_ID_TO_GET_QUERY_STATS = 3; if (!Bflg) cout << "Retrieving stats..." << endl; toid = TABLE_ID_TO_GET_QUERY_STATS; q = static_cast<ByteStream::quadbyte>(toid); statsStream.reset(); statsStream << q; mqc->write(statsStream); ByteStream bs_statsString; bs_statsString = mqc->read(); string statsString; bs_statsString >> statsString; string printStatsString; struct timeval startRunTime; parseStatsString (statsString, printStatsString, startRunTime); cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs" << endl; } //...Close this query/session mqc->write(eoq); jl.reset(); } catch(const exception& ex) { cout << "### SendPlan caught an exception: " << ex.what() << endl; } } // jl.reset(); CalpontSystemCatalog::removeCalpontSystemCatalog( sessionid ); config::Config::deleteInstanceMap(); delete mqc; return 0; }
bool DDLIndexPopulator::makeIndexStructs( ) { erydbSelectExecutionPlan csep; makeCsep(csep); ResourceManager rm; if (! fEC) { fEC = DistributedEngineComm::instance(rm); fEC->Open(); } SJLP jbl = joblist::JobListFactory::makeJobList(&csep, rm); boost::shared_ptr<erydbSystemCatalog> csc = erydbSystemCatalog::makeerydbSystemCatalog( fSessionID ); csc->identity(erydbSystemCatalog::EC); jbl->putEngineComm(fEC); /* ResultManager * result = jbl->GetResultManager(); result->setRunning(1); jbl->Execute(); */ jbl->doQuery(); erydbSystemCatalog::TableName tableName; tableName.schema = fTable.fSchema; tableName.table = fTable.fName; erydbSystemCatalog::OID tableOid = (csc->tableRID ( tableName )).objnum; erydbSystemCatalog::NJLSysDataList sysDataList; for (;;) { TableBand band; band = jbl->projectTable(tableOid); if (band.getRowCount() == 0) { // No more bands, table is done break; } band.convertToSysDataList(sysDataList, csc); break; } //size_t cnt = fColNames.size(); size_t i = 0; vector<ColumnResult*>::const_iterator it; vector<int>::const_iterator oid_iter; for (it = sysDataList.begin(); it != sysDataList.end(); it++) { if (isUnique()) fUniqueColResultList.push_back(*it); for ( oid_iter = fOidList.begin(); oid_iter != fOidList.end(); oid_iter++ ) { if ( (*it)->ColumnOID() == *oid_iter ) { erydbSystemCatalog::ColType coltype = makeIdxStruct(*it, fColNames.size(), csc); addColumnData(*it, coltype, i); } } i++; } return (fIdxValueList.size() && NO_ERROR == fResult.result ); }
//------------------------------------------------------------------------------ // Perform column projection and serialization for the given delivery step //------------------------------------------------------------------------------ void runStep(DeliveryStep& dstep, const string& message) { string nextBandMsg (message ); nextBandMsg += " - nextBand()"; string serializeMsg (message ); serializeMsg += " - serialize()"; int nextBandCount = 0; ByteStream bs; TableBand tb; //...Perform table band projection and serialization in succession #if 0 while (1) { // timer.start(nextBandMsg); tb = dstep.nextBand(); nextBandCount++; // timer.stop (nextBandMsg); // timer.start(serializeMsg); bs.reset(); tb.serialize(bs); // timer.stop( serializeMsg); if (tb.getRowCount() == 0) break; } //...Perform table band projection and serialization in parallel #else string thrCreateMsg (message ); thrCreateMsg += " - serialize-thrCreate"; string thrJoinMsg (message ); thrJoinMsg += " - serialize-thrJoin"; string serializeWaitMsg(message ); serializeWaitMsg += " - serialize-Wait"; //...Would prefer to record this label in projectThreadWrapper, but //...Stopwatch is not threadsafe, so safer to put here in main thread, //...where the other Stopwatch times are recorded. Note that this //...time will overlap the other timestamps we are recording. // timer.start(nextBandMsg); //...Start a second thread that will allow us to perform //...table projections in parallel with band serialization // timer.start(thrCreateMsg); TableBandQueueMgr tableBandMgr(&dstep,1); pthread_t projectionThread; pthread_create(&projectionThread, 0, projectThreadWrapper, &tableBandMgr ); // timer.stop (thrCreateMsg); while (1) { //...The amount of time we spend waiting will help tell us how //...much extra time is being spent constructing the table bands // timer.start(serializeWaitMsg); boost::scoped_ptr<TableBand> band(tableBandMgr.getNextTableBand()); nextBandCount++; // timer.stop (serializeWaitMsg); // timer.start(serializeMsg); bs.reset(); band->serialize(bs); // timer.stop( serializeMsg); if (band->getRowCount() == 0) break; } // timer.stop(nextBandMsg); // timer.start(thrJoinMsg); pthread_join(projectionThread, 0); // timer.stop (thrJoinMsg); #endif cout << nextBandCount << " table bands delivered" << endl; }