bool JoinPartition::getNextPartition(vector<RGData> *smallData, uint64_t *partitionID, JoinPartition **jp)
{
if (fileMode) {
ByteStream bs;
RGData rgData;
if (nextPartitionToReturn > 0)
return false;
//cout << "reading the small side" << endl;
nextSmallOffset = 0;
while (1) {
readByteStream(0, &bs);
if (bs.length() == 0)
break;
rgData.deserialize(bs);
//smallRG.setData(&rgData);
//cout << "read a smallRG with " << smallRG.getRowCount() << " rows" << endl;
smallData->push_back(rgData);
}
nextPartitionToReturn = 1;
*partitionID = uniqueID;
*jp = this;
return true;
}
bool ret = false;
while (!ret && nextPartitionToReturn < bucketCount) {
ret = buckets[nextPartitionToReturn]->getNextPartition(smallData, partitionID, jp);
if (!ret)
nextPartitionToReturn++;
}
return ret;
}
void TupleHavingStep::execute()
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
dlTimes.setFirstReadTime();
if (!more && cancelled())
{
fEndOfResult = true;
}
while (more && !fEndOfResult)
{
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
doHavingFilters();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (cancelled())
{
fEndOfResult = true;
}
else
{
fOutputDL->insert(rgDataOut);
}
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
}
catch(...)
{
catchHandler("TupleHavingStep execute caught an unknown exception",
tupleHavingStepErr, fErrorInfo, fSessionId);
}
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
fEndOfResult = true;
fOutputDL->endOfInput();
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
if (traceOn())
printCalTrace();
}
uint TupleConstantOnlyStep::nextBand(messageqcpp::ByteStream &bs)
{
RGData rgDataOut;
uint rowCount = 0;
if (!fEndOfResult)
{
try
{
bs.restart();
if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0)
dlTimes.setFirstReadTime();
rgDataOut.reinit(fRowGroupOut, 1);
fRowGroupOut.setData(&rgDataOut);
fillInConstants();
fRowGroupOut.serializeRGData(bs);
rowCount = fRowGroupOut.getRowCount();
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(tupleConstantStepErr);
}
catch(...)
{
catchHandler("TupleConstantStep next band caught an unknown exception", fSessionId);
if (status() == 0)
status(tupleConstantStepErr);
}
fEndOfResult = true;
}
else
{
// send an empty / error band
RGData rgData(fRowGroupOut, 0);
fRowGroupOut.setData(&rgData);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.setStatus(status());
fRowGroupOut.serializeRGData(bs);
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
}
return rowCount;
}
inline void CrossEngineStep::addRow(RGData &data)
{
fRowDelivered.setRid(fRowsReturned%fRowsPerGroup);
fRowDelivered.nextRow();
fRowGroupDelivered.incRowCount();
if (++fRowsReturned%fRowsPerGroup == 0)
{
fOutputDL->insert(data);
data.reinit(fRowGroupDelivered, fRowsPerGroup);
fRowGroupDelivered.setData(&data);
fRowGroupDelivered.resetRowGroup(fRowsReturned);
fRowGroupDelivered.getRow(0, &fRowDelivered);
}
}
void SubAdapterStep::execute()
{
RGData rgDataIn;
RGData rgDataOut;
Row rowIn;
Row rowFe;
Row rowOut;
fRowGroupIn.initRow(&rowIn);
fRowGroupOut.initRow(&rowOut);
RGData rowFeData;
bool usesFE = false;
if (fRowGroupFe.getColumnCount() != (uint32_t) -1)
{
usesFE = true;
fRowGroupFe.initRow(&rowFe, true);
rowFeData = RGData(fRowGroupFe, 1);
fRowGroupFe.setData(&rowFeData);
fRowGroupFe.getRow(0, &rowFe);
}
bool more = false;
try
{
fSubStep->run();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
while (more && !cancelled())
{
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
fRowGroupIn.getRow(0, &rowIn);
fRowGroupOut.getRow(0, &rowOut);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
{
if(fExpression.get() == NULL)
{
outputRow(rowIn, rowOut);
}
else if (!usesFE)
{
if(fExpression->evaluate(&rowIn))
{
outputRow(rowIn, rowOut);
}
}
else
{
copyRow(rowIn, &rowFe, rowIn.getColumnCount());
//memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize());
if(fExpression->evaluate(&rowFe))
{
outputRow(rowFe, rowOut);
}
}
rowIn.nextRow();
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
}
more = fInputDL->next(fInputIterator, &rgDataIn);
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
}
catch(...)
{
catchHandler("SubAdapterStep execute caught an unknown exception",
ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
}
if (cancelled())
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
// Bug 3136, let mini stats to be formatted if traceOn.
fOutputDL->endOfInput();
}
void TupleHavingStep::execute()
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
StepTeleStats sts;
sts.query_uuid = fQueryUuid;
sts.step_uuid = fStepUuid;
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
dlTimes.setFirstReadTime();
sts.msg_type = StepTeleStats::ST_START;
sts.total_units_of_work = 1;
postStepStartTele(sts);
if (!more && cancelled())
{
fEndOfResult = true;
}
while (more && !fEndOfResult)
{
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
doHavingFilters();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (cancelled())
{
fEndOfResult = true;
}
else
{
fOutputDL->insert(rgDataOut);
}
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
}
catch(...)
{
catchHandler("TupleHavingStep execute caught an unknown exception",
tupleHavingStepErr, fErrorInfo, fSessionId);
}
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
fEndOfResult = true;
fOutputDL->endOfInput();
sts.msg_type = StepTeleStats::ST_SUMMARY;
sts.total_units_of_work = sts.units_of_work_completed = 1;
sts.rows = fRowsReturned;
postStepSummaryTele(sts);
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
if (traceOn())
printCalTrace();
}
uint32_t TupleHavingStep::nextBand(messageqcpp::ByteStream &bs)
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
uint32_t rowCount = 0;
try
{
bs.restart();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (dlTimes.FirstReadTime().tv_sec ==0)
dlTimes.setFirstReadTime();
if (!more || cancelled())
{
fEndOfResult = true;
}
bool emptyRowGroup = true;
while (more && !fEndOfResult && emptyRowGroup)
{
if (cancelled())
{
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
break;
}
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
doHavingFilters();
if (fRowGroupOut.getRowCount() > 0)
{
emptyRowGroup = false;
fRowGroupOut.serializeRGData(bs);
rowCount = fRowGroupOut.getRowCount();
}
else
{
more = fInputDL->next(fInputIterator, &rgDataIn);
}
}
if (!more)
{
fEndOfResult = true;
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
fEndOfResult = true;
}
catch(...)
{
catchHandler("TupleHavingStep next band caught an unknown exception",
tupleHavingStepErr, fErrorInfo, fSessionId);
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
fEndOfResult = true;
}
if (fEndOfResult)
{
// send an empty / error band
rgDataOut.reinit(fRowGroupOut, 0);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.setStatus(status());
fRowGroupOut.serializeRGData(bs);
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
if (traceOn())
printCalTrace();
}
return rowCount;
}
uint64_t UpdatePackageProcessor::fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result,
const uint64_t uniqueId, const uint32_t tableOid)
{
ByteStream msg, msgBk, emsgBs;
RGData rgData;
uint32_t qb = 4;
msg << qb;
boost::scoped_ptr<rowgroup::RowGroup> rowGroup;
uint64_t rowsProcessed = 0;
uint32_t dbroot = 1;
bool metaData = false;
oam::OamCache* oamCache = oam::OamCache::makeOamCache();
std::vector<int> fPMs = oamCache->getModuleIds();
std::map<unsigned, bool> pmState;
string emsg;
string emsgStr;
bool err = false;
//boost::scoped_ptr<messageqcpp::MessageQueueClient> fExeMgr;
//fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1"));
try
{
for (unsigned i = 0; i < fPMs.size(); i++)
{
pmState[fPMs[i]] = true;
}
//timer.start("ExeMgr");
fExeMgr->write(msg);
fExeMgr->write(*(cpackage.get_ExecutionPlan()));
//cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl;
msg.restart();
emsgBs.restart();
msg = fExeMgr->read(); //error handling
if (msg.length() == 4)
{
msg >> qb;
if (qb != 0)
err = true;
}
else
{
qb = 999;
err = true;
}
if (err)
{
logging::Message::Args args;
logging::Message message(2);
args.add("Update Failed: ExeMgr Error");
args.add((int)qb);
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
//timer.finish();
return rowsProcessed;
}
emsgBs = fExeMgr->read();
if (emsgBs.length() == 0)
{
logging::Message::Args args;
logging::Message message(2);
args.add("Update Failed: ");
args.add("Lost connection to ExeMgr");
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
//timer.finish();
return rowsProcessed;
}
emsgBs >> emsgStr;
while (true)
{
if (fRollbackPending)
{
break;
}
msg.restart();
msgBk.restart();
msg = fExeMgr->read();
msgBk = msg;
if ( msg.length() == 0 )
{
cerr << "UpdatePackageProcessor::processPackage::fixupRows" << endl;
logging::Message::Args args;
logging::Message message(2);
args.add("Update Failed: ");
args.add("Lost connection to ExeMgr");
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
//timer.finish();
//return rowsProcessed;
break;
}
else
{
if (rowGroup.get() == NULL)
{
//This is mete data, need to send all PMs.
metaData = true;
//cout << "sending meta data" << endl;
//timer.start("Meta");
err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot);
rowGroup.reset(new rowgroup::RowGroup());
rowGroup->deserialize(msg);
qb = 100;
msg.restart();
msg << qb;
fExeMgr->write(msg);
metaData = false;
//timer.stop("Meta");
continue;
}
rgData.deserialize(msg, true);
rowGroup->setData(&rgData);
//rowGroup->setData(const_cast<uint8_t*>(msg.buf()));
err = (rowGroup->getStatus() != 0);
if (err)
{
//msgBk.advance(rowGroup->getDataSize());
string errorMsg;
msg >> errorMsg;
logging::Message::Args args;
logging::Message message(2);
args.add("Update Failed: ");
args.add(errorMsg);
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
DMLResult tmpResult;
receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
/* qb = 100;
//@Bug 4358 get rid of broken pipe error.
msg.restart();
msg << qb;
fExeMgr->write(msg);
*/ //timer.finish();
//return rowsProcessed;
//err = true;
break;
}
if (rowGroup->getRGData() == NULL)
{
msg.restart();
}
if (rowGroup->getRowCount() == 0) //done fetching
{
//timer.finish();
//need to receive all response
err = receiveAll( result, uniqueId, fPMs, pmState, tableOid);
//return rowsProcessed;
break;
}
if (rowGroup->getBaseRid() == (uint64_t) (-1))
{
continue; // @bug4247, not valid row ids, may from small side outer
}
dbroot = rowGroup->getDBRoot();
//cout << "dbroot in the rowgroup is " << dbroot << endl;
//timer.start("processRowgroup");
err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot);
//timer.stop("processRowgroup");
if (err)
{
//timer.finish();
LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID());
logging::Message::Args args1;
logging::Message msg1(1);
args1.add("SQL statement erroring out, need to receive all messages from WES");
msg1.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_DEBUG, msg1, logid);
DMLResult tmpResult;
receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
logging::Message::Args args2;
logging::Message msg2(1);
args2.add("SQL statement erroring out, received all messages from WES");
msg2.format( args2 );
logger.logMessage(LOG_TYPE_DEBUG, msg2, logid);
//@Bug 4358 get rid of broken pipe error.
/* msg.restart();
msg << qb;
fExeMgr->write(msg);
return rowsProcessed;
*/
//err = true;
break;
}
rowsProcessed += rowGroup->getRowCount();
}
}
if (fRollbackPending)
{
err = true;
// Response to user
cerr << "UpdatePackageProcessor::processPackage::fixupRows Rollback Pending" << endl;
//@Bug 4994 Cancelled job is not error
result.result = JOB_CANCELED;
// Log
LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID());
logging::Message::Args args1;
logging::Message msg1(1);
args1.add("SQL statement canceled by user");
msg1.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_DEBUG, msg1, logid);
// Clean out the pipe;
DMLResult tmpResult;
receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
}
// get stats from ExeMgr
if (!err)
{
qb = 3;
msg.restart();
msg << qb;
fExeMgr->write(msg);
msg = fExeMgr->read();
msg >> result.queryStats;
msg >> result.extendedStats;
msg >> result.miniStats;
result.stats.unserialize(msg);
}
void TupleAnnexStep::executeWithOrderBy()
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
while (more && !cancelled())
{
fRowGroupIn.setData(&rgDataIn);
fRowGroupIn.getRow(0, &fRowIn);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled(); ++i)
{
fOrderBy->processRow(fRowIn);
fRowIn.nextRow();
}
more = fInputDL->next(fInputIterator, &rgDataIn);
}
fOrderBy->finalize();
if (!cancelled())
{
while (fOrderBy->getData(rgDataIn))
{
if (fConstant == NULL &&
fRowGroupOut.getColumnCount() == fRowGroupIn.getColumnCount())
{
rgDataOut = rgDataIn;
fRowGroupOut.setData(&rgDataOut);
}
else
{
fRowGroupIn.setData(&rgDataIn);
fRowGroupIn.getRow(0, &fRowIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot());
fRowGroupOut.getRow(0, &fRowOut);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
{
if (fConstant)
fConstant->fillInConstants(fRowIn, fRowOut);
else
copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow();
fRowIn.nextRow();
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
}
}
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
catch(...)
{
catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn() && !fDelivery)
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
// Bug 3136, let mini stats to be formatted if traceOn.
fOutputDL->endOfInput();
}
void TupleAnnexStep::executeNoOrderByWithDistinct()
{
scoped_ptr<DistinctMap_t> distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this)));
vector<RGData> dataVec;
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
rgDataOut.reinit(fRowGroupOut);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
fRowGroupOut.initRow(&row1);
fRowGroupOut.initRow(&row2);
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
while (more && !cancelled() && !fLimitHit)
{
fRowGroupIn.setData(&rgDataIn);
fRowGroupIn.getRow(0, &fRowIn);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i)
{
pair<DistinctMap_t::iterator, bool> inserted;
if (fConstant)
fConstant->fillInConstants(fRowIn, fRowOut);
else
copyRow(fRowIn, &fRowOut);
++fRowsProcessed;
fRowIn.nextRow();
inserted = distinctMap->insert(fRowOut.getPointer());
if (inserted.second) {
fRowGroupOut.incRowCount();
fRowOut.nextRow();
if (UNLIKELY(++fRowsReturned >= fLimitCount))
{
fLimitHit = true;
fJobList->abortOnLimit((JobStep*) this);
}
if (UNLIKELY(fRowGroupOut.getRowCount() >= 8192))
{
dataVec.push_back(rgDataOut);
rgDataOut.reinit(fRowGroupOut);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
}
}
more = fInputDL->next(fInputIterator, &rgDataIn);
}
if (fRowGroupOut.getRowCount() > 0)
dataVec.push_back(rgDataOut);
for (vector<RGData>::iterator i = dataVec.begin(); i != dataVec.end(); i++)
{
rgDataOut = *i;
fRowGroupOut.setData(&rgDataOut);
fOutputDL->insert(rgDataOut);
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
catch(...)
{
catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn() && !fDelivery)
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
// Bug 3136, let mini stats to be formatted if traceOn.
fOutputDL->endOfInput();
}
void TupleAnnexStep::executeNoOrderBy()
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
while (more && !cancelled() && !fLimitHit)
{
fRowGroupIn.setData(&rgDataIn);
fRowGroupIn.getRow(0, &fRowIn);
// Get a new output rowgroup for each input rowgroup to preserve the rids
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot());
fRowGroupOut.getRow(0, &fRowOut);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i)
{
// skip first limit-start rows
if (fRowsProcessed++ < fLimitStart)
{
fRowIn.nextRow();
continue;
}
if (fConstant)
fConstant->fillInConstants(fRowIn, fRowOut);
else
copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
if (++fRowsReturned < fLimitCount)
{
fRowOut.nextRow();
fRowIn.nextRow();
}
else
{
fLimitHit = true;
fJobList->abortOnLimit((JobStep*) this);
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fOutputDL->insert(rgDataOut);
}
more = fInputDL->next(fInputIterator, &rgDataIn);
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
catch(...)
{
catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
if (status() == 0)
status(ERR_IN_PROCESS);
}
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn() && !fDelivery)
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
// Bug 3136, let mini stats to be formatted if traceOn.
fOutputDL->endOfInput();
}
uint TupleAnnexStep::nextBand(messageqcpp::ByteStream &bs)
{
RGData rgDataOut;
bool more = false;
uint rowCount = 0;
try
{
bs.restart();
more = fOutputDL->next(fOutputIterator, &rgDataOut);
if (traceOn() && dlTimes.FirstReadTime().tv_sec ==0)
dlTimes.setFirstReadTime();
if (more && !cancelled())
{
fRowGroupDeliver.setData(&rgDataOut);
fRowGroupDeliver.serializeRGData(bs);
rowCount = fRowGroupDeliver.getRowCount();
}
else
{
if (more)
more = fOutputDL->next(fOutputIterator, &rgDataOut);
fEndOfResult = true;
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(ERR_IN_DELIVERY);
while (more)
more = fOutputDL->next(fOutputIterator, &rgDataOut);
fEndOfResult = true;
}
catch(...)
{
catchHandler("TupleAnnexStep next band caught an unknown exception", fSessionId);
if (status() == 0)
status(ERR_IN_DELIVERY);
while (more)
more = fOutputDL->next(fOutputIterator, &rgDataOut);
fEndOfResult = true;
}
if (fEndOfResult)
{
// send an empty / error band
rgDataOut.reinit(fRowGroupDeliver, 0);
fRowGroupDeliver.setData(&rgDataOut);
fRowGroupDeliver.resetRowGroup(0);
fRowGroupDeliver.setStatus(status());
fRowGroupDeliver.serializeRGData(bs);
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
}
if (traceOn())
printCalTrace();
}
return rowCount;
}
int64_t JoinPartition::convertToSplitMode()
{
int i, j;
ByteStream bs;
RGData rgData;
uint32_t hash;
uint64_t tmp;
int64_t ret = -(int64_t)smallSizeOnDisk; // smallFile gets deleted
boost::scoped_array<uint32_t> rowDist(new uint32_t[bucketCount]);
uint32_t rowCount = 0;
memset(rowDist.get(), 0, sizeof(uint32_t) * bucketCount);
fileMode = false;
htSizeEstimate = 0;
smallSizeOnDisk = 0;
buckets.reserve(bucketCount);
for (i = 0; i < (int) bucketCount; i++)
buckets.push_back(boost::shared_ptr<JoinPartition>(new JoinPartition(*this, false)));
RowGroup &rg = smallRG;
Row &row = smallRow;
nextSmallOffset = 0;
while (1) {
readByteStream(0, &bs);
if (bs.length() == 0)
break;
rgData.deserialize(bs);
rg.setData(&rgData);
for (j = 0; j < (int) rg.getRowCount(); j++) {
rg.getRow(j, &row);
if (antiWithMatchNulls && hasNullJoinColumn(row)) {
if (needsAllNullRows || !gotNullRow) {
for (j = 0; j < (int) bucketCount; j++)
ret += buckets[j]->insertSmallSideRow(row);
gotNullRow = true;
}
continue;
}
if (typelessJoin)
hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount;
else {
if (UNLIKELY(row.isUnsigned(smallKeyCols[0])))
tmp = row.getUintField(smallKeyCols[0]);
else
tmp = row.getIntField(smallKeyCols[0]);
hash = hasher((char *) &tmp, 8, hashSeed);
hash = hasher.finalize(hash, 8) % bucketCount;
}
rowCount++;
rowDist[hash]++;
ret += buckets[hash]->insertSmallSideRow(row);
}
}
boost::filesystem::remove(smallFilename);
smallFilename.clear();
for (i = 0; i < (int) bucketCount; i++)
if (rowDist[i] == rowCount)
throw IDBExcept("All rows hashed to the same bucket", ERR_DBJ_DATA_DISTRIBUTION);
rg.setData(&buffer);
rg.resetRowGroup(0);
rg.getRow(0, &row);
return ret;
}
void TupleConstantStep::execute()
{
RGData rgDataIn;
RGData rgDataOut;
bool more = false;
try
{
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
if (!more && cancelled())
{
fEndOfResult = true;
}
while (more && !fEndOfResult)
{
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
fillInConstants();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (cancelled())
{
fEndOfResult = true;
}
else
{
fOutputDL->insert(rgDataOut);
}
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), fSessionId);
if (status() == 0)
status(tupleConstantStepErr);
}
catch(...)
{
catchHandler("TupleConstantStep execute caught an unknown exception", fSessionId);
if (status() == 0)
status(tupleConstantStepErr);
}
if (!fEndOfResult)
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
// Bug 3136, let mini stats to be formatted if traceOn.
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
fEndOfResult = true;
fOutputDL->endOfInput();
}
void SubAdapterStep::execute()
{
RGData rgDataIn;
RGData rgDataOut;
Row rowIn;
Row rowFe;
Row rowOut;
fRowGroupIn.initRow(&rowIn);
fRowGroupOut.initRow(&rowOut);
RGData rowFeData;
StepTeleStats sts;
sts.query_uuid = fQueryUuid;
sts.step_uuid = fStepUuid;
bool usesFE = false;
if (fRowGroupFe.getColumnCount() > 0)
{
usesFE = true;
fRowGroupFe.initRow(&rowFe, true);
rowFeData = RGData(fRowGroupFe, 1);
fRowGroupFe.setData(&rowFeData);
fRowGroupFe.getRow(0, &rowFe);
}
bool more = false;
try
{
sts.msg_type = StepTeleStats::ST_START;
sts.total_units_of_work = 1;
postStepStartTele(sts);
fSubStep->run();
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn()) dlTimes.setFirstReadTime();
while (more && !cancelled())
{
fRowGroupIn.setData(&rgDataIn);
rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
fRowGroupIn.getRow(0, &rowIn);
fRowGroupOut.getRow(0, &rowOut);
fRowsInput += fRowGroupIn.getRowCount();
for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
{
if(fExpression.get() == NULL)
{
outputRow(rowIn, rowOut);
}
else if (!usesFE)
{
if(fExpression->evaluate(&rowIn))
{
outputRow(rowIn, rowOut);
}
}
else
{
copyRow(rowIn, &rowFe, rowIn.getColumnCount());
//memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize());
if(fExpression->evaluate(&rowFe))
{
outputRow(rowFe, rowOut);
}
}
rowIn.nextRow();
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
}
more = fInputDL->next(fInputIterator, &rgDataIn);
}
}
catch(const std::exception& ex)
{
catchHandler(ex.what(), ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
}
catch(...)
{
catchHandler("SubAdapterStep execute caught an unknown exception",
ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
}
if (cancelled())
while (more)
more = fInputDL->next(fInputIterator, &rgDataIn);
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
printCalTrace();
}
sts.msg_type = StepTeleStats::ST_SUMMARY;
sts.total_units_of_work = sts.units_of_work_completed = 1;
sts.rows = fRowsReturned;
postStepSummaryTele(sts);
// Bug 3136, let mini stats to be formatted if traceOn.
fOutputDL->endOfInput();
}