Exemplo n.º 1
0
bool JoinPartition::getNextPartition(vector<RGData> *smallData, uint64_t *partitionID, JoinPartition **jp)
{

	if (fileMode) {
		ByteStream bs;
		RGData rgData;

		if (nextPartitionToReturn > 0)
			return false;

		//cout << "reading the small side" << endl;
		nextSmallOffset = 0;
		while (1) {
			readByteStream(0, &bs);
			if (bs.length() == 0)
				break;
			rgData.deserialize(bs);
			//smallRG.setData(&rgData);
			//cout << "read a smallRG with " << smallRG.getRowCount() << " rows" << endl;
			smallData->push_back(rgData);
		}
		nextPartitionToReturn = 1;
		*partitionID = uniqueID;
		*jp = this;
		return true;
	}

	bool ret = false;
	while (!ret && nextPartitionToReturn < bucketCount) {
		ret = buckets[nextPartitionToReturn]->getNextPartition(smallData, partitionID, jp);
		if (!ret)
			nextPartitionToReturn++;
	}
	return ret;
}
Exemplo n.º 2
0
void TupleHavingStep::execute()
{
	RGData rgDataIn;
	RGData rgDataOut;
	bool more = false;

	try
	{
		more = fInputDL->next(fInputIterator, &rgDataIn);
		dlTimes.setFirstReadTime();

		if (!more && cancelled())
		{
			fEndOfResult = true;
		}

		while (more && !fEndOfResult)
		{
			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);

			doHavingFilters();

			more = fInputDL->next(fInputIterator, &rgDataIn);
			if (cancelled())
			{
				fEndOfResult = true;
			}
			else
			{
				fOutputDL->insert(rgDataOut);
			}
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
	}
	catch(...)
	{
		catchHandler("TupleHavingStep execute caught an unknown exception",
					 tupleHavingStepErr, fErrorInfo, fSessionId);
	}

	while (more)
		more = fInputDL->next(fInputIterator, &rgDataIn);

	fEndOfResult = true;
	fOutputDL->endOfInput();

	dlTimes.setLastReadTime();
	dlTimes.setEndOfInputTime();

	if (traceOn())
		printCalTrace();
}
Exemplo n.º 3
0
uint TupleConstantOnlyStep::nextBand(messageqcpp::ByteStream &bs)
{
	RGData rgDataOut;
	uint rowCount = 0;

	if (!fEndOfResult)
	{
		try
		{
			bs.restart();

			if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0)
				dlTimes.setFirstReadTime();

			rgDataOut.reinit(fRowGroupOut, 1);
			fRowGroupOut.setData(&rgDataOut);

			fillInConstants();
			fRowGroupOut.serializeRGData(bs);
			rowCount = fRowGroupOut.getRowCount();
		}
		catch(const std::exception& ex)
		{
			catchHandler(ex.what(), fSessionId);
			if (status() == 0)
				status(tupleConstantStepErr);
		}
		catch(...)
		{
			catchHandler("TupleConstantStep next band caught an unknown exception", fSessionId);
			if (status() == 0)
				status(tupleConstantStepErr);
		}

		fEndOfResult = true;
	}
	else
	{
		// send an empty / error band
		RGData rgData(fRowGroupOut, 0);
		fRowGroupOut.setData(&rgData);
		fRowGroupOut.resetRowGroup(0);
		fRowGroupOut.setStatus(status());
		fRowGroupOut.serializeRGData(bs);

		if (traceOn())
		{
			dlTimes.setLastReadTime();
			dlTimes.setEndOfInputTime();
			printCalTrace();
		}
	}

	return rowCount;
}
Exemplo n.º 4
0
inline void CrossEngineStep::addRow(RGData &data)
{
	fRowDelivered.setRid(fRowsReturned%fRowsPerGroup);
	fRowDelivered.nextRow();
	fRowGroupDelivered.incRowCount();

	if (++fRowsReturned%fRowsPerGroup == 0)
	{
		fOutputDL->insert(data);
		data.reinit(fRowGroupDelivered, fRowsPerGroup);
		fRowGroupDelivered.setData(&data);
		fRowGroupDelivered.resetRowGroup(fRowsReturned);
		fRowGroupDelivered.getRow(0, &fRowDelivered);
	}
}
Exemplo n.º 5
0
void SubAdapterStep::execute()
{
	RGData rgDataIn;
	RGData rgDataOut;
	Row rowIn;
	Row rowFe;
	Row rowOut;
	fRowGroupIn.initRow(&rowIn);
	fRowGroupOut.initRow(&rowOut);

	RGData rowFeData;
	bool usesFE = false;
	if (fRowGroupFe.getColumnCount() != (uint32_t) -1)
	{
		usesFE = true;
		fRowGroupFe.initRow(&rowFe, true);
		rowFeData = RGData(fRowGroupFe, 1);
		fRowGroupFe.setData(&rowFeData);
		fRowGroupFe.getRow(0, &rowFe);
	}

	bool more = false;
	try
	{
		fSubStep->run();

		more = fInputDL->next(fInputIterator, &rgDataIn);
		if (traceOn()) dlTimes.setFirstReadTime();

		while (more && !cancelled())
		{
			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);
			fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());

			fRowGroupIn.getRow(0, &rowIn);
			fRowGroupOut.getRow(0, &rowOut);

			for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
			{
				if(fExpression.get() == NULL)
				{
					outputRow(rowIn, rowOut);
				}
				else if (!usesFE)
				{
					if(fExpression->evaluate(&rowIn))
					{
						outputRow(rowIn, rowOut);
					}
				}
				else
				{
					copyRow(rowIn, &rowFe, rowIn.getColumnCount());
					//memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize());
					if(fExpression->evaluate(&rowFe))
					{
						outputRow(rowFe, rowOut);
					}
				}

				rowIn.nextRow();
			}

			if (fRowGroupOut.getRowCount() > 0)
			{
				fRowsReturned += fRowGroupOut.getRowCount();
				fOutputDL->insert(rgDataOut);
			}

			more = fInputDL->next(fInputIterator, &rgDataIn);
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
	}
	catch(...)
	{
		catchHandler("SubAdapterStep execute caught an unknown exception",
						ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
	}

	if (cancelled())
		while (more)
			more = fInputDL->next(fInputIterator, &rgDataIn);

	if (traceOn())
	{
		dlTimes.setLastReadTime();
		dlTimes.setEndOfInputTime();
		printCalTrace();
	}

	// Bug 3136, let mini stats to be formatted if traceOn.
	fOutputDL->endOfInput();
}
Exemplo n.º 6
0
void TupleHavingStep::execute()
{
	RGData rgDataIn;
	RGData rgDataOut;
	bool more = false;
	StepTeleStats sts;
	sts.query_uuid = fQueryUuid;
	sts.step_uuid = fStepUuid;

	try
	{
		more = fInputDL->next(fInputIterator, &rgDataIn);
		dlTimes.setFirstReadTime();

		sts.msg_type = StepTeleStats::ST_START;
		sts.total_units_of_work = 1;
		postStepStartTele(sts);

		if (!more && cancelled())
		{
			fEndOfResult = true;
		}

		while (more && !fEndOfResult)
		{
			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);

			doHavingFilters();

			more = fInputDL->next(fInputIterator, &rgDataIn);
			if (cancelled())
			{
				fEndOfResult = true;
			}
			else
			{
				fOutputDL->insert(rgDataOut);
			}
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
	}
	catch(...)
	{
		catchHandler("TupleHavingStep execute caught an unknown exception",
					 tupleHavingStepErr, fErrorInfo, fSessionId);
	}

	while (more)
		more = fInputDL->next(fInputIterator, &rgDataIn);

	fEndOfResult = true;
	fOutputDL->endOfInput();

	sts.msg_type = StepTeleStats::ST_SUMMARY;
	sts.total_units_of_work = sts.units_of_work_completed = 1;
	sts.rows = fRowsReturned;
	postStepSummaryTele(sts);

	dlTimes.setLastReadTime();
	dlTimes.setEndOfInputTime();

	if (traceOn())
		printCalTrace();
}
Exemplo n.º 7
0
uint32_t TupleHavingStep::nextBand(messageqcpp::ByteStream &bs)
{
	RGData rgDataIn;
	RGData rgDataOut;
	bool more = false;
	uint32_t rowCount = 0;

	try
	{
		bs.restart();

		more = fInputDL->next(fInputIterator, &rgDataIn);
		if (dlTimes.FirstReadTime().tv_sec ==0)
            dlTimes.setFirstReadTime();

		if (!more || cancelled())
		{
			fEndOfResult = true;
		}

		bool emptyRowGroup = true;
		while (more && !fEndOfResult && emptyRowGroup)
		{
			if (cancelled())
			{
				while (more)
					more = fInputDL->next(fInputIterator, &rgDataIn);
				break;
			}

			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);

			doHavingFilters();

			if (fRowGroupOut.getRowCount() > 0)
			{
				emptyRowGroup = false;
				fRowGroupOut.serializeRGData(bs);
				rowCount = fRowGroupOut.getRowCount();
			}
			else
			{
				more = fInputDL->next(fInputIterator, &rgDataIn);
			}
		}

		if (!more)
		{
			fEndOfResult = true;
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), tupleHavingStepErr, fErrorInfo, fSessionId);
		while (more)
			more = fInputDL->next(fInputIterator, &rgDataIn);
		fEndOfResult = true;
	}
	catch(...)
	{
		catchHandler("TupleHavingStep next band caught an unknown exception",
					 tupleHavingStepErr, fErrorInfo, fSessionId);
		while (more)
			more = fInputDL->next(fInputIterator, &rgDataIn);
		fEndOfResult = true;
	}

	if (fEndOfResult)
	{
		// send an empty / error band
		rgDataOut.reinit(fRowGroupOut, 0);
		fRowGroupOut.setData(&rgDataOut);
		fRowGroupOut.resetRowGroup(0);
		fRowGroupOut.setStatus(status());
		fRowGroupOut.serializeRGData(bs);

		dlTimes.setLastReadTime();
		dlTimes.setEndOfInputTime();

		if (traceOn())
			printCalTrace();
	}

	return rowCount;
}
uint64_t UpdatePackageProcessor::fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result,
        const uint64_t uniqueId, const uint32_t tableOid)
{
    ByteStream msg, msgBk, emsgBs;
    RGData rgData;
    uint32_t qb = 4;
    msg << qb;
    boost::scoped_ptr<rowgroup::RowGroup> rowGroup;
    uint64_t rowsProcessed = 0;
    uint32_t dbroot = 1;
    bool metaData = false;
    oam::OamCache* oamCache = oam::OamCache::makeOamCache();
    std::vector<int> fPMs = oamCache->getModuleIds();
    std::map<unsigned, bool> pmState;
    string emsg;
    string emsgStr;
    bool err = false;

    //boost::scoped_ptr<messageqcpp::MessageQueueClient> fExeMgr;
    //fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1"));
    try
    {

        for (unsigned i = 0; i < fPMs.size(); i++)
        {
            pmState[fPMs[i]] = true;
        }

        //timer.start("ExeMgr");
        fExeMgr->write(msg);
        fExeMgr->write(*(cpackage.get_ExecutionPlan()));
        //cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl;
        msg.restart();
        emsgBs.restart();
        msg = fExeMgr->read(); //error handling

        if (msg.length() == 4)
        {
            msg >> qb;

            if (qb != 0)
                err = true;
        }
        else
        {
            qb = 999;
            err = true;
        }

        if (err)
        {
            logging::Message::Args args;
            logging::Message message(2);
            args.add("Update Failed: ExeMgr Error");
            args.add((int)qb);
            message.format(args);
            result.result = UPDATE_ERROR;
            result.message = message;
            //timer.finish();
            return rowsProcessed;
        }

        emsgBs = fExeMgr->read();

        if (emsgBs.length() == 0)
        {
            logging::Message::Args args;
            logging::Message message(2);
            args.add("Update Failed: ");
            args.add("Lost connection to ExeMgr");
            message.format(args);
            result.result = UPDATE_ERROR;
            result.message = message;
            //timer.finish();
            return rowsProcessed;
        }

        emsgBs >> emsgStr;

        while (true)
        {
            if (fRollbackPending)
            {
                break;
            }

            msg.restart();
            msgBk.restart();
            msg = fExeMgr->read();
            msgBk = msg;

            if ( msg.length() == 0 )
            {
                cerr << "UpdatePackageProcessor::processPackage::fixupRows" << endl;
                logging::Message::Args args;
                logging::Message message(2);
                args.add("Update Failed: ");
                args.add("Lost connection to ExeMgr");
                message.format(args);
                result.result = UPDATE_ERROR;
                result.message = message;
                //timer.finish();
                //return rowsProcessed;
                break;
            }
            else
            {
                if (rowGroup.get() == NULL)
                {
                    //This is mete data, need to send all PMs.
                    metaData = true;
                    //cout << "sending meta data" << endl;
                    //timer.start("Meta");
                    err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot);
                    rowGroup.reset(new rowgroup::RowGroup());
                    rowGroup->deserialize(msg);
                    qb = 100;
                    msg.restart();
                    msg << qb;
                    fExeMgr->write(msg);
                    metaData = false;
                    //timer.stop("Meta");
                    continue;
                }

                rgData.deserialize(msg, true);
                rowGroup->setData(&rgData);
                //rowGroup->setData(const_cast<uint8_t*>(msg.buf()));
                err = (rowGroup->getStatus() != 0);

                if (err)
                {
                    //msgBk.advance(rowGroup->getDataSize());
                    string errorMsg;
                    msg >> errorMsg;
                    logging::Message::Args args;
                    logging::Message message(2);
                    args.add("Update Failed: ");
                    args.add(errorMsg);
                    message.format(args);
                    result.result = UPDATE_ERROR;
                    result.message = message;
                    DMLResult tmpResult;
                    receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
                    /*					qb = 100;
                    					//@Bug 4358 get rid of broken pipe error.
                    					msg.restart();
                    					msg << qb;
                    					fExeMgr->write(msg);
                    */					//timer.finish();
                    //return rowsProcessed;
                    //err = true;
                    break;
                }

                if (rowGroup->getRGData() == NULL)
                {
                    msg.restart();
                }

                if (rowGroup->getRowCount() == 0)  //done fetching
                {
                    //timer.finish();
                    //need to receive all response
                    err = receiveAll( result, uniqueId, fPMs, pmState, tableOid);
                    //return rowsProcessed;
                    break;
                }

                if (rowGroup->getBaseRid() == (uint64_t) (-1))
                {
                    continue;  // @bug4247, not valid row ids, may from small side outer
                }

                dbroot = rowGroup->getDBRoot();
                //cout << "dbroot in the rowgroup is " << dbroot << endl;
                //timer.start("processRowgroup");
                err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot);

                //timer.stop("processRowgroup");
                if (err)
                {
                    //timer.finish();
                    LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID());
                    logging::Message::Args args1;
                    logging::Message msg1(1);
                    args1.add("SQL statement erroring out, need to receive all messages from WES");
                    msg1.format( args1 );
                    logging::Logger logger(logid.fSubsysID);
                    logger.logMessage(LOG_TYPE_DEBUG, msg1, logid);
                    DMLResult tmpResult;
                    receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
                    logging::Message::Args args2;
                    logging::Message msg2(1);
                    args2.add("SQL statement erroring out, received all messages from WES");
                    msg2.format( args2 );
                    logger.logMessage(LOG_TYPE_DEBUG, msg2, logid);
                    //@Bug 4358 get rid of broken pipe error.
                    /*					msg.restart();
                    					msg << qb;
                    					fExeMgr->write(msg);
                    					return rowsProcessed;
                    */
                    //err = true;
                    break;
                }

                rowsProcessed += rowGroup->getRowCount();
            }
        }

        if (fRollbackPending)
        {
            err = true;
            // Response to user
            cerr << "UpdatePackageProcessor::processPackage::fixupRows Rollback Pending" << endl;
            //@Bug 4994 Cancelled job is not error
            result.result = JOB_CANCELED;

            // Log
            LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID());
            logging::Message::Args args1;
            logging::Message msg1(1);
            args1.add("SQL statement canceled by user");
            msg1.format( args1 );
            logging::Logger logger(logid.fSubsysID);
            logger.logMessage(LOG_TYPE_DEBUG, msg1, logid);

            // Clean out the pipe;
            DMLResult tmpResult;
            receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid);
        }

        // get stats from ExeMgr
        if (!err)
        {
            qb = 3;
            msg.restart();
            msg << qb;
            fExeMgr->write(msg);
            msg = fExeMgr->read();
            msg >> result.queryStats;
            msg >> result.extendedStats;
            msg >> result.miniStats;
            result.stats.unserialize(msg);
        }
Exemplo n.º 9
0
void TupleAnnexStep::executeWithOrderBy()
{
    RGData rgDataIn;
    RGData rgDataOut;
    bool more = false;

    try
    {
        more = fInputDL->next(fInputIterator, &rgDataIn);
        if (traceOn()) dlTimes.setFirstReadTime();

        while (more && !cancelled())
        {
            fRowGroupIn.setData(&rgDataIn);
            fRowGroupIn.getRow(0, &fRowIn);

            for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled(); ++i)
            {
                fOrderBy->processRow(fRowIn);
                fRowIn.nextRow();
            }

            more = fInputDL->next(fInputIterator, &rgDataIn);
        }

        fOrderBy->finalize();

        if (!cancelled())
        {
            while (fOrderBy->getData(rgDataIn))
            {
                if (fConstant == NULL &&
                        fRowGroupOut.getColumnCount() == fRowGroupIn.getColumnCount())
                {
                    rgDataOut = rgDataIn;
                    fRowGroupOut.setData(&rgDataOut);
                }
                else
                {
                    fRowGroupIn.setData(&rgDataIn);
                    fRowGroupIn.getRow(0, &fRowIn);

                    rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
                    fRowGroupOut.setData(&rgDataOut);
                    fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
                    fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot());
                    fRowGroupOut.getRow(0, &fRowOut);

                    for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
                    {
                        if (fConstant)
                            fConstant->fillInConstants(fRowIn, fRowOut);
                        else
                            copyRow(fRowIn, &fRowOut);

                        fRowGroupOut.incRowCount();
                        fRowOut.nextRow();
                        fRowIn.nextRow();
                    }
                }

                if (fRowGroupOut.getRowCount() > 0)
                {
                    fRowsReturned += fRowGroupOut.getRowCount();
                    fOutputDL->insert(rgDataOut);
                }
            }
        }
    }
    catch(const std::exception& ex)
    {
        catchHandler(ex.what(), fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }
    catch(...)
    {
        catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }

    while (more)
        more = fInputDL->next(fInputIterator, &rgDataIn);

    if (traceOn() && !fDelivery)
    {
        dlTimes.setLastReadTime();
        dlTimes.setEndOfInputTime();
        printCalTrace();
    }

    // Bug 3136, let mini stats to be formatted if traceOn.
    fOutputDL->endOfInput();
}
Exemplo n.º 10
0
void TupleAnnexStep::executeNoOrderByWithDistinct()
{
    scoped_ptr<DistinctMap_t> distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this)));
    vector<RGData> dataVec;
    RGData rgDataIn;
    RGData rgDataOut;
    bool more = false;

    rgDataOut.reinit(fRowGroupOut);
    fRowGroupOut.setData(&rgDataOut);
    fRowGroupOut.resetRowGroup(0);
    fRowGroupOut.getRow(0, &fRowOut);

    fRowGroupOut.initRow(&row1);
    fRowGroupOut.initRow(&row2);

    try
    {
        more = fInputDL->next(fInputIterator, &rgDataIn);
        if (traceOn()) dlTimes.setFirstReadTime();

        while (more && !cancelled() && !fLimitHit)
        {
            fRowGroupIn.setData(&rgDataIn);
            fRowGroupIn.getRow(0, &fRowIn);

            for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i)
            {
                pair<DistinctMap_t::iterator, bool> inserted;
                if (fConstant)
                    fConstant->fillInConstants(fRowIn, fRowOut);
                else
                    copyRow(fRowIn, &fRowOut);

                ++fRowsProcessed;
                fRowIn.nextRow();

                inserted = distinctMap->insert(fRowOut.getPointer());
                if (inserted.second) {
                    fRowGroupOut.incRowCount();
                    fRowOut.nextRow();
                    if (UNLIKELY(++fRowsReturned >= fLimitCount))
                    {
                        fLimitHit = true;
                        fJobList->abortOnLimit((JobStep*) this);
                    }

                    if (UNLIKELY(fRowGroupOut.getRowCount() >= 8192))
                    {
                        dataVec.push_back(rgDataOut);
                        rgDataOut.reinit(fRowGroupOut);
                        fRowGroupOut.setData(&rgDataOut);
                        fRowGroupOut.resetRowGroup(0);
                        fRowGroupOut.getRow(0, &fRowOut);
                    }
                }
            }

            more = fInputDL->next(fInputIterator, &rgDataIn);
        }

        if (fRowGroupOut.getRowCount() > 0)
            dataVec.push_back(rgDataOut);

        for (vector<RGData>::iterator i = dataVec.begin(); i != dataVec.end(); i++)
        {
            rgDataOut = *i;
            fRowGroupOut.setData(&rgDataOut);
            fOutputDL->insert(rgDataOut);
        }
    }
    catch(const std::exception& ex)
    {
        catchHandler(ex.what(), fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }
    catch(...)
    {
        catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }


    while (more)
        more = fInputDL->next(fInputIterator, &rgDataIn);

    if (traceOn() && !fDelivery)
    {
        dlTimes.setLastReadTime();
        dlTimes.setEndOfInputTime();
        printCalTrace();
    }

    // Bug 3136, let mini stats to be formatted if traceOn.
    fOutputDL->endOfInput();
}
Exemplo n.º 11
0
void TupleAnnexStep::executeNoOrderBy()
{
    RGData rgDataIn;
    RGData rgDataOut;
    bool more = false;

    try
    {
        more = fInputDL->next(fInputIterator, &rgDataIn);
        if (traceOn()) dlTimes.setFirstReadTime();

        while (more && !cancelled() && !fLimitHit)
        {
            fRowGroupIn.setData(&rgDataIn);
            fRowGroupIn.getRow(0, &fRowIn);

            // Get a new output rowgroup for each input rowgroup to preserve the rids
            rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
            fRowGroupOut.setData(&rgDataOut);
            fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());
            fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot());
            fRowGroupOut.getRow(0, &fRowOut);

            for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i)
            {
                // skip first limit-start rows
                if (fRowsProcessed++ < fLimitStart)
                {
                    fRowIn.nextRow();
                    continue;
                }

                if (fConstant)
                    fConstant->fillInConstants(fRowIn, fRowOut);
                else
                    copyRow(fRowIn, &fRowOut);

                fRowGroupOut.incRowCount();
                if (++fRowsReturned < fLimitCount)
                {
                    fRowOut.nextRow();
                    fRowIn.nextRow();
                }
                else
                {
                    fLimitHit = true;
                    fJobList->abortOnLimit((JobStep*) this);
                }
            }

            if (fRowGroupOut.getRowCount() > 0)
            {
                fOutputDL->insert(rgDataOut);
            }

            more = fInputDL->next(fInputIterator, &rgDataIn);
        }
    }
    catch(const std::exception& ex)
    {
        catchHandler(ex.what(), fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }
    catch(...)
    {
        catchHandler("TupleAnnexStep execute caught an unknown exception", fSessionId);
        if (status() == 0)
            status(ERR_IN_PROCESS);
    }

    while (more)
        more = fInputDL->next(fInputIterator, &rgDataIn);


    if (traceOn() && !fDelivery)
    {
        dlTimes.setLastReadTime();
        dlTimes.setEndOfInputTime();
        printCalTrace();
    }

    // Bug 3136, let mini stats to be formatted if traceOn.
    fOutputDL->endOfInput();
}
Exemplo n.º 12
0
uint TupleAnnexStep::nextBand(messageqcpp::ByteStream &bs)
{
    RGData rgDataOut;
    bool more = false;
    uint rowCount = 0;

    try
    {
        bs.restart();

        more = fOutputDL->next(fOutputIterator, &rgDataOut);
        if (traceOn() && dlTimes.FirstReadTime().tv_sec ==0)
            dlTimes.setFirstReadTime();

        if (more && !cancelled())
        {
            fRowGroupDeliver.setData(&rgDataOut);
            fRowGroupDeliver.serializeRGData(bs);
            rowCount = fRowGroupDeliver.getRowCount();
        }
        else
        {
            if (more)
                more = fOutputDL->next(fOutputIterator, &rgDataOut);
            fEndOfResult = true;
        }
    }
    catch(const std::exception& ex)
    {
        catchHandler(ex.what(), fSessionId);
        if (status() == 0)
            status(ERR_IN_DELIVERY);
        while (more)
            more = fOutputDL->next(fOutputIterator, &rgDataOut);
        fEndOfResult = true;
    }
    catch(...)
    {
        catchHandler("TupleAnnexStep next band caught an unknown exception", fSessionId);
        if (status() == 0)
            status(ERR_IN_DELIVERY);
        while (more)
            more = fOutputDL->next(fOutputIterator, &rgDataOut);
        fEndOfResult = true;
    }

    if (fEndOfResult)
    {
        // send an empty / error band
        rgDataOut.reinit(fRowGroupDeliver, 0);
        fRowGroupDeliver.setData(&rgDataOut);
        fRowGroupDeliver.resetRowGroup(0);
        fRowGroupDeliver.setStatus(status());
        fRowGroupDeliver.serializeRGData(bs);

        if (traceOn())
        {
            dlTimes.setLastReadTime();
            dlTimes.setEndOfInputTime();
        }

        if (traceOn())
            printCalTrace();
    }

    return rowCount;
}
Exemplo n.º 13
0
int64_t JoinPartition::convertToSplitMode()
{
	int i, j;
	ByteStream bs;
	RGData rgData;
	uint32_t hash;
	uint64_t tmp;
	int64_t ret = -(int64_t)smallSizeOnDisk;    // smallFile gets deleted
	boost::scoped_array<uint32_t> rowDist(new uint32_t[bucketCount]);
	uint32_t rowCount = 0;

	memset(rowDist.get(), 0, sizeof(uint32_t) * bucketCount);
	fileMode = false;
	htSizeEstimate = 0;
	smallSizeOnDisk = 0;
	buckets.reserve(bucketCount);
	for (i = 0; i < (int) bucketCount; i++)
		buckets.push_back(boost::shared_ptr<JoinPartition>(new JoinPartition(*this, false)));

	RowGroup &rg = smallRG;
	Row &row = smallRow;
	nextSmallOffset = 0;
	while (1) {
		readByteStream(0, &bs);
		if (bs.length() == 0)
			break;
		rgData.deserialize(bs);
		rg.setData(&rgData);
		for (j = 0; j < (int) rg.getRowCount(); j++) {
			rg.getRow(j, &row);

			if (antiWithMatchNulls && hasNullJoinColumn(row)) {
				if (needsAllNullRows || !gotNullRow) {
					for (j = 0; j < (int) bucketCount; j++)
						ret += buckets[j]->insertSmallSideRow(row);
					gotNullRow = true;
				}
				continue;
			}

			if (typelessJoin)
				hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount;
			else {
				if (UNLIKELY(row.isUnsigned(smallKeyCols[0])))
					tmp = row.getUintField(smallKeyCols[0]);
				else
					tmp = row.getIntField(smallKeyCols[0]);
				hash = hasher((char *) &tmp, 8, hashSeed);
				hash = hasher.finalize(hash, 8) % bucketCount;
			}
			rowCount++;
			rowDist[hash]++;
			ret += buckets[hash]->insertSmallSideRow(row);
		}
	}
	boost::filesystem::remove(smallFilename);
	smallFilename.clear();

	for (i = 0; i < (int) bucketCount; i++)
		if (rowDist[i] == rowCount)
			throw IDBExcept("All rows hashed to the same bucket", ERR_DBJ_DATA_DISTRIBUTION);

	rg.setData(&buffer);
	rg.resetRowGroup(0);
	rg.getRow(0, &row);

	return ret;
}
Exemplo n.º 14
0
void TupleConstantStep::execute()
{
	RGData rgDataIn;
	RGData rgDataOut;
	bool more = false;

	try
	{
		more = fInputDL->next(fInputIterator, &rgDataIn);
		if (traceOn()) dlTimes.setFirstReadTime();

		if (!more && cancelled())
		{
			fEndOfResult = true;
		}

		while (more && !fEndOfResult)
		{
			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);

			fillInConstants();

			more = fInputDL->next(fInputIterator, &rgDataIn);
			if (cancelled())
			{
				fEndOfResult = true;
			}
			else
			{
				fOutputDL->insert(rgDataOut);
			}
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), fSessionId);
		if (status() == 0)
			status(tupleConstantStepErr);
	}
	catch(...)
	{
		catchHandler("TupleConstantStep execute caught an unknown exception", fSessionId);
		if (status() == 0)
			status(tupleConstantStepErr);
	}

	if (!fEndOfResult)
		while (more)
			more = fInputDL->next(fInputIterator, &rgDataIn);

	// Bug 3136, let mini stats to be formatted if traceOn.
	if (traceOn())
	{
		dlTimes.setLastReadTime();
		dlTimes.setEndOfInputTime();
		printCalTrace();
	}

	fEndOfResult = true;
	fOutputDL->endOfInput();
}
Exemplo n.º 15
0
void SubAdapterStep::execute()
{
	RGData rgDataIn;
	RGData rgDataOut;
	Row rowIn;
	Row rowFe;
	Row rowOut;
	fRowGroupIn.initRow(&rowIn);
	fRowGroupOut.initRow(&rowOut);

	RGData rowFeData;
	StepTeleStats sts;
	sts.query_uuid = fQueryUuid;
	sts.step_uuid = fStepUuid;
	bool usesFE = false;
	if (fRowGroupFe.getColumnCount() > 0)
	{
		usesFE = true;
		fRowGroupFe.initRow(&rowFe, true);
		rowFeData = RGData(fRowGroupFe, 1);
		fRowGroupFe.setData(&rowFeData);
		fRowGroupFe.getRow(0, &rowFe);
	}

	bool more = false;
	try
	{
		sts.msg_type = StepTeleStats::ST_START;
		sts.total_units_of_work = 1;
		postStepStartTele(sts);

		fSubStep->run();

		more = fInputDL->next(fInputIterator, &rgDataIn);
		if (traceOn()) dlTimes.setFirstReadTime();

		while (more && !cancelled())
		{
			fRowGroupIn.setData(&rgDataIn);
			rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount());
			fRowGroupOut.setData(&rgDataOut);
			fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid());

			fRowGroupIn.getRow(0, &rowIn);
			fRowGroupOut.getRow(0, &rowOut);

			fRowsInput += fRowGroupIn.getRowCount();

			for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
			{
				if(fExpression.get() == NULL)
				{
					outputRow(rowIn, rowOut);
				}
				else if (!usesFE)
				{
					if(fExpression->evaluate(&rowIn))
					{
						outputRow(rowIn, rowOut);
					}
				}
				else
				{
					copyRow(rowIn, &rowFe, rowIn.getColumnCount());
					//memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize());
					if(fExpression->evaluate(&rowFe))
					{
						outputRow(rowFe, rowOut);
					}
				}

				rowIn.nextRow();
			}

			if (fRowGroupOut.getRowCount() > 0)
			{
				fRowsReturned += fRowGroupOut.getRowCount();
				fOutputDL->insert(rgDataOut);
			}

			more = fInputDL->next(fInputIterator, &rgDataIn);
		}
	}
	catch(const std::exception& ex)
	{
		catchHandler(ex.what(), ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
	}
	catch(...)
	{
		catchHandler("SubAdapterStep execute caught an unknown exception",
						ERR_EXEMGR_MALFUNCTION, fErrorInfo, fSessionId);
	}

	if (cancelled())
		while (more)
			more = fInputDL->next(fInputIterator, &rgDataIn);

	if (traceOn())
	{
		dlTimes.setLastReadTime();
		dlTimes.setEndOfInputTime();
		printCalTrace();
	}

	sts.msg_type = StepTeleStats::ST_SUMMARY;
	sts.total_units_of_work = sts.units_of_work_completed = 1;
	sts.rows = fRowsReturned;
	postStepSummaryTele(sts);

	// Bug 3136, let mini stats to be formatted if traceOn.
	fOutputDL->endOfInput();
}