Lng32 ExHdfsFastExtractTcb::lobInterfaceInsert(ssize_t bytesToWrite)
{
Int64 requestTag = 0;
Int64 descSyskey = 0;
return ExpLOBInterfaceInsert(lobGlob_,
fileName_,
targetLocation_,
(Lng32)Lob_External_HDFS_File,
hdfsHost_,
hdfsPort_,
0,
NULL, //lobHandle == NULL -->simpleInsert
NULL,
NULL,
0,
NULL,
requestTag,
0,
descSyskey,
Lob_InsertDataSimple,
NULL,
Lob_None,//LobsSubOper so
1, //waitedOp
currBuffer_->data_,
bytesToWrite,
0, //bufferSize
myTdb().getHdfsReplication(), //replication
0 //blockSize
);
}
NABoolean ExSequenceTcb::removeOLAPBuffer()
{
if (lastOLAPBuffer_ == NULL || firstOLAPBuffer_ == NULL)
{
// LCOV_EXCL_START
return FALSE;
// LCOV_EXCL_STOP
}
if (lastOLAPBuffer_ == firstOLAPBuffer_)
{
// LCOV_EXCL_START
// since we keep a minimum number of buffers this code won't be visited
// this code should not be removed. if we decide to not keep a min number
// of buffers in the list it will be used
NADELETEBASIC( lastOLAPBuffer_ ,heap_);
numberOfOLAPBuffers_ = 0;
maxNumberHistoryRows_ = myTdb().maxHistoryRows_;
lastOLAPBuffer_ = NULL;
firstOLAPBuffer_ = NULL;
return TRUE;
// LCOV_EXCL_STOP
}
HashBuffer * tmpBuf = lastOLAPBuffer_;
lastOLAPBuffer_ = lastOLAPBuffer_->getPrev();
NADELETEBASIC( tmpBuf ,heap_);
lastOLAPBuffer_->setNext(NULL);
numberOfOLAPBuffers_ --;
maxNumberHistoryRows_ = numberOfOLAPBuffers_ * maxRowsInOLAPBuffer_;
return TRUE;
}
Int32 ExHdfsFastExtractTcb::fixup()
{
lobGlob_ = NULL;
ex_tcb::fixup();
if(!myTdb().getSkipWritingToFiles() &&
!myTdb().getBypassLibhdfs())
ExpLOBinterfaceInit
(lobGlob_, getGlobals()->getDefaultHeap(),TRUE);
modTS_ = myTdb().getModTSforDir();
return 0;
}
Lng32 ExHdfsFastExtractTcb::lobInterfaceDataModCheck()
{
return ExpLOBinterfaceDataModCheck(lobGlob_,
targetLocation_,
hdfsHost_,
hdfsPort_,
myTdb().getModTSforDir(),
0);
}
Int32 ExHdfsFastExtractTcb::fixup()
{
lobGlob_ = NULL;
ex_tcb::fixup();
if(!myTdb().getSkipWritingToFiles() &&
!myTdb().getBypassLibhdfs())
memset (hdfsHost_, '\0', sizeof(hdfsHost_));
strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
hdfsPort_ = myTdb().getHdfsPortNum();
ExpLOBinterfaceInit
(lobGlob_, getGlobals()->getDefaultHeap(),getGlobals()->castToExExeStmtGlobals()->getContext(),TRUE,hdfsHost_,hdfsPort_);
modTS_ = myTdb().getModTSforDir();
return 0;
}
Lng32 ExHdfsFastExtractTcb::lobInterfaceCreate()
{
return ExpLOBinterfaceCreate(lobGlob_,
fileName_,
targetLocation_,
(Lng32)Lob_External_HDFS_File,
hdfsHost_,
hdfsPort_,
0, //bufferSize -- 0 --> use default
myTdb().getHdfsReplication(), //replication
0 //bloclSize --0 -->use default
);
}
// work - doit...
//
//
short ExSequenceTcb::work()
{
// If there are no parent requests on the queue, then there cannot
// be anything to do here.
//
if (qparent_.down->isEmpty())
return WORK_OK;
ex_queue_entry * pentry_down;
ExSequencePrivateState * pstate;
ex_queue::down_request request;
// Take any new parent requests and pass them on to the child as long
// as the child's queue is not full. processedInputs_ maintains the
// Queue index of the last request that was passed on.
//
for(queue_index tail = qparent_.down->getTailIndex();
(processedInputs_ != tail) && (!qchild_.down->isFull());
processedInputs_++ )
{
pentry_down = qparent_.down->getQueueEntry(processedInputs_);
pstate = (ExSequencePrivateState*) pentry_down->pstate;
request = pentry_down->downState.request;
// If the request has already been cancelled don't pass it to the
// child. Instead, just mark the request as done. This will trigger
// a EOD reply when this request gets worked on.
//
if (request == ex_queue::GET_NOMORE)
{
// LCOV_EXCL_START
pstate->step_ = ExSeq_DONE;
// LCOV_EXCL_STOP
}
else
{
pstate->step_ = ExSeq_WORKING_READ;
// Pass the request to the child
//
ex_queue_entry * centry = qchild_.down->getTailEntry();
centry->downState.request = ex_queue::GET_ALL;
centry->downState.requestValue = 11;
centry->downState.parentIndex = processedInputs_;
centry->passAtp(pentry_down);
qchild_.down->insert();
}
} // end for processedInputs_
pentry_down = qparent_.down->getHeadEntry();
pstate = (ExSequencePrivateState*) pentry_down->pstate;
request = pentry_down->downState.request;
// Take any child replies and process them. Return the processed
// rows as long the parent queue has room.
//
while (1)
{
// If we have satisfied the parent request (or it was cancelled),
// then stop processing rows, cancel any outstanding child
// requests, and set this request to the CANCELLED state.
//
if ((pstate->step_ == ExSeq_WORKING_READ) ||
(pstate->step_ == ExSeq_WORKING_RETURN))
{
if ((request == ex_queue::GET_NOMORE) ||
((request == ex_queue::GET_N) &&
(pentry_down->downState.requestValue
<= (Lng32)pstate->matchCount_)))
{
qchild_.down->cancelRequestWithParentIndex
(qparent_.down->getHeadIndex());
pstate->step_ = ExSeq_CANCELLED;
}
}
switch (pstate->step_)
{
// ExSeq_CANCELLED
//
// Transition to this state from ...
// 1. ExSeq_Error - After the error has been processed.
// 2. ExSeq_Working - If enough rows have been returned.
// 3. ExSeq_Working - If the request was cancelled.
//
// Remain in this state until ..
// 1. All rows from the child including EOD are consumed
//
// Transition from this state to ...
// 1. ExSeq_DONE - In all cases.
//
case ExSeq_CANCELLED:
{
// There are no extra rows to process from the child yet,
// so try again later.
//
if (qchild_.up->isEmpty())
{
return WORK_OK;
}
ex_queue_entry * centry = qchild_.up->getHeadEntry();
ex_queue::up_status child_status = centry->upState.status;
// If this is the EOD, transition to the ExSeq_DONE state.
//
if (child_status == ex_queue::Q_NO_DATA)
pstate->step_ = ExSeq_DONE;
// Discard the child row.
qchild_.up->removeHead();
break;
}
// ExSeq_ERROR
//
// Transition to this state from ...
// 1. ExSeq_WORKING_READ - a child reply with the type SQLERROR.
// 2. ExSeq_WORKING_RETURN
// 3. ExSeq_OVERFLOW_READ
// 4. ExSeq_OVERFLOW_WRITE
// Remain in this state until ..
// 1. The error row has been returned to the parent.
//
// Transition from this state to ...
// 1. ExSeq_CANCELLED - In all cases.
//
case ExSeq_ERROR:
{
// If there is no room in the parent queue for the reply,
// try again later.
//
if (qparent_.up->isFull())
// LCOV_EXCL_START
return WORK_OK;
// LCOV_EXCL_STOP
ex_queue_entry *pentry_up = qparent_.up->getTailEntry();
// Cancel the child request - there must be a child request in
// progress to get to the ExSeq_ERROR state.
//
qchild_.down->cancelRequestWithParentIndex
(qparent_.down->getHeadIndex());
// Construct and return the error row.
//
if (workAtp_->getDiagsArea()) {
ComDiagsArea * da = workAtp_->getDiagsArea();
pentry_up->setDiagsArea(da);
da->incrRefCount();
workAtp_->setDiagsArea(0);
}
pentry_up->upState.status = ex_queue::Q_SQLERROR;
pentry_up->upState.parentIndex
= pentry_down->downState.parentIndex;
pentry_up->upState.downIndex = qparent_.down->getHeadIndex();
pentry_up->upState.setMatchNo(pstate->matchCount_);
qparent_.up->insert();
// Transition to the ExSeq_CANCELLED state.
//
pstate->step_ = ExSeq_CANCELLED;
break;
}
// ExSeq_WORKING_READ
//
// Transition to this state from ...
// 1. ExSeq_EMPTY - If a request is started.
// 2. ExSeq_WORKING_RETURN -
// 3. ExSeq_OVERFLOW_WRITE -
// Remain in this state until ...
// 1. All child replies including EOD have been processed.
// 2. A SQLERROR row is received.
// 3. Enough rows have been returned.
// 4. The request is cancelled.
// 5. End of partition is reached
// Transition from this state to ...
// 2. ExSeq_ERROR - If an SQLERROR rows is received.
// 3. ExSeq_CANCELLED - If the request is cancelled.
// 4. ExSeq_WORKING_RETURN
// 5. ExSeq_OVERFLOW_WRITE -
case ExSeq_WORKING_READ:
{
if(!isUnboundedFollowing() && isHistoryFull()) {
pstate->step_ = ExSeq_WORKING_RETURN;
break;
}
// If there are no replies, try again later.
//
if (qchild_.up->isEmpty())
return WORK_OK;
ex_queue_entry * centry = qchild_.up->getHeadEntry();
switch (centry->upState.status)
{
// A data row from the child.
//
case ex_queue::Q_OK_MMORE:
{
tupp_descriptor histTupp;
workAtp_->copyAtp(pentry_down->getAtp());
workAtp_->getTupp(myTdb().tuppIndex_) = &histTupp;
if ( checkPartitionChangeExpr() &&
currentHistRowPtr_)
{
workAtp_->getTupp
(myTdb().tuppIndex_).setDataPointer(currentHistRowPtr_);
// Check whether the partition changed
ex_expr::exp_return_type retCode = checkPartitionChangeExpr()->eval(workAtp_, centry->getAtp());
if (retCode == ex_expr::EXPR_ERROR)
{
// LCOV_EXCL_START
updateDiagsArea(centry);
pstate->step_ = ExSeq_ERROR;
break;
// LCOV_EXCL_STOP
}
if ( retCode == ex_expr::EXPR_FALSE)
{
setPartitionEnd(TRUE);
pstate->step_ = ExSeq_END_OF_PARTITION;
break;
}
}
if (isUnboundedFollowing() )
{
if (OLAPBuffersFlushed_)
{
OLAPBuffersFlushed_ = FALSE;// current row is the first one in first buffer already
}
else
{
NABoolean noMemory =
advanceHistoryRow( TRUE /* checkMemoryPressure */);
if (noMemory)
{
pstate->step_ = ExSeq_OVERFLOW_WRITE;
cluster_->nextBufferToFlush_ = firstOLAPBuffer_;
cluster_->afterLastBufferToFlush_ = NULL;//flush them all
// If it is the first overflow, for this partition
if ( ! memoryPressureDetected_ ) {
memoryPressureDetected_ = TRUE;
} // memory pressure detected
break;
}
}
}
else
{
advanceHistoryRow();
}
workAtp_->getTupp
(myTdb().tuppIndex_).setDataPointer(currentHistRowPtr_);
ex_expr::exp_return_type retCode = ex_expr::EXPR_OK;
// Apply the read phase sequence function expression to compute
// the values of the sequence functions.
if (sequenceExpr())
{
retCode = sequenceExpr()->eval(workAtp_, centry->getAtp());
if (retCode == ex_expr::EXPR_ERROR)
{
updateDiagsArea(centry);
pstate->step_ = ExSeq_ERROR;
break;
}
}
// merge the child's diags area into the work atp
updateDiagsArea(centry);
qchild_.up->removeHead();
break;
}
// The EOD from the child. Transition to ExSeq_DONE.
//
case ex_queue::Q_NO_DATA:
{
setPartitionEnd(TRUE);
if (isHistoryEmpty())
{
pstate->step_ = ExSeq_DONE;
qchild_.up->removeHead();
}
else
{
pstate->step_ = ExSeq_END_OF_PARTITION;
}
}
break;
// An SQLERROR from the child. Transition to ExSeq_ERROR.
//
case ex_queue::Q_SQLERROR:
updateDiagsArea(centry);
pstate->step_ = ExSeq_ERROR;
break;
}
}
break;
// ExSeq_WORKING_RETURN
//
// Transition to this state from ...
// 1. ExSeq_WORKING_READ -
// 2. ExSeq_OVERFLOW_READ -
// 3. ExSeq_END_OF_PARTITION -
// Remain in this state until ...
// 1. All rows are returned.
// 2. A SQLERROR row is received.
// 3. Enough rows have been returned.
//
// Transition from this state to ...
// 1. ExSeq_DONE - If all the child rows including EOD have
// been processed.
// 2. ExSeq_ERROR - If an SQLERROR rows is received.
// 3. ExSeq_CANCELLED - If enough rows have been returned.
// 4. ExSeq_CANCELLED - If the request is cancelled.
// 5. ExSeq_WORKING_RETURN
// 6. ExSeq_DONE
// 7. ExSeq_OVERFLOW_READ
case ExSeq_WORKING_RETURN:
{
// If there is not room in the parent Queue for the reply,
// try again later.
//
if (qparent_.up->isFull())
return WORK_OK;
if(isHistoryEmpty())
{
ex_queue_entry * centry = NULL;
if(!qchild_.up->isEmpty())
{
centry = qchild_.up->getHeadEntry();
}
if(centry && (centry->upState.status == ex_queue::Q_NO_DATA))
{
pstate->step_ = ExSeq_DONE;
qchild_.up->removeHead();
}
else
{
pstate->step_ = ExSeq_WORKING_READ;
if (getPartitionEnd())
{
initializeHistory();
}
}
break;
}
if(!canReturnRows() &&
!getPartitionEnd() &&
!isUnboundedFollowing() &&
!isOverflowStarted()) // redundant? because not unbounded ...
{
pstate->step_ = ExSeq_WORKING_READ;
break;
}
ex_queue_entry * pentry_up = qparent_.up->getTailEntry();
pentry_up->copyAtp(pentry_down);
// Try to allocate a tupp.
//
if (pool_->get_free_tuple(pentry_up->getTupp(myTdb().tuppIndex_),
recLen()))
// LCOV_EXCL_START
return WORK_POOL_BLOCKED;
// LCOV_EXCL_STOP
char *tuppData = pentry_up->getTupp
(myTdb().tuppIndex_).getDataPointer();
advanceReturnHistoryRow();
char *histData = currentRetHistRowPtr_;
pentry_up->getTupp
(myTdb().tuppIndex_).setDataPointer(histData);
ex_expr::exp_return_type retCode = ex_expr::EXPR_OK;
// Apply the return phase expression
if(returnExpr())
{
retCode = returnExpr()->eval(pentry_up->getAtp(),workAtp_);
if (retCode == ex_expr::EXPR_ERROR)
{
// LCOV_EXCL_START
pstate->step_ = ExSeq_ERROR;
break;
// LCOV_EXCL_STOP
}
}
retCode = ex_expr::EXPR_OK;
//Apply post predicate expression
if (postPred())
{
retCode = postPred()->eval(pentry_up->getAtp(),pentry_up->getAtp());
if (retCode == ex_expr::EXPR_ERROR)
{
// LCOV_EXCL_START
pstate->step_ = ExSeq_ERROR;
break;
// LCOV_EXCL_STOP
}
}
//pentry_up->getAtp()->display("return eval result", myTdb().getCriDescUp());
//
// Case-10-030724-7963: we are done pointing the tupp at the
// history buffer, so point it back to the SQL buffer.
//
pentry_up->getTupp
(myTdb().tuppIndex_).setDataPointer(tuppData);
switch(retCode) {
case ex_expr::EXPR_OK:
case ex_expr::EXPR_TRUE:
case ex_expr::EXPR_NULL:
// Copy the row that was computed in the history buffer,
// to the space previously allocated in the SQL buffer.
str_cpy_all(tuppData, histData, recLen());
// Return the processed row.
//
// Finalize the queue entry, then insert it
//
pentry_up->upState.status = ex_queue::Q_OK_MMORE;
pentry_up->upState.parentIndex
= pentry_down->downState.parentIndex;
pentry_up->upState.downIndex =
qparent_.down->getHeadIndex();
pstate->matchCount_++;
pentry_up->upState.setMatchNo(pstate->matchCount_);
qparent_.up->insert();
break;
// If the selection predicate returns FALSE,
// do not return the child row.
//
case ex_expr::EXPR_FALSE:
break;
// If the selection predicate returns an ERROR,
// go to the error processing state.
//
case ex_expr::EXPR_ERROR:
// LCOV_EXCL_START
pstate->step_ = ExSeq_ERROR;
// LCOV_EXCL_STOP
break;
}
// MV --
// Now, if there are no errors so far, evaluate the
// cancel expression
if ((pstate->step_ != ExSeq_ERROR) && cancelExpr())
{
// Temporarily point the tupp to the tail of the
// history buffer for evaluating the
// expressions.
//
pentry_up->getTupp
(myTdb().tuppIndex_).setDataPointer(histData);
retCode =
cancelExpr()->eval(pentry_up->getAtp(),pentry_up->getAtp());
// We are done pointing the tupp at the history
// buffer, so point it back to the SQL buffer.
//
pentry_up->getTupp
(myTdb().tuppIndex_).setDataPointer(tuppData);
if (retCode == ex_expr::EXPR_TRUE)
{
qchild_.down->cancelRequestWithParentIndex
(qparent_.down->getHeadIndex());
pstate->step_ = ExSeq_CANCELLED;
}
}
updateHistRowsToReturn();
if ( isOverflowStarted() )
{
numberOfRowsReturnedBeforeReadOF_ ++;
if (numberOfRowsReturnedBeforeReadOF_ == maxNumberOfRowsReturnedBeforeReadOF_)
{
firstOLAPBufferFromOF_ = currentRetOLAPBuffer_->getNext();
if (firstOLAPBufferFromOF_ == NULL)
{
firstOLAPBufferFromOF_ = firstOLAPBuffer_;
}
for( Int32 i = 0; i < numberOfWinOLAPBuffers_; i++)
{
firstOLAPBufferFromOF_ = firstOLAPBufferFromOF_->getNext();
if (firstOLAPBufferFromOF_ == NULL)
{
firstOLAPBufferFromOF_ = firstOLAPBuffer_;
}
}
numberOfOLAPBuffersFromOF_ = numberOfOLAPBuffers_ - numberOfWinOLAPBuffers_;
cluster_->nextBufferToRead_ = firstOLAPBufferFromOF_;
HashBuffer * afterLast = firstOLAPBufferFromOF_;
// last buffer to read into is the current buffer - maybe ?
for ( Lng32 bufcount = numberOfOLAPBuffersFromOF_ ;
bufcount ;
bufcount-- ) {
afterLast = afterLast->getNext() ;
// Don't cycle back if bufcount == 1 because the logic in
// Cluster::read relies on the NULL ptr to stop reading
if ( bufcount > 1 && ! afterLast )
afterLast = firstOLAPBuffer_; // cycle back
}
// The last buffer to read to is always the current buffer
// ex_assert ( afterLast == currentRetOLAPBuffer_->getNext(),
// "Miscalculated the last buffer to read into");
cluster_->afterLastBufferToRead_ = afterLast;
pstate->step_ = ExSeq_OVERFLOW_READ;
}
}
}
break;
// ExSeq_END_OF_PARTITION
//
// Transition to this state from ...
// 1. ExSeq_WORKING_READ -
// Transition from this state to ...
// 1. ExSeq_OVERFLOW_WRITE
// 2. ExSeq_WORKING_RETURN
case ExSeq_END_OF_PARTITION:
{
setPartitionEnd(TRUE);
if (lastRow_ && isUnboundedFollowing())
{
ex_assert(currentHistRowPtr_ != NULL, "ExSequenceTcb::work() - currentHistRowPtr_ is a NULL pointer");
str_cpy_all(lastRow_, currentHistRowPtr_, recLen());
}
if ( isOverflowStarted() ) // we are overflowing
{
cluster_->nextBufferToFlush_ = firstOLAPBuffer_;
// do not flush beyond the current buffer
cluster_->afterLastBufferToFlush_ = currentOLAPBuffer_->getNext();
pstate->step_ = ExSeq_OVERFLOW_WRITE;
}
else
{
pstate->step_ = ExSeq_WORKING_RETURN;
}
}
break;
// ExSeq_OVERFLOW_WRITE
//
// Transition to this state from ...
// 1. ExSeq_WORKING_READ -
// 2. ExSeq_END_OF_PARTITION -
// Remain in this state until ...
// 1. OLAPbuffers are written to oveflow space.
// 2. An error occurs
//
// Transition from this state to ...
// 1. ExSeq_OVERFLOW_READ
// 2. ExSeq_ERROR - If an error occurs
case ExSeq_OVERFLOW_WRITE:
{
if (!overflowEnabled_)
{
// LCOV_EXCL_START
// used for debugging when CmpCommon::getDefault(EXE_BMO_DISABLE_OVERFLOW)is set to off ;
updateDiagsArea(EXE_OLAP_OVERFLOW_NOT_SUPPORTED);
pstate->step_ = ExSeq_ERROR;
break;
// LCOV_EXCL_STOP
}
ex_assert(isUnboundedFollowing(),"");
if ( ! cluster_->flush(&rc_) ) { // flush the buffers
// LCOV_EXCL_START
// if no errors this code path is not visited
if ( rc_ )
{ // some error
updateDiagsArea( rc_);
pstate->step_ = ExSeq_ERROR;
break;
}
// LCOV_EXCL_STOP
// not all the buffers are completely flushed. An I/O is pending
// LCOV_EXCL_START
// maybe we cane remove in the future
return WORK_OK;
// LCOV_EXCL_STOP
}
// At this point -- all the buffers were completely flushed
OLAPBuffersFlushed_ = TRUE;
if (getPartitionEnd())
{
firstOLAPBufferFromOF_ = firstOLAPBuffer_;
numberOfOLAPBuffersFromOF_ = numberOfOLAPBuffers_;
cluster_->nextBufferToRead_ = firstOLAPBufferFromOF_;
// First time we read and fill all the buffers
cluster_->afterLastBufferToRead_ = NULL;
pstate->step_ = ExSeq_OVERFLOW_READ;
}
else
{
pstate->step_ = ExSeq_WORKING_READ;
}
}
break;
// ExSeq_OVERFLOW_READ
//
// Transition to this state from ...
// 1. ExSeq_OVERFLOW_WRITE
// 2. ExSeq_WORKING_RETURN
// Remain in this state until ...
// 1. OLAPbuffers are read from oveflow space.
// 2. An error occurs
//
// Transition from this state to ...
// 1. ExSeq_WORKING_RETURN
// 2. ExSeq_ERROR - If an error occurs
case ExSeq_OVERFLOW_READ:
{
assert(firstOLAPBufferFromOF_ &&
isUnboundedFollowing() );
if ( ! cluster_->read(&rc_) ) {
// LCOV_EXCL_START
if ( rc_ ) { // some error
updateDiagsArea( rc_);
pstate->step_ = ExSeq_ERROR;
break;
}
// LCOV_EXCL_STOP
// not all the buffers are completely read. An I/O is pending
// LCOV_EXCL_START
return WORK_OK;
// LCOV_EXCL_STOP
}
numberOfRowsReturnedBeforeReadOF_ = 0;
pstate->step_ = ExSeq_WORKING_RETURN;
}
break;
// ExSeq_DONE
//
// Transition to the state from ...
// 1. ExSeq_WORKING_RETURN - if all child rows have been processed.
// 2. ExSeq_CANCELLED - if all child rows have been consumed.
// 3. ExSeq_EMPTY - if the request was DOA.
//
// Remain in this state until ...
// 1. The EOD is returned to the parent.
//
// Transition from this state to ...
// 1. ExSeq_EMPTY - In all cases.
//
case ExSeq_DONE:
{
// If there is not any room in the parent's queue,
// try again later.
//
if (qparent_.up->isFull())
// LCOV_EXCL_START
return WORK_OK;
// LCOV_EXCL_STOP
ex_queue_entry * pentry_up = qparent_.up->getTailEntry();
pentry_up->upState.status = ex_queue::Q_NO_DATA;
pentry_up->upState.parentIndex
= pentry_down->downState.parentIndex;
pentry_up->upState.downIndex = qparent_.down->getHeadIndex();
pentry_up->upState.setMatchNo(pstate->matchCount_);
qparent_.down->removeHead();
qparent_.up->insert();
// Re-initialize pstate
//
pstate->step_ = ExSeq_EMPTY;
pstate->matchCount_ = 0;
workAtp_->release();
// Initialize the history buffer in preparation for the
// next request.
//
initializeHistory();
// If there are no more requests, simply return.
//
if (qparent_.down->isEmpty())
return WORK_OK;
// LCOV_EXCL_START
// If we haven't given to our child the new head
// index return and ask to be called again.
//
if (qparent_.down->getHeadIndex() == processedInputs_)
return WORK_CALL_AGAIN;
// Position at the new head of the request queue.
//
pentry_down = qparent_.down->getHeadEntry();
pstate = (ExSequencePrivateState*) pentry_down->pstate;
request = pentry_down->downState.request;
// LCOV_EXCL_STOP
}
break;
} // switch pstate->step_
} // while
}
void ExSequenceTcb::createCluster()
{
MemoryMonitor * memMonitor =
getGlobals()->castToExExeStmtGlobals()->getMemoryMonitor();
ULng32 availableMemory = memMonitor->getPhysMemInBytes() / 100
* myTdb().memUsagePercent_;
// if quota, and it's less than avail memory, then use that lower figure
if ( myTdb().memoryQuotaMB() > 0 &&
myTdb().memoryQuotaMB() * ONE_MEG < availableMemory )
// LCOV_EXCL_START
availableMemory = myTdb().memoryQuotaMB() * ONE_MEG ;
// LCOV_EXCL_STOP
ULng32 minMemQuotaMB = myTdb().isPossibleMultipleCalls() ?
myTdb().memoryQuotaMB() : 0 ;
// in case we recreate, delete the old objects (incl. old scratch file)
if ( cluster_ ) delete cluster_;
if ( clusterDb_ ) delete clusterDb_;
ULng32 minB4Chk = myTdb().getBmoMinMemBeforePressureCheck() * ONE_MEG;
clusterDb_ =
new(heap_) ClusterDB(ClusterDB::SEQUENCE_OLAP,
myTdb().OLAPBufferSize_,
NULL, // workAtp_,
tdb.getExplainNodeId(),
0, // ... hashJoinTdb().extRightRowAtpIndex1_,
0, // ... hashJoinTdb().extRightRowAtpIndex2_,
NULL, // ... rightSearchExpr_,
NULL, // ... buckets_,
1, // bucketCount_ - must be > 0
availableMemory,
memMonitor,
myTdb().pressureThreshold_,
getGlobals()->castToExExeStmtGlobals(),
&rc_,
myTdb().isNoOverflow(),
FALSE, /*isPartialGroupBy*/
0, // ... minBuffersToFlush_,
0, // ... numInBatch_,
myTdb().forceOverflowEvery(),
0, // ... forceHashLoopAfterNumBuffers()
0, // ... forceClusterSplitAfterMB(),
// first time it's uninitialized - would setup later!
ioEventHandler_, // set up at registerSubtasks
this, // the calling tcb
myTdb().scratchThresholdPct_,
myTdb().logDiagnostics(),
FALSE, // bufferedWrites(),
TRUE, // No early overflow based on cmp hints
myTdb().memoryQuotaMB(),
minMemQuotaMB,
minB4Chk, // BmoMinMemBeforePressureCheck
// next 4 are for early overflow (not used for OLAP)
0,
0,
0,
0,
0, // ... Hash-Table not resizable
NULL // getStatsEntry()
);
ex_assert( clusterDb_ , "No memory available for OLAP Operator");
clusterDb_->setScratchIOVectorSize(myTdb().scratchIOVectorSize());
switch(myTdb().getOverFlowMode())
{
// LCOV_EXCL_START
case SQLCLI_OFM_SSD_TYPE:
clusterDb_->setScratchOverflowMode(SCRATCH_SSD);
break;
case SQLCLI_OFM_MMAP_TYPE:
clusterDb_->setScratchOverflowMode(SCRATCH_MMAP);
break;
// LCOV_EXCL_STOP
default:
case SQLCLI_OFM_DISK_TYPE:
clusterDb_->setScratchOverflowMode(SCRATCH_DISK);
break;
}
#ifndef __EID
clusterDb_->setBMOMaxMemThresholdMB(myTdb().getBMOMaxMemThresholdMB());
#endif
cluster_ = new(heap_) Cluster(Cluster::IN_MEMORY,
clusterDb_,
NULL, // ... &buckets_[bucketIdx],
0, // ... bucketsPerCluster,
myTdb().recLen_, // Row Length
false,
false,
0, // ... extRightRowAtpIndex1_,
FALSE, // Use as an "outer cluster"
FALSE, // ... no bit map
NULL, // next cluster,
&rc_);
ex_assert( cluster_ , "No memory available for OLAP Operator");
}
ExFastExtractTcb::ExFastExtractTcb(
const ExFastExtractTdb &fteTdb,
const ex_tcb & childTcb,
ex_globals *glob)
: ex_tcb(fteTdb, 1, glob),
workAtp_(NULL),
outputPool_(NULL),
inputPool_(NULL),
childTcb_(&childTcb)
, inSqlBuffer_(NULL)
, childOutputTD_(NULL)
, sourceFieldsConvIndex_(NULL)
, currBuffer_(NULL)
, bufferAllocFailuresCount_(0)
, modTS_(-1)
{
ex_globals *stmtGlobals = getGlobals();
Space *globSpace = getSpace();
CollHeap *globHeap = getHeap();
heap_ = globHeap;
//convert to non constant to access the members.
ExFastExtractTdb *mytdb = (ExFastExtractTdb*)&fteTdb;
numBuffers_ = mytdb->getNumIOBuffers();
// Allocate queues to communicate with parent
allocateParentQueues(qParent_);
// get the queue that child use to communicate with me
qChild_ = childTcb.getParentQueue();
// Allocate the work ATP
if (myTdb().getWorkCriDesc())
workAtp_ = allocateAtp(myTdb().getWorkCriDesc(), globSpace);
// Fixup expressions
// NOT USED in M9
/*
if (myTdb().getInputExpression())
myTdb().getInputExpression()->fixup(0, getExpressionMode(), this,
globSpace, globHeap);
if (myTdb().getOutputExpression())
myTdb().getOutputExpression()->fixup(0, getExpressionMode(), this,
globSpace, globHeap);
*/
if (myTdb().getChildDataExpr())
myTdb().getChildDataExpr()->fixup(0,getExpressionMode(),this,
globSpace, globHeap, FALSE, glob);
//maybe we can move the below few line to the init section od work methods??
UInt32 numAttrs = myTdb().getChildTuple()->numAttrs();
sourceFieldsConvIndex_ = (int *)((NAHeap *)heap_)->allocateAlignedHeapMemory((UInt32)(sizeof(int) * numAttrs), 512, FALSE);
maxExtractRowLength_ = ROUND8(myTdb().getChildDataRowLen()) ;
const ULng32 sqlBufferSize = maxExtractRowLength_ +
ROUND8(sizeof(SqlBufferNormal)) +
sizeof(tupp_descriptor) +
16 ;//just in case
inSqlBuffer_ = (SqlBuffer *) new (heap_) char[sqlBufferSize];
inSqlBuffer_->driveInit(sqlBufferSize, TRUE, SqlBuffer::NORMAL_);
childOutputTD_ = inSqlBuffer_->add_tuple_desc(maxExtractRowLength_);
endOfData_ = FALSE;
} // ExFastExtractTcb::ExFastExtractTcb
ExWorkProcRetcode ExHdfsFastExtractTcb::work()
{
#ifdef __EID
// This class should not be instantiated in EID.
return WORK_BAD_ERROR;
#else
Lng32 retcode = 0;
SFW_RetCode sfwRetCode = SFW_OK;
ULng32 recSepLen = strlen(myTdb().getRecordSeparator());
ULng32 delimLen = strlen(myTdb().getDelimiter());
ULng32 nullLen =
(myTdb().getNullString() ? strlen(myTdb().getNullString()) : 0);
if (myTdb().getIsHiveInsert())
{
recSepLen = 1;
delimLen = 1;
}
if (getEmptyNullString()) //covers hive null case also
nullLen = 0;
ExOperStats *stats = NULL;
ExFastExtractStats *feStats = getFastExtractStats();
while (TRUE)
{
// if no parent request, return
if (qParent_.down->isEmpty())
return WORK_OK;
ex_queue_entry *pentry_down = qParent_.down->getHeadEntry();
const ex_queue::down_request request = pentry_down->downState.request;
const Lng32 value = pentry_down->downState.requestValue;
ExFastExtractPrivateState &pstate = *((ExFastExtractPrivateState *) pentry_down->pstate);
switch (pstate.step_)
{
case EXTRACT_NOT_STARTED:
{
pstate.step_= EXTRACT_CHECK_MOD_TS;
}
break;
case EXTRACT_CHECK_MOD_TS:
{
if ((! myTdb().getTargetFile()) ||
(myTdb().getModTSforDir() == -1))
{
pstate.step_ = EXTRACT_INITIALIZE;
break;
}
numBuffers_ = 0;
memset (hdfsHost_, '\0', sizeof(hdfsHost_));
strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
hdfsPort_ = myTdb().getHdfsPortNum();
memset (fileName_, '\0', sizeof(fileName_));
memset (targetLocation_, '\0', sizeof(targetLocation_));
snprintf(targetLocation_,999, "%s", myTdb().getTargetName());
retcode = lobInterfaceDataModCheck();
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
NULL, &intParam1,
&cliError,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceDataModCheck",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
if (retcode == 1) // check failed
{
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
pstate.step_= EXTRACT_INITIALIZE;
}
break;
case EXTRACT_INITIALIZE:
{
pstate.processingStarted_ = FALSE;
errorOccurred_ = FALSE;
//Allocate writeBuffers.
numBuffers_ = 1;
for (Int16 i = 0; i < numBuffers_; i++)
{
bool done = false;
Int64 input_datalen = myTdb().getHdfsIoBufferSize();
char * buf_addr = 0;
while ((!done) && input_datalen >= 32 * 1024)
{
buf_addr = 0;
buf_addr = (char *)((NAHeap *)heap_)->allocateAlignedHeapMemory((UInt32)input_datalen, 512, FALSE);
if (buf_addr)
{
done = true;
bufferPool_[i] = new (heap_) IOBuffer((char*) buf_addr, (Int32)input_datalen);
}
else
{
bufferAllocFailuresCount_++;
input_datalen = input_datalen / 2;
}
}
if (!done)
{
numBuffers_ = i;
break ; // if too few buffers have been allocated we will raise
} // an error later
}
if (feStats)
{
feStats->setBufferAllocFailuresCount(bufferAllocFailuresCount_);
feStats->setBuffersCount(numBuffers_);
}
ComDiagsArea *da = NULL;
if (!myTdb().getSkipWritingToFiles())
if (myTdb().getTargetFile() )
{
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
memset (hdfsHost_, '\0', sizeof(hdfsHost_));
strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
hdfsPort_ = myTdb().getHdfsPortNum();
memset (fileName_, '\0', sizeof(fileName_));
memset (targetLocation_, '\0', sizeof(targetLocation_));
time_t t;
time(&t);
char pt[30];
struct tm * curgmtime = gmtime(&t);
strftime(pt, 30, "%Y%m%d%H%M%S", curgmtime);
srand(getpid());
snprintf(targetLocation_,999, "%s", myTdb().getTargetName());
if (myTdb().getIsHiveInsert())
snprintf(fileName_,999, "%s%d-%s-%d", myTdb().getHiveTableName(), fileNum, pt,rand() % 1000);
else
snprintf(fileName_,999, "%s%d-%s-%d", "file", fileNum, pt,rand() % 1000);
if ((isSequenceFile() || myTdb().getBypassLibhdfs()) &&
!sequenceFileWriter_)
{
sequenceFileWriter_ = new(getHeap())
SequenceFileWriter((NAHeap *)getHeap());
sfwRetCode = sequenceFileWriter_->init();
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (isSequenceFile() || myTdb().getBypassLibhdfs())
{
strcat(targetLocation_, "//");
strcat(targetLocation_, fileName_);
if (isSequenceFile())
sfwRetCode = sequenceFileWriter_->open(targetLocation_, SFW_COMP_NONE);
else
sfwRetCode = sequenceFileWriter_->hdfsCreate(targetLocation_, isHdfsCompressed());
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else
{
retcode = 0;
retcode = lobInterfaceCreate();
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBinterfaceCreate",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (feStats)
{
feStats->setPartitionNumber(fileNum);
}
}
else
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"sockets are not supported");
pstate.step_ = EXTRACT_ERROR;
break;
}
for (UInt32 i = 0; i < myTdb().getChildTuple()->numAttrs(); i++)
{
Attributes * attr = myTdb().getChildTableAttr(i);
Attributes * attr2 = myTdb().getChildTableAttr2(i);
ex_conv_clause tempClause;
int convIndex = 0;
sourceFieldsConvIndex_[i] =
tempClause.find_case_index(
attr->getDatatype(),
0,
attr2->getDatatype(),
0,
0);
}
pstate.step_= EXTRACT_PASS_REQUEST_TO_CHILD;
}
break;
case EXTRACT_PASS_REQUEST_TO_CHILD:
{
// pass the parent request to the child downqueue
if (!qChild_.down->isFull())
{
ex_queue_entry * centry = qChild_.down->getTailEntry();
if (request == ex_queue::GET_N)
centry->downState.request = ex_queue::GET_ALL;
else
centry->downState.request = request;
centry->downState.requestValue = pentry_down->downState.requestValue;
centry->downState.parentIndex = qParent_.down->getHeadIndex();
// set the child's input atp
centry->passAtp(pentry_down->getAtp());
qChild_.down->insert();
pstate.processingStarted_ = TRUE;
}
else
// couldn't pass request to child, return
return WORK_OK;
pstate.step_ = EXTRACT_RETURN_ROWS_FROM_CHILD;
}
break;
case EXTRACT_RETURN_ROWS_FROM_CHILD:
{
if ((qChild_.up->isEmpty()))
{
return WORK_OK;
}
if (currBuffer_ == NULL)
{
currBuffer_ = bufferPool_[0];
memset(currBuffer_->data_, '\0',currBuffer_->bufSize_);
currBuffer_->bytesLeft_ = currBuffer_->bufSize_;
}
ex_queue_entry * centry = qChild_.up->getHeadEntry();
ComDiagsArea *cda = NULL;
ex_queue::up_status child_status = centry->upState.status;
switch (child_status)
{
case ex_queue::Q_OK_MMORE:
{
// for the very first row retruned from child
// include the header row if necessary
if ((pstate.matchCount_ == 0) && myTdb().getIncludeHeader())
{
if (!myTdb().getIsAppend())
{
Int32 headerLength = strlen(myTdb().getHeader());
char * target = currBuffer_->data_;
if (headerLength + 1 < currBuffer_->bufSize_)
{
strncpy(target, myTdb().getHeader(),headerLength);
target[headerLength] = '\n' ;
currBuffer_->bytesLeft_ -= headerLength+1 ;
}
else
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"header does not fit in buffer");
pstate.step_ = EXTRACT_ERROR;
break;
}
}
}
tupp_descriptor *dataDesc = childOutputTD_;
ex_expr::exp_return_type expStatus = ex_expr::EXPR_OK;
if (myTdb().getChildDataExpr())
{
UInt32 childTuppIndex = myTdb().childDataTuppIndex_;
workAtp_->getTupp(childTuppIndex) = dataDesc;
// Evaluate the child data expression. If diags are generated they
// will be left in the down entry ATP.
expStatus = myTdb().getChildDataExpr()->eval(centry->getAtp(), workAtp_);
workAtp_->getTupp(childTuppIndex).release();
if (expStatus == ex_expr::EXPR_ERROR)
{
updateWorkATPDiagsArea(centry);
pstate.step_ = EXTRACT_ERROR;
break;
}
} // if (myTdb().getChildDataExpr())
///////////////////////
char * targetData = currBuffer_->data_ + currBuffer_->bufSize_ - currBuffer_->bytesLeft_;
if (targetData == NULL)
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"targetData is NULL");
pstate.step_ = EXTRACT_ERROR;
break;
}
NABoolean convError = FALSE;
convertSQRowToString(nullLen, recSepLen, delimLen, dataDesc,
targetData, convError);
///////////////////////////////
pstate.matchCount_++;
if (!convError)
{
if (feStats)
{
feStats->incProcessedRowsCount();
}
pstate.successRowCount_ ++;
}
else
{
if (feStats)
{
feStats->incErrorRowsCount();
}
pstate.errorRowCount_ ++;
}
if (currBuffer_->bytesLeft_ < (Int32) maxExtractRowLength_)
{
pstate.step_ = EXTRACT_DATA_READY_TO_SEND;
}
}
break;
case ex_queue::Q_NO_DATA:
{
pstate.step_ = EXTRACT_DATA_READY_TO_SEND;
endOfData_ = TRUE;
pstate.processingStarted_ = FALSE ; // so that cancel does not
//wait for this Q_NO_DATA
}
break;
case ex_queue::Q_SQLERROR:
{
pstate.step_ = EXTRACT_ERROR;
}
break;
case ex_queue::Q_INVALID:
{
updateWorkATPDiagsArea(__FILE__,__LINE__,
"ExFastExtractTcb::work() Invalid state returned by child");
pstate.step_ = EXTRACT_ERROR;
}
break;
} // switch
qChild_.up->removeHead();
}
break;
case EXTRACT_DATA_READY_TO_SEND:
{
ssize_t bytesToWrite = currBuffer_->bufSize_ - currBuffer_->bytesLeft_;
if (!myTdb().getSkipWritingToFiles())
if (isSequenceFile())
{
sfwRetCode = sequenceFileWriter_->writeBuffer(currBuffer_->data_, bytesToWrite, myTdb().getRecordSeparator());
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else if (myTdb().getBypassLibhdfs())
{
sfwRetCode = sequenceFileWriter_->hdfsWrite(currBuffer_->data_, bytesToWrite);
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else
{
retcode = 0;
retcode = lobInterfaceInsert(bytesToWrite);
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceInsert",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (feStats)
{
feStats->incReadyToSendBuffersCount();
feStats->incReadyToSendBytes(currBuffer_->bufSize_ - currBuffer_->bytesLeft_);
}
currBuffer_ = NULL;
if (endOfData_)
{
pstate.step_ = EXTRACT_DONE;
}
else
{
pstate.step_ = EXTRACT_RETURN_ROWS_FROM_CHILD;
}
}
break;
case EXTRACT_ERROR:
{
// If there is no room in the parent queue for the reply,
// try again later.
//Later we may split this state into 2 one for cancel and one for query
if (qParent_.up->isFull())
return WORK_OK;
// Cancel the child request - there must be a child request in
// progress to get to the ERROR state.
if (pstate.processingStarted_)
{
qChild_.down->cancelRequestWithParentIndex(qParent_.down->getHeadIndex());
//pstate.processingStarted_ = FALSE;
}
while (pstate.processingStarted_ && pstate.step_ == EXTRACT_ERROR)
{
if (qChild_.up->isEmpty())
return WORK_OK;
ex_queue_entry * childEntry = qChild_.up->getHeadEntry();
ex_queue::up_status childStatus = childEntry->upState.status;
if (childStatus == ex_queue::Q_NO_DATA)
{
pstate.step_ = EXTRACT_DONE;
pstate.processingStarted_ = FALSE;
}
qChild_.up->removeHead();
}
ex_queue_entry *pentry_up = qParent_.up->getTailEntry();
pentry_up->copyAtp(pentry_down);
// Construct and return the error row.
//
if (workAtp_->getDiagsArea())
{
ComDiagsArea *diagsArea = pentry_up->getDiagsArea();
if (diagsArea == NULL)
{
diagsArea = ComDiagsArea::allocate(getGlobals()->getDefaultHeap());
pentry_up->setDiagsArea(diagsArea);
}
pentry_up->getDiagsArea()->mergeAfter(*workAtp_->getDiagsArea());
workAtp_->setDiagsArea(NULL);
}
pentry_up->upState.status = ex_queue::Q_SQLERROR;
pentry_up->upState.parentIndex
= pentry_down->downState.parentIndex;
pentry_up->upState.downIndex = qParent_.down->getHeadIndex();
pentry_up->upState.setMatchNo(pstate.matchCount_);
qParent_.up->insert();
//
errorOccurred_ = TRUE;
pstate.step_ = EXTRACT_DONE;
}
break;
case EXTRACT_DONE:
{
// If there is no room in the parent queue for the reply,
// try again later.
//
if (qParent_.up->isFull())
return WORK_OK;
if (!myTdb().getSkipWritingToFiles())
if (isSequenceFile())
{
sfwRetCode = sequenceFileWriter_->close();
if (!errorOccurred_ && sfwRetCode != SFW_OK )
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else if (myTdb().getBypassLibhdfs())
{
if (sequenceFileWriter_)
{
sfwRetCode = sequenceFileWriter_->hdfsClose();
if (!errorOccurred_ && sfwRetCode != SFW_OK )
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
}
else
{
retcode = lobInterfaceClose();
if (! errorOccurred_ && retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL,
(char*)"ExpLOBinterfaceCloseFile",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
//insertUpQueueEntry will insert Q_NO_DATA into the up queue and
//remove the head of the down queue
insertUpQueueEntry(ex_queue::Q_NO_DATA, NULL, TRUE);
errorOccurred_ = FALSE;
endOfData_ = FALSE;
//we need to set the next state so that the query can get re-executed
//and we start from the beginning again. Not sure if pstate will be
//valid anymore because insertUpQueueEntry() might have cleared it
//already.
pstate.step_ = EXTRACT_NOT_STARTED;
//exit out now and not break.
return WORK_OK;
}
break;
default:
{
ex_assert(FALSE, "Invalid state in ExHdfsFastExtractTcb ");
}
break;
} // switch(pstate.step_)
} // while
return WORK_OK;
#endif
}//ExHdfsFastExtractTcb::work()
void ExHdfsFastExtractTcb::convertSQRowToString(ULng32 nullLen,
ULng32 recSepLen,
ULng32 delimLen,
tupp_descriptor* dataDesc,
char* targetData,
NABoolean & convError) {
char* childRow = dataDesc->getTupleAddress();
ULng32 childRowLen = dataDesc->getAllocatedSize();
UInt32 vcActualLen = 0;
for (UInt32 i = 0; i < myTdb().getChildTuple()->numAttrs(); i++) {
Attributes * attr = myTdb().getChildTableAttr(i);
Attributes * attr2 = myTdb().getChildTableAttr2(i);
char *childColData = NULL; //childRow + attr->getOffset();
UInt32 childColLen = 0;
UInt32 maxTargetColLen = attr2->getLength();
//format is aligned format--
//----------
// field is varchar
if (attr->getVCIndicatorLength() > 0) {
childColData = childRow + *((UInt32*) (childRow + attr->getVoaOffset()));
childColLen = attr->getLength(childColData);
childColData += attr->getVCIndicatorLength();
} else { //source is fixed length
childColData = childRow + attr->getOffset();
childColLen = attr->getLength();
if ((attr->getCharSet() == CharInfo::ISO88591
|| attr->getCharSet() == CharInfo::UTF8) && childColLen > 0) {
// trim trailing blanks
while (childColLen > 0 && childColData[childColLen - 1] == ' ') {
childColLen--;
}
} else if (attr->getCharSet() == CharInfo::UCS2 && childColLen > 1) {
ex_assert(childColLen % 2 == 0, "invalid ucs2");
NAWchar* wChildColData = (NAWchar*) childColData;
Int32 wChildColLen = childColLen / 2;
while (wChildColLen > 0 && wChildColData[wChildColLen - 1] == L' ') {
wChildColLen--;
}
childColLen = wChildColLen * 2;
}
}
if (attr->getNullFlag()
&& ExpAlignedFormat::isNullValue(childRow + attr->getNullIndOffset(),
attr->getNullBitIndex())) {
// source is a null value.
nullLen = 0;
if (myTdb().getNullString()) {
nullLen = strlen(myTdb().getNullString());
memcpy(targetData, myTdb().getNullString(), nullLen);
}
targetData += nullLen;
currBuffer_->bytesLeft_ -= nullLen;
} else {
switch ((conv_case_index) sourceFieldsConvIndex_[i]) {
case CONV_ASCII_V_V:
case CONV_ASCII_F_V:
case CONV_UNICODE_V_V:
case CONV_UNICODE_F_V: {
if (childColLen > 0) {
memcpy(targetData, childColData, childColLen);
targetData += childColLen;
currBuffer_->bytesLeft_ -= childColLen;
}
}
break;
default:
ex_expr::exp_return_type err = convDoIt(childColData, childColLen,
attr->getDatatype(), attr->getPrecision(), attr->getScale(),
targetData, attr2->getLength(), attr2->getDatatype(),
attr2->getPrecision(), attr2->getScale(), (char*) &vcActualLen,
sizeof(vcActualLen), 0, 0, // diags may need to be added
(conv_case_index) sourceFieldsConvIndex_[i]);
if (err == ex_expr::EXPR_ERROR) {
convError = TRUE;
// not exit loop -- we will log the errenous row later
// do not cancel processing for this type of error???
}
targetData += vcActualLen;
currBuffer_->bytesLeft_ -= vcActualLen;
break;
} //switch
}
if (i == myTdb().getChildTuple()->numAttrs() - 1) {
strncpy(targetData, myTdb().getRecordSeparator(), recSepLen);
targetData += recSepLen;
currBuffer_->bytesLeft_ -= recSepLen;
} else {
strncpy(targetData, myTdb().getDelimiter(), delimLen);
targetData += delimLen;
currBuffer_->bytesLeft_ -= delimLen;
}
}
}
NABoolean ExHdfsFastExtractTcb::isHdfsCompressed()
{
return myTdb().getHdfsCompressed();
}
NABoolean ExHdfsFastExtractTcb::isSequenceFile()
{
return myTdb().getIsSequenceFile();
}
