ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
CollHeap*h,
ComDiagsArea** diagsArea)
{
Lng32 rc;
Lng32 lobOperStatus = checkLobOperStatus();
if (lobOperStatus == DO_NOTHING_)
return ex_expr::EXPR_OK;
char * result = op_data[0];
// get the lob name where data need to be inserted
char tgtLobNameBuf[100];
char * tgtLobName = ExpGetLOBname(objectUID_, lobNum(), tgtLobNameBuf, 100);
if (tgtLobName == NULL)
return ex_expr::EXPR_ERROR;
// call function with the lobname and source value
// to insert it in the LOB.
// Get back offset and len of the LOB.
Int64 descSyskey = 0;
char * lobHandle = NULL;
Lng32 handleLen = 0;
char lobHandleBuf[LOB_HANDLE_LEN];
// if (getExeGlobals()->lobGlobals()->getCurrLobOperInProgress())
if (lobOperStatus == CHECK_STATUS_)
{
lobHandle = lobHandleSaved_;
handleLen = lobHandleLenSaved_;
}
else
{
Int64 descTS = NA_JulianTimestamp();
lobHandle = lobHandleBuf;
ExpLOBoper::genLOBhandle(objectUID_, lobNum(), (short)lobStorageType(),
-1, descTS, -1,
descSchNameLen_, descSchName(),
handleLen, lobHandle);
}
LobsSubOper so = Lob_None;
if (fromFile())
so = Lob_File;
else if (fromString() || fromLoad())
so = Lob_Memory;
else if (fromLob())
so = Lob_Foreign_Lob;
else if (fromBuffer())
so = Lob_Buffer;
Lng32 waitedOp = 0;
#ifdef __EID
waitedOp = 0; // nowaited op from EID/TSE process
#else
waitedOp = 1;
#endif
//temptemp. Remove after ExLobsOper adds nowaited support.
waitedOp = 1;
// temp. Pass lobLen. When ExLobsOper fixes it so len is not needed during
// lob desc update, then remove this.
Int64 lobLen = getOperand(1)->getLength();
blackBoxLen_ = 0;
if (fromExternal())
{
blackBoxLen_ = getOperand(1)->getLength();
str_cpy_and_null(blackBox_, op_data[1], (Lng32)blackBoxLen_,
'\0', ' ', TRUE);
}
Lng32 cliError = 0;
char * lobData = NULL;
lobData= new(h) char[lobLen];
//send lobData only if it's a lob_file operation
if (so == Lob_File)
{
str_cpy_and_null(lobData,op_data[1],lobLen,'\0',' ',TRUE);
}
if (so == Lob_Buffer)
{
memcpy(&lobLen, op_data[2],sizeof(Int64));
}
LobsOper lo ;
if (lobOperStatus == CHECK_STATUS_)
lo = Lob_Check_Status;
else if (lobHandle == NULL)
lo = Lob_InsertDataSimple;
else
lo = Lob_InsertDesc;
rc = ExpLOBInterfaceInsert
(getExeGlobals()->lobGlobal(),
tgtLobName,
lobStorageLocation(),
lobStorageType(),
getLobHdfsServer(), getLobHdfsPort(),
handleLen, lobHandle,
&outHandleLen_, outLobHandle_,
blackBoxLen_, blackBox_,
requestTag_,
getExeGlobals()->lobGlobals()->xnId(),
descSyskey,
lo,
&cliError,
so,
waitedOp,
lobData, lobLen, getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
if (rc == LOB_ACCESS_PREEMPT)
{
// save the handle so it could be used after return from preempt
lobHandleLenSaved_ = handleLen;
str_cpy_all(lobHandleSaved_, lobHandle, handleLen);
return ex_expr::EXPR_PREEMPT;
}
if (rc < 0)
{
Lng32 intParam1 = -rc;
ExRaiseSqlError(h, diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceInsert",
getLobErrStr(intParam1));
return ex_expr::EXPR_ERROR;
}
// extract and update lob handle with the returned values
if (outHandleLen_ > 0)
{
ExpLOBoper::extractFromLOBhandle(NULL, NULL, NULL, NULL, &descSyskey,
NULL, NULL, NULL, outLobHandle_);
ExpLOBoper::updLOBhandle(descSyskey, 0, lobHandle);
}
str_cpy_all(result, lobHandle, handleLen);
getOperand(0)->setVarLength(handleLen, op_data[-MAX_OPERANDS]);
if (NOT fromExternal())
{
getExeGlobals()->lobGlobals()->lobLoadInfo()->
setLobHandle(lobNum(), handleLen, lobHandle);
}
return ex_expr::EXPR_OK;
}
ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen,
char * handle,
char *op_data[],
CollHeap*h,
ComDiagsArea** diagsArea)
{
Lng32 rc = 0;
Lng32 lobOperStatus = checkLobOperStatus();
if (lobOperStatus == DO_NOTHING_)
return ex_expr::EXPR_OK;
Lng32 lobType;
Int64 uid;
Lng32 lobNum;
Int64 lobLen;
// Lng32 flags;
Int64 descSyskey = -1;
// Int64 descTS = -1;
short numChunks = 0;
// short schNameLen = 0;
// char schName[500];
extractFromLOBhandle(NULL, &lobType, &lobNum, &uid,
&descSyskey, NULL, //&descTS,
NULL, NULL, //&schNameLen, schName,
handle);
// get the lob name where data need to be inserted
char tgtLobNameBuf[100];
char * tgtLobName = ExpGetLOBname(uid, lobNum, tgtLobNameBuf, 100);
if (tgtLobName == NULL)
return ex_expr::EXPR_ERROR;
lobLen = getOperand(1)->getLength();
char * lobData = NULL;
if(fromFile())
{
lobData = new (h) char[lobLen];
str_cpy_and_null(lobData,op_data[1],lobLen,'\0',' ',TRUE);
}
else
lobData = op_data[1];
if (fromBuffer())
{
memcpy(&lobLen, op_data[2],sizeof(Int64)); // user specified buffer length
memcpy(lobData,op_data[1],sizeof(Int64)); // user buffer address
}
LobsOper lo ;
if (lobOperStatus == CHECK_STATUS_)
lo = Lob_Check_Status;
else if (handle == NULL)
lo = Lob_InsertDataSimple;
else
lo = Lob_InsertData;
LobsSubOper so = Lob_None;
if (fromFile())
so = Lob_File;
else if (fromString() || fromLoad())
so = Lob_Memory;
else if (fromLob())
so = Lob_Foreign_Lob;
else if(fromBuffer())
so = Lob_Buffer;
Lng32 waitedOp = 0;
#ifdef __EID
waitedOp = 0; // nowaited op from EID/TSE process
#else
waitedOp = 1;
#endif
//temptemp. Remove after ExLobsOper adds nowaited support.
waitedOp = 1;
Lng32 cliError = 0;
blackBoxLen_ = 0;
if (fromLob())
{
Int64 srcDescKey = -1;
Int64 srcDescTS = -1;
char srcLobNameBuf[100];
char * srcLobName = NULL;
short srcSchNameLen = 0;
char srcSchName[500];
extractFromLOBhandle(NULL, &lobType, &lobNum, &uid,
&srcDescKey, &srcDescTS,
&srcSchNameLen, srcSchName,
op_data[1]);
// get the lob name where data will be read from
srcLobName = ExpGetLOBname(uid, lobNum, srcLobNameBuf, 100);
if (srcLobName == NULL)
return ex_expr::EXPR_ERROR;
}
else
{
rc = ExpLOBInterfaceInsert(getExeGlobals()->lobGlobal(),
tgtLobName,
lobStorageLocation(),
lobType,
getLobHdfsServer(), getLobHdfsPort(),
handleLen, handle,
&outHandleLen_, outLobHandle_,
blackBoxLen_, blackBox_,
requestTag_,
getExeGlobals()->lobGlobals()->xnId(),
descSyskey,
lo,
&cliError,
so,
waitedOp,
lobData,
lobLen,getLobMaxSize(),
getLobMaxChunkMemSize(),
getLobGCLimit());
}
if (rc == LOB_ACCESS_PREEMPT)
{
return ex_expr::EXPR_PREEMPT;
}
if (rc < 0)
{
Lng32 intParam1 = -rc;
ExRaiseSqlError(h, diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceInsert",
getLobErrStr(intParam1));
return ex_expr::EXPR_ERROR;
}
return ex_expr::EXPR_OK;
}
ex_expr::exp_return_type ExpLOBconvert::eval(char *op_data[],
CollHeap*h,
ComDiagsArea** diagsArea)
{
Lng32 rc = 0;
Lng32 lobOperStatus = checkLobOperStatus();
if (lobOperStatus == DO_NOTHING_)
return ex_expr::EXPR_OK;
char * result = op_data[0];
char * lobHandle = op_data[1];
char *tgtFileName = NULL;
Int32 handleLen = getOperand(1)->getLength(op_data[-MAX_OPERANDS+1]);
Int64 uid;
Lng32 lobType;
Lng32 lobNum;
Int64 descKey;
Int64 descTS;
Int16 flags;
short schNameLen = 0;
char schName[500];
LobsSubOper so;
Lng32 cliError = 0;
Lng32 waitedOp = 0;
Int64 lobLen = 0;
char *lobData = NULL;
#ifdef __EID
waitedOp = 0; // nowaited op from EID/TSE process
#else
waitedOp = 1;
#endif
//temptemp. Remove after ExLobsOper adds nowaited support.
waitedOp = 1;
extractFromLOBhandle(&flags, &lobType, &lobNum, &uid,
&descKey, &descTS,
&schNameLen, schName,
lobHandle);
// get the lob name where data need to be inserted
char lobNameBuf[100];
char * lobName = ExpGetLOBname(uid, lobNum, lobNameBuf, 100);
if(toFile())
{
so = Lob_File;
tgtFileName = tgtFileName_;
rc = ExpLOBInterfaceSelect(getExeGlobals()->lobGlobal(),
lobName,
lobStorageLocation(),
lobType,
getLobHdfsServer(), getLobHdfsPort(),
handleLen, lobHandle,
requestTag_,
so,
getExeGlobals()->lobGlobals()->xnId(),
(lobOperStatus == CHECK_STATUS_ ? 1 : 0),
waitedOp,
0, lobLen, lobLen, tgtFileName,getLobMaxChunkMemSize());
}
else if (toString())
{
so = Lob_Memory;
if (lobName == NULL)
return ex_expr::EXPR_ERROR;
lobLen = getConvertSize();
lobData = new(h) char[(Lng32)lobLen];
rc = ExpLOBInterfaceSelect(getExeGlobals()->lobGlobal(),
lobName,
lobStorageLocation(),
lobType,
getLobHdfsServer(), getLobHdfsPort(),
handleLen, lobHandle,
requestTag_,
so,
getExeGlobals()->lobGlobals()->xnId(),
(lobOperStatus == CHECK_STATUS_ ? 1 : 0),
waitedOp,
0, lobLen, lobLen, lobData,getLobMaxChunkMemSize());
if (rc == LOB_ACCESS_PREEMPT)
{
return ex_expr::EXPR_PREEMPT;
}
if (rc < 0)
{
Lng32 intParam1 = -rc;
ExRaiseSqlError(h, diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceSelect",
getLobErrStr(intParam1));
return ex_expr::EXPR_ERROR;
}
// store the length of substring in the varlen indicator.
if (getOperand(0)->getVCIndicatorLength() > 0)
{
getOperand(0)->setVarLength((UInt32)lobLen, op_data[-MAX_OPERANDS] );
if (lobLen > 0)
str_cpy_all(op_data[0], lobData, (Lng32)lobLen);
}
else
{
str_pad(result, getOperand(0)->getLength());
str_cpy_all(result, lobData, strlen(lobData));
}
NADELETEBASIC(lobData, h);
}
else
return ex_expr::EXPR_ERROR;
return ex_expr::EXPR_OK;
}
ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
CollHeap*h,
ComDiagsArea** diagsArea)
{
Lng32 rc;
Lng32 lobOperStatus = checkLobOperStatus();
if (lobOperStatus == DO_NOTHING_)
return ex_expr::EXPR_OK;
char * result = op_data[0];
char * lobHandle = NULL;
Lng32 handleLen = 0;
Lng32 sLobType;
Int64 sUid;
Lng32 sLobNum;
Int64 sDescSyskey = -1;
Int64 sDescTS = -1;
Int16 sFlags;
short sSchNameLen = 0;
char sSchName[500];
if (getOperand(2)->getNullFlag() &&
nullValue_)
{
ex_expr::exp_return_type err = insertDesc(op_data, h, diagsArea);
if (err == ex_expr::EXPR_ERROR)
return err;
char * handle = op_data[0];
handleLen = getOperand(0)->getLength();
err = insertData(handleLen, handle, op_data, h, diagsArea);
return err;
}
else
{
lobHandle = op_data[2];
handleLen = getOperand(2)->getLength(op_data[-MAX_OPERANDS+2]);
}
extractFromLOBhandle(&sFlags, &sLobType, &sLobNum, &sUid,
&sDescSyskey, &sDescTS,
&sSchNameLen, sSchName,
lobHandle); //op_data[2]);
// get the lob name where data need to be updated
char tgtLobNameBuf[100];
char * tgtLobName = ExpGetLOBname(sUid, sLobNum, tgtLobNameBuf, 100);
if (tgtLobName == NULL)
return ex_expr::EXPR_ERROR;
char fromLobNameBuf[100];
char * fromLobName = NULL;
Int64 fromDescKey = 0;
Int64 fromDescTS = 0;
short fromSchNameLen = 0;
char fromSchName[500];
if (0) // TBD. fromLob())
{
Lng32 fromLobType;
Int64 fromLobUid;
Lng32 fromLobNum;
Int16 fromFlags;
extractFromLOBhandle(&fromFlags, &fromLobType, &fromLobNum, &fromLobUid,
&fromDescKey, &fromDescTS,
&fromSchNameLen, fromSchName,
op_data[1]);
// get the lob name where data will be read from
fromLobName = ExpGetLOBname(fromLobUid, fromLobNum, fromLobNameBuf, 100);
if (fromLobName == NULL)
return ex_expr::EXPR_ERROR;
}
LobsSubOper so = Lob_None;
if (fromFile())
so = Lob_File;
else if (fromString())
so = Lob_Memory;
else if (fromLob())
so = Lob_Foreign_Lob;
else if (fromBuffer())
so= Lob_Buffer;
Lng32 waitedOp = 0;
#ifdef __EID
waitedOp = 0; // nowaited op from EID/TSE process
#else
waitedOp = 1;
#endif
//temptemp. Remove after ExLobsOper adds nowaited support.
waitedOp = 1;
Lng32 cliError = 0;
// call function with the lobname and source value
// to update it in the LOB.
// Get back offset and len of the LOB.
// Int64 offset = 0;
Int64 lobLen = getOperand(1)->getLength();
char * data = op_data[1];
if (fromBuffer())
{
memcpy(&lobLen, op_data[3],sizeof(Int64)); // user specified buffer length
memcpy(data,op_data[1],sizeof(Int64)); // user buffer address
}
if (isAppend())
{
rc = ExpLOBInterfaceUpdateAppend
(getExeGlobals()->lobGlobal(),
getLobHdfsServer(),
getLobHdfsPort(),
tgtLobName,
lobStorageLocation(),
handleLen, lobHandle,
&outHandleLen_, outLobHandle_,
requestTag_,
getExeGlobals()->lobGlobals()->xnId(),
(lobOperStatus == CHECK_STATUS_ ? 1 : 0),
waitedOp,
so,
sDescSyskey,
lobLen,
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
else
{
rc = ExpLOBInterfaceUpdate
(getExeGlobals()->lobGlobal(),
getLobHdfsServer(),
getLobHdfsPort(),
tgtLobName,
lobStorageLocation(),
handleLen, lobHandle,
&outHandleLen_, outLobHandle_,
requestTag_,
getExeGlobals()->lobGlobals()->xnId(),
(lobOperStatus == CHECK_STATUS_ ? 1 : 0),
waitedOp,
so,
sDescSyskey,
lobLen,
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
if (rc < 0)
{
Lng32 intParam1 = -rc;
ExRaiseSqlError(h, diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceUpdate",
getLobErrStr(intParam1));
return ex_expr::EXPR_ERROR;
}
// update lob handle with the returned values
str_cpy_all(result, lobHandle, handleLen);
// str_cpy_all(result, op_data[2], handleLen);
// ExpLOBoper::updLOBhandle(sDescSyskey, 0, result);
getOperand(0)->setVarLength(handleLen, op_data[-MAX_OPERANDS]);
return ex_expr::EXPR_OK;
}
ex_expr::exp_return_type ExpLOBdelete::eval(char *op_data[],
CollHeap*h,
ComDiagsArea** diagsArea)
{
Lng32 rc = 0;
Lng32 lobOperStatus = checkLobOperStatus();
if (lobOperStatus == DO_NOTHING_)
return ex_expr::EXPR_OK;
char * result = op_data[0];
Lng32 lobType;
Int64 uid;
Lng32 lobNum;
Int64 descSyskey;
Int64 descTS = -1;
extractFromLOBhandle(NULL, &lobType, &lobNum, &uid,
&descSyskey, NULL, //descTS,
NULL, NULL,
op_data[1]);
Lng32 handleLen = getOperand(1)->getLength(op_data[-MAX_OPERANDS+1]);
// get the lob name where data need to be deleted
char lobNameBuf[100];
char * lobName = ExpGetLOBname(uid, lobNum, lobNameBuf, 100);
if (lobName == NULL)
return ex_expr::EXPR_ERROR;
Lng32 waitedOp = 0;
#ifdef __EID
waitedOp = 0; // nowaited op from EID/TSE process
#else
waitedOp = 1;
#endif
//temptemp. Remove after ExLobsOper adds nowaited support.
waitedOp = 1;
Lng32 cliError = 0;
// call function with the lobname and offset to delete it.
rc = ExpLOBInterfaceDelete
(getExeGlobals()->lobGlobal(),
getLobHdfsServer(),
getLobHdfsPort(),
lobName,
lobStorageLocation(),
handleLen, op_data[1],
requestTag_,
getExeGlobals()->lobGlobals()->xnId(),
descSyskey,
// (getExeGlobals()->lobGlobals()->getCurrLobOperInProgress() ? 1 : 0),
(lobOperStatus == CHECK_STATUS_ ? 1 : 0),
waitedOp);
if (rc == LOB_ACCESS_PREEMPT)
{
return ex_expr::EXPR_PREEMPT;
}
if (rc < 0)
{
Lng32 intParam1 = -rc;
ExRaiseSqlError(h, diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceDelete",
getLobErrStr(intParam1));
return ex_expr::EXPR_ERROR;
}
return ex_expr::EXPR_OK;
}
//////////////////////////////////////////////////////
// work() for ExExeUtilHiveTruncateLegacyTcb
//////////////////////////////////////////////////////
short ExExeUtilHiveTruncateLegacyTcb::work()
{
short rc = 0;
Lng32 cliRC = 0;
// if no parent request, return
if (qparent_.down->isEmpty())
return WORK_OK;
// if no room in up queue, won't be able to return data/status.
// Come back later.
if (qparent_.up->isFull())
return WORK_OK;
ex_queue_entry * pentry_down = qparent_.down->getHeadEntry();
ExExeUtilPrivateState & pstate = *((ExExeUtilPrivateState*) pentry_down->pstate);
while (1)
{
switch (step_)
{
case INITIAL_:
{
step_ = EMPTY_DIRECTORY_;
}
break;
case EMPTY_DIRECTORY_:
{
cliRC = ExpLOBinterfaceEmptyDirectory(
lobGlob_,
(char*)"", //name is empty
(htTdb().getPartnLocation() ?
htTdb().getPartnLocation() :
htTdb().getTableLocation()),
Lob_HDFS_File,
htTdb().getHdfsHost(),
htTdb().getHdfsPort(),
0 ,
1 ,
0);
if (cliRC != 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -cliRC;
ExRaiseSqlError(getHeap(), &diagsArea_,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
NULL, &intParam1,
&cliError,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceEmptyDirectory",
getLobErrStr(intParam1));
char reason[200];
strcpy(reason, " ");
if (intParam1 == LOB_DIR_NAME_ERROR)
{
if (htTdb().getPartnLocation())
strcpy(reason, "Reason: specified partition does not exist");
else
strcpy(reason, "Reason: specified table location does not exist");
}
else if (intParam1 == LOB_DATA_FILE_DELETE_ERROR)
{
strcpy(reason, "Reason: error occurred during deletion of one or more files at the specified location");
}
ExRaiseSqlError(getHeap(), &diagsArea_,
(ExeErrorCode)(EXE_HIVE_TRUNCATE_ERROR), NULL,
NULL, NULL, NULL,
reason,
NULL, NULL);
step_ = ERROR_;
}
else
{
step_= DONE_;
}
}
break;
case ERROR_:
{
if (handleError())
return WORK_OK;
step_ = DONE_;
}
break;
case DONE_:
{
if (handleDone())
return WORK_OK;
step_ = INITIAL_;
return WORK_OK;
}
break;
} // switch
} // while
}
ExWorkProcRetcode ExHdfsFastExtractTcb::work()
{
#ifdef __EID
// This class should not be instantiated in EID.
return WORK_BAD_ERROR;
#else
Lng32 retcode = 0;
SFW_RetCode sfwRetCode = SFW_OK;
ULng32 recSepLen = strlen(myTdb().getRecordSeparator());
ULng32 delimLen = strlen(myTdb().getDelimiter());
ULng32 nullLen =
(myTdb().getNullString() ? strlen(myTdb().getNullString()) : 0);
if (myTdb().getIsHiveInsert())
{
recSepLen = 1;
delimLen = 1;
}
if (getEmptyNullString()) //covers hive null case also
nullLen = 0;
ExOperStats *stats = NULL;
ExFastExtractStats *feStats = getFastExtractStats();
while (TRUE)
{
// if no parent request, return
if (qParent_.down->isEmpty())
return WORK_OK;
ex_queue_entry *pentry_down = qParent_.down->getHeadEntry();
const ex_queue::down_request request = pentry_down->downState.request;
const Lng32 value = pentry_down->downState.requestValue;
ExFastExtractPrivateState &pstate = *((ExFastExtractPrivateState *) pentry_down->pstate);
switch (pstate.step_)
{
case EXTRACT_NOT_STARTED:
{
pstate.step_= EXTRACT_CHECK_MOD_TS;
}
break;
case EXTRACT_CHECK_MOD_TS:
{
if ((! myTdb().getTargetFile()) ||
(myTdb().getModTSforDir() == -1))
{
pstate.step_ = EXTRACT_INITIALIZE;
break;
}
numBuffers_ = 0;
memset (hdfsHost_, '\0', sizeof(hdfsHost_));
strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
hdfsPort_ = myTdb().getHdfsPortNum();
memset (fileName_, '\0', sizeof(fileName_));
memset (targetLocation_, '\0', sizeof(targetLocation_));
snprintf(targetLocation_,999, "%s", myTdb().getTargetName());
retcode = lobInterfaceDataModCheck();
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
NULL, &intParam1,
&cliError,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceDataModCheck",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
if (retcode == 1) // check failed
{
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
pstate.step_= EXTRACT_INITIALIZE;
}
break;
case EXTRACT_INITIALIZE:
{
pstate.processingStarted_ = FALSE;
errorOccurred_ = FALSE;
//Allocate writeBuffers.
numBuffers_ = 1;
for (Int16 i = 0; i < numBuffers_; i++)
{
bool done = false;
Int64 input_datalen = myTdb().getHdfsIoBufferSize();
char * buf_addr = 0;
while ((!done) && input_datalen >= 32 * 1024)
{
buf_addr = 0;
buf_addr = (char *)((NAHeap *)heap_)->allocateAlignedHeapMemory((UInt32)input_datalen, 512, FALSE);
if (buf_addr)
{
done = true;
bufferPool_[i] = new (heap_) IOBuffer((char*) buf_addr, (Int32)input_datalen);
}
else
{
bufferAllocFailuresCount_++;
input_datalen = input_datalen / 2;
}
}
if (!done)
{
numBuffers_ = i;
break ; // if too few buffers have been allocated we will raise
} // an error later
}
if (feStats)
{
feStats->setBufferAllocFailuresCount(bufferAllocFailuresCount_);
feStats->setBuffersCount(numBuffers_);
}
ComDiagsArea *da = NULL;
if (!myTdb().getSkipWritingToFiles())
if (myTdb().getTargetFile() )
{
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
memset (hdfsHost_, '\0', sizeof(hdfsHost_));
strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
hdfsPort_ = myTdb().getHdfsPortNum();
memset (fileName_, '\0', sizeof(fileName_));
memset (targetLocation_, '\0', sizeof(targetLocation_));
time_t t;
time(&t);
char pt[30];
struct tm * curgmtime = gmtime(&t);
strftime(pt, 30, "%Y%m%d%H%M%S", curgmtime);
srand(getpid());
snprintf(targetLocation_,999, "%s", myTdb().getTargetName());
if (myTdb().getIsHiveInsert())
snprintf(fileName_,999, "%s%d-%s-%d", myTdb().getHiveTableName(), fileNum, pt,rand() % 1000);
else
snprintf(fileName_,999, "%s%d-%s-%d", "file", fileNum, pt,rand() % 1000);
if ((isSequenceFile() || myTdb().getBypassLibhdfs()) &&
!sequenceFileWriter_)
{
sequenceFileWriter_ = new(getHeap())
SequenceFileWriter((NAHeap *)getHeap());
sfwRetCode = sequenceFileWriter_->init();
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (isSequenceFile() || myTdb().getBypassLibhdfs())
{
strcat(targetLocation_, "//");
strcat(targetLocation_, fileName_);
if (isSequenceFile())
sfwRetCode = sequenceFileWriter_->open(targetLocation_, SFW_COMP_NONE);
else
sfwRetCode = sequenceFileWriter_->hdfsCreate(targetLocation_, isHdfsCompressed());
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else
{
retcode = 0;
retcode = lobInterfaceCreate();
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBinterfaceCreate",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (feStats)
{
feStats->setPartitionNumber(fileNum);
}
}
else
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"sockets are not supported");
pstate.step_ = EXTRACT_ERROR;
break;
}
for (UInt32 i = 0; i < myTdb().getChildTuple()->numAttrs(); i++)
{
Attributes * attr = myTdb().getChildTableAttr(i);
Attributes * attr2 = myTdb().getChildTableAttr2(i);
ex_conv_clause tempClause;
int convIndex = 0;
sourceFieldsConvIndex_[i] =
tempClause.find_case_index(
attr->getDatatype(),
0,
attr2->getDatatype(),
0,
0);
}
pstate.step_= EXTRACT_PASS_REQUEST_TO_CHILD;
}
break;
case EXTRACT_PASS_REQUEST_TO_CHILD:
{
// pass the parent request to the child downqueue
if (!qChild_.down->isFull())
{
ex_queue_entry * centry = qChild_.down->getTailEntry();
if (request == ex_queue::GET_N)
centry->downState.request = ex_queue::GET_ALL;
else
centry->downState.request = request;
centry->downState.requestValue = pentry_down->downState.requestValue;
centry->downState.parentIndex = qParent_.down->getHeadIndex();
// set the child's input atp
centry->passAtp(pentry_down->getAtp());
qChild_.down->insert();
pstate.processingStarted_ = TRUE;
}
else
// couldn't pass request to child, return
return WORK_OK;
pstate.step_ = EXTRACT_RETURN_ROWS_FROM_CHILD;
}
break;
case EXTRACT_RETURN_ROWS_FROM_CHILD:
{
if ((qChild_.up->isEmpty()))
{
return WORK_OK;
}
if (currBuffer_ == NULL)
{
currBuffer_ = bufferPool_[0];
memset(currBuffer_->data_, '\0',currBuffer_->bufSize_);
currBuffer_->bytesLeft_ = currBuffer_->bufSize_;
}
ex_queue_entry * centry = qChild_.up->getHeadEntry();
ComDiagsArea *cda = NULL;
ex_queue::up_status child_status = centry->upState.status;
switch (child_status)
{
case ex_queue::Q_OK_MMORE:
{
// for the very first row retruned from child
// include the header row if necessary
if ((pstate.matchCount_ == 0) && myTdb().getIncludeHeader())
{
if (!myTdb().getIsAppend())
{
Int32 headerLength = strlen(myTdb().getHeader());
char * target = currBuffer_->data_;
if (headerLength + 1 < currBuffer_->bufSize_)
{
strncpy(target, myTdb().getHeader(),headerLength);
target[headerLength] = '\n' ;
currBuffer_->bytesLeft_ -= headerLength+1 ;
}
else
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"header does not fit in buffer");
pstate.step_ = EXTRACT_ERROR;
break;
}
}
}
tupp_descriptor *dataDesc = childOutputTD_;
ex_expr::exp_return_type expStatus = ex_expr::EXPR_OK;
if (myTdb().getChildDataExpr())
{
UInt32 childTuppIndex = myTdb().childDataTuppIndex_;
workAtp_->getTupp(childTuppIndex) = dataDesc;
// Evaluate the child data expression. If diags are generated they
// will be left in the down entry ATP.
expStatus = myTdb().getChildDataExpr()->eval(centry->getAtp(), workAtp_);
workAtp_->getTupp(childTuppIndex).release();
if (expStatus == ex_expr::EXPR_ERROR)
{
updateWorkATPDiagsArea(centry);
pstate.step_ = EXTRACT_ERROR;
break;
}
} // if (myTdb().getChildDataExpr())
///////////////////////
char * targetData = currBuffer_->data_ + currBuffer_->bufSize_ - currBuffer_->bytesLeft_;
if (targetData == NULL)
{
updateWorkATPDiagsArea(__FILE__,__LINE__,"targetData is NULL");
pstate.step_ = EXTRACT_ERROR;
break;
}
NABoolean convError = FALSE;
convertSQRowToString(nullLen, recSepLen, delimLen, dataDesc,
targetData, convError);
///////////////////////////////
pstate.matchCount_++;
if (!convError)
{
if (feStats)
{
feStats->incProcessedRowsCount();
}
pstate.successRowCount_ ++;
}
else
{
if (feStats)
{
feStats->incErrorRowsCount();
}
pstate.errorRowCount_ ++;
}
if (currBuffer_->bytesLeft_ < (Int32) maxExtractRowLength_)
{
pstate.step_ = EXTRACT_DATA_READY_TO_SEND;
}
}
break;
case ex_queue::Q_NO_DATA:
{
pstate.step_ = EXTRACT_DATA_READY_TO_SEND;
endOfData_ = TRUE;
pstate.processingStarted_ = FALSE ; // so that cancel does not
//wait for this Q_NO_DATA
}
break;
case ex_queue::Q_SQLERROR:
{
pstate.step_ = EXTRACT_ERROR;
}
break;
case ex_queue::Q_INVALID:
{
updateWorkATPDiagsArea(__FILE__,__LINE__,
"ExFastExtractTcb::work() Invalid state returned by child");
pstate.step_ = EXTRACT_ERROR;
}
break;
} // switch
qChild_.up->removeHead();
}
break;
case EXTRACT_DATA_READY_TO_SEND:
{
ssize_t bytesToWrite = currBuffer_->bufSize_ - currBuffer_->bytesLeft_;
if (!myTdb().getSkipWritingToFiles())
if (isSequenceFile())
{
sfwRetCode = sequenceFileWriter_->writeBuffer(currBuffer_->data_, bytesToWrite, myTdb().getRecordSeparator());
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else if (myTdb().getBypassLibhdfs())
{
sfwRetCode = sequenceFileWriter_->hdfsWrite(currBuffer_->data_, bytesToWrite);
if (sfwRetCode != SFW_OK)
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else
{
retcode = 0;
retcode = lobInterfaceInsert(bytesToWrite);
if (retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL, (char*)"ExpLOBInterfaceInsert",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
if (feStats)
{
feStats->incReadyToSendBuffersCount();
feStats->incReadyToSendBytes(currBuffer_->bufSize_ - currBuffer_->bytesLeft_);
}
currBuffer_ = NULL;
if (endOfData_)
{
pstate.step_ = EXTRACT_DONE;
}
else
{
pstate.step_ = EXTRACT_RETURN_ROWS_FROM_CHILD;
}
}
break;
case EXTRACT_ERROR:
{
// If there is no room in the parent queue for the reply,
// try again later.
//Later we may split this state into 2 one for cancel and one for query
if (qParent_.up->isFull())
return WORK_OK;
// Cancel the child request - there must be a child request in
// progress to get to the ERROR state.
if (pstate.processingStarted_)
{
qChild_.down->cancelRequestWithParentIndex(qParent_.down->getHeadIndex());
//pstate.processingStarted_ = FALSE;
}
while (pstate.processingStarted_ && pstate.step_ == EXTRACT_ERROR)
{
if (qChild_.up->isEmpty())
return WORK_OK;
ex_queue_entry * childEntry = qChild_.up->getHeadEntry();
ex_queue::up_status childStatus = childEntry->upState.status;
if (childStatus == ex_queue::Q_NO_DATA)
{
pstate.step_ = EXTRACT_DONE;
pstate.processingStarted_ = FALSE;
}
qChild_.up->removeHead();
}
ex_queue_entry *pentry_up = qParent_.up->getTailEntry();
pentry_up->copyAtp(pentry_down);
// Construct and return the error row.
//
if (workAtp_->getDiagsArea())
{
ComDiagsArea *diagsArea = pentry_up->getDiagsArea();
if (diagsArea == NULL)
{
diagsArea = ComDiagsArea::allocate(getGlobals()->getDefaultHeap());
pentry_up->setDiagsArea(diagsArea);
}
pentry_up->getDiagsArea()->mergeAfter(*workAtp_->getDiagsArea());
workAtp_->setDiagsArea(NULL);
}
pentry_up->upState.status = ex_queue::Q_SQLERROR;
pentry_up->upState.parentIndex
= pentry_down->downState.parentIndex;
pentry_up->upState.downIndex = qParent_.down->getHeadIndex();
pentry_up->upState.setMatchNo(pstate.matchCount_);
qParent_.up->insert();
//
errorOccurred_ = TRUE;
pstate.step_ = EXTRACT_DONE;
}
break;
case EXTRACT_DONE:
{
// If there is no room in the parent queue for the reply,
// try again later.
//
if (qParent_.up->isFull())
return WORK_OK;
if (!myTdb().getSkipWritingToFiles())
if (isSequenceFile())
{
sfwRetCode = sequenceFileWriter_->close();
if (!errorOccurred_ && sfwRetCode != SFW_OK )
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
else if (myTdb().getBypassLibhdfs())
{
if (sequenceFileWriter_)
{
sfwRetCode = sequenceFileWriter_->hdfsClose();
if (!errorOccurred_ && sfwRetCode != SFW_OK )
{
createSequenceFileError(sfwRetCode);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
}
else
{
retcode = lobInterfaceClose();
if (! errorOccurred_ && retcode < 0)
{
Lng32 cliError = 0;
Lng32 intParam1 = -retcode;
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(8442), NULL, &intParam1,
&cliError, NULL,
(char*)"ExpLOBinterfaceCloseFile",
getLobErrStr(intParam1));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
}
}
//insertUpQueueEntry will insert Q_NO_DATA into the up queue and
//remove the head of the down queue
insertUpQueueEntry(ex_queue::Q_NO_DATA, NULL, TRUE);
errorOccurred_ = FALSE;
endOfData_ = FALSE;
//we need to set the next state so that the query can get re-executed
//and we start from the beginning again. Not sure if pstate will be
//valid anymore because insertUpQueueEntry() might have cleared it
//already.
pstate.step_ = EXTRACT_NOT_STARTED;
//exit out now and not break.
return WORK_OK;
}
break;
default:
{
ex_assert(FALSE, "Invalid state in ExHdfsFastExtractTcb ");
}
break;
} // switch(pstate.step_)
} // while
return WORK_OK;
#endif
}//ExHdfsFastExtractTcb::work()