// append an ascii-version of FastExtract into cachewa.qryText_
void FastExtract::generateCacheKey(CacheWA &cwa) const
{
RelExpr::generateCacheKeyNode(cwa);
char buf[40];
cwa += " targType_ ";
str_itoa(getTargetType(), buf);
cwa += buf;
cwa += " targName_ ";
cwa += getTargetName();
cwa += " delim_ ";
cwa += getDelimiter();
cwa += " isAppend_ ";
cwa += isAppend() ? "1" : "0";
cwa += " includeHeader_ ";
cwa += includeHeader() ? "1" : "0";
cwa += " cType_ ";
str_itoa(getCompressionType(), buf);
cwa += buf;
cwa += " nullString_ ";
cwa += getNullString();
cwa += " recSep_ ";
cwa += getRecordSeparator();
generateCacheKeyForKids(cwa);
}
Common::SeekableReadStream *ERFFile::decompress(byte *compressedData, uint32 packedSize, uint32 unpackedSize) const {
switch (getCompressionType()) {
case 0:
// No compression
return new Common::MemoryReadStream(compressedData, packedSize, true);
case 1:
// Bioware Zlib
return decompressBiowareZlib(compressedData, packedSize, unpackedSize);
case 2:
case 3:
// Unknown
delete[] compressedData;
throw Common::Exception("Unknown ERF compression %d", getCompressionType());
case 7:
// Headerless Zlib
return decompressHeaderlessZlib(compressedData, packedSize, unpackedSize);
default:
// Invalid
delete[] compressedData;
throw Common::Exception("Invalid ERF compression %d", getCompressionType());
}
}
short
PhysicalFastExtract::codeGen(Generator *generator)
{
short result = 0;
Space *space = generator->getSpace();
CmpContext *cmpContext = generator->currentCmpContext();
const ULng32 downQueueMaxSize = getDefault(GEN_FE_SIZE_DOWN);
const ULng32 upQueueMaxSize = getDefault(GEN_FE_SIZE_UP);
const ULng32 defaultBufferSize = getDefault(GEN_FE_BUFFER_SIZE);
const ULng32 outputBufferSize = defaultBufferSize;
const ULng32 requestBufferSize = defaultBufferSize;
const ULng32 replyBufferSize = defaultBufferSize;
const ULng32 numOutputBuffers = getDefault(GEN_FE_NUM_BUFFERS);
// used in runtime stats
Cardinality estimatedRowCount = (Cardinality)
(getInputCardinality() * getEstRowsUsed()).getValue();
Int32 numChildren = getArity();
ex_cri_desc * givenDesc = generator->getCriDesc(Generator::DOWN);
ComTdb * childTdb = (ComTdb*) new (space) ComTdb();
ExplainTuple *firstExplainTuple = 0;
// Allocate a new map table for this child.
//
MapTable *localMapTable = generator->appendAtEnd();
generator->setCriDesc(givenDesc, Generator::DOWN);
child(0)->codeGen(generator);
childTdb = (ComTdb *)(generator->getGenObj());
firstExplainTuple = generator->getExplainTuple();
ComTdbFastExtract *newTdb = NULL;
char * targetName = NULL;
char * hiveTableName = NULL;
char * delimiter = NULL;
char * header = NULL;
char * nullString = NULL;
char * recordSeparator = NULL;
char * hdfsHostName = NULL;
Int32 hdfsPortNum = getHdfsPort();
char * newDelimiter = (char *)getDelimiter().data();
char specChar = '0';
if (!isHiveInsert() && isSpecialChar(newDelimiter, specChar))
{
newDelimiter = new (cmpContext->statementHeap()) char[2];
newDelimiter[0] = specChar;
newDelimiter[1] = '\0';
}
char * newRecordSep = (char *)getRecordSeparator().data();
specChar = '0';
if (!isHiveInsert() && isSpecialChar(newRecordSep, specChar))
{
newRecordSep = new (cmpContext->statementHeap()) char[2];
newRecordSep[0] = specChar;
newRecordSep[1] = '\0';
}
targetName = AllocStringInSpace(*space, (char *)getTargetName().data());
hdfsHostName = AllocStringInSpace(*space, (char *)getHdfsHostName().data());
hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data());
delimiter = AllocStringInSpace(*space, newDelimiter);
header = AllocStringInSpace(*space, (char *)getHeader().data());
nullString = AllocStringInSpace(*space, (char *)getNullString().data());
recordSeparator = AllocStringInSpace(*space, newRecordSep);
result = ft_codegen(generator,
*this, // RelExpr &relExpr
newTdb, // ComTdbUdr *&newTdb
estimatedRowCount,
targetName,
hdfsHostName,
hdfsPortNum,
hiveTableName,
delimiter,
header,
nullString,
recordSeparator,
downQueueMaxSize,
upQueueMaxSize,
outputBufferSize,
requestBufferSize,
replyBufferSize,
numOutputBuffers,
childTdb,
isSequenceFile());
if (!generator->explainDisabled())
{
generator->setExplainTuple(addExplainInfo(newTdb, firstExplainTuple, 0, generator));
}
if (getTargetType() == FILE)
newTdb->setTargetFile(1);
else if (getTargetType() == SOCKET)
newTdb->setTargetSocket(1);
else
GenAssert(0, "Unexpected Fast Extract target type")
if (isAppend())
newTdb->setIsAppend(1);
if (this->includeHeader())
newTdb->setIncludeHeader(1);
if (isHiveInsert())
{
newTdb->setIsHiveInsert(1);
newTdb->setIncludeHeader(0);
setOverwriteHiveTable( getOverwriteHiveTable());
}
else
{
if (includeHeader())
newTdb->setIncludeHeader(1);
}
if (getCompressionType() != NONE)
{
if (getCompressionType() == LZO)
newTdb->setCompressLZO(1);
else
GenAssert(0, "Unexpected Fast Extract compression type")
}
if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON)
newTdb->setPrintDiags(1);
return result;
}
ExplainTuple *PhysicalFastExtract::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, ComTdb *tdb,
Generator *generator)
{
NAString description = "Target_type: ";
if (getTargetType() == FILE)
{
if (isHiveInsert())
description += "hive table";
else
description += "file";
}
else if (getTargetType() == SOCKET)
description += "socket";
else
description += "none";
if (isHiveInsert())
{
NAString str = getTargetName();
size_t colonIndex = str.index(":", 1,0,NAString::ignoreCase);
while (colonIndex != NA_NPOS)
{
str = str.replace(colonIndex, 1, "_", 1);
colonIndex = str.index(":", 1,0,NAString::ignoreCase);
}
description += " location: ";
description += str;
}
if (isHiveInsert())
{
description += " table_name: ";
description += getHiveTableName();
}
else
{
description += " target_name: ";
description += getTargetName();
}
description += " delimiter: ";
description += getDelimiter();
if (isAppend())
description += " append: yes";
if ( !isHiveInsert() && includeHeader())
{
description += " header: ";
description += getHeader();
}
if (getCompressionType() != NONE)
{
description += " compression_type: ";
if (getCompressionType() == LZO)
description += "LZO";
else
description += "error";
}
description += " null_string: ";
description += getNullString();
description += " record_separator: ";
description += getRecordSeparator();
explainTuple->setDescription(description);
if (isHiveInsert())
explainTuple->setTableName(getHiveTableName());
return explainTuple;
}
