// append an ascii-version of FastExtract into cachewa.qryText_ void FastExtract::generateCacheKey(CacheWA &cwa) const { RelExpr::generateCacheKeyNode(cwa); char buf[40]; cwa += " targType_ "; str_itoa(getTargetType(), buf); cwa += buf; cwa += " targName_ "; cwa += getTargetName(); cwa += " delim_ "; cwa += getDelimiter(); cwa += " isAppend_ "; cwa += isAppend() ? "1" : "0"; cwa += " includeHeader_ "; cwa += includeHeader() ? "1" : "0"; cwa += " cType_ "; str_itoa(getCompressionType(), buf); cwa += buf; cwa += " nullString_ "; cwa += getNullString(); cwa += " recSep_ "; cwa += getRecordSeparator(); generateCacheKeyForKids(cwa); }
Common::SeekableReadStream *ERFFile::decompress(byte *compressedData, uint32 packedSize, uint32 unpackedSize) const { switch (getCompressionType()) { case 0: // No compression return new Common::MemoryReadStream(compressedData, packedSize, true); case 1: // Bioware Zlib return decompressBiowareZlib(compressedData, packedSize, unpackedSize); case 2: case 3: // Unknown delete[] compressedData; throw Common::Exception("Unknown ERF compression %d", getCompressionType()); case 7: // Headerless Zlib return decompressHeaderlessZlib(compressedData, packedSize, unpackedSize); default: // Invalid delete[] compressedData; throw Common::Exception("Invalid ERF compression %d", getCompressionType()); } }
short PhysicalFastExtract::codeGen(Generator *generator) { short result = 0; Space *space = generator->getSpace(); CmpContext *cmpContext = generator->currentCmpContext(); const ULng32 downQueueMaxSize = getDefault(GEN_FE_SIZE_DOWN); const ULng32 upQueueMaxSize = getDefault(GEN_FE_SIZE_UP); const ULng32 defaultBufferSize = getDefault(GEN_FE_BUFFER_SIZE); const ULng32 outputBufferSize = defaultBufferSize; const ULng32 requestBufferSize = defaultBufferSize; const ULng32 replyBufferSize = defaultBufferSize; const ULng32 numOutputBuffers = getDefault(GEN_FE_NUM_BUFFERS); // used in runtime stats Cardinality estimatedRowCount = (Cardinality) (getInputCardinality() * getEstRowsUsed()).getValue(); Int32 numChildren = getArity(); ex_cri_desc * givenDesc = generator->getCriDesc(Generator::DOWN); ComTdb * childTdb = (ComTdb*) new (space) ComTdb(); ExplainTuple *firstExplainTuple = 0; // Allocate a new map table for this child. // MapTable *localMapTable = generator->appendAtEnd(); generator->setCriDesc(givenDesc, Generator::DOWN); child(0)->codeGen(generator); childTdb = (ComTdb *)(generator->getGenObj()); firstExplainTuple = generator->getExplainTuple(); ComTdbFastExtract *newTdb = NULL; char * targetName = NULL; char * hiveTableName = NULL; char * delimiter = NULL; char * header = NULL; char * nullString = NULL; char * recordSeparator = NULL; char * hdfsHostName = NULL; Int32 hdfsPortNum = getHdfsPort(); char * newDelimiter = (char *)getDelimiter().data(); char specChar = '0'; if (!isHiveInsert() && isSpecialChar(newDelimiter, specChar)) { newDelimiter = new (cmpContext->statementHeap()) char[2]; newDelimiter[0] = specChar; newDelimiter[1] = '\0'; } char * newRecordSep = (char *)getRecordSeparator().data(); specChar = '0'; if (!isHiveInsert() && isSpecialChar(newRecordSep, specChar)) { newRecordSep = new (cmpContext->statementHeap()) char[2]; newRecordSep[0] = specChar; newRecordSep[1] = '\0'; } targetName = AllocStringInSpace(*space, (char *)getTargetName().data()); hdfsHostName = AllocStringInSpace(*space, (char *)getHdfsHostName().data()); hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data()); delimiter = AllocStringInSpace(*space, newDelimiter); header = AllocStringInSpace(*space, (char *)getHeader().data()); nullString = AllocStringInSpace(*space, (char *)getNullString().data()); recordSeparator = AllocStringInSpace(*space, newRecordSep); result = ft_codegen(generator, *this, // RelExpr &relExpr newTdb, // ComTdbUdr *&newTdb estimatedRowCount, targetName, hdfsHostName, hdfsPortNum, hiveTableName, delimiter, header, nullString, recordSeparator, downQueueMaxSize, upQueueMaxSize, outputBufferSize, requestBufferSize, replyBufferSize, numOutputBuffers, childTdb, isSequenceFile()); if (!generator->explainDisabled()) { generator->setExplainTuple(addExplainInfo(newTdb, firstExplainTuple, 0, generator)); } if (getTargetType() == FILE) newTdb->setTargetFile(1); else if (getTargetType() == SOCKET) newTdb->setTargetSocket(1); else GenAssert(0, "Unexpected Fast Extract target type") if (isAppend()) newTdb->setIsAppend(1); if (this->includeHeader()) newTdb->setIncludeHeader(1); if (isHiveInsert()) { newTdb->setIsHiveInsert(1); newTdb->setIncludeHeader(0); setOverwriteHiveTable( getOverwriteHiveTable()); } else { if (includeHeader()) newTdb->setIncludeHeader(1); } if (getCompressionType() != NONE) { if (getCompressionType() == LZO) newTdb->setCompressLZO(1); else GenAssert(0, "Unexpected Fast Extract compression type") } if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON) newTdb->setPrintDiags(1); return result; }
ExplainTuple *PhysicalFastExtract::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, ComTdb *tdb, Generator *generator) { NAString description = "Target_type: "; if (getTargetType() == FILE) { if (isHiveInsert()) description += "hive table"; else description += "file"; } else if (getTargetType() == SOCKET) description += "socket"; else description += "none"; if (isHiveInsert()) { NAString str = getTargetName(); size_t colonIndex = str.index(":", 1,0,NAString::ignoreCase); while (colonIndex != NA_NPOS) { str = str.replace(colonIndex, 1, "_", 1); colonIndex = str.index(":", 1,0,NAString::ignoreCase); } description += " location: "; description += str; } if (isHiveInsert()) { description += " table_name: "; description += getHiveTableName(); } else { description += " target_name: "; description += getTargetName(); } description += " delimiter: "; description += getDelimiter(); if (isAppend()) description += " append: yes"; if ( !isHiveInsert() && includeHeader()) { description += " header: "; description += getHeader(); } if (getCompressionType() != NONE) { description += " compression_type: "; if (getCompressionType() == LZO) description += "LZO"; else description += "error"; } description += " null_string: "; description += getNullString(); description += " record_separator: "; description += getRecordSeparator(); explainTuple->setDescription(description); if (isHiveInsert()) explainTuple->setTableName(getHiveTableName()); return explainTuple; }