void addResult(rowcount_t resultCount, MemoryBuffer &resultData, bool complete) { Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock(); Owned<IWUResult> result = updateWorkUnitResult(wu, resultName, resultSeq); if (appendOutput) result->addResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw); else result->setResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw); result->setResultRowCount(resultCount); result->setResultTotalRowCount(resultCount); resultData.clear(); if (complete) result->setResultStatus(ResultStatusCalculated); appendOutput = true; }
void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { rowcount_t slaveProcessed; mb.read(slaveProcessed); recordsProcessed += slaveProcessed; offset_t size, physicalSize; mb.read(size); mb.read(physicalSize); unsigned fileCrc; mb.read(fileCrc); CDateTime modifiedTime(mb); StringBuffer timeStr; modifiedTime.getString(timeStr); IPartDescriptor *partDesc = fileDesc->queryPart(slaveIdx); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); if (fileDesc->isCompressed()) props.setPropInt64("@compressedSize", physicalSize); props.setPropInt("@fileCrc", fileCrc); props.setProp("@modified", timeStr.str()); } }
virtual void done() { Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock(); Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence()); r->setResultStatus(ResultStatusCalculated); r->setResultLogicalName(outputName); r.clear(); wu.clear(); IPropertyTree &props = newIndexDesc->queryProperties(); props.setProp("@kind", "key"); if (0 != (helper->getFlags() & KDPexpires)) setExpiryTime(props, helper->getExpiryDays()); // Fill in some logical file properties here IPropertyTree &originalProps = originalDesc->queryProperties();; if (originalProps.queryProp("ECL")) props.setProp("ECL", originalProps.queryProp("ECL")); MemoryBuffer rLMB; if (originalProps.getPropBin("_record_layout", rLMB)) props.setPropBin("_record_layout", rLMB.length(), rLMB.toByteArray()); props.setPropInt("@formatCrc", originalProps.getPropInt("@formatCrc")); if (originalProps.getPropBool("@local")) props.setPropBool("@local", true); container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters); queryThorFileManager().publish(container.queryJob(), outputName, *newIndexDesc); CMasterActivity::done(); }
extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml) { MemoryBuffer data; if (!getResourceFromFile(filename, data, type, id)) return false; return decompressResource(data.length(), data.toByteArray(), xml); }
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result) { bool hasVersion = len && (*(const byte *)data == 0x80); MemoryBuffer src; src.setBuffer(len, const_cast<void *>(data), false); byte version = 1; if (hasVersion) { src.skip(1); src.read(version); } MemoryBuffer tgt; switch (version) { case 1: decompressToBuffer(tgt, src); break; default: throwUnexpected(); } tgt.append((char)0); unsigned expandedLen = tgt.length(); result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1); return true; }
void notify(MemoryBuffer &returndata) // if returns false should unsubscribe { if (hasaborted) { throw MakeStringException(-1,"Subscription notification aborted"); return; } size32_t dlen = returndata.length(); CMessageBuffer mb; mb.append(tag).append(sid).append(dlen).append(returndata); try { if (!queryWorldCommunicator().send(mb,dst,MPTAG_DALI_SUBSCRIPTION_FULFILL,1000*60*3)) { // Must reply in 3 Minutes // Kludge to avoid locking SDS on blocked client hasaborted = true; StringBuffer tmp; throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getUrlStr(tmp).str()); return; } } catch (IMP_Exception *e) { PrintExceptionLog(e,"Dali CSubscriptionStub"); hasaborted = true; throw; } }
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id) { #ifdef _WIN32 HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE); if (dllHandle == NULL) dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows if (dllHandle == NULL) { DBGLOG("Failed to load library %s: %d", filename, GetLastError()); return false; } HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type); if (!hrsrc) return false; size32_t len = SizeofResource(dllHandle, hrsrc); const void *rdata = (const void *) LoadResource(dllHandle, hrsrc); data.append(len, rdata); FreeLibrary(dllHandle); return true; #else bfd_init (); bfd *file = bfd_openr(filename, NULL); if (file) { StringBuffer sectionName; sectionName.append(type).append("_").append(id).append(".data"); SecScanParam param(data, sectionName.str()); if (bfd_check_format (file, bfd_object)) bfd_map_over_sections (file, secscan, ¶m); bfd_close (file); } return data.length() != 0; #endif }
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { offset_t size; mb.read(size); CDateTime modifiedTime(mb); IPartDescriptor *partDesc = newIndexDesc->queryPart(slaveIdx); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); if (!local && 0 == slaveIdx) { mb.read(size); CDateTime modifiedTime(mb); IPartDescriptor *partDesc = newIndexDesc->queryPart(newIndexDesc->numParts()-1); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); } } }
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id) { #ifdef _WIN32 HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE); if (dllHandle == NULL) dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows if (dllHandle == NULL) { DBGLOG("Failed to load library %s: %d", filename, GetLastError()); return false; } HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type); if (!hrsrc) return false; size32_t len = SizeofResource(dllHandle, hrsrc); const void *rdata = (const void *) LoadResource(dllHandle, hrsrc); data.append(len, rdata); FreeLibrary(dllHandle); return true; #elif defined (_USE_BINUTILS) CriticalBlock block(bfdCs); bfd_init (); bfd *file = bfd_openr(filename, NULL); if (file) { StringBuffer sectionName; sectionName.append(type).append("_").append(id).append(".data"); SecScanParam param(data, sectionName.str()); if (bfd_check_format (file, bfd_object)) bfd_map_over_sections (file, secscan, ¶m); bfd_close (file); } return data.length() != 0; #else struct stat stat_buf; VStringBuffer sectname("%s_%u", type, id); int fd = open(filename, O_RDONLY); if (fd == -1 || fstat(fd, &stat_buf) == -1) { DBGLOG("Failed to load library %s: %d", filename, errno); return false; } bool ok = false; __uint64 size = stat_buf.st_size; const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0); if (start_addr == MAP_FAILED) { DBGLOG("Failed to load library %s: %d", filename, errno); } else { ok = getResourceFromMappedFile(filename, start_addr, data, type, id); munmap((void *)start_addr, size); } close(fd); return ok; #endif }
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result) { MemoryBuffer tgt; decompressResource(len, data, tgt); tgt.append((char)0); unsigned expandedLen = tgt.length(); result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1); return true; }
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { rowcount_t rc; mb.read(rc); recordsProcessed += rc; } }
bool PagedWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset) { MemoryBuffer temp; MemoryBuffer2IDataVal wrapper(out); wuResult->getResultRaw(wrapper, offset, returnedMeta->getMaxRecordSize(), NULL, NULL); if (temp.length() == 0) return false; return true; }
size32_t ThorCompress(const void * src, size32_t srcSz, MemoryBuffer & dest, size32_t threshold) { size32_t prev = dest.length(); size32_t dSz = srcSz + sizeof(size32_t); void * d = dest.reserve(dSz); size32_t ret = ThorCompress(src, srcSz, d, dSz, threshold); dest.setLength(prev+ret); return ret; }
void setRtlFormat(IPropertyTree & properties, IOutputMetaData * meta) { if (meta && meta->queryTypeInfo()) { MemoryBuffer out; if (dumpTypeInfo(out, meta->querySerializedDiskMeta()->queryTypeInfo())) properties.setPropBin("_rtlType", out.length(), out.toByteArray()); } }
size32_t aesEncryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &publicKey) { // create random AES key and IV char randomAesKey[aesMaxKeySize]; char randomIV[aesBlockSize]; fillRandomData(aesMaxKeySize, randomAesKey); fillRandomData(aesBlockSize, randomIV); size32_t startSz = out.length(); DelayedSizeMarker mark(out); publicKeyEncrypt(out, aesMaxKeySize, randomAesKey, publicKey); mark.write(); out.append(aesBlockSize, randomIV); DelayedSizeMarker aesSz(out); aesEncrypt(out, inSz, inBytes, aesMaxKeySize, randomAesKey, randomIV); aesSz.write(); return out.length()-startSz; }
void CSendLogSerializer::LoadMap(MemoryBuffer& rawdata,StringBuffer& GUID, StringBuffer& line) { line.append(rawdata.length() -1, rawdata.toByteArray()); const char* strLine = line.str(); while(*strLine && *strLine != '\t' && *strLine != '\0') { GUID.append(*strLine); strLine++; } }
void CThorTransferGroup::send(SocketEndpoint &ep, CThorRowArray & group) { ISocket * sendSkt = ISocket::connect_wait(ep, 360*120*1000); // give it plenty of time, sequential in nature *could* be delayed in other side listening sendSkt->set_block_mode(BF_SYNC_TRANSFER_PULL,0,TRANSFER_TIMEOUT); MemoryBuffer mb; group.serialize(serializer,mb,false); sendSkt->send_block(mb.toByteArray(),mb.length()); sendSkt->close(); sendSkt->Release(); }
byte getWUSresult(MemoryBuffer &mb) { byte ret = 0; wusbuf.swapWith(mb); if (mb.length()==1) { mb.read(ret); mb.clear(); } return ret; }
extern DEFTYPE_API void serializeRecordMeta(MemoryBuffer & target, IDefRecordMeta * meta, bool compress) { if (compress) { MemoryBuffer temp; doSerializeRecordMeta(temp, meta); compressToBuffer(target, temp.length(), temp.toByteArray()); } else doSerializeRecordMeta(target, meta); }
void LdapUtils::bin2str(MemoryBuffer& from, StringBuffer& to) { const char* frombuf = from.toByteArray(); char tmp[3]; for(unsigned i = 0; i < from.length(); i++) { unsigned char c = frombuf[i]; sprintf(tmp, "%02X", c); tmp[2] = 0; to.append("\\").append(tmp); } }
CXRefNode::CXRefNode(IPropertyTree* pTreeRoot) { m_bChanged = false; try { m_XRefTree.set(pTreeRoot); pTreeRoot->getProp("@name",m_origName); //load up our tree with the data.....if there is data MemoryBuffer buff; pTreeRoot->getPropBin("data",buff); if (buff.length()) { m_dataStr.append(buff.length(),buff.toByteArray()); } //lets check to ensure we have the correct children inplace(Orphan,lost,found) } catch(...) { ERRLOG("Error in creation of XRefNode..."); } }
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out) { Owned<IFvDataSource> ds = readDataSource(in); if (!ds) { out.append(false); return; } __int64 requestedRow; in.read(requestedRow); unsigned startPos = out.length(); unsigned numRows = 0; out.append(true); // ok out.append(requestedRow); // start unsigned numRowsPos = out.length(); out.append(numRows); // total number of rows; loop { unsigned lengthPos = out.length(); out.append((unsigned)0); // size of this row. unsigned startRow = out.length(); if (raw) { if (!ds->getRawRow(out, requestedRow+numRows)) break; } else { if (!ds->getRow(out, requestedRow+numRows)) break; } if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE)) break; unsigned endRow = out.length(); out.rewrite(lengthPos); out.append(endRow-startRow); out.rewrite(endRow); numRows++; } if (numRows == 0) { out.rewrite(startPos); out.append(false); return; } unsigned totalLength = out.length(); out.rewrite(numRowsPos); out.append(numRows); out.rewrite(totalLength); }
StringBuffer &CXRefNode::serializeDirectories(StringBuffer &buf) { if(!m_directories.get()) { IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories"); if(directoriesBranch == 0) { directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree()); commit(); } StringBuffer tmpbuf; m_directories.set(directoriesBranch); } buf.clear(); MemoryBuffer data; m_directories->getPropBin("data",data); if (data.length()) { buf.append(data.length(),data.toByteArray()); } return buf; }
void ResourceManager::addManifestFromArchive(IPropertyTree *archive) { if (!archive) return; if (finalized) throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST"); ensureManifestInfo(); Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest"); ForEach(*manifests) { const char *xml = manifests->query().queryProp(NULL); Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml); Owned<IAttributeIterator> aiter = manifestSrc->getAttributes(); ForEach (*aiter) manifest->setProp(aiter->queryName(), aiter->queryValue()); StringBuffer manifestDir; if (manifestSrc->hasProp("@originalFilename")) splitDirTail(manifestSrc->queryProp("@originalFilename"), manifestDir); Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*"); ForEach(*iter) { IPropertyTree &item = iter->query(); if (streq(item.queryName(), "Resource") && item.hasProp("@filename")) { if (!item.hasProp("@type")) item.setProp("@type", "UNKNOWN"); const char *filename; if (item.hasProp("@originalFilename")) filename = item.queryProp("@originalFilename"); else filename = item.queryProp("@filename"); int id; if (getDuplicateResourceId(item.queryProp("@type"), NULL, filename, id)) { item.setPropInt("@id", (int)id); manifest->addPropTree("Resource", LINK(&item)); } else { VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename); MemoryBuffer content; archive->getPropBin(xpath.str(), content); addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item); } } else manifest->addPropTree(item.queryName(), LINK(&item)); } } }
bool receive(MemoryBuffer &mb) { #ifdef _TRACEBROADCAST ActPrintLog(activity, "Broadcast node %d Receiving on tag %d",nodeindex,(int)mpTag); #endif CMessageBuffer msg; rank_t sender; BooleanOnOff onOff(receiving); if (comm->recv(msg, RANK_ALL, mpTag, &sender)) { #ifdef _TRACEBROADCAST ActPrintLog(activity, "Broadcast node %d Received %d from %d",nodeindex, msg.length(), sender); #endif try { mb.swapWith(msg); msg.clear(); // send empty reply #ifdef _TRACEBROADCAST ActPrintLog(activity, "Broadcast node %d reply to %d",nodeindex, sender); #endif comm->reply(msg); if (aborted) return false; #ifdef _TRACEBROADCAST ActPrintLog(activity, "Broadcast node %d Received %d",nodeindex, mb.length()); #endif } catch (IException *e) { ActPrintLog(activity, e, "CBroadcaster::recv(2): exception"); throw; } } #ifdef _TRACEBROADCAST ActPrintLog(activity, "receive done"); #endif return (0 != mb.length()); }
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra) { if (local) i = 0; if (i >= maps.ordinality()) { mb.append((unsigned)0); return; } CSlaveMap &map = maps.item(i); unsigned nPos = mb.length(); unsigned n=0; mb.append(n); UnsignedArray parts; ForEachItemIn(m, map) parts.append(map.item(m).queryPartIndex()); MemoryBuffer extraMb; if (extra) { ForEachItemIn(m2, map) { unsigned xtraLen = 0; unsigned xtraPos = extraMb.length(); extraMb.append(xtraLen); IPartDescriptor &partDesc = map.item(m2); if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb)) { parts.zap(partDesc.queryPartIndex()); extraMb.rewrite(xtraPos); } else { xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen); extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen); } } }
bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id, bool addToManifest) { bool isCompressed=false; if (len>=32) //lzw assert if too small { isCompressed = true; MemoryBuffer compressed; compressResource(compressed, len, data); addNamed(type, compressed.length(), compressed.toByteArray(), manifestEntry, id, addToManifest, isCompressed); } else addNamed(type, len, data, manifestEntry, id, addToManifest, isCompressed); return isCompressed; }
void LZMACompressToBuffer(MemoryBuffer & out, size32_t len, const void * src) { CLZMA lzma; size32_t outbase = out.length(); size32_t *sz = (size32_t *)out.reserve(len+sizeof(size32_t)*2); *sz = len; sz++; *sz = lzma.compress(src,len,sz+1); if (*sz>len) { *sz = len; memcpy(sz+1,src,len); } else out.setLength(outbase+sizeof(size32_t)*2+*sz); }
bool PagedWorkUnitDataSource::loadBlock(__int64 startRow, offset_t startOffset) { MemoryBuffer temp; MemoryBuffer2IDataVal xxx(temp); RowBlock * rows; if (returnedMeta->isFixedSize()) { unsigned fixedSize = returnedMeta->fixedSize(); unsigned readSize = (WU_BLOCK_SIZE / fixedSize) * fixedSize; wuResult->getResultRaw(xxx, startOffset, readSize, NULL, NULL); if (temp.length() == 0) return false; rows = new FixedRowBlock(temp, startRow, startOffset, fixedSize); } else { wuResult->getResultRaw(xxx, startOffset, WU_BLOCK_SIZE, NULL, NULL); if (temp.length() == 0) return false; rows = new VariableRowBlock(temp, startRow, startOffset, returnedRecordSize, startOffset + temp.length() == totalSize); } cache.addRowsOwn(rows); return true; }
void sendFileChunk(const char * filename, offset_t offset, ISocket * socket) { FILE *in = fopen(filename, "rb"); unsigned size = 0; void * buff = NULL; if (in) { fseek(in, 0, SEEK_END); offset_t endOffset = ftell(in); fseek(in, offset, SEEK_SET); if (endOffset < offset) size = 0; else size = (unsigned)(endOffset - offset); if (size > CHUNK_SIZE) size = CHUNK_SIZE; buff = malloc(size); size_t numRead = fread(buff, 1, size, in); fclose(in); if (numRead != size) { printf("read from file %s failed (%u/%u)\n", filename, (unsigned)numRead, size); size = 0; } } else printf("read from file %s failed\n", filename); if (size > 0) { MemoryBuffer sendBuffer; unsigned rev = size + strlen(filename) + 10; rev |= 0x80000000; _WINREV(rev); sendBuffer.append(rev); sendBuffer.append('R'); rev = 0; // should put the sequence number here _WINREV(rev); sendBuffer.append(rev); rev = 0; // should put the # of recs in msg here _WINREV(rev); sendBuffer.append(rev); sendBuffer.append(strlen(filename)+1, filename); sendBuffer.append(size, buff); socket->write(sendBuffer.toByteArray(), sendBuffer.length()); } else { unsigned zeroLen = 0; socket->write(&zeroLen, sizeof(zeroLen)); } free(buff); }