void addResult(rowcount_t resultCount, MemoryBuffer &resultData, bool complete)
{
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> result = updateWorkUnitResult(wu, resultName, resultSeq);
if (appendOutput)
result->addResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw);
else
result->setResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw);
result->setResultRowCount(resultCount);
result->setResultTotalRowCount(resultCount);
resultData.clear();
if (complete)
result->setResultStatus(ResultStatusCalculated);
appendOutput = true;
}
void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
rowcount_t slaveProcessed;
mb.read(slaveProcessed);
recordsProcessed += slaveProcessed;
offset_t size, physicalSize;
mb.read(size);
mb.read(physicalSize);
unsigned fileCrc;
mb.read(fileCrc);
CDateTime modifiedTime(mb);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
IPartDescriptor *partDesc = fileDesc->queryPart(slaveIdx);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
if (fileDesc->isCompressed())
props.setPropInt64("@compressedSize", physicalSize);
props.setPropInt("@fileCrc", fileCrc);
props.setProp("@modified", timeStr.str());
}
}
virtual void done()
{
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(outputName);
r.clear();
wu.clear();
IPropertyTree &props = newIndexDesc->queryProperties();
props.setProp("@kind", "key");
if (0 != (helper->getFlags() & KDPexpires))
setExpiryTime(props, helper->getExpiryDays());
// Fill in some logical file properties here
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
props.setProp("ECL", originalProps.queryProp("ECL"));
MemoryBuffer rLMB;
if (originalProps.getPropBin("_record_layout", rLMB))
props.setPropBin("_record_layout", rLMB.length(), rLMB.toByteArray());
props.setPropInt("@formatCrc", originalProps.getPropInt("@formatCrc"));
if (originalProps.getPropBool("@local"))
props.setPropBool("@local", true);
container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
queryThorFileManager().publish(container.queryJob(), outputName, *newIndexDesc);
CMasterActivity::done();
}
extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml)
{
MemoryBuffer data;
if (!getResourceFromFile(filename, data, type, id))
return false;
return decompressResource(data.length(), data.toByteArray(), xml);
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
bool hasVersion = len && (*(const byte *)data == 0x80);
MemoryBuffer src;
src.setBuffer(len, const_cast<void *>(data), false);
byte version = 1;
if (hasVersion)
{
src.skip(1);
src.read(version);
}
MemoryBuffer tgt;
switch (version)
{
case 1:
decompressToBuffer(tgt, src);
break;
default:
throwUnexpected();
}
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
void notify(MemoryBuffer &returndata) // if returns false should unsubscribe
{
if (hasaborted) {
throw MakeStringException(-1,"Subscription notification aborted");
return;
}
size32_t dlen = returndata.length();
CMessageBuffer mb;
mb.append(tag).append(sid).append(dlen).append(returndata);
try {
if (!queryWorldCommunicator().send(mb,dst,MPTAG_DALI_SUBSCRIPTION_FULFILL,1000*60*3)) {
// Must reply in 3 Minutes
// Kludge to avoid locking SDS on blocked client
hasaborted = true;
StringBuffer tmp;
throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getUrlStr(tmp).str());
return;
}
}
catch (IMP_Exception *e) {
PrintExceptionLog(e,"Dali CSubscriptionStub");
hasaborted = true;
throw;
}
}
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
if (dllHandle == NULL)
dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
if (dllHandle == NULL)
{
DBGLOG("Failed to load library %s: %d", filename, GetLastError());
return false;
}
HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
if (!hrsrc)
return false;
size32_t len = SizeofResource(dllHandle, hrsrc);
const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
data.append(len, rdata);
FreeLibrary(dllHandle);
return true;
#else
bfd_init ();
bfd *file = bfd_openr(filename, NULL);
if (file)
{
StringBuffer sectionName;
sectionName.append(type).append("_").append(id).append(".data");
SecScanParam param(data, sectionName.str());
if (bfd_check_format (file, bfd_object))
bfd_map_over_sections (file, secscan, ¶m);
bfd_close (file);
}
return data.length() != 0;
#endif
}
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
offset_t size;
mb.read(size);
CDateTime modifiedTime(mb);
IPartDescriptor *partDesc = newIndexDesc->queryPart(slaveIdx);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
if (!local && 0 == slaveIdx)
{
mb.read(size);
CDateTime modifiedTime(mb);
IPartDescriptor *partDesc = newIndexDesc->queryPart(newIndexDesc->numParts()-1);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
}
}
}
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
if (dllHandle == NULL)
dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
if (dllHandle == NULL)
{
DBGLOG("Failed to load library %s: %d", filename, GetLastError());
return false;
}
HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
if (!hrsrc)
return false;
size32_t len = SizeofResource(dllHandle, hrsrc);
const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
data.append(len, rdata);
FreeLibrary(dllHandle);
return true;
#elif defined (_USE_BINUTILS)
CriticalBlock block(bfdCs);
bfd_init ();
bfd *file = bfd_openr(filename, NULL);
if (file)
{
StringBuffer sectionName;
sectionName.append(type).append("_").append(id).append(".data");
SecScanParam param(data, sectionName.str());
if (bfd_check_format (file, bfd_object))
bfd_map_over_sections (file, secscan, ¶m);
bfd_close (file);
}
return data.length() != 0;
#else
struct stat stat_buf;
VStringBuffer sectname("%s_%u", type, id);
int fd = open(filename, O_RDONLY);
if (fd == -1 || fstat(fd, &stat_buf) == -1)
{
DBGLOG("Failed to load library %s: %d", filename, errno);
return false;
}
bool ok = false;
__uint64 size = stat_buf.st_size;
const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
if (start_addr == MAP_FAILED)
{
DBGLOG("Failed to load library %s: %d", filename, errno);
}
else
{
ok = getResourceFromMappedFile(filename, start_addr, data, type, id);
munmap((void *)start_addr, size);
}
close(fd);
return ok;
#endif
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
MemoryBuffer tgt;
decompressResource(len, data, tgt);
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
rowcount_t rc;
mb.read(rc);
recordsProcessed += rc;
}
}
bool PagedWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
{
MemoryBuffer temp;
MemoryBuffer2IDataVal wrapper(out);
wuResult->getResultRaw(wrapper, offset, returnedMeta->getMaxRecordSize(), NULL, NULL);
if (temp.length() == 0)
return false;
return true;
}
size32_t ThorCompress(const void * src, size32_t srcSz, MemoryBuffer & dest, size32_t threshold)
{
size32_t prev = dest.length();
size32_t dSz = srcSz + sizeof(size32_t);
void * d = dest.reserve(dSz);
size32_t ret = ThorCompress(src, srcSz, d, dSz, threshold);
dest.setLength(prev+ret);
return ret;
}
void setRtlFormat(IPropertyTree & properties, IOutputMetaData * meta)
{
if (meta && meta->queryTypeInfo())
{
MemoryBuffer out;
if (dumpTypeInfo(out, meta->querySerializedDiskMeta()->queryTypeInfo()))
properties.setPropBin("_rtlType", out.length(), out.toByteArray());
}
}
size32_t aesEncryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &publicKey)
{
// create random AES key and IV
char randomAesKey[aesMaxKeySize];
char randomIV[aesBlockSize];
fillRandomData(aesMaxKeySize, randomAesKey);
fillRandomData(aesBlockSize, randomIV);
size32_t startSz = out.length();
DelayedSizeMarker mark(out);
publicKeyEncrypt(out, aesMaxKeySize, randomAesKey, publicKey);
mark.write();
out.append(aesBlockSize, randomIV);
DelayedSizeMarker aesSz(out);
aesEncrypt(out, inSz, inBytes, aesMaxKeySize, randomAesKey, randomIV);
aesSz.write();
return out.length()-startSz;
}
void CSendLogSerializer::LoadMap(MemoryBuffer& rawdata,StringBuffer& GUID, StringBuffer& line)
{
line.append(rawdata.length() -1, rawdata.toByteArray());
const char* strLine = line.str();
while(*strLine && *strLine != '\t' && *strLine != '\0')
{
GUID.append(*strLine);
strLine++;
}
}
void CThorTransferGroup::send(SocketEndpoint &ep, CThorRowArray & group)
{
ISocket * sendSkt = ISocket::connect_wait(ep, 360*120*1000); // give it plenty of time, sequential in nature *could* be delayed in other side listening
sendSkt->set_block_mode(BF_SYNC_TRANSFER_PULL,0,TRANSFER_TIMEOUT);
MemoryBuffer mb;
group.serialize(serializer,mb,false);
sendSkt->send_block(mb.toByteArray(),mb.length());
sendSkt->close();
sendSkt->Release();
}
byte getWUSresult(MemoryBuffer &mb)
{
byte ret = 0;
wusbuf.swapWith(mb);
if (mb.length()==1) {
mb.read(ret);
mb.clear();
}
return ret;
}
extern DEFTYPE_API void serializeRecordMeta(MemoryBuffer & target, IDefRecordMeta * meta, bool compress)
{
if (compress)
{
MemoryBuffer temp;
doSerializeRecordMeta(temp, meta);
compressToBuffer(target, temp.length(), temp.toByteArray());
}
else
doSerializeRecordMeta(target, meta);
}
void LdapUtils::bin2str(MemoryBuffer& from, StringBuffer& to)
{
const char* frombuf = from.toByteArray();
char tmp[3];
for(unsigned i = 0; i < from.length(); i++)
{
unsigned char c = frombuf[i];
sprintf(tmp, "%02X", c);
tmp[2] = 0;
to.append("\\").append(tmp);
}
}
CXRefNode::CXRefNode(IPropertyTree* pTreeRoot)
{
m_bChanged = false;
try
{
m_XRefTree.set(pTreeRoot);
pTreeRoot->getProp("@name",m_origName);
//load up our tree with the data.....if there is data
MemoryBuffer buff;
pTreeRoot->getPropBin("data",buff);
if (buff.length())
{
m_dataStr.append(buff.length(),buff.toByteArray());
}
//lets check to ensure we have the correct children inplace(Orphan,lost,found)
}
catch(...)
{
ERRLOG("Error in creation of XRefNode...");
}
}
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
Owned<IFvDataSource> ds = readDataSource(in);
if (!ds)
{
out.append(false);
return;
}
__int64 requestedRow;
in.read(requestedRow);
unsigned startPos = out.length();
unsigned numRows = 0;
out.append(true); // ok
out.append(requestedRow); // start
unsigned numRowsPos = out.length();
out.append(numRows); // total number of rows;
loop
{
unsigned lengthPos = out.length();
out.append((unsigned)0); // size of this row.
unsigned startRow = out.length();
if (raw)
{
if (!ds->getRawRow(out, requestedRow+numRows))
break;
}
else
{
if (!ds->getRow(out, requestedRow+numRows))
break;
}
if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
break;
unsigned endRow = out.length();
out.rewrite(lengthPos);
out.append(endRow-startRow);
out.rewrite(endRow);
numRows++;
}
if (numRows == 0)
{
out.rewrite(startPos);
out.append(false);
return;
}
unsigned totalLength = out.length();
out.rewrite(numRowsPos);
out.append(numRows);
out.rewrite(totalLength);
}
StringBuffer &CXRefNode::serializeDirectories(StringBuffer &buf)
{
if(!m_directories.get())
{
IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories");
if(directoriesBranch == 0)
{
directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree());
commit();
}
StringBuffer tmpbuf;
m_directories.set(directoriesBranch);
}
buf.clear();
MemoryBuffer data;
m_directories->getPropBin("data",data);
if (data.length())
{
buf.append(data.length(),data.toByteArray());
}
return buf;
}
void ResourceManager::addManifestFromArchive(IPropertyTree *archive)
{
if (!archive)
return;
if (finalized)
throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST");
ensureManifestInfo();
Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
ForEach(*manifests)
{
const char *xml = manifests->query().queryProp(NULL);
Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
ForEach (*aiter)
manifest->setProp(aiter->queryName(), aiter->queryValue());
StringBuffer manifestDir;
if (manifestSrc->hasProp("@originalFilename"))
splitDirTail(manifestSrc->queryProp("@originalFilename"), manifestDir);
Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
ForEach(*iter)
{
IPropertyTree &item = iter->query();
if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
{
if (!item.hasProp("@type"))
item.setProp("@type", "UNKNOWN");
const char *filename;
if (item.hasProp("@originalFilename"))
filename = item.queryProp("@originalFilename");
else
filename = item.queryProp("@filename");
int id;
if (getDuplicateResourceId(item.queryProp("@type"), NULL, filename, id))
{
item.setPropInt("@id", (int)id);
manifest->addPropTree("Resource", LINK(&item));
}
else
{
VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
MemoryBuffer content;
archive->getPropBin(xpath.str(), content);
addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
}
}
else
manifest->addPropTree(item.queryName(), LINK(&item));
}
}
}
bool receive(MemoryBuffer &mb)
{
#ifdef _TRACEBROADCAST
ActPrintLog(activity, "Broadcast node %d Receiving on tag %d",nodeindex,(int)mpTag);
#endif
CMessageBuffer msg;
rank_t sender;
BooleanOnOff onOff(receiving);
if (comm->recv(msg, RANK_ALL, mpTag, &sender))
{
#ifdef _TRACEBROADCAST
ActPrintLog(activity, "Broadcast node %d Received %d from %d",nodeindex, msg.length(), sender);
#endif
try
{
mb.swapWith(msg);
msg.clear(); // send empty reply
#ifdef _TRACEBROADCAST
ActPrintLog(activity, "Broadcast node %d reply to %d",nodeindex, sender);
#endif
comm->reply(msg);
if (aborted)
return false;
#ifdef _TRACEBROADCAST
ActPrintLog(activity, "Broadcast node %d Received %d",nodeindex, mb.length());
#endif
}
catch (IException *e)
{
ActPrintLog(activity, e, "CBroadcaster::recv(2): exception");
throw;
}
}
#ifdef _TRACEBROADCAST
ActPrintLog(activity, "receive done");
#endif
return (0 != mb.length());
}
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra)
{
if (local)
i = 0;
if (i >= maps.ordinality())
{
mb.append((unsigned)0);
return;
}
CSlaveMap &map = maps.item(i);
unsigned nPos = mb.length();
unsigned n=0;
mb.append(n);
UnsignedArray parts;
ForEachItemIn(m, map)
parts.append(map.item(m).queryPartIndex());
MemoryBuffer extraMb;
if (extra)
{
ForEachItemIn(m2, map)
{
unsigned xtraLen = 0;
unsigned xtraPos = extraMb.length();
extraMb.append(xtraLen);
IPartDescriptor &partDesc = map.item(m2);
if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb))
{
parts.zap(partDesc.queryPartIndex());
extraMb.rewrite(xtraPos);
}
else
{
xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen);
extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen);
}
}
}
bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id, bool addToManifest)
{
bool isCompressed=false;
if (len>=32) //lzw assert if too small
{
isCompressed = true;
MemoryBuffer compressed;
compressResource(compressed, len, data);
addNamed(type, compressed.length(), compressed.toByteArray(), manifestEntry, id, addToManifest, isCompressed);
}
else
addNamed(type, len, data, manifestEntry, id, addToManifest, isCompressed);
return isCompressed;
}
void LZMACompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
CLZMA lzma;
size32_t outbase = out.length();
size32_t *sz = (size32_t *)out.reserve(len+sizeof(size32_t)*2);
*sz = len;
sz++;
*sz = lzma.compress(src,len,sz+1);
if (*sz>len) {
*sz = len;
memcpy(sz+1,src,len);
}
else
out.setLength(outbase+sizeof(size32_t)*2+*sz);
}
bool PagedWorkUnitDataSource::loadBlock(__int64 startRow, offset_t startOffset)
{
MemoryBuffer temp;
MemoryBuffer2IDataVal xxx(temp);
RowBlock * rows;
if (returnedMeta->isFixedSize())
{
unsigned fixedSize = returnedMeta->fixedSize();
unsigned readSize = (WU_BLOCK_SIZE / fixedSize) * fixedSize;
wuResult->getResultRaw(xxx, startOffset, readSize, NULL, NULL);
if (temp.length() == 0)
return false;
rows = new FixedRowBlock(temp, startRow, startOffset, fixedSize);
}
else
{
wuResult->getResultRaw(xxx, startOffset, WU_BLOCK_SIZE, NULL, NULL);
if (temp.length() == 0)
return false;
rows = new VariableRowBlock(temp, startRow, startOffset, returnedRecordSize, startOffset + temp.length() == totalSize);
}
cache.addRowsOwn(rows);
return true;
}
void sendFileChunk(const char * filename, offset_t offset, ISocket * socket)
{
FILE *in = fopen(filename, "rb");
unsigned size = 0;
void * buff = NULL;
if (in)
{
fseek(in, 0, SEEK_END);
offset_t endOffset = ftell(in);
fseek(in, offset, SEEK_SET);
if (endOffset < offset)
size = 0;
else
size = (unsigned)(endOffset - offset);
if (size > CHUNK_SIZE)
size = CHUNK_SIZE;
buff = malloc(size);
size_t numRead = fread(buff, 1, size, in);
fclose(in);
if (numRead != size)
{
printf("read from file %s failed (%u/%u)\n", filename, (unsigned)numRead, size);
size = 0;
}
}
else
printf("read from file %s failed\n", filename);
if (size > 0)
{
MemoryBuffer sendBuffer;
unsigned rev = size + strlen(filename) + 10;
rev |= 0x80000000;
_WINREV(rev);
sendBuffer.append(rev);
sendBuffer.append('R');
rev = 0; // should put the sequence number here
_WINREV(rev);
sendBuffer.append(rev);
rev = 0; // should put the # of recs in msg here
_WINREV(rev);
sendBuffer.append(rev);
sendBuffer.append(strlen(filename)+1, filename);
sendBuffer.append(size, buff);
socket->write(sendBuffer.toByteArray(), sendBuffer.length());
}
else
{
unsigned zeroLen = 0;
socket->write(&zeroLen, sizeof(zeroLen));
}
free(buff);
}