void CachedPasswordProvider::serialize(MemoryBuffer & out)
{
unsigned num = passwords.ordinality();
out.append(num);
ForEachItemIn(idx, passwords)
passwords.item(idx).serialize(out);
}
virtual void init()
{
HashDistributeMasterBase::init();
// JCSMORE should common up some with indexread
IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();
StringBuffer scoped;
queryThorFileManager().addScope(container.queryJob(), helper->getIndexFileName(), scoped);
Owned<IDistributedFile> f = queryThorFileManager().lookup(container.queryJob(), helper->getIndexFileName());
if (!f)
throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
if (0 == f->numParts())
throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());
checkFormatCrc(this, f, helper->getFormatCrc(), true);
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
tlkMb.append(iFileIO->size());
::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);
queryThorFileManager().noteFileRead(container.queryJob(), f);
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
bool hasVersion = len && (*(const byte *)data == 0x80);
MemoryBuffer src;
src.setBuffer(len, const_cast<void *>(data), false);
byte version = 1;
if (hasVersion)
{
src.skip(1);
src.read(version);
}
MemoryBuffer tgt;
switch (version)
{
case 1:
decompressToBuffer(tgt, src);
break;
default:
throwUnexpected();
}
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
virtual void serializeStats(MemoryBuffer &mb)
{
CSlaveActivity::serializeStats(mb);
CriticalBlock b(joinHelperCrit);
rowcount_t p = joinhelper?joinhelper->getLhsProgress():0;
mb.append(p);
}
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
if (dllHandle == NULL)
dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
if (dllHandle == NULL)
{
DBGLOG("Failed to load library %s: %d", filename, GetLastError());
return false;
}
HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
if (!hrsrc)
return false;
size32_t len = SizeofResource(dllHandle, hrsrc);
const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
data.append(len, rdata);
FreeLibrary(dllHandle);
return true;
#else
bfd_init ();
bfd *file = bfd_openr(filename, NULL);
if (file)
{
StringBuffer sectionName;
sectionName.append(type).append("_").append(id).append(".data");
SecScanParam param(data, sectionName.str());
if (bfd_check_format (file, bfd_object))
bfd_map_over_sections (file, secscan, ¶m);
bfd_close (file);
}
return data.length() != 0;
#endif
}
virtual void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
IPartDescriptor *partDesc = fileDesc->queryPart(slave);
partDesc->serialize(dst);
dst.append(helper->getTempUsageCount());
}
void CDiskReadMasterBase::serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
OwnedRoxieString fileName(helper->getFileName());
dst.append(fileName);
dst.append(subfileLogicalFilenames.ordinality());
if (subfileLogicalFilenames.ordinality())
{
ForEachItemIn(s, subfileLogicalFilenames)
dst.append(subfileLogicalFilenames.item(s));
}
if (mapping)
mapping->serializeMap(slave, dst);
else
CSlavePartMapping::serializeNullMap(dst);
}
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
if (dllHandle == NULL)
dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
if (dllHandle == NULL)
{
DBGLOG("Failed to load library %s: %d", filename, GetLastError());
return false;
}
HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
if (!hrsrc)
return false;
size32_t len = SizeofResource(dllHandle, hrsrc);
const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
data.append(len, rdata);
FreeLibrary(dllHandle);
return true;
#elif defined (_USE_BINUTILS)
CriticalBlock block(bfdCs);
bfd_init ();
bfd *file = bfd_openr(filename, NULL);
if (file)
{
StringBuffer sectionName;
sectionName.append(type).append("_").append(id).append(".data");
SecScanParam param(data, sectionName.str());
if (bfd_check_format (file, bfd_object))
bfd_map_over_sections (file, secscan, ¶m);
bfd_close (file);
}
return data.length() != 0;
#else
struct stat stat_buf;
VStringBuffer sectname("%s_%u", type, id);
int fd = open(filename, O_RDONLY);
if (fd == -1 || fstat(fd, &stat_buf) == -1)
{
DBGLOG("Failed to load library %s: %d", filename, errno);
return false;
}
bool ok = false;
__uint64 size = stat_buf.st_size;
const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
if (start_addr == MAP_FAILED)
{
DBGLOG("Failed to load library %s: %d", filename, errno);
}
else
{
ok = getResourceFromMappedFile(filename, start_addr, data, type, id);
munmap((void *)start_addr, size);
}
close(fd);
return ok;
#endif
}
extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
{
mb.append((byte)0x80).append(resourceHeaderVersion);
if (compress)
compressToBuffer(mb, len, data);
else
appendToBuffer(mb, len, data);
}
void CWriteMasterBase::serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
dst.append(diskHelperBase->getFileName());
if (diskHelperBase->getFlags() & TDXtemporary)
{
unsigned usageCount = container.queryJob().queryWorkUnit().queryFileUsage(diskHelperBase->getFileName());
if (0 == usageCount) usageCount = diskHelperBase->getTempUsageCount();
dst.append(usageCount);
}
if (dlfn.isExternal())
{
fileDesc->queryPart(0)->serialize(dst);
dst.append(mpTag);
}
else
fileDesc->queryPart(targetOffset+slave)->serialize(dst);
}
static bool addWUQueryFilter(WUSortField *filters, unsigned &count, MemoryBuffer &buff, const char *name, WUSortField value)
{
if (!name || !*name)
return false;
filters[count++] = value;
buff.append(name);
return true;
}
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
Owned<IFvDataSource> ds = readDataSource(in);
if (!ds)
{
out.append(false);
return;
}
__int64 requestedRow;
in.read(requestedRow);
unsigned startPos = out.length();
unsigned numRows = 0;
out.append(true); // ok
out.append(requestedRow); // start
unsigned numRowsPos = out.length();
out.append(numRows); // total number of rows;
loop
{
unsigned lengthPos = out.length();
out.append((unsigned)0); // size of this row.
unsigned startRow = out.length();
if (raw)
{
if (!ds->getRawRow(out, requestedRow+numRows))
break;
}
else
{
if (!ds->getRow(out, requestedRow+numRows))
break;
}
if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
break;
unsigned endRow = out.length();
out.rewrite(lengthPos);
out.append(endRow-startRow);
out.rewrite(endRow);
numRows++;
}
if (numRows == 0)
{
out.rewrite(startPos);
out.append(false);
return;
}
unsigned totalLength = out.length();
out.rewrite(numRowsPos);
out.append(numRows);
out.rewrite(totalLength);
}
virtual void serializeCreateContext(MemoryBuffer &mb)
{
// bit of hack, need to tell slave if wuidread converted to diskread (see master activity)
CMasterGraphElement::serializeCreateContext(mb);
if (kind == TAKworkunitread)
{
if (!activity)
doCreateActivity();
IHThorArg *helper = activity->queryHelper();
IHThorDiskReadArg *diskHelper = QUERYINTERFACE(helper, IHThorDiskReadArg);
mb.append(NULL != diskHelper); // flag to slaves that they should create diskread
if (diskHelper)
{
OwnedRoxieString fileName(diskHelper->getFileName());
mb.append(fileName);
}
}
}
void RemoteDataSourceServer::doCmdFetchRaw(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
Owned<IFvDataSource> ds = readDataSource(in);
if (!ds)
{
out.append(false);
return;
}
__int64 requestedOffset;
in.read(requestedOffset);
MemoryBuffer temp;
bool ok = ds->fetchRawRow(temp, requestedOffset);
out.append(ok); // ok
out.append(temp.length());
out.append(temp.length(), temp.toByteArray());
}
bool FullWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
{
size32_t length;
const void * data = rows->fetchRow(offset, length);
if (!data)
return false;
out.append(length, data);
return true;
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
MemoryBuffer tgt;
decompressResource(len, data, tgt);
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
void CDiskReadSlaveActivityBase::serializeStats(MemoryBuffer &mb)
{
CSlaveActivity::serializeStats(mb);
mb.append(diskProgress);
CRuntimeStatisticCollection activeStats(diskReadRemoteStatistics);
if (partHandler)
partHandler->gatherStats(activeStats);
activeStats.serialize(mb);
}
virtual void serializeStats(MemoryBuffer &mb)
{
CSlaveActivity::serializeStats(mb);
CriticalBlock b(joinHelperCrit);
rowcount_t p = joinhelper?joinhelper->getLhsProgress():0;
mb.append(p);
CRuntimeStatisticCollection mergedStats(spillStats);
mergeStats(mergedStats, sorter); // No danger of a race with reset() because that never replaces a valid sorter
mergedStats.serialize(mb);
}
void serializeBoolArray(MemoryBuffer & out, unsigned len, const bool * values)
{
for (unsigned i = 0; i < len; i+= 8)
{
unsigned char next = 0;
unsigned max = i+8 <= len ? 8 : len - i;
for (unsigned j=0; j<max; j++)
if (values[i+j]) next |= (1 << j);
out.append(next);
}
}
void CDiskWriteSlaveActivityBase::serializeStats(MemoryBuffer &mb)
{
CriticalBlock block(statsCs);
ProcessSlaveActivity::serializeStats(mb);
mb.append(replicateDone);
CRuntimeStatisticCollection activeStats(fileStats);
mergeStats(activeStats, outputIO);
activeStats.serialize(mb);
}
void CDiskWriteSlaveActivityBase::processDone(MemoryBuffer &mb)
{
if (abortSoon)
return;
rowcount_t _processed = processed & THORDATALINK_COUNT_MASK;
Owned<IFile> ifile = createIFile(fName);
offset_t sz = ifile->size();
if (-1 != sz)
container.queryJob().queryIDiskUsage().increase(sz);
mb.append(_processed).append(compress?uncompressedBytesWritten:sz).append(sz);
unsigned crc = compress?~0:fileCRC.get();
mb.append(calcFileCrc?crc:0);
CDateTime createTime, modifiedTime, accessedTime;
ifile->getTime(&createTime, &modifiedTime, &accessedTime);
// round file time down to nearest sec. Nanosec accuracy is not preserved elsewhere and can lead to mismatch later.
unsigned hour, min, sec, nanosec;
modifiedTime.getTime(hour, min, sec, nanosec);
modifiedTime.setTime(hour, min, sec, 0);
modifiedTime.serialize(mb);
}
void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
{
SessionId session;
StringAttr username, password, logicalName;
in.read(session);
in.read(username).read(password);
in.read(logicalName);
DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session);
Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
unique_id_t id = addDataSource(session, ds);
out.append((unsigned short)CurRemoteVersion);
out.append(id);
out.append(ds->numRows(false));
ds->queryMetaData()->serialize(out);
out.append(ds->isIndex());
DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id);
}
static bool serializeWUInfo(IConstWorkUnitInfo &info,MemoryBuffer &mb)
{
fixedAppend(mb,24,info.queryWuid());
varAppendMax(mb,64,info.queryUser());
varAppendMax(mb,64,info.queryClusterName());
varAppendMax(mb,64,""); // roxiecluster is obsolete
varAppendMax(mb,256,info.queryJobName());
fixedAppend(mb,10,info.queryStateDesc());
fixedAppend(mb,7,info.queryPriorityDesc());
short int prioritylevel = info.getPriorityLevel();
mb.append(prioritylevel);
fixedAppend(mb,20,""); // Created timestamp
fixedAppend(mb,20,""); // Modified timestamp
mb.append(true);
mb.append(info.isProtected());
if (mb.length()>WORKUNIT_SERVICES_BUFFER_MAX) {
mb.clear().append(WUS_STATUS_OVERFLOWED);
return false;
}
return true;
}
static void serializeElement(MemoryBuffer & target, IDefRecordElement * elem)
{
byte kind = elem ? elem->getKind() : DEKnone;
target.append(kind);
switch (kind)
{
case DEKnone:
break;
case DEKrecord:
{
size32_t maxSize = elem->getMaxSize();
unsigned numChildren = elem->numChildren();
target.append(maxSize).append(numChildren);
for (unsigned i=0; i < numChildren; i++)
serializeElement(target, elem->queryChild(i));
break;
}
case DEKifblock:
{
IValue * value = elem->queryCompareValue();
serializeValue(target, value);
serializeElement(target, elem->queryChild(0));
serializeElement(target, elem->queryChild(1));
break;
}
case DEKfield:
{
_ATOM name = elem->queryName();
ITypeInfo * type = elem->queryType();
size32_t maxSize = elem->getMaxSize();
serializeAtom(target, name);
type->serialize(target);
serializeElement(target, elem->queryChild(0));
target.append(maxSize);
break;
}
default:
throwUnexpected();
}
}
void init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
AggregateSlaveBase::init(data, slaveData);
appendOutputLinked(this);
helper = (IHThorAggregateArg *)queryHelper();
if (firstNode())
{
mpTag = createReplyTag();
slaveData.append((int &)mpTag);
}
}
void FileFormat::serialize(MemoryBuffer & out) const
{
out.append((byte)type);
switch (type)
{
case FFTfixed:
case FFTblocked:
out.append(recordSize);
break;
case FFTcsv:
case FFTutf:
case FFTutf8: case FFTutf8n:
case FFTutf16: case FFTutf16be: case FFTutf16le:
case FFTutf32: case FFTutf32be: case FFTutf32le:
out.append(maxRecordSize);
::serialize(out, separate);
::serialize(out, quote);
::serialize(out, terminate);
::serialize(out, rowTag);
break;
}
}
void ProcessSlaveActivity::serializeStats(MemoryBuffer &mb)
{
CSlaveActivity::serializeStats(mb);
mb.append(processed);
#ifdef TIME_ACTIVITIES
if (timeActivities)
{
unsigned __int64 nowCycles = get_cycles_now();
totalCycles += nowCycles-lastCycles;
lastCycles = nowCycles;
}
#endif
}
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra)
{
if (local)
i = 0;
if (i >= maps.ordinality())
{
mb.append((unsigned)0);
return;
}
CSlaveMap &map = maps.item(i);
unsigned nPos = mb.length();
unsigned n=0;
mb.append(n);
UnsignedArray parts;
ForEachItemIn(m, map)
parts.append(map.item(m).queryPartIndex());
MemoryBuffer extraMb;
if (extra)
{
ForEachItemIn(m2, map)
{
unsigned xtraLen = 0;
unsigned xtraPos = extraMb.length();
extraMb.append(xtraLen);
IPartDescriptor &partDesc = map.item(m2);
if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb))
{
parts.zap(partDesc.queryPartIndex());
extraMb.rewrite(xtraPos);
}
else
{
xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen);
extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen);
}
}
}
bool FullWorkUnitDataSource::init()
{
bool ok = WorkUnitDataSource::init();
if (ok)
{
MemoryBuffer temp;
MemoryBuffer2IDataVal xxx(temp);
//Nasty. Single sets are represented as the same way as datasets (with an extra flag for all)
//however need to represent as a single row containing a set, which has a different format.
if (wuResult->isResultScalar() && returnedMeta->isSingleSet())
{
temp.append(wuResult->getResultIsAll());
temp.append((size32_t)wuResult->getResultRawSize(0, 0));
}
wuResult->getResultRaw(xxx, NULL, NULL);
if (returnedMeta->isFixedSize())
rows.setown(new FixedRowBlock(temp, 0, 0, returnedMeta->fixedSize()));
else
rows.setown(new VariableRowBlock(temp, 0, 0, returnedRecordSize, true));
}
return ok;
}
void DataSourceMetaData::serialize(MemoryBuffer & buffer) const
{
//NB: Update NullDataSourceMeta if this changes....
buffer.append(numFieldsToIgnore);
buffer.append(randomIsOk);
buffer.append(hasMixedContent);
buffer.append(isStoredFixedWidth);
buffer.append(minRecordSize);
buffer.append(keyedSize);
buffer.append(maxRecordSize);
unsigned numFields = fields.ordinality();
buffer.append(numFields);
for (unsigned idx=0; idx < numFields; idx++)
{
fields.item(idx).serialize(buffer);
}
}