void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
IPartDescriptor *partDesc = fileDesc->queryPart(slave);
partDesc->serialize(dst);
dst.append(helper->getTempUsageCount());
}
void done()
{
if (!container.queryOwner().queryOwner() || container.queryOwner().isGlobal()) // I am in a child query
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
OwnedRoxieString fname(helper->getFileName());
container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
}
}
void open()
{
char drive [_MAX_DRIVE];
char dir [_MAX_DIR];
char fname [_MAX_DIR];
char ext [_MAX_EXT];
_splitpath(fileName.str(), drive, dir, fname, ext);
StringBuffer directory;
directory.append(drive).append(dir);
Owned<IFile> cd = createIFile(directory.str());
cd->createDirectory();
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
Owned<ICompressor> ecomp;
if (ekeylen!=0)
{
ecomp.setown(createAESCompressor256(ekeylen,ekey));
memset(ekey,0,ekeylen);
free(ekey);
compress = true;
}
Owned<IFile> file = createIFile(fileName.str());
Owned<IFileIO> iFileIO;
bool fixedRecordSize = queryRowMetaData()->isFixedSize();
size32_t minrecsize = queryRowMetaData()->getMinRecordSize();
if (fixedRecordSize)
ActPrintLog("SPILL: created fixed output %s recsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
else
ActPrintLog("SPILL: created variable output %s, minrecsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
unsigned rwFlags = (DEFAULT_RWFLAGS & ~rw_autoflush); // flushed by close()
if (compress)
rwFlags |= rw_compress;
else
rwFlags |= rw_crc; // only if !compress
if (grouped)
rwFlags |= rw_grouped;
out.setown(createRowWriter(file, this, rwFlags));
}
void init()
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
IArrayOf<IGroup> groups;
OwnedRoxieString fname(helper->getFileName());
fillClusterArray(container.queryJob(), fname, clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, true, TDWnoreplicate+TDXtemporary));
IPropertyTree &props = fileDesc->queryProperties();
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (helper->getFlags() & TDWnewcompress) || 0 != (helper->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (0 != (helper->getFlags() & TDXgrouped))
fileDesc->queryProperties().setPropBool("@grouped", true);
props.setProp("@kind", "flat");
if (container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
}
}