virtual void done()
{
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(outputName);
r.clear();
wu.clear();
IPropertyTree &props = newIndexDesc->queryProperties();
props.setProp("@kind", "key");
if (0 != (helper->getFlags() & KDPexpires))
setExpiryTime(props, helper->getExpiryDays());
// Fill in some logical file properties here
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
props.setProp("ECL", originalProps.queryProp("ECL"));
MemoryBuffer rLMB;
if (originalProps.getPropBin("_record_layout", rLMB))
props.setPropBin("_record_layout", rLMB.length(), rLMB.toByteArray());
props.setPropInt("@formatCrc", originalProps.getPropInt("@formatCrc"));
if (originalProps.getPropBool("@local"))
props.setPropBool("@local", true);
container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
queryThorFileManager().publish(container.queryJob(), outputName, *newIndexDesc);
CMasterActivity::done();
}
void CWriteMasterBase::publish()
{
if (published) return;
published = true;
if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
{
StringBuffer scopedName;
queryThorFileManager().addScope(container.queryJob(), diskHelperBase->getFileName(), scopedName);
updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), scopedName.str(), recordsProcessed);
}
IPropertyTree &props = fileDesc->queryProperties();
props.setPropInt64("@recordCount", recordsProcessed);
if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
{
setExpiryTime(props, diskHelperBase->getExpiryDays());
if (TDWupdate & diskHelperBase->getFlags())
{
unsigned eclCRC;
unsigned __int64 totalCRC;
diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
props.setPropInt("@eclCRC", eclCRC);
props.setPropInt64("@totalCRC", totalCRC);
}
}
container.queryTempHandler()->registerFile(diskHelperBase->getFileName(), container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
if (!dlfn.isExternal())
{
bool mangle = 0 != (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp));
queryThorFileManager().publish(container.queryJob(), diskHelperBase->getFileName(), mangle, *fileDesc, NULL, targetOffset);
}
}
virtual void done()
{
StringBuffer scopedName;
OwnedRoxieString outputName(helper->getOutputName());
queryThorFileManager().addScope(container.queryJob(), outputName, scopedName);
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(scopedName.str());
r.clear();
wu.clear();
IPropertyTree &patchProps = patchDesc->queryProperties();
if (0 != (helper->getFlags() & KDPexpires))
setExpiryTime(patchProps, helper->getExpiryDays());
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
patchProps.setProp("ECL", originalProps.queryProp("ECL"));
if (originalProps.getPropBool("@local"))
patchProps.setPropBool("@local", true);
container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
Owned<IDistributedFile> patchFile;
// set part sizes etc
queryThorFileManager().publish(container.queryJob(), outputName, false, *patchDesc, &patchFile, 0, false);
try { // set file size
if (patchFile) {
__int64 fs = patchFile->getFileSize(true,false);
if (fs!=-1)
patchFile->queryAttributes().setPropInt64("@size",fs);
}
}
catch (IException *e) {
EXCLOG(e,"keydiff setting file size");
e->Release();
}
// Add a new 'Patch' description to the secondary key.
DistributedFilePropertyLock lock(newIndexFile);
IPropertyTree &fileProps = lock.queryAttributes();
StringBuffer path("Patch[@name=\"");
path.append(scopedName.str()).append("\"]");
IPropertyTree *patch = fileProps.queryPropTree(path.str());
if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
patch->setProp("@name", scopedName.str());
unsigned checkSum;
if (patchFile->getFileCheckSum(checkSum))
patch->setPropInt64("@checkSum", checkSum);
IPropertyTree *index = patch->setPropTree("Index", createPTree());
index->setProp("@name", originalIndexFile->queryLogicalName());
if (originalIndexFile->getFileCheckSum(checkSum))
index->setPropInt64("@checkSum", checkSum);
originalIndexFile->setAccessed();
newIndexFile->setAccessed();
}
void CWriteMasterBase::publish()
{
if (published) return;
published = true;
if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), fileName, recordsProcessed);
IPropertyTree &props = fileDesc->queryProperties();
props.setPropInt64("@recordCount", recordsProcessed);
if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
{
if (0 != (diskHelperBase->getFlags() & TDWexpires))
setExpiryTime(props, diskHelperBase->getExpiryDays());
if (TDWupdate & diskHelperBase->getFlags())
{
unsigned eclCRC;
unsigned __int64 totalCRC;
diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
props.setPropInt("@eclCRC", eclCRC);
props.setPropInt64("@totalCRC", totalCRC);
}
}
container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
if (!dlfn.isExternal())
{
bool temporary = 0 != (diskHelperBase->getFlags()&TDXtemporary);
if (!temporary && (queryJob().querySlaves() < fileDesc->numParts()))
{
// create empty parts for a fileDesc being published that is larger than this clusters
size32_t recordSize = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
recordSize = diskRowMeta->getMinRecordSize();
if (0 != (diskHelperBase->getFlags() & TDXgrouped))
recordSize += 1;
}
unsigned compMethod = COMPRESS_METHOD_LZW;
// rowdiff used if recordSize > 0, else fallback to compMethod
if (getOptBool(THOROPT_COMP_FORCELZW, false))
{
recordSize = 0; // by default if fixed length (recordSize set), row diff compression is used. This forces compMethod.
compMethod = COMPRESS_METHOD_LZW;
}
else if (getOptBool(THOROPT_COMP_FORCEFLZ, false))
compMethod = COMPRESS_METHOD_FASTLZ;
else if (getOptBool(THOROPT_COMP_FORCELZ4, false))
compMethod = COMPRESS_METHOD_LZ4;
bool blockCompressed;
bool compressed = fileDesc->isCompressed(&blockCompressed);
for (unsigned clusterIdx=0; clusterIdx<fileDesc->numClusters(); clusterIdx++)
{
StringBuffer clusterName;
fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore());
PROGLOG("Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str());
unsigned p=0;
while (p<fileDesc->numParts())
{
if (p == targetOffset)
p += queryJob().querySlaves();
IPartDescriptor *partDesc = fileDesc->queryPart(p);
CDateTime createTime, modifiedTime;
for (unsigned c=0; c<partDesc->numCopies(); c++)
{
RemoteFilename rfn;
partDesc->getFilename(c, rfn);
StringBuffer path;
rfn.getPath(path);
try
{
ensureDirectoryForFile(path.str());
OwnedIFile iFile = createIFile(path.str());
Owned<IFileIO> iFileIO;
if (compressed)
iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod));
else
iFileIO.setown(iFile->open(IFOcreate));
dbgassertex(iFileIO.get());
iFileIO.clear();
// ensure copies have matching datestamps, as they would do normally (backupnode expects it)
if (partDesc->numCopies() > 1)
{
if (0 == c)
iFile->getTime(&createTime, &modifiedTime, NULL);
else
iFile->setTime(&createTime, &modifiedTime, NULL);
}
}
catch (IException *e)
{
if (0 == c)
throw;
Owned<IThorException> e2 = MakeThorException(e);
e->Release();
e2->setAction(tea_warning);
queryJob().fireException(e2);
}
}
partDesc->queryProperties().setPropInt64("@size", 0);
p++;
}
clusterIdx++;
}
}
queryThorFileManager().publish(container.queryJob(), fileName, *fileDesc, NULL);
}
}
