void init()
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
IArrayOf<IGroup> groups;
OwnedRoxieString fname(helper->getFileName());
fillClusterArray(container.queryJob(), fname, clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, true, TDWnoreplicate+TDXtemporary));
IPropertyTree &props = fileDesc->queryProperties();
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (helper->getFlags() & TDWnewcompress) || 0 != (helper->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (0 != (helper->getFlags() & TDXgrouped))
fileDesc->queryProperties().setPropBool("@grouped", true);
props.setProp("@kind", "flat");
if (container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
}
}
virtual void init()
{
HashDistributeMasterBase::init();
// JCSMORE should common up some with indexread
IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();
StringBuffer scoped;
OwnedRoxieString indexFileName(helper->getIndexFileName());
queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped);
file.setown(queryThorFileManager().lookup(container.queryJob(), indexFileName));
if (!file)
throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
if (0 == file->numParts())
throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());
checkFormatCrc(this, file, helper->getFormatCrc(), true);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
tlkMb.append(iFileIO->size());
::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);
queryThorFileManager().noteFileRead(container.queryJob(), file);
}
void init()
{
helper = (IHThorKeyDiffArg *)queryHelper();
OwnedRoxieString origName(helper->getOriginalName());
OwnedRoxieString updatedName(helper->getUpdatedName());
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), origName));
newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedName));
if (originalIndexFile->numParts() != newIndexFile->numParts())
throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", origName.get(), updatedName.get());
if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
throw MakeActivityException(this, 0, "Diffing super files not supported");
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
newIndexDesc.setown(newIndexFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());
queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile);
queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile);
IArrayOf<IGroup> groups;
OwnedRoxieString outputName(helper->getOutputName());
fillClusterArray(container.queryJob(), outputName, clusters, groups);
patchDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
}
void CWriteMasterBase::publish()
{
if (published) return;
published = true;
if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
{
StringBuffer scopedName;
queryThorFileManager().addScope(container.queryJob(), diskHelperBase->getFileName(), scopedName);
updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), scopedName.str(), recordsProcessed);
}
IPropertyTree &props = fileDesc->queryProperties();
props.setPropInt64("@recordCount", recordsProcessed);
if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
{
setExpiryTime(props, diskHelperBase->getExpiryDays());
if (TDWupdate & diskHelperBase->getFlags())
{
unsigned eclCRC;
unsigned __int64 totalCRC;
diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
props.setPropInt("@eclCRC", eclCRC);
props.setPropInt64("@totalCRC", totalCRC);
}
}
container.queryTempHandler()->registerFile(diskHelperBase->getFileName(), container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
if (!dlfn.isExternal())
{
bool mangle = 0 != (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp));
queryThorFileManager().publish(container.queryJob(), diskHelperBase->getFileName(), mangle, *fileDesc, NULL, targetOffset);
}
}
void CDiskReadMasterBase::init()
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
fileName.setown(helper->getFileName());
file.setown(queryThorFileManager().lookup(container.queryJob(), fileName, 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true));
if (file)
{
if (file->numParts() > 1)
fileDesc.setown(getConfiguredFileDescriptor(*file));
else
fileDesc.setown(file->getFileDescriptor());
reInit = 0 != (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename));
if (container.queryLocal() || helper->canMatchAny()) // if local, assume may match
{
bool local;
if (0 == (TDXtemporary & helper->getFlags())) // don't add temp files
{
queryThorFileManager().noteFileRead(container.queryJob(), file);
local = container.queryLocal();
}
else
local = false;
mapping.setown(getFileSlaveMaps(file->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), local, false, hash, file->querySuperFile()));
}
if (0 != (helper->getFlags() & TDRfilenamecallback)) // only get/serialize if using virtual file name fields
{
IDistributedSuperFile *super = file->querySuperFile();
if (super)
{
unsigned numsubs = super->numSubFiles(true);
unsigned s=0;
for (; s<numsubs; s++)
{
IDistributedFile &subfile = super->querySubFile(s, true);
subfileLogicalFilenames.append(subfile.queryLogicalName());
}
}
}
validateFile(file);
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fileName.get());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fileName.get());
}
}
void CWriteMasterBase::init()
{
published = false;
recordsProcessed = 0;
dlfn.set(diskHelperBase->getFileName());
if (diskHelperBase->getFlags() & TDWextend)
{
assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), diskHelperBase->getFileName(), false, true);
if (file.get())
{
fileDesc.setown(file->getFileDescriptor());
queryThorFileManager().noteFileRead(container.queryJob(), file, true);
}
}
if (dlfn.isExternal())
mpTag = container.queryJob().allocateMPTag(); // used
if (NULL == fileDesc.get())
{
bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags());
unsigned idx=0;
while (diskHelperBase->queryCluster(idx))
clusters.append(diskHelperBase->queryCluster(idx++));
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), diskHelperBase->getFileName(), clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), diskHelperBase->getFileName(), clusters, groups, overwriteok, diskHelperBase->getFlags()));
if (1 == groups.ordinality())
targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());
IPropertyTree &props = fileDesc->queryProperties();
if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
props.setPropBool("@owned", true);
if (diskHelperBase->getFlags() & TDWresult)
props.setPropBool("@result", true);
const char *rececl= diskHelperBase->queryRecordECL();
if (rececl&&*rececl)
props.setProp("ECL", rececl);
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
diskHelperBase->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
publish();
}
}
}
virtual void done()
{
StringBuffer scopedName;
OwnedRoxieString outputName(helper->getOutputName());
queryThorFileManager().addScope(container.queryJob(), outputName, scopedName);
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(scopedName.str());
r.clear();
wu.clear();
IPropertyTree &patchProps = patchDesc->queryProperties();
if (0 != (helper->getFlags() & KDPexpires))
setExpiryTime(patchProps, helper->getExpiryDays());
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
patchProps.setProp("ECL", originalProps.queryProp("ECL"));
if (originalProps.getPropBool("@local"))
patchProps.setPropBool("@local", true);
container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
Owned<IDistributedFile> patchFile;
// set part sizes etc
queryThorFileManager().publish(container.queryJob(), outputName, false, *patchDesc, &patchFile, 0, false);
try { // set file size
if (patchFile) {
__int64 fs = patchFile->getFileSize(true,false);
if (fs!=-1)
patchFile->queryAttributes().setPropInt64("@size",fs);
}
}
catch (IException *e) {
EXCLOG(e,"keydiff setting file size");
e->Release();
}
// Add a new 'Patch' description to the secondary key.
DistributedFilePropertyLock lock(newIndexFile);
IPropertyTree &fileProps = lock.queryAttributes();
StringBuffer path("Patch[@name=\"");
path.append(scopedName.str()).append("\"]");
IPropertyTree *patch = fileProps.queryPropTree(path.str());
if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
patch->setProp("@name", scopedName.str());
unsigned checkSum;
if (patchFile->getFileCheckSum(checkSum))
patch->setPropInt64("@checkSum", checkSum);
IPropertyTree *index = patch->setPropTree("Index", createPTree());
index->setProp("@name", originalIndexFile->queryLogicalName());
if (originalIndexFile->getFileCheckSum(checkSum))
index->setPropInt64("@checkSum", checkSum);
originalIndexFile->setAccessed();
newIndexFile->setAccessed();
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString fname(helper->getFileName());
Owned<IDistributedFile> fetchFile = queryThorFileManager().lookup(container.queryJob(), fname, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true);
if (fetchFile)
{
if (isFileKey(fetchFile))
throw MakeActivityException(this, 0, "Attempting to read index as a flat file: %s", fname.get());
Owned<IFileDescriptor> fileDesc = getConfiguredFileDescriptor(*fetchFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fetchFile->queryLogicalName());
queryJobChannel().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFile->queryLogicalName());
mapping.setown(getFileSlaveMaps(fetchFile->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), container.queryLocalOrGrouped(), false, NULL, fetchFile->querySuperFile()));
mapping->serializeFileOffsetMap(offsetMapMb);
addReadFile(fetchFile);
}
}
virtual void init()
{
CSortBaseActivityMaster::init();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
IHThorAlgorithm *algo = static_cast<IHThorAlgorithm *>(helper->selectInterface(TAIalgorithm_1));
OwnedRoxieString algoname(algo->getAlgorithm());
unsigned flags = algo->getAlgorithmFlags();
if (algoname && (0 != stricmp(algoname, "quicksort")))
{
Owned<IException> e = MakeActivityException(this, 0, "Ignoring, unsupported sort order algorithm '%s'", algoname.get());
reportExceptionToWorkunit(container.queryJob().queryWorkUnit(), e);
}
OwnedRoxieString cosortlogname(helper->getSortedFilename());
if (cosortlogname&&*cosortlogname)
{
Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
addReadFile(coSortFile);
Owned<IFileDescriptor> fileDesc = coSortFile->getFileDescriptor();
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
}
virtual void done()
{
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(outputName);
r.clear();
wu.clear();
IPropertyTree &props = newIndexDesc->queryProperties();
props.setProp("@kind", "key");
if (0 != (helper->getFlags() & KDPexpires))
setExpiryTime(props, helper->getExpiryDays());
// Fill in some logical file properties here
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
props.setProp("ECL", originalProps.queryProp("ECL"));
MemoryBuffer rLMB;
if (originalProps.getPropBin("_record_layout", rLMB))
props.setPropBin("_record_layout", rLMB.length(), rLMB.toByteArray());
props.setPropInt("@formatCrc", originalProps.getPropInt("@formatCrc"));
if (originalProps.getPropBool("@local"))
props.setPropBool("@local", true);
container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
queryThorFileManager().publish(container.queryJob(), outputName, *newIndexDesc);
CMasterActivity::done();
}
void done()
{
if (!container.queryOwner().queryOwner() || container.queryOwner().isGlobal()) // I am in a child query
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
OwnedRoxieString fname(helper->getFileName());
container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
}
}
void CDiskReadMasterBase::done()
{
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
fileDesc.clear();
if (!abortSoon) // in case query has relinquished control of file usage to another query (e.g. perists)
{
if (0 != (helper->getFlags() & TDXupdateaccessed))
queryThorFileManager().updateAccessTime(container.queryJob(), helper->getFileName());
}
}
void preStart(size32_t parentExtractSz, const byte *parentExtract)
{
CMasterActivity::preStart(parentExtractSz, parentExtract);
IHThorKeyPatchArg *helper = (IHThorKeyPatchArg *) queryHelper();
if (0==(KDPoverwrite & helper->getFlags()))
{
if (KDPvaroutputname & helper->getFlags()) return;
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputName, false, true);
if (file)
throw MakeActivityException(this, TE_OverwriteNotSpecified, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", file->queryLogicalName());
}
}
void CWriteMasterBase::preStart(size32_t parentExtractSz, const byte *parentExtract)
{
CMasterActivity::preStart(parentExtractSz, parentExtract);
if (TAKdiskwrite == container.getKind())
{
if (0 == ((TDXvarfilename|TDXtemporary|TDXjobtemp) & diskHelperBase->getFlags()))
{
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), diskHelperBase->getFileName(), false, true);
if (file)
{
if (0 == (TDWextend+TDWoverwrite & diskHelperBase->getFlags()))
throw MakeActivityException(this, TE_OverwriteNotSpecified, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", file->queryLogicalName());
checkSuperFileOwnership(*file);
}
}
}
}
virtual void init()
{
CMasterActivity::init();
helper = (IHThorKeyPatchArg *)queryHelper();
OwnedRoxieString originalHelperName(helper->getOriginalName());
OwnedRoxieString patchHelperName(helper->getPatchName());
OwnedRoxieString outputHelperName(helper->getOutputName());
StringBuffer expandedFileName;
queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false);
originalName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), patchHelperName, expandedFileName.clear(), false);
patchName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
outputName.set(expandedFileName);
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName));
if (originalIndexFile->numParts() != patchFile->numParts())
throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get());
if (originalIndexFile->querySuperFile() || patchFile->querySuperFile())
throw MakeActivityException(this, 0, "Patching super files not supported");
addReadFile(originalIndexFile);
addReadFile(patchFile);
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
patchDesc.setown(patchFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local && width > 1)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves());
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), outputName, clusters, groups);
newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
if (!local)
newIndexDesc->queryPart(newIndexDesc->numParts()-1)->queryProperties().setProp("@kind", "topLevelKey");
}
virtual void init()
{
CDiskReadMasterVF::init();
bool canMatch = container.queryLocalOrGrouped() || helper->canMatchAny(); // if local, assume may match
if (!canMatch)
totalCountKnown = true; // totalCount = 0;
else if (!container.queryLocalOrGrouped())
{
if (!helper->hasSegmentMonitors() && !helper->hasFilter() && !(helper->getFlags() & TDXtemporary))
{
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helper->getFileName(), 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()));
if (file.get() && canMatch)
{
if (0 != (TDRunfilteredcount & helper->getFlags()) && file->queryAttributes().hasProp("@recordCount"))
{
totalCount = (rowcount_t)file->queryAttributes().getPropInt64("@recordCount");
if (totalCount > stopAfter)
totalCount = stopAfter;
totalCountKnown = true;
}
}
}
}
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString helperFileName = helper->getFileName();
StringBuffer expandedFileName;
queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName, false);
fileName.set(expandedFileName);
dlfn.set(fileName);
isLocal = 0 != (TIWlocal & helper->getFlags());
IOutputMetaData * diskSize = helper->queryDiskRecordSize();
unsigned minSize = diskSize->getMinRecordSize();
if (minSize > KEYBUILD_MAXLENGTH)
throw MakeActivityException(this, 0, "Index minimum record length (%d) exceeds %d internal limit", minSize, KEYBUILD_MAXLENGTH);
unsigned maxSize;
if (diskSize->isVariableSize())
{
if (TIWmaxlength & helper->getFlags())
maxSize = helper->getMaxKeySize();
else
maxSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
}
else
maxSize = diskSize->getFixedSize();
if (maxSize > KEYBUILD_MAXLENGTH)
throw MakeActivityException(this, 0, "Index maximum record length (%d) exceeds %d internal limit. Maximum size = %d, try adjusting index MAXLENGTH", maxSize, KEYBUILD_MAXLENGTH, minSize);
singlePartKey = 0 != (helper->getFlags() & TIWsmall) || dlfn.isExternal();
clusters.kill();
unsigned idx=0;
while (true)
{
OwnedRoxieString cluster(helper->getCluster(idx));
if(!cluster)
break;
clusters.append(cluster);
idx++;
}
IArrayOf<IGroup> groups;
if (singlePartKey)
{
isLocal = true;
buildTlk = false;
}
else if (!isLocal || globals->getPropBool("@buildLocalTlks", true))
buildTlk = true;
fillClusterArray(container.queryJob(), fileName, clusters, groups);
unsigned restrictedWidth = 0;
if (TIWhaswidth & helper->getFlags())
{
restrictedWidth = helper->getWidth();
if (restrictedWidth > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported, can't refactor to width(%d) larger than host cluster(%d)", restrictedWidth, container.queryJob().querySlaves());
else if (restrictedWidth < container.queryJob().querySlaves())
{
if (!isLocal)
throw MakeActivityException(this, 0, "Unsupported, refactoring to few parts only supported for local indexes.");
assertex(!singlePartKey);
unsigned gwidth = groups.item(0).ordinality();
if (0 != container.queryJob().querySlaves() % gwidth)
throw MakeActivityException(this, 0, "Unsupported, refactored target size (%d) must be factor of thor cluster width (%d)", groups.item(0).ordinality(), container.queryJob().querySlaves());
if (0 == restrictedWidth)
restrictedWidth = gwidth;
ForEachItemIn(g, groups)
{
IGroup &group = groups.item(g);
if (gwidth != groups.item(g).ordinality())
throw MakeActivityException(this, 0, "Unsupported, cannot output multiple refactored widths, targeting cluster '%s' and '%s'", clusters.item(0), clusters.item(g));
if (gwidth != restrictedWidth)
groups.replace(*group.subset((unsigned)0, restrictedWidth), g);
}
refactor = true;
}
void CWriteMasterBase::publish()
{
if (published) return;
published = true;
if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), fileName, recordsProcessed);
IPropertyTree &props = fileDesc->queryProperties();
props.setPropInt64("@recordCount", recordsProcessed);
if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
{
if (0 != (diskHelperBase->getFlags() & TDWexpires))
setExpiryTime(props, diskHelperBase->getExpiryDays());
if (TDWupdate & diskHelperBase->getFlags())
{
unsigned eclCRC;
unsigned __int64 totalCRC;
diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
props.setPropInt("@eclCRC", eclCRC);
props.setPropInt64("@totalCRC", totalCRC);
}
}
container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
if (!dlfn.isExternal())
{
bool temporary = 0 != (diskHelperBase->getFlags()&TDXtemporary);
if (!temporary && (queryJob().querySlaves() < fileDesc->numParts()))
{
// create empty parts for a fileDesc being published that is larger than this clusters
size32_t recordSize = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
recordSize = diskRowMeta->getMinRecordSize();
if (0 != (diskHelperBase->getFlags() & TDXgrouped))
recordSize += 1;
}
unsigned compMethod = COMPRESS_METHOD_LZW;
// rowdiff used if recordSize > 0, else fallback to compMethod
if (getOptBool(THOROPT_COMP_FORCELZW, false))
{
recordSize = 0; // by default if fixed length (recordSize set), row diff compression is used. This forces compMethod.
compMethod = COMPRESS_METHOD_LZW;
}
else if (getOptBool(THOROPT_COMP_FORCEFLZ, false))
compMethod = COMPRESS_METHOD_FASTLZ;
else if (getOptBool(THOROPT_COMP_FORCELZ4, false))
compMethod = COMPRESS_METHOD_LZ4;
bool blockCompressed;
bool compressed = fileDesc->isCompressed(&blockCompressed);
for (unsigned clusterIdx=0; clusterIdx<fileDesc->numClusters(); clusterIdx++)
{
StringBuffer clusterName;
fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore());
PROGLOG("Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str());
unsigned p=0;
while (p<fileDesc->numParts())
{
if (p == targetOffset)
p += queryJob().querySlaves();
IPartDescriptor *partDesc = fileDesc->queryPart(p);
CDateTime createTime, modifiedTime;
for (unsigned c=0; c<partDesc->numCopies(); c++)
{
RemoteFilename rfn;
partDesc->getFilename(c, rfn);
StringBuffer path;
rfn.getPath(path);
try
{
ensureDirectoryForFile(path.str());
OwnedIFile iFile = createIFile(path.str());
Owned<IFileIO> iFileIO;
if (compressed)
iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod));
else
iFileIO.setown(iFile->open(IFOcreate));
dbgassertex(iFileIO.get());
iFileIO.clear();
// ensure copies have matching datestamps, as they would do normally (backupnode expects it)
if (partDesc->numCopies() > 1)
{
if (0 == c)
iFile->getTime(&createTime, &modifiedTime, NULL);
else
iFile->setTime(&createTime, &modifiedTime, NULL);
}
}
catch (IException *e)
{
if (0 == c)
throw;
Owned<IThorException> e2 = MakeThorException(e);
e->Release();
e2->setAction(tea_warning);
queryJob().fireException(e2);
}
}
partDesc->queryProperties().setPropInt64("@size", 0);
p++;
}
clusterIdx++;
}
}
queryThorFileManager().publish(container.queryJob(), fileName, *fileDesc, NULL);
}
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString indexFileName(helper->getIndexFileName());
Owned<IDistributedFile> dataFile;
Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
keyReadWidth = container.queryJob().querySlaves();
initMb.clear();
initMb.append(indexFileName.get());
if (helper->diskAccessRequired())
numTags += 2;
initMb.append(numTags);
unsigned t=0;
for (; t<numTags; t++)
{
tags[t] = container.queryJob().allocateMPTag();
initMb.append(tags[t]);
}
bool keyHasTlk = false;
if (indexFile)
{
unsigned numParts = 0;
localKey = indexFile->queryAttributes().getPropBool("@local");
if (container.queryLocalData() && !localKey)
throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
IDistributedSuperFile *superIndex = indexFile->querySuperFile();
unsigned superIndexWidth = 0;
unsigned numSuperIndexSubs = 0;
if (superIndex)
{
numSuperIndexSubs = superIndex->numSubFiles(true);
bool first=true;
// consistency check
Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
ForEach(*iter)
{
IDistributedFile &f = iter->query();
unsigned np = f.numParts()-1;
IDistributedFilePart &part = f.queryPart(np);
const char *kind = part.queryAttributes().queryProp("@kind");
bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
if (first)
{
first = false;
keyHasTlk = hasTlk;
superIndexWidth = f.numParts();
if (keyHasTlk)
--superIndexWidth;
}
else
{
if (hasTlk != keyHasTlk)
throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
if (keyHasTlk && superIndexWidth != f.numParts()-1)
throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
}
}
if (keyHasTlk)
numParts = superIndexWidth * numSuperIndexSubs;
else
numParts = superIndex->numParts();
}
else
{
numParts = indexFile->numParts();
if (numParts)
{
const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
if (keyHasTlk)
--numParts;
}
}
if (numParts)
{
initMb.append(numParts);
initMb.append(superIndexWidth); // 0 if not superIndex
initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
unsigned p=0;
UnsignedArray parts;
for (; p<numParts; p++)
parts.append(p);
indexFileDesc->serializeParts(initMb, parts);
if (localKey)
keyHasTlk = false; // not used
initMb.append(keyHasTlk);
if (keyHasTlk)
{
if (numSuperIndexSubs)
initMb.append(numSuperIndexSubs);
else
initMb.append((unsigned)1);
Owned<IDistributedFileIterator> iter;
IDistributedFile *f;
if (superIndex)
{
iter.setown(superIndex->getSubFileIterator(true));
f = &iter->query();
}
else
f = indexFile;
loop
{
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
size32_t tlkSz = (size32_t)iFileIO->size();
initMb.append(tlkSz);
::read(iFileIO, 0, tlkSz, initMb);
if (!iter || !iter->next())
break;
f = &iter->query();
}
}
if (helper->diskAccessRequired())
{
OwnedRoxieString fetchFilename(helper->getFileName());
if (fetchFilename)
{
dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
if (dataFile)
{
if (superIndex)
throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
dataReadWidth = container.queryJob().querySlaves();
Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
}
else
indexFile.clear();
}
}
}
else
virtual void process()
{
ActPrintLog("process");
CMasterActivity::process();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
StringBuffer skewV;
double skewError;
container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
if (skewV.length())
skewError = atof(skewV.str());
else
{
skewError = helper->getSkew();
if (!skewError)
{
container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
if (skewV.length())
skewError = atof(skewV.str());
}
}
container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
double skewWarning = defaultSkewWarning;
unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
if (!skewThreshold)
{
skewThreshold = helper->getThreshold();
if (!skewThreshold)
skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
}
StringBuffer cosortfilenames;
const char *cosortlogname = helper->getSortedFilename();
if (cosortlogname&&*cosortlogname) {
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
queryThorFileManager().noteFileRead(container.queryJob(), file);
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext());
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
try {
imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif);
if (barrier->wait(false)) { // local sort complete
size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024);
if (!imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"))) {
Owned<IThorException> e = MakeActivityException(this, TE_SortFailedSkewExceeded,"SORT failed, skew exceeded");
fireException(e);
}
barrier->wait(false); // merge complete
}
imaster->SortDone();
}
catch (IException *e)
{
ActPrintLog(e, "WARNING: exception during sort");
throw;
}
::Release(imaster);
ActPrintLog("process exit");
}