void init()
{
IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
IArrayOf<IGroup> groups;
OwnedRoxieString fname(helper->getFileName());
fillClusterArray(container.queryJob(), fname, clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, true, TDWnoreplicate+TDXtemporary));
IPropertyTree &props = fileDesc->queryProperties();
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (helper->getFlags() & TDWnewcompress) || 0 != (helper->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (0 != (helper->getFlags() & TDXgrouped))
fileDesc->queryProperties().setPropBool("@grouped", true);
props.setProp("@kind", "flat");
if (container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
}
}
void init()
{
helper = (IHThorKeyDiffArg *)queryHelper();
OwnedRoxieString origName(helper->getOriginalName());
OwnedRoxieString updatedName(helper->getUpdatedName());
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), origName));
newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedName));
if (originalIndexFile->numParts() != newIndexFile->numParts())
throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", origName.get(), updatedName.get());
if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
throw MakeActivityException(this, 0, "Diffing super files not supported");
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
newIndexDesc.setown(newIndexFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());
queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile);
queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile);
IArrayOf<IGroup> groups;
OwnedRoxieString outputName(helper->getOutputName());
fillClusterArray(container.queryJob(), outputName, clusters, groups);
patchDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
}
virtual void init()
{
OwnedRoxieString fname(helper->getFileName());
dlfn.set(fname);
isLocal = 0 != (TIWlocal & helper->getFlags());
unsigned minSize = helper->queryDiskRecordSize()->getMinRecordSize();
if (minSize > KEYBUILD_MAXLENGTH)
throw MakeActivityException(this, 0, "Index minimum record length (%d) exceeds %d internal limit", minSize, KEYBUILD_MAXLENGTH);
unsigned maxSize = helper->queryDiskRecordSize()->getRecordSize(NULL);
if (maxSize > KEYBUILD_MAXLENGTH)
throw MakeActivityException(this, 0, "Index maximum record length (%d) exceeds %d internal limit. Minimum size = %d, try setting index MAXLENGTH", maxSize, KEYBUILD_MAXLENGTH, minSize);
singlePartKey = 0 != (helper->getFlags() & TIWsmall) || dlfn.isExternal();
clusters.kill();
unsigned idx=0;
while (true)
{
OwnedRoxieString cluster(helper->getCluster(idx));
if(!cluster)
break;
clusters.append(cluster);
idx++;
}
IArrayOf<IGroup> groups;
if (singlePartKey)
{
isLocal = true;
buildTlk = false;
}
else if (!isLocal || globals->getPropBool("@buildLocalTlks", true))
buildTlk = true;
fillClusterArray(container.queryJob(), fname, clusters, groups);
unsigned restrictedWidth = 0;
if (TIWhaswidth & helper->getFlags())
{
restrictedWidth = helper->getWidth();
if (restrictedWidth > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported, can't refactor to width(%d) larger than host cluster(%d)", restrictedWidth, container.queryJob().querySlaves());
else if (restrictedWidth < container.queryJob().querySlaves())
{
if (!isLocal)
throw MakeActivityException(this, 0, "Unsupported, refactoring to few parts only supported for local indexes.");
assertex(!singlePartKey);
unsigned gwidth = groups.item(0).ordinality();
if (0 != container.queryJob().querySlaves() % gwidth)
throw MakeActivityException(this, 0, "Unsupported, refactored target size (%d) must be factor of thor cluster width (%d)", groups.item(0).ordinality(), container.queryJob().querySlaves());
if (0 == restrictedWidth)
restrictedWidth = gwidth;
ForEachItemIn(g, groups)
{
IGroup &group = groups.item(g);
if (gwidth != groups.item(g).ordinality())
throw MakeActivityException(this, 0, "Unsupported, cannot output multiple refactored widths, targeting cluster '%s' and '%s'", clusters.item(0), clusters.item(g));
if (gwidth != restrictedWidth)
groups.replace(*group.subset((unsigned)0, restrictedWidth), g);
}
refactor = true;
}
void CWriteMasterBase::init()
{
published = false;
recordsProcessed = 0;
dlfn.set(diskHelperBase->getFileName());
if (diskHelperBase->getFlags() & TDWextend)
{
assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), diskHelperBase->getFileName(), false, true);
if (file.get())
{
fileDesc.setown(file->getFileDescriptor());
queryThorFileManager().noteFileRead(container.queryJob(), file, true);
}
}
if (dlfn.isExternal())
mpTag = container.queryJob().allocateMPTag(); // used
if (NULL == fileDesc.get())
{
bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags());
unsigned idx=0;
while (diskHelperBase->queryCluster(idx))
clusters.append(diskHelperBase->queryCluster(idx++));
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), diskHelperBase->getFileName(), clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), diskHelperBase->getFileName(), clusters, groups, overwriteok, diskHelperBase->getFlags()));
if (1 == groups.ordinality())
targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());
IPropertyTree &props = fileDesc->queryProperties();
if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
props.setPropBool("@owned", true);
if (diskHelperBase->getFlags() & TDWresult)
props.setPropBool("@result", true);
const char *rececl= diskHelperBase->queryRecordECL();
if (rececl&&*rececl)
props.setProp("ECL", rececl);
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
diskHelperBase->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
publish();
}
}
}
virtual void init()
{
CMasterActivity::init();
helper = (IHThorKeyPatchArg *)queryHelper();
OwnedRoxieString originalHelperName(helper->getOriginalName());
OwnedRoxieString patchHelperName(helper->getPatchName());
OwnedRoxieString outputHelperName(helper->getOutputName());
StringBuffer expandedFileName;
queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false);
originalName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), patchHelperName, expandedFileName.clear(), false);
patchName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
outputName.set(expandedFileName);
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName));
if (originalIndexFile->numParts() != patchFile->numParts())
throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get());
if (originalIndexFile->querySuperFile() || patchFile->querySuperFile())
throw MakeActivityException(this, 0, "Patching super files not supported");
addReadFile(originalIndexFile);
addReadFile(patchFile);
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
patchDesc.setown(patchFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local && width > 1)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves());
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), outputName, clusters, groups);
newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
if (!local)
newIndexDesc->queryPart(newIndexDesc->numParts()-1)->queryProperties().setProp("@kind", "topLevelKey");
}
