void init() { IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper(); IArrayOf<IGroup> groups; OwnedRoxieString fname(helper->getFileName()); fillClusterArray(container.queryJob(), fname, clusters, groups); fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, true, TDWnoreplicate+TDXtemporary)); IPropertyTree &props = fileDesc->queryProperties(); bool blockCompressed=false; void *ekey; size32_t ekeylen; helper->getEncryptKey(ekeylen,ekey); if (ekeylen) { memset(ekey,0,ekeylen); free(ekey); props.setPropBool("@encrypted", true); blockCompressed = true; } else if (0 != (helper->getFlags() & TDWnewcompress) || 0 != (helper->getFlags() & TDXcompress)) blockCompressed = true; if (blockCompressed) props.setPropBool("@blockCompressed", true); if (0 != (helper->getFlags() & TDXgrouped)) fileDesc->queryProperties().setPropBool("@grouped", true); props.setProp("@kind", "flat"); if (container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query { // do early, because this will be local act. and will not come back to master until end of owning graph. container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters); queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc); } }
void init() { helper = (IHThorKeyDiffArg *)queryHelper(); OwnedRoxieString origName(helper->getOriginalName()); OwnedRoxieString updatedName(helper->getUpdatedName()); originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), origName)); newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedName)); if (originalIndexFile->numParts() != newIndexFile->numParts()) throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", origName.get(), updatedName.get()); if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile()) throw MakeActivityException(this, 0, "Diffing super files not supported"); width = originalIndexFile->numParts(); originalDesc.setown(originalIndexFile->getFileDescriptor()); newIndexDesc.setown(newIndexFile->getFileDescriptor()); Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1); const char *kind = tlkDesc->queryProperties().queryProp("@kind"); local = NULL == kind || 0 != stricmp("topLevelKey", kind); if (!local) width--; // 1 part == No n distributed / Monolithic key if (width > container.queryJob().querySlaves()) throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves()); queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile); queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile); IArrayOf<IGroup> groups; OwnedRoxieString outputName(helper->getOutputName()); fillClusterArray(container.queryJob(), outputName, clusters, groups); patchDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width)); }
virtual void init() { OwnedRoxieString fname(helper->getFileName()); dlfn.set(fname); isLocal = 0 != (TIWlocal & helper->getFlags()); unsigned minSize = helper->queryDiskRecordSize()->getMinRecordSize(); if (minSize > KEYBUILD_MAXLENGTH) throw MakeActivityException(this, 0, "Index minimum record length (%d) exceeds %d internal limit", minSize, KEYBUILD_MAXLENGTH); unsigned maxSize = helper->queryDiskRecordSize()->getRecordSize(NULL); if (maxSize > KEYBUILD_MAXLENGTH) throw MakeActivityException(this, 0, "Index maximum record length (%d) exceeds %d internal limit. Minimum size = %d, try setting index MAXLENGTH", maxSize, KEYBUILD_MAXLENGTH, minSize); singlePartKey = 0 != (helper->getFlags() & TIWsmall) || dlfn.isExternal(); clusters.kill(); unsigned idx=0; while (true) { OwnedRoxieString cluster(helper->getCluster(idx)); if(!cluster) break; clusters.append(cluster); idx++; } IArrayOf<IGroup> groups; if (singlePartKey) { isLocal = true; buildTlk = false; } else if (!isLocal || globals->getPropBool("@buildLocalTlks", true)) buildTlk = true; fillClusterArray(container.queryJob(), fname, clusters, groups); unsigned restrictedWidth = 0; if (TIWhaswidth & helper->getFlags()) { restrictedWidth = helper->getWidth(); if (restrictedWidth > container.queryJob().querySlaves()) throw MakeActivityException(this, 0, "Unsupported, can't refactor to width(%d) larger than host cluster(%d)", restrictedWidth, container.queryJob().querySlaves()); else if (restrictedWidth < container.queryJob().querySlaves()) { if (!isLocal) throw MakeActivityException(this, 0, "Unsupported, refactoring to few parts only supported for local indexes."); assertex(!singlePartKey); unsigned gwidth = groups.item(0).ordinality(); if (0 != container.queryJob().querySlaves() % gwidth) throw MakeActivityException(this, 0, "Unsupported, refactored target size (%d) must be factor of thor cluster width (%d)", groups.item(0).ordinality(), container.queryJob().querySlaves()); if (0 == restrictedWidth) restrictedWidth = gwidth; ForEachItemIn(g, groups) { IGroup &group = groups.item(g); if (gwidth != groups.item(g).ordinality()) throw MakeActivityException(this, 0, "Unsupported, cannot output multiple refactored widths, targeting cluster '%s' and '%s'", clusters.item(0), clusters.item(g)); if (gwidth != restrictedWidth) groups.replace(*group.subset((unsigned)0, restrictedWidth), g); } refactor = true; }
void CWriteMasterBase::init() { published = false; recordsProcessed = 0; dlfn.set(diskHelperBase->getFileName()); if (diskHelperBase->getFlags() & TDWextend) { assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp))); Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), diskHelperBase->getFileName(), false, true); if (file.get()) { fileDesc.setown(file->getFileDescriptor()); queryThorFileManager().noteFileRead(container.queryJob(), file, true); } } if (dlfn.isExternal()) mpTag = container.queryJob().allocateMPTag(); // used if (NULL == fileDesc.get()) { bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags()); unsigned idx=0; while (diskHelperBase->queryCluster(idx)) clusters.append(diskHelperBase->queryCluster(idx++)); IArrayOf<IGroup> groups; fillClusterArray(container.queryJob(), diskHelperBase->getFileName(), clusters, groups); fileDesc.setown(queryThorFileManager().create(container.queryJob(), diskHelperBase->getFileName(), clusters, groups, overwriteok, diskHelperBase->getFlags())); if (1 == groups.ordinality()) targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup()); IPropertyTree &props = fileDesc->queryProperties(); if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary)) props.setPropBool("@owned", true); if (diskHelperBase->getFlags() & TDWresult) props.setPropBool("@result", true); const char *rececl= diskHelperBase->queryRecordECL(); if (rececl&&*rececl) props.setProp("ECL", rececl); bool blockCompressed=false; void *ekey; size32_t ekeylen; diskHelperBase->getEncryptKey(ekeylen,ekey); if (ekeylen) { memset(ekey,0,ekeylen); free(ekey); props.setPropBool("@encrypted", true); blockCompressed = true; } else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress)) blockCompressed = true; if (blockCompressed) props.setPropBool("@blockCompressed", true); if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query { // do early, because this will be local act. and will not come back to master until end of owning graph. publish(); } } }
virtual void init() { CMasterActivity::init(); helper = (IHThorKeyPatchArg *)queryHelper(); OwnedRoxieString originalHelperName(helper->getOriginalName()); OwnedRoxieString patchHelperName(helper->getPatchName()); OwnedRoxieString outputHelperName(helper->getOutputName()); StringBuffer expandedFileName; queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false); originalName.set(expandedFileName); queryThorFileManager().addScope(container.queryJob(), patchHelperName, expandedFileName.clear(), false); patchName.set(expandedFileName); queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false); outputName.set(expandedFileName); originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName)); patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName)); if (originalIndexFile->numParts() != patchFile->numParts()) throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get()); if (originalIndexFile->querySuperFile() || patchFile->querySuperFile()) throw MakeActivityException(this, 0, "Patching super files not supported"); addReadFile(originalIndexFile); addReadFile(patchFile); width = originalIndexFile->numParts(); originalDesc.setown(originalIndexFile->getFileDescriptor()); patchDesc.setown(patchFile->getFileDescriptor()); Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1); const char *kind = tlkDesc->queryProperties().queryProp("@kind"); local = NULL == kind || 0 != stricmp("topLevelKey", kind); if (!local && width > 1) width--; // 1 part == No n distributed / Monolithic key if (width > container.queryJob().querySlaves()) throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves()); IArrayOf<IGroup> groups; fillClusterArray(container.queryJob(), outputName, clusters, groups); newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width)); if (!local) newIndexDesc->queryPart(newIndexDesc->numParts()-1)->queryProperties().setProp("@kind", "topLevelKey"); }