virtual void init() { CMasterActivity::init(); OwnedRoxieString fname(helper->getFileName()); Owned<IDistributedFile> fetchFile = queryThorFileManager().lookup(container.queryJob(), fname, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true); if (fetchFile) { if (isFileKey(fetchFile)) throw MakeActivityException(this, 0, "Attempting to read index as a flat file: %s", fname.get()); Owned<IFileDescriptor> fileDesc = getConfiguredFileDescriptor(*fetchFile); void *ekey; size32_t ekeylen; helper->getFileEncryptKey(ekeylen,ekey); bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted"); if (0 != ekeylen) { memset(ekey,0,ekeylen); free(ekey); if (!encrypted) { Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fetchFile->queryLogicalName()); queryJobChannel().fireException(e); } } else if (encrypted) throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFile->queryLogicalName()); mapping.setown(getFileSlaveMaps(fetchFile->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), container.queryLocalOrGrouped(), false, NULL, fetchFile->querySuperFile())); mapping->serializeFileOffsetMap(offsetMapMb); addReadFile(fetchFile); } }
virtual void init() { CSortBaseActivityMaster::init(); IHThorSortArg *helper = (IHThorSortArg *)queryHelper(); IHThorAlgorithm *algo = static_cast<IHThorAlgorithm *>(helper->selectInterface(TAIalgorithm_1)); OwnedRoxieString algoname(algo->getAlgorithm()); unsigned flags = algo->getAlgorithmFlags(); if (algoname && (0 != stricmp(algoname, "quicksort"))) { Owned<IException> e = MakeActivityException(this, 0, "Ignoring, unsupported sort order algorithm '%s'", algoname.get()); reportExceptionToWorkunit(container.queryJob().queryWorkUnit(), e); } OwnedRoxieString cosortlogname(helper->getSortedFilename()); if (cosortlogname&&*cosortlogname) { Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname); addReadFile(coSortFile); Owned<IFileDescriptor> fileDesc = coSortFile->getFileDescriptor(); unsigned o; for (o=0; o<fileDesc->numParts(); o++) { Owned<IPartDescriptor> partDesc = fileDesc->getPart(o); if (cosortfilenames.length()) cosortfilenames.append("|"); // JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly. getPartFilename(*partDesc, 0, cosortfilenames); } } }
virtual void init() { HashDistributeMasterBase::init(); // JCSMORE should common up some with indexread IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper(); StringBuffer scoped; OwnedRoxieString indexFileName(helper->getIndexFileName()); queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped); Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), indexFileName); if (!file) throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str()); if (0 == file->numParts()) throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str()); checkFormatCrc(this, file, helper->getFormatCrc(), true); Owned<IFileDescriptor> fileDesc = file->getFileDescriptor(); Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1); if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind"))) throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str()); unsigned location; OwnedIFile iFile; StringBuffer filePath; if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath)) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName()); OwnedIFileIO iFileIO = iFile->open(IFOread); assertex(iFileIO); tlkMb.append(iFileIO->size()); ::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb); addReadFile(file); }
virtual void init() { CMasterActivity::init(); helper = (IHThorKeyPatchArg *)queryHelper(); OwnedRoxieString originalHelperName(helper->getOriginalName()); OwnedRoxieString patchHelperName(helper->getPatchName()); OwnedRoxieString outputHelperName(helper->getOutputName()); StringBuffer expandedFileName; queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false); originalName.set(expandedFileName); queryThorFileManager().addScope(container.queryJob(), patchHelperName, expandedFileName.clear(), false); patchName.set(expandedFileName); queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false); outputName.set(expandedFileName); originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName)); patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName)); if (originalIndexFile->numParts() != patchFile->numParts()) throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get()); if (originalIndexFile->querySuperFile() || patchFile->querySuperFile()) throw MakeActivityException(this, 0, "Patching super files not supported"); addReadFile(originalIndexFile); addReadFile(patchFile); width = originalIndexFile->numParts(); originalDesc.setown(originalIndexFile->getFileDescriptor()); patchDesc.setown(patchFile->getFileDescriptor()); Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1); const char *kind = tlkDesc->queryProperties().queryProp("@kind"); local = NULL == kind || 0 != stricmp("topLevelKey", kind); if (!local && width > 1) width--; // 1 part == No n distributed / Monolithic key if (width > container.queryJob().querySlaves()) throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves()); IArrayOf<IGroup> groups; fillClusterArray(container.queryJob(), outputName, clusters, groups); newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width)); if (!local) newIndexDesc->queryPart(newIndexDesc->numParts()-1)->queryProperties().setProp("@kind", "topLevelKey"); }
void CDiskReadMasterBase::init() { CMasterActivity::init(); IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper(); bool mangle = 0 != (helper->getFlags() & (TDXtemporary|TDXjobtemp)); OwnedRoxieString helperFileName = helper->getFileName(); StringBuffer expandedFileName; queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName, mangle); fileName.set(expandedFileName); Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true); if (file) { if (file->isExternal() && (helper->getFlags() & TDXcompress)) file->queryAttributes().setPropBool("@blockCompressed", true); if (file->numParts() > 1) fileDesc.setown(getConfiguredFileDescriptor(*file)); else fileDesc.setown(file->getFileDescriptor()); reInit = 0 != (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)); if (container.queryLocal() || helper->canMatchAny()) // if local, assume may match { bool temp = 0 != (TDXtemporary & helper->getFlags()); bool local; if (temp) local = false; else local = container.queryLocal(); mapping.setown(getFileSlaveMaps(file->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), local, false, hash, file->querySuperFile())); addReadFile(file, temp); } if (0 != (helper->getFlags() & TDRfilenamecallback)) // only get/serialize if using virtual file name fields { IDistributedSuperFile *super = file->querySuperFile(); if (super) { unsigned numsubs = super->numSubFiles(true); unsigned s=0; for (; s<numsubs; s++) { IDistributedFile &subfile = super->querySubFile(s, true); subfileLogicalFilenames.append(subfile.queryLogicalName()); } } } validateFile(file); void *ekey; size32_t ekeylen; helper->getEncryptKey(ekeylen,ekey); bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted"); if (0 != ekeylen) { memset(ekey,0,ekeylen); free(ekey); if (!encrypted) { Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fileName.get()); queryJobChannel().fireException(e); } } else if (encrypted) throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fileName.get()); } }