virtual void init()
{
CMasterActivity::init();
OwnedRoxieString fname(helper->getFileName());
Owned<IDistributedFile> fetchFile = queryThorFileManager().lookup(container.queryJob(), fname, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true);
if (fetchFile)
{
if (isFileKey(fetchFile))
throw MakeActivityException(this, 0, "Attempting to read index as a flat file: %s", fname.get());
Owned<IFileDescriptor> fileDesc = getConfiguredFileDescriptor(*fetchFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fetchFile->queryLogicalName());
queryJobChannel().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFile->queryLogicalName());
mapping.setown(getFileSlaveMaps(fetchFile->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), container.queryLocalOrGrouped(), false, NULL, fetchFile->querySuperFile()));
mapping->serializeFileOffsetMap(offsetMapMb);
addReadFile(fetchFile);
}
}
virtual void init()
{
CSortBaseActivityMaster::init();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
IHThorAlgorithm *algo = static_cast<IHThorAlgorithm *>(helper->selectInterface(TAIalgorithm_1));
OwnedRoxieString algoname(algo->getAlgorithm());
unsigned flags = algo->getAlgorithmFlags();
if (algoname && (0 != stricmp(algoname, "quicksort")))
{
Owned<IException> e = MakeActivityException(this, 0, "Ignoring, unsupported sort order algorithm '%s'", algoname.get());
reportExceptionToWorkunit(container.queryJob().queryWorkUnit(), e);
}
OwnedRoxieString cosortlogname(helper->getSortedFilename());
if (cosortlogname&&*cosortlogname)
{
Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
addReadFile(coSortFile);
Owned<IFileDescriptor> fileDesc = coSortFile->getFileDescriptor();
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
}
virtual void init()
{
HashDistributeMasterBase::init();
// JCSMORE should common up some with indexread
IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();
StringBuffer scoped;
OwnedRoxieString indexFileName(helper->getIndexFileName());
queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped);
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), indexFileName);
if (!file)
throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
if (0 == file->numParts())
throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());
checkFormatCrc(this, file, helper->getFormatCrc(), true);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
tlkMb.append(iFileIO->size());
::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);
addReadFile(file);
}
virtual void init()
{
CMasterActivity::init();
helper = (IHThorKeyPatchArg *)queryHelper();
OwnedRoxieString originalHelperName(helper->getOriginalName());
OwnedRoxieString patchHelperName(helper->getPatchName());
OwnedRoxieString outputHelperName(helper->getOutputName());
StringBuffer expandedFileName;
queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false);
originalName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), patchHelperName, expandedFileName.clear(), false);
patchName.set(expandedFileName);
queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
outputName.set(expandedFileName);
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName));
if (originalIndexFile->numParts() != patchFile->numParts())
throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get());
if (originalIndexFile->querySuperFile() || patchFile->querySuperFile())
throw MakeActivityException(this, 0, "Patching super files not supported");
addReadFile(originalIndexFile);
addReadFile(patchFile);
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
patchDesc.setown(patchFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local && width > 1)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves());
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), outputName, clusters, groups);
newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
if (!local)
newIndexDesc->queryPart(newIndexDesc->numParts()-1)->queryProperties().setProp("@kind", "topLevelKey");
}
void CDiskReadMasterBase::init()
{
CMasterActivity::init();
IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
bool mangle = 0 != (helper->getFlags() & (TDXtemporary|TDXjobtemp));
OwnedRoxieString helperFileName = helper->getFileName();
StringBuffer expandedFileName;
queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName, mangle);
fileName.set(expandedFileName);
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true);
if (file)
{
if (file->isExternal() && (helper->getFlags() & TDXcompress))
file->queryAttributes().setPropBool("@blockCompressed", true);
if (file->numParts() > 1)
fileDesc.setown(getConfiguredFileDescriptor(*file));
else
fileDesc.setown(file->getFileDescriptor());
reInit = 0 != (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename));
if (container.queryLocal() || helper->canMatchAny()) // if local, assume may match
{
bool temp = 0 != (TDXtemporary & helper->getFlags());
bool local;
if (temp)
local = false;
else
local = container.queryLocal();
mapping.setown(getFileSlaveMaps(file->queryLogicalName(), *fileDesc, container.queryJob().queryUserDescriptor(), container.queryJob().querySlaveGroup(), local, false, hash, file->querySuperFile()));
addReadFile(file, temp);
}
if (0 != (helper->getFlags() & TDRfilenamecallback)) // only get/serialize if using virtual file name fields
{
IDistributedSuperFile *super = file->querySuperFile();
if (super)
{
unsigned numsubs = super->numSubFiles(true);
unsigned s=0;
for (; s<numsubs; s++)
{
IDistributedFile &subfile = super->querySubFile(s, true);
subfileLogicalFilenames.append(subfile.queryLogicalName());
}
}
}
validateFile(file);
void *ekey;
size32_t ekeylen;
helper->getEncryptKey(ekeylen,ekey);
bool encrypted = fileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", fileName.get());
queryJobChannel().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fileName.get());
}
}
