virtual void init()
{
HashDistributeMasterBase::init();
// JCSMORE should common up some with indexread
IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();
StringBuffer scoped;
OwnedRoxieString indexFileName(helper->getIndexFileName());
queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped);
file.setown(queryThorFileManager().lookup(container.queryJob(), indexFileName));
if (!file)
throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
if (0 == file->numParts())
throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());
checkFormatCrc(this, file, helper->getFormatCrc(), true);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
tlkMb.append(iFileIO->size());
::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);
queryThorFileManager().noteFileRead(container.queryJob(), file);
}
extern bool isCompressedIndex(const char *filename)
{
OwnedIFile file = createIFile(filename);
OwnedIFileIO io = file->open(IFOread);
unsigned __int64 size = file->size();
if (size)
{
KeyHdr hdr;
if (io->read(0, sizeof(hdr), &hdr) == sizeof(hdr))
{
SwapBigEndian(hdr);
if (size % hdr.nodeSize == 0 && hdr.phyrec == size-1 && hdr.root && hdr.root % hdr.nodeSize == 0 && hdr.ktype & (HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY))
{
NodeHdr root;
if (io->read(hdr.root, sizeof(root), &root) == sizeof(root))
{
SwapBigEndian(root);
return root.leftSib==0 && root.rightSib==0;
}
}
}
}
return false;
}
bool loadPwds()
{
try
{
if (!pwFile.length())
throw MakeStringException(-1, "htpasswd Password file not specified");
Owned<IFile> file = createIFile(pwFile.str());
if (!file->exists())
{
userMap.kill();
throw MakeStringException(-1, "htpasswd Password file does not exist");
}
bool isDir;
offset_t size;
CDateTime whenChanged;
file->getInfo(isDir,size,whenChanged);
if (isDir)
{
userMap.kill();
throw MakeStringException(-1, "htpasswd Password file specifies a directory");
}
if (0 == whenChanged.compare(pwFileLastMod))
return true;//Don't reload if file unchanged
userMap.kill();
OwnedIFileIO io = file->open(IFOread);
if (!io)
throw MakeStringException(-1, "htpasswd Unable to open Password file");
MemoryBuffer mb;
size32_t count = read(io, 0, (size32_t)-1, mb);
if (0 == count)
throw MakeStringException(-1, "htpasswd Password file is empty");
mb.append((char)NULL);
char * p = (char*)mb.toByteArray();
char *saveptr;
const char * seps = "\f\r\n";
char * next = strtok_r(p, seps, &saveptr);
if (next)
{
do
{
char * colon = strchr(next,':');
if (NULL == colon)
throw MakeStringException(-1, "htpasswd Password file appears malformed");
*colon = (char)NULL;
userMap.setValue(next, colon+1);//username, enctypted password
} while (next = strtok_r(NULL, seps, &saveptr));
}
io->close();
pwFileLastMod = whenChanged;//remember when last changed
}
catch(IException*)
{
throw MakeStringException(-1, "htpasswd Exception accessing Password file");
}
return true;
}
extern jhtree_decl void validateKeyFile(const char *filename, offset_t nodePos)
{
OwnedIFile file = createIFile(filename);
OwnedIFileIO io = file->open(IFOread);
if (!io)
throw MakeStringException(1, "Invalid key %s: cannot open file", filename);
unsigned __int64 size = file->size();
if (!size)
throw MakeStringException(2, "Invalid key %s: zero size", filename);
KeyHdr hdr;
if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr))
throw MakeStringException(4, "Invalid key %s: failed to read key header", filename);
CKeyHdr keyHdr;
keyHdr.load(hdr);
_WINREV(hdr.phyrec);
_WINREV(hdr.root);
_WINREV(hdr.nodeSize);
if (hdr.phyrec != size-1)
throw MakeStringException(5, "Invalid key %s: phyrec was %" I64F "d, expected %" I64F "d", filename, hdr.phyrec, size-1);
if (size % hdr.nodeSize)
throw MakeStringException(3, "Invalid key %s: size %" I64F "d is not a multiple of key node size (%d)", filename, size, hdr.nodeSize);
if (!hdr.root || hdr.root % hdr.nodeSize !=0)
throw MakeStringException(6, "Invalid key %s: invalid root pointer %" I64F "x", filename, hdr.root);
NodeHdr root;
if (io->read(hdr.root, sizeof(root), &root) != sizeof(root))
throw MakeStringException(7, "Invalid key %s: failed to read root node", filename);
_WINREV(root.rightSib);
_WINREV(root.leftSib);
if (root.leftSib || root.rightSib)
throw MakeStringException(8, "Invalid key %s: invalid root node sibling pointers 0x%" I64F "x, 0x%" I64F "x (expected 0,0)", filename, root.leftSib, root.rightSib);
for (offset_t nodeOffset = (nodePos ? nodePos : hdr.nodeSize); nodeOffset < (nodePos ? nodePos+1 : size); nodeOffset += hdr.nodeSize)
{
MemoryAttr ma;
char *buffer = (char *) ma.allocate(hdr.nodeSize);
{
MTIME_SECTION(queryActiveTimer(), "JHTREE read index node");
io->read(nodeOffset, hdr.nodeSize, buffer);
}
CJHTreeNode theNode;
{
MTIME_SECTION(queryActiveTimer(), "JHTREE load index node");
theNode.load(&keyHdr, buffer, nodeOffset, true);
}
NodeHdr *nodeHdr = (NodeHdr *) buffer;
SwapBigEndian(*nodeHdr);
if (!nodeHdr->isValid(hdr.nodeSize))
throw MakeStringException(9, "Invalid key %s: invalid node header at position 0x%" I64F "x", filename, nodeOffset);
if (nodeHdr->leftSib >= size || nodeHdr->rightSib >= size)
throw MakeStringException(9, "Invalid key %s: out of range sibling pointers 0x%" I64F "x, 0x%" I64F "x at position 0x%" I64F "x", filename, nodeHdr->leftSib, nodeHdr->rightSib, nodeOffset);
if (nodeHdr->crc32)
{
unsigned crc = crc32(buffer + sizeof(NodeHdr), nodeHdr->keyBytes, 0);
if (crc != nodeHdr->crc32)
throw MakeStringException(9, "Invalid key %s: crc mismatch at position 0x%" I64F "x", filename, nodeOffset);
}
else
{
// MORE - if we felt so inclined, we could decode the node and check records were in ascending order
}
}
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString indexFileName(helper->getIndexFileName());
Owned<IDistributedFile> dataFile;
Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
keyReadWidth = container.queryJob().querySlaves();
initMb.clear();
initMb.append(indexFileName.get());
if (helper->diskAccessRequired())
numTags += 2;
initMb.append(numTags);
unsigned t=0;
for (; t<numTags; t++)
{
tags[t] = container.queryJob().allocateMPTag();
initMb.append(tags[t]);
}
bool keyHasTlk = false;
if (indexFile)
{
unsigned numParts = 0;
localKey = indexFile->queryAttributes().getPropBool("@local");
if (container.queryLocalData() && !localKey)
throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
IDistributedSuperFile *superIndex = indexFile->querySuperFile();
unsigned superIndexWidth = 0;
unsigned numSuperIndexSubs = 0;
if (superIndex)
{
numSuperIndexSubs = superIndex->numSubFiles(true);
bool first=true;
// consistency check
Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
ForEach(*iter)
{
IDistributedFile &f = iter->query();
unsigned np = f.numParts()-1;
IDistributedFilePart &part = f.queryPart(np);
const char *kind = part.queryAttributes().queryProp("@kind");
bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
if (first)
{
first = false;
keyHasTlk = hasTlk;
superIndexWidth = f.numParts();
if (keyHasTlk)
--superIndexWidth;
}
else
{
if (hasTlk != keyHasTlk)
throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
if (keyHasTlk && superIndexWidth != f.numParts()-1)
throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
}
}
if (keyHasTlk)
numParts = superIndexWidth * numSuperIndexSubs;
else
numParts = superIndex->numParts();
}
else
{
numParts = indexFile->numParts();
if (numParts)
{
const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
if (keyHasTlk)
--numParts;
}
}
if (numParts)
{
initMb.append(numParts);
initMb.append(superIndexWidth); // 0 if not superIndex
initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
unsigned p=0;
UnsignedArray parts;
for (; p<numParts; p++)
parts.append(p);
indexFileDesc->serializeParts(initMb, parts);
if (localKey)
keyHasTlk = false; // not used
initMb.append(keyHasTlk);
if (keyHasTlk)
{
if (numSuperIndexSubs)
initMb.append(numSuperIndexSubs);
else
initMb.append((unsigned)1);
Owned<IDistributedFileIterator> iter;
IDistributedFile *f;
if (superIndex)
{
iter.setown(superIndex->getSubFileIterator(true));
f = &iter->query();
}
else
f = indexFile;
loop
{
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
size32_t tlkSz = (size32_t)iFileIO->size();
initMb.append(tlkSz);
::read(iFileIO, 0, tlkSz, initMb);
if (!iter || !iter->next())
break;
f = &iter->query();
}
}
if (helper->diskAccessRequired())
{
OwnedRoxieString fetchFilename(helper->getFileName());
if (fetchFilename)
{
dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
if (dataFile)
{
if (superIndex)
throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
dataReadWidth = container.queryJob().querySlaves();
Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
}
else
indexFile.clear();
}
}
}
else