void DataSourceMetaData::extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray & types)
{
unsigned curOffset = 0;
ForEachItemIn(i, fields)
{
if (curOffset >= keyedSize)
break;
DataSourceMetaItem & cur = fields.item(i);
switch (cur.flags)
{
case FVFFnone:
{
offsets.append(curOffset);
types.append(*LINK(cur.type));
unsigned size = cur.type->getSize();
assertex(size != UNKNOWN_LENGTH);
curOffset += size;
break;
}
case FVFFbeginrecord:
case FVFFendrecord:
break;
default:
throwUnexpected();
}
}
offsets.append(curOffset);
assertex(curOffset == keyedSize);
}
void init()
{
// prepare topology.
unsigned rootNodes = container.queryJob().querySlaves();
unsigned res = MERGE_GRANULARITY;
sD = new MemoryBuffer[rootNodes];
UnsignedArray nodes1, nodes2;
UnsignedArray *currentLevel = &nodes1, *nextLevel = &nodes2;
unsigned n = 0; while (n<rootNodes) currentLevel->append(n++);
while (rootNodes > 1)
{
assertex(rootNodes);
unsigned r = (rootNodes+(res-1))/res; // groups
unsigned t = 0;
n = 0;
bool first = true;
loop
{
if (first)
{
first = false;
nextLevel->append(currentLevel->item(n));
}
else
{
#ifdef _DEBUG
unsigned node = nextLevel->tos();
unsigned item = currentLevel->item(n);
ActPrintLog("Adding to node=%d, item=%d", node, item);
#endif
sD[nextLevel->tos()].append(currentLevel->item(n));
}
n++;
if (n>=rootNodes) break;
t += r;
if (t>=rootNodes)
{
t -= rootNodes;
first = true;
}
}
assertex(sD[nextLevel->tos()].length()); // something must have been added
n = 0;
while (n<nextLevel->ordinality()) sD[nextLevel->item(n++)].append(0); // terminator
#ifdef _DEBUG
ActPrintLog("EOL");
#endif
rootNodes = nextLevel->ordinality();
UnsignedArray *tmp = currentLevel;
currentLevel = nextLevel;
nextLevel = tmp;
nextLevel->kill();
}
}
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra)
{
if (local)
i = 0;
if (i >= maps.ordinality())
{
mb.append((unsigned)0);
return;
}
CSlaveMap &map = maps.item(i);
unsigned nPos = mb.length();
unsigned n=0;
mb.append(n);
UnsignedArray parts;
ForEachItemIn(m, map)
parts.append(map.item(m).queryPartIndex());
MemoryBuffer extraMb;
if (extra)
{
ForEachItemIn(m2, map)
{
unsigned xtraLen = 0;
unsigned xtraPos = extraMb.length();
extraMb.append(xtraLen);
IPartDescriptor &partDesc = map.item(m2);
if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb))
{
parts.zap(partDesc.queryPartIndex());
extraMb.rewrite(xtraPos);
}
else
{
xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen);
extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen);
}
}
}
bool Cws_machineEx::doStartStop(IEspContext &context, StringArray& addresses, char* userName, char* password, bool bStop,
IEspStartStopResponse &resp)
{
bool containCluster = false;
double version = context.getClientVersion();
const int ordinality= addresses.ordinality();
UnsignedArray threadHandles;
IArrayOf<IEspStartStopResult> resultsArray;
for (int index=0; index<ordinality; index++)
{
const char* address0 = addresses.item(index);
//address passed in is of the form "192.168.1.4:EspProcess:2:path1"
StringArray sArray;
sArray.appendList(addresses.item(index), ":");
if (sArray.ordinality() < 4)
throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments");
Owned<IEspStartStopResult> pResult = static_cast<IEspStartStopResult*>(new CStartStopResult(""));
const char* address = sArray.item(0);
const char* compType= sArray.item(1);
const char* OS = sArray.item(3);//index 2 is component name
const char* path = sArray.item(4);
if (!(address && *address && compType && *compType && OS && *OS && path && *path))
throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Invalid input");
if (!stricmp(compType, "ThorCluster") || !stricmp(compType, "RoxieCluster"))
containCluster = true;
#ifndef OLD_START_STOP
{
char* configAddress = NULL;
char* props1 = (char*) strchr(address, '|');
if (props1)
{
configAddress = props1+1;
*props1 = '\0';
}
else
{
configAddress = (char*) address;
}
StringBuffer newAddress;
ConvertAddress(address0, newAddress);
pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
pResult->setAddress ( address );//can be either IP or name of component
pResult->setCompType( compType );
if (version > 1.04)
{
pResult->setName( path );
const char* pStr2 = strstr(path, "LexisNexis");
if (pStr2)
{
char name[256];
const char* pStr1 = strchr(pStr2, '|');
if (!pStr1)
{
strcpy(name, pStr2+11);
}
else
{
strncpy(name, pStr2+11, pStr1 - pStr2 -11);
name[pStr1 - pStr2 -11] = 0;
}
pResult->setName( name );
}
}
pResult->setOS( atoi(OS) );
pResult->setPath( path );
resultsArray.append(*pResult.getLink());
CStartStopThreadParam* pThreadReq;
pThreadReq = new CStartStopThreadParam(address, configAddress, bStop, m_useDefaultHPCCInit, this, context);
pThreadReq->setResultObject( pResult );
if (userName && *userName)
pThreadReq->setUserID( userName );
if (password && *password)
pThreadReq->setPassword( password );
PooledThreadHandle handle = m_threadPool->start( pThreadReq );
threadHandles.append(handle);
}
#else
{
StringBuffer newAddress;
ConvertAddress(address0, newAddress);
char* pStr = (char*) strchr(address, '|');;
if (pStr)
pStr[0] = 0;
pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
pResult->setAddress ( address );//can be either IP or name of component
pResult->setCompType( compType );
pResult->setOS( atoi(OS) );
pResult->setPath( path );
resultsArray.append(*pResult.getLink());
CStartStopThreadParam* pThreadReq;
pThreadReq = new CStartStopThreadParam(address, bStop, this, context);
pThreadReq->setResultObject( pResult );
if (userName && *userName)
pThreadReq->setUserID( userName );
if (password && *password)
pThreadReq->setPassword( password );
PooledThreadHandle handle = m_threadPool->start( pThreadReq );
threadHandles.append(handle);
}
#endif
}
//block for worker theads to finish, if necessary, and then collect results
//
PooledThreadHandle* pThreadHandle = threadHandles.getArray();
unsigned i=threadHandles.ordinality();
while (i--)
{
m_threadPool->join(*pThreadHandle, 30000);//abort after 30 secs in remote possibility that the command blocks
pThreadHandle++;
}
resp.setStartStopResults(resultsArray);
resp.setStop(bStop);
if (version > 1.08)
{
resp.setContainCluster(containCluster);
}
return true;
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString indexFileName(helper->getIndexFileName());
Owned<IDistributedFile> dataFile;
Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
keyReadWidth = container.queryJob().querySlaves();
initMb.clear();
initMb.append(indexFileName.get());
if (helper->diskAccessRequired())
numTags += 2;
initMb.append(numTags);
unsigned t=0;
for (; t<numTags; t++)
{
tags[t] = container.queryJob().allocateMPTag();
initMb.append(tags[t]);
}
bool keyHasTlk = false;
if (indexFile)
{
unsigned numParts = 0;
localKey = indexFile->queryAttributes().getPropBool("@local");
if (container.queryLocalData() && !localKey)
throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
IDistributedSuperFile *superIndex = indexFile->querySuperFile();
unsigned superIndexWidth = 0;
unsigned numSuperIndexSubs = 0;
if (superIndex)
{
numSuperIndexSubs = superIndex->numSubFiles(true);
bool first=true;
// consistency check
Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
ForEach(*iter)
{
IDistributedFile &f = iter->query();
unsigned np = f.numParts()-1;
IDistributedFilePart &part = f.queryPart(np);
const char *kind = part.queryAttributes().queryProp("@kind");
bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
if (first)
{
first = false;
keyHasTlk = hasTlk;
superIndexWidth = f.numParts();
if (keyHasTlk)
--superIndexWidth;
}
else
{
if (hasTlk != keyHasTlk)
throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
if (keyHasTlk && superIndexWidth != f.numParts()-1)
throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
}
}
if (keyHasTlk)
numParts = superIndexWidth * numSuperIndexSubs;
else
numParts = superIndex->numParts();
}
else
{
numParts = indexFile->numParts();
if (numParts)
{
const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
if (keyHasTlk)
--numParts;
}
}
if (numParts)
{
initMb.append(numParts);
initMb.append(superIndexWidth); // 0 if not superIndex
initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
unsigned p=0;
UnsignedArray parts;
for (; p<numParts; p++)
parts.append(p);
indexFileDesc->serializeParts(initMb, parts);
if (localKey)
keyHasTlk = false; // not used
initMb.append(keyHasTlk);
if (keyHasTlk)
{
if (numSuperIndexSubs)
initMb.append(numSuperIndexSubs);
else
initMb.append((unsigned)1);
Owned<IDistributedFileIterator> iter;
IDistributedFile *f;
if (superIndex)
{
iter.setown(superIndex->getSubFileIterator(true));
f = &iter->query();
}
else
f = indexFile;
loop
{
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
size32_t tlkSz = (size32_t)iFileIO->size();
initMb.append(tlkSz);
::read(iFileIO, 0, tlkSz, initMb);
if (!iter || !iter->next())
break;
f = &iter->query();
}
}
if (helper->diskAccessRequired())
{
OwnedRoxieString fetchFilename(helper->getFileName());
if (fetchFilename)
{
dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
if (dataFile)
{
if (superIndex)
throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
dataReadWidth = container.queryJob().querySlaves();
Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
}
else
indexFile.clear();
}
}
}
else