void DataSourceMetaData::extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray & types) { unsigned curOffset = 0; ForEachItemIn(i, fields) { if (curOffset >= keyedSize) break; DataSourceMetaItem & cur = fields.item(i); switch (cur.flags) { case FVFFnone: { offsets.append(curOffset); types.append(*LINK(cur.type)); unsigned size = cur.type->getSize(); assertex(size != UNKNOWN_LENGTH); curOffset += size; break; } case FVFFbeginrecord: case FVFFendrecord: break; default: throwUnexpected(); } } offsets.append(curOffset); assertex(curOffset == keyedSize); }
void init() { // prepare topology. unsigned rootNodes = container.queryJob().querySlaves(); unsigned res = MERGE_GRANULARITY; sD = new MemoryBuffer[rootNodes]; UnsignedArray nodes1, nodes2; UnsignedArray *currentLevel = &nodes1, *nextLevel = &nodes2; unsigned n = 0; while (n<rootNodes) currentLevel->append(n++); while (rootNodes > 1) { assertex(rootNodes); unsigned r = (rootNodes+(res-1))/res; // groups unsigned t = 0; n = 0; bool first = true; loop { if (first) { first = false; nextLevel->append(currentLevel->item(n)); } else { #ifdef _DEBUG unsigned node = nextLevel->tos(); unsigned item = currentLevel->item(n); ActPrintLog("Adding to node=%d, item=%d", node, item); #endif sD[nextLevel->tos()].append(currentLevel->item(n)); } n++; if (n>=rootNodes) break; t += r; if (t>=rootNodes) { t -= rootNodes; first = true; } } assertex(sD[nextLevel->tos()].length()); // something must have been added n = 0; while (n<nextLevel->ordinality()) sD[nextLevel->item(n++)].append(0); // terminator #ifdef _DEBUG ActPrintLog("EOL"); #endif rootNodes = nextLevel->ordinality(); UnsignedArray *tmp = currentLevel; currentLevel = nextLevel; nextLevel = tmp; nextLevel->kill(); } }
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra) { if (local) i = 0; if (i >= maps.ordinality()) { mb.append((unsigned)0); return; } CSlaveMap &map = maps.item(i); unsigned nPos = mb.length(); unsigned n=0; mb.append(n); UnsignedArray parts; ForEachItemIn(m, map) parts.append(map.item(m).queryPartIndex()); MemoryBuffer extraMb; if (extra) { ForEachItemIn(m2, map) { unsigned xtraLen = 0; unsigned xtraPos = extraMb.length(); extraMb.append(xtraLen); IPartDescriptor &partDesc = map.item(m2); if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb)) { parts.zap(partDesc.queryPartIndex()); extraMb.rewrite(xtraPos); } else { xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen); extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen); } } }
bool Cws_machineEx::doStartStop(IEspContext &context, StringArray& addresses, char* userName, char* password, bool bStop, IEspStartStopResponse &resp) { bool containCluster = false; double version = context.getClientVersion(); const int ordinality= addresses.ordinality(); UnsignedArray threadHandles; IArrayOf<IEspStartStopResult> resultsArray; for (int index=0; index<ordinality; index++) { const char* address0 = addresses.item(index); //address passed in is of the form "192.168.1.4:EspProcess:2:path1" StringArray sArray; sArray.appendList(addresses.item(index), ":"); if (sArray.ordinality() < 4) throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments"); Owned<IEspStartStopResult> pResult = static_cast<IEspStartStopResult*>(new CStartStopResult("")); const char* address = sArray.item(0); const char* compType= sArray.item(1); const char* OS = sArray.item(3);//index 2 is component name const char* path = sArray.item(4); if (!(address && *address && compType && *compType && OS && *OS && path && *path)) throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Invalid input"); if (!stricmp(compType, "ThorCluster") || !stricmp(compType, "RoxieCluster")) containCluster = true; #ifndef OLD_START_STOP { char* configAddress = NULL; char* props1 = (char*) strchr(address, '|'); if (props1) { configAddress = props1+1; *props1 = '\0'; } else { configAddress = (char*) address; } StringBuffer newAddress; ConvertAddress(address0, newAddress); pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component pResult->setAddress ( address );//can be either IP or name of component pResult->setCompType( compType ); if (version > 1.04) { pResult->setName( path ); const char* pStr2 = strstr(path, "LexisNexis"); if (pStr2) { char name[256]; const char* pStr1 = strchr(pStr2, '|'); if (!pStr1) { strcpy(name, pStr2+11); } else { strncpy(name, pStr2+11, pStr1 - pStr2 -11); name[pStr1 - pStr2 -11] = 0; } pResult->setName( name ); } } pResult->setOS( atoi(OS) ); pResult->setPath( path ); resultsArray.append(*pResult.getLink()); CStartStopThreadParam* pThreadReq; pThreadReq = new CStartStopThreadParam(address, configAddress, bStop, m_useDefaultHPCCInit, this, context); pThreadReq->setResultObject( pResult ); if (userName && *userName) pThreadReq->setUserID( userName ); if (password && *password) pThreadReq->setPassword( password ); PooledThreadHandle handle = m_threadPool->start( pThreadReq ); threadHandles.append(handle); } #else { StringBuffer newAddress; ConvertAddress(address0, newAddress); char* pStr = (char*) strchr(address, '|');; if (pStr) pStr[0] = 0; pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component pResult->setAddress ( address );//can be either IP or name of component pResult->setCompType( compType ); pResult->setOS( atoi(OS) ); pResult->setPath( path ); resultsArray.append(*pResult.getLink()); CStartStopThreadParam* pThreadReq; pThreadReq = new CStartStopThreadParam(address, bStop, this, context); pThreadReq->setResultObject( pResult ); if (userName && *userName) pThreadReq->setUserID( userName ); if (password && *password) pThreadReq->setPassword( password ); PooledThreadHandle handle = m_threadPool->start( pThreadReq ); threadHandles.append(handle); } #endif } //block for worker theads to finish, if necessary, and then collect results // PooledThreadHandle* pThreadHandle = threadHandles.getArray(); unsigned i=threadHandles.ordinality(); while (i--) { m_threadPool->join(*pThreadHandle, 30000);//abort after 30 secs in remote possibility that the command blocks pThreadHandle++; } resp.setStartStopResults(resultsArray); resp.setStop(bStop); if (version > 1.08) { resp.setContainCluster(containCluster); } return true; }
virtual void init() { CMasterActivity::init(); OwnedRoxieString indexFileName(helper->getIndexFileName()); Owned<IDistributedFile> dataFile; Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true); unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0); if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves()) keyReadWidth = container.queryJob().querySlaves(); initMb.clear(); initMb.append(indexFileName.get()); if (helper->diskAccessRequired()) numTags += 2; initMb.append(numTags); unsigned t=0; for (; t<numTags; t++) { tags[t] = container.queryJob().allocateMPTag(); initMb.append(tags[t]); } bool keyHasTlk = false; if (indexFile) { unsigned numParts = 0; localKey = indexFile->queryAttributes().getPropBool("@local"); if (container.queryLocalData() && !localKey) throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local"); checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true); Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor(); IDistributedSuperFile *superIndex = indexFile->querySuperFile(); unsigned superIndexWidth = 0; unsigned numSuperIndexSubs = 0; if (superIndex) { numSuperIndexSubs = superIndex->numSubFiles(true); bool first=true; // consistency check Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true); ForEach(*iter) { IDistributedFile &f = iter->query(); unsigned np = f.numParts()-1; IDistributedFilePart &part = f.queryPart(np); const char *kind = part.queryAttributes().queryProp("@kind"); bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey) if (first) { first = false; keyHasTlk = hasTlk; superIndexWidth = f.numParts(); if (keyHasTlk) --superIndexWidth; } else { if (hasTlk != keyHasTlk) throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin"); if (keyHasTlk && superIndexWidth != f.numParts()-1) throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin"); } } if (keyHasTlk) numParts = superIndexWidth * numSuperIndexSubs; else numParts = superIndex->numParts(); } else { numParts = indexFile->numParts(); if (numParts) { const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind"); keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); if (keyHasTlk) --numParts; } } if (numParts) { initMb.append(numParts); initMb.append(superIndexWidth); // 0 if not superIndex initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0); unsigned p=0; UnsignedArray parts; for (; p<numParts; p++) parts.append(p); indexFileDesc->serializeParts(initMb, parts); if (localKey) keyHasTlk = false; // not used initMb.append(keyHasTlk); if (keyHasTlk) { if (numSuperIndexSubs) initMb.append(numSuperIndexSubs); else initMb.append((unsigned)1); Owned<IDistributedFileIterator> iter; IDistributedFile *f; if (superIndex) { iter.setown(superIndex->getSubFileIterator(true)); f = &iter->query(); } else f = indexFile; loop { unsigned location; OwnedIFile iFile; StringBuffer filePath; Owned<IFileDescriptor> fileDesc = f->getFileDescriptor(); Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1); if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath)) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName()); OwnedIFileIO iFileIO = iFile->open(IFOread); assertex(iFileIO); size32_t tlkSz = (size32_t)iFileIO->size(); initMb.append(tlkSz); ::read(iFileIO, 0, tlkSz, initMb); if (!iter || !iter->next()) break; f = &iter->query(); } } if (helper->diskAccessRequired()) { OwnedRoxieString fetchFilename(helper->getFileName()); if (fetchFilename) { dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true)); if (dataFile) { if (superIndex) throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported"); Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile); void *ekey; size32_t ekeylen; helper->getFileEncryptKey(ekeylen,ekey); bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted"); if (0 != ekeylen) { memset(ekey,0,ekeylen); free(ekey); if (!encrypted) { Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName()); container.queryJob().fireException(e); } } else if (encrypted) throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get()); unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0); if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves()) dataReadWidth = container.queryJob().querySlaves(); Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth); dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL)); dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear()); } else indexFile.clear(); } } } else