Beispiel #1
0
    void init()
    {
        // prepare topology.

        unsigned rootNodes = container.queryJob().querySlaves();

        unsigned res = MERGE_GRANULARITY;

        sD = new MemoryBuffer[rootNodes];
        UnsignedArray nodes1, nodes2;
        UnsignedArray *currentLevel = &nodes1, *nextLevel = &nodes2;
        unsigned n = 0; while (n<rootNodes) currentLevel->append(n++);
        while (rootNodes > 1)
        {
            assertex(rootNodes);

            unsigned r = (rootNodes+(res-1))/res; // groups
            unsigned t = 0;
            n = 0;
            bool first = true;
            loop
            {
                if (first)
                {
                    first = false;
                    nextLevel->append(currentLevel->item(n));
                }
                else
                {
#ifdef _DEBUG
                    unsigned node = nextLevel->tos();
                    unsigned item = currentLevel->item(n);
                    ActPrintLog("Adding to node=%d, item=%d", node, item);
#endif
                    sD[nextLevel->tos()].append(currentLevel->item(n));
                }
                n++;
                if (n>=rootNodes) break;
                t += r;
                if (t>=rootNodes)
                {
                    t -= rootNodes;
                    first = true;
                }
            }
            
            assertex(sD[nextLevel->tos()].length()); // something must have been added
            n = 0;
            while (n<nextLevel->ordinality()) sD[nextLevel->item(n++)].append(0); // terminator
#ifdef _DEBUG
            ActPrintLog("EOL");
#endif
            rootNodes = nextLevel->ordinality();

            UnsignedArray *tmp = currentLevel;
            currentLevel = nextLevel;
            nextLevel = tmp;
            nextLevel->kill();
        }
    }
Beispiel #2
0
void DataSourceMetaData::extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray & types)
{
    unsigned curOffset = 0;
    ForEachItemIn(i, fields)
    {
        if (curOffset >= keyedSize)
            break;

        DataSourceMetaItem & cur = fields.item(i);
        switch (cur.flags)
        {
        case FVFFnone:
            {
                offsets.append(curOffset);
                types.append(*LINK(cur.type));
                unsigned size = cur.type->getSize();
                assertex(size != UNKNOWN_LENGTH);
                curOffset += size;
                break;
            }
        case FVFFbeginrecord:
        case FVFFendrecord:
            break;
        default:
            throwUnexpected();
        }
    }
    offsets.append(curOffset);
    assertex(curOffset == keyedSize);
}
Beispiel #3
0
NlpMatchPath::NlpMatchPath(const UnsignedArray & _ids, const UnsignedArray & _indices)
{
    assert(_ids.ordinality() == _indices.ordinality());
    ForEachItemIn(idx, _ids)
    {
        ids.append(_ids.item(idx));
        indices.append(_indices.item(idx));
    }
Beispiel #4
0
bool CCronAtSchedule::match(UnsignedArray &a,unsigned v,unsigned &next)
{
    if (a.ordinality()==0) {
        next = v;
        return true;
    }
    ForEachItemIn(i,a) {
        unsigned n = a.item(i);
        if (n>=v) {
            next = n;
            return true;
        }
    }
MultiLexer::MultiLexer(const AsciiDfa & _tokens, const AsciiDfa & _skip, const UnsignedArray & _endTokenChars, unsigned _eofId) : tokens(_tokens), skip(_skip)
{
    eofId = _eofId;
    _clear(isEndToken);
    ForEachItemIn(idx, _endTokenChars)
    {
        unsigned next = _endTokenChars.item(idx);
        if (next < 256)
            isEndToken[next] = true;
    }
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra)
{
    if (local)
        i = 0;
    if (i >= maps.ordinality())
    {
        mb.append((unsigned)0);
        return;
    }

    CSlaveMap &map = maps.item(i);
    unsigned nPos = mb.length();
    unsigned n=0;
    mb.append(n);
    UnsignedArray parts;
    ForEachItemIn(m, map)
        parts.append(map.item(m).queryPartIndex());
    MemoryBuffer extraMb;
    if (extra)
    {
        ForEachItemIn(m2, map)
        {
            unsigned xtraLen = 0;
            unsigned xtraPos = extraMb.length();
            extraMb.append(xtraLen);
            IPartDescriptor &partDesc = map.item(m2);
            if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb))
            {
                parts.zap(partDesc.queryPartIndex());
                extraMb.rewrite(xtraPos);
            }
            else
            {
                xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen);
                extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen);
            }
        }
    }
bool Cws_machineEx::doStartStop(IEspContext &context, StringArray& addresses, char* userName, char* password, bool bStop,
                                                     IEspStartStopResponse &resp)
{
    bool containCluster = false;
    double version = context.getClientVersion();
    const int ordinality= addresses.ordinality();

    UnsignedArray threadHandles;
    IArrayOf<IEspStartStopResult> resultsArray;

    for (int index=0; index<ordinality; index++)
    {
        const char* address0 = addresses.item(index);

        //address passed in is of the form "192.168.1.4:EspProcess:2:path1"
        StringArray sArray;
        sArray.appendList(addresses.item(index), ":");

        if (sArray.ordinality() < 4)
            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments");

        Owned<IEspStartStopResult> pResult = static_cast<IEspStartStopResult*>(new CStartStopResult(""));
        const char* address = sArray.item(0);
        const char* compType= sArray.item(1);
        const char* OS        = sArray.item(3);//index 2 is component name
        const char* path      = sArray.item(4);

        if (!(address && *address && compType && *compType && OS && *OS && path && *path))
            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Invalid input");

        if (!stricmp(compType, "ThorCluster") || !stricmp(compType, "RoxieCluster"))
            containCluster = true;

#ifndef OLD_START_STOP
        {
            char* configAddress = NULL;
            char* props1 = (char*) strchr(address, '|');
            if (props1)
            {
                configAddress = props1+1;
                *props1 = '\0';
            }
            else
            {
                configAddress = (char*) address;
            }

            StringBuffer newAddress;
            ConvertAddress(address0, newAddress);
            pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
            pResult->setAddress ( address );//can be either IP or name of component
            pResult->setCompType( compType );
            if (version > 1.04)
            {       
                pResult->setName( path );
                const char* pStr2 = strstr(path, "LexisNexis");
                if (pStr2)
                {
                    char name[256];
                    const char* pStr1 = strchr(pStr2, '|');
                    if (!pStr1)
                    {
                        strcpy(name, pStr2+11);
                    }
                    else
                    {
                        strncpy(name, pStr2+11, pStr1 - pStr2 -11);
                        name[pStr1 - pStr2 -11] = 0;
                    }
                    pResult->setName( name );
                }   
            }
            
            pResult->setOS( atoi(OS) ); 
            pResult->setPath( path );

            resultsArray.append(*pResult.getLink());

            CStartStopThreadParam* pThreadReq;
            pThreadReq = new CStartStopThreadParam(address, configAddress, bStop, m_useDefaultHPCCInit, this, context);
            pThreadReq->setResultObject( pResult );

            if (userName && *userName)
                pThreadReq->setUserID( userName );
            if (password && *password)
                pThreadReq->setPassword( password );

            PooledThreadHandle handle = m_threadPool->start( pThreadReq );
            threadHandles.append(handle);
        }
#else
        {
            StringBuffer newAddress;
            ConvertAddress(address0, newAddress);
            char* pStr = (char*) strchr(address, '|');;
            if (pStr)
                pStr[0] = 0;

            pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
            pResult->setAddress ( address );//can be either IP or name of component
            pResult->setCompType( compType );
            pResult->setOS( atoi(OS) ); 
            pResult->setPath( path );

            resultsArray.append(*pResult.getLink());

            CStartStopThreadParam* pThreadReq;
            pThreadReq = new CStartStopThreadParam(address, bStop, this, context);
            pThreadReq->setResultObject( pResult );

            if (userName && *userName)
                pThreadReq->setUserID( userName );
            if (password && *password)
                pThreadReq->setPassword( password );

            PooledThreadHandle handle = m_threadPool->start( pThreadReq );
            threadHandles.append(handle);
        }
#endif
    }

    //block for worker theads to finish, if necessary, and then collect results
    //
    PooledThreadHandle* pThreadHandle = threadHandles.getArray();
    unsigned i=threadHandles.ordinality();
    while (i--) 
    {
        m_threadPool->join(*pThreadHandle, 30000);//abort after 30 secs in remote possibility that the command blocks
        pThreadHandle++;
    }

    resp.setStartStopResults(resultsArray);
    resp.setStop(bStop);

    if (version > 1.08)
    {
        resp.setContainCluster(containCluster);
    }
    return true;
}
Beispiel #8
0
    virtual void init()
    {
        CMasterActivity::init();
        OwnedRoxieString indexFileName(helper->getIndexFileName());
        Owned<IDistributedFile> dataFile;
        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);

        unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
        if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
            keyReadWidth = container.queryJob().querySlaves();
        

        initMb.clear();
        initMb.append(indexFileName.get());
        if (helper->diskAccessRequired())
            numTags += 2;
        initMb.append(numTags);
        unsigned t=0;
        for (; t<numTags; t++)
        {
            tags[t] = container.queryJob().allocateMPTag();
            initMb.append(tags[t]);
        }
        bool keyHasTlk = false;
        if (indexFile)
        {
            unsigned numParts = 0;
            localKey = indexFile->queryAttributes().getPropBool("@local");

            if (container.queryLocalData() && !localKey)
                throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");

            checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
            Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
            IDistributedSuperFile *superIndex = indexFile->querySuperFile();
            unsigned superIndexWidth = 0;
            unsigned numSuperIndexSubs = 0;
            if (superIndex)
            {
                numSuperIndexSubs = superIndex->numSubFiles(true);
                bool first=true;
                // consistency check
                Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
                ForEach(*iter)
                {
                    IDistributedFile &f = iter->query();
                    unsigned np = f.numParts()-1;
                    IDistributedFilePart &part = f.queryPart(np);
                    const char *kind = part.queryAttributes().queryProp("@kind");
                    bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
                    if (first)
                    {
                        first = false;
                        keyHasTlk = hasTlk;
                        superIndexWidth = f.numParts();
                        if (keyHasTlk)
                            --superIndexWidth;
                    }
                    else
                    {
                        if (hasTlk != keyHasTlk)
                            throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
                        if (keyHasTlk && superIndexWidth != f.numParts()-1)
                            throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
                    }
                }
                if (keyHasTlk)
                    numParts = superIndexWidth * numSuperIndexSubs;
                else
                    numParts = superIndex->numParts();
            }
            else
            {
                numParts = indexFile->numParts();
                if (numParts)
                {
                    const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
                    keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
                    if (keyHasTlk)
                        --numParts;
                }
            }
            if (numParts)
            {
                initMb.append(numParts);
                initMb.append(superIndexWidth); // 0 if not superIndex
                initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
                unsigned p=0;
                UnsignedArray parts;
                for (; p<numParts; p++)
                    parts.append(p);
                indexFileDesc->serializeParts(initMb, parts);
                if (localKey)
                    keyHasTlk = false; // not used
                initMb.append(keyHasTlk);
                if (keyHasTlk)
                {
                    if (numSuperIndexSubs)
                        initMb.append(numSuperIndexSubs);
                    else
                        initMb.append((unsigned)1);

                    Owned<IDistributedFileIterator> iter;
                    IDistributedFile *f;
                    if (superIndex)
                    {
                        iter.setown(superIndex->getSubFileIterator(true));
                        f = &iter->query();
                    }
                    else
                        f = indexFile;
                    loop
                    {
                        unsigned location;
                        OwnedIFile iFile;
                        StringBuffer filePath;
                        Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
                        Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
                        if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
                            throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
                        OwnedIFileIO iFileIO = iFile->open(IFOread);
                        assertex(iFileIO);

                        size32_t tlkSz = (size32_t)iFileIO->size();
                        initMb.append(tlkSz);
                        ::read(iFileIO, 0, tlkSz, initMb);

                        if (!iter || !iter->next())
                            break;
                        f = &iter->query();
                    }
                }
                if (helper->diskAccessRequired())
                {
                    OwnedRoxieString fetchFilename(helper->getFileName());
                    if (fetchFilename)
                    {
                        dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
                        if (dataFile)
                        {
                            if (superIndex)
                                throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
                            Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
                            void *ekey;
                            size32_t ekeylen;
                            helper->getFileEncryptKey(ekeylen,ekey);
                            bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
                            if (0 != ekeylen)
                            {
                                memset(ekey,0,ekeylen);
                                free(ekey);
                                if (!encrypted)
                                {
                                    Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
                                    container.queryJob().fireException(e);
                                }
                            }
                            else if (encrypted)
                                throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
                            unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
                            if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
                                dataReadWidth = container.queryJob().querySlaves();
                            Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
                            dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
                            dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
                        }
                        else
                            indexFile.clear();
                    }
                }
            }
            else