void CachedPasswordProvider::serialize(MemoryBuffer & out)
{
    unsigned num = passwords.ordinality();
    out.append(num);
    ForEachItemIn(idx, passwords)
        passwords.item(idx).serialize(out);
}
    virtual void init()
    {
        HashDistributeMasterBase::init();

        // JCSMORE should common up some with indexread
        IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();

        StringBuffer scoped;
        queryThorFileManager().addScope(container.queryJob(), helper->getIndexFileName(), scoped);
        Owned<IDistributedFile> f = queryThorFileManager().lookup(container.queryJob(), helper->getIndexFileName());
        if (!f)
            throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
        if (0 == f->numParts())
            throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());

        checkFormatCrc(this, f, helper->getFormatCrc(), true);
        Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
        Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
        if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
            throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
        unsigned location;
        OwnedIFile iFile;
        StringBuffer filePath;
        if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
            throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
        OwnedIFileIO iFileIO = iFile->open(IFOread);
        assertex(iFileIO);

        tlkMb.append(iFileIO->size());
        ::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);

        queryThorFileManager().noteFileRead(container.queryJob(), f);
    }
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
    bool hasVersion = len && (*(const byte *)data == 0x80);
    MemoryBuffer src;
    src.setBuffer(len, const_cast<void *>(data), false);
    byte version = 1;
    if (hasVersion)
    {
        src.skip(1);
        src.read(version);
    }

    MemoryBuffer tgt;
    switch (version)
    {
    case 1:
        decompressToBuffer(tgt, src);
        break;
    default:
        throwUnexpected();
    }

    tgt.append((char)0);
    unsigned expandedLen = tgt.length();
    result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
    return true;
}
 virtual void serializeStats(MemoryBuffer &mb)
 {
     CSlaveActivity::serializeStats(mb);
     CriticalBlock b(joinHelperCrit);
     rowcount_t p = joinhelper?joinhelper->getLhsProgress():0;
     mb.append(p);
 }
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
    HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
    if (dllHandle == NULL)
        dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
    if (dllHandle == NULL)
    {
        DBGLOG("Failed to load library %s: %d", filename, GetLastError());
        return false;
    }
    HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
    if (!hrsrc)
        return false;
    size32_t len = SizeofResource(dllHandle, hrsrc);
    const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
    data.append(len, rdata);
    FreeLibrary(dllHandle);
    return true;
#else
    bfd_init ();
    bfd *file = bfd_openr(filename, NULL);
    if (file)
    {
        StringBuffer sectionName;
        sectionName.append(type).append("_").append(id).append(".data");
        SecScanParam param(data, sectionName.str());
        if (bfd_check_format (file, bfd_object))
            bfd_map_over_sections (file, secscan, &param);
        bfd_close (file);
   }
   return data.length() != 0;
#endif
}
Exemple #6
0
 virtual void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
 {
     IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
     IPartDescriptor *partDesc = fileDesc->queryPart(slave);
     partDesc->serialize(dst);
     dst.append(helper->getTempUsageCount());
 }
void CDiskReadMasterBase::serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
    IHThorDiskReadBaseArg *helper = (IHThorDiskReadBaseArg *) queryHelper();
    OwnedRoxieString fileName(helper->getFileName());
    dst.append(fileName);
    dst.append(subfileLogicalFilenames.ordinality());
    if (subfileLogicalFilenames.ordinality())
    {
        ForEachItemIn(s, subfileLogicalFilenames)
            dst.append(subfileLogicalFilenames.item(s));
    }
    if (mapping)
        mapping->serializeMap(slave, dst);
    else
        CSlavePartMapping::serializeNullMap(dst);
}
Exemple #8
0
extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
{
#ifdef _WIN32
    HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
    if (dllHandle == NULL)
        dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
    if (dllHandle == NULL)
    {
        DBGLOG("Failed to load library %s: %d", filename, GetLastError());
        return false;
    }
    HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
    if (!hrsrc)
        return false;
    size32_t len = SizeofResource(dllHandle, hrsrc);
    const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
    data.append(len, rdata);
    FreeLibrary(dllHandle);
    return true;
#elif defined (_USE_BINUTILS)
    CriticalBlock block(bfdCs);
    bfd_init ();
    bfd *file = bfd_openr(filename, NULL);
    if (file)
    {
        StringBuffer sectionName;
        sectionName.append(type).append("_").append(id).append(".data");
        SecScanParam param(data, sectionName.str());
        if (bfd_check_format (file, bfd_object))
            bfd_map_over_sections (file, secscan, &param);
        bfd_close (file);
   }
   return data.length() != 0;
#else
    struct stat stat_buf;
    VStringBuffer sectname("%s_%u", type, id);
    int fd = open(filename, O_RDONLY);
    if (fd == -1 || fstat(fd, &stat_buf) == -1)
    {
        DBGLOG("Failed to load library %s: %d", filename, errno);
        return false;
    }

    bool ok = false;
    __uint64 size = stat_buf.st_size;
    const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
    if (start_addr == MAP_FAILED)
    {
        DBGLOG("Failed to load library %s: %d", filename, errno);
    }
    else
    {
        ok = getResourceFromMappedFile(filename, start_addr, data, type, id);
        munmap((void *)start_addr, size);
    }
    close(fd);
    return ok;
#endif
}
Exemple #9
0
extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
{
    mb.append((byte)0x80).append(resourceHeaderVersion);
    if (compress)
        compressToBuffer(mb, len, data);
    else
        appendToBuffer(mb, len, data);
}
Exemple #10
0
void CWriteMasterBase::serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
    dst.append(diskHelperBase->getFileName());
    if (diskHelperBase->getFlags() & TDXtemporary)
    {
        unsigned usageCount = container.queryJob().queryWorkUnit().queryFileUsage(diskHelperBase->getFileName());
        if (0 == usageCount) usageCount = diskHelperBase->getTempUsageCount();
        dst.append(usageCount);
    }
    if (dlfn.isExternal())
    {
        fileDesc->queryPart(0)->serialize(dst);
        dst.append(mpTag);
    }
    else
        fileDesc->queryPart(targetOffset+slave)->serialize(dst);
}
static bool addWUQueryFilter(WUSortField *filters, unsigned &count, MemoryBuffer &buff, const char *name, WUSortField value)
{
    if (!name || !*name)
        return false;
    filters[count++] = value;
    buff.append(name);
    return true;
}
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
    Owned<IFvDataSource> ds = readDataSource(in);
    if (!ds)
    {
        out.append(false);
        return;
    }

    __int64 requestedRow;
    in.read(requestedRow);

    unsigned startPos = out.length();
    unsigned numRows = 0;
    out.append(true);                       // ok
    out.append(requestedRow);       // start 

    unsigned numRowsPos = out.length();
    out.append(numRows);                // total number of rows;
    loop
    {
        unsigned lengthPos = out.length();
        out.append((unsigned)0);                // size of this row.
        unsigned startRow = out.length();
        if (raw)
        {
            if (!ds->getRawRow(out, requestedRow+numRows))
                break;
        }
        else
        {
            if (!ds->getRow(out, requestedRow+numRows))
                break;
        }
        if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
            break;
        unsigned endRow = out.length();
        out.rewrite(lengthPos);
        out.append(endRow-startRow);
        out.rewrite(endRow);
        numRows++;
    }

    if (numRows == 0)
    {
        out.rewrite(startPos);
        out.append(false);
        return;
    }

    unsigned totalLength = out.length();
    out.rewrite(numRowsPos);
    out.append(numRows);
    out.rewrite(totalLength);
}
 virtual void serializeCreateContext(MemoryBuffer &mb)
 {
     // bit of hack, need to tell slave if wuidread converted to diskread (see master activity)
     CMasterGraphElement::serializeCreateContext(mb);
     if (kind == TAKworkunitread)
     {
         if (!activity)
             doCreateActivity();
         IHThorArg *helper = activity->queryHelper();
         IHThorDiskReadArg *diskHelper = QUERYINTERFACE(helper, IHThorDiskReadArg);
         mb.append(NULL != diskHelper); // flag to slaves that they should create diskread
         if (diskHelper)
         {
             OwnedRoxieString fileName(diskHelper->getFileName());
             mb.append(fileName);
         }
     }
 }
void RemoteDataSourceServer::doCmdFetchRaw(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
    Owned<IFvDataSource> ds = readDataSource(in);
    if (!ds)
    {
        out.append(false);
        return;
    }

    __int64 requestedOffset;
    in.read(requestedOffset);

    MemoryBuffer temp;
    bool ok = ds->fetchRawRow(temp, requestedOffset);
    out.append(ok);                     // ok
    out.append(temp.length());
    out.append(temp.length(), temp.toByteArray());
}
bool FullWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
{
    size32_t length;
    const void * data = rows->fetchRow(offset, length);
    if (!data)
        return false;
    out.append(length, data);
    return true;
}
Exemple #16
0
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
    MemoryBuffer tgt;
    decompressResource(len, data, tgt);
    tgt.append((char)0);
    unsigned expandedLen = tgt.length();
    result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
    return true;
}
void CDiskReadSlaveActivityBase::serializeStats(MemoryBuffer &mb)
{
    CSlaveActivity::serializeStats(mb);
    mb.append(diskProgress);

    CRuntimeStatisticCollection activeStats(diskReadRemoteStatistics);
    if (partHandler)
        partHandler->gatherStats(activeStats);
    activeStats.serialize(mb);
}
    virtual void serializeStats(MemoryBuffer &mb)
    {
        CSlaveActivity::serializeStats(mb);
        CriticalBlock b(joinHelperCrit);
        rowcount_t p = joinhelper?joinhelper->getLhsProgress():0;
        mb.append(p);

        CRuntimeStatisticCollection mergedStats(spillStats);
        mergeStats(mergedStats, sorter);    // No danger of a race with reset() because that never replaces a valid sorter
        mergedStats.serialize(mb);
    }
Exemple #19
0
void serializeBoolArray(MemoryBuffer & out, unsigned len, const bool * values)
{
    for (unsigned i = 0; i < len; i+= 8)
    {
        unsigned char next = 0;
        unsigned max = i+8 <= len ? 8 : len - i;
        for (unsigned j=0; j<max; j++)
            if (values[i+j]) next |= (1 << j);
        out.append(next);
    }
}
void CDiskWriteSlaveActivityBase::serializeStats(MemoryBuffer &mb)
{
    CriticalBlock block(statsCs);

    ProcessSlaveActivity::serializeStats(mb);
    mb.append(replicateDone);

    CRuntimeStatisticCollection activeStats(fileStats);
    mergeStats(activeStats, outputIO);
    activeStats.serialize(mb);
}
void CDiskWriteSlaveActivityBase::processDone(MemoryBuffer &mb)
{
    if (abortSoon)
        return;
    rowcount_t _processed = processed & THORDATALINK_COUNT_MASK;
    Owned<IFile> ifile = createIFile(fName);
    offset_t sz = ifile->size();
    if (-1 != sz)
        container.queryJob().queryIDiskUsage().increase(sz);
    mb.append(_processed).append(compress?uncompressedBytesWritten:sz).append(sz);
    unsigned crc = compress?~0:fileCRC.get();
    mb.append(calcFileCrc?crc:0);

    CDateTime createTime, modifiedTime, accessedTime;
    ifile->getTime(&createTime, &modifiedTime, &accessedTime);
    // round file time down to nearest sec. Nanosec accuracy is not preserved elsewhere and can lead to mismatch later.
    unsigned hour, min, sec, nanosec;
    modifiedTime.getTime(hour, min, sec, nanosec);
    modifiedTime.setTime(hour, min, sec, 0);
    modifiedTime.serialize(mb);
}
void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
{
    SessionId session;
    StringAttr username, password, logicalName;

    in.read(session);
    in.read(username).read(password);
    in.read(logicalName);

    DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session);
    Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
    unique_id_t id = addDataSource(session, ds);

    out.append((unsigned short)CurRemoteVersion);
    out.append(id);
    out.append(ds->numRows(false));
    ds->queryMetaData()->serialize(out);
    out.append(ds->isIndex());

    DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id);
}
static bool serializeWUInfo(IConstWorkUnitInfo &info,MemoryBuffer &mb)
{
    fixedAppend(mb,24,info.queryWuid());
    varAppendMax(mb,64,info.queryUser());
    varAppendMax(mb,64,info.queryClusterName());
    varAppendMax(mb,64,""); // roxiecluster is obsolete
    varAppendMax(mb,256,info.queryJobName());
    fixedAppend(mb,10,info.queryStateDesc());
    fixedAppend(mb,7,info.queryPriorityDesc());
    short int prioritylevel = info.getPriorityLevel();
    mb.append(prioritylevel);
    fixedAppend(mb,20,"");  // Created timestamp
    fixedAppend(mb,20,"");  // Modified timestamp
    mb.append(true);
    mb.append(info.isProtected());
    if (mb.length()>WORKUNIT_SERVICES_BUFFER_MAX) {
        mb.clear().append(WUS_STATUS_OVERFLOWED);
        return false;
    }
    return true;
}
Exemple #24
0
static void serializeElement(MemoryBuffer & target, IDefRecordElement * elem)
{
    byte kind = elem ? elem->getKind() : DEKnone;
    target.append(kind);
    switch (kind)
    {
    case DEKnone:
        break;
    case DEKrecord:
        {
            size32_t maxSize = elem->getMaxSize();
            unsigned numChildren = elem->numChildren();
            target.append(maxSize).append(numChildren);
            for (unsigned i=0; i < numChildren; i++)
                serializeElement(target, elem->queryChild(i));
            break;
        }
    case DEKifblock:
        {
            IValue * value = elem->queryCompareValue();
            serializeValue(target, value);
            serializeElement(target, elem->queryChild(0));
            serializeElement(target, elem->queryChild(1));
            break;
        }
    case DEKfield:
        {
            _ATOM name = elem->queryName();
            ITypeInfo * type = elem->queryType();
            size32_t maxSize = elem->getMaxSize();
            serializeAtom(target, name);
            type->serialize(target);
            serializeElement(target, elem->queryChild(0));
            target.append(maxSize);
            break;
        }
    default:
        throwUnexpected();
    }
}
    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
    {
        AggregateSlaveBase::init(data, slaveData);

        appendOutputLinked(this);
        helper = (IHThorAggregateArg *)queryHelper();

        if (firstNode())
        {
            mpTag = createReplyTag();
            slaveData.append((int &)mpTag);
        }
    }
Exemple #26
0
void FileFormat::serialize(MemoryBuffer & out) const
{
    out.append((byte)type);
    switch (type)
    {
    case FFTfixed:
    case FFTblocked:
        out.append(recordSize);
        break;
    case FFTcsv:
    case FFTutf:
    case FFTutf8: case FFTutf8n:
    case FFTutf16: case FFTutf16be: case FFTutf16le:
    case FFTutf32: case FFTutf32be: case FFTutf32le:
        out.append(maxRecordSize);
        ::serialize(out, separate);
        ::serialize(out, quote);
        ::serialize(out, terminate);
        ::serialize(out, rowTag);
        break;
    }
}
Exemple #27
0
void ProcessSlaveActivity::serializeStats(MemoryBuffer &mb)
{
    CSlaveActivity::serializeStats(mb);
    mb.append(processed);
#ifdef TIME_ACTIVITIES
    if (timeActivities)
    {
        unsigned __int64 nowCycles = get_cycles_now();
        totalCycles += nowCycles-lastCycles;
        lastCycles = nowCycles;
    }
#endif
}
void CSlavePartMapping::serializeMap(unsigned i, MemoryBuffer &mb, IGetSlaveData *extra)
{
    if (local)
        i = 0;
    if (i >= maps.ordinality())
    {
        mb.append((unsigned)0);
        return;
    }

    CSlaveMap &map = maps.item(i);
    unsigned nPos = mb.length();
    unsigned n=0;
    mb.append(n);
    UnsignedArray parts;
    ForEachItemIn(m, map)
        parts.append(map.item(m).queryPartIndex());
    MemoryBuffer extraMb;
    if (extra)
    {
        ForEachItemIn(m2, map)
        {
            unsigned xtraLen = 0;
            unsigned xtraPos = extraMb.length();
            extraMb.append(xtraLen);
            IPartDescriptor &partDesc = map.item(m2);
            if (!extra->getData(m2, partDesc.queryPartIndex(), extraMb))
            {
                parts.zap(partDesc.queryPartIndex());
                extraMb.rewrite(xtraPos);
            }
            else
            {
                xtraLen = (extraMb.length()-xtraPos)-sizeof(xtraLen);
                extraMb.writeDirect(xtraPos, sizeof(xtraLen), &xtraLen);
            }
        }
    }
bool FullWorkUnitDataSource::init()
{
    bool ok = WorkUnitDataSource::init();
    if (ok)
    {
        MemoryBuffer temp;
        MemoryBuffer2IDataVal xxx(temp);

        //Nasty.  Single sets are represented as the same way as datasets (with an extra flag for all)
        //however need to represent as a single row containing a set, which has a different format.
        if (wuResult->isResultScalar() && returnedMeta->isSingleSet())
        {
            temp.append(wuResult->getResultIsAll());
            temp.append((size32_t)wuResult->getResultRawSize(0, 0));
        }
        wuResult->getResultRaw(xxx, NULL, NULL);

        if (returnedMeta->isFixedSize())
            rows.setown(new FixedRowBlock(temp, 0, 0, returnedMeta->fixedSize()));
        else
            rows.setown(new VariableRowBlock(temp, 0, 0, returnedRecordSize, true));
    }
    return ok;
}
void DataSourceMetaData::serialize(MemoryBuffer & buffer) const
{
    //NB: Update NullDataSourceMeta if this changes....
    buffer.append(numFieldsToIgnore);
    buffer.append(randomIsOk);
    buffer.append(hasMixedContent);
    buffer.append(isStoredFixedWidth);
    buffer.append(minRecordSize);
    buffer.append(keyedSize);
    buffer.append(maxRecordSize);

    unsigned numFields = fields.ordinality();
    buffer.append(numFields);
    for (unsigned idx=0; idx < numFields; idx++)
    {
        fields.item(idx).serialize(buffer);
    }
}