Пример #1
0
 void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
 {
     IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
     IPartDescriptor *partDesc = fileDesc->queryPart(slave);
     partDesc->serialize(dst);
     dst.append(helper->getTempUsageCount());
 }
Пример #2
0
 void done()
 {
     if (!container.queryOwner().queryOwner() || container.queryOwner().isGlobal()) // I am in a child query
     {
         IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
         OwnedRoxieString fname(helper->getFileName());
         container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
         queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
     }
 }
    void open()
    {
        char drive       [_MAX_DRIVE];
        char dir         [_MAX_DIR];
        char fname       [_MAX_DIR];
        char ext         [_MAX_EXT];
        _splitpath(fileName.str(), drive, dir, fname, ext);

        StringBuffer directory;
        directory.append(drive).append(dir);

        Owned<IFile> cd = createIFile(directory.str());
        cd->createDirectory();

        IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
        void *ekey;
        size32_t ekeylen;
        helper->getEncryptKey(ekeylen,ekey);
        Owned<ICompressor> ecomp;
        if (ekeylen!=0)
        {
            ecomp.setown(createAESCompressor256(ekeylen,ekey));
            memset(ekey,0,ekeylen);
            free(ekey);
            compress = true;
        }
        Owned<IFile> file = createIFile(fileName.str());
        Owned<IFileIO> iFileIO;
        bool fixedRecordSize = queryRowMetaData()->isFixedSize();
        size32_t minrecsize = queryRowMetaData()->getMinRecordSize();

        if (fixedRecordSize)
            ActPrintLog("SPILL: created fixed output %s recsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
        else
            ActPrintLog("SPILL: created variable output %s, minrecsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
        unsigned rwFlags = (DEFAULT_RWFLAGS & ~rw_autoflush); // flushed by close()
        if (compress)
            rwFlags |= rw_compress;
        else
            rwFlags |= rw_crc; // only if !compress
        if (grouped)
            rwFlags |= rw_grouped;
        out.setown(createRowWriter(file, this, rwFlags));
    }
Пример #4
0
 void init()
 {
     IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
     IArrayOf<IGroup> groups;
     OwnedRoxieString fname(helper->getFileName());
     fillClusterArray(container.queryJob(), fname, clusters, groups);
     fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, true, TDWnoreplicate+TDXtemporary));
     IPropertyTree &props = fileDesc->queryProperties();
     bool blockCompressed=false;
     void *ekey;
     size32_t ekeylen;
     helper->getEncryptKey(ekeylen,ekey);
     if (ekeylen)
     {
         memset(ekey,0,ekeylen);
         free(ekey);
         props.setPropBool("@encrypted", true);      
         blockCompressed = true;
     }
     else if (0 != (helper->getFlags() & TDWnewcompress) || 0 != (helper->getFlags() & TDXcompress))
         blockCompressed = true;
     if (blockCompressed)
         props.setPropBool("@blockCompressed", true);        
     if (0 != (helper->getFlags() & TDXgrouped))
         fileDesc->queryProperties().setPropBool("@grouped", true);
     props.setProp("@kind", "flat");
     if (container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
     { // do early, because this will be local act. and will not come back to master until end of owning graph.
         container.queryTempHandler()->registerFile(fname, container.queryOwner().queryGraphId(), helper->getTempUsageCount(), TDXtemporary & helper->getFlags(), getDiskOutputKind(helper->getFlags()), &clusters);
         queryThorFileManager().publish(container.queryJob(), fname, true, *fileDesc);
     }
 }