Exemplo n.º 1
0
    IRowStream *load(
        IRowStream *in,
        IRowInterfaces *rowif,
        ICompare *icompare,
        bool alldisk,
        bool &abort,
        bool &isempty,
        const char *tracename,
        bool isstable, unsigned maxcores)
    {
        overflowcount = 0;
        unsigned nrecs;
        serializer.set(rowif->queryRowSerializer());
        rows.setSizing(true,false);
#ifdef _FULL_TRACE
        PROGLOG("CThorRowSortedLoader load start");
#endif
        isempty = true;
        while (!abort) {
            rows.clear();
            bool hasoverflowed = false;
            nrecs = rows.load(*in,true,abort,&hasoverflowed);
            if (nrecs)
                isempty = false;
            if (hasoverflowed)
                overflowcount++;
#ifdef _FULL_TRACE
            PROGLOG("rows loaded overflowed = %s",hasoverflowed?"true":"false");
#endif
            if (nrecs&&icompare)
                rows.sort(*icompare,isstable,maxcores);
            numrows += nrecs;
            totalsize += rows.totalSize();
            if (!hasoverflowed&&!alldisk)
                break;
#ifdef _FULL_TRACE
            PROGLOG("CThorRowSortedLoader spilling %u",(unsigned)rows.totalSize());
#endif
            alldisk = true;
            unsigned idx = newAuxFile();
            Owned<IExtRowWriter> writer = createRowWriter(&auxfiles.item(idx),rowif->queryRowSerializer(),rowif->queryRowAllocator(),false,false,false);
            rows.save(writer);
            writer->flush();
#ifdef _FULL_TRACE
            PROGLOG("CThorRowSortedLoader spilt");
#endif
            if (!hasoverflowed)
                break;
        }
        ForEachItemIn(i,auxfiles) {
#ifdef _FULL_TRACE
            PROGLOG("CThorRowSortedLoader spill file(%d) %s %"I64F"d",i,auxfiles.item(i).queryFilename(),auxfiles.item(i).size());
#endif
            instrms.append(*createSimpleRowStream(&auxfiles.item(i),rowif));
        }
Exemplo n.º 2
0
    offset_t write(IRowStream *input)
    {
        StringBuffer tempname;
        GetTempName(tempname,"srtmrg",false);
        dataFile.setown(createIFile(tempname.str()));
        Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);

        bool overflowed = false;
        ActPrintLog(&activity, "Local Overflow Merge start");
        unsigned ret=0;
        loop
        {
            const void *_row = input->nextRow();
            if (!_row)
                break;
            ret++;

            OwnedConstThorRow row = _row;
            offset_t start = output->getPosition();
            output->putRow(row.getLink());
            idx++;
            if (idx==interval)
            {
                idx = 0;
                if (!sampleRows.append(row.getClear()))
                {
                    // JCSMORE used to check if 'isFull()' here, but only to warn
                    // I think this is bad news, if has run out of room here...
                    // should at least warn in workunit I suspect
                    overflowsize = output->getPosition();
                    if (!overflowed)
                    {
                        WARNLOG("Sample buffer full");
                        overflowed = true;
                    }
                }
            }
            writeidxofs(start);
        }
        output->flush();
        offset_t end = output->getPosition();
        output.clear();
        writeidxofs(end);
        if (idxFileIO)
        {
            idxFileStream->flush();
            idxFileStream.clear();
            idxFileIO.clear();
        }
        if (overflowed)
            WARNLOG("Overflowed by %"I64F"d", overflowsize);
        ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
        return end;
    }