Ejemplo n.º 1
0
 virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
 {
     PARENT::setInputStream(index, _input, consumerOrdered);
     // JCSMORE - not sure why you ever want a look ahead on a sink like this?
     if (!isFastThrough(input))
         setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
 }
Ejemplo n.º 2
0
 void start()
 {
     ActivityTimer s(totalCycles, timeActivities, NULL);
     dataLinkStart();
     input = inputs.item(0);
     unsigned spillPriority = container.queryGrouped() ? SPILL_PRIORITY_GROUPSORT : SPILL_PRIORITY_LARGESORT;
     iLoader.setown(createThorRowLoader(*this, queryRowInterfaces(input), iCompare, unstable ? stableSort_none : stableSort_earlyAlloc, rc_mixed, spillPriority));
     startInput(input);
     eoi = false;
     if (container.queryGrouped())
         out.setown(iLoader->loadGroup(input, abortSoon));
     else
         out.setown(iLoader->load(input, abortSoon));
     if (0 == iLoader->numRows())
         eoi = true;
 }
Ejemplo n.º 3
0
 virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
 {
     PARENT::setInputStream(index, _input, consumerOrdered);
     setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
 }
Ejemplo n.º 4
0
 void start()
 {
     ActivityTimer s(totalCycles, timeActivities);
     input = inputs.item(0);
     try
     {
         try { 
             startInput(input); 
         }
         catch (IException *e)
         {
             fireException(e);
             barrier->cancel();
             throw;
         }
         catch (CATCHALL)
         {
             Owned<IException> e = MakeActivityException(this, 0, "Unknown exception starting sort input");
             fireException(e);
             barrier->cancel();
             throw;
         }
         dataLinkStart();
         
         Linked<IRowInterfaces> rowif = queryRowInterfaces(input);
         Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
         sorter->Gather(
             rowif,
             input,
             helper->queryCompare(),
             helper->queryCompareLeftRight(),
             NULL,helper->querySerialize(),
             NULL,
             false,
             isUnstable(),
             abortSoon,
             auxrowif);
         stopInput(input);
         input = NULL;
         if (abortSoon)
         {
             ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting");
             barrier->cancel();
             return;
         }
     }
     catch (IException *e)
     {
         fireException(e);
         barrier->cancel();
         throw;
     }
     catch (CATCHALL)
     {
         Owned<IException> e = MakeActivityException(this, 0, "Unknown exception gathering sort input");
         fireException(e);
         barrier->cancel();
         throw;
     }
     ActPrintLog("SORT waiting barrier.1");
     if (!barrier->wait(false)) {
         Sleep(1000); // let original error through
         throw MakeThorException(TE_BarrierAborted,"SORT: Barrier Aborted");
     }
     ActPrintLog("SORT barrier.1 raised");
     output.setown(sorter->startMerge(totalrows));
 }
Ejemplo n.º 5
0
 virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
 {
     PARENT::setInputStream(index, _input, consumerOrdered);
     if (global) // only want lookahead if global (hence serial)
         setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
 }
Ejemplo n.º 6
0
 int run()
 {
     if (!started) {
         try {
             in->start();
             started = true;
         }
         catch(IException * e)
         {
             ActPrintLog(&activity, e, "ThorLookaheadCache starting input");
             startexception.setown(e);
             if (asyncstart) 
                 notify->onInputStarted(startexception);
             running = false;
             stopped = true;
             startsem.signal();
             return 0;
         }
     }
     try {
         StringBuffer temp;
         if (allowspill)
             GetTempName(temp,"lookahd",true);
         assertex(bufsize);
         if (allowspill)
             smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in)));
         else
             smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize));
         if (notify) 
             notify->onInputStarted(NULL);
         startsem.signal();
         Linked<IRowWriter> writer = smartbuf->queryWriter();
         if (preserveLhsGrouping)
         {
             while (required&&running)
             {
                 OwnedConstThorRow row = in->nextRow();
                 if (!row)
                 {
                     row.setown(in->nextRow());
                     if (!row)
                         break;
                     else
                         writer->putRow(NULL); // eog
                 }
                 ++count;
                 writer->putRow(row.getClear());
                 if (required!=RCUNBOUND)
                     required--;
             }
         }
         else
         {
             while (required&&running)
             {
                 OwnedConstThorRow row = in->ungroupedNextRow();
                 if (!row)
                     break;
                 ++count;
                 writer->putRow(row.getClear());
                 if (required!=RCUNBOUND)
                     required--;
             }
         }
     }
     catch(IException * e)
     {
         ActPrintLog(&activity, e, "ThorLookaheadCache get exception");
         getexception.setown(e);
     }   
     if (notify)
         notify->onInputFinished(count);
     if (smartbuf)
         smartbuf->queryWriter()->flush();
     running = false;
     try {
         if (in)
             in->stop();
     }
     catch(IException * e)
     {
         ActPrintLog(&activity, e, "ThorLookaheadCache stop exception");
         if (!getexception.get())
             getexception.setown(e);
     }   
     return 0;
 }
Ejemplo n.º 7
0
 virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
 {
     PARENT::setInputStream(index, _input, consumerOrdered);
     inputCounter->setInputStream(inputStream);
     setLookAhead(0, createRowStreamLookAhead(this, inputCounter.get(), queryRowInterfaces(input), CHOOSESETSPLUS_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage())); // read all input
 }
Ejemplo n.º 8
0
 virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
 {
     PARENT::setInputStream(index, _input, consumerOrdered);
     if (!isFastThrough(input))
         setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), CHOOSESETS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
 }
Ejemplo n.º 9
0
void CDiskWriteSlaveActivityBase::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered)
{
    PARENT::setInputStream(index, _input, consumerOrdered);
    if (dlfn.isExternal() && !firstNode())
        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
}