WorkerInfoQueueItem ProcessorQueue::TryGettingFittingJob(WorkerInfoQueue** curQ)
{
  WorkerInfoQueueItem item;

  /*try to get an item from the work queue*/
  WorkerInfoQueue * pQ = GetQueue();
  assert(pQ);
  item = pQ->TryGetItem();
  if (item.private_data != NULL){
    *curQ = pQ;
    return item;
  }
  /*if heterogeneous execution try GPU Q if cpu queue was empty*/
  if (useHeterogenousCompute())
  {
    pQ = GetGpuQueue();
    assert(pQ);
    item = pQ->TryGetItem();
    if (item.private_data != NULL){
      *curQ = pQ;
      return item;
    }
  }

  /*if both tries came up empty wait on cpu Q */
  pQ = GetQueue();
  assert(pQ);
  *curQ = pQ;
  return pQ->GetItem();

}
Example #2
0
void *FileSDatLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;
  int flow_buffer_size = master_img_loader->flow_buffer_size;
  for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);


    //    cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
    //    cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (!ChipIdDecoder::IsProtonChip())  //P2 and Long compute?
      PauseForLongCompute ( cur_flow,cur_image_loader );
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();

  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;

  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
void *BkgFitWorkerCpu(void *arg)
{
  ProcessorQueue* pq = static_cast<ProcessorQueue*>(arg);
  assert(pq);

  WorkerInfoQueue* curQ = NULL;
  bool done = false;
  WorkerInfoQueueItem item;
  while (!done)
  {
    //item = TryGettingFittingJobForCpuFromQueue(pq, &curQ);
    item = pq->TryGettingFittingJob(&curQ);
    if (item.finished == true)
    {
      // we are no longer needed...go away!
      done = true;
      curQ->DecrementDone();
      continue;
    }

    int event = * ( (int *) item.private_data);

    if (event == MULTI_FLOW_REGIONAL_FIT)
    {
      DoMultiFlowRegionalFit(item);
    }
    else if (event == INITIAL_FLOW_BLOCK_ALLBEAD_FIT)
    {
      DoInitialBlockOfFlowsAllBeadFit(item);
    }
    else if (event == INITIAL_FLOW_BLOCK_REMAIN_REGIONAL_FIT)
    {
      DoInitialBlockOfFlowsRemainingRegionalFit(item);
    }
    else if (event == SINGLE_FLOW_FIT) {
      DoSingleFlowFitAndPostProcessing(item);
    }
    else if (event == POST_FIT_STEPS) {
      DoPostFitProcessing(item);
    }
    else if (event == imageInitBkgModel)
    {
      DoConstructSignalProcessingFitterAndData (item);
    }

    // indicate we finished that bit of work
    curQ->DecrementDone();
  }

  return (NULL);
}
Example #4
0
void *FileSDatLoadWorker ( void *arg )
{
  WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg );
  assert ( q );

  bool done = false;

  while ( !done )
  {
    WorkerInfoQueueItem item = q->GetItem();

    if ( item.finished == true )
    {
      // we are no longer needed...go away!
      done = true;
      q->DecrementDone();
      continue;
    }
    ClockTimer timer;
    ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;
    SynchDat &sdat = one_img_loader->sdat[one_img_loader->cur_buffer];

    //    int flow_buffer_for_flow = one_img_loader->cur_buffer;
    if ( one_img_loader->inception_state->img_control.col_flicker_correct ) {
      ComparatorNoiseCorrector cnc;
      Mask &mask = *(one_img_loader->mask);
      for (size_t rIx = 0; rIx < sdat.GetNumBin(); rIx++) {
        TraceChunk &chunk = sdat.GetChunk(rIx);
        // Copy over temp mask for normalization
        Mask m(chunk.mWidth, chunk.mHeight);
        for (size_t r = 0; r < chunk.mHeight; r++) {
          for (size_t c = 0; c < chunk.mWidth; c++) {
            m[r*chunk.mWidth+c] = mask[(r+chunk.mRowStart) * mask.W() + (c+chunk.mColStart)];
          }
        }
        cnc.CorrectComparatorNoise(&chunk.mData[0], chunk.mHeight, chunk.mWidth, chunk.mDepth, 
                                   &m, one_img_loader->inception_state->img_control.col_flicker_correct_verbose);
      } 
    }
    // @todo output trace and dc offset info
    sdat.SubDcOffset();
    SetReadCompleted(one_img_loader);
    size_t usec = timer.GetMicroSec();
    fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow, usec / 1.0e6);

    q->DecrementDone();
  }

  return ( NULL );
}
Example #5
0
void *FileLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;

  prctl(PR_SET_NAME,"FileLoader",0,0,0);


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );


  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );


  //   int numWorkers = 1;
  int numWorkers = numCores() /4; // @TODO - this should be subject to inception_state options
  numWorkers = ( numWorkers < 4 ? 4:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  WqInfo_t wq_info[numWorkers];

  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      wq_info[cworker].threadNum=cworker;
      wq_info[cworker].wq = loadWorkQ;
      int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
                               &wq_info[cworker] );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;

  //time_t start, end;
  int flow_buffer_size = master_img_loader->flow_buffer_size;

  // this loop goes over the individual image loaders
  for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow;  // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
    if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
      JustLoadOneImageWithPinnedUpdate(cur_image_loader);
    }
    //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion

    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (!ChipIdDecoder::IsProtonChip())
      PauseForLongCompute ( cur_flow,cur_image_loader );
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();
  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;
  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #6
0
void *FileLoadWorker ( void *arg )
{
  WqInfo_t *wqinfo = (WqInfo_t *)arg;
  int threadNum=wqinfo->threadNum;
  WorkerInfoQueue *q = wqinfo->wq;
  assert ( q );

  bool done = false;
  char dateStr[256];
  struct tm newtime;
  time_t ltime;
  double T1=0,T2=0,T3,T4;

  char name[20];
  sprintf(name,"FileLdWkr%d",threadNum);
  prctl(PR_SET_NAME,name,0,0,0);

  const double noiseThreshold = 0.;  // to be set by command line option
  FluidPotentialCorrector fpCorr(noiseThreshold);

  while ( !done )
  {
    WorkerInfoQueueItem item = q->GetItem();

    if ( item.finished == true )
    {
      // we are no longer needed...go away!
      done = true;
      q->DecrementDone();
      continue;
    }
    ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;

    ClockTimer timer;
    Timer tmr;
    Image *img = &one_img_loader->img[one_img_loader->cur_buffer];

    if (one_img_loader->inception_state->img_control.threaded_file_access)
    {
      tmr.restart();
      if ( !img->LoadRaw ( one_img_loader->name) )
      {
        fprintf ( stdout, "ERROR: Failed to load raw image %s\n", one_img_loader->name );
        exit ( EXIT_FAILURE );
      }

      T1=tmr.elapsed();
      tmr.restart();
      if(img->raw->imageState & IMAGESTATE_QuickPinnedPixelDetect)
        one_img_loader->pinnedInFlow->QuickUpdate ( one_img_loader->flow, img);
      else
        one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, img,(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0));

      T2=tmr.elapsed();
    }

    tmr.restart();


    // correct in-channel electrical cross-talk
    ImageTransformer::XTChannelCorrect ( one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->img[one_img_loader->cur_buffer].results_folder ); // buffer_ix

    // testing of lossy compression
    if(ImageTransformer::PCATest[0]) {
      AdvComprTest(one_img_loader->name,&one_img_loader->img[one_img_loader->cur_buffer],ImageTransformer::PCATest,false/*one_img_loader->inception_state->img_control.col_flicker_correct*/ );
    }
    // col noise correction (if done during lossy compression will already have happened.
    else if ( !(img->raw->imageState & IMAGESTATE_ComparatorCorrected) &&
              one_img_loader->inception_state->img_control.col_flicker_correct )
    {
        if( one_img_loader->inception_state->img_control.col_pair_pixel_xtalk_correct ){
            PairPixelXtalkCorrector xtalkCorrector;
            float xtalk_fraction = one_img_loader->inception_state->img_control.pair_xtalk_fraction;
            xtalkCorrector.Correct(one_img_loader->img[one_img_loader->cur_buffer].raw, xtalk_fraction);
        }
        if (one_img_loader->inception_state->img_control.corr_noise_correct){
      	  CorrNoiseCorrector rnc;
      	  rnc.CorrectCorrNoise(one_img_loader->img[one_img_loader->cur_buffer].raw,3,one_img_loader->inception_state->bfd_control.beadfindThumbnail );
        }


      if(one_img_loader->inception_state->bfd_control.beadfindThumbnail)
      {
        //ComparatorNoiseCorrector
        ComparatorNoiseCorrector cnc;
        cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose);
      } else {
        ComparatorNoiseCorrector cnc;
        cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose, one_img_loader->inception_state->img_control.aggressive_cnc,false,threadNum );
      }
    }
//#define DEBUG_IMAGE_CORR_ISSUES 1
#ifdef DEBUG_IMAGE_CORR_ISSUES

        char newName[1024];
        char *nptr,*ptr=one_img_loader->name;
        while((nptr = strstr(ptr,"/")))ptr=nptr+1;
        if(*ptr == '/')
        	ptr++;
        sprintf(newName,"%s_proc",one_img_loader->name);

        Acq saver;
        saver.SetData ( &one_img_loader->img[one_img_loader->cur_buffer] );
        saver.WriteVFC(newName, 0, 0, one_img_loader->img[one_img_loader->cur_buffer].raw->cols, one_img_loader->img[one_img_loader->cur_buffer].raw->rows);
#endif
    T3=tmr.elapsed();
    tmr.restart();

    // Fluid potential corrector
    const bool correctFluidPotential = one_img_loader->inception_state->img_control.fluid_potential_correct;
    const double noiseThreshold = (double) one_img_loader->inception_state->img_control.fluid_potential_threshold;
    if (correctFluidPotential){
        fpCorr.setThreshold(noiseThreshold);

                // parse rowsum file name to load
                const std::string datFileName = one_img_loader->name;
                printf("dat file name %s\n", datFileName.c_str());
                const size_t pos1 = datFileName.find_last_of('/');
                const size_t pos2 = datFileName.find_last_of('.');
                const std::string rowsumFileName = datFileName.substr(0, pos1) + "/../rowsum/" + datFileName.substr(pos1+1, pos2-pos1-1) + ".hwsum";
                // printf("rowsum file name %s pos1 %u pos2 %u\n", rowsumFileName.c_str(), (uint32_t) pos1, (uint32_t) pos2);


                // determine if the data file is thumbnail or block
        const bool isThumbnail = one_img_loader->inception_state->bfd_control.beadfindThumbnail;
        if(isThumbnail){
                fpCorr.setIsThumbnail();
        }

        // set image
        const unsigned int regionXSize = one_img_loader->inception_state->loc_context.regionXSize;
        const unsigned int regionYSize = one_img_loader->inception_state->loc_context.regionYSize;
        const char nucChar = one_img_loader->inception_state->flow_context.ReturnNucForNthFlow(one_img_loader->flow);
        fpCorr.setImage(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask,  regionYSize, regionXSize, nucChar);
        //printf("nucChar is %c\n", nucChar);


        // load sensing electrode data
        if (isThumbnail){
                const unsigned int numRows = one_img_loader->img[one_img_loader->cur_buffer].raw->rows;
                fpCorr.loadSensingElectrodeDataThumbnail(rowsumFileName, numRows);

        } else {
                // determine startRow and endRow
                one_img_loader->img[one_img_loader->cur_buffer].SetOffsetFromChipOrigin(one_img_loader->name);
                const unsigned int startRow = one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y;
                const unsigned int endRow = startRow + one_img_loader->img[one_img_loader->cur_buffer].raw->rows;
       //         printf("offset x y = %d %d, start and end rows are %u,%u\n", one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_x,one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y, startRow, endRow);
                fpCorr.loadSensingElectrodeData(rowsumFileName, startRow, endRow);
        }

        // correct fluid potential
         if (fpCorr.readRowSumIsSuccess()){
                 fpCorr.doCorrection();
         } else {
                 printf("fluidPotentialCorrector skipped:  Cannot find rowsum file %s \n", rowsumFileName.c_str());
         }

    }


    // dump dc offset one_img_loaderrmation before we do any normalization
    DumpDcOffset ( one_img_loader );
    int buffer_ix = one_img_loader->cur_buffer;

    // setting the mean of frames to zero will be done by the bkgmodel as soon as its loaded.

// the traces will be zero'd by the bknd model loader anyway, no need to do them here.
 //   img->SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd,0 );


    // calculate the smooth pH step amplitude in empty wells across the whole image
    if ( one_img_loader->doEmptyWellNormalization )
    {
      // this is redundant with what's in the EmptyTrace class, but this method of spatial normalization
      // is going to go away soon, so not worth fixing
      MaskType referenceMask = MaskReference;

      if ( one_img_loader->inception_state->bkg_control.trace_control.use_dud_and_empty_wells_as_reference )
        referenceMask = ( MaskType ) ( MaskReference | MaskDud );

      one_img_loader->img[buffer_ix].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span );
#if 0
      char fname[512];
      sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow );
      FILE *ewampfile = fopen ( fname,"w" );
      for ( int row=0;row < one_img_loader->img[buffer_ix].GetRows();row++ )
      {
        int col;

        for ( col=0;col < ( one_img_loader->img[buffer_ix].GetCols()-1 ); col++ )
          fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) );

        fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) );
      }
      fclose ( ewampfile );
#endif
    }

    // dump pH step debug one_img_loader to file (its really fast, don't worry)
    DumpStep ( one_img_loader );
    if ( one_img_loader->doRawBkgSubtract )
    {
      one_img_loader->img[buffer_ix].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx,
                                                                   one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true );
      if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer
        printf ( "Notify user: NN empty subtraction in effect.  No further warnings will be given. %d \n",one_img_loader->flow );
    }
    T4=tmr.elapsed();

    //    printf ( "Allow model to go %d \n", one_img_loader->flow );

    SetReadCompleted(one_img_loader);

    size_t usec = timer.GetMicroSec();

    ltime=time(&ltime);
    localtime_r(&ltime, &newtime);
    strftime(dateStr,sizeof(dateStr),"%H:%M:%S", &newtime);

    fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.2lf(ld=%.2f pin=%.2f cnc=%.2f xt=%.2f sem=%.2lf cache=%.2lf) sec %s\n",
              one_img_loader->flow , usec / 1.0e6, T1, T2, T3, T4, img->SemaphoreWaitTime, img->CacheAccessTime, dateStr);
    fflush(stdout);
    fprintf(stdout, "File: %s\n", one_img_loader->name);
    fflush(stdout);
    q->DecrementDone();
  }

  return ( NULL );
}
Example #7
0
void *FileSDatLoadWorker ( void *arg )
{
  WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg );
  assert ( q );

  bool done = false;
  TraceChunkSerializer serializer;
  while ( !done )
  {
    WorkerInfoQueueItem item = q->GetItem();

    if ( item.finished == true )
    {
      // we are no longer needed...go away!
      done = true;
      q->DecrementDone();
      continue;
    }
    ClockTimer timer;
    ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;
    bool ok = serializer.Read ( one_img_loader->name, one_img_loader->sdat[one_img_loader->cur_buffer] );
    if (!ok) {
      ION_ABORT("Couldn't load file: " + ToStr(one_img_loader->name));
    }
    one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer],(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0));
    //    one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer] );

    SynchDat &sdat = one_img_loader->sdat[one_img_loader->cur_buffer];
    // if ( ImageTransformer::gain_correction != NULL )
    //   ImageTransformer::GainCorrectImage ( &sdat );
    //   ImageTransformer::GainCorrectImage ( &one_img_loader->sdat[one_img_loader->cur_buffer] );

    //    int buffer_ix = one_img_loader->cur_buffer;
    if ( one_img_loader->inception_state->img_control.col_flicker_correct ) {
      ComparatorNoiseCorrector cnc;
      Mask &mask = *(one_img_loader->mask);
      for (size_t rIx = 0; rIx < sdat.GetNumBin(); rIx++) {
        TraceChunk &chunk = sdat.GetChunk(rIx);
        // Copy over temp mask for normalization
        Mask m(chunk.mWidth, chunk.mHeight);
        for (size_t r = 0; r < chunk.mHeight; r++) {
          for (size_t c = 0; c < chunk.mWidth; c++) {
            m[r*chunk.mWidth+c] = mask[(r+chunk.mRowStart) * mask.W() + (c+chunk.mColStart)];
          }
        }
        cnc.CorrectComparatorNoise(&chunk.mData[0], chunk.mHeight, chunk.mWidth, chunk.mDepth,
            &m, one_img_loader->inception_state->img_control.col_flicker_correct_verbose,
            one_img_loader->inception_state->img_control.aggressive_cnc);
      }
    }
    // @todo output trace and dc offset info
    sdat.AdjustForDrift();
    sdat.SubDcOffset();
    SetReadCompleted(one_img_loader);
    size_t usec = timer.GetMicroSec();
    fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow, usec / 1.0e6);

    q->DecrementDone();
  }

  return ( NULL );
}
Example #8
0
void *FileSDatLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  //int numWorkers = 1;
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;
  Timer timer_file_access;
  //double file_access_time = 0;
  int flow_buffer_size = master_img_loader->flow_buffer_size;
  for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    TraceChunkSerializer serializer;
    timer_file_access.restart();
    bool ok = serializer.Read ( cur_image_loader->name, cur_image_loader->sdat[cur_image_loader->cur_buffer] );
    if (!ok) {
      ION_ABORT("Couldn't load file: " + ToStr(cur_image_loader->name));
    }
    // if ( ImageTransformer::gain_correction != NULL )
    //   ImageTransformer::GainCorrectImage ( &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
  
    //file_access_time += timer_file_access.elapsed();
    fprintf ( stdout, "File access = %0.2lf sec.\n", timer_file_access.elapsed() );
    cur_image_loader->pinnedInFlow->Update ( cur_image_loader->flow, &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
    cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
    cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );
    
    if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
      PauseForLongCompute ( cur_flow,cur_image_loader );

    /*if ( CheckFlowForWrite ( cur_flow,false ) )
    {
      fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
      file_access_time = 0;
    }*/
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();

  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;

  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #9
0
void *FileLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );


  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  // int numWorkers = 1;
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;

  //time_t start, end;
  int flow_buffer_size = master_img_loader->flow_buffer_size;

  Timer timer_file_access;
  double file_access_time = 0;
  // this loop goes over the individual image loaders
  for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow;  // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
    timer_file_access.restart();
    if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
      JustLoadOneImageWithPinnedUpdate(cur_image_loader);
    }
    else {
//      JustCacheOneImage(cur_image_loader);
    }
    file_access_time += timer_file_access.elapsed();
    //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion

    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
      PauseForLongCompute ( cur_flow,cur_image_loader );

    if ( CheckFlowForWrite ( cur_flow,false ) )
    {
      fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
      file_access_time = 0;
    }
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();
  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;
  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #10
0
void *FileLoadWorker ( void *arg )
{
  WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg );
  assert ( q );

  bool done = false;

  while ( !done )
  {
    WorkerInfoQueueItem item = q->GetItem();

    if ( item.finished == true )
    {
      // we are no longer needed...go away!
      done = true;
      q->DecrementDone();
      continue;
    }
    ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;

    ClockTimer timer;

    if (one_img_loader->inception_state->img_control.threaded_file_access)
        JustLoadOneImageWithPinnedUpdate(one_img_loader);

    // col noise correction
    if ( one_img_loader->inception_state->img_control.col_flicker_correct )
    {
        if(one_img_loader->inception_state->bfd_control.beadfindThumbnail)
        {
          //ComparatorNoiseCorrector
          ComparatorNoiseCorrector cnc;
          cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose);
        } else {
            ComparatorNoiseCorrector cnc;
            cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose);
        }
    }

    // dump dc offset one_img_loaderrmation before we do any normalization
    DumpDcOffset ( one_img_loader );
    int flow_buffer_for_flow = one_img_loader->cur_buffer;

    one_img_loader->img[flow_buffer_for_flow].SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd );

    // correct in-channel electrical cross-talk
    ImageTransformer::XTChannelCorrect ( one_img_loader->img[flow_buffer_for_flow].raw, one_img_loader->img[flow_buffer_for_flow].results_folder );
    // calculate the smooth pH step amplitude in empty wells across the whole image
    if ( one_img_loader->doEmptyWellNormalization )
    {
      // this is redundant with what's in the EmptyTrace class, but this method of spatial normalization
      // is going to go away soon, so not worth fixing
      MaskType referenceMask = MaskReference;

      if ( one_img_loader->inception_state->bkg_control.use_dud_and_empty_wells_as_reference )
        referenceMask = ( MaskType ) ( MaskReference | MaskDud );

      one_img_loader->img[flow_buffer_for_flow].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span );
#if 0
      char fname[512];
      sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow );
      FILE *ewampfile = fopen ( fname,"w" );
      for ( int row=0;row < one_img_loader->img[flow_buffer_for_flow].GetRows();row++ )
      {
        int col;

        for ( col=0;col < ( one_img_loader->img[flow_buffer_for_flow].GetCols()-1 ); col++ )
          fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) );

        fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) );
      }
      fclose ( ewampfile );
#endif
    }

    // dump pH step debug one_img_loader to file (its really fast, don't worry)
    DumpStep ( one_img_loader );
    if ( one_img_loader->doRawBkgSubtract )
    {
      one_img_loader->img[flow_buffer_for_flow].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx,
          one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true );
      if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer
        printf ( "Notify user: NN empty subtraction in effect.  No further warnings will be given. %d \n",one_img_loader->flow );
    }

    printf ( "Allow model to go %d\n", one_img_loader->flow );

    SetReadCompleted(one_img_loader);

    size_t usec = timer.GetMicroSec();

    fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow , usec / 1.0e6);

    q->DecrementDone();
  }

  return ( NULL );
}