WorkerInfoQueueItem ProcessorQueue::TryGettingFittingJob(WorkerInfoQueue** curQ) { WorkerInfoQueueItem item; /*try to get an item from the work queue*/ WorkerInfoQueue * pQ = GetQueue(); assert(pQ); item = pQ->TryGetItem(); if (item.private_data != NULL){ *curQ = pQ; return item; } /*if heterogeneous execution try GPU Q if cpu queue was empty*/ if (useHeterogenousCompute()) { pQ = GetGpuQueue(); assert(pQ); item = pQ->TryGetItem(); if (item.private_data != NULL){ *curQ = pQ; return item; } } /*if both tries came up empty wait on cpu Q */ pQ = GetQueue(); assert(pQ); *curQ = pQ; return pQ->GetItem(); }
void *FileSDatLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options numWorkers = ( numWorkers < 1 ? 1:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker, loadWorkQ ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; int flow_buffer_size = master_img_loader->flow_buffer_size; for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); // cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift(); // cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset(); item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (!ChipIdDecoder::IsProtonChip()) //P2 and Long compute? PauseForLongCompute ( cur_flow,cur_image_loader ); } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *BkgFitWorkerCpu(void *arg) { ProcessorQueue* pq = static_cast<ProcessorQueue*>(arg); assert(pq); WorkerInfoQueue* curQ = NULL; bool done = false; WorkerInfoQueueItem item; while (!done) { //item = TryGettingFittingJobForCpuFromQueue(pq, &curQ); item = pq->TryGettingFittingJob(&curQ); if (item.finished == true) { // we are no longer needed...go away! done = true; curQ->DecrementDone(); continue; } int event = * ( (int *) item.private_data); if (event == MULTI_FLOW_REGIONAL_FIT) { DoMultiFlowRegionalFit(item); } else if (event == INITIAL_FLOW_BLOCK_ALLBEAD_FIT) { DoInitialBlockOfFlowsAllBeadFit(item); } else if (event == INITIAL_FLOW_BLOCK_REMAIN_REGIONAL_FIT) { DoInitialBlockOfFlowsRemainingRegionalFit(item); } else if (event == SINGLE_FLOW_FIT) { DoSingleFlowFitAndPostProcessing(item); } else if (event == POST_FIT_STEPS) { DoPostFitProcessing(item); } else if (event == imageInitBkgModel) { DoConstructSignalProcessingFitterAndData (item); } // indicate we finished that bit of work curQ->DecrementDone(); } return (NULL); }
void *FileSDatLoadWorker ( void *arg ) { WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg ); assert ( q ); bool done = false; while ( !done ) { WorkerInfoQueueItem item = q->GetItem(); if ( item.finished == true ) { // we are no longer needed...go away! done = true; q->DecrementDone(); continue; } ClockTimer timer; ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data; SynchDat &sdat = one_img_loader->sdat[one_img_loader->cur_buffer]; // int flow_buffer_for_flow = one_img_loader->cur_buffer; if ( one_img_loader->inception_state->img_control.col_flicker_correct ) { ComparatorNoiseCorrector cnc; Mask &mask = *(one_img_loader->mask); for (size_t rIx = 0; rIx < sdat.GetNumBin(); rIx++) { TraceChunk &chunk = sdat.GetChunk(rIx); // Copy over temp mask for normalization Mask m(chunk.mWidth, chunk.mHeight); for (size_t r = 0; r < chunk.mHeight; r++) { for (size_t c = 0; c < chunk.mWidth; c++) { m[r*chunk.mWidth+c] = mask[(r+chunk.mRowStart) * mask.W() + (c+chunk.mColStart)]; } } cnc.CorrectComparatorNoise(&chunk.mData[0], chunk.mHeight, chunk.mWidth, chunk.mDepth, &m, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } } // @todo output trace and dc offset info sdat.SubDcOffset(); SetReadCompleted(one_img_loader); size_t usec = timer.GetMicroSec(); fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow, usec / 1.0e6); q->DecrementDone(); } return ( NULL ); }
void *FileLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; prctl(PR_SET_NAME,"FileLoader",0,0,0); WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); // int numWorkers = 1; int numWorkers = numCores() /4; // @TODO - this should be subject to inception_state options numWorkers = ( numWorkers < 4 ? 4:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); WqInfo_t wq_info[numWorkers]; { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { wq_info[cworker].threadNum=cworker; wq_info[cworker].wq = loadWorkQ; int t = pthread_create ( &work_thread, NULL, FileLoadWorker, &wq_info[cworker] ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; //time_t start, end; int flow_buffer_size = master_img_loader->flow_buffer_size; // this loop goes over the individual image loaders for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order if (!cur_image_loader->inception_state->img_control.threaded_file_access) { JustLoadOneImageWithPinnedUpdate(cur_image_loader); } //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (!ChipIdDecoder::IsProtonChip()) PauseForLongCompute ( cur_flow,cur_image_loader ); } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *FileLoadWorker ( void *arg ) { WqInfo_t *wqinfo = (WqInfo_t *)arg; int threadNum=wqinfo->threadNum; WorkerInfoQueue *q = wqinfo->wq; assert ( q ); bool done = false; char dateStr[256]; struct tm newtime; time_t ltime; double T1=0,T2=0,T3,T4; char name[20]; sprintf(name,"FileLdWkr%d",threadNum); prctl(PR_SET_NAME,name,0,0,0); const double noiseThreshold = 0.; // to be set by command line option FluidPotentialCorrector fpCorr(noiseThreshold); while ( !done ) { WorkerInfoQueueItem item = q->GetItem(); if ( item.finished == true ) { // we are no longer needed...go away! done = true; q->DecrementDone(); continue; } ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data; ClockTimer timer; Timer tmr; Image *img = &one_img_loader->img[one_img_loader->cur_buffer]; if (one_img_loader->inception_state->img_control.threaded_file_access) { tmr.restart(); if ( !img->LoadRaw ( one_img_loader->name) ) { fprintf ( stdout, "ERROR: Failed to load raw image %s\n", one_img_loader->name ); exit ( EXIT_FAILURE ); } T1=tmr.elapsed(); tmr.restart(); if(img->raw->imageState & IMAGESTATE_QuickPinnedPixelDetect) one_img_loader->pinnedInFlow->QuickUpdate ( one_img_loader->flow, img); else one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, img,(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0)); T2=tmr.elapsed(); } tmr.restart(); // correct in-channel electrical cross-talk ImageTransformer::XTChannelCorrect ( one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->img[one_img_loader->cur_buffer].results_folder ); // buffer_ix // testing of lossy compression if(ImageTransformer::PCATest[0]) { AdvComprTest(one_img_loader->name,&one_img_loader->img[one_img_loader->cur_buffer],ImageTransformer::PCATest,false/*one_img_loader->inception_state->img_control.col_flicker_correct*/ ); } // col noise correction (if done during lossy compression will already have happened. else if ( !(img->raw->imageState & IMAGESTATE_ComparatorCorrected) && one_img_loader->inception_state->img_control.col_flicker_correct ) { if( one_img_loader->inception_state->img_control.col_pair_pixel_xtalk_correct ){ PairPixelXtalkCorrector xtalkCorrector; float xtalk_fraction = one_img_loader->inception_state->img_control.pair_xtalk_fraction; xtalkCorrector.Correct(one_img_loader->img[one_img_loader->cur_buffer].raw, xtalk_fraction); } if (one_img_loader->inception_state->img_control.corr_noise_correct){ CorrNoiseCorrector rnc; rnc.CorrectCorrNoise(one_img_loader->img[one_img_loader->cur_buffer].raw,3,one_img_loader->inception_state->bfd_control.beadfindThumbnail ); } if(one_img_loader->inception_state->bfd_control.beadfindThumbnail) { //ComparatorNoiseCorrector ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } else { ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose, one_img_loader->inception_state->img_control.aggressive_cnc,false,threadNum ); } } //#define DEBUG_IMAGE_CORR_ISSUES 1 #ifdef DEBUG_IMAGE_CORR_ISSUES char newName[1024]; char *nptr,*ptr=one_img_loader->name; while((nptr = strstr(ptr,"/")))ptr=nptr+1; if(*ptr == '/') ptr++; sprintf(newName,"%s_proc",one_img_loader->name); Acq saver; saver.SetData ( &one_img_loader->img[one_img_loader->cur_buffer] ); saver.WriteVFC(newName, 0, 0, one_img_loader->img[one_img_loader->cur_buffer].raw->cols, one_img_loader->img[one_img_loader->cur_buffer].raw->rows); #endif T3=tmr.elapsed(); tmr.restart(); // Fluid potential corrector const bool correctFluidPotential = one_img_loader->inception_state->img_control.fluid_potential_correct; const double noiseThreshold = (double) one_img_loader->inception_state->img_control.fluid_potential_threshold; if (correctFluidPotential){ fpCorr.setThreshold(noiseThreshold); // parse rowsum file name to load const std::string datFileName = one_img_loader->name; printf("dat file name %s\n", datFileName.c_str()); const size_t pos1 = datFileName.find_last_of('/'); const size_t pos2 = datFileName.find_last_of('.'); const std::string rowsumFileName = datFileName.substr(0, pos1) + "/../rowsum/" + datFileName.substr(pos1+1, pos2-pos1-1) + ".hwsum"; // printf("rowsum file name %s pos1 %u pos2 %u\n", rowsumFileName.c_str(), (uint32_t) pos1, (uint32_t) pos2); // determine if the data file is thumbnail or block const bool isThumbnail = one_img_loader->inception_state->bfd_control.beadfindThumbnail; if(isThumbnail){ fpCorr.setIsThumbnail(); } // set image const unsigned int regionXSize = one_img_loader->inception_state->loc_context.regionXSize; const unsigned int regionYSize = one_img_loader->inception_state->loc_context.regionYSize; const char nucChar = one_img_loader->inception_state->flow_context.ReturnNucForNthFlow(one_img_loader->flow); fpCorr.setImage(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, regionYSize, regionXSize, nucChar); //printf("nucChar is %c\n", nucChar); // load sensing electrode data if (isThumbnail){ const unsigned int numRows = one_img_loader->img[one_img_loader->cur_buffer].raw->rows; fpCorr.loadSensingElectrodeDataThumbnail(rowsumFileName, numRows); } else { // determine startRow and endRow one_img_loader->img[one_img_loader->cur_buffer].SetOffsetFromChipOrigin(one_img_loader->name); const unsigned int startRow = one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y; const unsigned int endRow = startRow + one_img_loader->img[one_img_loader->cur_buffer].raw->rows; // printf("offset x y = %d %d, start and end rows are %u,%u\n", one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_x,one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y, startRow, endRow); fpCorr.loadSensingElectrodeData(rowsumFileName, startRow, endRow); } // correct fluid potential if (fpCorr.readRowSumIsSuccess()){ fpCorr.doCorrection(); } else { printf("fluidPotentialCorrector skipped: Cannot find rowsum file %s \n", rowsumFileName.c_str()); } } // dump dc offset one_img_loaderrmation before we do any normalization DumpDcOffset ( one_img_loader ); int buffer_ix = one_img_loader->cur_buffer; // setting the mean of frames to zero will be done by the bkgmodel as soon as its loaded. // the traces will be zero'd by the bknd model loader anyway, no need to do them here. // img->SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd,0 ); // calculate the smooth pH step amplitude in empty wells across the whole image if ( one_img_loader->doEmptyWellNormalization ) { // this is redundant with what's in the EmptyTrace class, but this method of spatial normalization // is going to go away soon, so not worth fixing MaskType referenceMask = MaskReference; if ( one_img_loader->inception_state->bkg_control.trace_control.use_dud_and_empty_wells_as_reference ) referenceMask = ( MaskType ) ( MaskReference | MaskDud ); one_img_loader->img[buffer_ix].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span ); #if 0 char fname[512]; sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow ); FILE *ewampfile = fopen ( fname,"w" ); for ( int row=0;row < one_img_loader->img[buffer_ix].GetRows();row++ ) { int col; for ( col=0;col < ( one_img_loader->img[buffer_ix].GetCols()-1 ); col++ ) fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) ); fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) ); } fclose ( ewampfile ); #endif } // dump pH step debug one_img_loader to file (its really fast, don't worry) DumpStep ( one_img_loader ); if ( one_img_loader->doRawBkgSubtract ) { one_img_loader->img[buffer_ix].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx, one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true ); if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer printf ( "Notify user: NN empty subtraction in effect. No further warnings will be given. %d \n",one_img_loader->flow ); } T4=tmr.elapsed(); // printf ( "Allow model to go %d \n", one_img_loader->flow ); SetReadCompleted(one_img_loader); size_t usec = timer.GetMicroSec(); ltime=time(<ime); localtime_r(<ime, &newtime); strftime(dateStr,sizeof(dateStr),"%H:%M:%S", &newtime); fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.2lf(ld=%.2f pin=%.2f cnc=%.2f xt=%.2f sem=%.2lf cache=%.2lf) sec %s\n", one_img_loader->flow , usec / 1.0e6, T1, T2, T3, T4, img->SemaphoreWaitTime, img->CacheAccessTime, dateStr); fflush(stdout); fprintf(stdout, "File: %s\n", one_img_loader->name); fflush(stdout); q->DecrementDone(); } return ( NULL ); }
void *FileSDatLoadWorker ( void *arg ) { WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg ); assert ( q ); bool done = false; TraceChunkSerializer serializer; while ( !done ) { WorkerInfoQueueItem item = q->GetItem(); if ( item.finished == true ) { // we are no longer needed...go away! done = true; q->DecrementDone(); continue; } ClockTimer timer; ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data; bool ok = serializer.Read ( one_img_loader->name, one_img_loader->sdat[one_img_loader->cur_buffer] ); if (!ok) { ION_ABORT("Couldn't load file: " + ToStr(one_img_loader->name)); } one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer],(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0)); // one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer] ); SynchDat &sdat = one_img_loader->sdat[one_img_loader->cur_buffer]; // if ( ImageTransformer::gain_correction != NULL ) // ImageTransformer::GainCorrectImage ( &sdat ); // ImageTransformer::GainCorrectImage ( &one_img_loader->sdat[one_img_loader->cur_buffer] ); // int buffer_ix = one_img_loader->cur_buffer; if ( one_img_loader->inception_state->img_control.col_flicker_correct ) { ComparatorNoiseCorrector cnc; Mask &mask = *(one_img_loader->mask); for (size_t rIx = 0; rIx < sdat.GetNumBin(); rIx++) { TraceChunk &chunk = sdat.GetChunk(rIx); // Copy over temp mask for normalization Mask m(chunk.mWidth, chunk.mHeight); for (size_t r = 0; r < chunk.mHeight; r++) { for (size_t c = 0; c < chunk.mWidth; c++) { m[r*chunk.mWidth+c] = mask[(r+chunk.mRowStart) * mask.W() + (c+chunk.mColStart)]; } } cnc.CorrectComparatorNoise(&chunk.mData[0], chunk.mHeight, chunk.mWidth, chunk.mDepth, &m, one_img_loader->inception_state->img_control.col_flicker_correct_verbose, one_img_loader->inception_state->img_control.aggressive_cnc); } } // @todo output trace and dc offset info sdat.AdjustForDrift(); sdat.SubDcOffset(); SetReadCompleted(one_img_loader); size_t usec = timer.GetMicroSec(); fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow, usec / 1.0e6); q->DecrementDone(); } return ( NULL ); }
void *FileSDatLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options //int numWorkers = 1; numWorkers = ( numWorkers < 1 ? 1:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker, loadWorkQ ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; Timer timer_file_access; //double file_access_time = 0; int flow_buffer_size = master_img_loader->flow_buffer_size; for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); TraceChunkSerializer serializer; timer_file_access.restart(); bool ok = serializer.Read ( cur_image_loader->name, cur_image_loader->sdat[cur_image_loader->cur_buffer] ); if (!ok) { ION_ABORT("Couldn't load file: " + ToStr(cur_image_loader->name)); } // if ( ImageTransformer::gain_correction != NULL ) // ImageTransformer::GainCorrectImage ( &cur_image_loader->sdat[cur_image_loader->cur_buffer] ); //file_access_time += timer_file_access.elapsed(); fprintf ( stdout, "File access = %0.2lf sec.\n", timer_file_access.elapsed() ); cur_image_loader->pinnedInFlow->Update ( cur_image_loader->flow, &cur_image_loader->sdat[cur_image_loader->cur_buffer] ); cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift(); cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset(); item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (ChipIdDecoder::GetGlobalChipId() != ChipId900) PauseForLongCompute ( cur_flow,cur_image_loader ); /*if ( CheckFlowForWrite ( cur_flow,false ) ) { fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time); file_access_time = 0; }*/ } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *FileLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options // int numWorkers = 1; numWorkers = ( numWorkers < 1 ? 1:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { int t = pthread_create ( &work_thread, NULL, FileLoadWorker, loadWorkQ ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; //time_t start, end; int flow_buffer_size = master_img_loader->flow_buffer_size; Timer timer_file_access; double file_access_time = 0; // this loop goes over the individual image loaders for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order timer_file_access.restart(); if (!cur_image_loader->inception_state->img_control.threaded_file_access) { JustLoadOneImageWithPinnedUpdate(cur_image_loader); } else { // JustCacheOneImage(cur_image_loader); } file_access_time += timer_file_access.elapsed(); //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (ChipIdDecoder::GetGlobalChipId() != ChipId900) PauseForLongCompute ( cur_flow,cur_image_loader ); if ( CheckFlowForWrite ( cur_flow,false ) ) { fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time); file_access_time = 0; } } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *FileLoadWorker ( void *arg ) { WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg ); assert ( q ); bool done = false; while ( !done ) { WorkerInfoQueueItem item = q->GetItem(); if ( item.finished == true ) { // we are no longer needed...go away! done = true; q->DecrementDone(); continue; } ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data; ClockTimer timer; if (one_img_loader->inception_state->img_control.threaded_file_access) JustLoadOneImageWithPinnedUpdate(one_img_loader); // col noise correction if ( one_img_loader->inception_state->img_control.col_flicker_correct ) { if(one_img_loader->inception_state->bfd_control.beadfindThumbnail) { //ComparatorNoiseCorrector ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } else { ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } } // dump dc offset one_img_loaderrmation before we do any normalization DumpDcOffset ( one_img_loader ); int flow_buffer_for_flow = one_img_loader->cur_buffer; one_img_loader->img[flow_buffer_for_flow].SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd ); // correct in-channel electrical cross-talk ImageTransformer::XTChannelCorrect ( one_img_loader->img[flow_buffer_for_flow].raw, one_img_loader->img[flow_buffer_for_flow].results_folder ); // calculate the smooth pH step amplitude in empty wells across the whole image if ( one_img_loader->doEmptyWellNormalization ) { // this is redundant with what's in the EmptyTrace class, but this method of spatial normalization // is going to go away soon, so not worth fixing MaskType referenceMask = MaskReference; if ( one_img_loader->inception_state->bkg_control.use_dud_and_empty_wells_as_reference ) referenceMask = ( MaskType ) ( MaskReference | MaskDud ); one_img_loader->img[flow_buffer_for_flow].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span ); #if 0 char fname[512]; sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow ); FILE *ewampfile = fopen ( fname,"w" ); for ( int row=0;row < one_img_loader->img[flow_buffer_for_flow].GetRows();row++ ) { int col; for ( col=0;col < ( one_img_loader->img[flow_buffer_for_flow].GetCols()-1 ); col++ ) fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) ); fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) ); } fclose ( ewampfile ); #endif } // dump pH step debug one_img_loader to file (its really fast, don't worry) DumpStep ( one_img_loader ); if ( one_img_loader->doRawBkgSubtract ) { one_img_loader->img[flow_buffer_for_flow].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx, one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true ); if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer printf ( "Notify user: NN empty subtraction in effect. No further warnings will be given. %d \n",one_img_loader->flow ); } printf ( "Allow model to go %d\n", one_img_loader->flow ); SetReadCompleted(one_img_loader); size_t usec = timer.GetMicroSec(); fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow , usec / 1.0e6); q->DecrementDone(); } return ( NULL ); }