void *FileLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options // int numWorkers = 1; numWorkers = ( numWorkers < 1 ? 1:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { int t = pthread_create ( &work_thread, NULL, FileLoadWorker, loadWorkQ ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; //time_t start, end; int flow_buffer_size = master_img_loader->flow_buffer_size; Timer timer_file_access; double file_access_time = 0; // this loop goes over the individual image loaders for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order timer_file_access.restart(); if (!cur_image_loader->inception_state->img_control.threaded_file_access) { JustLoadOneImageWithPinnedUpdate(cur_image_loader); } else { // JustCacheOneImage(cur_image_loader); } file_access_time += timer_file_access.elapsed(); //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (ChipIdDecoder::GetGlobalChipId() != ChipId900) PauseForLongCompute ( cur_flow,cur_image_loader ); if ( CheckFlowForWrite ( cur_flow,false ) ) { fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time); file_access_time = 0; } } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *FileLoader ( void *arg ) { ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg; prctl(PR_SET_NAME,"FileLoader",0,0,0); WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size ); ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size]; SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader ); // int numWorkers = 1; int numWorkers = numCores() /4; // @TODO - this should be subject to inception_state options numWorkers = ( numWorkers < 4 ? 4:numWorkers ); fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers ); WqInfo_t wq_info[numWorkers]; { int cworker; pthread_t work_thread; // spawn threads for doing background correction/fitting work for ( cworker = 0; cworker < numWorkers; cworker++ ) { wq_info[cworker].threadNum=cworker; wq_info[cworker].wq = loadWorkQ; int t = pthread_create ( &work_thread, NULL, FileLoadWorker, &wq_info[cworker] ); pthread_detach(work_thread); if ( t ) fprintf ( stderr, "Error starting thread\n" ); } } WorkerInfoQueueItem item; //time_t start, end; int flow_buffer_size = master_img_loader->flow_buffer_size; // this loop goes over the individual image loaders for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ ) { ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer]; int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead); //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order if (!cur_image_loader->inception_state->img_control.threaded_file_access) { JustLoadOneImageWithPinnedUpdate(cur_image_loader); } //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion item.finished = false; item.private_data = cur_image_loader; loadWorkQ->PutItem ( item ); if (!ChipIdDecoder::IsProtonChip()) PauseForLongCompute ( cur_flow,cur_image_loader ); } // wait for all of the images to be processed loadWorkQ->WaitTillDone(); KillQueue ( loadWorkQ,numWorkers ); delete loadWorkQ; delete[] n_image_loaders; master_img_loader->finished = true; return NULL; }
void *FileLoadWorker ( void *arg ) { WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg ); assert ( q ); bool done = false; while ( !done ) { WorkerInfoQueueItem item = q->GetItem(); if ( item.finished == true ) { // we are no longer needed...go away! done = true; q->DecrementDone(); continue; } ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data; ClockTimer timer; if (one_img_loader->inception_state->img_control.threaded_file_access) JustLoadOneImageWithPinnedUpdate(one_img_loader); // col noise correction if ( one_img_loader->inception_state->img_control.col_flicker_correct ) { if(one_img_loader->inception_state->bfd_control.beadfindThumbnail) { //ComparatorNoiseCorrector ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } else { ComparatorNoiseCorrector cnc; cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose); } } // dump dc offset one_img_loaderrmation before we do any normalization DumpDcOffset ( one_img_loader ); int flow_buffer_for_flow = one_img_loader->cur_buffer; one_img_loader->img[flow_buffer_for_flow].SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd ); // correct in-channel electrical cross-talk ImageTransformer::XTChannelCorrect ( one_img_loader->img[flow_buffer_for_flow].raw, one_img_loader->img[flow_buffer_for_flow].results_folder ); // calculate the smooth pH step amplitude in empty wells across the whole image if ( one_img_loader->doEmptyWellNormalization ) { // this is redundant with what's in the EmptyTrace class, but this method of spatial normalization // is going to go away soon, so not worth fixing MaskType referenceMask = MaskReference; if ( one_img_loader->inception_state->bkg_control.use_dud_and_empty_wells_as_reference ) referenceMask = ( MaskType ) ( MaskReference | MaskDud ); one_img_loader->img[flow_buffer_for_flow].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span ); #if 0 char fname[512]; sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow ); FILE *ewampfile = fopen ( fname,"w" ); for ( int row=0;row < one_img_loader->img[flow_buffer_for_flow].GetRows();row++ ) { int col; for ( col=0;col < ( one_img_loader->img[flow_buffer_for_flow].GetCols()-1 ); col++ ) fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) ); fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[flow_buffer_for_flow].getEmptyWellAmplitude ( row,col ) ); } fclose ( ewampfile ); #endif } // dump pH step debug one_img_loader to file (its really fast, don't worry) DumpStep ( one_img_loader ); if ( one_img_loader->doRawBkgSubtract ) { one_img_loader->img[flow_buffer_for_flow].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx, one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true ); if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer printf ( "Notify user: NN empty subtraction in effect. No further warnings will be given. %d \n",one_img_loader->flow ); } printf ( "Allow model to go %d\n", one_img_loader->flow ); SetReadCompleted(one_img_loader); size_t usec = timer.GetMicroSec(); fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow , usec / 1.0e6); q->DecrementDone(); } return ( NULL ); }