Example #1
0
void *FileSDatLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;
  int flow_buffer_size = master_img_loader->flow_buffer_size;
  for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);


    //    cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
    //    cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (!ChipIdDecoder::IsProtonChip())  //P2 and Long compute?
      PauseForLongCompute ( cur_flow,cur_image_loader );
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();

  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;

  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #2
0
void *FileLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;

  prctl(PR_SET_NAME,"FileLoader",0,0,0);


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );


  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );


  //   int numWorkers = 1;
  int numWorkers = numCores() /4; // @TODO - this should be subject to inception_state options
  numWorkers = ( numWorkers < 4 ? 4:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  WqInfo_t wq_info[numWorkers];

  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      wq_info[cworker].threadNum=cworker;
      wq_info[cworker].wq = loadWorkQ;
      int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
                               &wq_info[cworker] );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;

  //time_t start, end;
  int flow_buffer_size = master_img_loader->flow_buffer_size;

  // this loop goes over the individual image loaders
  for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow;  // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
    if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
      JustLoadOneImageWithPinnedUpdate(cur_image_loader);
    }
    //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion

    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (!ChipIdDecoder::IsProtonChip())
      PauseForLongCompute ( cur_flow,cur_image_loader );
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();
  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;
  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #3
0
void *FileSDatLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  //int numWorkers = 1;
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;
  Timer timer_file_access;
  //double file_access_time = 0;
  int flow_buffer_size = master_img_loader->flow_buffer_size;
  for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    TraceChunkSerializer serializer;
    timer_file_access.restart();
    bool ok = serializer.Read ( cur_image_loader->name, cur_image_loader->sdat[cur_image_loader->cur_buffer] );
    if (!ok) {
      ION_ABORT("Couldn't load file: " + ToStr(cur_image_loader->name));
    }
    // if ( ImageTransformer::gain_correction != NULL )
    //   ImageTransformer::GainCorrectImage ( &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
  
    //file_access_time += timer_file_access.elapsed();
    fprintf ( stdout, "File access = %0.2lf sec.\n", timer_file_access.elapsed() );
    cur_image_loader->pinnedInFlow->Update ( cur_image_loader->flow, &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
    cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
    cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );
    
    if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
      PauseForLongCompute ( cur_flow,cur_image_loader );

    /*if ( CheckFlowForWrite ( cur_flow,false ) )
    {
      fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
      file_access_time = 0;
    }*/
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();

  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;

  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}
Example #4
0
void *FileLoader ( void *arg )
{
  ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;


  WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );


  ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
  SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );

  int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
  // int numWorkers = 1;
  numWorkers = ( numWorkers < 1 ? 1:numWorkers );
  fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
  {
    int cworker;
    pthread_t work_thread;
    // spawn threads for doing background correction/fitting work
    for ( cworker = 0; cworker < numWorkers; cworker++ )
    {
      int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
                               loadWorkQ );
      pthread_detach(work_thread);
      if ( t )
        fprintf ( stderr, "Error starting thread\n" );
    }
  }

  WorkerInfoQueueItem item;

  //time_t start, end;
  int flow_buffer_size = master_img_loader->flow_buffer_size;

  Timer timer_file_access;
  double file_access_time = 0;
  // this loop goes over the individual image loaders
  for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
  {
    ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];

    int cur_flow = cur_image_loader->flow;  // each job is an n_image_loaders item

    DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
    //***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
    timer_file_access.restart();
    if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
      JustLoadOneImageWithPinnedUpdate(cur_image_loader);
    }
    else {
//      JustCacheOneImage(cur_image_loader);
    }
    file_access_time += timer_file_access.elapsed();
    //*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion

    item.finished = false;
    item.private_data = cur_image_loader;
    loadWorkQ->PutItem ( item );

    if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
      PauseForLongCompute ( cur_flow,cur_image_loader );

    if ( CheckFlowForWrite ( cur_flow,false ) )
    {
      fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
      file_access_time = 0;
    }
  }

  // wait for all of the images to be processed
  loadWorkQ->WaitTillDone();
  KillQueue ( loadWorkQ,numWorkers );

  delete loadWorkQ;
  delete[] n_image_loaders;

  master_img_loader->finished = true;

  return NULL;
}