void *FileSDatLoader ( void *arg )
{
ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;
WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );
int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
numWorkers = ( numWorkers < 1 ? 1:numWorkers );
fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
{
int cworker;
pthread_t work_thread;
// spawn threads for doing background correction/fitting work
for ( cworker = 0; cworker < numWorkers; cworker++ )
{
int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
loadWorkQ );
pthread_detach(work_thread);
if ( t )
fprintf ( stderr, "Error starting thread\n" );
}
}
WorkerInfoQueueItem item;
int flow_buffer_size = master_img_loader->flow_buffer_size;
for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
{
ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];
int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
// cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
// cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
item.finished = false;
item.private_data = cur_image_loader;
loadWorkQ->PutItem ( item );
if (!ChipIdDecoder::IsProtonChip()) //P2 and Long compute?
PauseForLongCompute ( cur_flow,cur_image_loader );
}
// wait for all of the images to be processed
loadWorkQ->WaitTillDone();
KillQueue ( loadWorkQ,numWorkers );
delete loadWorkQ;
delete[] n_image_loaders;
master_img_loader->finished = true;
return NULL;
}
void *FileLoader ( void *arg )
{
ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;
prctl(PR_SET_NAME,"FileLoader",0,0,0);
WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );
// int numWorkers = 1;
int numWorkers = numCores() /4; // @TODO - this should be subject to inception_state options
numWorkers = ( numWorkers < 4 ? 4:numWorkers );
fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
WqInfo_t wq_info[numWorkers];
{
int cworker;
pthread_t work_thread;
// spawn threads for doing background correction/fitting work
for ( cworker = 0; cworker < numWorkers; cworker++ )
{
wq_info[cworker].threadNum=cworker;
wq_info[cworker].wq = loadWorkQ;
int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
&wq_info[cworker] );
pthread_detach(work_thread);
if ( t )
fprintf ( stderr, "Error starting thread\n" );
}
}
WorkerInfoQueueItem item;
//time_t start, end;
int flow_buffer_size = master_img_loader->flow_buffer_size;
// this loop goes over the individual image loaders
for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
{
ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];
int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
//***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
JustLoadOneImageWithPinnedUpdate(cur_image_loader);
}
//*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion
item.finished = false;
item.private_data = cur_image_loader;
loadWorkQ->PutItem ( item );
if (!ChipIdDecoder::IsProtonChip())
PauseForLongCompute ( cur_flow,cur_image_loader );
}
// wait for all of the images to be processed
loadWorkQ->WaitTillDone();
KillQueue ( loadWorkQ,numWorkers );
delete loadWorkQ;
delete[] n_image_loaders;
master_img_loader->finished = true;
return NULL;
}
void *FileSDatLoader ( void *arg )
{
ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;
WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );
int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
//int numWorkers = 1;
numWorkers = ( numWorkers < 1 ? 1:numWorkers );
fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
{
int cworker;
pthread_t work_thread;
// spawn threads for doing background correction/fitting work
for ( cworker = 0; cworker < numWorkers; cworker++ )
{
int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
loadWorkQ );
pthread_detach(work_thread);
if ( t )
fprintf ( stderr, "Error starting thread\n" );
}
}
WorkerInfoQueueItem item;
Timer timer_file_access;
//double file_access_time = 0;
int flow_buffer_size = master_img_loader->flow_buffer_size;
for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
{
ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];
int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
TraceChunkSerializer serializer;
timer_file_access.restart();
bool ok = serializer.Read ( cur_image_loader->name, cur_image_loader->sdat[cur_image_loader->cur_buffer] );
if (!ok) {
ION_ABORT("Couldn't load file: " + ToStr(cur_image_loader->name));
}
// if ( ImageTransformer::gain_correction != NULL )
// ImageTransformer::GainCorrectImage ( &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
//file_access_time += timer_file_access.elapsed();
fprintf ( stdout, "File access = %0.2lf sec.\n", timer_file_access.elapsed() );
cur_image_loader->pinnedInFlow->Update ( cur_image_loader->flow, &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
item.finished = false;
item.private_data = cur_image_loader;
loadWorkQ->PutItem ( item );
if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
PauseForLongCompute ( cur_flow,cur_image_loader );
/*if ( CheckFlowForWrite ( cur_flow,false ) )
{
fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
file_access_time = 0;
}*/
}
// wait for all of the images to be processed
loadWorkQ->WaitTillDone();
KillQueue ( loadWorkQ,numWorkers );
delete loadWorkQ;
delete[] n_image_loaders;
master_img_loader->finished = true;
return NULL;
}
void *FileLoader ( void *arg )
{
ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;
WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );
int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
// int numWorkers = 1;
numWorkers = ( numWorkers < 1 ? 1:numWorkers );
fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
{
int cworker;
pthread_t work_thread;
// spawn threads for doing background correction/fitting work
for ( cworker = 0; cworker < numWorkers; cworker++ )
{
int t = pthread_create ( &work_thread, NULL, FileLoadWorker,
loadWorkQ );
pthread_detach(work_thread);
if ( t )
fprintf ( stderr, "Error starting thread\n" );
}
}
WorkerInfoQueueItem item;
//time_t start, end;
int flow_buffer_size = master_img_loader->flow_buffer_size;
Timer timer_file_access;
double file_access_time = 0;
// this loop goes over the individual image loaders
for ( int i_buffer = 0; i_buffer < flow_buffer_size; i_buffer++ )
{
ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];
int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
//***We are doing this on this thread so we >load< in sequential order that pinned in Flow updates in sequential order
timer_file_access.restart();
if (!cur_image_loader->inception_state->img_control.threaded_file_access) {
JustLoadOneImageWithPinnedUpdate(cur_image_loader);
}
else {
// JustCacheOneImage(cur_image_loader);
}
file_access_time += timer_file_access.elapsed();
//*** now we can do the rest of the computation for an image, including dumping in a multiply threaded fashion
item.finished = false;
item.private_data = cur_image_loader;
loadWorkQ->PutItem ( item );
if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
PauseForLongCompute ( cur_flow,cur_image_loader );
if ( CheckFlowForWrite ( cur_flow,false ) )
{
fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
file_access_time = 0;
}
}
// wait for all of the images to be processed
loadWorkQ->WaitTillDone();
KillQueue ( loadWorkQ,numWorkers );
delete loadWorkQ;
delete[] n_image_loaders;
master_img_loader->finished = true;
return NULL;
}