static void DoThreadedSignalProcessing(
OptArgs &opts,
CommandLineOpts &inception_state,
const ComplexMask &from_beadfind_mask,
const char *chipType,
const ImageSpecClass &my_image_spec,
SlicedPrequel &my_prequel_setup,
const SeqListClass &my_keys,
const bool pass_tau,
const BkgFitterTracker *bkg_fitter_tracker
)
{
// This is the main routine, more or less, as far as the background model goes.
// Our job is to wait for the ImageLoader to load everything necessary,
// and then apply the background model to flow blocks.
// So... Let's start with the monitoring tools, for gathering statistics on these runs.
MemUsage ( "StartingBackground" );
time_t init_start;
time ( &init_start );
bool restart = not inception_state.bkg_control.signal_chunks.restart_from.empty();
const std::string wellsFile = string(inception_state.sys_context.wellsFilePath) + "/" +
inception_state.sys_context.wellsFileName;
const FlowBlockSequence & flow_block_sequence =
inception_state.bkg_control.signal_chunks.flow_block_sequence;
// shut off gpu multi flow fit if starting flow is not in first 20
if ( ! flow_block_sequence.HasFlowInFirstFlowBlock( inception_state.flow_context.startingFlow ) )
{
inception_state.bkg_control.gpuControl.gpuMultiFlowFit = 0;
}
Json::Value json_params;
bool regional_sampling_def = false;
string sChipType = chipType;
if(ChipIdDecoder::IsProtonChip())
{
regional_sampling_def = true;
}
bool regional_sampling = RetrieveParameterBool(opts, json_params, '-', "regional-sampling", regional_sampling_def);
// shut off gpu multifit if regional sampling is not enabled
if(!regional_sampling)
{
inception_state.bkg_control.gpuControl.gpuMultiFlowFit = 0;
}
bool convert = RetrieveParameterBool(opts, json_params, '-', "wells-save-as-ushort", false);
float lower = RetrieveParameterFloat(opts, json_params, '-', "wells-convert-low", -5.0);
float upper = RetrieveParameterFloat(opts, json_params, '-', "wells-convert-high", 28.0);
// Make a BkgFitterTracker object, and then either load its state from above,
// or build it fresh.
BkgFitterTracker GlobalFitter ( my_prequel_setup.num_regions );
if( restart ){
// Get the object that came from serialization.
GlobalFitter = *bkg_fitter_tracker;
}
else {
GlobalFitter.global_defaults.flow_global.SetFlowOrder ( inception_state.flow_context.flowOrder ); // @TODO: 2nd duplicated code instance
// Build everything
json_params["chipType"] = chipType;
json_params["results_folder"] = inception_state.sys_context.GetResultsFolder();
GlobalFitter.global_defaults.SetOpts(opts, json_params);
// >does not open wells file<
fprintf(stdout, "Opening wells file %s ... ", wellsFile.c_str());
RawWells preWells ( inception_state.sys_context.wellsFilePath, inception_state.sys_context.wellsFileName, convert, lower, upper );
fprintf(stdout, "done\n");
CreateWellsFileForWriting ( preWells,from_beadfind_mask.my_mask, inception_state,
inception_state.flow_context.GetNumFlows(),
my_image_spec.rows, my_image_spec.cols, chipType );
// build trace tracking
GlobalFitter.SetUpTraceTracking ( my_prequel_setup, inception_state, my_image_spec,
from_beadfind_mask,
flow_block_sequence.MaxFlowsInAnyFlowBlock() );
GlobalFitter.AllocateRegionData(my_prequel_setup.region_list.size(), & inception_state );
}
// One way (fresh creation) or the other (serialization), we need to allocate scratch space,
// now that the GlobalFitter has been built.
GlobalFitter.AllocateSlicedChipScratchSpace( flow_block_sequence.MaxFlowsInAnyFlowBlock() );
// plan (this happens whether we're from-disk or not):
GlobalFitter.PlanComputation ( inception_state.bkg_control);
// tweaking global defaults for bkg model if GPU is used
if (GlobalFitter.IsGpuAccelerationUsed()) {
GlobalFitter.global_defaults.signal_process_control.amp_guess_on_gpu =
(inception_state.bkg_control.gpuControl.gpuSingleFlowFit &
inception_state.bkg_control.gpuControl.gpuAmpGuess);
}
// do we have a wells file?
ION_ASSERT( isFile(wellsFile.c_str()), "Wells file "+ wellsFile + " does not exist" );
ChunkyWells rawWells ( inception_state.sys_context.wellsFilePath,
inception_state.sys_context.wellsFileName,
inception_state.bkg_control.signal_chunks.save_wells_flow,
inception_state.flow_context.startingFlow,
inception_state.flow_context.endingFlow
);
// Image Loading thread setup to grab flows in the background
// ImageTracker constructed to load flows
// must contact the GlobalFitter data that it will be associated with
// from thin air each time:
ImageTracker my_img_set ( inception_state.flow_context.getFlowSpan(),
inception_state.img_control.ignoreChecksumErrors,
inception_state.img_control.doSdat,
inception_state.img_control.total_timeout );
my_img_set.SetUpImageLoaderInfo ( inception_state, from_beadfind_mask, my_image_spec,
flow_block_sequence );
my_img_set.DecideOnRawDatsToBufferForThisFlowBlock();
my_img_set.FireUpThreads();
master_fit_type_table *LevMarSparseMatrices = 0;
// The number of flow blocks here is going to the hdf5 writer,
// which needs to build structures containing all the flow blocks,
// not just the ones that might be part of this run.
int num_flow_blocks = flow_block_sequence.FlowBlockCount( 0, inception_state.flow_context.endingFlow );
GlobalFitter.ThreadedInitialization ( rawWells, inception_state, from_beadfind_mask,
inception_state.sys_context.GetResultsFolder(),
my_image_spec,
my_prequel_setup.smooth_t0_est,
my_prequel_setup.region_list,
my_prequel_setup.region_timing, my_keys, restart,
num_flow_blocks );
// need to have initialized the regions for this
GlobalFitter.SetRegionProcessOrder (inception_state);
// init trace-output for the --bkg-debug-trace-sse/xyflow/rcflow options
GlobalFitter.InitBeads_xyflow(inception_state);
// Get the GPU ready, if we're using it.
if (GlobalFitter.IsGpuAccelerationUsed()) {
GlobalFitter.DetermineAndSetGPUAllocationAndKernelParams( inception_state.bkg_control, KEY_LEN,
flow_block_sequence.MaxFlowsInAnyFlowBlock() );
}
GlobalFitter.SpinUpGPUThreads();
float washoutThreshold = RetrieveParameterFloat(opts, json_params, '-', "bkg-washout-threshold", WASHOUT_THRESHOLD);
GlobalFitter.setWashoutThreshold(washoutThreshold);
int washoutFlowDetection = RetrieveParameterInt(opts, json_params, '-', "bkg-washout-flow-detection", WASHOUT_FLOW_DETECTION);
GlobalFitter.setWashoutFlowDetection(washoutFlowDetection);
MemUsage ( "AfterBgInitialization" );
time_t init_end;
time ( &init_end );
fprintf ( stdout, "InitModel: %0.3lf sec.\n", difftime ( init_end,init_start ) );
// JZ start flow data writer thread
pthread_t flowDataWriterThread;
SemQueue packQueue;
SemQueue writeQueue;
int saveQueueSize = RetrieveParameterInt(opts, json_params, '-', "wells-save-queue-size", 0);
if(saveQueueSize > 0) {
unsigned int queueSize = (unsigned int)saveQueueSize;
packQueue.init(queueSize);
writeQueue.init(queueSize);
size_t stepSize = rawWells.GetStepSize();
size_t flowDepth = inception_state.bkg_control.signal_chunks.save_wells_flow;
unsigned int spaceSize = my_image_spec.rows * my_image_spec.cols;
unsigned int bufferSize = stepSize * stepSize * flowDepth;
for(int item = 0; item < saveQueueSize; ++item) {
ChunkFlowData* chunkData = new ChunkFlowData(spaceSize, flowDepth, bufferSize);
packQueue.enQueue(chunkData);
}
writeFlowDataFuncArg writerArg;
writerArg.filePath = rawWells.GetHdf5FilePath();
writerArg.numCols = my_image_spec.cols;
writerArg.stepSize = stepSize;
writerArg.packQueuePtr = &packQueue;
writerArg.writeQueuePtr = &writeQueue;
pthread_create(&flowDataWriterThread, NULL, WriteFlowDataFunc, &writerArg);
}
bool saveCopies = RetrieveParameterBool(opts, json_params, '-', "wells-save-number-copies", false);
if(saveCopies)
{
rawWells.SetSaveCopies(0);
}
// process all flows...
// using actual flow values
Timer flow_block_timer;
Timer signal_proc_timer;
for ( int flow = inception_state.flow_context.startingFlow;
flow < (int)inception_state.flow_context.endingFlow; flow++ )
{
FlowBlockSequence::const_iterator flow_block = flow_block_sequence.BlockAtFlow( flow );
if ( flow == flow_block->begin() || flow == inception_state.flow_context.startingFlow ) {
flow_block_timer.restart();
// Build some matrices to work with.
LevMarSparseMatrices = new master_fit_type_table( GlobalFitter.global_defaults.flow_global,
flow_block->begin(), max( KEY_LEN - flow_block->begin() , 0),
flow_block->size() );
}
// coordinate with the ImageLoader threads for this flow to be read in
// WaitForFlowToLoad guarantees all flows up this one have been read in
my_img_set.WaitForFlowToLoad ( flow );
// ----- handle set up for processing this flow before we do anything needing
bool last_flow = ( ( flow ) == ( inception_state.flow_context.GetNumFlows()- 1 ) ); // actually the literal >last< flow, not just the flow in a chunk, so we can handle not having a full chunk.
// done with set up for anything this flow needs
signal_proc_timer.restart();
// computation that modifies data
// isolate this object so it can carry out actions in any order it chooses.
GlobalFitter.ExecuteFitForFlow ( flow, my_img_set, last_flow,
max( KEY_LEN - flow_block->begin(), 0 ),
LevMarSparseMatrices, & inception_state );
// Find the flow that's the last runnable flow in the "mixed_last_flow" block.
int applyFlow = flow_block_sequence.BlockAtFlow(
inception_state.bkg_control.polyclonal_filter.mixed_last_flow - 1 )->end() - 1;
if ( flow == applyFlow and inception_state.bkg_control.polyclonal_filter.enable )
ApplyClonalFilter ( *from_beadfind_mask.my_mask,
inception_state.sys_context.GetResultsFolder(),
GlobalFitter.sliced_chip, inception_state.bkg_control.polyclonal_filter );
// no more computation
signal_proc_timer.elapsed();
printf ( "SigProc: pure compute time for flow %d: %.1f sec.\n",
flow, signal_proc_timer.elapsed());
MemUsage ( "Memory_Flow: " + ToStr ( flow ) );
if (inception_state.bkg_control.pest_control.bkg_debug_files) {
GlobalFitter.DumpBkgModelRegionInfo ( inception_state.sys_context.GetResultsFolder(),
flow, last_flow, flow_block );
GlobalFitter.DumpBkgModelBeadInfo( inception_state.sys_context.GetResultsFolder(),
flow,last_flow,
inception_state.bkg_control.pest_control.debug_bead_only>0, flow_block );
}
// hdf5 dump of bead and regional parameters in bkgmodel
if (inception_state.bkg_control.pest_control.bkg_debug_files)
GlobalFitter.all_params_hdf.IncrementalWrite ( flow, last_flow, flow_block,
flow_block_sequence.FlowBlockIndex( flow ) );
// done capturing parameters, close out this flow
// Needed for 318 chips. Decide how many DATs to read ahead for every flow block
// also report timing for block of 20 flows from reading dat to writing 1.wells for this block
if ( flow == flow_block->end() - 1 )
my_img_set.DecideOnRawDatsToBufferForThisFlowBlock();
// End of block cleanup.
if ( (flow == flow_block->end() - 1 ) || last_flow) {
// Make sure that we've written everything.
if(saveQueueSize > 0) {
rawWells.DoneUpThroughFlow( flow, &packQueue, &writeQueue );
}
else {
rawWells.DoneUpThroughFlow( flow );
}
// report timing for block of 20 flows from reading dat to writing 1.wells for this block
fprintf ( stdout, "Flow Block compute time for flow %d to %d: %.1f sec.\n",
flow_block->begin(), flow, flow_block_timer.elapsed());
// Cleanup.
delete LevMarSparseMatrices;
LevMarSparseMatrices = 0;
}
// coordinate with the ImageLoader threads that this flow is done with
// and release resources associated with this image
// my_img_set knows what buffer is associated with the absolute flow
my_img_set.FinishFlow ( flow );
}
if(saveCopies)
{
rawWells.WriteWellsCopies();
}
if(saveQueueSize > 0) {
pthread_join(flowDataWriterThread, NULL);
packQueue.clear();
writeQueue.clear();
}
if ( not inception_state.bkg_control.signal_chunks.restart_next.empty() ){
string filePath = inception_state.sys_context.analysisLocation + inception_state.bkg_control.signal_chunks.restart_next;
ofstream outStream(filePath.c_str(), ios_base::trunc);
assert(outStream.good());
//boost::archive::text_oarchive outArchive(outStream);
boost::archive::binary_oarchive outArchive(outStream);
// get region associated objects on disk first
time_t begin_save_time;
time ( &begin_save_time );
const ComplexMask *from_beadfind_mask_ptr = &from_beadfind_mask;
BkgFitterTracker *GlobalFitter_ptr = &GlobalFitter;
string git_hash = IonVersion::GetGitHash();
outArchive
<< git_hash
<< my_prequel_setup
<< from_beadfind_mask_ptr
<< GlobalFitter_ptr;
outStream.close();
time_t finish_save_time;
time ( &finish_save_time );
fprintf ( stdout, "Writing restart state to archive %s took %0.1f secs",
filePath.c_str(), difftime ( finish_save_time, begin_save_time ));
}
rawWells.Close();
rawWells.GetWriteTimer().PrintMilliSeconds(std::cout, "Timer: Wells total writing time:");
GlobalFitter.UnSpinGPUThreads ();
if ( inception_state.bkg_control.signal_chunks.updateMaskAfterBkgModel )
from_beadfind_mask.pinnedInFlow->UpdateMaskWithPinned ( from_beadfind_mask.my_mask ); //update maskPtr
from_beadfind_mask.pinnedInFlow->DumpSummaryPinsPerFlow ( inception_state.sys_context.GetResultsFolder() );
}
void DoThreadedSignalProcessing ( CommandLineOpts &inception_state, ComplexMask &from_beadfind_mask, char *chipType,
ImageSpecClass &my_image_spec, SlicedPrequel &my_prequel_setup,SeqListClass &my_keys, bool pass_tau,
BkgFitterTracker *bkg_fitter_tracker)
{
MemUsage ( "StartingBackground" );
time_t init_start;
time ( &init_start );
bool restart = not inception_state.bkg_control.restart_from.empty();
BkgFitterTracker GlobalFitter ( my_prequel_setup.num_regions );
const std::string wellsFile = string(inception_state.sys_context.wellsFilePath) + "/" + inception_state.sys_context.wellsFileName;
MakeDecisionOnGpuMultiFlowFit(inception_state);
if( restart ){
GlobalFitter = *bkg_fitter_tracker;
}
else {
GlobalFitter.global_defaults.flow_global.SetFlowOrder ( inception_state.flow_context.flowOrder ); // @TODO: 2nd duplicated code instance
// Build everything
SetBkgModelGlobalDefaults ( GlobalFitter.global_defaults, inception_state.bkg_control,chipType,inception_state.sys_context.GetResultsFolder() );
// >does not open wells file<
fprintf(stdout, "Opening wells file %s ... ", wellsFile.c_str());
RawWells preWells ( inception_state.sys_context.wellsFilePath, inception_state.sys_context.wellsFileName );
fprintf(stdout, "done\n");
CreateWellsFileForWriting ( preWells,from_beadfind_mask.my_mask, inception_state, NUMFB,
inception_state.flow_context.GetNumFlows(), my_image_spec.rows, my_image_spec.cols, chipType );
// build trace tracking
GlobalFitter.SetUpTraceTracking ( my_prequel_setup, inception_state, my_image_spec, from_beadfind_mask );
GlobalFitter.AllocateRegionData(my_prequel_setup.region_list.size());
}
TinyInitializeUglyStaticForSignalProcessing ( GlobalFitter.global_defaults , inception_state);
// plan (this happens whether we're from-disk or not):
GlobalFitter.PlanComputation ( inception_state.bkg_control );
// do we have a wells file?
ION_ASSERT( isFile(wellsFile.c_str()), "Wells file "+ wellsFile + " does not exist" );
RawWells rawWells ( inception_state.sys_context.wellsFilePath, inception_state.sys_context.wellsFileName );
// plan (this happens whether we're from-disk or not):
GlobalFitter.ThreadedInitialization ( rawWells, inception_state, from_beadfind_mask, inception_state.sys_context.GetResultsFolder(), my_image_spec,
my_prequel_setup.smooth_t0_est,my_prequel_setup.region_list, my_prequel_setup.region_timing, my_keys, restart);
MemUsage ( "AfterBgInitialization" );
time_t init_end;
time ( &init_end );
fprintf ( stdout, "InitModel: %0.3lf sec.\n", difftime ( init_end,init_start ) );
// Image Loading thread setup to grab flows in the background
// ImageTracker constructed to load flows
// must contact the GlobalFitter data that it will be associated with
// from thin air each time:
ImageTracker my_img_set ( inception_state.flow_context.getFlowSpan(),inception_state.img_control.ignoreChecksumErrors,inception_state.img_control.doSdat,inception_state.img_control.total_timeout );
my_img_set.SetUpImageLoaderInfo ( inception_state, from_beadfind_mask, my_image_spec );
my_img_set.DecideOnRawDatsToBufferForThisFlowBlock();
my_img_set.FireUpThreads();
// Now do threaded solving, going through all the flows
GlobalFitter.SpinUp();
// need to have initialized the regions for this
GlobalFitter.SetRegionProcessOrder ();
// determine maximum beads in a region for gpu memory allocations
GlobalFitter.DetermineMaxLiveBeadsAndFramesAcrossAllRegionsForGpu();
// ideally these are part of the rawWells object itself
int write_well_flow_interval = inception_state.bkg_control.saveWellsFrequency*NUMFB; // goes with rawWells
int flow_to_write_wells = -1000; // never happens unless we set it to happen
// process all flows...
// using actual flow values
Timer flow_block_timer;
Timer signal_proc_timer;
for ( int flow = inception_state.flow_context.startingFlow; flow < (int)inception_state.flow_context.endingFlow; flow++ )
{
if ((flow % NUMFB) == 0)
flow_block_timer.restart();
// coordinate with the ImageLoader threads for this flow to be read in
// WaitForFlowToLoad guarantees all flows up this one have been read in
my_img_set.WaitForFlowToLoad ( flow );
// ----- handle set up for processing this flow before we do anything needing
bool last_flow = ( ( flow ) == ( inception_state.flow_context.GetNumFlows()- 1 ) ); // actually the literal >last< flow, not just the flow in a chunk, so we can handle not having a full chunk.
// always write intervals starting at wherever we are starting
// logic here: open wells file at startingFlow, tell at what flow we need to write things out.
if (NeedToOpenWellChunk(flow-inception_state.flow_context.startingFlow, write_well_flow_interval))
{
// chunk size is flow interval unless we run out of things to do in this interval
int chunk_depth = FigureChunkDepth(flow,inception_state.flow_context.endingFlow,write_well_flow_interval);
OpenExistingWellsForOneChunk(rawWells,flow,chunk_depth); // start
flow_to_write_wells = flow+chunk_depth-1;
}
// done with set up for anything this flow needs
signal_proc_timer.restart();
// computation that modifies data
GlobalFitter.ExecuteFitForFlow ( flow,my_img_set,last_flow ); // isolate this object so it can carry out actions in any order it chooses.
ApplyClonalFilter ( *from_beadfind_mask.my_mask, inception_state.sys_context.GetResultsFolder(), GlobalFitter.sliced_chip,inception_state.bkg_control.enableBkgModelClonalFilter, flow );
// no more computation
signal_proc_timer.elapsed();
fprintf ( stdout, "SigProc: pure compute time for flow %d: %.1f sec.\n", flow, signal_proc_timer.elapsed());
MemUsage ( "Memory_Flow: " + ToStr ( flow ) );
// capture the regional parameters every 20 flows, plus one bead per region at "random"
// @TODO replace with clean hdf5 interface for sampling beads and region parameters
GlobalFitter.DumpBkgModelRegionInfo ( inception_state.sys_context.GetResultsFolder(),flow,last_flow );
GlobalFitter.DumpBkgModelBeadInfo ( inception_state.sys_context.GetResultsFolder(),flow,last_flow, inception_state.bkg_control.debug_bead_only>0 );
WriteSampleRegion(inception_state.sys_context.GetResultsFolder(), GlobalFitter, flow, inception_state.bkg_control.region_vfrc_debug);
// variables should be >captured< at the end of fitting
// and then the hdf5 dump happens across all threads as we synchronize
GlobalFitter.all_params_hdf.IncrementalWrite ( flow, last_flow );
// done capturing parameters, close out this flow
// logic here: wells file knows when it needs to write something out
if (flow==flow_to_write_wells)
WriteOneChunkAndClose(rawWells);
// Needed for 318 chips. Decide how many DATs to read ahead for every block of NUMFB flows
// also report timing for block of 20 flows from reading dat to writing 1.wells for this block
if ((flow % NUMFB) == (NUMFB - 1))
my_img_set.DecideOnRawDatsToBufferForThisFlowBlock();
// report timing for block of 20 flows from reading dat to writing 1.wells for this block
if (((flow % NUMFB) == (NUMFB - 1)) || last_flow)
fprintf ( stdout, "Flow Block compute time for flow %d to %d: %.1f sec.\n",
((flow + 1) - NUMFB), flow, flow_block_timer.elapsed());
// coordinate with the ImageLoader threads that this flow is done with
// and release resources associated with this image
// my_img_set knows what buffer is associated with the absolute flow
my_img_set.FinishFlow ( flow );
// stop GPU thread computing doing fitting of first block of flows
if (flow == (NUMFB - 1))
GlobalFitter.UnSpinMultiFlowFitGpuThreads();
}
if ( not inception_state.bkg_control.restart_next.empty() ){
string filePath = inception_state.sys_context.analysisLocation + inception_state.bkg_control.restart_next;
ofstream outStream(filePath.c_str(), ios_base::trunc);
assert(outStream.good());
//boost::archive::text_oarchive outArchive(outStream);
boost::archive::binary_oarchive outArchive(outStream);
// get region associated objects on disk first
time_t begin_save_time;
time ( &begin_save_time );
ComplexMask *from_beadfind_mask_ptr = &from_beadfind_mask;
BkgFitterTracker *GlobalFitter_ptr = &GlobalFitter;
string svn_rev = IonVersion::GetSvnRev();
outArchive
<< svn_rev
<< my_prequel_setup
<< from_beadfind_mask_ptr
<< GlobalFitter_ptr;
outStream.close();
time_t finish_save_time;
time ( &finish_save_time );
fprintf ( stdout, "Writing restart state to archive %s took %0.1f secs",
filePath.c_str(), difftime ( finish_save_time, begin_save_time ));
}
rawWells.Close();
GlobalFitter.UnSpinSingleFlowFitGpuThreads ();
TinyDestroyUglyStaticForSignalProcessing();
if ( inception_state.bkg_control.updateMaskAfterBkgModel )
from_beadfind_mask.pinnedInFlow->UpdateMaskWithPinned ( from_beadfind_mask.my_mask ); //update maskPtr
from_beadfind_mask.pinnedInFlow->DumpSummaryPinsPerFlow ( inception_state.sys_context.GetResultsFolder() );
}