//@TODO: Bad to have side effects on every image load from now on
// should be >explicit< pass of gain correction information to image loader
void ImageTransformer::CalculateGainCorrectionFromBeadfindFlow (char *_datDir, bool gain_debug_output)
{
// calculate gain of each well from the beadfind and use for correction of all images thereafter
Mask *gainCalcMask;
std::string datdir = _datDir;
std::string preBeadFind = datdir + "/beadfind_pre_0003.dat";
Image bfImg;
bfImg.SetImgLoadImmediate (false);
bool loaded = bfImg.LoadRaw (preBeadFind.c_str());
if (!loaded)
{
ION_ABORT ("*Error* - No beadfind file found, did beadfind run? are files transferred? (" + preBeadFind + ")");
}
gainCalcMask = new Mask (bfImg.GetCols(),bfImg.GetRows());
bfImg.FilterForPinned (gainCalcMask, MaskEmpty, false);
bfImg.SetMeanOfFramesToZero (1,3);
//@TODO: implicit global variable->explicit global variable-> explicit variable
GainCalculationFromBeadfind (gainCalcMask,bfImg.raw);
printf ("Computed gain for each pixel using beadind image\n");
if (gain_debug_output)
{
DumpTextGain(bfImg.GetCols(),bfImg.GetRows());
}
bfImg.Close();
delete gainCalcMask;
}
void MergeAcq::CheckImagesAlign() {
const RawImage *first = mFirst->GetImage();
const RawImage *second = mSecond->GetImage();
ION_ASSERT(mFirst != NULL && mSecond != NULL, "Can't merge null images.");
ION_ASSERT(first->rows == second->rows || first->cols == second->cols,
"Images must mactch in dimension with common edge.");
ION_ASSERT(first->frames == second->frames, "Images must have same number of frames");
if ( !((mSecondRowStart == (size_t)first->rows && mSecondColStart == 0) ||
(mSecondColStart == 0 && mSecondColStart == (size_t)first->cols))) {
ION_ABORT("Image 2 must start where image 1 starts.");
}
}
// checks to see if the special lsrowimage.dat file exists in the experiment directory. If it does,
// this image is used to generate custom channel correction coefficients. If not, the method silently
// returns (and subsequent analysis uses the default correction).
void ImageTransformer::CalibrateChannelXTCorrection ( const char *exp_dir,const char *filename, bool wait_for_prerun )
{
// only allow this to be done once
if ( custom_correction_data != NULL )
return;
// LSRowImageProcessor can generate a correction for the 314, but application of the correction is much more
// difficult than for 316/318, and the expected benefit is not as high, so for now...we're skipping the 314
if ( ( ChipIdDecoder::GetGlobalChipId() != ChipId316 ) && ( ChipIdDecoder::GetGlobalChipId() != ChipId318 ) && ( ChipIdDecoder::GetGlobalChipId() != ChipId316v2 ) )
return;
int len = strlen ( exp_dir ) +strlen ( filename ) + 2;
char full_fname[len];
sprintf ( full_fname,"%s/%s",exp_dir,filename );
if ( wait_for_prerun ) {
std::string preRun = exp_dir;
preRun = preRun + "/prerun_0000.dat";
std::string acq0 = exp_dir;
acq0 = acq0 + "/acq_0000.dat";
uint32_t waitTime = RETRY_INTERVAL;
int32_t timeOut = TOTAL_TIMEOUT;
//--- Wait up to 3600 seconds for a file to be available
bool okToProceed = false;
while ( timeOut > 0 ) {
//--- do our checkpoint files exist?
if ( isFile ( preRun.c_str() ) || isFile ( acq0.c_str() ) ) {
okToProceed = true;
break;
}
fprintf ( stdout, "Waiting to load crosstalk params in %s\n", full_fname );
sleep ( waitTime );
timeOut -= waitTime;
}
if ( !okToProceed ) {
ION_ABORT ( "Couldn't find gateway files for: " + ToStr ( full_fname ) );
}
// We got the files we expected so if the xtalk file isn't there then warn.
if ( !isFile ( full_fname ) ) {
ION_WARN ( "Didn't find xtalk file: " + ToStr ( full_fname ) );
}
}
LSRowImageProcessor lsrowproc;
custom_correction_data = lsrowproc.GenerateCorrection ( full_fname );
if ( custom_correction_data != NULL )
selected_chip_xt_vectors = custom_correction_data->GetCorrectionDescriptor();
}
bool RawWellsV1::OpenForRead(bool wantMemMap)
{
int n = 0; //number elements read
char wellFileName[MAX_PATH_LENGTH];
sprintf_s(wellFileName, sizeof(wellFileName), "%s/%s", experimentPath, rawWellsName);
fp = NULL;
hFile = 0;
fileMemPtr = NULL;
hdr.flowOrder = NULL;
if (wantMemMap)
hFile = open(wellFileName, O_RDONLY);
else
fopen_s(&fp, wellFileName, "rb");
if (fp) {
n = fread(&hdr.numWells, sizeof(hdr.numWells), 1, fp);
assert(n==1);
n = fread(&hdr.numFlows, sizeof(hdr.numFlows), 1, fp);
assert(n==1);
hdr.flowOrder = (char *)malloc(sizeof(char) * hdr.numFlows);
n = fread(hdr.flowOrder, sizeof(char), hdr.numFlows, fp);
assert(n==hdr.numFlows);
} else if (hFile) {
struct stat result;
fstat(hFile, &result);
rawWellsSize = result.st_size;
fileMemPtr = mmap(0, rawWellsSize, PROT_READ, MAP_SHARED, hFile, 0);
if (fileMemPtr) {
unsigned char *ptr = (unsigned char *)fileMemPtr;
memcpy(&hdr.numWells, ptr, sizeof(hdr.numWells)); ptr += sizeof(hdr.numWells);
memcpy(&hdr.numFlows, ptr, sizeof(hdr.numFlows)); ptr += sizeof(hdr.numFlows);
hdr.flowOrder = (char *)malloc(sizeof(char) * hdr.numFlows);
memcpy(hdr.flowOrder, ptr, hdr.numFlows); ptr += hdr.numFlows;
}
} else {
ION_ABORT("Couldn't open RawWellsV1 file: " + std::string(wellFileName));
return (true); //indicate there was an error
}
flowValuesSize = sizeof(float) * hdr.numFlows;
data.flowValues = (float *)malloc(flowValuesSize);
dataBlockSize = sizeof(unsigned int) + 2 * sizeof(unsigned short);
return (false); // no error
}
void MergeAcq::Merge(Image &combo) {
CheckImagesAlign();
RawImage *raw = new RawImage();
const RawImage *first = mFirst->GetImage();
const RawImage *second = mSecond->GetImage();
/* do either compressed or uncompressed. */
if (first->image != NULL) {
raw->rows = first->rows + mSecondRowStart;
raw->cols = first->cols + mSecondColStart;
raw->frames = first->frames;
raw->uncompFrames = first->uncompFrames;
raw->compFrames = first->compFrames;
raw->channels = first->channels;
raw->interlaceType = first->interlaceType;
raw->frameStride = raw->cols * raw->rows;
raw->baseFrameRate = first->baseFrameRate;
// for some reason these are null with bb images.
if (first->interpolatedFrames != NULL) {
raw->interpolatedFrames = (int *)malloc(sizeof(int)*raw->uncompFrames);
memcpy(raw->interpolatedFrames, first->interpolatedFrames, sizeof(int)*raw->uncompFrames);
}
if (first->interpolatedMult != NULL) {
raw->interpolatedMult = (float *)malloc(sizeof(float)*raw->uncompFrames);
memcpy(raw->interpolatedMult, first->interpolatedMult, sizeof(float)*raw->uncompFrames);
}
// Note = use malloc as that is how
uint64_t size = (uint64_t)raw->rows * raw->cols * raw->frames * sizeof(float);
raw->image = (short *)malloc(size);
memset(raw->image, 0, size);
raw->timestamps = (int *)malloc(raw->frames * sizeof(int));
memcpy(raw->timestamps, first->timestamps, raw->frames * sizeof(int));
FillNewImage(first->image, first->rows, first->cols,
second->image, second->rows, second->cols,
mSecondRowStart, mSecondColStart,
raw->rows, raw->cols, raw->frames, raw->image);
combo.SetImage(raw);
}
else {
delete raw;
raw = NULL;
ION_ABORT("Don't support the compImage buffer");
}
}
void RegionAnalysis::analyze(vector<float> *_cf, vector<float> *_ie, vector<float> *_dr, RawWells *_wells, Mask *_mask, SequenceItem *_libraryInfo, CommandLineOpts *_clo,
const string& _flowOrder, int _numFlows, int numWorkers)
{
printf("RegionAnalysis::analyze start (with %d threads)\n", numWorkers);
// By storing these values as class members, they become visible to all worker threads.
cf = _cf;
ie = _ie;
dr = _dr;
mask = _mask;
libraryInfo = _libraryInfo;
clo = _clo;
flowOrder = _flowOrder;
numFlows = _numFlows;
// Initialize region wells reader
assert (wellsReader.OpenForRead(_wells, mask->W(), mask->H(), clo->cfe_control.cfiedrRegionsX, clo->cfe_control.cfiedrRegionsY));
// Launch worker threads and await their completion
pthread_mutex_t tmp_mutex = PTHREAD_MUTEX_INITIALIZER;
common_output_mutex = &tmp_mutex;
pthread_t worker_id[numWorkers];
for (int iWorker = 0; iWorker < numWorkers; iWorker++)
if (pthread_create(&worker_id[iWorker], NULL, RegionAnalysisWorker, this))
ION_ABORT("*Error* - problem starting thread");
for (int iWorker = 0; iWorker < numWorkers; iWorker++)
pthread_join(worker_id[iWorker], NULL);
printf("RegionAnalysis::analyze end\n");
}
// Load optimized defaults from GeneticOptimizer runs
void GlobalDefaultsForBkgModel::SetGoptDefaults ( char *fname )
{
struct stat fstatus;
int status;
FILE *param_file;
char *line;
int nChar = MAX_LINE_LEN;
float d[10];
int num = 0;
line = new char[MAX_LINE_LEN];
status = stat ( fname,&fstatus );
if ( status == 0 )
{
// file exists
printf ( "GOPT: loading parameters from %s\n",fname );
param_file=fopen ( fname,"rt" );
bool done = false;
while ( !done )
{
int bytes_read = getline ( &line, ( size_t * ) &nChar,param_file );
if ( bytes_read > 0 )
{
if ( bytes_read >= MAX_LINE_LEN || bytes_read < 0 )
{
ION_ABORT ( "Read: " + ToStr ( bytes_read ) + " into a buffer only: " +
ToStr ( MAX_LINE_LEN ) + " long for line: '" + ToStr ( line ) + "'" );
}
line[bytes_read]='\0';
if ( strncmp ( "km_const",line,8 ) == 0 )
{
num = sscanf ( line,"km_const: %f %f %f %f",&d[0],&d[1],&d[2],&d[3] );
if ( num > 0 )
for ( int i=0;i<NUMNUC;i++ ) region_param_start.kmax_default[i] = d[i];
}
if ( strncmp ( "krate",line,5 ) == 0 )
{
num = sscanf ( line,"krate: %f %f %f %f",&d[0],&d[1],&d[2],&d[3] );
if ( num > 0 )
for ( int i=0;i<NUMNUC;i++ ) region_param_start.krate_default[i] = d[i];
}
if ( strncmp ( "d_coeff",line,7 ) == 0 )
{
num = sscanf ( line,"d_coeff: %f %f %f %f",&d[0],&d[1],&d[2],&d[3] );
if ( num > 0 )
for ( int i=0;i<NUMNUC;i++ ) region_param_start.d_default[i] = d[i];
}
if ( strncmp ( "n_to_uM_conv",line,12 ) == 0 )
num = sscanf ( line,"n_to_uM_conv: %f",®ion_param_start.molecules_to_micromolar_conv );
if ( strncmp ( "sens",line,4 ) == 0 )
num = sscanf ( line,"sens: %f",®ion_param_start.sens_default );
if ( strncmp ( "tau_R_m",line,7 ) == 0 )
num = sscanf ( line,"tau_R_m: %f",®ion_param_start.tau_R_m_default );
if ( strncmp ( "tau_R_o",line,7 ) == 0 )
num = sscanf ( line,"tau_R_o: %f",®ion_param_start.tau_R_o_default );
if ( strncmp ( "sigma_mult", line, 10 ) == 0 )
{
num = sscanf ( line,"sigma_mult: %f %f %f %f", &d[0],&d[1],&d[2],&d[3] );
for ( int i=0;i<num;i++ ) region_param_start.sigma_mult_default[i]=d[i];
}
if ( strncmp ( "t_mid_nuc_delay", line, 15 ) == 0 )
{
num = sscanf ( line,"t_mid_nuc_delay: %f %f %f %f", &d[0],&d[1],&d[2],&d[3] );
for ( int i=0;i<num;i++ ) region_param_start.t_mid_nuc_delay_default[i]=d[i];
}
if ( strncmp ( "emphasis",line,8 ) == 0 )
{
num = sscanf ( line,"emphasis: %f %f %f %f %f %f %f %f", &d[0],&d[1],&d[2],&d[3],&d[4],&d[5],&d[6],&d[7] );
for ( int i=0;i<num;i++ ) data_control.emp[i]=d[i];
}
if ( strncmp ( "emp_amp",line,7 ) == 0 )
num = sscanf ( line,"emp_amplitude: %f",&data_control.emphasis_ampl_default );
if ( strncmp ( "emp_width",line,7 ) == 0 )
num = sscanf ( line,"emp_width: %f",&data_control.emphasis_width_default );
if ( strncmp ( "clonal_call_scale",line,17 ) == 0 )
{
num = sscanf ( line,"clonal_call_scale: %f %f %f %f %f", &d[0],&d[1],&d[2],&d[3],&d[4] );
for ( int i=0;i<num;i++ ) fitter_defaults.clonal_call_scale[i]=d[i];
}
if ( strncmp ( "shrink_factor:", line, 14 ) == 0 )
{
float t_shrink;
num = sscanf ( line,"shrink_factor: %f", &t_shrink );
fitter_defaults.shrink_factor = t_shrink;
}
if ( strncmp ( "nuc_flow_timing", line, 15 ) == 0 )
{
num = sscanf ( line,"nuc_flow_frame_width: %f", &d[0] );
data_control.nuc_flow_frame_width = d[0];
}
if ( strncmp ( "time_compression", line, 16 ) == 0 )
{
num = sscanf ( line,"time_compression: %f %f %f", &d[0], &d[1],&d[2] );
data_control.time_left_avg = d[0];
data_control.time_start_detail = d[1];
data_control.time_stop_detail = d[2];
}
}
else
done = true;
}
fclose ( param_file );
DumpExcitingParameters("default");
}
else
printf ( "GOPT: parameter file %s does not exist\n",fname );
delete [] line;
}
void PhaseEstimator::DoPhaseEstimation(RawWells *wells, Mask *mask, const ion::FlowOrder& flow_order, const vector<KeySequence>& keys,
int region_size_x, int region_size_y, bool use_single_core)
{
flow_order_.SetFlowOrder(flow_order.str(), min(flow_order.num_flows(), 120));
keys_ = keys;
chip_size_x_ = mask->W();
chip_size_y_ = mask->H();
region_size_x_ = region_size_x;
region_size_y_ = region_size_y;
printf("Phase estimation mode = %s\n", phasing_estimator_.c_str());
if (phasing_estimator_ == "override") {
// Nothing to do!
} else if (phasing_estimator_ == "spatial-refiner") {
int num_workers = max(numCores(), 2);
if (use_single_core)
num_workers = 1;
wells->Close();
wells->OpenForIncrementalRead();
SpatialRefiner(wells, mask, num_workers);
} else if (phasing_estimator_ == "spatial-refiner-2") {
int num_workers = max(numCores(), 2);
if (use_single_core)
num_workers = 1;
wells->Close();
wells->OpenForIncrementalRead();
train_subset_count_ = 2;
train_subset_cf_.resize(train_subset_count_);
train_subset_ie_.resize(train_subset_count_);
train_subset_dr_.resize(train_subset_count_);
train_subset_regions_x_.resize(train_subset_count_);
train_subset_regions_y_.resize(train_subset_count_);
for (train_subset_ = 0; train_subset_ < train_subset_count_; ++train_subset_) {
SpatialRefiner(wells, mask, num_workers);
train_subset_cf_[train_subset_] = result_cf_;
train_subset_ie_[train_subset_] = result_ie_;
train_subset_dr_[train_subset_] = result_dr_;
train_subset_regions_x_[train_subset_] = result_regions_x_;
train_subset_regions_y_[train_subset_] = result_regions_y_;
}
} else
ION_ABORT("Requested phase estimator is not recognized");
// Compute mean cf, ie, dr
average_cf_ = 0;
average_ie_ = 0;
average_dr_ = 0;
int count = 0;
for (int r = 0; r < result_regions_x_*result_regions_y_; r++) {
if (result_cf_[r] || result_ie_[r] || result_dr_[r]) {
average_cf_ += result_cf_[r];
average_ie_ += result_ie_[r];
average_dr_ += result_dr_[r];
count++;
}
}
if (count > 0) {
average_cf_ /= count;
average_ie_ /= count;
average_dr_ /= count;
}
}
void PhaseEstimator::SpatialRefiner(RawWells *wells, Mask *mask, int num_workers)
{
printf("PhaseEstimator::analyze start\n");
num_regions_x_ = (chip_size_x_+region_size_x_-1) / region_size_x_;
num_regions_y_ = (chip_size_y_+region_size_y_-1) / region_size_y_;
num_regions_ = num_regions_x_ * num_regions_y_;
int num_levels = 1;
if (num_regions_x_ >= 2 and num_regions_y_ >= 2 and max_phasing_levels_ >= 2)
num_levels = 2;
if (num_regions_x_ >= 4 and num_regions_y_ >= 4 and max_phasing_levels_ >= 3)
num_levels = 3;
if (num_regions_x_ >= 8 and num_regions_y_ >= 8 and max_phasing_levels_ >= 4)
num_levels = 4;
printf("Using numEstimatorFlows %d, chip is %d x %d, region is %d x %d, numRegions is %d x %d, numLevels %d\n",
flow_order_.num_flows(), chip_size_x_, chip_size_y_, region_size_x_, region_size_y_, num_regions_x_, num_regions_y_, num_levels);
// Step 1. Use mask to build region density map
region_num_reads_.assign(num_regions_, 0);
for (int x = 0; x < chip_size_x_; x++)
for (int y = 0; y < chip_size_y_; y++)
if (mask->Match(x, y, (MaskType)(MaskTF|MaskLib)) and
!mask->Match(x, y, MaskFilteredBadResidual) and
!mask->Match(x, y, MaskFilteredBadPPF) and
!mask->Match(x, y, MaskFilteredBadKey))
region_num_reads_[(x/region_size_x_) + (y/region_size_y_)*num_regions_x_]++;
// Step 2. Build the tree of estimation subblocks.
int max_subblocks = 2*4*4*4;
vector<Subblock> subblocks;
subblocks.reserve(max_subblocks);
subblocks.push_back(Subblock());
subblocks.back().cf = 0.0;
subblocks.back().ie = 0.0;
subblocks.back().dr = 0.0;
subblocks.back().begin_x = 0;
subblocks.back().end_x = num_regions_x_;
subblocks.back().begin_y = 0;
subblocks.back().end_y = num_regions_y_;
subblocks.back().level = 1;
subblocks.back().pos_x = 0;
subblocks.back().pos_y = 0;
subblocks.back().superblock = NULL;
for (unsigned int idx = 0; idx < subblocks.size(); idx++) {
Subblock &s = subblocks[idx];
if (s.level == num_levels) {
s.subblocks[0] = NULL;
s.subblocks[1] = NULL;
s.subblocks[2] = NULL;
s.subblocks[3] = NULL;
continue;
}
int cut_x = (s.begin_x + s.end_x) / 2;
int cut_y = (s.begin_y + s.end_y) / 2;
for (int i = 0; i < 4; i++) {
subblocks.push_back(s);
subblocks.back().cf = -1.0;
subblocks.back().ie = -1.0;
subblocks.back().dr = -1.0;
subblocks.back().level++;
subblocks.back().superblock = &s;
s.subblocks[i] = &subblocks.back();
}
s.subblocks[0]->end_x = cut_x;
s.subblocks[0]->end_y = cut_y;
s.subblocks[0]->pos_x = (s.pos_x << 1);
s.subblocks[0]->pos_y = (s.pos_y << 1);
s.subblocks[1]->begin_x = cut_x;
s.subblocks[1]->end_y = cut_y;
s.subblocks[1]->pos_x = (s.pos_x << 1) + 1;
s.subblocks[1]->pos_y = (s.pos_y << 1);
s.subblocks[2]->end_x = cut_x;
s.subblocks[2]->begin_y = cut_y;
s.subblocks[2]->pos_x = (s.pos_x << 1);
s.subblocks[2]->pos_y = (s.pos_y << 1) + 1;
s.subblocks[3]->begin_x = cut_x;
s.subblocks[3]->begin_y = cut_y;
s.subblocks[3]->pos_x = (s.pos_x << 1) + 1;
s.subblocks[3]->pos_y = (s.pos_y << 1) + 1;
}
// Step 3. Populate region search order in lowermost subblocks
for (unsigned int idx = 0; idx < subblocks.size(); idx++) {
Subblock &s = subblocks[idx];
if (s.level != num_levels)
continue;
s.sorted_regions.reserve((s.end_x - s.begin_x) * (s.end_y - s.begin_y));
for (int region_x = s.begin_x; region_x < s.end_x; region_x++)
for (int region_y = s.begin_y; region_y < s.end_y; region_y++)
s.sorted_regions.push_back(region_x + region_y*num_regions_x_);
sort(s.sorted_regions.begin(), s.sorted_regions.end(), CompareDensity(region_num_reads_));
}
// Step 4. Populate region search order in remaining subblocks
for (int level = num_levels-1; level >= 1; --level) {
for (unsigned int idx = 0; idx < subblocks.size(); idx++) {
Subblock &s = subblocks[idx];
if (s.level != level)
continue;
assert(s.subblocks[0] != NULL);
assert(s.subblocks[1] != NULL);
assert(s.subblocks[2] != NULL);
assert(s.subblocks[3] != NULL);
unsigned int sum_regions = s.subblocks[0]->sorted_regions.size()
+ s.subblocks[1]->sorted_regions.size()
+ s.subblocks[2]->sorted_regions.size()
+ s.subblocks[3]->sorted_regions.size();
s.sorted_regions.reserve(sum_regions);
vector<int>::iterator V0 = s.subblocks[0]->sorted_regions.begin();
vector<int>::iterator V1 = s.subblocks[1]->sorted_regions.begin();
vector<int>::iterator V2 = s.subblocks[2]->sorted_regions.begin();
vector<int>::iterator V3 = s.subblocks[3]->sorted_regions.begin();
while (s.sorted_regions.size() < sum_regions) {
if (V0 != s.subblocks[0]->sorted_regions.end())
s.sorted_regions.push_back(*V0++);
if (V2 != s.subblocks[2]->sorted_regions.end())
s.sorted_regions.push_back(*V2++);
if (V1 != s.subblocks[1]->sorted_regions.end())
s.sorted_regions.push_back(*V1++);
if (V3 != s.subblocks[3]->sorted_regions.end())
s.sorted_regions.push_back(*V3++);
}
}
}
// Step 5. Show time. Spawn multiple worker threads to do phasing estimation
region_reads_.clear();
region_reads_.resize(num_regions_);
action_map_.assign(num_regions_,0);
subblock_map_.assign(num_regions_,' ');
pthread_mutex_init(®ion_loader_mutex_, NULL);
pthread_mutex_init(&job_queue_mutex_, NULL);
pthread_cond_init(&job_queue_cond_, NULL);
wells_ = wells;
mask_ = mask;
job_queue_.push_back(&subblocks[0]);
jobs_in_progress_ = 0;
pthread_t worker_id[num_workers];
for (int worker = 0; worker < num_workers; worker++)
if (pthread_create(&worker_id[worker], NULL, EstimatorWorkerWrapper, this))
ION_ABORT("*Error* - problem starting thread");
for (int worker = 0; worker < num_workers; worker++)
pthread_join(worker_id[worker], NULL);
pthread_cond_destroy(&job_queue_cond_);
pthread_mutex_destroy(&job_queue_mutex_);
pthread_mutex_destroy(®ion_loader_mutex_);
// Print a silly action map
//! @todo Get rid of action map once confidence in spatial refiner performance is high
for (int region_y = 0; region_y < num_regions_y_; region_y++) {
for (int region_x = 0; region_x < num_regions_x_; region_x++) {
int region = region_x + region_y * num_regions_x_;
if (action_map_[region] == 0)
printf(" ");
else
printf("%d",action_map_[region]);
printf("%c", subblock_map_[region]);
}
printf("\n");
}
// Crunching complete. Retrieve phasing estimates
result_regions_x_ = 1 << (num_levels-1);
result_regions_y_ = 1 << (num_levels-1);
result_cf_.assign(result_regions_x_*result_regions_y_,0.0);
result_ie_.assign(result_regions_x_*result_regions_y_,0.0);
result_dr_.assign(result_regions_x_*result_regions_y_,0.0);
for (unsigned int idx = 0; idx < subblocks.size(); idx++) {
Subblock *current = &subblocks[idx];
if (current->level != num_levels)
continue;
while (current) {
if (current->cf >= 0) {
result_cf_[subblocks[idx].pos_y + result_regions_y_ * subblocks[idx].pos_x] = current->cf;
result_ie_[subblocks[idx].pos_y + result_regions_y_ * subblocks[idx].pos_x] = current->ie;
result_dr_[subblocks[idx].pos_y + result_regions_y_ * subblocks[idx].pos_x] = current->dr;
break;
}
current = current->superblock;
}
}
printf("PhaseEstimator::analyze end\n");
}
void EmptyTraceTracker::Allocate(const Mask *bfmask, const ImageSpecClass &imgSpec, int flow_block_size)
{
// assumes regions are indexed over a small non-negative range
imgFrames.resize(maxNumRegions);
// sdat handling
SynchDat sdat;
bool doSdat = inception_state.img_control.doSdat;
for (unsigned int i=0; i<regions.size(); i++){
Region region = regions[i];
imgFrames[region.index] = imgSpec.uncompFrames;
}
emptyTracesForBMFitter.resize(maxNumRegions);
for (int i=0; i<maxNumRegions; i++)
emptyTracesForBMFitter[i] = NULL;
for (unsigned int i=0; i<regions.size(); i++){
Region region = regions[i];
// allocate only once, if changed need to deallocate
assert ((region.index < maxNumRegions) & (region.index>=0));
assert (emptyTracesForBMFitter[region.index] == NULL);
// allocate empty traces, current algorithm is 1 per region
// each region is also used by a BkgModelFitters
EmptyTrace *emptyTrace = AllocateEmptyTrace(region, imgFrames[region.index], flow_block_size);
emptyTrace->SetTrimWildTraceOptions(inception_state.bkg_control.trace_control.do_ref_trace_trim,
inception_state.bkg_control.trace_control.span_inflator_min,
inception_state.bkg_control.trace_control.span_inflator_mult,
inception_state.bkg_control.trace_control.cutoff_quantile,
global_defaults.data_control.nuc_flow_frame_width);
emptyTrace->T0EstimateToMap(sep_t0_est, ®ion, bfmask);
if (doSdat){
// make the empty trace's notion of time conform to an sdat region's
// notion of time mapping the upper left corner of the empty trace's
// region to the corresponding sdat region. Note that multiple sdat
// regions may cover the empty trace's region
TraceChunkSerializer serializer;
char sdatFile[1024];
sprintf (sdatFile, "%s/%s%04d.%s", inception_state.sys_context.dat_source_directory,inception_state.img_control.acqPrefix,
0, inception_state.img_control.sdatSuffix.c_str() );
bool ok = serializer.Read ( sdatFile, sdat);
if (!ok) {
ION_ABORT("Couldn't load file: " + ToStr(sdatFile));
}
emptyTrace->SetTimeFromSdatChunk(region, sdat);
}
emptyTrace->CountReferenceTraces(region, bfmask);
//fprintf(stdout, "Found %d reference traces starting at %d in region %d\n", cnt, ((EmptyTraceRecorder *)emptyTrace)->regionIndicesStartIndex, region.index);
// assign the empty trace to the lookup vector BkgModelFitter can use
// all regions must match an entry into emptyTracesForBMFitter so
// every BkgModelFitter can find an EmptyTrace for its region.
emptyTracesForBMFitter[region.index] = emptyTrace;
}
if (inception_state.bkg_control.trace_control.do_ref_trace_trim)
InitializeDumpOutlierTracesFile();
}
void PhaseEstimator::DoPhaseEstimation(RawWells *wells, Mask *mask, const ion::FlowOrder& flow_order,
const vector<KeySequence>& keys, bool use_single_core)
{
// We only load / process what is necessary
flow_order_.SetFlowOrder(flow_order.str(), min(flow_order.num_flows(), phasing_end_flow_+20));
keys_ = keys;
// Do we have enough flows to do phase estimation?
// Check and, if necessary, adjust flow interval for estimation,
if (not have_phase_estimates_) {
if (flow_order_.num_flows() < 50) {
phasing_estimator_ = "override";
cout << "PhaseEstimator WARNING: Not enough flows to estimate phase; using default values." << endl;
}
else {
// Make sure we have at least 30 flows to estimate over
if (phasing_end_flow_ - phasing_start_flow_ < 30) {
phasing_end_flow_ = min(phasing_start_flow_+30, flow_order_.num_flows());
phasing_start_flow_ = phasing_end_flow_ - 30; // We are guaranteed to have at least 50 flows
cout << "PhaseEstimator WARNING: Shifting phase estimation window to flows " << phasing_start_flow_ << "-" << phasing_end_flow_ << endl;
cerr << "PhaseEstimator WARNING: Shifting phase estimation window to flows " << phasing_start_flow_ << "-" << phasing_end_flow_ << endl;
}
// Check boundaries of estimation window and adjust if necessary,
// try to keep estimation window size if possible, but don't start before flow 20
if (phasing_end_flow_ > flow_order_.num_flows()) {
phasing_start_flow_ = max(20, (phasing_start_flow_ - phasing_end_flow_ + flow_order_.num_flows()) );
phasing_end_flow_ = flow_order_.num_flows();
cout << "PhaseEstimator WARNING: Shifting phase estimation window to flows " << phasing_start_flow_ << "-" << phasing_end_flow_ << endl;
cerr << "PhaseEstimator WARNING: Shifting phase estimation window to flows " << phasing_start_flow_ << "-" << phasing_end_flow_ << endl;
}
}
}
// ------------------------------------
if (phasing_estimator_ == "override") {
if (not have_phase_estimates_)
SetPhaseParameters(init_cf_, init_ie_, init_dr_);
} else if (phasing_estimator_ == "spatial-refiner") {
int num_workers = max(numCores(), 2);
if (use_single_core)
num_workers = 1;
wells->Close();
wells->OpenForIncrementalRead();
SpatialRefiner(wells, mask, num_workers);
} else if (phasing_estimator_ == "spatial-refiner-2") {
int num_workers = max(numCores(), 2);
if (use_single_core)
num_workers = 1;
wells->Close();
wells->OpenForIncrementalRead();
train_subset_count_ = 2;
train_subset_cf_.resize(train_subset_count_);
train_subset_ie_.resize(train_subset_count_);
train_subset_dr_.resize(train_subset_count_);
train_subset_regions_x_.resize(train_subset_count_);
train_subset_regions_y_.resize(train_subset_count_);
for (train_subset_ = 0; train_subset_ < train_subset_count_; ++train_subset_) {
SpatialRefiner(wells, mask, num_workers);
train_subset_cf_[train_subset_] = result_cf_;
train_subset_ie_[train_subset_] = result_ie_;
train_subset_dr_[train_subset_] = result_dr_;
train_subset_regions_x_[train_subset_] = result_regions_x_;
train_subset_regions_y_[train_subset_] = result_regions_y_;
}
} else
ION_ABORT("Requested phase estimator is not recognized");
// Compute mean cf, ie, dr
average_cf_ = 0;
average_ie_ = 0;
average_dr_ = 0;
int count = 0;
for (int r = 0; r < result_regions_x_*result_regions_y_; r++) {
if (result_cf_.at(r) || result_ie_.at(r) || result_dr_.at(r)) {
average_cf_ += result_cf_[r];
average_ie_ += result_ie_[r];
average_dr_ += result_dr_[r];
count++;
}
}
if (count > 0) {
average_cf_ /= count;
average_ie_ /= count;
average_dr_ /= count;
}
have_phase_estimates_ = true;
}
void *FileSDatLoadWorker ( void *arg )
{
WorkerInfoQueue *q = static_cast<WorkerInfoQueue *> ( arg );
assert ( q );
bool done = false;
TraceChunkSerializer serializer;
while ( !done )
{
WorkerInfoQueueItem item = q->GetItem();
if ( item.finished == true )
{
// we are no longer needed...go away!
done = true;
q->DecrementDone();
continue;
}
ClockTimer timer;
ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;
bool ok = serializer.Read ( one_img_loader->name, one_img_loader->sdat[one_img_loader->cur_buffer] );
if (!ok) {
ION_ABORT("Couldn't load file: " + ToStr(one_img_loader->name));
}
one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer],(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0));
// one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, &one_img_loader->sdat[one_img_loader->cur_buffer] );
SynchDat &sdat = one_img_loader->sdat[one_img_loader->cur_buffer];
// if ( ImageTransformer::gain_correction != NULL )
// ImageTransformer::GainCorrectImage ( &sdat );
// ImageTransformer::GainCorrectImage ( &one_img_loader->sdat[one_img_loader->cur_buffer] );
// int buffer_ix = one_img_loader->cur_buffer;
if ( one_img_loader->inception_state->img_control.col_flicker_correct ) {
ComparatorNoiseCorrector cnc;
Mask &mask = *(one_img_loader->mask);
for (size_t rIx = 0; rIx < sdat.GetNumBin(); rIx++) {
TraceChunk &chunk = sdat.GetChunk(rIx);
// Copy over temp mask for normalization
Mask m(chunk.mWidth, chunk.mHeight);
for (size_t r = 0; r < chunk.mHeight; r++) {
for (size_t c = 0; c < chunk.mWidth; c++) {
m[r*chunk.mWidth+c] = mask[(r+chunk.mRowStart) * mask.W() + (c+chunk.mColStart)];
}
}
cnc.CorrectComparatorNoise(&chunk.mData[0], chunk.mHeight, chunk.mWidth, chunk.mDepth,
&m, one_img_loader->inception_state->img_control.col_flicker_correct_verbose,
one_img_loader->inception_state->img_control.aggressive_cnc);
}
}
// @todo output trace and dc offset info
sdat.AdjustForDrift();
sdat.SubDcOffset();
SetReadCompleted(one_img_loader);
size_t usec = timer.GetMicroSec();
fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.5lf sec\n", one_img_loader->flow, usec / 1.0e6);
q->DecrementDone();
}
return ( NULL );
}
void *FileSDatLoader ( void *arg )
{
ImageLoadWorkInfo *master_img_loader = ( ImageLoadWorkInfo * ) arg;
WorkerInfoQueue *loadWorkQ = new WorkerInfoQueue ( master_img_loader->flow_buffer_size );
ImageLoadWorkInfo *n_image_loaders = new ImageLoadWorkInfo[master_img_loader->flow_buffer_size];
SetUpIndividualImageLoaders ( n_image_loaders,master_img_loader );
int numWorkers = numCores() /2; // @TODO - this should be subject to inception_state options
//int numWorkers = 1;
numWorkers = ( numWorkers < 1 ? 1:numWorkers );
fprintf ( stdout, "FileLoader: numWorkers threads = %d\n", numWorkers );
{
int cworker;
pthread_t work_thread;
// spawn threads for doing background correction/fitting work
for ( cworker = 0; cworker < numWorkers; cworker++ )
{
int t = pthread_create ( &work_thread, NULL, FileSDatLoadWorker,
loadWorkQ );
pthread_detach(work_thread);
if ( t )
fprintf ( stderr, "Error starting thread\n" );
}
}
WorkerInfoQueueItem item;
Timer timer_file_access;
//double file_access_time = 0;
int flow_buffer_size = master_img_loader->flow_buffer_size;
for ( int i_buffer = 0; i_buffer < flow_buffer_size;i_buffer++ )
{
ImageLoadWorkInfo *cur_image_loader = &n_image_loaders[i_buffer];
int cur_flow = cur_image_loader->flow; // each job is an n_image_loaders item
DontReadAheadOfSignalProcessing (cur_image_loader, master_img_loader->lead);
TraceChunkSerializer serializer;
timer_file_access.restart();
bool ok = serializer.Read ( cur_image_loader->name, cur_image_loader->sdat[cur_image_loader->cur_buffer] );
if (!ok) {
ION_ABORT("Couldn't load file: " + ToStr(cur_image_loader->name));
}
// if ( ImageTransformer::gain_correction != NULL )
// ImageTransformer::GainCorrectImage ( &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
//file_access_time += timer_file_access.elapsed();
fprintf ( stdout, "File access = %0.2lf sec.\n", timer_file_access.elapsed() );
cur_image_loader->pinnedInFlow->Update ( cur_image_loader->flow, &cur_image_loader->sdat[cur_image_loader->cur_buffer] );
cur_image_loader->sdat[cur_image_loader->cur_buffer].AdjustForDrift();
cur_image_loader->sdat[cur_image_loader->cur_buffer].SubDcOffset();
item.finished = false;
item.private_data = cur_image_loader;
loadWorkQ->PutItem ( item );
if (ChipIdDecoder::GetGlobalChipId() != ChipId900)
PauseForLongCompute ( cur_flow,cur_image_loader );
/*if ( CheckFlowForWrite ( cur_flow,false ) )
{
fprintf (stdout, "File access Time for flow %d to %d: %.1f sec\n", ( ( cur_flow+1 ) - NUMFB ), cur_flow, file_access_time);
file_access_time = 0;
}*/
}
// wait for all of the images to be processed
loadWorkQ->WaitTillDone();
KillQueue ( loadWorkQ,numWorkers );
delete loadWorkQ;
delete[] n_image_loaders;
master_img_loader->finished = true;
return NULL;
}
