ExitCodes outputTo(ostream& os)
{
//-------------------------------------------------------------
// Parameter handling
//-------------------------------------------------------------
// File names
String in = getStringOption_("in");
// File type
FileHandler fh;
FileTypes::Type in_type = FileTypes::nameToType(getStringOption_("in_type"));
if (in_type == FileTypes::UNKNOWN)
{
in_type = fh.getType(in);
writeDebug_(String("Input file type: ") + FileTypes::typeToName(in_type), 2);
}
if (in_type == FileTypes::UNKNOWN)
{
writeLog_("Error: Could not determine input file type!");
return PARSE_ERROR;
}
MSExperiment<Peak1D> exp;
FeatureMap feat;
ConsensusMap cons;
if (in_type == FileTypes::FEATUREXML) //features
{
FeatureXMLFile().load(in, feat);
feat.updateRanges();
}
else if (in_type == FileTypes::CONSENSUSXML) //consensus features
{
ConsensusXMLFile().load(in, cons);
cons.updateRanges();
}
//-------------------------------------------------------------
// meta information
//-------------------------------------------------------------
if (getFlag_("m"))
{
os << endl
<< "-- General information --" << endl
<< endl
<< "file name: " << in << endl
<< "file type: " << FileTypes::typeToName(in_type) << endl;
//basic info
os << endl
<< "-- Meta information --" << endl
<< endl;
if (in_type == FileTypes::FEATUREXML) //features
{
os << "Document id : " << feat.getIdentifier() << endl << endl;
}
else if (in_type == FileTypes::CONSENSUSXML) //consensus features
{
os << "Document id : " << cons.getIdentifier() << endl << endl;
}
}
//-------------------------------------------------------------
// data processing
//-------------------------------------------------------------
if (getFlag_("p"))
{
//basic info
os << endl
<< "-- Data processing information --" << endl
<< endl;
//get data processing info
vector<DataProcessing> dp;
if (in_type == FileTypes::FEATUREXML) //features
{
dp = feat.getDataProcessing();
}
else if (in_type == FileTypes::CONSENSUSXML) //consensus features
{
dp = cons.getDataProcessing();
}
int i = 0;
for (vector<DataProcessing>::iterator it = dp.begin(); it != dp.end(); ++it)
{
os << "Data processing " << i << endl;
os << "\tcompletion_time: " << (*it).getCompletionTime().getDate() << 'T' << (*it).getCompletionTime().getTime() << endl;
os << "\tsoftware name: " << (*it).getSoftware().getName() << " version " << (*it).getSoftware().getVersion() << endl;
for (set<DataProcessing::ProcessingAction>::const_iterator paIt = (*it).getProcessingActions().begin(); paIt != (*it).getProcessingActions().end(); ++paIt)
{
os << "\t\tprocessing action: " << DataProcessing::NamesOfProcessingAction[*paIt] << endl;
}
}
++i;
}
//-------------------------------------------------------------
// statistics
//-------------------------------------------------------------
if (getFlag_("s"))
{
//-------------------------------------------------------------
// Content statistics
//-------------------------------------------------------------
Map<String, int> meta_names;
if (in_type == FileTypes::FEATUREXML) //features
{
os << "Number of features: " << feat.size() << endl
<< endl
<< "Ranges:" << endl
<< " retention time: " << String::number(feat.getMin()[Peak2D::RT], 2) << " : " << String::number(feat.getMax()[Peak2D::RT], 2) << endl
<< " mass-to-charge: " << String::number(feat.getMin()[Peak2D::MZ], 2) << " : " << String::number(feat.getMax()[Peak2D::MZ], 2) << endl
<< " intensity: " << String::number(feat.getMinInt(), 2) << " : " << String::number(feat.getMaxInt(), 2) << endl
<< endl;
// Charge distribution
Map<UInt, UInt> charges;
for (Size i = 0; i < feat.size(); ++i)
{
charges[feat[i].getCharge()]++;
}
os << "Charge distribution" << endl;
for (Map<UInt, UInt>::const_iterator it = charges.begin();
it != charges.end(); ++it)
{
os << "charge " << it->first << ": " << it->second << endl;
}
}
else if (in_type == FileTypes::CONSENSUSXML) //consensus features
{
map<Size, UInt> num_consfeat_of_size;
for (ConsensusMap::const_iterator cmit = cons.begin();
cmit != cons.end(); ++cmit)
{
++num_consfeat_of_size[cmit->size()];
}
os << endl << "Number of consensus features:" << endl;
for (map<Size, UInt>::reverse_iterator i = num_consfeat_of_size.rbegin(); i != num_consfeat_of_size.rend(); ++i)
{
os << " of size " << setw(2) << i->first << ": " << setw(6) << i->second << endl;
}
os << " total: " << setw(6) << cons.size() << endl << endl;
os << "Ranges:" << endl
<< " retention time: " << String::number(cons.getMin()[Peak2D::RT], 2) << " : " << String::number(cons.getMax()[Peak2D::RT], 2) << endl
<< " mass-to-charge: " << String::number(cons.getMin()[Peak2D::MZ], 2) << " : " << String::number(cons.getMax()[Peak2D::MZ], 2) << endl
<< " intensity: " << String::number(cons.getMinInt(), 2) << " : " << String::number(cons.getMaxInt(), 2) << endl;
// file descriptions
const ConsensusMap::FileDescriptions& descs = cons.getFileDescriptions();
if (!descs.empty())
{
os << endl <<
"File descriptions:" << endl;
for (ConsensusMap::FileDescriptions::const_iterator it = descs.begin(); it != descs.end(); ++it)
{
os << " - " << it->second.filename << endl
<< " identifier: " << it->first << endl
<< " label : " << it->second.label << endl
<< " size : " << it->second.size << endl;
}
}
}
os << endl
<< "-- Summary Statistics --" << endl
<< endl;
}
if (in_type == FileTypes::FEATUREXML) //features
{
feat.sortByRT();
vector<double> slice_stats;
Size n = getIntOption_("n");
Size begin = 0;
Size end = 0;
os << "#slice\tRT_begin\tRT_end\tnumber_of_features\ttic\t"
<< "int_mean\tint_stddev\tint_min\tint_max\tint_median\tint_lowerq\tint_upperq\t"
<< "mz_mean\tmz_stddev\tmz_min\tmz_max\tmz_median\tmz_lowerq\tmz_upperq\t"
<< "width_mean\twidth_stddev\twidth_min\twidth_max\twidth_median\twidth_lowerq\twidth_upperq\t"
<< "qual_mean\tqual_stddev\tqual_min\tqual_max\tqual_median\tqual_lowerq\tqual_upperq\t"
<< "rt_qual_mean\trt_qual_stddev\trt_qual_min\trt_qual_max\trt_qual_median\trt_qual_lowerq\trt_qual_upperq\t"
<< "mz_qual_mean\tmz_qual_stddev\tmz_qual_min\tmz_qual_max\tmz_qual_median\tmz_qual_lowerq\tmz_qual_upperq"
<< endl;
double rt_begin = 0.0;
for (Size slice = 0; slice < n; ++slice)
{
// Determine slice boundaries.
double rt_end = feat.back().getRT() / (double)n * (slice + 1);
for (end = begin; end < feat.size() && feat[end].getRT() < rt_end; ++end) {}
// Compute statistics on all features in this slice.
slice_stats = sliceStatistics(feat, begin, end);
// Write the beginning and end of the slices to the output as well as the slice index.
os << slice << "\t" << rt_begin << "\t" << rt_end << "\t" << end - begin << "\t";
// Write the statistics as a line of an csv file
copy(slice_stats.begin(), slice_stats.end(), ostream_iterator<double>(os, "\t"));
os << endl;
begin = end;
rt_begin = rt_end;
}
}
else if (in_type == FileTypes::CONSENSUSXML) //consensus features
{
Size size = cons.size();
vector<double> intensities;
intensities.reserve(size);
vector<double> qualities(size);
qualities.reserve(size);
vector<double> widths(size);
widths.reserve(size);
vector<double> rt_delta_by_elems;
vector<double> rt_aad_by_elems;
vector<double> rt_aad_by_cfs;
rt_aad_by_cfs.reserve(size);
vector<double> mz_delta_by_elems;
vector<double> mz_aad_by_elems;
vector<double> mz_aad_by_cfs;
mz_aad_by_cfs.reserve(size);
vector<double> it_delta_by_elems;
vector<double> it_aad_by_elems;
vector<double> it_aad_by_cfs;
it_aad_by_cfs.reserve(size);
for (ConsensusMap::const_iterator cm_iter = cons.begin();
cm_iter != cons.end(); ++cm_iter)
{
double rt_aad = 0;
double mz_aad = 0;
double it_aad = 0;
intensities.push_back(cm_iter->getIntensity());
qualities.push_back(cm_iter->getQuality());
widths.push_back(cm_iter->getWidth());
for (ConsensusFeature::HandleSetType::const_iterator hs_iter = cm_iter->begin();
hs_iter != cm_iter->end(); ++hs_iter)
{
double rt_diff = hs_iter->getRT() - cm_iter->getRT();
rt_delta_by_elems.push_back(rt_diff);
if (rt_diff < 0)
{
rt_diff = -rt_diff;
}
rt_aad_by_elems.push_back(rt_diff);
rt_aad += rt_diff;
double mz_diff = hs_iter->getMZ() - cm_iter->getMZ();
mz_delta_by_elems.push_back(mz_diff);
if (mz_diff < 0)
{
mz_diff = -mz_diff;
}
mz_aad_by_elems.push_back(mz_diff);
mz_aad += mz_diff;
double it_ratio = hs_iter->getIntensity() / (cm_iter->getIntensity() ? cm_iter->getIntensity() : 1.);
it_delta_by_elems.push_back(it_ratio);
if (it_ratio < 1.)
{
it_ratio = 1. / it_ratio;
}
it_aad_by_elems.push_back(it_ratio);
it_aad += it_ratio;
}
if (!cm_iter->empty())
{
rt_aad /= cm_iter->size();
mz_aad /= cm_iter->size();
it_aad /= cm_iter->size();
} // otherwise rt_aad etc. are 0 anyway
rt_aad_by_cfs.push_back(rt_aad);
mz_aad_by_cfs.push_back(mz_aad);
it_aad_by_cfs.push_back(it_aad);
}
OpenMS::SomeStatistics some_statistics;
os.precision(writtenDigits(ConsensusFeature::IntensityType()));
os << "Intensities of consensus features:" << endl << some_statistics(intensities) << endl;
os.precision(writtenDigits(ConsensusFeature::QualityType()));
os << "Qualities of consensus features:" << endl << some_statistics(qualities) << endl;
os.precision(writtenDigits(ConsensusFeature::CoordinateType()));
os << "Retention time differences ( element-center, weight 1 per element):" << endl << some_statistics(rt_delta_by_elems) << endl;
os << "Absolute retention time differences ( |element-center|, weight 1 per element):" << endl << some_statistics(rt_aad_by_elems) << endl;
os << "Average absolute differences of retention time within consensus features ( |element-center|, weight 1 per consensus features):" << endl << some_statistics(rt_aad_by_cfs) << endl;
os.precision(writtenDigits(ConsensusFeature::CoordinateType()));
os << "Mass-to-charge differences ( element-center, weight 1 per element):" << endl << some_statistics(mz_delta_by_elems) << endl;
os << "Absolute differences of mass-to-charge ( |element-center|, weight 1 per element):" << endl << some_statistics(mz_aad_by_elems) << endl;
os << "Average absolute differences of mass-to-charge within consensus features ( |element-center|, weight 1 per consensus features):" << endl << some_statistics(mz_aad_by_cfs) << endl;
os.precision(writtenDigits(ConsensusFeature::IntensityType()));
os << "Intensity ratios ( element/center, weight 1 per element):" << endl << some_statistics(it_delta_by_elems) << endl;
os << "Relative intensity error ( max{(element/center),(center/element)}, weight 1 per element):" << endl << some_statistics(it_aad_by_elems) << endl;
os << "Average relative intensity error within consensus features ( max{(element/center),(center/element)}, weight 1 per consensus features):" << endl << some_statistics(it_aad_by_cfs) << endl;
}
return EXECUTION_OK;
}
void PoseClusteringShiftSuperimposer::run(const ConsensusMap & map_model, const ConsensusMap & map_scene, TransformationDescription & transformation)
{
typedef ConstRefVector<ConsensusMap> PeakPointerArray_;
typedef Math::LinearInterpolation<double, double> LinearInterpolationType_;
LinearInterpolationType_ shift_hash_;
// OLD STUFF
// LinearInterpolationType_ scaling_hash_1;
// LinearInterpolationType_ scaling_hash_2;
// LinearInterpolationType_ shift_hash_;
// LinearInterpolationType_ rt_high_hash_;
/// Maximum deviation in mz of two partner points
const double mz_pair_max_distance = param_.getValue("mz_pair_max_distance");
/// Size of each shift bucket
const double shift_bucket_size = param_.getValue("shift_bucket_size");
const UInt struc_elem_length_datapoints = 21; // MAGIC ALERT: number of data points in structuring element for tophat filter, which removes baseline from histogram
const double scaling_histogram_crossing_slope = 3.0; // MAGIC ALERT: used when distinguishing noise level and enriched histogram bins
const double scaling_cutoff_stdev_multiplier = 1.5; // MAGIC ALERT: multiplier for stdev in cutoff for outliers
const UInt loops_mean_stdev_cutoff = 3; // MAGIC ALERT: number of loops in stdev cutoff for outliers
startProgress(0, 100, "shift pose clustering");
UInt actual_progress = 0;
setProgress(++actual_progress);
// Optionally, we will write dumps of the hash table buckets.
bool do_dump_buckets = false;
String dump_buckets_basename;
if (param_.getValue("dump_buckets") != "")
{
do_dump_buckets = true;
dump_buckets_basename = param_.getValue("dump_buckets");
}
setProgress(++actual_progress);
// Even more optionally, we will write dumps of the hashed pairs.
bool do_dump_pairs = false;
String dump_pairs_basename;
if (param_.getValue("dump_pairs") != "")
{
do_dump_pairs = true;
dump_pairs_basename = param_.getValue("dump_pairs");
}
setProgress(++actual_progress);
//**************************************************************************
// Select the most abundant data points only. After that, disallow modifications
// (we tend to have annoying issues with const_iterator versus iterator).
PeakPointerArray_ model_map_ini(map_model.begin(), map_model.end());
const PeakPointerArray_ & model_map(model_map_ini);
PeakPointerArray_ scene_map_ini(map_scene.begin(), map_scene.end());
const PeakPointerArray_ & scene_map(scene_map_ini);
{
// truncate the data as necessary
// casting to SignedSize is done on PURPOSE here! (num_used_points will be maximal if -1 is used)
const Size num_used_points = (SignedSize) param_.getValue("num_used_points");
if (model_map_ini.size() > num_used_points)
{
model_map_ini.sortByIntensity(true);
model_map_ini.resize(num_used_points);
}
model_map_ini.sortByComparator(Peak2D::MZLess());
setProgress(++actual_progress);
if (scene_map_ini.size() > num_used_points)
{
scene_map_ini.sortByIntensity(true);
scene_map_ini.resize(num_used_points);
}
scene_map_ini.sortByComparator(Peak2D::MZLess());
setProgress(++actual_progress);
// Note: model_map_ini and scene_map_ini will not be used further below
}
setProgress((actual_progress = 10));
//**************************************************************************
// Preprocessing
// get RT ranges (NOTE: we trust that min and max have been updated in the
// ConsensusMap::convert() method !)
const double model_low = map_model.getMin()[ConsensusFeature::RT];
const double scene_low = map_scene.getMin()[ConsensusFeature::RT];
const double model_high = map_model.getMax()[ConsensusFeature::RT];
const double scene_high = map_scene.getMax()[ConsensusFeature::RT];
// OLD STUFF
// const double rt_low = (maps[0].getMin()[ConsensusFeature::RT] + maps[1].getMin()[ConsensusFeature::RT]) / 2.;
// const double rt_high = (maps[0].getMax()[ConsensusFeature::RT] + maps[1].getMax()[ConsensusFeature::RT]) / 2.;
// Initialize the hash tables: shift_hash_
// OLD STUFF: was: rt_scaling_hash_, rt_low_hash_, and rt_high_hash_
{
// (over)estimate the required number of buckets for shifting
double max_shift = param_.getValue("max_shift");
// actually the largest possible shift can be much smaller, depending on the data
do
{
if (max_shift < 0)
max_shift = -max_shift;
// ...ml@@@mh........ , ........ml@@@mh...
// ........sl@@@sh... , ...sl@@@sh........
double diff;
diff = model_high - scene_low;
if (diff < 0)
diff = -diff;
if (max_shift > diff)
max_shift = diff;
diff = model_low - scene_high;
if (diff < 0)
diff = -diff;
if (max_shift > diff)
max_shift = diff;
}
while (0);
const Int shift_buckets_num_half = 4 + (Int) ceil((max_shift) / shift_bucket_size);
const Int shift_buckets_num = 1 + 2 * shift_buckets_num_half;
shift_hash_.getData().clear();
shift_hash_.getData().resize(shift_buckets_num);
shift_hash_.setMapping(shift_bucket_size, shift_buckets_num_half, 0);
}
setProgress(++actual_progress);
//**************************************************************************
// compute the ratio of the total intensities of both maps, for normalization
double total_intensity_ratio;
do
{
double total_int_model_map = 0;
for (Size i = 0; i < model_map.size(); ++i)
{
total_int_model_map += model_map[i].getIntensity();
}
setProgress(++actual_progress);
double total_int_scene_map = 0;
for (Size i = 0; i < scene_map.size(); ++i)
{
total_int_scene_map += scene_map[i].getIntensity();
}
setProgress(++actual_progress);
// ... and finally ...
total_intensity_ratio = total_int_model_map / total_int_scene_map;
}
while (0); // (the extra syntax helps with code folding in eclipse!)
setProgress((actual_progress = 20));
/// The serial number is incremented for each invocation of this, to avoid overwriting of hash table dumps.
static Int dump_buckets_serial = 0;
++dump_buckets_serial;
//**************************************************************************
// Hashing
// Compute the transformations between each point pair in the model map
// and each point pair in the scene map and hash the shift
// transformation.
// To speed up the calculation of the final transformation, we confine the number of
// considered point pairs. We match a point p in the model map only onto those points p'
// in the scene map that lie in a certain mz interval.
Size const model_map_size = model_map.size(); // i /* OLD STUFF: also: j */
Size const scene_map_size = scene_map.size(); // k /* OLD STUFF: also: l */
const double winlength_factor_baseline = 0.1; // MAGIC ALERT: Each window is given unit weight. If there are too many pairs for a window, the individual contributions will be very small, but running time will be high, so we provide a cutoff for this. Typically this will exclude compounds which elute over the whole retention time range from consideration.
///////////////////////////////////////////////////////////////////
// Hashing: Estimate the shift
do // begin of hashing (the extra syntax helps with code folding in eclipse!)
{
String dump_pairs_filename;
std::ofstream dump_pairs_file;
if (do_dump_pairs)
{
dump_pairs_filename = dump_pairs_basename + String(dump_buckets_serial);
dump_pairs_file.open(dump_pairs_filename.c_str());
dump_pairs_file << "#" << ' ' << "i" << ' ' << "k" << std::endl;
}
setProgress(++actual_progress);
// first point in model map
for (Size i = 0, i_low = 0, i_high = 0, k_low = 0, k_high = 0; i < model_map_size - 1; ++i)
{
setProgress(actual_progress + float(i) / model_map_size * 10.f);
// Adjust window around i in model map
while (i_low < model_map_size && model_map[i_low].getMZ() < model_map[i].getMZ() - mz_pair_max_distance)
++i_low;
while (i_high < model_map_size && model_map[i_high].getMZ() <= model_map[i].getMZ() + mz_pair_max_distance)
++i_high;
double i_winlength_factor = 1. / (i_high - i_low);
i_winlength_factor -= winlength_factor_baseline;
if (i_winlength_factor <= 0)
continue;
// Adjust window around k in scene map
while (k_low < scene_map_size && scene_map[k_low].getMZ() < model_map[i].getMZ() - mz_pair_max_distance)
++k_low;
while (k_high < scene_map_size && scene_map[k_high].getMZ() <= model_map[i].getMZ() + mz_pair_max_distance)
++k_high;
// first point in scene map
for (Size k = k_low; k < k_high; ++k)
{
double k_winlength_factor = 1. / (k_high - k_low);
k_winlength_factor -= winlength_factor_baseline;
if (k_winlength_factor <= 0)
continue;
// compute similarity of intensities i k
double similarity_ik;
{
const double int_i = model_map[i].getIntensity();
const double int_k = scene_map[k].getIntensity() * total_intensity_ratio;
similarity_ik = (int_i < int_k) ? int_i / int_k : int_k / int_i;
// weight is inverse proportional to number of elements with similar mz
similarity_ik *= i_winlength_factor;
similarity_ik *= k_winlength_factor;
// VV_(int_i<<' '<<int_k<<' '<<int_similarity_ik);
}
// compute the transformation (i) -> (k)
double shift = model_map[i].getRT() - scene_map[k].getRT();
// hash the images of scaling, rt_low and rt_high into their respective hash tables
shift_hash_.addValue(shift, similarity_ik);
if (do_dump_pairs)
{
dump_pairs_file << i << ' ' << model_map[i].getRT() << ' ' << model_map[i].getMZ() << ' ' << k << ' ' << scene_map[k].getRT() << ' '
<< scene_map[k].getMZ() << ' ' << similarity_ik << ' ' << std::endl;
}
} // k
} // i
}
while (0); // end of hashing (the extra syntax helps with code folding in eclipse!)
setProgress((actual_progress = 30));
///////////////////////////////////////////////////////////////////
// work on shift_hash_
// double shift_low;
// double shift_centroid;
// double shift_high;
// OLD STUFF
// double shift_low;
double shift_centroid;
// double shift_high;
do
{
UInt filtering_stage = 0;
// optionally, dump before filtering
String dump_buckets_filename;
std::ofstream dump_buckets_file;
if (do_dump_buckets)
{
dump_buckets_filename = dump_buckets_basename + "_" + String(dump_buckets_serial);
dump_buckets_file.open(dump_buckets_filename.c_str());
VV_(dump_buckets_filename);
dump_buckets_file << "# shift hash table buckets dump ( scale, height ) : " << dump_buckets_filename << std::endl;
dump_buckets_file << "# unfiltered hash data\n";
for (Size index = 0; index < shift_hash_.getData().size(); ++index)
{
const double image = shift_hash_.index2key(index);
const double height = shift_hash_.getData()[index];
dump_buckets_file << filtering_stage << '\t' << index << '\t' << image << '\t' << height << '\n';
}
dump_buckets_file << '\n';
}
++filtering_stage;
setProgress(++actual_progress);
// apply tophat filter to histogram
MorphologicalFilter morph_filter;
Param morph_filter_param;
morph_filter_param.setValue("struc_elem_unit", "DataPoints");
morph_filter_param.setValue("struc_elem_length", double(struc_elem_length_datapoints));
morph_filter_param.setValue("method", "tophat");
morph_filter.setParameters(morph_filter_param);
LinearInterpolationType_::container_type buffer(shift_hash_.getData().size());
morph_filter.filterRange(shift_hash_.getData().begin(), shift_hash_.getData().end(), buffer.begin());
shift_hash_.getData().swap(buffer);
// optionally, dump after filtering
if (do_dump_buckets)
{
dump_buckets_file << "# tophat filtered hash data\n";
for (Size index = 0; index < shift_hash_.getData().size(); ++index)
{
const double image = shift_hash_.index2key(index);
const double height = shift_hash_.getData()[index];
dump_buckets_file << filtering_stage << '\t' << index << '\t' << image << '\t' << height << '\n';
}
dump_buckets_file << '\n';
}
setProgress(++actual_progress);
++filtering_stage;
// compute freq_cutoff using a fancy criterion to distinguish between the noise level of the histogram and enriched histogram bins
double freq_cutoff_low;
do
{
{
std::copy(shift_hash_.getData().begin(), shift_hash_.getData().end(), buffer.begin());
std::sort(buffer.begin(), buffer.end(), std::greater<double>());
double freq_intercept = shift_hash_.getData().front();
double freq_slope = (shift_hash_.getData().back() - shift_hash_.getData().front()) / double(buffer.size())
/ scaling_histogram_crossing_slope;
if (!freq_slope || !buffer.size())
{
// in fact these conditions are actually impossible, but let's be really sure ;-)
freq_cutoff_low = 0;
}
else
{
Size index = 1; // not 0 (!)
while (buffer[index] >= freq_intercept + freq_slope * double(index))
{
++index;
}
freq_cutoff_low = buffer[--index]; // note that we have index >= 1
}
}
}
while (0);
setProgress(++actual_progress);
// apply freq_cutoff, setting smaller values to zero
for (Size index = 0; index < shift_hash_.getData().size(); ++index)
{
if (shift_hash_.getData()[index] < freq_cutoff_low)
{
shift_hash_.getData()[index] = 0;
}
}
setProgress(++actual_progress);
// optionally, dump after noise filtering using freq_cutoff
if (do_dump_buckets)
{
dump_buckets_file << "# after freq_cutoff, which is: " << freq_cutoff_low << '\n';
for (Size index = 0; index < shift_hash_.getData().size(); ++index)
{
const double image = shift_hash_.index2key(index);
const double height = shift_hash_.getData()[index];
dump_buckets_file << filtering_stage << '\t' << index << '\t' << image << '\t' << height << '\n';
}
dump_buckets_file << '\n';
}
setProgress(++actual_progress);
// iterative cut-off based on mean and stdev - relies upon scaling_cutoff_stdev_multiplier which is a bit hard to set right.
{
Math::BasicStatistics<double> statistics;
std::vector<double>::const_iterator data_begin = shift_hash_.getData().begin();
const Size data_size = shift_hash_.getData().size();
Size data_range_begin = 0;
Size data_range_end = data_size;
for (UInt loop = 0; loop < loops_mean_stdev_cutoff; ++loop) // MAGIC ALERT: number of loops
{
statistics.update(data_begin + data_range_begin, data_begin + data_range_end);
double mean = statistics.mean() + data_range_begin;
double stdev = sqrt(statistics.variance());
data_range_begin = floor(std::max<double>(mean - scaling_cutoff_stdev_multiplier * stdev, 0));
data_range_end = ceil(std::min<double>(mean + scaling_cutoff_stdev_multiplier * stdev + 1, data_size));
const double outside_mean = shift_hash_.index2key(mean);
const double outside_stdev = stdev * shift_hash_.getScale();
// shift_low = (outside_mean - outside_stdev);
shift_centroid = (outside_mean);
// shift_high = (outside_mean + outside_stdev);
if (do_dump_buckets)
{
dump_buckets_file << "# loop: " << loop << " mean: " << outside_mean << " stdev: " << outside_stdev << " (mean-stdev): "
<< outside_mean - outside_stdev << " (mean+stdev): " << outside_mean + outside_stdev
<< " data_range_begin: " << data_range_begin << " data_range_end: "
<< data_range_end << std::endl;
}
}
setProgress(++actual_progress);
}
if (do_dump_buckets)
{
dump_buckets_file << "# EOF" << std::endl;
dump_buckets_file.close();
}
setProgress(80);
}
while (0);
//************************************************************************************
// Estimate transform
// Compute the shifts at the low and high ends by looking at (around) the fullest bins.
double intercept;
#if 1 // yes of course, use centroids for images of rt_low and rt_high
intercept = shift_centroid;
#else // ooh, use maximum bins instead (Note: this is a fossil which would disregard most of the above computations! The code is left here for developers/debugging only.)
const Size rt_low_max_index = std::distance(shift_hash_.getData().begin(),
std::max_element(shift_hash_.getData().begin(), shift_hash_.getData().end()));
intercept = shift_hash_.index2key(rt_low_max_index);
#endif
VV_(intercept);
setProgress(++actual_progress);
// set trafo
{
Param params;
params.setValue("slope", 1.0);
params.setValue("intercept", intercept);
TransformationDescription trafo;
trafo.fitModel("linear", params);
transformation = trafo;
}
setProgress(++actual_progress);
endProgress();
return;
} // run()