void testNoiseFloorVarianceCalculation()
{
if (os_) *os_ << "testNoiseFloorVarianceCalculation()\n";
if (os_) *os_ << setprecision(10);
// test noise floor calculation on sample frequency data
string filename = "FrequencyDataTest.cfd.temp.txt";
ofstream temp(filename.c_str());
temp << sampleFrequencyData_;
temp.close();
FrequencyData fd(filename);
boost::filesystem::remove(filename);
double result = fd.cutoffNoiseFloor();
if (os_) *os_ << "result: " << result << endl;
unit_assert_equal(result, 29000, 1000);
// test noise floor calculation on sample mass data
FrequencyData fdMasses;
CalibrationParameters cp = CalibrationParameters::thermo_FT();
for (RawMassDatum* p=sampleMassData_; p!=sampleMassData_+sampleMassDataSize_; ++p)
fdMasses.data().push_back(FrequencyDatum(cp.frequency(p->mz), p->intensity));
fdMasses.analyze();
double result2 = fdMasses.cutoffNoiseFloor();
if (os_) *os_ << "result2: " << result2 << endl;
unit_assert_equal(result2, 6000, 1000);
}
int noise(const vector<string>& args)
{
if (args.size() != 2) throw runtime_error(usage_);
const string& inputFilename = args[0];
const string& outputFilename = args[1];
const FrequencyData in(inputFilename);
FrequencyData out;
FrequencyData::container& data = out.data();
const double sd = 1/sqrt(2.);
Random::initialize();
for (FrequencyData::const_iterator it=in.data().begin(); it!=in.data().end(); ++it)
{
complex<double> noise(Random::gaussian(sd), Random::gaussian(sd));
data.push_back(FrequencyDatum(it->x, noise));
}
out.analyze();
out.write(outputFilename);
return 0;
}
void testFind()
{
FrequencyData fd;
copy(data_, data_+dataSize_, back_inserter(fd.data()));
if (os_) copy(fd.data().begin(), fd.data().end(), ostream_iterator<FrequencyDatum>(*os_, "\n"));
fd.analyze();
PeakData pd;
pd.scans.resize(3);
const double noiseFactor = 1;
auto_ptr<PeakDetectorNaive> pdn1 = PeakDetectorNaive::create(noiseFactor, 1);
pdn1->findPeaks(fd, pd.scans[0]);
unit_assert(pd.scans[0].peakFamilies.size() == 3);
auto_ptr<PeakDetectorNaive> pdn2 = PeakDetectorNaive::create(noiseFactor, 2);
pdn2->findPeaks(fd, pd.scans[1]);
unit_assert(pd.scans[1].peakFamilies.size() == 2);
auto_ptr<PeakDetectorNaive> pdn3 = PeakDetectorNaive::create(noiseFactor, 3);
pdn3->findPeaks(fd, pd.scans[2]);
unit_assert(pd.scans[2].peakFamilies.size() == 1);
if (os_)
{
*os_ << "pd:\n" << pd << endl;
for (unsigned int i=0; i<pd.scans.size(); i++)
{
*os_ << "scan " << i << ":\n";
*os_ << pd.scans[i] << endl;
}
}
}
void initializeWithTestData(FrequencyData& fd)
{
for (TestDatum* datum=testData_; datum<testData_+testDataSize_; datum++)
fd.data().push_back(FrequencyDatum(datum->frequency,
complex<double>(datum->real, datum->imaginary)));
fd.observationDuration(testDataObservationDuration_);
fd.calibrationParameters(CalibrationParameters(testDataCalibrationA_, testDataCalibrationB_));
fd.analyze();
}
void sinusoidTest()
{
TransientData td;
td.observationDuration(T_);
td.A(A_);
td.B(B_);
td.data().resize(sampleCount_);
SinusoidSignal signal;
td.add(signal);
td.write("sinusoid.dat");
FrequencyData fd;
td.computeFFT(1, fd);
fd.write("sinusoid.cfd");
}
int main(int argc, char* argv[])
{
if (argc < 4)
{
cout << "Usage: tlperror model.tlp data.cfd radius\n";
return 1;
}
const char* filenameModel = argv[1];
const char* filenameData = argv[2];
int radius = atoi(argv[3]);
try
{
if (radius < 1)
throw runtime_error("Positive radius required.\n");
TruncatedLorentzianParameters tlp(filenameModel);
const FrequencyData fdOriginal(filenameData);
auto_ptr<TruncatedLorentzianEstimator> estimator = TruncatedLorentzianEstimator::create();
const FrequencyData fd(fdOriginal, fdOriginal.max(), radius);
double N = fd.data().size();
double error = estimator->error(fd, tlp);
double normalizedError = estimator->normalizedError(fd, tlp);
double noiseVariance = fd.noiseFloor() * fd.noiseFloor();
double zscore = (error/noiseVariance - N) / (sqrt(N));
double sumSquaresModel = estimator->sumSquaresModel(fd, tlp);
cout << "# N error normalizedError noiseVariance z-score sumSquaresModel\n";
cout << N << " ";
cout << error << " ";
cout << normalizedError << " ";
cout << noiseVariance << " ";
cout << zscore << " ";
cout << sumSquaresModel << " ";
cout << endl;
return 0;
}
catch (exception& e)
{
cout << e.what() << endl;
return 1;
}
}
void NoisyData::errorFunctionCheck()
{
cout << "errorFunctionCheck()\n";
const ublas::vector<double>& p = tlp_.parameters();
for (double delta=1.; delta>1e-9; delta/=10.)
{
TruncatedLorentzianParameters tlp_shifted(tlp_);
tlp_shifted.f0 += delta;
const ublas::vector<double>& p_shifted = tlp_shifted.parameters();
double e = 0;
double e_shifted = 0;
double de1 = 0;
double de2 = 0;
for (FrequencyData::iterator it=fd_.data().begin(); it!=fd_.data().end(); ++it)
{
double f = it->x;
complex<double> L = L_(f, p);
complex<double> L_shifted = L_(f, p_shifted);
complex<double> noise = it->y - L;
complex<double> dL = L_.dp(f, p)(TruncatedLorentzian::F0);
complex<double> d2L = L_.dp2(f, p)(TruncatedLorentzian::F0, TruncatedLorentzian::F0);
e += norm(noise);
e_shifted += norm(it->y - L_shifted);
complex<double> term1 = dL + d2L*delta;
complex<double> term2 = dL*delta + .5*d2L*delta*delta - noise;
de1 += 2*real(term1 * conj(term2));
complex<double> term = norm(dL)*delta - dL*conj(noise) - d2L*conj(noise)*delta;
de2 += 2*real(term);
}
double differential = (e_shifted - e) / delta;
cout << delta << " " <<
differential << " " <<
de1 << " " <<
de2 << endl;
}
}
void NoisyData::generateData()
{
const ublas::vector<double>& p = tlp_.parameters();
const int sampleCount = 21;
double delta = 1/tlp_.T;
double fBegin = tlp_.f0 - delta*(sampleCount/2) + delta*relativeGridOffset_;
double peakHeight = abs(L_(tlp_.f0, p));
double noiseLevel = peakHeight / signalToNoise_;
noiseVariance_ = noiseLevel*noiseLevel;
for (int i=0; i<sampleCount; i++)
{
double f = fBegin + i*delta;
complex<double> value = L_(f, p);
complex<double> noise(Random::gaussian(noiseLevel/sqrt(2.)), Random::gaussian(noiseLevel/sqrt(2.)));
fd_.data().push_back(FrequencyDatum(f, value+noise));
}
fd_.calibration(Calibration(1.075e8, -3.455e8));
fd_.observationDuration(tlp_.T);
fd_.analyze(); // recache
}
void NoisyData::analyzeData()
{
const ublas::vector<double>& p = tlp_.parameters();
double sum_norm_dLdf0 = 0;
double sum_Re_dLdf0_conj_noise = 0;
double sum_Re_d2Ldf02_conj_noise = 0;
for (FrequencyData::iterator it=fd_.data().begin(); it!=fd_.data().end(); ++it)
{
double f = it->x;
complex<double> noise = it->y - L_(f, p);
complex<double> dLdf0 = L_.dp(f, p)(TruncatedLorentzian::F0);
complex<double> d2Ldf02 = L_.dp2(f, p)(TruncatedLorentzian::F0, TruncatedLorentzian::F0);
sum_Re_dLdf0_conj_noise += real(dLdf0 * conj(noise));
sum_norm_dLdf0 += norm(dLdf0);
sum_Re_d2Ldf02_conj_noise += real(d2Ldf02 * conj(noise));
}
predictedDeltaF0_ = sum_Re_dLdf0_conj_noise / (sum_norm_dLdf0 - sum_Re_d2Ldf02_conj_noise);
crBound_ = noiseVariance_ / sum_norm_dLdf0;
}
int calibrate(const FrequencyData& fd, const peakdata::Scan& scan,
const Configuration& config, const string& outputDirectory)
{
try
{
const string& pdbFilename = config.calibrate_database_filename;
cout << "Using database " << pdbFilename << endl;
auto_ptr<MassDatabase> mdb = MassDatabase::createFromPeptideDatabase(pdbFilename);
vector<Calibrator::Measurement> measurements = createMeasurements(scan);
if (measurements.empty())
throw runtime_error("No measurements read.");
CalibrationParameters cp = fd.calibrationParameters();
bfs::create_directories(outputDirectory);
cerr << "Running calibrator..." << flush;
auto_ptr<Calibrator> calibrator = Calibrator::create(*mdb,
measurements,
cp,
config.calibrate_initial_error_estimate,
config.calibrate_error_estimator_iteration_count,
outputDirectory);
for (unsigned int i=0; i<config.calibrate_calibrator_iteration_count; i++)
calibrator->iterate();
cerr << "done.\n";
cout << "A: " << calibrator->parameters().A << " B: " << calibrator->parameters().B << endl;
}
catch (exception& e)
{
cerr << e.what() << endl;
}
return 0;
}
void PeakDetectorNaiveImpl::findPeaks(const FrequencyData& fd, peakdata::Scan& result) const
{
result.scanNumber = fd.scanNumber();
result.retentionTime = fd.retentionTime();
result.observationDuration = fd.observationDuration();
result.calibrationParameters = fd.calibrationParameters();
result.peakFamilies.clear();
const double noiseLevel = sqrt(fd.variance());
const double threshold = noiseLevel * noiseFactor_;
for (FrequencyData::const_iterator it=fd.data().begin(); it!=fd.data().end(); ++it)
if (isPeak(it, fd.data(), threshold, detectionRadius_))
{
result.peakFamilies.push_back(PeakFamily());
PeakFamily& peakFamily = result.peakFamilies.back();
peakFamily.peaks.push_back(Peak());
Peak& peak = peakFamily.peaks.back();
peak.frequency = it->x;
peak.intensity = it->y.real();
peak.phase = it->y.imag();
}
}
void test()
{
// create some data, f(x) = abs(5-(x-2))
FrequencyData fd;
FrequencyData::container& data = fd.data();
for (int i=-5; i<=5; i++)
data.push_back(FrequencyDatum(i+2, 5-abs(i)));
fd.analyze(); // recache after changing data
// verify peak()
FrequencyData::const_iterator max = fd.max();
unit_assert(max->x == 2);
unit_assert(max->y == 5.);
// verify stats
unit_assert(fd.mean() == 25./11);
unit_assert(fd.meanSquare() == 85./11);
unit_assert(fd.sumSquares() == 85.);
unit_assert_equal(fd.variance(), 85./11 - 25.*25/11/11, 1e-12);
// write out data
if (os_) *os_ << "Writing " << filename1 << endl;
fd.write(filename1, FrequencyData::Text);
// read into const FrequencyData
string filename2 = "FrequencyDataTest.output2.txt";
FrequencyData fd2(filename1, FrequencyData::Text);
// verify normalize()
fd2.normalize();
unit_assert(fd2.shift() == -2);
unit_assert(fd2.scale() == 1./5);
max = fd2.max();
unit_assert(max->x == 0);
unit_assert(max->y == 1.);
// verify transform(shift, scale)
fd2.transform(-fd2.shift(), 1./fd2.scale());
// verify read/write
if (os_) *os_ << "Writing " << filename2 << endl;
fd2.write(filename2, FrequencyData::Text);
diff(filename1, filename2);
// test subrange
string filename3 = "FrequencyDataTest.output3.txt";
FrequencyData fd3(fd2, fd2.data().begin(), fd2.max()); // copy first half
if (os_) *os_ << "Writing " << filename3 << endl;
fd3.write(filename3, FrequencyData::Text);
FrequencyData fd4(fd2, fd2.max(), fd2.data().end()); // copy second half
ofstream os(filename3.c_str(), ios::app);
fd4.write(os, FrequencyData::Text);
os.close();
diff(filename1, filename3);
// read/write binary, and metadata
fd.scanNumber(555);
fd.retentionTime(444);
fd.calibrationParameters(CalibrationParameters(1,1));
fd.observationDuration(666);
fd.noiseFloor(777);
string filename4a = "FrequencyDataTest.output4a.txt";
if (os_) *os_ << "Writing " << filename4a << endl;
fd.write(filename4a, FrequencyData::Text);
string filenameBinary1 = "FrequencyDataTest.output1.cfd";
if (os_) *os_ << "Writing " << filenameBinary1 << endl;
fd.write(filenameBinary1);
FrequencyData fd5(filenameBinary1);
unit_assert(fd5.observationDuration() == 666);
fd5.observationDuration(fd.observationDurationEstimatedFromData());
unit_assert(fd5.scanNumber() == 555);
unit_assert(fd5.retentionTime() == 444);
unit_assert(fd5.observationDuration() == 1);
unit_assert(fd5.noiseFloor() == 777);
if (os_) *os_ << "Calibration: " << fd5.calibrationParameters().A << " " << fd5.calibrationParameters().B << endl;
string filename4b = "FrequencyDataTest.output4b.txt";
if (os_) *os_ << "Writing " << filename4b << endl;
fd5.write(filename4b, FrequencyData::Text);
diff(filename4a, filename4b);
fd.calibrationParameters(CalibrationParameters());
// test window
FrequencyData window1(fd, data.begin()+1, 2);
FrequencyData window2(fd, fd.max(), 1);
FrequencyData window3(fd, data.end()-2, 2);
string filename5 = "FrequencyDataTest.output5.txt";
if (os_) *os_ << "Writing " << filename5 << endl;
ofstream os5(filename5.c_str());
window1.write(os5, FrequencyData::Text);
window2.write(os5, FrequencyData::Text);
window3.write(os5, FrequencyData::Text);
os5.close();
diff(filename1, filename5);
}