NumericalComparison<double> CompareWells(const string &queryFile, const string &goldFile,
float epsilon, double maxAbsVal) {
NumericalComparison<double> compare(epsilon);
string queryDir, queryWells, goldDir, goldWells;
FillInDirName(queryFile, queryDir, queryWells);
FillInDirName(goldFile, goldDir, goldWells);
RawWells queryW(queryDir.c_str(), queryWells.c_str());
RawWells goldW(goldDir.c_str(), goldWells.c_str());
struct WellData goldData;
goldData.flowValues = NULL;
struct WellData queryData;
queryData.flowValues = NULL;
cout << "Opening query." << endl;
queryW.OpenForRead();
cout << "Opening gold." << endl;
goldW.OpenForRead();
unsigned int numFlows = goldW.NumFlows();
while( !queryW.ReadNextRegionData(&queryData) ) {
assert(!goldW.ReadNextRegionData(&goldData));
for (unsigned int i = 0; i < numFlows; i++) {
if (isfinite(queryData.flowValues[i]) && isfinite(goldData.flowValues[i]) &&
(fabs(queryData.flowValues[i]) < maxAbsVal && fabs(goldData.flowValues[i]) < maxAbsVal)) {
compare.AddPair(queryData.flowValues[i], goldData.flowValues[i]);
}
}
}
const SampleStats<double> ssX = compare.GetXStats();
const SampleStats<double> ssY = compare.GetYStats();
cout << "query values: " << ssX.GetMean() << " +/- " << ssX.GetSD() << endl;
cout << "gold values: " << ssY.GetMean() << " +/- " << ssY.GetSD() << endl;
return compare;
}
void Traces::CalcIncorporationRegionStart(size_t rowStart, size_t rowEnd,
size_t colStart, size_t colEnd,
SampleStats<float> &starts, MaskType maskType) {
FindSlopeChange<float> finder;
double xDist = mCol - ((colStart + colEnd)/2.0);
double yDist = mRow - ((rowStart + rowEnd)/2.0);
int offset = floor(xDist * yDist * 3.738038e-06);
size_t startNucFrame = 0;
size_t iGuessStart = 14+offset, iGuessEnd = 24+offset;
// size_t iRangeStart = 10, iRangeEnd = 60, iGuessStart = 16, iGuessEnd = 45;
// size_t iRangeStart = 0, iRangeEnd = 25, iGuessStart = 3, iGuessEnd = 10;
/* 15-25 is usual start
17-25
*/
Mask &mask = *mMask;
vector<SampleQuantiles<float> >frameAvgs(80);
for (size_t i = 0; i < frameAvgs.size(); i++) {
frameAvgs[i].Init(200);
}
vector<float> trace;
vector<float> traceBuffer;
for (size_t rowIx = rowStart; rowIx < rowEnd; rowIx++) {
for (size_t colIx = colStart; colIx < colEnd; colIx++) {
size_t idx = RowColToIndex(rowIx, colIx);
if ((mask[idx] & maskType) && mFlags[idx] == OK) {
GetTraces(idx, traceBuffer);
for (size_t i = 0; i < frameAvgs.size(); i++) {
frameAvgs[i].AddValue(traceBuffer[i]);
}
}
}
}
if (frameAvgs[0].GetNumSeen() < MIN_T0_PROBES) {
return;
}
trace.resize(frameAvgs.size());
for (size_t i = 0; i < frameAvgs.size(); i++) {
trace[i] = frameAvgs[i].GetMedian();
}
bool ok = true;
// finder.findChangeIndex(startNucFrame, startSumSeq, trace, iRangeStart, iRangeEnd, iGuessStart, iGuessEnd);
if (startNucFrame >= iGuessEnd || startNucFrame <= iGuessStart) {
ok = false;
startNucFrame = -2;
}
if (ok && frameAvgs[0].GetNumSeen() >= MIN_T0_PROBES ) {
starts.AddValue(startNucFrame-.3);
}
}
int main(int argc, const char *argv[]) {
OptArgs opts;
TraceConfig config;
string inputDir;
string outputDir;
bool help;
opts.ParseCmdLine(argc, argv);
opts.GetOption(inputDir, "", '-', "source-dir");
opts.GetOption(outputDir, "", '-', "output-dir");
opts.GetOption(config.precision, "5", '-', "precision");
opts.GetOption(config.numEvec, "7", '-', "num-evec");
opts.GetOption(config.doDebug, "false", '-', "debug-files");
opts.GetOption(config.compressionType, "delta", '-', "compression");
opts.GetOption(config.numFlows, "-1", '-', "num-flows");
opts.GetOption(config.numCores, "6", '-', "num-cores");
opts.GetOption(config.errCon,"0",'-',"err-con");
opts.GetOption(config.rankGood,"0",'-',"rank-good");
opts.GetOption(config.pivot,"0",'-',"pivot");
opts.GetOption(help, "false", 'h', "help");
opts.GetOption(config.isThumbnail, "false", '-', "thumbnail");
opts.GetOption(config.use_hard_est, "false",'-', "use-hard-est");
opts.GetOption(config.t0_hard, "0", '-', "t0-hard");
opts.GetOption(config.tmid_hard, "0", '-', "tmid-hard");
opts.GetOption(config.sigma_hard, "0", '-', "sigma-hard");
opts.GetOption(config.row_step, "100", '-', "row-step");
opts.GetOption(config.col_step, "100", '-', "col-step");
opts.GetOption(config.bg_param, "", '-', "region-param");
opts.GetOption(config.grind_acq_0, "0", '-', "grind-acq0");
if(help || inputDir.empty() || outputDir.empty()) {
usage();
}
char *explog_path = NULL;
explog_path = MakeExpLogPathFromDatDir(inputDir.c_str());
int numFlows = config.numFlows;
if (numFlows < 0) {
numFlows = GetTotalFlows(explog_path);
}
// Check and setup our compression type
TraceChunkSerializer serializer;
serializer.SetRecklessAbandon(true);
if (config.compressionType == "svd") {
SvdDatCompress *dc = new SvdDatCompress(config.precision, config.numEvec);
serializer.SetCompressor(dc);
cout << "Doing lossy svd compression. (" << serializer.GetCompressionType() << ")" << endl;
}
// else if (config.compressionType == "svd+") {
// SvdDatCompressPlus *dc = new SvdDatCompressPlus();
// serializer.SetCompressor(dc);
// cout << "Doing lossy svd compression. (" << serializer.GetCompressionType() << ")" << endl;
// }
// else if (config.compressionType == "svd++") {
// SvdDatCompressPlusPlus *dc = new SvdDatCompressPlusPlus();
// if (config.errCon >0 )
// dc->SetErrCon(config.errCon);
// if (config.rankGood > 0 )
// dc->SetRankGood(config.rankGood);
// if (config.pivot > 0)
// dc->SetPivot(config.pivot);
// serializer.SetCompressor(dc);
// cout << "Doing lossy svd compression for good traces and delta for bad ones. (" << serializer.GetCompressionType() << ")" << endl;
// }
else if (config.compressionType == "delta") {
VencoLossless *venco = new VencoLossless();
serializer.SetCompressor(venco);
cout << "Doing lossless delta compression. (" << serializer.GetCompressionType() << ")" << endl;
}
else if (config.compressionType == "delta-plain") {
DeltaComp *delta = new DeltaComp();
serializer.SetCompressor(delta);
cout << "Doing lossless delta plain compression. (" << serializer.GetCompressionType() << ")" << endl;
}
else if (config.compressionType == "delta-plain-fst") {
DeltaCompFst *delta = new DeltaCompFst();
serializer.SetCompressor(delta);
cout << "Doing lossless delta plain fast compression. (" << serializer.GetCompressionType() << ")" << endl;
}
else if (config.compressionType == "delta-plain-fst-smx") {
DeltaCompFstSmX *delta = new DeltaCompFstSmX();
serializer.SetCompressor(delta);
cout << "Doing lossless delta plain fast compression. (" << serializer.GetCompressionType() << ")" << endl;
}
else if (config.compressionType == "none") {
TraceCompressor *vanilla = new TraceNoCompress();
serializer.SetCompressor(vanilla);
cout << "Doing no compression. (" << serializer.GetCompressionType() << ")" << endl;
}
else {
ION_ABORT("Don't recognize compression type: " + config.compressionType);
}
const char *id = GetChipId(explog_path);
if (explog_path) free (explog_path);
ChipIdDecoder::SetGlobalChipId(id);
ImageTransformer::CalibrateChannelXTCorrection(inputDir.c_str(), "lsrowimage.dat");
Image bfImg1;
string bfFile = inputDir + "/beadfind_pre_0003.dat";
bfImg1.LoadRaw(bfFile.c_str());
const RawImage *bf1raw = bfImg1.GetImage();
Mask mask(bf1raw->cols, bf1raw->rows);
ImageTransformer::XTChannelCorrect(bfImg1.raw,bfImg1.results_folder);
bfImg1.FilterForPinned (&mask, MaskEmpty, false);
Image bfImg2;
string bfFile2 = inputDir + "/beadfind_pre_0001.dat";
bfImg2.LoadRaw(bfFile2.c_str());
ImageTransformer::XTChannelCorrect(bfImg2.raw,bfImg1.results_folder);
bfImg2.FilterForPinned (&mask, MaskEmpty, false);
const RawImage *bf2raw = bfImg2.GetImage();
GridMesh<T0Prior> t0Prior;
T0Calc bfT0;
/* Calc t0 and get prior. */
cout << "Doing beadfind t0" << endl;
GenerateBfT0Prior(config, bf1raw->image, bf1raw->baseFrameRate, bf1raw->rows, bf1raw->cols,
bf1raw->frames, bf1raw->timestamps,
config.row_step, config.col_step, &mask, bfT0, t0Prior);
GridMesh<T0Prior> t0Prior2;
T0Calc bfT02;
GenerateBfT0Prior(config, bf2raw->image, bf2raw->baseFrameRate, bf2raw->rows, bf2raw->cols,
bf2raw->frames, bf2raw->timestamps,
config.row_step, config.col_step, &mask, bfT02, t0Prior2);
SigmaTMidNucEstimation sigmaEst;
sigmaEst.Init(config.rate_sigma_intercept, config.rate_sigma_slope,
config.t0_tmid_intercept, config.t0_tmid_slope, bf1raw->baseFrameRate);
GridMesh<SigmaEst> sigmaTMid;
bfImg1.Close();
bfImg2.Close();
// Calculate individual well t0 by looking at neighboring regions
vector<float> wellT0;
bfT0.CalcIndividualT0(wellT0, 0);
vector<float> wellT02;
bfT02.CalcIndividualT0(wellT02, 0);
for (size_t i =0; i< wellT0.size();i++) {
if (wellT0[i] > 0 && wellT02[i] > 0) {
wellT0[i] = (wellT0[i] + wellT02[i])/2.0f;
}
else {
wellT0[i] = max(wellT0[i], wellT02[i]);
}
}
// Average the region level t0, should we do this first and then just do sinle well level?
for (size_t bIx = 0; bIx < bfT0.GetNumRegions(); bIx++) {
double t1 = bfT0.GetT0(bIx);
double t2 = bfT02.GetT0(bIx);
if (t1 > 0 && t2 > 0) {
t1 = (t1 + t2)/2.0;
}
else {
t1 = max(t1,t2);
}
bfT0.SetT0(bIx, t1);
}
// Single thread first dat
for (size_t datIx = 0; datIx < 1; ++datIx) {
cout << "Doing: " << datIx << endl;
char buffer[2048];
snprintf(buffer, sizeof(buffer), "%s/acq_%.4d.dat", inputDir.c_str(), (int) datIx);
string datFile = buffer;
/* Use prior to calculate t0 and slope. */
Image datImg;
T0Calc t0;
datImg.LoadRaw(datFile.c_str());
// ImageTransformer::XTChannelCorrect(datImg.raw,datImg.results_folder);
const RawImage *datRaw = datImg.GetImage();
/* Estimate sigma and t_mid_nuc */
if (datIx == 0) {
cout << "Doing acquisition t0" << endl;
GenerateAcqT0Prior(config, datRaw->image, datRaw->baseFrameRate, datRaw->rows, datRaw->cols,
datRaw->frames, datRaw->timestamps,
config.row_step, config.col_step, &mask, t0, t0Prior);
ClockTimer timer;
cout << "Estimating sigma." << endl;
sigmaTMid.Init(datRaw->rows, datRaw->cols, config.row_step, config.col_step);
for (size_t bIx = 0; bIx < t0.GetNumRegions(); bIx++) {
t0.SetT0(bIx, bfT0.GetT0(bIx));
}
int neighbors = 2;
if (config.isThumbnail) {
cout << "Doing thumbnail version of slope." << endl;
neighbors = 1;
}
EstimateSigmaValue(t0, sigmaEst, sigmaTMid, neighbors);
timer.PrintMilliSeconds(cout,"Sigma Est took:");
string sigmaFile = outputDir + "/sigma_tmid_est.txt";
OutputSigmaTmidEstimates(sigmaTMid, sigmaFile.c_str());
}
/* For each region do shifting */
ClockTimer timer;
cout << "Shifting traces" << endl;
timer.StartTimer();
// ShiftTraces(bfT0, wellT0, datRaw->frames, datRaw->baseFrameRate, datRaw->timestamps, datRaw->image);
timer.PrintMilliSeconds(cout,"Shift took:");
if (!config.bg_param.empty()) {
DataCube<int> rowsCols;
DataCube<float> tmidSigma;
DataCube<float> fitTmidSigma;
string path = config.bg_param + ":/region/region_location";
if (!H5File::ReadDataCube(path, rowsCols)) {
ION_ABORT("Couldn't read file: " + path);
}
path = config.bg_param + ":/region/region_init_param";
if (!H5File::ReadDataCube(path, fitTmidSigma)) {
ION_ABORT("Couldn't read file: " + path);
}
for (size_t i = 0; i < rowsCols.GetNumX(); i++) {
int row = rowsCols.At(i,1,0);
int col = rowsCols.At(i,0,0);
SigmaEst &est = sigmaTMid.GetItemByRowCol(row, col);
float tmid_est = fitTmidSigma.At(i,0,0);
float sigma_est = fitTmidSigma.At(i,1,0);
est.mTMidNuc = tmid_est;
est.mSigma = sigma_est;
}
string fitSigmaFile = outputDir + "/bg_fit_sigma_tmid_est.txt";
OutputSigmaTmidEstimates(sigmaTMid, fitSigmaFile.c_str());
// path = config.bg_param + ":/region/region_init_param";
// if (!H5File::ReadMatrix(path, tmidSigma)) {
// ION_ABORT("Couldn't read file: " + path);
// }
// for (size_t i = 0; i < rowsCols.n_rows; i++) {
// int row = rowsCols.at(i,0);
// int col = rowsCols.at(i,1);
// SigmaEst &est = sigmaTMid.GetItemByRowCol(row, col);
// float tmid_est = tmidSigma.at(i,0);
// float sigma_est = tmidSigma.at(i,1);
// est.mTMidNuc = tmid_est;
// est.mSigma = sigma_est;
// }
// string sigmaFile = outputDir + "/supplied_sigma_tmid_est.txt";
// OutputSigmaTmidEstimates(sigmaTMid, sigmaFile.c_str());
}
else if (config.use_hard_est) {
for (size_t i = 0; i < bfT0.GetNumRegions(); i++) {
bfT0.SetT0(i,config.t0_hard * datRaw->baseFrameRate + config.time_start_slop);
}
for (size_t i = 0; i < sigmaTMid.GetNumBin(); i++) {
SigmaEst &est = sigmaTMid.GetItem(i);
est.mTMidNuc = config.tmid_hard;
est.mSigma = config.sigma_hard;
est.mT0 = config.t0_hard;
}
}
/* Use t0 and sigma to get the time compression bkgModel wants. */
cout << "Generating chunks" << endl;
// GridMesh<TraceChunk> traceChunks;
SynchDat sdat;
if (datIx == 0 && config.grind_acq_0 > 0) {
int nTimes = config.grind_acq_0;
timer.StartTimer();
size_t processMicroSec = 0;
size_t hdf5MicroSec = 0;
size_t compressMicroSec = 0;
size_t convertMicroSec = 0;
for (int i = 0; i <nTimes; i++) {
//GridMesh<TraceChunk> traceChunken;
SynchDat sdatIn;
AddMetaData(sdat, datRaw, datIx);
ClockTimer convTimer;
GenerateDataChunks(config, bfT0, datRaw, config.row_step, config.col_step, sigmaTMid, sdatIn.mChunks,datImg);
convertMicroSec += convTimer.GetMicroSec();
snprintf(buffer, sizeof(buffer), "%s/acq_%.4d.sdat", outputDir.c_str(), (int)datIx);
serializer.Write(buffer, sdatIn);
processMicroSec += serializer.computeMicroSec;
hdf5MicroSec += serializer.ioMicroSec;
compressMicroSec += serializer.compressMicroSec;
}
size_t usec = timer.GetMicroSec();
cout << "Took: " << usec / 1.0e6 << " seconds, " << usec / (nTimes * 1.0f) << " usec per write." << endl;
timer.PrintMilliSeconds(cout,"Chunks took:");
cout << "Read took: " << processMicroSec / (1e3 * nTimes) << " milli seconds per sdat compute." << endl;
cout << "Read took: " << hdf5MicroSec / (1e3 * nTimes) << " milli seconds per sdat hdf5." << endl;
cout << "Read took: " << compressMicroSec / (1e3 * nTimes) << " milli seconds per sdat compressing." << endl;
cout << "Read took: " << convertMicroSec / (1e3 * nTimes) << " milli seconds per sdat converting." << endl;
exit(0);
}
else {
timer.StartTimer();
AddMetaData(sdat, datRaw, datIx);
GenerateDataChunks(config, bfT0, datRaw, config.row_step, config.col_step, sigmaTMid, sdat.mChunks,datImg);
timer.PrintMilliSeconds(cout,"Chunks took:");
if (datIx == 0 && config.doDebug) {
OutputTraceChunks(sdat.mChunks,"flow_0_data_chunks.txt");
}
}
datImg.Close();
/* Serialize onto disk. */
snprintf(buffer, sizeof(buffer), "%s/acq_%.4d.sdat", outputDir.c_str(), (int)datIx);
serializer.Write(buffer, sdat);
/* Read back in first flow for checking */
if (datIx == 0) {
TraceChunkSerializer readSerializer;
readSerializer.SetRecklessAbandon(true);
// GridMesh<TraceChunk> traceChunksIn;
SynchDat sdatIn;
readSerializer.Read(buffer, sdatIn);
if (datIx == 0 && config.doDebug) {
OutputTraceChunks(sdatIn.mChunks, "flow_0_data_chunks_read.txt");
}
SampleQuantiles<float> s(50000);
SampleQuantiles<float> s2(50000);
SampleQuantiles<float> sAbs(50000);
SampleStats<double> ss;
int diffCount = 0;
for (size_t bIx = 0; bIx < sdatIn.mChunks.mBins.size(); bIx++) {
if (sdatIn.mChunks.mBins[bIx].mT0 != sdat.mChunks.mBins[bIx].mT0) {
cout << "Got: " << sdatIn.mChunks.mBins[bIx].mT0 << " vs: " << sdat.mChunks.mBins[bIx].mT0 << endl;
exit(1);
}
for (size_t i = 0; i < sdatIn.mChunks.mBins[bIx].mData.size(); i++) {
double diff = (double)sdatIn.mChunks.mBins[bIx].mData[i] - (double)sdat.mChunks.mBins[bIx].mData[i];
if (!std::isfinite(diff)) {
cout << "NaNs!!" << endl;
}
if (diffCount < 10 && fabs(diff) > .00001) { // != 0) {
diffCount++;
cout << "Bin: " << bIx << " well: " << i << " diff is: " << diff << endl;
}
s.AddValue(diff);
sAbs.AddValue(fabs(diff));
ss.AddValue(sqrt(diff * diff));
s2.AddValue(sqrt(diff * diff));
}
}
cout << "Median rms: " << s2.GetMedian() << " Avg: " << ss.GetMean() << " diff: " << s.GetMedian() << endl;
cout << "Abs(diff) Quantiles:" << endl;
for (size_t i = 0; i <= 100; i+=10) {
cout << i << "\t" << sAbs.GetQuantile(i/100.0) << endl;
}
}
}
// do the next N flows multithreaded
if (numFlows > 1) {
PJobQueue jQueue (config.numCores, numFlows-1);
vector<CreateSDat> jobs(numFlows -1);
// for (int i = 0; i < 4; i++) {
// char buffer[2048];
// snprintf(buffer, sizeof(buffer), "%s/beadfind_pre_%.4d.dat", inputDir.c_str(), (int) i);
// string input = buffer;
// snprintf(buffer, sizeof(buffer), "%s/beadfind_pre_%.4d.sdat", outputDir.c_str(), (int)i);
// string output = buffer;
// jobs[i].Init(&config, input, output, &wellT0, &bfT0, &sigmaTMid);
// jQueue.AddJob(jobs[i]);
// }
// jQueue.WaitUntilDone();
for (int i = 1; i < numFlows; i++) {
char buffer[2048];
snprintf(buffer, sizeof(buffer), "%s/acq_%.4d.dat", inputDir.c_str(), (int) i);
string input = buffer;
snprintf(buffer, sizeof(buffer), "%s/acq_%.4d.sdat", outputDir.c_str(), (int)i);
string output = buffer;
jobs[i-1].Init(&config, input, output, &wellT0, &bfT0, &sigmaTMid, i);
jQueue.AddJob(jobs[i-1]);
}
jQueue.WaitUntilDone();
}
/* Serialize into backbround models */
cout << "Done." << endl;
}
void KeyClassifier::ClassifyKnownTauE(std::vector<KeySeq> &keys,
ZeromerModelBulk<double> &bg,
Mat<double> &wellFlows,
Mat<double> &refFlows,
Col<double> &time,
const Col<double> &incorp,
double minSnr,
double tauE,
KeyFit &fit,
TraceStore<double> &store,
Mat<double> &predicted) {
param.set_size(2);// << 0 << 0;
param[0] = 0;
param[1] = 0;
fit.keyIndex = -1;
fit.snr = 0;
fit.sd = 0;
fit.mad = -1;
signal.set_size(wellFlows.n_cols);
projSignal.set_size(wellFlows.n_cols);
Col<double> weights = ones<vec>(store.GetNumFrames());
if (fit.bestKey >= 0) {
fit.bestKey = -1;
}
fit.ok = -1;
size_t frameStart = min(FRAME_START,wellFlows.n_rows);
size_t frameEnd = min(FRAME_END,wellFlows.n_rows);
for (size_t keyIx = 0; keyIx < keys.size(); keyIx++) {
double keyMinSnr = std::max(minSnr, keys[keyIx].minSnr);
double tauB = 0;
bg.FitWell(fit.wellIdx, store, keys[keyIx], weights, mDist, mValues);
param.at(0) = tauB;
param.at(1) = tauE;
onemerIncorpMad.Clear();
onemerProjMax.Init(10);
onemerProjMax.Clear();
onemerSig.Clear();
zeromerSig.Clear();
zeroStats.Clear();
traceSd.Clear();
sigVar.Clear();
onemerProj.Clear();
for (size_t flowIx = 0; flowIx < wellFlows.n_cols; flowIx++) {
bg.ZeromerPrediction(fit.wellIdx, flowIx, store, refFlows.unsafe_col(flowIx),p);
double sig = 0;
SampleStats<double> mad;
diff = wellFlows.unsafe_col(flowIx) - p;
for (size_t frameIx = frameStart; frameIx < frameEnd; frameIx++) {
sig += diff.at(frameIx);
}
signal.at(flowIx) = sig;
/* uvec indices; */
double pSig = std::numeric_limits<double>::quiet_NaN();
if (incorp.n_rows == diff.n_rows) {
pSig = GetProjection(diff, incorp);
}
projSignal.at(flowIx) = pSig;
sigVar.AddValue(sig);
if (keys[keyIx].flows[flowIx] == 0) {
for (size_t frameIx = frameStart; frameIx < frameEnd; frameIx++) {
mad.AddValue(fabs(diff.at(frameIx)));
}
zeroStats.AddValue(mad.GetMean());
zeromerSig.AddValue(sig);
}
else if (keys[keyIx].flows[flowIx] == 1 && flowIx < keys[keyIx].usableKeyFlows) {
onemerSig.AddValue(sig);
// double maxValue = 0;
// for (size_t fIx = frameStart; fIx < frameEnd-1; fIx++) {
// maxValue = max(maxValue, (diff.at(fIx)+diff.at(fIx+1))/2);
// }
// projSignal.at(flowIx) = maxValue;
// onemerProjMax.AddValue(maxValue);
if (isfinite(pSig) && incorp.n_rows == p.n_rows) {
onemerProj.AddValue(pSig);
double maxSig = 0;
for (size_t frameIx = frameStart; frameIx < frameEnd; frameIx++) {
double projVal = pSig * incorp.at(frameIx);
maxSig = max(maxSig, projVal);
onemerIncorpMad.AddValue(fabs(projVal - (wellFlows.at(frameIx,flowIx) - p.at(frameIx))));
}
onemerProjMax.AddValue(maxSig);
}
}
}
double snr = (onemerSig.GetMedian() - zeromerSig.GetMedian()) / ((onemerSig.GetIqrSd() + zeromerSig.GetIqrSd() + SDFUDGE)/2);
float sd = sigVar.GetSD();
if (!isfinite(sd) || isnan(sd)) {
sd = 0;
}
if ((snr >= fit.snr || (isfinite(snr) && !isfinite(fit.snr))) && snr >= keyMinSnr ) {
fit.keyIndex = keyIx;
fit.bestKey = keyIx;
fit.mad = zeroStats.GetMean();
fit.snr = snr;
fit.param = param;
fit.sd = sd;
fit.onemerAvg = onemerSig.GetCount() > 0 ? onemerSig.GetMedian() : std::numeric_limits<double>::quiet_NaN();
fit.peakSig = onemerProjMax.GetCount() > 0 ? onemerProjMax.GetMedian() : std::numeric_limits<double>::quiet_NaN();
fit.onemerProjAvg = onemerProj.GetCount() > 0 ? onemerProj.GetMean() : std::numeric_limits<double>::quiet_NaN();
fit.projResid = onemerIncorpMad.GetCount() > 0 ? onemerIncorpMad.GetMean() : std::numeric_limits<double>::quiet_NaN();
fit.ok = true;
for (size_t flowIx = 0; flowIx < wellFlows.n_cols; flowIx++) {
bg.ZeromerPrediction(fit.wellIdx, flowIx, store, refFlows.unsafe_col(flowIx),p);
copy(p.begin(), p.end(), predicted.begin_col(flowIx));
}
}
else if (keyIx == 0) { // || snr > fit.snr) { // just set default...
fit.bestKey = keyIx;
fit.mad = zeroStats.GetMean();
fit.snr = snr;
fit.param = param;
fit.sd = sd;
fit.onemerAvg = onemerSig.GetCount() > 0 ? onemerSig.GetMedian() : std::numeric_limits<double>::quiet_NaN();
fit.peakSig = onemerProjMax.GetCount() > 0 ? onemerProjMax.GetMedian() : std::numeric_limits<double>::quiet_NaN();
fit.onemerProjAvg = onemerProj.GetCount() > 0 ? onemerProj.GetMean() : std::numeric_limits<double>::quiet_NaN();
fit.projResid = onemerIncorpMad.GetCount() > 0 ? onemerIncorpMad.GetMean() : std::numeric_limits<double>::quiet_NaN();
fit.ok = true;
for (size_t flowIx = 0; flowIx < wellFlows.n_cols; flowIx++) {
bg.ZeromerPrediction(fit.wellIdx, flowIx, store, refFlows.unsafe_col(flowIx),p);
copy(p.begin(), p.end(), predicted.begin_col(flowIx));
}
}
}
// Reset the params to the right key
if (fit.keyIndex < 0) {
bg.FitWell(fit.wellIdx, store, keys[0], weights, mDist, mValues);
}
else {
bg.FitWell(fit.wellIdx, store, keys[fit.keyIndex], weights, mDist, mValues);
}
if (!isfinite(fit.mad)) {
fit.ok = 0;
fit.mad = std::numeric_limits<float>::max();
}
}
void Traces::CalcIncorpBreakRegionStart(size_t rowStart, size_t rowEnd,
size_t colStart, size_t colEnd,
SampleStats<float> &starts) { //, MaskType maskType) {
FindSlopeChange<double> finder;
float startNucFrame = 0, startSumSeq = 0, slope = 0, yIntercept = 0;
int numFrames = std::min(100, (int)mFrames);
int maxSearch = std::min(70, (int)mFrames-FRAME_ZERO_WINDOW);
//size_t iRangeStart = 5+offset, iRangeEnd = 52+offset, iGuessStart = 10+offset, iGuessEnd = 46+offset;
// size_t iRangeStart = 0, iRangeEnd = 25, iGuessStart = 3, iGuessEnd = 10;
/* 15-25 is usual start
17-25
*/
Mask &mask = *mMask;
int sampleSize = 100;
vector<SampleQuantiles<float> >frameAvgs(numFrames);
for (size_t i = 0; i < frameAvgs.size(); i++) {
frameAvgs[i].Init(sampleSize);
}
// vector<SampleStats<float> > frameAvgs(numFrames);
vector<double> trace;
vector<float> traceBuffer;
int seen = 0;
for (size_t rowIx = rowStart; rowIx < rowEnd; rowIx++) {
for (size_t colIx = colStart; colIx < colEnd; colIx++) {
size_t idx = RowColToIndex(rowIx, colIx);
// if ((mask[idx] & maskType) && mFlags[idx] == OK) {
if (!(mask[idx] & MaskPinned) && !(mask[idx] & MaskExclude) && mFlags[idx] == OK) {
GetTraces(idx, traceBuffer);
seen++;
for (size_t i = 0; i < frameAvgs.size(); i++) {
frameAvgs[i].AddValue(traceBuffer[i]);
}
}
}
}
if(frameAvgs[0].GetNumSeen() <= MIN_T0_PROBES) {
return;
}
trace.resize(frameAvgs.size());
for (size_t i = 0; i < frameAvgs.size(); i++) {
// trace[i] = frameAvgs[i].GetMedian();
trace[i] = frameAvgs[i].GetMedian();
}
double zero = 0, hinge = 0;
bool ok = finder.findNonZeroRatioChangeIndex(startNucFrame,startSumSeq,
slope, yIntercept,
0, maxSearch, zero, hinge, trace, 5);
if (mRefOut != NULL) {
std::ostream &o = *mRefOut;
o << mFlow << "\t" << rowStart << "\t" << rowEnd << "\t" << colStart << "\t" << colEnd << "\t"
<< ok << "\t" << startNucFrame << "\t" << frameAvgs[0].GetNumSeen();
for (size_t i = 0; i < trace.size(); i++) {
o << "\t" << trace[i];
}
o << endl;
}
if (ok && frameAvgs[0].GetNumSeen() >= MIN_T0_PROBES ) {
starts.AddValue(startNucFrame);
}
}
void Traces::Init(Image *img, Mask *mask, int startFrame, int endFrame,
int dcOffsetStart, int dcOffsetEnd) {
mRefOut = NULL;
mFlow = -1;
mT0Step = 32;
mUseMeshNeighbors = 1;
mRow = (img->GetImage())->rows;
mCol = (img->GetImage())->cols;
startFrame = std::max(startFrame, 0);
if( endFrame > 0 )
endFrame = std::min(endFrame, (int)img->GetUnCompFrames());
else
endFrame = (int)img->GetUnCompFrames();
mFrames = endFrame - startFrame;
mTimes.resize(mFrames);
// copy(&raw->timestamps[0], &raw->timestamps[0] + mFrames, mTimes.begin());
// need to take into account variable frame compression here... ??
if (mIndexes.size() != mRow*mCol) {
mIndexes.resize(mRow*mCol);
}
fill(mIndexes.begin(), mIndexes.end(), -1);
vector<float> tBuff(mFrames, 0);
SampleStats<float> traceStats;
std::vector<float> sdTrace(mRow *mCol, 0);
// Figure out how much memory to allocate for data pool
int count = 0;
for (size_t rowIx = 0; rowIx < mRow; rowIx++) {
for (size_t colIx = 0; colIx < mCol; colIx++) {
size_t traceIdx = RowColToIndex(rowIx, colIx);
if (mask == NULL || !((*mask)[traceIdx] & MaskExclude)) {
count++;
}
}
}
// Allocate memory
if (mRawTraces != NULL) {
delete [] mRawTraces ;
}
mRawTraces = new int8_t[count * mFrames];
SampleQuantiles<float> chipTraceSdQuant(10000);
count = 0;
for (size_t rowIx = 0; rowIx < mRow; rowIx++) {
for (size_t colIx = 0; colIx < mCol; colIx++) {
size_t traceIdx = RowColToIndex(rowIx, colIx);
if (mask == NULL || !((*mask)[traceIdx] & MaskExclude)) {
mIndexes[traceIdx] = count++ * mFrames;
double mean = 0;
if (dcOffsetEnd > 0 && dcOffsetStart > 0) {
assert(dcOffsetEnd > dcOffsetStart);
for (int frameIx = dcOffsetStart; frameIx < dcOffsetEnd; frameIx++) {
mean += img->GetInterpolatedValue(frameIx,colIx,rowIx);
}
mean = mean / (dcOffsetEnd - dcOffsetStart);
}
float val = img->GetInterpolatedValue(startFrame,colIx,rowIx) - mean;
traceStats.Clear();
for (int frameIx = 0; frameIx < endFrame; frameIx++) {
val = (img->GetInterpolatedValue(frameIx,colIx,rowIx) - mean);
tBuff[frameIx] = val;
traceStats.AddValue(tBuff[frameIx]);
}
sdTrace[traceIdx] = traceStats.GetSD();
chipTraceSdQuant.AddValue(sdTrace[traceIdx]);
SetTraces(traceIdx, tBuff, mRawTraces);
}
}
}
mFlags.resize(mRow*mCol, 0);
double traceSDThresh = chipTraceSdQuant.GetMedian() - 5 * (chipTraceSdQuant.GetQuantile(.5) - chipTraceSdQuant.GetQuantile(.25));
traceSDThresh = max(0.0, traceSDThresh);
int badCount = 0;
for (size_t wellIx = 0; wellIx < sdTrace.size(); wellIx++) {
if (mask != NULL && ((*mask)[wellIx] & MaskExclude)) {
continue;
}
if (sdTrace[wellIx] <= traceSDThresh) {
mFlags[wellIx] = BAD_VAR;
badCount++;
}
}
cout << "Found: " << badCount << " wells <= sd: " << traceSDThresh << endl;
mT0.resize(0);
if(mask != NULL) {
if (mMask != NULL)
delete mMask;
mMask = new Mask(mask);
size_t size = mFlags.size();
for (size_t i = 0; i < size; i++) {
if ((*mMask)[i] & MaskExclude) {
mFlags[i] = NOT_AVAIL;
}
}
}
else {
mMask = NULL;
}
mSampleMap.resize(mRow * mCol);
mCurrentData = mRawTraces;
fill(mSampleMap.begin(), mSampleMap.end(), -1);
}
NumericalComparison<double> CompareWells(
const string &queryFile,
const string &goldFile,
float epsilon,
double maxAbsVal,
DumpMismatches &dump) {
NumericalComparison<double> compare(epsilon);
string queryDir, queryWells, goldDir, goldWells;
FillInDirName(queryFile, queryDir, queryWells);
FillInDirName(goldFile, goldDir, goldWells);
RawWells queryW(queryDir.c_str(), queryWells.c_str());
RawWells goldW(goldDir.c_str(), goldWells.c_str());
struct WellData goldData;
goldData.flowValues = NULL;
struct WellData queryData;
queryData.flowValues = NULL;
cout << "Opening query." << endl;
queryW.OpenForRead();
cout << "Opening gold." << endl;
goldW.OpenForRead();
// check if any 1.wells is saved as unsigned short
bool ushortg = goldW.GetSaveAsUShort();
bool ushortq = queryW.GetSaveAsUShort();
bool difType = false;
string fileName;
if(ushortg && (!ushortq))
{
difType = true;
cout << "RawWellsEquivalent WARNING: " << goldFile << " is saved as unsigned short and \n" << queryFile << " is saved as float. \nYou may want to re-run it with a bigger epsilon." << endl;
fileName = goldFile;
}
if(ushortq && (!ushortg))
{
difType = true;
cout << "RawWellsEquivalent WARNING: " << queryFile << " is saved as unsigned short and \n" << goldFile << " is saved as float. \nYou may want to re-run it with a bigger epsilon." << endl;
fileName = queryFile;
}
unsigned int numFlows = goldW.NumFlows();
while( !queryW.ReadNextRegionData(&queryData) ) {
assert(!goldW.ReadNextRegionData(&goldData));
for (unsigned int i = 0; i < numFlows; i++) {
if(difType)
{
float flowValues2 = -1.0;
if(ushortg)
{
flowValues2 = goldData.flowValues[i];
}
else if(ushortq)
{
flowValues2 = queryData.flowValues[i];
}
if(isfinite(flowValues2) && fabs(flowValues2) < maxAbsVal)
{
if(ushortg && isfinite(queryData.flowValues[i]) && fabs(queryData.flowValues[i]) < maxAbsVal)
{
compare.AddPair(queryData.flowValues[i], flowValues2);
}
else if(ushortq && isfinite(goldData.flowValues[i]) && fabs(goldData.flowValues[i]) < maxAbsVal)
{
compare.AddPair(flowValues2, goldData.flowValues[i]);
}
}
}
else
{
if (isfinite(queryData.flowValues[i]) && isfinite(goldData.flowValues[i]) &&
(fabs(queryData.flowValues[i]) < maxAbsVal && fabs(goldData.flowValues[i]) < maxAbsVal))
{
compare.AddPair(queryData.flowValues[i], goldData.flowValues[i]);
if (abs(goldData.flowValues[i] - queryData.flowValues[i]) > epsilon) {
if (dump.needDump())
dump.dumpMismatchWellData(goldData, queryData, i);
}
}
}
}
}
const SampleStats<double> ssX = compare.GetXStats();
const SampleStats<double> ssY = compare.GetYStats();
cout << "query values: " << ssX.GetMean() << " +/- " << ssX.GetSD() << endl;
cout << "gold values: " << ssY.GetMean() << " +/- " << ssY.GetSD() << endl;
return compare;
}
