void ComparatorNoiseCorrector::DebugSaveCorrection(int row_start, int row_end)
{
int y,frame;
int frameStride=rows*cols;
short *srcPtr;
short *corPtr;
Image loader2;
loader2.LoadRaw ( "acq_0000.dat", 0, true, false );
// now subtract each neighbor-subtracted comparator signal from the
// pixels that are connected to that comparator
printf("%s:\n",__FUNCTION__);
for (frame = 0; frame < frames; frame++)
{
for (y = row_start; y < row_end; y++)
{
for(int x = 0; x < cols; x++)
{
int comparator=y&(ncomp-1);
srcPtr = &loader2.raw->image[frame * frameStride + y * cols + x];
corPtr = &mCorrection_ACC(x,comparator,frame);
*srcPtr = (short)(8192 + *corPtr);
}
}
}
char *newName=(char *)"acq_0000.dat_oldcorr";
Acq saver;
saver.SetData ( &loader2 );
saver.WriteVFC(newName, 0, 0, cols, rows);
}
TEST(MergeAcq_Test, TopMergeTest) {
Image top;
Image bottom;
Image combo;
MergeAcq merger;
// Load up our test files
const char *file = "test.dat";
ION_ASSERT(top.LoadRaw(file), "Couldn't load file.");
ION_ASSERT(bottom.LoadRaw(file), "Couldn't load file.");
merger.SetFirstImage(&bottom);
merger.SetSecondImage(&top, bottom.GetRows(), 0); // starting vertically raised but columns the same.
// Merge the two images into a single image
merger.Merge(combo);
cout << "Done." << endl;
// Write out the merged image for some testing
Acq acq;
acq.SetData(&combo);
acq.WriteVFC("combo.dat", 0, 0, combo.GetCols(), combo.GetRows());
Image test;
ION_ASSERT(test.LoadRaw("combo.dat"), "Couldn't load file.");
// Check to make sure that the combined file has same values as original
for (int row = 0; row < top.GetRows(); row++) {
for (int col = 0; col < top.GetCols(); col++) {
for (int frame = 0; frame < top.GetFrames(); frame++) {
EXPECT_EQ(top.At(row, col, frame), bottom.At(row,col,frame));
short orig = top.At(row, col, frame);
short combined = test.At(row+top.GetRows(),col,frame);
EXPECT_EQ(orig, combined);
combined = test.At(row,col,frame);
EXPECT_EQ(orig, combined);
}
}
}
}
void ComparatorNoiseCorrector::DebugSaveComparatorMask(int time)
{
int y,frame;
int frameStride=rows*cols;
short *srcPtr;
int *corPtr;
// int lw=cols/VEC8_SIZE;
Image loader2;
loader2.LoadRaw ( "acq_0000.dat", 0, true, false );
// now subtract each neighbor-subtracted comparator signal from the
// pixels that are connected to that comparator
for (frame = 0; frame < frames; frame++)
{
for (y = 0; y < rows; y++)
{
for(int x = 0; x < cols; x++)
{
int comparator=y&(ncomp-1);
srcPtr = &loader2.raw->image[frame * frameStride + y * cols + x];
corPtr = &mComparator_mask[x*ncomp + comparator];
*srcPtr = (short)(8192.0f + *corPtr);
}
}
}
char newName[1024];
sprintf(newName,"acq_0000.dat_oldmask%d",time);
Acq saver;
saver.SetData ( &loader2 );
saver.WriteVFC(newName, 0, 0, cols, rows);
}
void ComparatorNoiseCorrector::DebugSaveAvgNum()
{
int y,frame;
int frameStride=rows*cols;
short *srcPtr;
float *corPtr;
// int lw=cols/VEC8_SIZE;
Image loader2;
loader2.LoadRaw ( "acq_0000.dat", 0, true, false );
// now subtract each neighbor-subtracted comparator signal from the
// pixels that are connected to that comparator
for (frame = 0; frame < frames; frame++)
{
for (y = 0; y < rows; y++)
{
for(int x = 0; x < cols; x++)
{
int comparator=y&(ncomp-1);
srcPtr = &loader2.raw->image[frame * frameStride + y * cols + x];
corPtr = &mAvg_num[x*2+comparator];//mAvg_num_ACC(x,comparator);
*srcPtr = (short)(/*8192.0f +*/ *corPtr);
}
}
}
char *newName=(char *)"acq_0000.dat_oldavgnum";
Acq saver;
saver.SetData ( &loader2 );
saver.WriteVFC(newName, 0, 0, cols, rows);
}
int main(int argc, const char *argv[]) {
OptArgs opts;
opts.ParseCmdLine(argc, argv);
bool help;
string topFile, bottomFile, outFile;
opts.GetOption(topFile, "", '-', "top");
opts.GetOption(bottomFile, "", '-', "bottom");
opts.GetOption(outFile, "", '-', "merged");
opts.GetOption(help, "false", 'h', "help");
if (help || argc == 1) {
usage();
}
ION_ASSERT(!topFile.empty() && !bottomFile.empty() && !outFile.empty(),
"Need top, bottom and merged files. use --help for details.");
MergeAcq merger;
Image top;
Image bottom;
Image combo;
cout << "Loading images." << endl;
ION_ASSERT(top.LoadRaw(topFile.c_str()), "Couldn't load file.");
ION_ASSERT(bottom.LoadRaw(bottomFile.c_str()), "Couldn't load file.");
merger.SetFirstImage(&bottom);
merger.SetSecondImage(&top, bottom.GetRows(), 0); // starting vertically raised but columns the same.
cout << "Merging." << endl;
merger.Merge(combo);
Acq acq;
cout << "Saving. " << endl;
acq.SetData(&combo);
acq.WriteVFC(outFile.c_str(), 0, 0, combo.GetCols(), combo.GetRows());
cout << "Done." << endl;
return 0;
}
void ComparatorNoiseCorrector::DebugSaveComparatorSigs(int state)
{
Image loader2;
loader2.LoadRaw ( "acq_0000.dat", 0, true, false );
int y,frame;
int frameStride=rows*cols;
short *srcPtr;
float *corPtr;
// int lw=cols/VEC8_SIZE;
// now subtract each neighbor-subtracted comparator signal from the
// pixels that are connected to that comparator
for (frame = 0; frame < frames; frame++)
{
for (y = 0; y < rows; y++)
{
for(int x = 0; x < cols; x++)
{
int comparator=y&(ncomp-1);
srcPtr = &loader2.raw->image[frame * frameStride + y * cols + x];
if(!state)
corPtr = &mComparator_sigs_ACC(x,comparator,frame);
else
corPtr = &mComparator_sigs[(2*x+comparator)*frames+frame];
*srcPtr = (short)(8192.0f + *corPtr);
}
}
}
char newName[256];
sprintf(newName,"acq_0000.dat_oldsig%d",state);
Acq saver;
saver.SetData ( &loader2 );
saver.WriteVFC(newName, 0, 0, cols, rows);
}
void ProcAbsPos::handleStart(ProjAcq& pAcq, AbsPos2D<float> const& pos) const {
Acq* acq = pAcq.getAcq();
Mat rgb = acq->getMat(BGR);
#ifdef WRITE_IMAGES
imwrite("rgb.png", rgb);
#endif
#ifdef COMP_HSV
Mat im_hsv = acq->getMat(HSV);
#ifdef HSV_TO_HGRAY
for (Mat_<Vec3b>::iterator it = im_hsv.begin<Vec3b>();
it != im_hsv.end<Vec3b>(); it++) {
(*it)[1] = (*it)[0];
(*it)[2] = (*it)[0];
}
#elif defined(HSV_TO_VGRAY)
for (Mat_<Vec3b>::iterator it = im_hsv.begin<Vec3b>();
it != im_hsv.end<Vec3b>(); it++) {
(*it)[0] = (*it)[2];
(*it)[1] = (*it)[2];
}
#endif
#ifdef WRITE_IMAGES
imwrite("hsv.png", im_hsv);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow("hsv", im_hsv);
#endif /* SHOW_IMAGES */
#endif /* COMP_HSV */
#ifdef COMP_SIMULATED
// simulate acquisition
Mat _im3 = getSimulatedAt(pAcq, pos);
string _bn("_im3-sa");
#ifdef WRITE_IMAGES
imwrite(_bn + ".png", _im3);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow(bn, _im3);
#endif /* SHOW_IMAGES */
#ifdef COMP_HSV
// show simulated acquisition in hsv
Mat im3_hsv = im3.clone();
cvtColor(im3, im3_hsv, COLOR_BGR2HSV);
#ifdef HSV_TO_HGRAY
for (Mat_<Vec3b>::iterator it = im3_hsv.begin<Vec3b>();
it != im3_hsv.end<Vec3b>(); it++) {
(*it)[1] = (*it)[0];
(*it)[2] = (*it)[0];
}
#elif defined(HSV_TO_VGRAY)
for(Mat_<Vec3b>::iterator it = im3_hsv.begin<Vec3b>(); it != im3_hsv.end<Vec3b>(); it++) {
(*it)[0] = (*it)[2];
(*it)[1] = (*it)[2];
}
#endif
#ifdef WRITE_IMAGES
imwrite(bn + "_hsv.png", im3_hsv);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow(bn + "_hsv", im3_hsv);
#endif /* SHOW_IMAGES */
Mat diff_hsv = im_hsv - im3_hsv;
#ifdef WRITE_IMAGES
imwrite(bn + "_diff_hsv.png", diff_hsv);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow(bn + "_diff_hsv", diff_hsv);
#endif /* SHOW_IMAGES */
#endif /* COMP_HSV */
#ifdef COMP_TESTPOINTS
Mat _im3_tp = getTestPointsAt(pAcq, pos.getTransform().getReverse());
_bn = "_im3_tp-sa";
#ifdef WRITE_IMAGES
imwrite(_bn + ".png", _im3_tp);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow(bn, im3_tp);
#endif /* SHOW_IMAGES */
#endif /* COMP_TESTPOINTS */
#endif /* COMP_SIMULATED */
#ifdef COMP_TESTPOINTS
Mat _rgb_tp = rgb.clone();
addTestPointsAtTo(_rgb_tp, pAcq, pos.getTransform().getReverse());
_bn = "_im_tp-sa";
#ifdef WRITE_IMAGES
imwrite(_bn + ".png", _rgb_tp);
#endif /* WRITE_IMAGES */
#ifdef SHOW_IMAGES
imshow(bn, rgb_tp);
#endif /* SHOW_IMAGES */
#endif /* COMP_TESTPOINTS */
}
float ProcAbsPos::getEnergy(ProjAcq& pAcq, const Pos& robPos) {
// vector<TestPoint> posDependentTP = getPosDependentTP(robPos);
float E = 0;
Acq* acq = pAcq.getAcq();
Cam const* cam = acq->getCam();
Mat im = acq->getMat(HSV);
Transform2D<float> tr_pg2rob(robPos.getTransform());
Transform2D<float> tr_rob2pg = tr_pg2rob.getReverse();
// get corners and edges of cam fov projected on playground
Mat camCorners[4];
camCorners[0] = tr_rob2pg.transformLinPos(pAcq.cam2plane(cam->getTopLeft()));
camCorners[1] = tr_rob2pg.transformLinPos(pAcq.cam2plane(cam->getTopRight()));
camCorners[2] = tr_rob2pg.transformLinPos(pAcq.cam2plane(cam->getBottomRight()));
camCorners[3] = tr_rob2pg.transformLinPos(pAcq.cam2plane(cam->getBottomLeft()));
float xMin = MIN(MIN(camCorners[0].at<float>(0), camCorners[1].at<float>(0)), MIN(camCorners[2].at<float>(0), camCorners[3].at<float>(0)));
float yMin = MIN(MIN(camCorners[0].at<float>(1), camCorners[1].at<float>(1)), MIN(camCorners[2].at<float>(1), camCorners[3].at<float>(1)));
float xMax = MAX(MAX(camCorners[0].at<float>(0), camCorners[1].at<float>(0)), MAX(camCorners[2].at<float>(0), camCorners[3].at<float>(0)));
float yMax = MAX(MAX(camCorners[0].at<float>(1), camCorners[1].at<float>(1)), MAX(camCorners[2].at<float>(1), camCorners[3].at<float>(1)));
#ifdef COMP_PLAYGROUND
Point2i camCornersPoints[4];
#endif /* COMP_PLAYGROUND */
Mat camEdges[4];
for (int i = 0; i < 4; i++) {
#ifdef COMP_PLAYGROUND
camCornersPoints[i] = getInPGIm(camCorners[i]);
#endif /* COMP_PLAYGROUND */
camEdges[i] = camCorners[(i + 1) % 4] - camCorners[i];
}
#ifdef COMP_PLAYGROUND
Mat pg_fov = pg.clone();
// draw projected FOV
Scalar color_fov(0, 0, 0);
for (int i = 0; i < 4; i++) {
line(pg_fov, camCornersPoints[i], camCornersPoints[(i + 1) % 4], color_fov, 4);
}
// draw robot position
Point2i rP = getInPGIm(robPos.getLinPos());
Point2i rP_x = getInPGIm(robPos.getLinPos() + Point2D<float>(3, 0).rotate(robPos.theta()).toCv());
line(pg_fov, rP, rP_x, Scalar(0, 0, 255), 4);
Point2i rP_y = getInPGIm(robPos.getLinPos() + Point2D<float>(0, 3).rotate(robPos.theta()).toCv());
line(pg_fov, rP, rP_y, Scalar(0, 255, 0), 4);
#endif /* COMP_PLAYGROUND */
// get Energy...
int nb = 0;
for (TestPoint& tp : staticTP) {
cv::Mat tp_pos = tp.getPos();
float tp_x = tp_pos.at<float>(0);
float tp_y = tp_pos.at<float>(1);
if (tp_x > xMax || tp_x < xMin || tp_y > yMax || tp_y < yMin) {
continue;
}
// check if testpoint seen by camera
int i;
for (i = 0; i < 4; i++) {
Mat vi = tp_pos - camCorners[i];
double cross = vi.at<float>(0) * camEdges[i].at<float>(1) -
vi.at<float>(1) * camEdges[i].at<float>(0);
if (cross < 0) {
break;
}
}
if (i < 4) {
continue;
}
nb++;
#ifdef COMP_PLAYGROUND
// testpoint is in cam field of view, draw it in playground
pg_fov.at<Vec3b>(getInPGIm(tp_pos)) = Vec3b(255, 255, 255);
#endif /* COMP_PLAYGROUND */
// get position of testpoint in original camera image
cv::Mat tp_cmRob = tr_pg2rob.transformLinPos(tp_pos);
cv::Mat tp_pxCam = pAcq.plane2cam(tp_cmRob);
int x = int(round(tp_pxCam.at<float>(0)));
int y = int(round(tp_pxCam.at<float>(1)));
assert(x >= 0 && x < cam->getSize().width);
assert(y >= 0 && y < cam->getSize().height);
// get hue of selected pixel
float hue = float(im.at<Vec3b>(y, x)[0]) / 255.;
float sat = float(im.at<Vec3b>(y, x)[1]) / 255.;
float val = float(im.at<Vec3b>(y, x)[2]) / 255.;
// assert(hue >= 0. && hue <= 1.);
// assert(255*hue == im.at<Vec3b>(y, x)[0]);
// get cost of testpoint given this hue
E += tp.getCost(hue, sat, val);
}
// for(const TestPoint& tp : posDependentTP){
// Pt tp_cmRob = tr_pg2rob.transformLinPos(tp.getPos());
// Pt tp_pxCam = pAcq->plane2Cam(tp_cmRob);
//
// const Scalar& px = im.at<Scalar>(tp_pxCam.x, tp_pxCam.y);
//
// E += tp.getCost(px(0));
// }
E /= nb;
#ifdef COMP_PLAYGROUND
#ifdef SHOW_IMAGES
imshow("pg", pg_fov);
#endif /* SHOW_IMAGES */
#ifdef WRITE_IMAGES
imwrite("pg.png", pg_fov);
#endif /* WRITE_IMAGES */
#endif /* COMP_PLAYGROUND */
return E;
}
void DoCrop(OptionsCls &options)
{
char name[MAX_PATH_LENGTH];
char destName[MAX_PATH_LENGTH];
int i;
Image loader;
Acq saver;
int mode = 0;
i = 0;
bool allocate = true;
int nameListLen;
char **nameList;
int maxMode=2;
//@WARNING: >must< copy beadfind_post_0000 if it exists, or Analysis will >fail to complete< on PGM run crops
// This is an inobvious bug due to the fact that the Image routine "ReadyToLoad" checks for >the next< acq file
// or explog_final.txt, >or< beadfind_post. If none of those three exist, we wait forever.
// not all runs appear to have explog_final.txt copied around, so can't depend on that.
const char* defaultNameList[] = {"beadfind_pre_0000.dat", "beadfind_pre_0001.dat", "beadfind_pre_0002.dat", "beadfind_pre_0003.dat",
"beadfind_post_0000.dat", "beadfind_post_0001.dat", "beadfind_post_0002.dat", "beadfind_post_0003.dat",
"prerun_0000.dat", "prerun_0001.dat", "prerun_0002.dat", "prerun_0003.dat"
};
// if requested...do not bother waiting for the files to show up
if ( options.dont_retry )
loader.SetTimeout ( 1,1 );
if ( options.oneFile != NULL ) {
nameList = &options.oneFile;
nameListLen = 1;
maxMode=1;
} else {
nameList = const_cast<char**> ( defaultNameList );
nameListLen = sizeof ( defaultNameList ) /sizeof ( defaultNameList[0] );
}
// Create results folder
umask ( 0 ); // make permissive permissions so its easy to delete.
if ( mkdir ( options.destPath, 0777 ) ) {
if ( errno == EEXIST ) {
//already exists? well okay...
} else {
perror ( options.destPath );
exit ( 1 );
}
}
if(!options.skipCopy)
{
// Copy explog.txt file: all .txt files
char cmd[1024];
sprintf ( cmd, "cp -v %s/*.txt %s", options.expPath, options.destPath );
if(system ( cmd ) != 0 )
printf("failed to copy txt files from src\n");
// Copy lsrowimage.txt file
const char *filesToMove[] = {
"lsrowimage.dat",
"gainimage.dat",
"reimage.dat",
"rmsimage.dat"
};
for(int iFile=0; iFile < 4; iFile++) {
sprintf ( cmd, "cp -v %s/%s %s", options.expPath, filesToMove[iFile], options.destPath);
if(system ( cmd ) == 1)
fprintf (stdout, "No %s file found\n",filesToMove[iFile]);
}
}
while ( mode < maxMode ) {
if ( mode == 1 ) {
sprintf ( name, "%s/acq_%04d.dat", options.expPath, i );
sprintf ( destName, "%s/acq_%04d.dat", options.destPath, i );
} else if ( mode == 0 ) {
if ( i >= nameListLen ){
mode++;
i=0;
continue;
}
sprintf ( name, "%s/%s", options.expPath, nameList[i] );
sprintf ( destName, "%s/%s", options.destPath, nameList[i] );
} else
break;
if ( loader.LoadRaw ( name, 0, allocate, false, false ) ) {
allocate = false;
const RawImage *raw = loader.GetImage();
DetermineCropWidthAndHeight ( options.cropx, options.cropy, options.cropw, options.croph, raw->cols, raw->rows );
struct timeval tv;
double startT;
double stopT;
gettimeofday ( &tv, NULL );
startT = ( double ) tv.tv_sec + ( ( double ) tv.tv_usec/1000000 );
if(options.OverSample_skip || options.OverSample_combine)
DoOverSample(options,loader);
if(options.chipType)
ChipIdDecoder::SetGlobalChipId(options.chipType);
if(options.applyXtalkCorrection1){
PairPixelXtalkCorrector xtalkCorrector;
xtalkCorrector.Correct(loader.raw, options.applyXtalkCorrection1);
}
if(options.applyRowCorrection){
printf("Applying row noise correction\n");
CorrNoiseCorrector rnc;
rnc.CorrectCorrNoise(loader.raw,options.applyColCorrection?3:1,true,true,false );
}
else if(options.applyColCorrection){
printf("Applying column noise correction\n");
CorrNoiseCorrector rnc;
rnc.CorrectCorrNoise(loader.raw,0,true,true,false );
}
if(options.applyXtalkCorrection2){
PairPixelXtalkCorrector xtalkCorrector;
xtalkCorrector.Correct(loader.raw, options.applyXtalkCorrection2);
}
if(options.doColumnCorrectionTn)
{
ComparatorNoiseCorrector cnc;
cnc.CorrectComparatorNoiseThumbnail(loader.raw, NULL, 50,50, false);
} else if(options.doColumnCorrection){
ComparatorNoiseCorrector cnc;
bool beadfind = false;//((strstr(name,"beadfind_pre"))?true:false);
cnc.CorrectComparatorNoise(loader.raw, NULL, false, options.cncType,beadfind );
}
if(options.cmpFile){
// subtract this file before continuing
Image loader2;
loader2.LoadRaw ( options.cmpFile, 0, allocate, 2 );
if(options.OverSample_skip || options.OverSample_combine)
DoOverSample(options,loader2);
if(options.doColumnCorrectionTn)
{
ComparatorNoiseCorrector cnc;
cnc.CorrectComparatorNoiseThumbnail(loader2.raw, NULL, 100,100, false);
} else if(options.doColumnCorrection){
ComparatorNoiseCorrector cnc;
bool beadfind = ((strstr(options.cmpFile,"beadfind_pre"))?true:false);
cnc.CorrectComparatorNoise(loader2.raw, NULL, 0, options.cncType,beadfind );
}
{
short int *rawPtr = loader.raw->image;
short int *rawPtr2 = loader2.raw->image;
int frameStride=loader.raw->cols*loader.raw->rows;
for(int frame=0;frame<loader.raw->frames;frame++){
for(int idx=0;idx<frameStride;idx++){
*rawPtr -= *rawPtr2;
*rawPtr += 8192;
rawPtr++;
rawPtr2++;
}
}
}
printf("subtracted %s from the file\n",options.cmpFile);
}
if(options.subtractAverage)
{
// subtract off the mean trace
SubtractOffMean(loader);
printf("subtracted mean from trace\n");
}
// testing of lossy compression
if(!options.pfc && ImageTransformer::PCATest[0]) {
AdvComprTest(name,&loader,ImageTransformer::PCATest,true );
}
saver.SetData ( &loader );
if(options.T0Test)
saver.doT0Compression();
if ( options.regBasedAcq && i == 0 ) {
if ( options.excludeMask )
saver.GenerateExcludeMaskRegions ( ( const char* ) options.excludeMaskFile );
if ( options.useSeparatorT0File )
saver.PopulateCroppedRegionalAcquisitionWindow ( ( const char* ) options.separatorIn, "t0Map.txt",
options.cropx, options.cropy, options.cropw, options.croph, raw->timestamps[0] );
else
saver.ParseT0File ( ( const char* ) options.t0In, "t0Map.txt",
options.cropx, options.cropy, options.cropw, options.croph, raw->timestamps[0] );
}
printf ( "Converting raw data %d %d frames: %d UncompFrames: %d\n", raw->cols, raw->rows, raw->frames, raw->uncompFrames );
if ( options.doAscii ) {
if ( !saver.WriteAscii ( destName, options.cropx, options.cropy, options.cropw, options.croph ) )
break;
} else {
if ( options.vfc ) {
if(options.pfc) {
saver.WritePFV(destName, options.cropx, options.cropy, options.cropw, options.croph, options.PCAOpts);
}
else if ( options.regBasedAcq ) {
if ( !saver.WriteFrameAveragedRegionBasedAcq ( destName, options.cropx, options.cropy, options.cropw, options.croph ) )
break;
} else if ( options.TimeBasedAcq ) {
if ( !saver.WriteTimeBasedAcq ( destName, options.cropx, options.cropy, options.cropw, options.croph ) )
break;
} else {
if ( !saver.WriteVFC ( destName, options.cropx, options.cropy, options.cropw, options.croph ) )
break;
}
} else {
if ( !saver.Write ( destName, options.cropx, options.cropy, options.cropw, options.croph ) )
break;
}
}
gettimeofday ( &tv, NULL );
stopT = ( double ) tv.tv_sec + ( ( double ) tv.tv_usec/1000000 );
printf ( "Converted: %s in %0.2lf sec\n", name,stopT - startT );
fflush ( stdout );
i++;
} else {
if ( ( mode == 0 && i >= 12 ) || ( mode ==1 ) ) {
mode++;
i = 0;
allocate = true;
} else
i++;
}
}
}
void *FileLoadWorker ( void *arg )
{
WqInfo_t *wqinfo = (WqInfo_t *)arg;
int threadNum=wqinfo->threadNum;
WorkerInfoQueue *q = wqinfo->wq;
assert ( q );
bool done = false;
char dateStr[256];
struct tm newtime;
time_t ltime;
double T1=0,T2=0,T3,T4;
char name[20];
sprintf(name,"FileLdWkr%d",threadNum);
prctl(PR_SET_NAME,name,0,0,0);
const double noiseThreshold = 0.; // to be set by command line option
FluidPotentialCorrector fpCorr(noiseThreshold);
while ( !done )
{
WorkerInfoQueueItem item = q->GetItem();
if ( item.finished == true )
{
// we are no longer needed...go away!
done = true;
q->DecrementDone();
continue;
}
ImageLoadWorkInfo *one_img_loader = ( ImageLoadWorkInfo * ) item.private_data;
ClockTimer timer;
Timer tmr;
Image *img = &one_img_loader->img[one_img_loader->cur_buffer];
if (one_img_loader->inception_state->img_control.threaded_file_access)
{
tmr.restart();
if ( !img->LoadRaw ( one_img_loader->name) )
{
fprintf ( stdout, "ERROR: Failed to load raw image %s\n", one_img_loader->name );
exit ( EXIT_FAILURE );
}
T1=tmr.elapsed();
tmr.restart();
if(img->raw->imageState & IMAGESTATE_QuickPinnedPixelDetect)
one_img_loader->pinnedInFlow->QuickUpdate ( one_img_loader->flow, img);
else
one_img_loader->pinnedInFlow->Update ( one_img_loader->flow, img,(ImageTransformer::gain_correction?ImageTransformer::gain_correction:0));
T2=tmr.elapsed();
}
tmr.restart();
// correct in-channel electrical cross-talk
ImageTransformer::XTChannelCorrect ( one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->img[one_img_loader->cur_buffer].results_folder ); // buffer_ix
// testing of lossy compression
if(ImageTransformer::PCATest[0]) {
AdvComprTest(one_img_loader->name,&one_img_loader->img[one_img_loader->cur_buffer],ImageTransformer::PCATest,false/*one_img_loader->inception_state->img_control.col_flicker_correct*/ );
}
// col noise correction (if done during lossy compression will already have happened.
else if ( !(img->raw->imageState & IMAGESTATE_ComparatorCorrected) &&
one_img_loader->inception_state->img_control.col_flicker_correct )
{
if( one_img_loader->inception_state->img_control.col_pair_pixel_xtalk_correct ){
PairPixelXtalkCorrector xtalkCorrector;
float xtalk_fraction = one_img_loader->inception_state->img_control.pair_xtalk_fraction;
xtalkCorrector.Correct(one_img_loader->img[one_img_loader->cur_buffer].raw, xtalk_fraction);
}
if (one_img_loader->inception_state->img_control.corr_noise_correct){
CorrNoiseCorrector rnc;
rnc.CorrectCorrNoise(one_img_loader->img[one_img_loader->cur_buffer].raw,3,one_img_loader->inception_state->bfd_control.beadfindThumbnail );
}
if(one_img_loader->inception_state->bfd_control.beadfindThumbnail)
{
//ComparatorNoiseCorrector
ComparatorNoiseCorrector cnc;
cnc.CorrectComparatorNoiseThumbnail(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->loc_context.regionXSize,one_img_loader->inception_state->loc_context.regionYSize, one_img_loader->inception_state->img_control.col_flicker_correct_verbose);
} else {
ComparatorNoiseCorrector cnc;
cnc.CorrectComparatorNoise(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, one_img_loader->inception_state->img_control.col_flicker_correct_verbose, one_img_loader->inception_state->img_control.aggressive_cnc,false,threadNum );
}
}
//#define DEBUG_IMAGE_CORR_ISSUES 1
#ifdef DEBUG_IMAGE_CORR_ISSUES
char newName[1024];
char *nptr,*ptr=one_img_loader->name;
while((nptr = strstr(ptr,"/")))ptr=nptr+1;
if(*ptr == '/')
ptr++;
sprintf(newName,"%s_proc",one_img_loader->name);
Acq saver;
saver.SetData ( &one_img_loader->img[one_img_loader->cur_buffer] );
saver.WriteVFC(newName, 0, 0, one_img_loader->img[one_img_loader->cur_buffer].raw->cols, one_img_loader->img[one_img_loader->cur_buffer].raw->rows);
#endif
T3=tmr.elapsed();
tmr.restart();
// Fluid potential corrector
const bool correctFluidPotential = one_img_loader->inception_state->img_control.fluid_potential_correct;
const double noiseThreshold = (double) one_img_loader->inception_state->img_control.fluid_potential_threshold;
if (correctFluidPotential){
fpCorr.setThreshold(noiseThreshold);
// parse rowsum file name to load
const std::string datFileName = one_img_loader->name;
printf("dat file name %s\n", datFileName.c_str());
const size_t pos1 = datFileName.find_last_of('/');
const size_t pos2 = datFileName.find_last_of('.');
const std::string rowsumFileName = datFileName.substr(0, pos1) + "/../rowsum/" + datFileName.substr(pos1+1, pos2-pos1-1) + ".hwsum";
// printf("rowsum file name %s pos1 %u pos2 %u\n", rowsumFileName.c_str(), (uint32_t) pos1, (uint32_t) pos2);
// determine if the data file is thumbnail or block
const bool isThumbnail = one_img_loader->inception_state->bfd_control.beadfindThumbnail;
if(isThumbnail){
fpCorr.setIsThumbnail();
}
// set image
const unsigned int regionXSize = one_img_loader->inception_state->loc_context.regionXSize;
const unsigned int regionYSize = one_img_loader->inception_state->loc_context.regionYSize;
const char nucChar = one_img_loader->inception_state->flow_context.ReturnNucForNthFlow(one_img_loader->flow);
fpCorr.setImage(one_img_loader->img[one_img_loader->cur_buffer].raw, one_img_loader->mask, regionYSize, regionXSize, nucChar);
//printf("nucChar is %c\n", nucChar);
// load sensing electrode data
if (isThumbnail){
const unsigned int numRows = one_img_loader->img[one_img_loader->cur_buffer].raw->rows;
fpCorr.loadSensingElectrodeDataThumbnail(rowsumFileName, numRows);
} else {
// determine startRow and endRow
one_img_loader->img[one_img_loader->cur_buffer].SetOffsetFromChipOrigin(one_img_loader->name);
const unsigned int startRow = one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y;
const unsigned int endRow = startRow + one_img_loader->img[one_img_loader->cur_buffer].raw->rows;
// printf("offset x y = %d %d, start and end rows are %u,%u\n", one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_x,one_img_loader->img[one_img_loader->cur_buffer].raw->chip_offset_y, startRow, endRow);
fpCorr.loadSensingElectrodeData(rowsumFileName, startRow, endRow);
}
// correct fluid potential
if (fpCorr.readRowSumIsSuccess()){
fpCorr.doCorrection();
} else {
printf("fluidPotentialCorrector skipped: Cannot find rowsum file %s \n", rowsumFileName.c_str());
}
}
// dump dc offset one_img_loaderrmation before we do any normalization
DumpDcOffset ( one_img_loader );
int buffer_ix = one_img_loader->cur_buffer;
// setting the mean of frames to zero will be done by the bkgmodel as soon as its loaded.
// the traces will be zero'd by the bknd model loader anyway, no need to do them here.
// img->SetMeanOfFramesToZero ( one_img_loader->normStart, one_img_loader->normEnd,0 );
// calculate the smooth pH step amplitude in empty wells across the whole image
if ( one_img_loader->doEmptyWellNormalization )
{
// this is redundant with what's in the EmptyTrace class, but this method of spatial normalization
// is going to go away soon, so not worth fixing
MaskType referenceMask = MaskReference;
if ( one_img_loader->inception_state->bkg_control.trace_control.use_dud_and_empty_wells_as_reference )
referenceMask = ( MaskType ) ( MaskReference | MaskDud );
one_img_loader->img[buffer_ix].CalculateEmptyWellLocalScaleForFlow ( * ( one_img_loader->pinnedInFlow ),one_img_loader->mask,one_img_loader->flow,referenceMask,one_img_loader->smooth_span );
#if 0
char fname[512];
sprintf ( fname,"ewampimg_%04d.txt",one_img_loader->flow );
FILE *ewampfile = fopen ( fname,"w" );
for ( int row=0;row < one_img_loader->img[buffer_ix].GetRows();row++ )
{
int col;
for ( col=0;col < ( one_img_loader->img[buffer_ix].GetCols()-1 ); col++ )
fprintf ( ewampfile,"%9.5f\t",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) );
fprintf ( ewampfile,"%9.5f\n",one_img_loader->img[buffer_ix].getEmptyWellAmplitude ( row,col ) );
}
fclose ( ewampfile );
#endif
}
// dump pH step debug one_img_loader to file (its really fast, don't worry)
DumpStep ( one_img_loader );
if ( one_img_loader->doRawBkgSubtract )
{
one_img_loader->img[buffer_ix].SubtractLocalReferenceTrace ( one_img_loader->mask, MaskBead, MaskReference, one_img_loader->NNinnerx,
one_img_loader->NNinnery,one_img_loader->NNouterx,one_img_loader->NNoutery, false, true, true );
if ( one_img_loader->flow==0 ) // absolute first flow,not flow buffer
printf ( "Notify user: NN empty subtraction in effect. No further warnings will be given. %d \n",one_img_loader->flow );
}
T4=tmr.elapsed();
// printf ( "Allow model to go %d \n", one_img_loader->flow );
SetReadCompleted(one_img_loader);
size_t usec = timer.GetMicroSec();
ltime=time(<ime);
localtime_r(<ime, &newtime);
strftime(dateStr,sizeof(dateStr),"%H:%M:%S", &newtime);
fprintf ( stdout, "FileLoadWorker: ImageProcessing time for flow %d: %0.2lf(ld=%.2f pin=%.2f cnc=%.2f xt=%.2f sem=%.2lf cache=%.2lf) sec %s\n",
one_img_loader->flow , usec / 1.0e6, T1, T2, T3, T4, img->SemaphoreWaitTime, img->CacheAccessTime, dateStr);
fflush(stdout);
fprintf(stdout, "File: %s\n", one_img_loader->name);
fflush(stdout);
q->DecrementDone();
}
return ( NULL );
}
int main(int argc, char *argv[])
{
//int cropx = 624, cropy = 125, cropw = 100, croph = 100;
int cropx = 0, cropy = 0, cropw = 0, croph = 0;
char *expPath = const_cast<char*>(".");
char *destPath = const_cast<char*>("./converted");
char *oneFile = NULL;
int alternate_sampling=0;
int doAscii = 0;
int vfc = 1;
int regBasedAcq = 0;
int TimeBasedAcq = 0;
int excludeMask = 0;
int useSeparatorT0File = true;
char* separatorIn = const_cast<char*>("./separator.summary.txt");
char* t0In = const_cast<char*>("./T0file.txt");
char* excludeMaskFile = "/opt/ion/config/exclusionMask_318.bin";
int dont_retry = 0;
if (argc == 1) {
usage(cropx, cropy, cropw, croph);
}
int argcc = 1;
while (argcc < argc) {
switch (argv[argcc][1]) {
case 'a':
doAscii = 1;
break;
case 'x':
argcc++;
cropx = atoi(argv[argcc]);
break;
case 'y':
argcc++;
cropy = atoi(argv[argcc]);
break;
case 'w':
argcc++;
cropw = atoi(argv[argcc]);
break;
case 'h':
argcc++;
// don't segfault if called with -h (user expects help)
if (argcc >= argc) {
usage(cropx, cropy, cropw, croph);
}
croph = atoi(argv[argcc]);
break;
case 's':
argcc++;
expPath = argv[argcc];
break;
case 'f':
argcc++;
oneFile = argv[argcc];
break;
case 'z':
dont_retry = 1;
break;
case 'c':
vfc=1;
cropx=0;
cropy=0;
cropw=0;
croph=0;
break;
case 'n':
vfc=0;
break;
case 'b':
alternate_sampling=1;
break;
case 'v':
fprintf (stdout, "%s", IonVersion::GetFullVersion("Crop").c_str());
exit (0);
break;
case 'H':
usage(cropx, cropy, cropw, croph);
break;
case 'd':
argcc++;
destPath = argv[argcc];
break;
case 'r':
regBasedAcq=1;
break;
case 'l':
TimeBasedAcq=1;
break;
case 't':
argcc++;
separatorIn = argv[argcc];
break;
case 'e':
argcc++;
t0In = argv[argcc];
useSeparatorT0File = false;
break;
case 'g':
argcc++;
excludeMaskFile = argv[argcc];
excludeMask = 1;
break;
default:
argcc++;
fprintf (stdout, "\n");
}
argcc++;
}
char name[MAX_PATH_LENGTH];
char destName[MAX_PATH_LENGTH];
int i;
Image loader;
Acq saver;
int mode = 0;
i = 0;
bool allocate = true;
char **nameList;
char *defaultNameList[] = {"beadfind_post_0000.dat", "beadfind_post_0001.dat", "beadfind_post_0002.dat", "beadfind_post_0003.dat",
"beadfind_pre_0000.dat", "beadfind_pre_0001.dat", "beadfind_pre_0002.dat", "beadfind_pre_0003.dat",
"prerun_0000.dat", "prerun_0001.dat", "prerun_0002.dat", "prerun_0003.dat", "prerun_0004.dat"};
int nameListLen;
// Turn off this feature which sucks the life out of performance.
PWR::PinnedWellReporter::Instance( false );
// if requested...do not bother waiting for the files to show up
if (dont_retry)
loader.SetTimeout(1,1);
if (oneFile != NULL)
{
nameList = &oneFile;
nameListLen = 1;
mode = 1;
}
else
{
nameList = defaultNameList;
nameListLen = sizeof(defaultNameList)/sizeof(defaultNameList[0]);
}
// Create results folder
umask (0); // make permissive permissions so its easy to delete.
if (mkdir (destPath, 0777))
{
if (errno == EEXIST) {
//already exists? well okay...
}
else {
perror (destPath);
exit (1);
}
}
// Copy explog.txt file: all .txt files
char cmd[1024];
sprintf (cmd, "cp -v %s/*.txt %s", expPath, destPath);
assert(system(cmd) == 0);
while (mode < 2) {
if (mode == 0) {
sprintf(name, "%s/acq_%04d.dat", expPath, i);
sprintf(destName, "%s/acq_%04d.dat", destPath, i);
} else if (mode == 1) {
if(i >= nameListLen)
break;
sprintf(name, "%s/%s", expPath, nameList[i]);
sprintf(destName, "%s/%s", destPath, nameList[i]);
} else
break;
if (loader.LoadRaw(name, 0, allocate, false)) {
allocate = false;
const RawImage *raw = loader.GetImage();
DetermineCropWidthAndHeight(cropx, cropy, cropw, croph, raw->cols, raw->rows);
struct timeval tv;
double startT;
double stopT;
gettimeofday(&tv, NULL);
startT = (double) tv.tv_sec + ((double) tv.tv_usec/1000000);
saver.SetData(&loader);
if (regBasedAcq && i == 0) {
if (excludeMask)
saver.GenerateExcludeMaskRegions((const char*)excludeMaskFile);
if (useSeparatorT0File)
saver.PopulateCroppedRegionalAcquisitionWindow((const char*)separatorIn, "t0Map.txt",
cropx, cropy, cropw, croph, raw->timestamps[0]);
else
saver.ParseT0File((const char*)t0In, "t0Map.txt",
cropx, cropy, cropw, croph, raw->timestamps[0]);
}
printf("Converting raw data %d %d frames: %d UncompFrames: %d\n", raw->cols, raw->rows, raw->frames, raw->uncompFrames);
if (doAscii) {
if (!saver.WriteAscii(destName, cropx, cropy, cropw, croph))
break;
}
else {
if(vfc)
{
if (regBasedAcq) {
if (!saver.WriteFrameAveragedRegionBasedAcq(destName, cropx, cropy, cropw, croph))
break;
}
else if (TimeBasedAcq) {
if (!saver.WriteTimeBasedAcq(destName, cropx, cropy, cropw, croph))
break;
}
else {
if (!saver.WriteVFC(destName, cropx, cropy, cropw, croph))
break;
}
}
else
{
if (!saver.Write(destName, cropx, cropy, cropw, croph))
break;
}
}
gettimeofday(&tv, NULL);
stopT = (double) tv.tv_sec + ((double) tv.tv_usec/1000000);
printf("Converted: %s in %0.2lf sec\n", name,stopT - startT);
fflush (stdout);
i++;
} else {
if ((mode == 1 && i >= 12) || (mode == 0)) {
mode++;
i = 0;
allocate = true;
} else
i++;
}
}
}