double benchmark_computeDescriptor(int N, int num_feats)
{
CImage img;
getTestImage(0, img);
CFeatureExtraction fExt;
fExt.profiler.enable();
fExt.options.featsType = featFASTER9;
for (int i = 0; i < N; i++)
{
CFeatureList fs;
fExt.detectFeatures(img, fs, 0 /*id*/, num_feats);
fExt.computeDescriptors(img, fs, DESCRIPTOR_TYPE);
}
return fExt.profiler.getMeanTime("computeDescriptors");
}
// ------------------------------------------------------
// Benchmark: Spin descriptor
// ------------------------------------------------------
double feature_extraction_test_Spin_desc( int N, int h )
{
CTicTac tictac;
// Generate a random image
CImage img;
getTestImage(0,img);
CFeatureExtraction fExt;
CFeatureList featsHarris;
fExt.options.SpinImagesOptions.radius = 13;
fExt.options.SpinImagesOptions.hist_size_distance = 10;
fExt.options.SpinImagesOptions.hist_size_intensity = 10;
fExt.detectFeatures( img, featsHarris );
tictac.Tic();
for (int i=0;i<N;i++)
fExt.computeDescriptors( img, featsHarris, descSpinImages );
const double T = tictac.Tac()/N;
return T;
}
// ------------------------------------------------------
// Benchmark: SIFT descriptor only
// ------------------------------------------------------
double feature_extraction_test_SIFT_desc( int N, int h )
{
CTicTac tictac;
// Generate a random image
CImage img;
getTestImage(0,img);
CFeatureExtraction fExt;
CFeatureList featsHarris;
fExt.options.featsType = featHarris;
fExt.detectFeatures( img, featsHarris );
tictac.Tic();
for (int i=0;i<N;i++)
fExt.computeDescriptors( img, featsHarris, descSIFT );
const double T = tictac.Tac()/N;
// cout << "SIFT desc: " << featsHarris.size();
return T;
}
// ------------------------------------------------------
// TestMatchingComparative
// ------------------------------------------------------
void TestMatchingComparative()
{
// Take two images
string imgL = myDataDir + string("imL_p01.jpg"); // Left image
string imgR = myDataDir + string("imR_p01.jpg"); // Right image
CImage im1, im2;
im1.loadFromFile( imgL );
im2.loadFromFile( imgR );
size_t imW = im1.getWidth();
size_t imH = im1.getHeight();
CFeatureExtraction fExt;
fExt.options.featsType = featFAST;
fExt.options.patchSize = 21;
fExt.options.SIFTOptions.implementation = CFeatureExtraction::Hess;
// Find FAST features
CFeatureList list1, list2;
fExt.detectFeatures( im1, list1, 150 );
// Compute SIFT & SURF descriptors
fExt.computeDescriptors( im1, list1, descSIFT );
fExt.computeDescriptors( im1, list1, descSURF );
fExt.detectFeatures( im2, list2, 150 );
// Compute SIFT & SURF descriptors
fExt.computeDescriptors( im2, list2, descSIFT );
fExt.computeDescriptors( im2, list2, descSURF );
CFeatureList::iterator it1, it2;
for( it1 = list1.begin(); it1 != list1.end(); ++it1 )
im1.cross( (*it1)->x, (*it1)->y, TColor::red, '+');
for( it2 = list2.begin(); it2 != list2.end(); ++it2 )
im2.cross( (*it2)->x, (*it2)->y, TColor::red, '+');
CDisplayWindow win, win2;
win.setPos(0,0);
win2.setPos(0,imH*1.5);
CImage joinimage, copyjoinimage, copyInfoImage;
size_t imW2 = 1280;
size_t imH2 = 150;
CImage infoimage( imW2, imH2, CH_RGB );
joinimage.joinImagesHorz( im1, im2 );
infoimage.filledRectangle( 0, 0, imW2, imH2, TColor(150,150,150) );
infoimage.textOut( 20, imH2-53, "SAD", TColor::blue );
infoimage.textOut( 20, imH2-41, "NCC", TColor::blue );
infoimage.textOut( 20, imH2-29, "SIFT", TColor::blue );
infoimage.textOut( 20, imH2-17, "SURF", TColor::blue );
for( it1 = list1.begin(); it1 != list1.end(); ++it1 )
{
copyInfoImage = infoimage;
copyjoinimage = joinimage;
copyjoinimage.line( (*it1)->x, 0, (*it1)->x, imH, TColor::green ); // Horiz
copyjoinimage.line( (*it1)->x+imW, 0, (*it1)->x+imW, imH, TColor::green ); // Horiz
copyjoinimage.line( 0, (*it1)->y, imW+imW, (*it1)->y, TColor::green ); // Epipolar
copyjoinimage.drawCircle( (*it1)->x, (*it1)->y, 4, TColor::green, 2 ); // Keypoint
copyInfoImage.update_patch( (*it1)->patch, 0, 0 );
bool firstMatch = true;
int cnt = 0;
int px = 80;
double minsad = 1.0, maxncc = 0.0;
float minsiftd = 1.0f, minsurfd = 1.0f;
int idxsad = 0, idxncc = 0, idxsiftd = 0, idxsurfd = 0;
for( it2 = list2.begin(); it2 != list2.end(); ++it2 )
{
if( fabs((*it1)->y-(*it2)->y) <= 1.0 && (*it1)->x > (*it2)->x )
{
// Compute matching with SAD and Correlation and SIFT/SURF?
// Use epipolar constraints
// Compute SAD
double sad = mrpt::vision::computeSAD( (*it1)->patch, (*it2)->patch );
if( sad < minsad )
{
minsad = sad;
idxsad = cnt;
}
// Compute Correlation
double ncc;
size_t u, v;
mrpt::vision::openCV_cross_correlation( (*it1)->patch, (*it2)->patch, u, v, ncc );
if( ncc > maxncc )
{
maxncc = ncc;
idxncc = cnt;
}
// Compute distance between descriptors SIFT
float siftd = (*it1)->descriptorSIFTDistanceTo( *(*it2) );
if( siftd < minsiftd )
{
minsiftd = siftd;
idxsiftd = cnt;
}
// Compute distance between descriptors SIFT
float surfd = (*it1)->descriptorSURFDistanceTo( *(*it2) );
if( surfd < minsurfd )
{
minsurfd = surfd;
idxsurfd = cnt;
}
// Plot images + features + each candidate + difference score
if( firstMatch )
{
copyjoinimage.line( (*it1)->x+imW, 0, (*it1)->x+imW, imH, TColor::green ); // Limit line (only the first time)
firstMatch = false;
} // end-if
copyjoinimage.drawCircle( (*it2)->x+imW, (*it2)->y, 4, TColor::blue, 2 ); // Keypoint
double rx0, rx1, ry0, ry1, tx, ty;
rx0 = (*it2)->x+imW-15;
rx1 = (*it2)->x+imW;
tx = (*it2)->x+imW-13;
if( cnt % 2 )
{
ry0 = (*it2)->y-20;
ry1 = (*it2)->y-10;
ty = (*it2)->y-22;
}
else
{
ry0 = (*it2)->y+10;
ry1 = (*it2)->y+20;
ty = (*it2)->y+8;
}
copyjoinimage.filledRectangle( rx0, ry0, rx1, ry1, TColor(150,150,150) );
copyjoinimage.textOut( tx, ty, format("%d", cnt), TColor::blue );
px = 80+cnt*50;
if( px + fExt.options.patchSize > imW2 )
continue;
copyInfoImage.update_patch( (*it2)->patch, px, 30 );
copyInfoImage.textOut( px, imH2-70, format("%d", cnt), TColor::blue );
copyInfoImage.textOut( px, imH2-53, format("%.2f", sad), TColor::blue );
copyInfoImage.textOut( px, imH2-41, format("%.2f", ncc), TColor::blue );
copyInfoImage.textOut( px, imH2-29, format("%.2f", siftd), TColor::blue );
copyInfoImage.textOut( px, imH2-17, format("%.2f", surfd), TColor::blue );
cnt++;
} // end if
} // end for it2
copyInfoImage.textOut( 80+idxsad*50, imH2-53, format("%.2f", minsad), TColor::green );
copyInfoImage.textOut( 80+idxncc*50, imH2-41, format("%.2f", maxncc), TColor::green );
copyInfoImage.textOut( 80+idxsiftd*50, imH2-29, format("%.2f", minsiftd), TColor::green );
copyInfoImage.textOut( 80+idxsurfd*50, imH2-17, format("%.2f", minsurfd), TColor::green );
win.showImage( copyjoinimage );
win2.showImage( copyInfoImage );
mrpt::system::pause();
} // end for it1
// Save to file
// Check number of good features
} // end TestMatchingComparative
// ------------------------------------------------------
// TestExtractFeatures
// ------------------------------------------------------
void TestExtractFeatures()
{
CDisplayWindow wind1,wind2,wind3,wind4,wind5;
CFeatureExtraction fExt;
CFeatureList featsHarris, featsKLT, featsSIFT_Hess, featsSIFT_Lowe, featsSIFT_Vedaldi, featsSURF, featsFAST;
CImage img;
if (!img.loadFromFile(the_img_for_extract_feats ))
{
cerr << "Cannot load " << the_img_for_extract_feats << endl;
return;
}
cout << "Loaded test image: " << endl << the_img_for_extract_feats << endl;
cout << "--------------------------------------------------------------------------" << endl << endl;
CTicTac tictac;
fExt.options.patchSize = 0;
cout << "Detect Harris features... [f_harris.txt]" << endl;
tictac.Tic();
fExt.options.featsType = featHarris;
fExt.detectFeatures( img, featsHarris );
cout << "Detected " << featsHarris.size() << " features in ";
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsHarris.saveToTextFile("f_harris.txt");
wind1.setWindowTitle("Harris detected features");
wind1.showImageAndPoints(img, featsHarris);
cout << "Detect FAST features... [f_fast.txt]" << endl;
tictac.Tic();
fExt.options.featsType = featFAST;
fExt.options.FASTOptions.threshold = 15; //150;
fExt.options.FASTOptions.min_distance = 4;
fExt.options.FASTOptions.use_KLT_response = true;
fExt.detectFeatures( img, featsFAST, 0, 500 /* max num feats */ );
cout << "Detected " << featsFAST.size() << " features in ";
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsFAST.saveToTextFile("f_fast.txt");
wind5.setWindowTitle("FAST detected features");
wind5.showImageAndPoints( img, featsFAST );
cout << "Computing SIFT descriptors only ... [f_harris+sift.txt]" << endl;
tictac.Tic();
fExt.options.SIFTOptions.implementation = CFeatureExtraction::Hess;
fExt.computeDescriptors( img, featsHarris, descSIFT );
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsHarris.saveToTextFile("f_harris+sift.txt");
cout << "Extracting KLT features... [f_klt.txt]" << endl;
tictac.Tic();
fExt.options.featsType = featKLT;
fExt.options.KLTOptions.threshold = 0.05f;
fExt.options.KLTOptions.radius = 5;
fExt.detectFeatures( img, featsKLT, 0, 10 );
cout << "Detected " << featsKLT.size() << " features in ";
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsKLT.saveToTextFile("f_klt.txt");
wind2.setWindowTitle("KLT detected features");
wind2.showImageAndPoints( img, featsKLT );
cout << "Extracting SIFT features... [f_sift_hess.txt]" << endl;
tictac.Tic();
fExt.options.featsType = featSIFT;
fExt.options.SIFTOptions.implementation = CFeatureExtraction::Hess;
fExt.detectFeatures( img, featsSIFT_Hess );
cout << "Detected " << featsSIFT_Hess.size() << " features in ";
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsSIFT_Hess.saveToTextFile("f_sift_hess.txt");
wind3.setWindowTitle("SIFT Hess detected features");
wind3.showImageAndPoints( img, featsSIFT_Hess );
cout << "Extracting SURF features... [f_surf.txt]" << endl;
tictac.Tic();
fExt.options.featsType = featSURF;
fExt.detectFeatures( img, featsSURF );
cout << "Detected " << featsSURF.size() << " features in ";
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsSURF.saveToTextFile("f_surf.txt");
wind4.setWindowTitle("SURF detected features");
wind4.showImageAndPoints( img, featsSURF );
cout << "Computing spin images descriptors only ... [f_harris+spinimgs.txt]" << endl;
tictac.Tic();
fExt.options.SpinImagesOptions.radius = 13;
fExt.options.SpinImagesOptions.hist_size_distance = 10;
fExt.options.SpinImagesOptions.hist_size_intensity = 10;
fExt.computeDescriptors( img, featsHarris, descSpinImages );
cout << format(" %.03fms",tictac.Tac()*1000) << endl << endl;
featsHarris.saveToTextFile("f_harris+spinimgs.txt");
mrpt::system::pause();
return;
}
/************************************************************************************************
* Start Function function *
************************************************************************************************/
string PlaceRecognition::startPlaceRecognition(CFeatureExtraction fext)
{
ofstream training_file;
CTicTac feature_time;
feature_time.Tic();
/// stores the labels for the i'th image instance for training and testing
/// images
int training_labels[len_training];
int testing_labels[len_testing];
/// The training model is built here all features are extracted in this
/// part, takes 30 seconds for 900+900 images
if (!trained_flag)
{
for (int i = 0; i < len_training; i++)
{
training[i].loadFromFile(training_paths.at(i));
fext.detectFeatures(training[i], feats_training[i], 0, numFeats);
fext.computeDescriptors(
training[i], feats_training[i], desc_to_compute);
}
for (int i = 0; i < len_testing; i++)
{
testing[i].loadFromFile(testing_paths.at(i));
fext.detectFeatures(testing[i], feats_testing[i], 0, numFeats);
fext.computeDescriptors(
testing[i], feats_testing[i], desc_to_compute);
}
trained_flag = true;
feats_testing_org = feats_testing;
} /// end of if feature extraction flag (trained flag)
CTicTac label_time;
label_time.Tic();
computeLabels(training_paths, training_count, training_labels);
computeLabels(testing_paths, testing_count, testing_labels);
int len_train_words;
len_train_words = 0;
// int len_test_words = 0;
for (int i = 0; i < len_training; i++)
len_train_words += feats_training[i].size();
if (!training_file_written_flag)
{
training_words2 = new vector<float>[len_train_words];
training_words1 = new vector<uint8_t>[len_train_words];
}
int training_word_labels[len_train_words];
CTicTac training_time;
training_time.Tic();
training_file.open("training_images_features.txt");
if (!training_file_written_flag)
{
training_file.clear();
int kount = 0;
for (int i = 0; i < len_training; i++)
{
training_file << feats_training[i].size();
for (unsigned int j = 0; j < feats_training[i].size(); j++, kount++)
{
if (descriptor_selected == 0)
{
vector<uint8_t> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.SIFT;
training_words1[kount] =
feats_training[i].getByID(j).get()->descriptors.SIFT;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
training_file << (int)temp_feat.at(k) << " ";
}
else if (descriptor_selected == 1)
{
vector<float> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.SURF;
training_words2[kount] =
feats_training[i].getByID(j).get()->descriptors.SURF;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
{
training_file << temp_feat.at(k) << " ";
}
}
else if (descriptor_selected == 2)
{
vector<float> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.SpinImg;
training_words2[kount] =
feats_training[i].getByID(j).get()->descriptors.SpinImg;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
{
training_file << temp_feat.at(k) << " ";
}
}
else if (descriptor_selected == 3)
;
// //!< Polar image descriptor
else if (descriptor_selected == 4)
;
// //!< Log-Polar image descriptor
else if (descriptor_selected == 5)
{
vector<uint8_t> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.ORB;
training_words1[kount] =
feats_training[i].getByID(j).get()->descriptors.ORB;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
{
int temp_var; //= (int) temp_feat.at(k);
temp_var = (int)training_words1[kount].at(k);
training_file << temp_var << " ";
}
}
else if (descriptor_selected == 6)
{
vector<uint8_t> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.BLD;
training_words1[kount] =
feats_training[i].getByID(j).get()->descriptors.BLD;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
training_file << (int)temp_feat.at(k) << " ";
}
else if (descriptor_selected == 7)
{
vector<uint8_t> temp_feat;
temp_feat =
feats_training[i].getByID(j).get()->descriptors.LATCH;
training_words1[kount] =
feats_training[i].getByID(j).get()->descriptors.LATCH;
training_word_labels[kount] = training_labels[i];
for (unsigned int k = 0; k < temp_feat.size(); k++)
training_file << (int)temp_feat.at(k) << " ";
}
training_file << " #" << training_labels[i] << " $"
<< training_word_labels[kount] << endl;
} // end of inner for loop for number of key-points
} // end of outer for loop for number of images
training_file.close();
// testing_file.close();
this->training_words_org = training_words2;
this->training_words_org2 = training_words1;
this->training_word_labels_org = training_word_labels;
training_word_labels_org = new int[kount];
for (int i = 0; i < kount; i++)
{
training_word_labels_org[i] = training_word_labels[i];
}
this->total_vocab_size_org = len_train_words;
this->training_file_written_flag = true;
} // end of writting training features to a file
CTicTac testing_time;
testing_time.Tic();
/// now extracting features for Place Recognition for testing dataset
int predicted_classes[len_testing];
CTicTac time_prediction;
time_prediction.Tic();
int predicted_Label = 1;
if (descriptor_selected == 1)
predicted_Label = predictLabel(
feats_testing_org, training_words_org, training_word_labels_org,
total_vocab_size_org, current_index_test_image);
else
predicted_Label = predictLabel2(
feats_testing_org, training_words_org2, training_word_labels_org,
total_vocab_size_org, current_index_test_image);
current_index_test_image++;
/// use a bag of words kind of framework here
predicted_classes[current_index_test_image] = predicted_Label;
if (predicted_classes[current_index_test_image] ==
testing_labels[current_index_test_image])
correct++;
else
incorrect++;
stringstream output;
output << endl
<< endl
<< "PLACE RECOGNITION RESULTS " << endl
<< endl
<< "actual label : "
<< findPlaceName(
testing_labels[current_index_test_image % len_testing])
<< ".\n"
<< endl
<< " predicted label : " << findPlaceName(predicted_Label) << ".\n"
<< endl
<< " correct = " << correct << " incorrect = " << incorrect << ".\n"
<< " Current Accuracy: "
<< 100.00 * (double)correct / (double)(incorrect + correct) << " % "
<< endl
<< " image " << current_index_test_image << " of " << len_testing
<< endl;
return output.str();
}