/***************************************************************************** // The knn matching with k = 2 // This code performs the matching and the refinement. // @paraam query_image: the input image // @param matches_out: a pointer that stores the output matches. It is necessary for // pose estimation. */ int knn_match(cv::Mat& query_image, std::vector< cv::DMatch> * matches_out) { // variabels that keep the query keypoints and query descriptors std::vector<cv::KeyPoint> keypointsQuery; cv::Mat descriptorQuery; // Temporary variables for the matching results std::vector< std::vector< cv::DMatch> > matches1; std::vector< std::vector< cv::DMatch> > matches2; std::vector< std::vector< cv::DMatch> > matches_opt1; ////////////////////////////////////////////////////////////////////// // 1. Detect the keypoints // This line detects keypoints in the query image _detector->detect(query_image, keypointsQuery); // If keypoints were found, descriptors are extracted. if(keypointsQuery.size() > 0) { // extract descriptors _extractor->compute( query_image, keypointsQuery, descriptorQuery); } ////////////////////////////////////////////////////////////////////////////// // 2. Here we match the descriptors with the database descriptors. // with k-nearest neighbors with k=2 _matcher.knnMatch(descriptorQuery , matches1, 2); #ifdef DEBUG_OUT std::cout << "Found " << matches1.size() << " matching feature descriptors out of " << _matcher.getTrainDescriptors().size() << " database descriptors." << std::endl; #endif ////////////////////////////////////////////////////////////////////////////// // 3 Filter the matches. // Accept only matches (knn with k=2) which belong ot one images // The database tree within _matcher contains descriptors of all input images. // We expect that both nearest neighbors must belong to one image. // Otherwise we can remove the result. // Along with this, we count which reference image has the highest number of matches. // At this time, we consinder this image as the searched image. // we init the variable hit with 0 std::vector<int> hits(_num_ref_images); for (int i=0; i<_num_ref_images; i++) { hits[i] = 0; } // the loop runs through all matches and comparees the image indes // imgIdx. The must be equal otherwise the matches belong to two // different reference images. for (int i=0; i<matches1.size(); i++) { // The comparison. if(matches1[i].at(0).imgIdx == matches1[i].at(1).imgIdx) { // we keep it matches_opt1.push_back(matches1[i]); // and count a hit hits[matches1[i].at(0).imgIdx]++; } } #ifdef DEBUG_OUT std::cout << "Optimized " << matches_opt1.size() << " feature descriptors." << std::endl; #endif // Now we search for the highest number of hits in our hit array // The variable max_idx keeps the image id. // The variable max_value the amount of hits. int max_idx = -1; int max_value = 0; for (int i=0; i<_num_ref_images; i++) { #ifdef DEBUG_OUT std::cout << "for " << i << " : " << hits[i] << std::endl; #endif if(hits[i] > max_value) { max_value = hits[i]; max_idx = i; } } /////////////////////////////////////////////////////// // 4. The cross-match // At this time, we test the database agains the query descriptors. // The variable max_id stores the reference image id. Thus, we test only // the descriptors that belong to max_idx agains the query descriptors. _matcher.knnMatch(_descriptorsRefDB[max_idx], descriptorQuery, matches2, 2); /////////////////////////////////////////////////////// // 5. Refinement; Ratio test // The ratio test only accept matches which are clear without ambiguity. // The best hit must be closer to the query descriptors than the second hit. int removed = ratioTest(matches_opt1); #ifdef DEBUG_OUT std::cout << "Removed " << removed << " matched." << std::endl; #endif removed = ratioTest(matches2); #ifdef DEBUG_OUT std::cout << "Removed " << removed << " matched." << std::endl; #endif /////////////////////////////////////////////////////// // 6. Refinement; Symmetry test // We only accept matches which appear in both knn-matches. // It should not matter whether we test the database against the query desriptors // or the query descriptors against the database. // If we do not find the same solution in both directions, we toss the match. std::vector<cv::DMatch> symMatches; symmetryTest( matches_opt1, matches2, symMatches); #ifdef DEBUG_OUT std::cout << "Kept " << symMatches.size() << " matches after symetry test test." << std::endl; #endif /////////////////////////////////////////////////////// // 7. Refinement; Epipolar constraint // We perform a Epipolar test using the RANSAC method. if(symMatches.size() > 25) { matches_out->clear(); ransacTest( symMatches, _keypointsRefDB[max_idx], keypointsQuery, *matches_out); } #ifdef DEBUG_OUT std::cout << "Kept " << matches_out->size() << " matches after RANSAC test." << std::endl; #endif /////////////////////////////////////////////////////// // 8. Draw this image on screen. cv::Mat out; cv::drawMatches(feature_map_database[max_idx]._ref_image , _keypointsRefDB[max_idx], query_image, keypointsQuery, *matches_out, out, cv::Scalar(255,255,255), cv::Scalar(0,0,255)); std::string num_matches_str; std::strstream conv; conv << matches_out->size(); conv >> num_matches_str; std::string text; text.append( num_matches_str); text.append("( " + _num_ref_features_in_db_str + " total)"); text.append(" matches were found in reference image "); text.append( feature_map_database[max_idx]._ref_image_str); putText(out, text, cvPoint(20,20), cv::FONT_HERSHEY_COMPLEX_SMALL, 1.0, cvScalar(0,255,255), 1, CV_AA); cv::imshow("result", out); if (run_video) cv::waitKey(1); else cv::waitKey(); // Delete the images query_image.release(); out.release(); return max_idx; }
/***************************************************************************** // MAIN */ int main(int argc, const char * argv[]) { //************************************************************************* // 1. Read the input files // This code reads the arguments from the input variable argv which is supposed to contain the // path of the input and reference database. std::string teachdb_folder, querydb_folder; if (argc > 2) { std::string command = argv[2]; std::string type = argv[1]; if(type.compare("-SIFT")== 0) { _ftype = SIFT; std::cout << "NFT with SIFT feature detector and extractor." << std::endl; } else if(type.compare("-SURF")== 0) { _ftype = SURF; std::cout << "NFT with SURF feature detector and extractor." << std::endl; } else if(type.compare("-ORB")== 0) { _ftype = ORB; std::cout << "NFT with ORB feature detector and extractor." << std::endl; } if(command.compare("-file") == 0) { if(argc > 4) { teachdb_folder = argv[3]; querydb_folder = argv[4]; run_video = false; } else { std::cout << "No folder with query or reference images has been specified" << std::endl; std::cout << "Call: ./HCI571X_Feature_Matching -file folder_reference folder_query" << std::endl; system("pause"); exit(0); } } else if(command.compare("-video") == 0) { run_video = true; if(argc > 4) { teachdb_folder = argv[3]; device_id = atoi(argv[4]); } } } else { std::cout << "No command has been specified. use -file or -video" << std::endl; system("pause"); exit(0); } // Read the filenames inside the teach database directory. std::vector<std::string> ref_filename; readDirFiles(teachdb_folder, &ref_filename); // Read the filenames inside the query database directory. std::vector<std::string> query_filename; readDirFiles(querydb_folder, &query_filename); //************************************************************************* // 2. Create a detector and a descriptor extractor // In this case, the SIFT detector and extractor are used // Corner detector if(_ftype == SIFT)_detector = new cv::SiftFeatureDetector(_num_feature, _octaves, _contrast_threshold, _edge_threshold, _sigma); else if(_ftype == SURF)_detector = new cv::SurfFeatureDetector( _hessianThreshold, _surf_Octaves, _surf_OctaveLayers, _surf_extended, _surf_upright ); else if(_ftype == ORB)_detector = new cv::OrbFeatureDetector(1000); // Corner extractor if(_ftype == SIFT) _extractor = new cv::SiftDescriptorExtractor(_num_feature, _octaves, _contrast_threshold, _edge_threshold, _sigma); else if(_ftype == SURF) _extractor = new cv::SurfDescriptorExtractor( _hessianThreshold, _surf_Octaves, _surf_OctaveLayers, _surf_extended, _surf_upright ); else if(_ftype == ORB)_extractor = new cv::OrbDescriptorExtractor(1000); // Check whether files are in the database list. if(ref_filename.size() == 0) { std::cout << "STOP: no files in the reference database!!! Specify a folder or a set of files." << std::cout; system("pause"); return -1; } //************************************************************************* // 3. Init the database // The code reads all the images in ref_filename, detect keypoints, extract descriptors and // stores them in the datbase variables. init_database(std::string(teachdb_folder), ref_filename); //************************************************************************* // 4. The data of the database _descriptorsRefDB is added to the featue matcher // and the mathcer is trained _matcher.add(_descriptorsRefDB); _matcher.train(); // Read the number of reference images in the database _num_ref_images = _matcher.getTrainDescriptors().size(); //************************************************************************* // 5. Here we run the matching. // for images from files if(!run_video) { if(_mtype == KNN)run_matching( querydb_folder, query_filename); else if(_mtype == BRUTEFORCE) run_bf_matching(querydb_folder, query_filename); else { std::cout << "No matching type specified. Specify a matching type" << std::endl; system("pause"); } } else // and image from a video camera { if(_mtype == KNN)run_matching( device_id); else if(_mtype == BRUTEFORCE) run_bf_matching(device_id); else { std::cout << "No matching type specified. Specify a matching type" << std::endl; system("pause"); } } //************************************************************************* // 6. Cleanup: release the keypoint detector and feature descriptor extractor _extractor.release(); _detector.release(); return 1; }