int main()
{
//Change the default position of files!!!
string files_dir = "/home/shushman/Desktop/Codes/ml-project/files/";
//Change the default position of images!
string image_dir = "/home/shushman/Desktop/Codes/ml-project/test-images/";
string vocabpath,indexpath,labelpath;
keypath = files_dir+"keypoints";
vocabpath = files_dir+"vocabulary.yml";
FileStorage fs_vocab(vocabpath, FileStorage::READ );
cout<<"vocabulary file loaded!"<<endl;
indexpath = files_dir+"inverted_index.yml";
FileStorage fs_inv(indexpath, FileStorage::READ );
cout<<"index file loaded!"<<endl;
labelpath = files_dir+"labels.yml";
FileStorage fs_labels(labelpath, FileStorage::READ );
cout<<"labels file loaded!"<<endl;
Mat train_mat;
Mat label_mat;
fs_vocab["vocabulary"] >> vocabulary;//The training data matrix
vocabulary.convertTo(vocabulary,CV_32F);
fs_labels["labels"] >> labels;//The label data matrix
for(int i=0;i<VOCAB_SIZE;i++)
{
char n[50];
sprintf(n,"inverted_index_%d",i);
fs_inv[n] >> inverted_index[i];
}
fs_labels.release();
fs_inv.release();
fs_vocab.release();
bowDE.setVocabulary(vocabulary);
char ch;
string image_fname;
do{
cout<<"Enter the name of the image to test: "<<endl;
cin>>image_fname;
Mat img = imread(image_dir+image_fname,1);
int result = test(img);
if(result>0)
cout<<"Result is "<<bill_values[result]<<endl;
else
cout<<"Can't say!"<<endl;
cout<<"Do you wish to continue? (y/n)";
cin>>ch;
}while(ch!='n');
return 0;
}
void createConfusionMatrix()
{
for (unsigned int i = 0; i < categories.size(); i++)
{
vector<string> catFileNames = getTestFileNames(i);
vector<int> categoryLine;
for (unsigned int j = 0; j < categories.size(); j++)
categoryLine.push_back(0);
for (unsigned int j = 0; j < catFileNames.size(); j++)
{
Mat img, descriptor;
openImage(catFileNames[j], img);
vector<KeyPoint> keypoints;
detector->detect(img, keypoints);
bowExtractor.compute(img, keypoints, descriptor);
int catIndex = (int)svm.predict(descriptor);
categoryLine[catIndex]++;
}
for (unsigned int j = 0; j < categories.size(); j++)
cout << categoryLine[j] << ",";
cout << endl;
confusionMatrix.push_back(categoryLine);
}
saveConfusionMatrix();
}
void computeDescriptors()
{
for (unsigned int i = 0; i < categories.size(); i++)
{
vector<float> tempDescriptors;
string categoryName = categories[i];
vector<string> fileNames = getFileNames(categoryName);
int fileNumber = fileNames.size();
if (debug)
fileNumber = NUMBER_OF_FILES_CAT;
for (int j = 0; j < fileNumber; j++)
{
Mat image, descriptors;
vector<KeyPoint> keypoints;
openImage(fileNames[j], image);
detector->detect(image, keypoints);
bowExtractor.compute(image, keypoints, descriptors);
for (int k = 0; k < descriptors.cols; k++)
tempDescriptors.push_back(descriptors.at<float>(0, k));
descriptorsVector.push_back(tempDescriptors);
tempDescriptors.clear();
}
}
}
void loadVocabulary()
{
FileStorage fs(DATABASE_FILENAME, FileStorage::READ);
Mat vocabulary;
fs["Vocabulary"] >> vocabulary;
bowExtractor.setVocabulary(vocabulary);
fs.release();
}
int test(Mat img2)
{
//Obtains the result for a particular image. 0 for ten, 1 for twenty, 2 for fifty, 3 for hundred, 4 for five-hundred and 5 for thousand
// <0 when unsure (There are some other conditions for -1/-2 but just put all <0 as unsure)
Mat img;
cvtColor(img2,img,CV_BGR2GRAY);
Mat mask=grabcut_seg(img2);
if(img.cols>640)
{
resize(img,img,Size(640,img.rows*(640.0/img.cols)));
}
vector<KeyPoint> keypoints;
detector.detect(img,keypoints);
remove_keypoints(keypoints,mask);
//cout<<keypoints.size()<<" keypoints after masking"<<endl;
if(keypoints.size()<20)
{
return -1;
}
Mat descriptors;
vector<vector<int> > pointIdxsOfClusters;
bowDE.compute(img,keypoints,descriptors,&pointIdxsOfClusters);
int dot_result=sort_by_dot_product(descriptors);
re_rank_geo(keypoints,pointIdxsOfClusters);
//cout<<"After spatial re-ranking"<<endl;
int res=get_top_geoscore();
int result=labels.at<uchar>(res,0);
for(int i=0;i<10;i++)
{
res=get_top_geoscore();
result=labels.at<uchar>(res,0);
}
int response=max_decision();
for(int i=0;i<6;i++)
{
decision[i]=0;
}
return response;
}
void TestIndividualImage(string imagePath, string categoryName, int database_type)
{
//Load image
Mat testImage = imread(imagePath);
//Load database
vector<Mat> trainData, trainLabels;
vector<float> videoFPS;
vector<string> _listDataName = SetupModel(categoryName, trainData, trainLabels, videoFPS, database_type);
//Load dictionary for BOW model
if (database_type == 1)
bowDE.setVocabulary(LoadBOWDictionaryFromFile("Data/Training_Data/" + categoryName + "_BOW_dictionary.xml"));
clock_t t;
t = clock();
//---Extract feature based on database_type
Mat queryFeature;
if (database_type == 1)
queryFeature = ExtractBOWFeature(bowDE, detector, testImage);
else
queryFeature = ExtractMPEGFeature(testImage);
//For each test image, retrieve list of video and check if there's any correct video in candidate list
vector<int> predictLabels = RetrieveVideo(_listDataName.size(), queryFeature, trainData);
if (predictLabels.size() > 0)
{
cout << "This image can belong to: " << endl;
for (int j = 0; j < 3; j++)
{
int predictLabel = predictLabels[j];
cout << GetName(_listDataName[predictLabel]) << endl;
// string videoName = "Data/Raws/" + categoryName + "/" + _listRawClass[predictLabel];
RetrieveShot(videoFPS[predictLabel], trainData[predictLabel], trainLabels[predictLabel], queryFeature);
}
cout << endl;
}
else
{
cout << "No video match the query image" << endl << endl;
}
t = clock() - t;
cout << "It took " << ((float)t) / CLOCKS_PER_SEC << " seconds to complete all tasks" << endl;
}
void computeWeights()
{
Mat voc = bowExtractor.getVocabulary();
for (unsigned int i = 0; i < occurrencesInImageVector.size(); i++)
{
vector<float> temp;
for (unsigned int j = 0; j < occurrencesInImageVector[i].size(); j++)
{
float weight = (float)occurrencesInImageVector[i][j] / NUMBER_OF_WORDS *
log(occurrencesInImageVector.size() / occurrencesVector[j]);
temp.push_back(weight + descriptorsVector[i][j]);
}
indexes.push_back(temp);
}
}
/*
dataTraining dunction is a function that will allow us to store both character features and their corresponding float value for the ascii chracter they represent. As an example, 'A' is stored as 65.
The function takes a vector of images, a reference to a Mat array of labels which will hold the float value of ascii characters, a reference to a Mat array of trainingData, SurfFeatureDetector detector and a reference to BOWImgDescriptorExtractor and also the character we are training for.
We compute the keypoints of a character and their features (bowDescriptor) and store these features in a Mat array called trainingData. We also store the corresponding label for the features as the float value of the ascii character we are seeing.
The function will access and modify two Mat arrays holding features and the matching labels.
*/
void dataTraining (vector<Mat> images, Mat &labels, Mat &trainingData, SiftFeatureDetector detector, BOWImgDescriptorExtractor &bowDE, char character) {
vector<KeyPoint> keypoints;
Mat bowDescriptor;
cout<<images.size()<<" images"<<endl;
bool verify = false;
for (int j=0; j<images.size(); j++) {
Mat image = images.at(j);
detector.detect(image, keypoints);
bowDE.compute(image, keypoints, bowDescriptor);
cout<<keypoints.size()<<" "<<bowDescriptor.size()<<endl;
if (bowDescriptor.size()!=Size(0,0)) {
verify=true;
trainingData.push_back(bowDescriptor);
labels.push_back((float) character);
}
}
cout<<labels.rows<<" "<<trainingData.rows<<endl<<endl;
}
vector<Mat> getHistAndLabels(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, int dictionarySize) {
// setup variable and object I need
IplImage *img2;
Mat labels(0, 1, CV_32FC1);
Mat trainingData(0, dictionarySize, CV_32FC1);
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
vector<string> files = vector<string>();
helper.GetFileList(EVAL_DIR, files);
float labelVal;
for (unsigned int iz = 0; iz < files.size(); iz++) {
int isImage = helper.instr(files[iz], "jpg", 0, true);
if (isImage > 0) {
string sFileName = TRAINING_DIR;
sFileName.append(files[iz]);
const char * imageName = sFileName.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
trainingData.push_back(bowDescriptor1);
labelVal = iz+1;
labels.push_back(labelVal);
}
}
}
vector<Mat> retVec;
retVec.push_back(trainingData);
retVec.push_back(labels);
return retVec;
}
Mat getSingleImageHistogram(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, string evalFile) {
// setup variable and object I need
IplImage *img2;
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
int isImage = helper.instr(evalFile, "jpg", 0, true);
if (isImage > 0) {
const char * imageName = evalFile.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
}
}
return bowDescriptor1;
}
vector<pair<int, float> > paintingSearch(const Mat& QI,const SparseMat& hists, BOWImgDescriptorExtractor bow,
vector<KeyPoint>& kp,vector<vector<int> >& hist)
{
ORB detect;
vector<pair<int, float> > imageResponse;
detect(QI, Mat(), kp);
Mat imgDescriptor;
bow.compute(QI, kp, imgDescriptor, &hist);
Mat nhist = normalise(hist);
Mat answer = mul(hists, nhist);
for(int i=0; i<answer.rows; i++)
{
imageResponse.push_back(pair<int, float>(i, answer.at<float>(i)));
}
std::sort(imageResponse.begin(), imageResponse.end(), irComparer2);
return imageResponse;
}
float getClassMatch(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, IplImage* &img2, int dictionarySize, string sFileName, CvSVM &svm) {
float response;
vector<KeyPoint> keypoint2;
Mat bowDescriptor2;
Mat evalData(0, dictionarySize, CV_32FC1);
Mat groundTruth(0, 1, CV_32FC1);
Mat results(0, 1, CV_32FC1);
detector.detect(img2, keypoint2);
bowDE.compute(img2, keypoint2, bowDescriptor2);
//evalData.push_back(bowDescriptor2);
//groundTruth.push_back((float) classID);
response = svm.predict(bowDescriptor2);
//results.push_back(response);
return response;
}
Mat getHistograms(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, int dictionarySize, vector<string> &collectionFilenames, string evalDir) {
// setup variable and object I need
IplImage *img2;
Mat trainingData(0, dictionarySize, CV_32FC1);
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
vector<string> files = vector<string>();
helper.GetFileList(evalDir, files);
cout << "Number of Collection Files to Process: " << files.size()-2 << endl;
for (unsigned int iz = 0; iz < files.size(); iz++) {
int isImage = helper.instr(files[iz], "jpg", 0, true);
if (isImage > 0) {
cout << " Processing " << files[iz] << endl;
collectionFilenames.push_back(files[iz]);
string sFileName = EVAL_DIR;
sFileName.append(files[iz]);
const char * imageName = sFileName.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
trainingData.push_back(bowDescriptor1);
}
}
}
return trainingData;
}
void TestShotRetrieval(string categoryName, int database_type, int num_retrieve)
{
string resultFile = "Data/Result_" + categoryName + "_ShotRetrieval.txt";
ofstream out(resultFile);
//Load database
vector<Mat> trainData, trainLabels;
vector<float> videoFPS;
vector<string> _listDataName = SetupModel(categoryName, trainData, trainLabels, videoFPS, database_type);
//Load dictionary for BOW model
if (database_type == 1)
bowDE.setVocabulary(LoadBOWDictionaryFromFile("Data/Training_Data/" + categoryName + "_BOW_dictionary.xml"));
//Get list of test frames
string pathTest = "Data/Test_images/" + categoryName + "/";
vector<string> _listTestClass = ReadFileList(pathTest);
vector<vector<float>> _listWeightScore;
for (int index = 0; index < _listTestClass.size(); index++)
{
vector<float> avgWeigthScore;
avgWeigthScore.resize(num_retrieve);
string pathClass = pathTest + _listTestClass[index] + "/";
vector<string> _listTestImage = ReadFileList(pathClass);
for (int i = 0; i < _listTestImage.size(); i++)
{
string testpath = pathClass + _listTestImage[i];
Mat testImage = imread(testpath);
//---Extract feature based on database_type
Mat queryFeature;
if (database_type == 1)
queryFeature = ExtractBOWFeature(bowDE, detector, testImage);
else
queryFeature = ExtractMPEGFeature(testImage);
//For each test image, retrieve list of shot and check if there's any correct shot in candidate list
int trueShotID = IdentifyShotFromKeyFrame(_listTestImage[i]);
cout << "Retrieve shot in video " << GetName(_listDataName[index]) << " using " << _listTestImage[i] << endl;
vector<int> _listShotID = RetrieveShot(videoFPS[index], trainData[index], trainLabels[index], queryFeature);
//Calculate precision and recall
for (int numRetrieve = 0; numRetrieve < num_retrieve; numRetrieve++)
{
float weightScore = ShotRetrievalPerformance(_listShotID, trueShotID, numRetrieve+1);
avgWeigthScore[numRetrieve] += weightScore;
}
cout << endl;
}
for (int i = 0; i < num_retrieve; i++)
{
avgWeigthScore[i] /= _listTestImage.size();
}
_listWeightScore.push_back(avgWeigthScore);
}
vector<float> avgWeightScorePerNumRetrieval;
avgWeightScorePerNumRetrieval.resize(num_retrieve);
for (int i = 0; i < _listWeightScore.size(); i++)
{
float avgPrecision = 0.0f;
out << _listTestClass[i].c_str() << endl;
for (int j = 0; j < num_retrieve; j++)
{
out << j+1 << ". " << _listWeightScore[i][j] << endl;
avgWeightScorePerNumRetrieval[j] += _listWeightScore[i][j];
}
out << endl;
}
out << endl << "Average weight score: " << endl;
for (int i = 0; i < num_retrieve; i++)
{
avgWeightScorePerNumRetrieval[i] /= (float)_listWeightScore.size();
out << i + 1 << ". " << avgWeightScorePerNumRetrieval[i] << endl;
}
out.close();
}
void TestVideoRetrieval(string categoryName, int database_type)
{
//Load database
vector<Mat> trainData, trainLabels;
vector<float> videoFPS;
vector<string> _listDataName = SetupModel(categoryName, trainData, trainLabels, videoFPS, database_type);
//Load dictionary for BOW model
if (database_type == 1)
bowDE.setVocabulary(LoadBOWDictionaryFromFile("Data/Training_Data/" + categoryName + "_BOW_dictionary.xml"));
float avgReciprocalRank = 0.0f;
float avgPrecision = 0.0f;
float avgRecall = 0.0f;
string pathTest = "Data/Test_images/" + categoryName + "/";
vector<string> _listTestClass = ReadFileList(pathTest);
int countTotal = 0;
for (int index = 0; index < _listTestClass.size(); index++)
{
float avgClassRR = 0.0f;
string pathClass = pathTest + _listTestClass[index] + "/";
vector<string> _listTestImage = ReadFileList(pathClass);
for (int i = 0; i < _listTestImage.size(); i++)
{
string testpath = pathClass + _listTestImage[i];
Mat testImage = imread(testpath);
//---Extract feature based on database_type
Mat queryFeature;
if (database_type == 1)
queryFeature = ExtractBOWFeature(bowDE, detector, testImage);
else
queryFeature = ExtractMPEGFeature(testImage);
//For each test image, retrieve list of video and check if there's any correct video in candidate list
int countMatch = 0;
int countMatchRank = 0;
int countRetrievedMatch = 0;
int numMatchRetrieve = 3;
vector<int> predictLabels = RetrieveVideo(_listTestClass.size(), queryFeature, trainData);
//Calculate precision and recall from retrieved list
if (predictLabels.size() > 0)
{
cout << _listTestImage[i] << " in video " << _listTestClass[index] << " can belong to: " << endl;
for (int j = 0; j < predictLabels.size(); j++)
{
int predictLabel = predictLabels[j];
cout << _listTestClass[predictLabel] << endl;
countMatchRank++;
if (predictLabel == index)
{
if (j < numMatchRetrieve)
{
countRetrievedMatch++;
}
countMatch++;
break;
}
}
cout << endl;
}
else
{
cout << "No video match the query image" << endl << endl;
}
float reciprocalrank = 0.0f;
float precision = 0.0f;
float recall = 0.0f;
if (predictLabels.size() > 0)
{
reciprocalrank = (float)countMatch / (float)countMatchRank;
precision = (float)countRetrievedMatch / (float)numMatchRetrieve;
recall = countRetrievedMatch;
}
avgClassRR += reciprocalrank;
avgPrecision += precision;
avgRecall += recall;
}
countTotal += _listTestImage.size();
avgReciprocalRank += avgClassRR;
}
avgReciprocalRank = avgReciprocalRank / countTotal * 100.0f;
avgPrecision = avgPrecision / countTotal * 100.0f;
avgRecall = avgRecall / countTotal * 100.0f;
cout << "MAP = " << avgReciprocalRank << endl;
cout << "Average Precision = " << avgPrecision << endl;
cout << "Average Recall = " << avgRecall << endl;
}
int main(int argc, char** argv)
{
initModule_nonfree();
FileStorage fs;
if (!fs.open(DATABASE_FILENAME, FileStorage::READ))
{
/// Local Features Detection
cout << "Detecting all Local Features" << endl;
detecLocalFeatures();
cout << "Done\n" << endl;
/// Gather Features
cout << "Gathering all Local Features" << endl;
gatherLocalFeatures();
cout << "Done\n" << endl;
/// Cluster Features
cout << "Clustering Features" << endl;
clusterFeatures();
cout << "Done\n" << endl;
/// Write Vocabulary
cout << "Saving Vocabulary" << endl;
saveVocabulary();
cout << "Done\n" << endl;
}
else
{
/// Load Vocabulary
cout << "Loading Vocabulary" << endl;
loadVocabulary();
cout << "Done\n" << endl;
}
if (!fs.open(INDEXES_FILENAME, FileStorage::READ))
{
/// Compute Descriptors
cout << "Computing Descriptors" << endl;
computeDescriptors();
cout << "Done\n" << endl;
/// Save Descriptors
cout << "Saving Descriptors" << endl;
saveDescriptors();
cout << "Done\n" << endl;
/// Compute Occurrences
cout << "Computing Occurrences" << endl;
computeOccurrences();
cout << "Done\n" << endl;
/// Compute Weights
cout << "Computing Weights" << endl;
computeWeights();
cout << "Done\n" << endl;
/// Save Indexes
cout << "Saving Indexes" << endl;
saveIndexes();
cout << "Done\n" << endl;
}
else
{
/// Load Descriptors
cout << "Loading Descriptors" << endl;
loadDescriptors();
cout << "Done\n" << endl;
/// Load Indexes
cout << "Loading Indexes" << endl;
loadIndexes();
cout << "Done\n" << endl;
}
if (!fs.open(CLASSIFIERS_FILENAME, FileStorage::READ))
{
/// Train Classifier
cout << "Training Classifier" << endl;
trainClassifier();
cout << "Done\n" << endl;
}
else
{
/// Load Classifier
cout << "Loading Classifier" << endl;
loadClassifier();
cout << "Done\n" << endl;
}
if (!existFile(CONFUSION_MATRIX_FILENAME))
{
/// Create Confusion Matrix
cout << "Creating Confusion Matrix" << endl;
createConfusionMatrix();
saveConfusionMatrix();
cout << "Done\n" << endl;
}
else loadConfusionMatrix();
/// Print Confusion Matrix
showConfusionMatrix();
cout << endl;
/// Create Confusion Table
cout << "Creating Confusion Table" << endl;
createConfusionTable();
cout << "Done\n" << endl;
//showTablesOfConfusion();
/// Create Average Table of Confusion
cout << "Creating Average Table of Confusion" << endl;
createAverageTableOfConfusion();
cout << "Done\n" << endl;
showAverageTableOfConfusion();
showStatistics();
cout << "\n\n\nFinding Category" << endl;
Mat img;
openImage(TEST_IMG_FILENAME, img);
vector<KeyPoint> keypoints;
detector->detect(img, keypoints);
Mat descriptor;
bowExtractor.compute(img, keypoints, descriptor);
int catIndex = (int) svm.predict(descriptor);
cout << "Image Category: " << categories[catIndex] << endl;
cout << "Done\n" << endl;
cout << "Calculating Nearest Neighbors" << endl;
vector<int> neighbors = nearestNeighbors(descriptor);
neighbors = nearestNeighborsCat(neighbors, catIndex);
imshow("Original", img);
for (int i = 0; i < NUMBER_OF_FILES_RETRIVED; i++)
{
String windowName = "Nearest Neighbor ";
windowName.append(to_string(i));
imshow(windowName, getImage(neighbors[i]));
}
cout << "Done\n" << endl;
waitKey(0);
return 0;
}
void clusterFeatures()
{
Mat dictionary = bowTrainer.cluster();
bowExtractor.setVocabulary(dictionary);
}
void saveVocabulary()
{
FileStorage fs(DATABASE_FILENAME, FileStorage::WRITE);
fs << "Vocabulary" << bowExtractor.getVocabulary();
fs.release();
}
