void computeDescriptors()
{
for (unsigned int i = 0; i < categories.size(); i++)
{
vector<float> tempDescriptors;
string categoryName = categories[i];
vector<string> fileNames = getFileNames(categoryName);
int fileNumber = fileNames.size();
if (debug)
fileNumber = NUMBER_OF_FILES_CAT;
for (int j = 0; j < fileNumber; j++)
{
Mat image, descriptors;
vector<KeyPoint> keypoints;
openImage(fileNames[j], image);
detector->detect(image, keypoints);
bowExtractor.compute(image, keypoints, descriptors);
for (int k = 0; k < descriptors.cols; k++)
tempDescriptors.push_back(descriptors.at<float>(0, k));
descriptorsVector.push_back(tempDescriptors);
tempDescriptors.clear();
}
}
}
void createConfusionMatrix()
{
for (unsigned int i = 0; i < categories.size(); i++)
{
vector<string> catFileNames = getTestFileNames(i);
vector<int> categoryLine;
for (unsigned int j = 0; j < categories.size(); j++)
categoryLine.push_back(0);
for (unsigned int j = 0; j < catFileNames.size(); j++)
{
Mat img, descriptor;
openImage(catFileNames[j], img);
vector<KeyPoint> keypoints;
detector->detect(img, keypoints);
bowExtractor.compute(img, keypoints, descriptor);
int catIndex = (int)svm.predict(descriptor);
categoryLine[catIndex]++;
}
for (unsigned int j = 0; j < categories.size(); j++)
cout << categoryLine[j] << ",";
cout << endl;
confusionMatrix.push_back(categoryLine);
}
saveConfusionMatrix();
}
int test(Mat img2)
{
//Obtains the result for a particular image. 0 for ten, 1 for twenty, 2 for fifty, 3 for hundred, 4 for five-hundred and 5 for thousand
// <0 when unsure (There are some other conditions for -1/-2 but just put all <0 as unsure)
Mat img;
cvtColor(img2,img,CV_BGR2GRAY);
Mat mask=grabcut_seg(img2);
if(img.cols>640)
{
resize(img,img,Size(640,img.rows*(640.0/img.cols)));
}
vector<KeyPoint> keypoints;
detector.detect(img,keypoints);
remove_keypoints(keypoints,mask);
//cout<<keypoints.size()<<" keypoints after masking"<<endl;
if(keypoints.size()<20)
{
return -1;
}
Mat descriptors;
vector<vector<int> > pointIdxsOfClusters;
bowDE.compute(img,keypoints,descriptors,&pointIdxsOfClusters);
int dot_result=sort_by_dot_product(descriptors);
re_rank_geo(keypoints,pointIdxsOfClusters);
//cout<<"After spatial re-ranking"<<endl;
int res=get_top_geoscore();
int result=labels.at<uchar>(res,0);
for(int i=0;i<10;i++)
{
res=get_top_geoscore();
result=labels.at<uchar>(res,0);
}
int response=max_decision();
for(int i=0;i<6;i++)
{
decision[i]=0;
}
return response;
}
/*
dataTraining dunction is a function that will allow us to store both character features and their corresponding float value for the ascii chracter they represent. As an example, 'A' is stored as 65.
The function takes a vector of images, a reference to a Mat array of labels which will hold the float value of ascii characters, a reference to a Mat array of trainingData, SurfFeatureDetector detector and a reference to BOWImgDescriptorExtractor and also the character we are training for.
We compute the keypoints of a character and their features (bowDescriptor) and store these features in a Mat array called trainingData. We also store the corresponding label for the features as the float value of the ascii character we are seeing.
The function will access and modify two Mat arrays holding features and the matching labels.
*/
void dataTraining (vector<Mat> images, Mat &labels, Mat &trainingData, SiftFeatureDetector detector, BOWImgDescriptorExtractor &bowDE, char character) {
vector<KeyPoint> keypoints;
Mat bowDescriptor;
cout<<images.size()<<" images"<<endl;
bool verify = false;
for (int j=0; j<images.size(); j++) {
Mat image = images.at(j);
detector.detect(image, keypoints);
bowDE.compute(image, keypoints, bowDescriptor);
cout<<keypoints.size()<<" "<<bowDescriptor.size()<<endl;
if (bowDescriptor.size()!=Size(0,0)) {
verify=true;
trainingData.push_back(bowDescriptor);
labels.push_back((float) character);
}
}
cout<<labels.rows<<" "<<trainingData.rows<<endl<<endl;
}
vector<Mat> getHistAndLabels(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, int dictionarySize) {
// setup variable and object I need
IplImage *img2;
Mat labels(0, 1, CV_32FC1);
Mat trainingData(0, dictionarySize, CV_32FC1);
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
vector<string> files = vector<string>();
helper.GetFileList(EVAL_DIR, files);
float labelVal;
for (unsigned int iz = 0; iz < files.size(); iz++) {
int isImage = helper.instr(files[iz], "jpg", 0, true);
if (isImage > 0) {
string sFileName = TRAINING_DIR;
sFileName.append(files[iz]);
const char * imageName = sFileName.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
trainingData.push_back(bowDescriptor1);
labelVal = iz+1;
labels.push_back(labelVal);
}
}
}
vector<Mat> retVec;
retVec.push_back(trainingData);
retVec.push_back(labels);
return retVec;
}
vector<pair<int, float> > paintingSearch(const Mat& QI,const SparseMat& hists, BOWImgDescriptorExtractor bow,
vector<KeyPoint>& kp,vector<vector<int> >& hist)
{
ORB detect;
vector<pair<int, float> > imageResponse;
detect(QI, Mat(), kp);
Mat imgDescriptor;
bow.compute(QI, kp, imgDescriptor, &hist);
Mat nhist = normalise(hist);
Mat answer = mul(hists, nhist);
for(int i=0; i<answer.rows; i++)
{
imageResponse.push_back(pair<int, float>(i, answer.at<float>(i)));
}
std::sort(imageResponse.begin(), imageResponse.end(), irComparer2);
return imageResponse;
}
float getClassMatch(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, IplImage* &img2, int dictionarySize, string sFileName, CvSVM &svm) {
float response;
vector<KeyPoint> keypoint2;
Mat bowDescriptor2;
Mat evalData(0, dictionarySize, CV_32FC1);
Mat groundTruth(0, 1, CV_32FC1);
Mat results(0, 1, CV_32FC1);
detector.detect(img2, keypoint2);
bowDE.compute(img2, keypoint2, bowDescriptor2);
//evalData.push_back(bowDescriptor2);
//groundTruth.push_back((float) classID);
response = svm.predict(bowDescriptor2);
//results.push_back(response);
return response;
}
Mat getSingleImageHistogram(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, string evalFile) {
// setup variable and object I need
IplImage *img2;
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
int isImage = helper.instr(evalFile, "jpg", 0, true);
if (isImage > 0) {
const char * imageName = evalFile.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
}
}
return bowDescriptor1;
}
Mat getHistograms(SurfFeatureDetector &detector, BOWImgDescriptorExtractor &bowDE, int dictionarySize, vector<string> &collectionFilenames, string evalDir) {
// setup variable and object I need
IplImage *img2;
Mat trainingData(0, dictionarySize, CV_32FC1);
vector<KeyPoint> keypoint1;
Mat bowDescriptor1;
Helper helper;
vector<string> files = vector<string>();
helper.GetFileList(evalDir, files);
cout << "Number of Collection Files to Process: " << files.size()-2 << endl;
for (unsigned int iz = 0; iz < files.size(); iz++) {
int isImage = helper.instr(files[iz], "jpg", 0, true);
if (isImage > 0) {
cout << " Processing " << files[iz] << endl;
collectionFilenames.push_back(files[iz]);
string sFileName = EVAL_DIR;
sFileName.append(files[iz]);
const char * imageName = sFileName.c_str ();
img2 = cvLoadImage(imageName,0);
if (img2) {
detector.detect(img2, keypoint1);
bowDE.compute(img2, keypoint1, bowDescriptor1);
trainingData.push_back(bowDescriptor1);
}
}
}
return trainingData;
}
int main(int argc, char** argv)
{
initModule_nonfree();
FileStorage fs;
if (!fs.open(DATABASE_FILENAME, FileStorage::READ))
{
/// Local Features Detection
cout << "Detecting all Local Features" << endl;
detecLocalFeatures();
cout << "Done\n" << endl;
/// Gather Features
cout << "Gathering all Local Features" << endl;
gatherLocalFeatures();
cout << "Done\n" << endl;
/// Cluster Features
cout << "Clustering Features" << endl;
clusterFeatures();
cout << "Done\n" << endl;
/// Write Vocabulary
cout << "Saving Vocabulary" << endl;
saveVocabulary();
cout << "Done\n" << endl;
}
else
{
/// Load Vocabulary
cout << "Loading Vocabulary" << endl;
loadVocabulary();
cout << "Done\n" << endl;
}
if (!fs.open(INDEXES_FILENAME, FileStorage::READ))
{
/// Compute Descriptors
cout << "Computing Descriptors" << endl;
computeDescriptors();
cout << "Done\n" << endl;
/// Save Descriptors
cout << "Saving Descriptors" << endl;
saveDescriptors();
cout << "Done\n" << endl;
/// Compute Occurrences
cout << "Computing Occurrences" << endl;
computeOccurrences();
cout << "Done\n" << endl;
/// Compute Weights
cout << "Computing Weights" << endl;
computeWeights();
cout << "Done\n" << endl;
/// Save Indexes
cout << "Saving Indexes" << endl;
saveIndexes();
cout << "Done\n" << endl;
}
else
{
/// Load Descriptors
cout << "Loading Descriptors" << endl;
loadDescriptors();
cout << "Done\n" << endl;
/// Load Indexes
cout << "Loading Indexes" << endl;
loadIndexes();
cout << "Done\n" << endl;
}
if (!fs.open(CLASSIFIERS_FILENAME, FileStorage::READ))
{
/// Train Classifier
cout << "Training Classifier" << endl;
trainClassifier();
cout << "Done\n" << endl;
}
else
{
/// Load Classifier
cout << "Loading Classifier" << endl;
loadClassifier();
cout << "Done\n" << endl;
}
if (!existFile(CONFUSION_MATRIX_FILENAME))
{
/// Create Confusion Matrix
cout << "Creating Confusion Matrix" << endl;
createConfusionMatrix();
saveConfusionMatrix();
cout << "Done\n" << endl;
}
else loadConfusionMatrix();
/// Print Confusion Matrix
showConfusionMatrix();
cout << endl;
/// Create Confusion Table
cout << "Creating Confusion Table" << endl;
createConfusionTable();
cout << "Done\n" << endl;
//showTablesOfConfusion();
/// Create Average Table of Confusion
cout << "Creating Average Table of Confusion" << endl;
createAverageTableOfConfusion();
cout << "Done\n" << endl;
showAverageTableOfConfusion();
showStatistics();
cout << "\n\n\nFinding Category" << endl;
Mat img;
openImage(TEST_IMG_FILENAME, img);
vector<KeyPoint> keypoints;
detector->detect(img, keypoints);
Mat descriptor;
bowExtractor.compute(img, keypoints, descriptor);
int catIndex = (int) svm.predict(descriptor);
cout << "Image Category: " << categories[catIndex] << endl;
cout << "Done\n" << endl;
cout << "Calculating Nearest Neighbors" << endl;
vector<int> neighbors = nearestNeighbors(descriptor);
neighbors = nearestNeighborsCat(neighbors, catIndex);
imshow("Original", img);
for (int i = 0; i < NUMBER_OF_FILES_RETRIVED; i++)
{
String windowName = "Nearest Neighbor ";
windowName.append(to_string(i));
imshow(windowName, getImage(neighbors[i]));
}
cout << "Done\n" << endl;
waitKey(0);
return 0;
}
