void MosseFilter::addTraining(cv::Mat image, cv::Point p) {
cv::Mat g = createPointTarget(p,size_,sigma_);
cv::Mat F = preprocessImage(image);
cv::Mat G = applyFFT(g);
if(filter_.empty())
filter_ = addComplexPlane(cv::Mat::zeros(size_,cv::DataType<float>::type));
cv::Mat conjF = F;
//conjF[1] = -conjF[1];
for(int x = 0; x < conjF.cols; x++)
for(int y = 0; y < conjF.rows; y++)
conjF.at<float[2]>(y,x)[1] = -conjF.at<float[2]>(y,x)[1];
N_ += G.mul(conjF);
D_ += F.mul(conjF) + 100;
//N_ += mulC(G,conjF);
//D_ += mulC(F,conjF) + 100;
//filter_ = N_/D_;
filter_ = divC(N_,D_);
}
ColorFilter::ColorFilter(Mat image, Mat characterMask, Config* config)
{
timespec startTime;
getTimeMonotonic(&startTime);
this->config = config;
this->debug = config->debugColorFiler;
this->grayscale = imageIsGrayscale(image);
if (this->debug)
cout << "ColorFilter: isGrayscale = " << grayscale << endl;
this->hsv = Mat(image.size(), image.type());
cvtColor( image, this->hsv, CV_BGR2HSV );
preprocessImage();
this->charMask = characterMask;
this->colorMask = Mat(image.size(), CV_8U);
findCharColors();
if (config->debugTiming)
{
timespec endTime;
getTimeMonotonic(&endTime);
cout << " -- ColorFilter Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
}
BBox * ColorTracker::track(cv::Mat & img, double x1, double y1, double x2, double y2)
{
double width = x2-x1;
double height = y2-y1;
im1_old = im1.clone();
im2_old = im2.clone();
im3_old = im3.clone();
preprocessImage(img);
//MS with scale estimation
double scale = 1;
int iter = 0;
cv::Point modeCenter = histMeanShiftIsotropicScale(x1, y1, x2, y2, &scale, &iter);
width = 0.7*width + 0.3*width*scale;
height = 0.7*height + 0.3*height*scale;
//MS with anisotropic scale estimation
// double scale = 1.;
// double new_width = 1., new_height = 1.;
// cv::Point modeCenter = histMeanShiftAnisotropicScale(x1, y1, x2, y2, &new_width, &new_height);
// width = new_width;
// height = new_height;
//Forward-Backward validation
if (std::abs(std::log(scale)) > 0.05){
cv::Mat tmp_im1 = im1;
cv::Mat tmp_im2 = im2;
cv::Mat tmp_im3 = im3;
im1 = im1_old;
im2 = im2_old;
im3 = im3_old;
double scaleB = 1;
histMeanShiftIsotropicScale(modeCenter.x - width/2, modeCenter.y - height/2, modeCenter.x + width/2, modeCenter.y + height/2, &scaleB);
im1 = tmp_im1;
im2 = tmp_im2;
im3 = tmp_im3;
if (std::abs(std::log(scale*scaleB)) > 0.1){
double alfa = 0.1*(defaultWidth/(float)(x2-x1));
width = (0.9 - alfa)*(x2-x1) + 0.1*(x2-x1)*scale + alfa*defaultWidth;
height = (0.9 - alfa)*(y2-y1) + 0.1*(y2-y1)*scale + alfa*defaultHeight;
}
}
BBox * retBB = new BBox();
retBB->setBBox(modeCenter.x - width/2, modeCenter.y - height/2, width, height, 1, 1);
lastPosition.setBBox(modeCenter.x - width/2, modeCenter.y - height/2, width, height, 1, 1);
frame++;
return retBB;
}
//--------------------------------------------------------------
void testApp::draw() {
if (mov.isFrameNew()) {
//img = ofImage(mov.getPixelsRef());
loadFrameToImage();
preprocessImage();
}
scaled.draw(150,0);
}
void testApp::guiEvent(ofxUIEventArgs &e) {
if (e.widget->getName() == "FRAME") {
ofxUISlider *slider = (ofxUISlider *) e.widget;
int frame = (int)slider->getScaledValue();
mov.setFrame(frame);
}
else if (e.widget->getName() == "SCALE_AMT") {
ofxUISlider *slider = (ofxUISlider *) e.widget;
scale = slider->getScaledValue();
preprocessImage();
}
else if (e.widget->getName() == "BLUR_AMT") {
ofxUISlider *slider = (ofxUISlider *) e.widget;
//todo: int?
medianSize = (int)slider->getScaledValue();
cout << medianSize;
preprocessImage();
}
else if (e.widget->getName() == "RUN_OCR") {
runOcr();
}
}
std::vector<Rectangle> RectangleDetector::processImage(cv::Mat input)
{
// Variable Initializations
input.copyTo(image);
preprocessImage();
findRectangles();
if (! foundRectangle) return std::vector<Rectangle>();
populateRectangles();
if (! foundRectangle) return std::vector<Rectangle>();
filterUniqueRectangles();
removeSimilarRectangles();
if (! foundRectangle) return std::vector<Rectangle>();
removeMarkedRectangles();
std::cout << "Returning output Rectangles..." << std::endl;
return outputRectangles;
}
cv::Mat MosseFilter::correlate(cv::Mat image) {
cv::Mat preprocessed = preprocessImage(image);
//cv::Mat G = filter_.mul(preprocessed); // Doesn't works :(
/*
cv::Mat G = addComplexPlane(cv::Mat(filter_.rows,filter_.cols,cv::DataType<float>::type));
for(int x = 0; x < filter_.cols; x++)
for(int y = 0; y < filter_.rows; y++) {
float x1 = filter_.at<float[2]>(y,x)[0];
float x2 = preprocessed.at<float[2]>(y,x)[0];
float y1 = filter_.at<float[2]>(y,x)[1];
float y2 = preprocessed.at<float[2]>(y,x)[1];
G.at<float[2]>(y,x)[0] = x1*x2-y1*y2;
G.at<float[2]>(y,x)[1] = x1*y2+y1*x2;
}
*/
cv::Mat G = mulC(filter_,preprocessed);
cv::Mat g;
cv::idft(G,g);
cv::Mat splitted[2];
cv::split(g,splitted);
double min, max;
cv::Point minp, maxp;
cv::minMaxLoc(splitted[0],&min,&max,&minp,&maxp);
cv::Mat res(splitted[0].rows,splitted[0].cols,cv::DataType<unsigned char>::type);
for(int x = 0; x < splitted[0].cols; x++)
for(int y = 0; y < splitted[0].rows; y++)
res.at<unsigned char>(y,x) = (abs(min) + splitted[0].at<float>(y,x))*255/(max+abs(min));
cv::circle(res,maxp,20,cv::Scalar(255,255,255));
return res;
//return splitted[0];
}
void ColorTracker::init(cv::Mat & img, int x1, int y1, int x2, int y2)
{
im1 = cv::Mat( img.rows, img.cols, CV_8UC1 );
im2 = cv::Mat( img.rows, img.cols, CV_8UC1 );
im3 = cv::Mat( img.rows, img.cols, CV_8UC1 );
//boundary checks
y1 = std::max(0, y1);
y2 = std::min(img.rows-1, y2);
x1 = std::max(0, x1);
x2 = std::min(img.cols-1, x2);
preprocessImage(img);
extractForegroundHistogram(x1, y1, x2, y2, q_hist);
q_orig_hist = q_hist;
extractBackgroundHistogram(x1, y1, x2, y2, b_hist);
Histogram b_weights = b_hist;
b_weights.transformToWeights();
q_hist.multiplyByWeights(&b_weights);
lastPosition.setBBox(x1, y1, x2-x1, y2-y1, 1, 1);
defaultWidth = x2-x1;
defaultHeight = y2-y1;
sumIter = 0;
frame = 0;
double w2 = (x2-x1)/2.;
double h2 = (y2-y1)/2.;
double cx = x1 + w2;
double cy = y1 + h2;
double wh = w2+5.;
double hh = h2+5.;
double Sbg = 0, Sfg = 0;
for(int i = y1; i < y2+1; i++) {
const uchar *Mi1 = im1.ptr<uchar>(i);
const uchar *Mi2 = im2.ptr<uchar>(i);
const uchar *Mi3 = im3.ptr<uchar>(i);
double tmp_y = std::pow((cy - i) / hh, 2);
for (int j = x1; j < x1 + 1; j++) {
double arg = std::pow((cx - j) / wh, 2) + tmp_y;
if (arg > 1)
continue;
//likelihood weights
double wqi = 1.0;
double wbi = sqrt(b_hist.getValue(Mi1[j], Mi2[j], Mi3[j]) /
q_orig_hist.getValue(Mi1[j], Mi2[j], Mi3[j]));
Sbg += (wqi < wbi) ? q_orig_hist.getValue(Mi1[j], Mi2[j], Mi3[j])
: 0.0;
Sfg += q_orig_hist.getValue(Mi1[j], Mi2[j], Mi3[j]);
}
}
//wAvgBg = 0.5
wAvgBg = std::max(0.1, std::min(Sbg/Sfg, 0.5));
bound1 = 0.05;
bound2 = 0.1;
}
/**
* Apply Fast Template Matching algorithm
*/
vector<Point2f> FAsTMatch::apply(Mat& original_image, Mat& original_template ) {
/* Preprocess the image and template first */
image = preprocessImage( original_image );
templ = preprocessImage( original_template );
int r1x = 0.5 * (templ.cols - 1),
r1y = 0.5 * (templ.rows - 1),
r2x = 0.5 * (image.cols - 1),
r2y = 0.5 * (image.rows - 1);
float min_trans_x = -(r2x - r1x * minScale),
max_trans_x = -min_trans_x,
min_trans_y = -(r2y - r1y * minScale),
max_trans_y = -min_trans_y,
min_rotation = -M_PI,
max_rotation = M_PI;
/* Create the matching grid / net */
MatchNet net( templ.cols, templ.rows, delta, min_trans_x, max_trans_x, min_trans_y, max_trans_y,
min_rotation, max_rotation, minScale, maxScale );
/* Smooth our images */
GaussianBlur( templ, templ, Size(0, 0), 2.0, 2.0 );
GaussianBlur( image, image, Size(0, 0), 2.0, 2.0 );
int no_of_points = round( 10 / (epsilon * epsilon) );
/* Randomly sample points */
Mat xs( 1, no_of_points, CV_32SC1 ),
ys( 1, no_of_points, CV_32SC1 );
rng.fill( xs, RNG::UNIFORM, 1, templ.cols );
rng.fill( ys, RNG::UNIFORM, 1, templ.rows );
int level = 0;
float delta_fact = 1.511f;
float new_delta = delta;
MatchConfig best_config;
Mat best_trans;
vector<double> best_distances(20, 0.0);
double best_distance;
vector<double> distances;
vector<bool> insiders;
while( true ) {
level++;
/* First create configurations based on our net */
vector<MatchConfig> configs = createListOfConfigs( net, templ.size(), image.size() );
int configs_count = static_cast<int>(configs.size());
/* Convert the configurations into affine matrices */
vector<Mat> affines = configsToAffine( configs, insiders );
/* Filter out configurations that fall outside of the boundaries */
/* the internal logic of configsToAffine has more information */
vector<MatchConfig> temp_configs;
for( int i = 0; i < insiders.size(); i++ )
if( insiders[i] == true )
temp_configs.push_back( configs[i] );
configs = temp_configs;
/* For the configs, calculate the scores / distances */
distances = evaluateConfigs( image, templ, affines, xs, ys, photometricInvariance );
/* Find the minimum distance */
auto min_itr = min_element( distances.begin(), distances.end() );
int min_index = static_cast<int>(min_itr - distances.begin());
best_distance = distances[min_index];
best_distances[level] = best_distance;
best_config = configs[min_index];
best_trans = best_config.getAffineMatrix();
/* Conditions to exit the loop */
if( (best_distance < 0.005) || ((level > 2) && (best_distance < 0.015)) || level >= 20 )
break;
if( level > 3 ) {
float mean_value = std::accumulate( best_distances.begin() + level - 3, best_distances.begin() + level - 1, 0 ) * 1.0 / distances.size();
if( best_distance > mean_value * 0.97 )
break;
}
float thresh;
bool too_high_percentage;
/* Get the good configurations that falls between certain thresholds */
vector<MatchConfig> good_configs = getGoodConfigsByDistance( configs, best_distance, new_delta, distances, thresh, too_high_percentage );
if ((too_high_percentage && (best_distance > 0.05) && ((level==1) && (configs_count < 7.5e6)) ) ||
((best_distance > 0.1) && ((level==1) && (configs_count < 5e6)) ) ) {
static float factor = 0.9;
new_delta = new_delta * factor;
level = 0;
net = net * factor;
configs = createListOfConfigs( net, templ.size(), image.size() );
}
else {
new_delta = new_delta / delta_fact;
vector<MatchConfig> expanded_configs = randomExpandConfigs( good_configs, net, level, 80, delta_fact );
configs.clear();
configs.insert( configs.end(), good_configs.begin(), good_configs.end() );
configs.insert( configs.end(), expanded_configs.begin(), expanded_configs.end() );
}
/* Randomly sample points again */
rng.fill( xs, RNG::UNIFORM, 1, templ.cols );
rng.fill( ys, RNG::UNIFORM, 1, templ.rows );
}
/* Return the rectangle corners based on the best affine transformation */
return calcCorners( image.size(), templ.size(), best_trans );
}
void testApp::runOcr() {
loadFrameToImage();
preprocessImage();
ocrResult = tess.findHOCRText(scaled);
cout << ocrResult;
}
bool load_fddb(std::vector< std::unique_ptr<NeuralNet::Image> >& images,
size_t num_image, bool uses_first)
{
const size_t patch_size = 32;
size_t loaded_image = 0;
std::ifstream ellipse_list;
if (uses_first)
{
ellipse_list.open("data-fddb/ellipse-1to5.txt");
}
else
{
ellipse_list.open("data-fddb/ellipse-6to10.txt");
}
while (loaded_image < num_image)
{
std::string file_line;
if (!std::getline(ellipse_list, file_line))
{
std::cout << "end-of-file of ellipse file" << std::endl;
return false;
}
file_line.insert(file_line.size(), ".ppm");
file_line.insert(0, "data-fddb/");
auto img_ptr = NeuralNet::loadPPMImage(file_line.c_str());
if (!img_ptr)
{
std::cout << "missing file " << file_line << std::endl;
return false;
}
std::string line_str;
if (!std::getline(ellipse_list, line_str))
{
std::cout << "end-of-file of ellipse file" << std::endl;
return false;
}
std::istringstream iss(line_str);
size_t face_count;
iss >> face_count;
size_t actual_count = 0;
for (size_t i = 0; i < face_count && actual_count+loaded_image < num_image; i++)
{
if (!std::getline(ellipse_list, line_str))
{
std::cout << "end-of-file of ellipse file" << std::endl;
return false;
}
std::istringstream iss2(line_str);
float majsize, minsize, tilt, centx, centy;
iss2 >> majsize >> minsize >> tilt >> centx >> centy;
if (majsize < patch_size)
continue;
auto crop_ptr = NeuralNet::cropImage(img_ptr,
centx - majsize/2,
centy - minsize/2,
majsize,
majsize);
auto fit_ptr = NeuralNet::fitImageTo(crop_ptr, patch_size, patch_size);
images.push_back(preprocessImage(fit_ptr));
actual_count++;
}
loaded_image += actual_count;
}
return true;
}
int main(int argc, char* argv[])
{
const size_t imageCount = 28;
const size_t test_lfw = 1000;
const size_t test_fddb = 1000;
const size_t test_nonface = 1000;
pm_init(argv[0], 0);
std::cout << "initiating network..." << std::endl;
Json::Value networkValue;
// load neural net setting
if (!load_test(networkValue))
{
return 0;
}
NeuralNet::Network network(networkValue);
std::cout << "loading settings finished" << std::endl;
// train only if the argument is given to this program
bool do_train = false;
if (argc >= 2 && !strncmp(argv[1], "train", 5))
{
do_train = true;
}
// loading images for evaluation
std::vector< std::unique_ptr<NeuralNet::Image> > imageList;
for (size_t i = 1; i <= imageCount; i++)
{
std::ostringstream oss;
oss << "eval-image/" << i << ".bmp";
imageList.push_back(preprocessImage(
NeuralNet::loadBitmapImage(oss.str().c_str())));
}
auto originalImgData = getRawDataFromImages(imageList);
imageList.clear();
std::cout << "loading evaluation images (original) finished" << std::endl;
std::vector< std::unique_ptr<NeuralNet::Image> > fddbTestImages;
if (!load_fddb(fddbTestImages, test_fddb, false))
{
std::cout << "error loading FDDB test images" << std::endl;
return false;
}
auto fddbTestData = getRawDataFromImages(fddbTestImages);
fddbTestImages.clear();
std::cout << "loading evaluation images (FDDB) finished" << std::endl;
// loading LFW images for evaluation
std::vector< std::unique_ptr<NeuralNet::Image> > lfwTestImages;
std::ifstream lfw_name_file("image-lfw/peopleDevTest.txt");
for (size_t i = 1; i <= test_lfw; i++)
{
std::string person_line;
if (!std::getline(lfw_name_file, person_line))
{
std::cout << "end-of-file of LFW names" << std::endl;
return 0;
}
std::istringstream iss(person_line);
std::string person_name;
std::getline(iss, person_name, '\t');
std::string file_name("image-lfw/");
file_name.append(person_name);
file_name.append("/");
file_name.append(person_name);
file_name.append("_0001.ppm");
lfwTestImages.push_back(preprocessImage(
NeuralNet::loadPPMImage(file_name.c_str())));
}
auto lfwTestData = getRawDataFromImages(lfwTestImages);
lfwTestImages.clear();
std::cout << "loading evaluation images (LFW) finished" << std::endl;
std::vector< std::unique_ptr<NeuralNet::Image> > nonFaceImages;
if (!load_nonface_patch(nonFaceImages, test_nonface, 1))
{
std::cout << "error loading evaluation non-face images" << std::endl;
return 0;
}
auto nonFaceTestData = getRawDataFromImages(nonFaceImages);
nonFaceImages.clear();
std::cout << "loading evaluation images (non-face) finished" << std::endl;
// put test sets into the network
std::vector< std::vector<int> > clist1;
clist1.emplace_back(test_lfw, 0);
network.registerTestSet("LFW", std::move(lfwTestData), std::move(clist1));
std::vector< std::vector<int> > clist2;
clist2.emplace_back(test_nonface, 1);
network.registerTestSet("non-face", std::move(nonFaceTestData),
std::move(clist2));
std::vector< std::vector<int> > clist3;
clist3.emplace_back(test_fddb, 0);
network.registerTestSet("FDDB", std::move(fddbTestData), std::move(clist3));
std::vector< std::vector<int> > clist4;
std::vector<int> clist4_temp;
for (size_t i = 0; i < 14; i++)
clist4_temp.push_back(0);
for (size_t i = 0; i < 14; i++)
clist4_temp.push_back(1);
clist4.push_back(clist4_temp);
network.registerTestSet("original", std::move(originalImgData),
std::move(clist4));
std::chrono::time_point<std::chrono::system_clock> time_point_start, time_point_finish;
if (do_train)
{
std::vector<float> data;
std::vector< std::vector<int> > category;
category.push_back(std::vector<int>());
// load face images
if (!load_faces(data, category))
{
return 0;
}
std::cout << "loading face images (for training) finished" << std::endl;
// load non-face images
if (!load_non_faces(data, category))
{
return 0;
}
std::cout << "loading non-face images (for training) finished" << std::endl;
std::cout << "training begin, timer start" << std::endl;
time_point_start = std::chrono::system_clock::now();
// actual training goes here
network.train(data, category);
time_point_finish = std::chrono::system_clock::now();
std::cout << "timer finished" << std::endl;
}
else
{
network.loadFromFiles();
std::cout << "network weight/bias load complete, timer start" << std::endl;
time_point_start = std::chrono::system_clock::now();
network.evaluateTestSets();
time_point_finish = std::chrono::system_clock::now();
std::cout << "timer finished" << std::endl;
}
std::chrono::duration<double> elapsed_seconds = time_point_finish - time_point_start;
std::cout << "elapsed time: " << elapsed_seconds.count() << "s" << std::endl;
}
bool load_nonface_patch(std::vector< std::unique_ptr<NeuralNet::Image> >& images,
size_t num_image, size_t train_set)
{
const size_t set_size = 2000;
const size_t patch_size = 32;
const size_t sample_per_img = 10;
const size_t max_sample_per_img = 1000;
const float var_thresh = 0.0007;
size_t lbound = set_size * train_set + 1;
size_t ubound = lbound + set_size;
std::vector<size_t> idxes;
for (size_t i = lbound; i < ubound; i++)
idxes.push_back(i);
std::random_device rd;
std::mt19937 rgen(rd());
std::shuffle(idxes.begin(), idxes.end(), rgen);
size_t num_failed_imgs = 0;
size_t num_accept_imgs = 0;
size_t img_count = 0;
size_t loaded_patch = 0;
while (loaded_patch < num_image && img_count < idxes.size())
{
std::ostringstream oss;
oss << "image-nonface/img_" << idxes[img_count] << ".bmp";
auto img_ptr = NeuralNet::loadBitmapImage(oss.str().c_str());
if (!img_ptr)
{
std::cout << "missing file " << oss.str() << std::endl;
return false;
}
std::uniform_int_distribution<size_t> dis_w(0, img_ptr->getWidth() - patch_size);
std::uniform_int_distribution<size_t> dis_h(0, img_ptr->getHeight() - patch_size);
size_t added_patch = 0;
for (size_t i=0; i<max_sample_per_img && added_patch < sample_per_img
&& added_patch + loaded_patch < num_image; i++)
{
auto cropImage = NeuralNet::cropImage(
img_ptr, dis_w(rgen), dis_h(rgen), patch_size, patch_size);
auto grayImage = NeuralNet::grayscaleImage(cropImage);
if (NeuralNet::getVariance(grayImage) <= var_thresh)
{
num_failed_imgs++;
}
else
{
num_accept_imgs++;
}
if (NeuralNet::getVariance(grayImage) <= var_thresh)
continue;
images.push_back(preprocessImage(cropImage));
added_patch++;
}
loaded_patch += added_patch;
img_count++;
}
std::cout << "accept=" << num_accept_imgs << " reject="
<< num_failed_imgs << std::endl;
if (loaded_patch == num_image)
return true;
return false;
}
