TEST(hole_filling_test, elliptical_hole_on_repeated_texture_should_give_good_result)
{
Mat img = imread("test_images/brick_pavement.jpg");
convert_for_computation(img, 0.5f);
// Add some hole
Mat hole_mask = Mat::zeros(img.size(), CV_8U);
Point center(100, 110);
Size axis(20, 5);
float angle = 20;
ellipse(hole_mask, center, axis, angle, 0, 360, Scalar(255), -1);
int patch_size = 7;
HoleFilling hf(img, hole_mask, patch_size);
// Dump image with hole as black region.
Mat img_with_hole_bgr;
cvtColor(img, img_with_hole_bgr, CV_Lab2BGR);
img_with_hole_bgr.setTo(Scalar(0,0,0), hole_mask);
imwrite("brick_pavement_hole.exr", img_with_hole_bgr);
// Dump reconstructed image
Mat filled = hf.run();
cvtColor(filled, filled, CV_Lab2BGR);
imwrite("brick_pavement_hole_filled.exr", filled);
// The reconstructed image should be close to the original one, in this very simple case.
Mat img_bgr;
cvtColor(img, img_bgr, CV_Lab2BGR);
double ssd = norm(img_bgr, filled, cv::NORM_L2SQR);
EXPECT_LT(ssd, 0.2);
}
TEST(hole_filling_test, rectangular_hole_on_repeated_texture_should_give_good_result)
{
Mat img = imread("test_images/brick_pavement.jpg");
convert_for_computation(img, 0.5f);
// Add some hole
Mat hole_mask = Mat::zeros(img.size(), CV_8U);
hole_mask(Rect(72, 65, 5, 20)) = 255;
int patch_size = 7;
HoleFilling hf(img, hole_mask, patch_size);
// Dump image with hole as black region.
Mat img_with_hole_bgr;
cvtColor(img, img_with_hole_bgr, CV_Lab2BGR);
img_with_hole_bgr.setTo(Scalar(0,0,0), hole_mask);
imwrite("brick_pavement_hole.exr", img_with_hole_bgr);
// Dump reconstructed image
Mat filled = hf.run();
cvtColor(filled, filled, CV_Lab2BGR);
imwrite("brick_pavement_hole_filled.exr", filled);
// The reconstructed image should be close to the original one, in this very simple case.
Mat img_bgr;
cvtColor(img, img_bgr, CV_Lab2BGR);
double ssd = norm(img_bgr, filled, cv::NORM_L2SQR);
EXPECT_LT(ssd, 0.2);
}
bool VideoStitcher::appendImage(string fileName) {
Mat image = imread(fileName, CV_LOAD_IMAGE_COLOR);
if (!openOut(image.size())) {
return false;
}
_out.write(image);
return true;
}
const Mat FileListImageSource::getImage() const
{
if (index < 0 || index >= static_cast<int>(files.size()))
return Mat();
Mat image = imread(files[index].string(), CV_LOAD_IMAGE_COLOR);
if (image.empty())
throw runtime_error("image '" + files[index].string() + "' could not be loaded");
return image;
}
int main(){
Mat img = imread("img.jpg");
Size s = img.size();
Mat gray;
cvtColor(img, gray, CV_BGR2GRAY);
Mat tmp = imread("mask.png");
Mat mask;
cvtColor(tmp,mask, CV_BGR2GRAY);
Mat grad;
int scale = 1;
int delta = 0;
int ddepth = CV_16S;
/// Generate grad_x and grad_y
Mat grad_x, grad_y;
Mat abs_grad_x, abs_grad_y;
Sobel( gray, grad_x, ddepth, 1, 0, 3, scale, delta, BORDER_DEFAULT );
convertScaleAbs( grad_x, abs_grad_x );
Sobel( gray, grad_y, ddepth, 0, 1, 3, scale, delta, BORDER_DEFAULT );
convertScaleAbs( grad_y, abs_grad_y );
/// Total Gradient (approximate)
addWeighted( abs_grad_x, 0.5, abs_grad_y, 0.5, 0, grad );
imwrite("grad.jpg",grad);
Mat result = DTOCS(grad,mask);
imwrite("DTOCS.png",result);
Mat new_result = WDTOCS(grad,mask);
imwrite("WDTOCS.png",new_result);
}
int main(int argc, char **argv)
{
//Create a CMT object
Config config("FAST", "BRISK","RANSAC",0.5);
CMT cmt(config);
//Initialization bounding box
Rect rect;
//Parse args
int challenge_flag = 0;
int loop_flag = 0;
int verbose_flag = 0;
int bbox_flag = 0;
int skip_frames = 0;
int skip_msecs = 0;
int output_flag = 0;
string input_path;
string output_path;
const int detector_cmd = 1000;
const int descriptor_cmd = 1001;
const int bbox_cmd = 1002;
const int no_scale_cmd = 1003;
const int with_rotation_cmd = 1004;
const int skip_cmd = 1005;
const int skip_msecs_cmd = 1006;
const int output_file_cmd = 1007;
struct option longopts[] =
{
//No-argument options
{"challenge", no_argument, &challenge_flag, 1},
{"loop", no_argument, &loop_flag, 1},
{"verbose", no_argument, &verbose_flag, 1},
{"no-scale", no_argument, 0, no_scale_cmd},
{"with-rotation", no_argument, 0, with_rotation_cmd},
//Argument options
{"bbox", required_argument, 0, bbox_cmd},
{"detector", required_argument, 0, detector_cmd},
{"descriptor", required_argument, 0, descriptor_cmd},
{"output-file", required_argument, 0, output_file_cmd},
{"skip", required_argument, 0, skip_cmd},
{"skip-msecs", required_argument, 0, skip_msecs_cmd},
{0, 0, 0, 0}
};
int index = 0;
int c;
while((c = getopt_long(argc, argv, "v", longopts, &index)) != -1)
{
switch (c)
{
case 'v':
verbose_flag = true;
break;
case bbox_cmd:
{
//TODO: The following also accepts strings of the form %f,%f,%f,%fxyz...
string bbox_format = "%f,%f,%f,%f";
float x,y,w,h;
int ret = sscanf(optarg, bbox_format.c_str(), &x, &y, &w, &h);
if (ret != 4)
{
cerr << "bounding box must be given in format " << bbox_format << endl;
return 1;
}
bbox_flag = 1;
rect = Rect(x,y,w,h);
}
break;
case detector_cmd:
cmt.str_detector = optarg;
break;
case descriptor_cmd:
cmt.str_descriptor = optarg;
break;
case output_file_cmd:
output_path = optarg;
output_flag = 1;
break;
case skip_cmd:
{
int ret = sscanf(optarg, "%d", &skip_frames);
if (ret != 1)
{
skip_frames = 0;
}
}
break;
case skip_msecs_cmd:
{
int ret = sscanf(optarg, "%d", &skip_msecs);
if (ret != 1)
{
skip_msecs = 0;
}
}
break;
case no_scale_cmd:
cmt.consensus.estimate_scale = false;
break;
case with_rotation_cmd:
cmt.consensus.estimate_rotation = true;
break;
case '?':
return 1;
}
}
// Can only skip frames or milliseconds, not both.
if (skip_frames > 0 && skip_msecs > 0)
{
cerr << "You can only skip frames, or milliseconds, not both." << endl;
return 1;
}
//One argument remains
if (optind == argc - 1)
{
input_path = argv[optind];
}
else if (optind < argc - 1)
{
cerr << "Only one argument is allowed." << endl;
return 1;
}
//Set up logging
FILELog::ReportingLevel() = verbose_flag ? logDEBUG : logINFO;
Output2FILE::Stream() = stdout; //Log to stdout
//Challenge mode
if (challenge_flag)
{
//Read list of images
ifstream im_file("images.txt");
vector<string> files;
string line;
while(getline(im_file, line ))
{
files.push_back(line);
}
//Read region
ifstream region_file("region.txt");
vector<float> coords = getNextLineAndSplitIntoFloats(region_file);
if (coords.size() == 4) {
rect = Rect(coords[0], coords[1], coords[2], coords[3]);
}
else if (coords.size() == 8)
{
//Split into x and y coordinates
vector<float> xcoords;
vector<float> ycoords;
for (size_t i = 0; i < coords.size(); i++)
{
if (i % 2 == 0) xcoords.push_back(coords[i]);
else ycoords.push_back(coords[i]);
}
float xmin = *min_element(xcoords.begin(), xcoords.end());
float xmax = *max_element(xcoords.begin(), xcoords.end());
float ymin = *min_element(ycoords.begin(), ycoords.end());
float ymax = *max_element(ycoords.begin(), ycoords.end());
rect = Rect(xmin, ymin, xmax-xmin, ymax-ymin);
cout << "Found bounding box" << xmin << " " << ymin << " " << xmax-xmin << " " << ymax-ymin << endl;
}
else {
cerr << "Invalid Bounding box format" << endl;
return 0;
}
//Read first image
Mat im0 = imread(files[0]);
Mat im0_gray;
cvtColor(im0, im0_gray, CV_BGR2GRAY);
//Initialize cmt
cmt.initialize(im0_gray, rect);
//Write init region to output file
ofstream output_file("output.txt");
output_file << rect.x << ',' << rect.y << ',' << rect.width << ',' << rect.height << std::endl;
//Process images, write output to file
for (size_t i = 1; i < files.size(); i++)
{
FILE_LOG(logINFO) << "Processing frame " << i << "/" << files.size();
Mat im = imread(files[i]);
Mat im_gray;
cvtColor(im, im_gray, CV_BGR2GRAY);
cmt.processFrame(im_gray);
if (verbose_flag)
{
display(im, cmt);
}
rect = cmt.bb_rot.boundingRect();
output_file << rect.x << ',' << rect.y << ',' << rect.width << ',' << rect.height << std::endl;
}
output_file.close();
return 0;
}
//Normal mode
//Create window
namedWindow(WIN_NAME);
VideoCapture cap;
bool show_preview = true;
//If no input was specified
if (input_path.length() == 0)
{
cap.open(0); //Open default camera device
}
//Else open the video specified by input_path
else
{
cap.open(input_path);
if (skip_frames > 0)
{
cap.set(CV_CAP_PROP_POS_FRAMES, skip_frames);
}
if (skip_msecs > 0)
{
cap.set(CV_CAP_PROP_POS_MSEC, skip_msecs);
// Now which frame are we on?
skip_frames = (int) cap.get(CV_CAP_PROP_POS_FRAMES);
}
show_preview = false;
}
//If it doesn't work, stop
if(!cap.isOpened())
{
cerr << "Unable to open video capture." << endl;
return -1;
}
//Show preview until key is pressed
while (show_preview)
{
Mat preview;
cap >> preview;
screenLog(preview, "Press a key to start selecting an object.");
imshow(WIN_NAME, preview);
char k = waitKey(10);
if (k != -1) {
show_preview = false;
}
}
//Get initial image
Mat im0;
cap >> im0;
//If no bounding was specified, get it from user
if (!bbox_flag)
{
rect = getRect(im0, WIN_NAME);
}
FILE_LOG(logINFO) << "Using " << rect.x << "," << rect.y << "," << rect.width << "," << rect.height
<< " as initial bounding box.";
//Convert im0 to grayscale
Mat im0_gray;
if (im0.channels() > 1) {
cvtColor(im0, im0_gray, CV_BGR2GRAY);
} else {
im0_gray = im0;
}
//Initialize CMT
cmt.initialize(im0_gray, rect);
int frame = skip_frames;
//Open output file.
ofstream output_file;
if (output_flag)
{
int msecs = (int) cap.get(CV_CAP_PROP_POS_MSEC);
output_file.open(output_path.c_str());
output_file << OUT_FILE_COL_HEADERS << endl;
output_file << frame << "," << msecs << ",";
output_file << cmt.points_active.size() << ",";
output_file << write_rotated_rect(cmt.bb_rot) << endl;
}
//Main loop
while (true)
{
frame++;
Mat im;
//If loop flag is set, reuse initial image (for debugging purposes)
if (loop_flag) im0.copyTo(im);
else cap >> im; //Else use next image in stream
if (im.empty()) break; //Exit at end of video stream
Mat im_gray;
if (im.channels() > 1) {
cvtColor(im, im_gray, CV_BGR2GRAY);
} else {
im_gray = im;
}
//Let CMT process the frame
cmt.processFrame(im_gray);
//Output.
if (output_flag)
{
int msecs = (int) cap.get(CV_CAP_PROP_POS_MSEC);
output_file << frame << "," << msecs << ",";
output_file << cmt.points_active.size() << ",";
output_file << write_rotated_rect(cmt.bb_rot) << endl;
}
else
{
//TODO: Provide meaningful output
FILE_LOG(logINFO) << "#" << frame << " active: " << cmt.points_active.size();
}
//Display image and then quit if requested.
char key = display(im, cmt);
if(key == 'q') break;
}
//Close output file.
if (output_flag) output_file.close();
return 0;
}
int main(int argc, char **argv) {
if (argc != 4 && argc != 5) {
printf(
"\nUsage:\n"
"\t%s -cf <in.imgs.folder> <out.keys.folder> {in.startimg.id}\n\n"
"\t%s -featsel <in.query.keys.folder> <in.queries.mask.folder> <out.queries.maskedkeys.folder>\n\n"
"\t%s -visualkp <in.query.keys.folder> <in.query.imgs.folder> <out.query.withkeys.folder>\n\n"
"Options:\n"
"\t-cf: compute features\t\n"
"\t-featseal: apply mask to a set of keypoints file and write the result to a specific folder\t\n"
"\t-visualkp: visualize mask application result\t\n\n", argv[0], argv[0], argv[0]);
return EXIT_FAILURE;
}
vector<string> folderFiles;
int result = FileUtils::readFolder(argv[2], folderFiles);
if (result == EXIT_FAILURE) {
return result;
}
if (string(argv[1]).compare("-cf") == 0) {
vector<string>::iterator start_image;
if (argc == 4) {
// Set first image as start point of the loop over the folder of images
start_image = folderFiles.begin();
} else {
// Set received argument as start point of the loop over the folder of images
start_image = std::find(folderFiles.begin(), folderFiles.end(),
argv[4]);
}
for (vector<string>::iterator image = start_image;
image != folderFiles.end(); ++image) {
if ((*image).find(".jpg") != string::npos) {
printf("%s\n", (*image).c_str());
Features features = detectAndDescribeFeatures(
argv[2] + string("/") + (*image));
string descriptorFileName(argv[3]);
descriptorFileName += "/"
+ (*image).substr(0, (*image).size() - 4) + ".key";
writeFeaturesToFile(descriptorFileName, features);
}
}
} else if (string(argv[1]).compare("-featsel") == 0) {
string keypointsFolderPath(argv[2]);
string masksFolderPath(argv[3]);
string outputFolderPath(argv[4]);
vector<string> maskFiles;
Features features, selectedFeatures;
for (string filename : folderFiles) {
if (filename.find(".key") != string::npos) {
string keypointFilepath = keypointsFolderPath + "/" + filename;
readKeypoints(keypointFilepath.c_str(), features.keypoints,
features.descriptors);
StringUtils::split(filename.c_str(), '/').back();
filename.resize(filename.size() - 4);
string maskPath = masksFolderPath + "/" + filename
+ MASK_FILE_EXTENSION;
vector<Point2f> polygon = readMask(maskPath.c_str());
printf("Selecting features\n");
int count = 0;
selectedFeatures.keypoints.clear();
for (KeyPoint p : features.keypoints) {
int inCont = pointPolygonTest(polygon, p.pt, false);
if (inCont != -1) {
selectedFeatures.keypoints.push_back(p);
selectedFeatures.descriptors.push_back(
features.descriptors.row(count));
}
count++;
}
printf(" Selected [%d] features\n",
(int) selectedFeatures.keypoints.size());
string outputFeaturesPath = outputFolderPath + "/" + filename
+ KEYPOINT_FILE_EXTENSION;
writeFeaturesToFile(outputFeaturesPath, selectedFeatures);
}
}
} else if (string(argv[1]).compare("-visualkp") == 0) {
string keypointsFolderPath(argv[2]);
string imagesFolderPath(argv[3]);
string outputImagesFolderPath(argv[4]);
Features features;
for (string filename : folderFiles) {
if (filename.find(".key") != string::npos) {
string keypointFilepath = keypointsFolderPath + "/" + filename;
readKeypoints(keypointFilepath.c_str(), features.keypoints,
features.descriptors);
string imgPath = imagesFolderPath + "/"
+ StringUtils::parseImgFilename(filename);
printf("Reading image [%s]\n", imgPath.c_str());
Mat img = imread(imgPath, CV_LOAD_IMAGE_COLOR);
drawKeypoints(img, features.keypoints, img, cvScalar(255, 0, 0),
DrawMatchesFlags::DEFAULT);
string imgWithKeysPath = outputImagesFolderPath + "/"
+ StringUtils::parseImgFilename(filename, "_with_keys");
printf("Writing image [%s]\n", imgWithKeysPath.c_str());
cv::imwrite(imgWithKeysPath, img);
}
}
}
return EXIT_SUCCESS;
}
// Uses OpenCV
ImageData* loadImageData(string imgFilename) {
Mat imgMat = imread(imgFilename.c_str()); // BGR
return new ImageData(imgMat, CV_BGR2RGB);
}
const Mat FileListImageSource::getImage() const
{
if (index < 0 || index >= files.size())
return Mat();
return imread(files[index].string(), 1);
}
Params ImageRetriever::train( const string& dirname, const Params& preconf )
{
path dir = dirname;
RuntimeCheck(is_directory(dir), "Error: failed to load image directory.");
vector<string> names;
// load image names
for (auto it = directory_iterator(dir); it != directory_iterator(); ++it)
names.push_back((it->path()).string());
// shuffle the images
std::random_shuffle(names.begin(), names.end());
const bool with_vocabulary = not preconf.vocabulary.empty();
const float hessian = preconf.hessian;
const int voclen = preconf.voclen;
const int binlen = preconf.binarylen;
const int imgmaxlen = preconf.imgmaxlen;
// load images and extract local feature for training.
auto lfextractor = m_bwextractor.lfextractor();
const int maxnum = 1000 * voclen;
const int dims = lfextractor.size();
Mat descs(maxnum, dims, DataType<float>::type); //TODO add auto configuration for feature type.
int count = 0;
for (auto it = names.begin(); it != names.end(); ++it) {
std::cout << "\t" << *it << std::endl;
Mat image = imread(*it);
image = reformed(image, imgmaxlen);
auto lfs = lfextractor.compute(image);
Mat desc = std::get<1>(lfs);
int num = desc.rows;
desc.copyTo(descs.rowRange(count, count+num));
count += num;
if (count > maxnum)
break;
}
// copy the valid descriptors for training.
if (maxnum > count)
descs = descs.rowRange(0, count);
Params param = preconf;
// train the parameters and store in Param struct.
if (with_vocabulary) {
auto ret = m_bwextractor.train(descs, preconf.vocabulary, binlen);
param.projection = std::get<0>(ret);
param.thresholds = std::get<1>(ret);
}
else {
auto ret = m_bwextractor.train(descs, voclen, binlen);
param.vocabulary = std::get<0>(ret);
param.projection = std::get<1>(ret);
param.thresholds = std::get<2>(ret);
}
return param;
}
