// Rotates the box by the angle given by rotation.
// If the blob is a diacritic, then only small rotations for skew
// correction can be applied.
void BLOBNBOX::rotate_box(FCOORD rotation) {
if (IsDiacritic()) {
ASSERT_HOST(rotation.x() >= kCosSmallAngle)
ICOORD top_pt((box.left() + box.right()) / 2, base_char_top_);
ICOORD bottom_pt(top_pt.x(), base_char_bottom_);
top_pt.rotate(rotation);
base_char_top_ = top_pt.y();
bottom_pt.rotate(rotation);
base_char_bottom_ = bottom_pt.y();
box.rotate(rotation);
} else {
box.rotate(rotation);
set_diacritic_box(box);
}
}
// (Re)Fit a line to the stored points. Returns false if the line
// is degenerate. Althougth the TabVector code mostly doesn't care about the
// direction of lines, XAtY would give silly results for a horizontal line.
// The class is mostly aimed at use for vertical lines representing
// horizontal tab stops.
bool TabVector::Fit(ICOORD vertical, bool force_parallel) {
needs_refit_ = false;
if (boxes_.empty()) {
// Don't refit something with no boxes, as that only happens
// in Evaluate, and we don't want to end up with a zero vector.
if (!force_parallel)
return false;
// If we are forcing parallel, then we just need to set the sort_key_.
ICOORD midpt = startpt_;
midpt += endpt_;
midpt /= 2;
sort_key_ = SortKey(vertical, midpt.x(), midpt.y());
return startpt_.y() != endpt_.y();
}
if (!force_parallel && !IsRagged()) {
// Use a fitted line as the vertical.
DetLineFit linepoints;
BLOBNBOX_C_IT it(&boxes_);
// Fit a line to all the boxes in the list.
for (it.mark_cycle_pt(); !it.cycled_list(); it.forward()) {
BLOBNBOX* bbox = it.data();
TBOX box = bbox->bounding_box();
int x1 = IsRightTab() ? box.right() : box.left();
ICOORD boxpt(x1, box.bottom());
linepoints.Add(boxpt);
if (it.at_last()) {
ICOORD top_pt(x1, box.top());
linepoints.Add(top_pt);
}
}
linepoints.Fit(&startpt_, &endpt_);
if (startpt_.y() != endpt_.y()) {
vertical = endpt_;
vertical -= startpt_;
}
}
int start_y = startpt_.y();
int end_y = endpt_.y();
sort_key_ = IsLeftTab() ? MAX_INT32 : -MAX_INT32;
BLOBNBOX_C_IT it(&boxes_);
// Choose a line parallel to the vertical such that all boxes are on the
// correct side of it.
mean_width_ = 0;
int width_count = 0;
for (it.mark_cycle_pt(); !it.cycled_list(); it.forward()) {
BLOBNBOX* bbox = it.data();
TBOX box = bbox->bounding_box();
mean_width_ += box.width();
++width_count;
int x1 = IsRightTab() ? box.right() : box.left();
// Test both the bottom and the top, as one will be more extreme, depending
// on the direction of skew.
int bottom_y = box.bottom();
int top_y = box.top();
int key = SortKey(vertical, x1, bottom_y);
if (IsLeftTab() == (key < sort_key_)) {
sort_key_ = key;
startpt_ = ICOORD(x1, bottom_y);
}
key = SortKey(vertical, x1, top_y);
if (IsLeftTab() == (key < sort_key_)) {
sort_key_ = key;
startpt_ = ICOORD(x1, top_y);
}
if (it.at_first())
start_y = bottom_y;
if (it.at_last())
end_y = top_y;
}
if (width_count > 0) {
mean_width_ = (mean_width_ + width_count - 1) / width_count;
}
endpt_ = startpt_ + vertical;
needs_evaluation_ = true;
if (start_y != end_y) {
// Set the ends of the vector to fully include the first and last blobs.
startpt_.set_x(XAtY(vertical, sort_key_, start_y));
startpt_.set_y(start_y);
endpt_.set_x(XAtY(vertical, sort_key_, end_y));
endpt_.set_y(end_y);
return true;
}
return false;
}
// extract patches from database images
bool ObjPatchMatcher::PreparePatchDB(DatasetName db_name) {
patch_meta.objects.clear();
cout<<patch_meta.objects.max_size()<<endl;
DataManagerInterface* db_man = NULL;
if(db_name == DB_NYU2_RGBD)
db_man = new NYUDepth2DataMan;
if(db_name == DB_SALIENCY_RGBD)
db_man = new RGBDECCV14;
srand(time(0));
FileInfos imgfns, dmapfns;
db_man->GetImageList(imgfns);
random_shuffle(imgfns.begin(), imgfns.end());
imgfns.erase(imgfns.begin()+30, imgfns.end());
db_man->GetDepthmapList(imgfns, dmapfns);
map<string, vector<VisualObject>> gt_masks;
db_man->LoadGTMasks(imgfns, gt_masks);
int gt_obj_cnt = 0;
gt_obj_masks.clear();
for(size_t i=0; i<imgfns.size(); i++) {
// color
Mat cimg = imread(imgfns[i].filepath);
Size newsz(400, 400);
//tools::ToolFactory::compute_downsample_ratio(Size(cimg.cols, cimg.rows), 400, newsz);
resize(cimg, cimg, newsz);
// depth
Mat dmap_float;
db_man->LoadDepthData(dmapfns[i].filepath, dmap_float);
resize(dmap_float, dmap_float, newsz);
dmap_float.convertTo(dmap_float, CV_32F, 1000);
Mat cmp_mask;
compare(dmap_float, 800, cmp_mask, CMP_LT);
dmap_float.setTo(800, cmp_mask);
compare(dmap_float, 7000, cmp_mask, CMP_GT);
dmap_float.setTo(7000, cmp_mask);
dmap_float = (dmap_float-800)/(7000-800);
// get label image
Mat lable_mask = Mat::zeros(newsz.height, newsz.width, CV_8U);
Mat label_id_mask = Mat::zeros(newsz.height, newsz.width, CV_32S)-1;
vector<VisualObject>& gt_objs = gt_masks[imgfns[i].filename];
for(auto& cur_gt : gt_objs) {
//if( !valid_cls[cur_gt.category_id] ) continue;
resize(cur_gt.visual_data.mask, cur_gt.visual_data.mask, newsz);
label_id_mask.setTo(gt_obj_cnt++, cur_gt.visual_data.mask);
gt_obj_masks.push_back(cur_gt.visual_data.mask);
lable_mask.setTo(1, cur_gt.visual_data.mask);
}
imshow("color", cimg);
ImgVisualizer::DrawFloatImg("depth", dmap_float);
imshow("label", lable_mask*255);
waitKey(10);
// do edge detection to locate boundary point quickly
Mat gray_img, edge_map, gray_img_float;
cvtColor(cimg, gray_img, CV_BGR2GRAY);
gray_img.convertTo(gray_img_float, CV_32F, 1.f/255);
Canny(gray_img, edge_map, 10, 50);
Mat grad_x, grad_y, grad_mag;
Sobel(gray_img_float, grad_x, CV_32F, 1, 0);
Sobel(gray_img_float, grad_y, CV_32F, 0, 1);
magnitude(grad_x, grad_y, grad_mag);
Mat pts3d, normal_map;
if(use_depth) {
Feature3D feat3d;
feat3d.ComputeKinect3DMap(dmap_float, pts3d, false);
feat3d.ComputeNormalMap(pts3d, normal_map);
}
// selected patch indicator, avoid very close duplicate patches
Mat picked_patch_mask = Mat::zeros(lable_mask.rows, lable_mask.cols, CV_8U);
// extract patches
for(int r=patch_size.height/2; r<edge_map.rows-patch_size.height/2; r+=3) {
for(int c=patch_size.width/2; c<edge_map.cols-patch_size.width/2; c+=3) {
if( edge_map.at<uchar>(r,c) == 0 ) continue;
// only use perfect boundary points
Point center_pt(c, r), left_pt(c-1, r), right_pt(c+1, r), top_pt(c, r-1), bottom_pt(c, r+1);
if(lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(left_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(right_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(top_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(bottom_pt) );
//continue;
// set picked
picked_patch_mask.at<uchar>(center_pt) = 1;
VisualObject cur_patch;
cur_patch.meta_data.img_path = imgfns[i].filepath;
Rect box(c-patch_size.width/2, r-patch_size.height/2, patch_size.width, patch_size.height);
cur_patch.visual_data.bbox = box;
// find which object is dominant
map<int, int> obj_label_cnt;
int max_num = 0;
cur_patch.meta_data.category_id = -1;
for(int rr=box.y; rr<box.br().y; rr++) for(int cc=box.x; cc<box.br().x; cc++) {
int cur_id = label_id_mask.at<int>(rr,cc);
if(cur_id != -1)
{
obj_label_cnt[cur_id]++;
if(obj_label_cnt[cur_id] > max_num)
{
max_num = obj_label_cnt[cur_id];
cur_patch.meta_data.category_id= cur_id;
}
}
}
/*vector<ImgWin> boxes; boxes.push_back(box);
ImgVisualizer::DrawWinsOnImg("", cimg, boxes);
if(cur_patch.category_id != -1) {
imshow("mask", gt_obj_masks[cur_patch.category_id]*255);
waitKey(0);
}*/
lable_mask(box).convertTo(cur_patch.visual_data.mask, CV_32F);
// extract feature vector
gray_img_float(box).copyTo( cur_patch.visual_data.custom_feats["gray"] );
//grad_mag(box).copyTo( cur_patch.visual_desc.extra_features["gradient"] );
if(use_depth) {
normal_map(box).copyTo( cur_patch.visual_data.custom_feats["normal"] );
dmap_float(box).copyTo( cur_patch.visual_data.custom_feats["depth"] );
/*ImgVisualizer::DrawFloatImg("depthmask", cur_patch.visual_desc.extra_features["depth"]);
cout<<"new box"<<endl;
waitKey(0);*/
}
Mat feat;
ComputePatchFeat(cur_patch.visual_data.custom_feats, feat);
patch_data.push_back(feat);
patch_meta.objects.push_back(cur_patch);
}
//}
//}
}
cout<<"finished image: "<<i<<"/"<<imgfns.size()<<endl;
cout<<"db size: "<<patch_data.rows<<endl;
}
cout<<"total patch number: "<<patch_data.rows<<endl;
if(use_code) {
// compress to codes
LSHCoder lsh_coder;
if( !lsh_coder.LearnOptimalCodes(patch_data, 64, patch_keys) ) {
return false;
}
}
return true;
}
