// Rotates the box by the angle given by rotation. // If the blob is a diacritic, then only small rotations for skew // correction can be applied. void BLOBNBOX::rotate_box(FCOORD rotation) { if (IsDiacritic()) { ASSERT_HOST(rotation.x() >= kCosSmallAngle) ICOORD top_pt((box.left() + box.right()) / 2, base_char_top_); ICOORD bottom_pt(top_pt.x(), base_char_bottom_); top_pt.rotate(rotation); base_char_top_ = top_pt.y(); bottom_pt.rotate(rotation); base_char_bottom_ = bottom_pt.y(); box.rotate(rotation); } else { box.rotate(rotation); set_diacritic_box(box); } }
// (Re)Fit a line to the stored points. Returns false if the line // is degenerate. Althougth the TabVector code mostly doesn't care about the // direction of lines, XAtY would give silly results for a horizontal line. // The class is mostly aimed at use for vertical lines representing // horizontal tab stops. bool TabVector::Fit(ICOORD vertical, bool force_parallel) { needs_refit_ = false; if (boxes_.empty()) { // Don't refit something with no boxes, as that only happens // in Evaluate, and we don't want to end up with a zero vector. if (!force_parallel) return false; // If we are forcing parallel, then we just need to set the sort_key_. ICOORD midpt = startpt_; midpt += endpt_; midpt /= 2; sort_key_ = SortKey(vertical, midpt.x(), midpt.y()); return startpt_.y() != endpt_.y(); } if (!force_parallel && !IsRagged()) { // Use a fitted line as the vertical. DetLineFit linepoints; BLOBNBOX_C_IT it(&boxes_); // Fit a line to all the boxes in the list. for (it.mark_cycle_pt(); !it.cycled_list(); it.forward()) { BLOBNBOX* bbox = it.data(); TBOX box = bbox->bounding_box(); int x1 = IsRightTab() ? box.right() : box.left(); ICOORD boxpt(x1, box.bottom()); linepoints.Add(boxpt); if (it.at_last()) { ICOORD top_pt(x1, box.top()); linepoints.Add(top_pt); } } linepoints.Fit(&startpt_, &endpt_); if (startpt_.y() != endpt_.y()) { vertical = endpt_; vertical -= startpt_; } } int start_y = startpt_.y(); int end_y = endpt_.y(); sort_key_ = IsLeftTab() ? MAX_INT32 : -MAX_INT32; BLOBNBOX_C_IT it(&boxes_); // Choose a line parallel to the vertical such that all boxes are on the // correct side of it. mean_width_ = 0; int width_count = 0; for (it.mark_cycle_pt(); !it.cycled_list(); it.forward()) { BLOBNBOX* bbox = it.data(); TBOX box = bbox->bounding_box(); mean_width_ += box.width(); ++width_count; int x1 = IsRightTab() ? box.right() : box.left(); // Test both the bottom and the top, as one will be more extreme, depending // on the direction of skew. int bottom_y = box.bottom(); int top_y = box.top(); int key = SortKey(vertical, x1, bottom_y); if (IsLeftTab() == (key < sort_key_)) { sort_key_ = key; startpt_ = ICOORD(x1, bottom_y); } key = SortKey(vertical, x1, top_y); if (IsLeftTab() == (key < sort_key_)) { sort_key_ = key; startpt_ = ICOORD(x1, top_y); } if (it.at_first()) start_y = bottom_y; if (it.at_last()) end_y = top_y; } if (width_count > 0) { mean_width_ = (mean_width_ + width_count - 1) / width_count; } endpt_ = startpt_ + vertical; needs_evaluation_ = true; if (start_y != end_y) { // Set the ends of the vector to fully include the first and last blobs. startpt_.set_x(XAtY(vertical, sort_key_, start_y)); startpt_.set_y(start_y); endpt_.set_x(XAtY(vertical, sort_key_, end_y)); endpt_.set_y(end_y); return true; } return false; }
// extract patches from database images bool ObjPatchMatcher::PreparePatchDB(DatasetName db_name) { patch_meta.objects.clear(); cout<<patch_meta.objects.max_size()<<endl; DataManagerInterface* db_man = NULL; if(db_name == DB_NYU2_RGBD) db_man = new NYUDepth2DataMan; if(db_name == DB_SALIENCY_RGBD) db_man = new RGBDECCV14; srand(time(0)); FileInfos imgfns, dmapfns; db_man->GetImageList(imgfns); random_shuffle(imgfns.begin(), imgfns.end()); imgfns.erase(imgfns.begin()+30, imgfns.end()); db_man->GetDepthmapList(imgfns, dmapfns); map<string, vector<VisualObject>> gt_masks; db_man->LoadGTMasks(imgfns, gt_masks); int gt_obj_cnt = 0; gt_obj_masks.clear(); for(size_t i=0; i<imgfns.size(); i++) { // color Mat cimg = imread(imgfns[i].filepath); Size newsz(400, 400); //tools::ToolFactory::compute_downsample_ratio(Size(cimg.cols, cimg.rows), 400, newsz); resize(cimg, cimg, newsz); // depth Mat dmap_float; db_man->LoadDepthData(dmapfns[i].filepath, dmap_float); resize(dmap_float, dmap_float, newsz); dmap_float.convertTo(dmap_float, CV_32F, 1000); Mat cmp_mask; compare(dmap_float, 800, cmp_mask, CMP_LT); dmap_float.setTo(800, cmp_mask); compare(dmap_float, 7000, cmp_mask, CMP_GT); dmap_float.setTo(7000, cmp_mask); dmap_float = (dmap_float-800)/(7000-800); // get label image Mat lable_mask = Mat::zeros(newsz.height, newsz.width, CV_8U); Mat label_id_mask = Mat::zeros(newsz.height, newsz.width, CV_32S)-1; vector<VisualObject>& gt_objs = gt_masks[imgfns[i].filename]; for(auto& cur_gt : gt_objs) { //if( !valid_cls[cur_gt.category_id] ) continue; resize(cur_gt.visual_data.mask, cur_gt.visual_data.mask, newsz); label_id_mask.setTo(gt_obj_cnt++, cur_gt.visual_data.mask); gt_obj_masks.push_back(cur_gt.visual_data.mask); lable_mask.setTo(1, cur_gt.visual_data.mask); } imshow("color", cimg); ImgVisualizer::DrawFloatImg("depth", dmap_float); imshow("label", lable_mask*255); waitKey(10); // do edge detection to locate boundary point quickly Mat gray_img, edge_map, gray_img_float; cvtColor(cimg, gray_img, CV_BGR2GRAY); gray_img.convertTo(gray_img_float, CV_32F, 1.f/255); Canny(gray_img, edge_map, 10, 50); Mat grad_x, grad_y, grad_mag; Sobel(gray_img_float, grad_x, CV_32F, 1, 0); Sobel(gray_img_float, grad_y, CV_32F, 0, 1); magnitude(grad_x, grad_y, grad_mag); Mat pts3d, normal_map; if(use_depth) { Feature3D feat3d; feat3d.ComputeKinect3DMap(dmap_float, pts3d, false); feat3d.ComputeNormalMap(pts3d, normal_map); } // selected patch indicator, avoid very close duplicate patches Mat picked_patch_mask = Mat::zeros(lable_mask.rows, lable_mask.cols, CV_8U); // extract patches for(int r=patch_size.height/2; r<edge_map.rows-patch_size.height/2; r+=3) { for(int c=patch_size.width/2; c<edge_map.cols-patch_size.width/2; c+=3) { if( edge_map.at<uchar>(r,c) == 0 ) continue; // only use perfect boundary points Point center_pt(c, r), left_pt(c-1, r), right_pt(c+1, r), top_pt(c, r-1), bottom_pt(c, r+1); if(lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(left_pt) && lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(right_pt) && lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(top_pt) && lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(bottom_pt) ); //continue; // set picked picked_patch_mask.at<uchar>(center_pt) = 1; VisualObject cur_patch; cur_patch.meta_data.img_path = imgfns[i].filepath; Rect box(c-patch_size.width/2, r-patch_size.height/2, patch_size.width, patch_size.height); cur_patch.visual_data.bbox = box; // find which object is dominant map<int, int> obj_label_cnt; int max_num = 0; cur_patch.meta_data.category_id = -1; for(int rr=box.y; rr<box.br().y; rr++) for(int cc=box.x; cc<box.br().x; cc++) { int cur_id = label_id_mask.at<int>(rr,cc); if(cur_id != -1) { obj_label_cnt[cur_id]++; if(obj_label_cnt[cur_id] > max_num) { max_num = obj_label_cnt[cur_id]; cur_patch.meta_data.category_id= cur_id; } } } /*vector<ImgWin> boxes; boxes.push_back(box); ImgVisualizer::DrawWinsOnImg("", cimg, boxes); if(cur_patch.category_id != -1) { imshow("mask", gt_obj_masks[cur_patch.category_id]*255); waitKey(0); }*/ lable_mask(box).convertTo(cur_patch.visual_data.mask, CV_32F); // extract feature vector gray_img_float(box).copyTo( cur_patch.visual_data.custom_feats["gray"] ); //grad_mag(box).copyTo( cur_patch.visual_desc.extra_features["gradient"] ); if(use_depth) { normal_map(box).copyTo( cur_patch.visual_data.custom_feats["normal"] ); dmap_float(box).copyTo( cur_patch.visual_data.custom_feats["depth"] ); /*ImgVisualizer::DrawFloatImg("depthmask", cur_patch.visual_desc.extra_features["depth"]); cout<<"new box"<<endl; waitKey(0);*/ } Mat feat; ComputePatchFeat(cur_patch.visual_data.custom_feats, feat); patch_data.push_back(feat); patch_meta.objects.push_back(cur_patch); } //} //} } cout<<"finished image: "<<i<<"/"<<imgfns.size()<<endl; cout<<"db size: "<<patch_data.rows<<endl; } cout<<"total patch number: "<<patch_data.rows<<endl; if(use_code) { // compress to codes LSHCoder lsh_coder; if( !lsh_coder.LearnOptimalCodes(patch_data, 64, patch_keys) ) { return false; } } return true; }