typename array_nd<T, S>::create_new permute(
const array_nd<T, S>& a, const array<size_t, D>& sz)
{
CHECK (a.ndims() == sz.length(), eshape());
array<size_t, stack<3> > newsz(a.ndims());
for (size_t i = 0; i < a.ndims(); i++)
newsz[i] = a.size_nd()[sz[i] - 1];
return reshape(a, newsz);
}
// extract patches from database images
bool ObjPatchMatcher::PreparePatchDB(DatasetName db_name) {
patch_meta.objects.clear();
cout<<patch_meta.objects.max_size()<<endl;
DataManagerInterface* db_man = NULL;
if(db_name == DB_NYU2_RGBD)
db_man = new NYUDepth2DataMan;
if(db_name == DB_SALIENCY_RGBD)
db_man = new RGBDECCV14;
srand(time(0));
FileInfos imgfns, dmapfns;
db_man->GetImageList(imgfns);
random_shuffle(imgfns.begin(), imgfns.end());
imgfns.erase(imgfns.begin()+30, imgfns.end());
db_man->GetDepthmapList(imgfns, dmapfns);
map<string, vector<VisualObject>> gt_masks;
db_man->LoadGTMasks(imgfns, gt_masks);
int gt_obj_cnt = 0;
gt_obj_masks.clear();
for(size_t i=0; i<imgfns.size(); i++) {
// color
Mat cimg = imread(imgfns[i].filepath);
Size newsz(400, 400);
//tools::ToolFactory::compute_downsample_ratio(Size(cimg.cols, cimg.rows), 400, newsz);
resize(cimg, cimg, newsz);
// depth
Mat dmap_float;
db_man->LoadDepthData(dmapfns[i].filepath, dmap_float);
resize(dmap_float, dmap_float, newsz);
dmap_float.convertTo(dmap_float, CV_32F, 1000);
Mat cmp_mask;
compare(dmap_float, 800, cmp_mask, CMP_LT);
dmap_float.setTo(800, cmp_mask);
compare(dmap_float, 7000, cmp_mask, CMP_GT);
dmap_float.setTo(7000, cmp_mask);
dmap_float = (dmap_float-800)/(7000-800);
// get label image
Mat lable_mask = Mat::zeros(newsz.height, newsz.width, CV_8U);
Mat label_id_mask = Mat::zeros(newsz.height, newsz.width, CV_32S)-1;
vector<VisualObject>& gt_objs = gt_masks[imgfns[i].filename];
for(auto& cur_gt : gt_objs) {
//if( !valid_cls[cur_gt.category_id] ) continue;
resize(cur_gt.visual_data.mask, cur_gt.visual_data.mask, newsz);
label_id_mask.setTo(gt_obj_cnt++, cur_gt.visual_data.mask);
gt_obj_masks.push_back(cur_gt.visual_data.mask);
lable_mask.setTo(1, cur_gt.visual_data.mask);
}
imshow("color", cimg);
ImgVisualizer::DrawFloatImg("depth", dmap_float);
imshow("label", lable_mask*255);
waitKey(10);
// do edge detection to locate boundary point quickly
Mat gray_img, edge_map, gray_img_float;
cvtColor(cimg, gray_img, CV_BGR2GRAY);
gray_img.convertTo(gray_img_float, CV_32F, 1.f/255);
Canny(gray_img, edge_map, 10, 50);
Mat grad_x, grad_y, grad_mag;
Sobel(gray_img_float, grad_x, CV_32F, 1, 0);
Sobel(gray_img_float, grad_y, CV_32F, 0, 1);
magnitude(grad_x, grad_y, grad_mag);
Mat pts3d, normal_map;
if(use_depth) {
Feature3D feat3d;
feat3d.ComputeKinect3DMap(dmap_float, pts3d, false);
feat3d.ComputeNormalMap(pts3d, normal_map);
}
// selected patch indicator, avoid very close duplicate patches
Mat picked_patch_mask = Mat::zeros(lable_mask.rows, lable_mask.cols, CV_8U);
// extract patches
for(int r=patch_size.height/2; r<edge_map.rows-patch_size.height/2; r+=3) {
for(int c=patch_size.width/2; c<edge_map.cols-patch_size.width/2; c+=3) {
if( edge_map.at<uchar>(r,c) == 0 ) continue;
// only use perfect boundary points
Point center_pt(c, r), left_pt(c-1, r), right_pt(c+1, r), top_pt(c, r-1), bottom_pt(c, r+1);
if(lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(left_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(right_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(top_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(bottom_pt) );
//continue;
// set picked
picked_patch_mask.at<uchar>(center_pt) = 1;
VisualObject cur_patch;
cur_patch.meta_data.img_path = imgfns[i].filepath;
Rect box(c-patch_size.width/2, r-patch_size.height/2, patch_size.width, patch_size.height);
cur_patch.visual_data.bbox = box;
// find which object is dominant
map<int, int> obj_label_cnt;
int max_num = 0;
cur_patch.meta_data.category_id = -1;
for(int rr=box.y; rr<box.br().y; rr++) for(int cc=box.x; cc<box.br().x; cc++) {
int cur_id = label_id_mask.at<int>(rr,cc);
if(cur_id != -1)
{
obj_label_cnt[cur_id]++;
if(obj_label_cnt[cur_id] > max_num)
{
max_num = obj_label_cnt[cur_id];
cur_patch.meta_data.category_id= cur_id;
}
}
}
/*vector<ImgWin> boxes; boxes.push_back(box);
ImgVisualizer::DrawWinsOnImg("", cimg, boxes);
if(cur_patch.category_id != -1) {
imshow("mask", gt_obj_masks[cur_patch.category_id]*255);
waitKey(0);
}*/
lable_mask(box).convertTo(cur_patch.visual_data.mask, CV_32F);
// extract feature vector
gray_img_float(box).copyTo( cur_patch.visual_data.custom_feats["gray"] );
//grad_mag(box).copyTo( cur_patch.visual_desc.extra_features["gradient"] );
if(use_depth) {
normal_map(box).copyTo( cur_patch.visual_data.custom_feats["normal"] );
dmap_float(box).copyTo( cur_patch.visual_data.custom_feats["depth"] );
/*ImgVisualizer::DrawFloatImg("depthmask", cur_patch.visual_desc.extra_features["depth"]);
cout<<"new box"<<endl;
waitKey(0);*/
}
Mat feat;
ComputePatchFeat(cur_patch.visual_data.custom_feats, feat);
patch_data.push_back(feat);
patch_meta.objects.push_back(cur_patch);
}
//}
//}
}
cout<<"finished image: "<<i<<"/"<<imgfns.size()<<endl;
cout<<"db size: "<<patch_data.rows<<endl;
}
cout<<"total patch number: "<<patch_data.rows<<endl;
if(use_code) {
// compress to codes
LSHCoder lsh_coder;
if( !lsh_coder.LearnOptimalCodes(patch_data, 64, patch_keys) ) {
return false;
}
}
return true;
}
