void Tesseract::output_pass( //Tess output pass //send to api
PAGE_RES_IT &page_res_it,
const TBOX *target_word_box) {
BLOCK_RES *block_of_last_word;
inT16 block_id;
BOOL8 force_eol; //During output
BLOCK *nextblock; //block of next word
WERD *nextword; //next word
page_res_it.restart_page ();
block_of_last_word = NULL;
while (page_res_it.word () != NULL) {
check_debug_pt (page_res_it.word (), 120);
if (target_word_box)
{
TBOX current_word_box=page_res_it.word ()->word->bounding_box();
FCOORD center_pt((current_word_box.right()+current_word_box.left())/2,(current_word_box.bottom()+current_word_box.top())/2);
if (!target_word_box->contains(center_pt))
{
page_res_it.forward ();
continue;
}
}
if (tessedit_write_block_separators &&
block_of_last_word != page_res_it.block ()) {
block_of_last_word = page_res_it.block ();
block_id = block_of_last_word->block->index();
}
force_eol = (tessedit_write_block_separators &&
(page_res_it.block () != page_res_it.next_block ())) ||
(page_res_it.next_word () == NULL);
if (page_res_it.next_word () != NULL)
nextword = page_res_it.next_word ()->word;
else
nextword = NULL;
if (page_res_it.next_block () != NULL)
nextblock = page_res_it.next_block ()->block;
else
nextblock = NULL;
//regardless of tilde crunching
write_results(page_res_it,
determine_newline_type(page_res_it.word()->word,
page_res_it.block()->block,
nextword, nextblock), force_eol);
page_res_it.forward();
}
}
//Calculates matching cost and show matching results on the window
float GraphMatch::drawMatches(vector<NodeSig> ns1, vector<NodeSig> ns2,
Mat img1, Mat img2)
{
Mat assignment, cost;
//Construct [rowsxcols] cost matrix using pairwise
//distance between node signature
int rows = ns1.size();
int cols = ns2.size();
int** cost_matrix = (int**)calloc(rows,sizeof(int*));
for(int i = 0; i < rows; i++)
{
cost_matrix[i] = (int*)calloc(cols,sizeof(int));
for(int j = 0; j < cols; j++)
{
//Multiplied by constant because hungarian algorithm accept integer defined cost
//matrix. We later divide by constant for correction
cost_matrix[i][j] = GraphMatch::calcN2NDistance(ns1[i], ns2[j])*MULTIPLIER_1000;
}
}
hungarian_problem_t p;
// initialize the hungarian_problem using the cost matrix
hungarian_init(&p, cost_matrix , rows, cols, HUNGARIAN_MODE_MINIMIZE_COST);
// solve the assignment problem
hungarian_solve(&p);
//Convert results to OpenCV format
assignment = array2Mat8U(p.assignment,rows,cols);
cost = array2Mat32F(cost_matrix,rows,cols);
//Divide for correction
cost = cost / MULTIPLIER_1000;
//free variables
hungarian_free(&p);
// Calculate match score and
// show matching results given assignment and cost matrix
Mat img_merged;
float matching_cost = 0;
//Concatenate two images
hconcat(img1, img2, img_merged);
//Get non-zero indices which defines the optimal match(assignment)
vector<Point> nonzero_locs;
findNonZero(assignment,nonzero_locs);
//Draw optimal match
for(int i = 0; i < nonzero_locs.size(); i++)
{
Point p1 = ns1[nonzero_locs[i].y].center;
Point p2 = ns2[nonzero_locs[i].x].center+Point(img1.size().width,0);
circle(img_merged,p1,MATCH_CIRCLE_RADIUS,MATCH_LINE_COLOR);
circle(img_merged,p2,MATCH_CIRCLE_RADIUS,MATCH_LINE_COLOR);
line(img_merged,p1,p2,MATCH_LINE_COLOR,MATCH_LINE_WIDTH);
float dist = cost.at<float>(nonzero_locs[i].y, nonzero_locs[i].x);
matching_cost = matching_cost + dist;
//Draw cost on match image
stringstream ss;
ss << dist;
string cost_str = ss.str();
Point center_pt((p1.x + p2.x) / 2.0, (p1.y + p2.y) / 2.0);
//putText(img_merged, cost_str, center_pt, cv::FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 1);
}
//Show matching results image on the window
emit showMatchImage(mat2QImage(img_merged));
return matching_cost;
}
// extract patches from database images
bool ObjPatchMatcher::PreparePatchDB(DatasetName db_name) {
patch_meta.objects.clear();
cout<<patch_meta.objects.max_size()<<endl;
DataManagerInterface* db_man = NULL;
if(db_name == DB_NYU2_RGBD)
db_man = new NYUDepth2DataMan;
if(db_name == DB_SALIENCY_RGBD)
db_man = new RGBDECCV14;
srand(time(0));
FileInfos imgfns, dmapfns;
db_man->GetImageList(imgfns);
random_shuffle(imgfns.begin(), imgfns.end());
imgfns.erase(imgfns.begin()+30, imgfns.end());
db_man->GetDepthmapList(imgfns, dmapfns);
map<string, vector<VisualObject>> gt_masks;
db_man->LoadGTMasks(imgfns, gt_masks);
int gt_obj_cnt = 0;
gt_obj_masks.clear();
for(size_t i=0; i<imgfns.size(); i++) {
// color
Mat cimg = imread(imgfns[i].filepath);
Size newsz(400, 400);
//tools::ToolFactory::compute_downsample_ratio(Size(cimg.cols, cimg.rows), 400, newsz);
resize(cimg, cimg, newsz);
// depth
Mat dmap_float;
db_man->LoadDepthData(dmapfns[i].filepath, dmap_float);
resize(dmap_float, dmap_float, newsz);
dmap_float.convertTo(dmap_float, CV_32F, 1000);
Mat cmp_mask;
compare(dmap_float, 800, cmp_mask, CMP_LT);
dmap_float.setTo(800, cmp_mask);
compare(dmap_float, 7000, cmp_mask, CMP_GT);
dmap_float.setTo(7000, cmp_mask);
dmap_float = (dmap_float-800)/(7000-800);
// get label image
Mat lable_mask = Mat::zeros(newsz.height, newsz.width, CV_8U);
Mat label_id_mask = Mat::zeros(newsz.height, newsz.width, CV_32S)-1;
vector<VisualObject>& gt_objs = gt_masks[imgfns[i].filename];
for(auto& cur_gt : gt_objs) {
//if( !valid_cls[cur_gt.category_id] ) continue;
resize(cur_gt.visual_data.mask, cur_gt.visual_data.mask, newsz);
label_id_mask.setTo(gt_obj_cnt++, cur_gt.visual_data.mask);
gt_obj_masks.push_back(cur_gt.visual_data.mask);
lable_mask.setTo(1, cur_gt.visual_data.mask);
}
imshow("color", cimg);
ImgVisualizer::DrawFloatImg("depth", dmap_float);
imshow("label", lable_mask*255);
waitKey(10);
// do edge detection to locate boundary point quickly
Mat gray_img, edge_map, gray_img_float;
cvtColor(cimg, gray_img, CV_BGR2GRAY);
gray_img.convertTo(gray_img_float, CV_32F, 1.f/255);
Canny(gray_img, edge_map, 10, 50);
Mat grad_x, grad_y, grad_mag;
Sobel(gray_img_float, grad_x, CV_32F, 1, 0);
Sobel(gray_img_float, grad_y, CV_32F, 0, 1);
magnitude(grad_x, grad_y, grad_mag);
Mat pts3d, normal_map;
if(use_depth) {
Feature3D feat3d;
feat3d.ComputeKinect3DMap(dmap_float, pts3d, false);
feat3d.ComputeNormalMap(pts3d, normal_map);
}
// selected patch indicator, avoid very close duplicate patches
Mat picked_patch_mask = Mat::zeros(lable_mask.rows, lable_mask.cols, CV_8U);
// extract patches
for(int r=patch_size.height/2; r<edge_map.rows-patch_size.height/2; r+=3) {
for(int c=patch_size.width/2; c<edge_map.cols-patch_size.width/2; c+=3) {
if( edge_map.at<uchar>(r,c) == 0 ) continue;
// only use perfect boundary points
Point center_pt(c, r), left_pt(c-1, r), right_pt(c+1, r), top_pt(c, r-1), bottom_pt(c, r+1);
if(lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(left_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(right_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(top_pt) &&
lable_mask.at<uchar>(center_pt) == lable_mask.at<uchar>(bottom_pt) );
//continue;
// set picked
picked_patch_mask.at<uchar>(center_pt) = 1;
VisualObject cur_patch;
cur_patch.meta_data.img_path = imgfns[i].filepath;
Rect box(c-patch_size.width/2, r-patch_size.height/2, patch_size.width, patch_size.height);
cur_patch.visual_data.bbox = box;
// find which object is dominant
map<int, int> obj_label_cnt;
int max_num = 0;
cur_patch.meta_data.category_id = -1;
for(int rr=box.y; rr<box.br().y; rr++) for(int cc=box.x; cc<box.br().x; cc++) {
int cur_id = label_id_mask.at<int>(rr,cc);
if(cur_id != -1)
{
obj_label_cnt[cur_id]++;
if(obj_label_cnt[cur_id] > max_num)
{
max_num = obj_label_cnt[cur_id];
cur_patch.meta_data.category_id= cur_id;
}
}
}
/*vector<ImgWin> boxes; boxes.push_back(box);
ImgVisualizer::DrawWinsOnImg("", cimg, boxes);
if(cur_patch.category_id != -1) {
imshow("mask", gt_obj_masks[cur_patch.category_id]*255);
waitKey(0);
}*/
lable_mask(box).convertTo(cur_patch.visual_data.mask, CV_32F);
// extract feature vector
gray_img_float(box).copyTo( cur_patch.visual_data.custom_feats["gray"] );
//grad_mag(box).copyTo( cur_patch.visual_desc.extra_features["gradient"] );
if(use_depth) {
normal_map(box).copyTo( cur_patch.visual_data.custom_feats["normal"] );
dmap_float(box).copyTo( cur_patch.visual_data.custom_feats["depth"] );
/*ImgVisualizer::DrawFloatImg("depthmask", cur_patch.visual_desc.extra_features["depth"]);
cout<<"new box"<<endl;
waitKey(0);*/
}
Mat feat;
ComputePatchFeat(cur_patch.visual_data.custom_feats, feat);
patch_data.push_back(feat);
patch_meta.objects.push_back(cur_patch);
}
//}
//}
}
cout<<"finished image: "<<i<<"/"<<imgfns.size()<<endl;
cout<<"db size: "<<patch_data.rows<<endl;
}
cout<<"total patch number: "<<patch_data.rows<<endl;
if(use_code) {
// compress to codes
LSHCoder lsh_coder;
if( !lsh_coder.LearnOptimalCodes(patch_data, 64, patch_keys) ) {
return false;
}
}
return true;
}
