// represents and OR search free for AStar
SearchTree train_search_tree_agglomerative(map<string,NNTemplateType>&allTemplates)
{
SearchTree search_tree_root;
// initiailize the bottom layer
unordered_map<int, unordered_map<int,double> > costs;
list<SearchTree > clusters(allTemplates.size());
function<void (SearchTree&node)> update_dists_for_node = [&](SearchTree&node)
{
static atomic<int> completed(0);
log_once(printfpp("++computed distance %d",completed++));
for(auto & node_k : clusters)
{
double cost = node.Templ.merge_cost(node_k.Templ);
static mutex m; lock_guard<mutex> l(m);
costs[node.id][node_k.id] = cost;
costs[node_k.id][node.id] = cost;
}
};
auto iter = allTemplates.begin();
for(int cter = 0; iter != allTemplates.end(); ++cter, ++iter)
{
auto jter = clusters.begin();
std::advance(jter,cter);
SearchTree&curTree = *jter;
curTree.Templ = iter->second;
curTree.id = seq_id++;
curTree.pose_uuid = iter->first;
}
TaskBlock compute_init_dists("compute_init_dists");
log_once(printfpp("++Computing Initial Distances for %d clusters",(int)clusters.size()));
for(auto & curTree : clusters)
{
SearchTree*tp = &curTree;
compute_init_dists.add_callee([&,tp]()
{
update_dists_for_node(*tp);
});
}
compute_init_dists.execute();
log_once(printfpp("--Computing Initial Distances for %d clusters",(int)clusters.size()));
// merge layers until we get the root
merge_clusters_agglom(seq_id,clusters,costs,update_dists_for_node);
// store the root
search_tree_root = clusters.front();
// print the search tree
log_search_tree(search_tree_root);
return search_tree_root;
}
void message(int level, const char *format, ...)
{
va_list args;
if (level >= logLevel) {
va_start(args, format);
log_once(stderr, level, format, args);
va_end(args);
}
if (messageFile != NULL) {
va_start(args, format);
log_once(messageFile, level, format, args);
va_end(args);
}
#ifdef HAVE_VSYSLOG
if (_logToSyslog) {
int priority = LOG_USER;
switch(level) {
case MESS_REALDEBUG:
priority |= LOG_DEBUG;
break;
case MESS_DEBUG:
case MESS_VERBOSE:
case MESS_NORMAL:
priority |= LOG_INFO;
break;
case MESS_ERROR:
priority |= LOG_ERR;
break;
case MESS_FATAL:
priority |= LOG_CRIT;
break;
default:
priority |= LOG_INFO;
break;
};
va_start(args, format);
vsyslog(priority, format, args);
va_end(args);
}
#endif
if (level == MESS_FATAL)
exit(1);
}
Image *Resources::get_library_image(Atom library_name, int image_num) {
static std::unordered_set<Atom> library_warnings;
static std::unordered_set<std::pair<Atom, int>> library_image_warnings;
ImageLibraryResource *lib = get_image_library(library_name);
if (!lib) {
log_once(warning, library_warnings, library_name, "No image library: " << library_name);
return nullptr;
}
auto found = lib->images.find(image_num);
if (found == lib->images.end()) {
log_once(warning, library_image_warnings, std::make_pair(library_name, image_num), "No image at position " << image_num << " in library " << library_name);
return nullptr;
}
return found->second.get();
}
void log_search_tree(SearchTree&tree_root)
{
if(not g_params.option_is_set("NN_LOG_TREE"))
return;
// for calcing the branching factor
int internal_nodes = 0;
int internal_node_children = 0;
function<void (SearchTree&,string)> expandFn = [&](SearchTree&node,string prefix) -> void
{
Mat RGB = imageeq("",node.Templ.getTIm(),false,false);
string filename = string("bound") + uuid() + ".jpg";
imwrite(prefix + "/" + filename,RGB);
if(node.children.size() > 0)
{
internal_nodes++;
internal_node_children += node.children.size();
}
for(auto & child : node.children)
{
string new_dir = printfpp("%s/%d",prefix.c_str(),child->id);
assert(boost::filesystem::create_directory(new_dir));
expandFn(*child,new_dir);
}
};
// traverse the tree.
expandFn(tree_root,params::out_dir());
double mean_branching_factor = static_cast<double>(internal_node_children)/internal_nodes;
log_once(safe_printf("mean_branching_factor = %",mean_branching_factor));
}
/// SECTION: Combo with RGB and Depth
ComboComputer_RGBPlusDepth::ComboComputer_RGBPlusDepth(Size win_size, Size block_size, Size block_stride, Size cell_size):
FeatureBinCombiner(win_size, block_size, block_stride, cell_size)
{
computers.clear();
// for now, use ZHOG and ZAREA
computers.push_back(shared_ptr<DepthFeatComputer>(
new HOGComputer18p4_General(selectDepthFn,win_size,block_size,block_stride,cell_size)));
//computers.push_back(shared_ptr<DepthFeatComputer>(
//new HOGComputer_Area(win_size,block_size,block_stride,cell_size)));
computers.push_back(shared_ptr<DepthFeatComputer>(
new HOGComputer18p4_General([](const ImRGBZ&im)-> const Mat&{return im.gray();},win_size,block_size,block_stride,cell_size)));
//computers.push_back(shared_ptr<DepthFeatComputer>(
//new FaceFeature(win_size,block_size,block_stride,cell_size)));
// skin
//computers.push_back(shared_ptr<DepthFeatComputer>(
//new SkinFeatureComputer(win_size,block_size,block_stride,cell_size)));
normalize_weights();
//weights[1] *= HOA_IMPORTANCE; // double the weight of HoA
//assert(std::dynamic_pointer_cast<HOGComputer_Area>(computers[1]));
//weights[3] = 1; // don't normalize the face feature weight.
for(int iter = 0; iter < computers.size(); ++iter)
log_once(printfpp("ComboComputer_RGBPlusDepth:%d weight = %f",
iter,weights[iter]));
}
static void suppress_most_common_pixel_hack(Mat&z,Rect handBB)
{
// hack, remove the most common pixel
std::map<float,double> counts;
Rect imBB(Point(0,0),z.size());
for(int yIter = 0; yIter < z.rows; ++yIter)
for(int xIter = 0; xIter < z.cols; ++xIter)
if(!handBB.contains(Point(xIter,yIter)))
{
float zz = z.at<float>(yIter,xIter);
if(goodNumber(zz) && params::MIN_Z() <= zz && zz <= params::MAX_Z())
counts[zz] ++;
}
// find the larget count
float common_z, common_zs_count = -inf;
for(auto & pair : counts)
if(pair.second > common_zs_count)
{
common_z = pair.first;
common_zs_count = pair.second;
}
log_once(safe_printf("note: suppressing % %",common_z,common_zs_count));
// suppress!
for(int yIter = 0; yIter < z.rows; ++yIter)
for(int xIter = 0; xIter < z.cols; ++xIter)
if(!handBB.contains(Point(xIter,yIter)))
{
float&zz = z.at<float>(yIter,xIter);
if(zz == common_z)
zz = inf;
}
}
LibHandMetadata::LibHandMetadata(
string filename,
const Mat& RGB, const Mat& Z,
const Mat& segmentation, const Mat&semantics,
Rect handBB,
libhand::HandRenderer::JointPositionMap& joint_positions,
CustomCamera camera, bool read_only) :
filename(filename), RGB(RGB), Z(Z), joint_positions(joint_positions),
handBB(handBB), camera(camera), read_only(read_only)
{
// export for Greg
if(!read_only)
{
int id = getId(filename);
string save_dir = boost::filesystem::path(filename).parent_path().string();
log_once(printfpp("save_dir = %s",save_dir.c_str()));
export_annotations(printfpp(
(save_dir + "/gen%d.annotations").c_str(),id));
imwrite(printfpp((save_dir +"/gen%d.jpg").c_str(),id),RGB);
imwrite(printfpp((save_dir +"/segmentation%d.jpg").c_str(),id),segmentation);
imwrite(printfpp((save_dir +"/semantics%d.png").c_str(),id),semantics);
}
else
unlink(filename.c_str());
log_metric_bb();
}
///
/// SECTION: Pedestrian Model
///
DetectionSet StackedModel::detect_dpm2occ(const ImRGBZ& im, DetectionFilter filter) const
{
DetectionFilter part_fitler = filter;
part_fitler.nmax = numeric_limits<int>::max();
part_fitler.thresh = -inf;
DetectionSet occ_dets = occ_model->detect(im,part_fitler);
DetectionSet dpm_dets = dpm_model->detect(im,part_fitler);
DetectionSet merged_dets;
for(int iter = 0; iter < dpm_dets.size(); ++iter)
{
DetectorResult dpm_det = dpm_dets[iter];
DetectorResult new_det;
//DetectorResult occ_det = nearest_neighbour(occ_dets,dpm_det->BB);
for(DetectorResult occ_det : occ_dets)
{
double sf = std::sqrt(occ_det->BB.area()/dpm_det->BB.area());
if(
occ_det->real >= .5 &&
rectIntersect(occ_det->BB,dpm_det->BB) > BaselineModel_ExternalKITTI::BB_OL_THRESH &&
clamp<double>(.5,sf,2) == sf)
{
// update the resp
double new_resp = dpm_det->resp; //dpm_platt->prob();// * occ_platt->prob(occ_det->resp);
if(!new_det or new_det->resp < new_resp)
{
new_det.reset(new Detection(*dpm_det));
new_det->resp = new_resp;
}
}
if(new_det)
merged_dets.push_back(new_det);
}
}
log_once(printfpp("merged %d of %d dets",(int)merged_dets.size(),(int)dpm_dets.size()));
//filter.apply(merged_dets);
log_once(printfpp("filt 2 %d of %d dets",(int)merged_dets.size(),(int)dpm_dets.size()));
return merged_dets;
}
DetectionSet StackedModel::detect_occ2dpm(const ImRGBZ& im, DetectionFilter filter) const
{
DetectionFilter part_fitler = filter;
part_fitler.nmax = numeric_limits<int>::max();
part_fitler.thresh = -inf;
DetectionSet occ_dets = occ_model->detect(im,part_fitler);
DetectionSet dpm_dets = dpm_model->detect(im,part_fitler);
DetectionSet merged_dets;
for(int iter = 0; iter < occ_dets.size(); ++iter)
{
DetectorResult occ_det = occ_dets[iter];
auto occFeatFn = occ_det->feature;
DetectorResult dpm_det = nearest_neighbour(dpm_dets,occ_det->BB);
decltype(occFeatFn) dpmFeatFn;
if(rectIntersect(occ_det->BB,dpm_det->BB) > BaselineModel_ExternalKITTI::BB_OL_THRESH)
{
occ_det->resp = dpm_det->resp + learner->getB();
//occ_det->resp = ;//dpm_platt->prob(dpm_det->resp) * occ_platt->prob(occ_det->resp);
dpmFeatFn = dpm_det->feature;
}
else
{
occ_det->resp = -inf; //dot(dpm_model->getW(),BaselineModel_ExternalKITTI::minFeat()) + learner->getB();
dpmFeatFn = []() {
return vector<double> {BaselineModel_ExternalKITTI::minFeat()};
};
}
occ_det->feature = [this,occFeatFn,dpmFeatFn]()
{
map<string,SparseVector> feats{
{"dpm_model",dpmFeatFn()},
{"occ_model",occFeatFn()}};
return feat_interpreation->pack(feats);
};
merged_dets.push_back(occ_det);
}
log_once(printfpp("merged %d of %d dets",(int)merged_dets.size(),(int)occ_dets.size()));
//filter.apply(merged_dets);
return merged_dets;
}
map< string, AnnotationBoundingBox > Metadata_Simple::get_positives()
{
map<string,AnnotationBoundingBox> implicit_abbs = MetaData::get_essential_positives(HandBB_RGB);
for(auto && pair : explicit_abbs)
{
auto found = implicit_abbs.find(pair.first);
if(found == implicit_abbs.end() or static_cast<Rect_<double>>(found->second) == Rect_<double>())
{
implicit_abbs[pair.first] = pair.second;
}
else
{
implicit_abbs[pair.first] = pair.second;
log_once(safe_printf("Metadata_Simple::get_positives % overwrite for part %",filename,pair.first));
}
}
return implicit_abbs;
}
DetectionSet StackedModel::detect_dpmThenOcc(const ImRGBZ& im, DetectionFilter filter) const
{
// get the detections from the DPM
DetectionFilter part_fitler = filter;
part_fitler.nmax = numeric_limits<int>::max();
part_fitler.thresh = -inf;
DetectionSet dpm_dets = dpm_model->detect(im,part_fitler);
DetectionSet dets;
float min_world_area, max_world_area;
area_model->validRange(params::world_area_variance,true,min_world_area,max_world_area);
for(auto & dpm_det : dpm_dets)
{
AnnotationBoundingBox abb;
abb.write(dpm_det->BB);
// consider these depths...
bool could_be_real = false;
vector<float> depths = manifoldFn_prng(im,abb,50);
for(float depth : depths)
{
// check the box's depth
double bb_world_area = im.camera.worldAreaForImageArea(depth,abb);
if(bb_world_area < min_world_area || bb_world_area > max_world_area)
continue;
// check that the box is real
abb.depth = depth;
auto info = occ_model->extract(im, abb,false);
//cout << printfpp("%real = %f\n",real);
if(info.occ <= .80 && info.bg <= .5)
could_be_real = true;
}
if(could_be_real)
dets.push_back(dpm_det);
}
log_once(printfpp("areaModel filtered %d => %d",(int)dpm_dets.size(),(int)dets.size()));
return dets;
}
ComboComputer_Depth::ComboComputer_Depth(Size win_size, Size block_size, Size block_stride, Size cell_size): FeatureBinCombiner(win_size, block_size, block_stride, cell_size)
{
computers.clear();
// for now, use ZHOG and ZAREA
// Depth HOG
computers.push_back(shared_ptr<DepthFeatComputer>(
new HOGComputer18p4_General(selectDepthFn,win_size,block_size,block_stride,cell_size)));
// HoA
//computers.push_back(shared_ptr<DepthFeatComputer>(
//new HOGComputer_Area(win_size,block_size,block_stride,cell_size)));
normalize_weights();
// update the normalized weights
//weights[1] *= HOA_IMPORTANCE;
//assert(std::dynamic_pointer_cast<HOGComputer_Area>(computers[1]));
for(int iter = 0; iter < computers.size(); ++iter)
log_once(printfpp("ComboComputer_Depth:%d weight = %f",
iter,weights[iter]));
}
MetaData_YML_Backed* load_ICL(string base_path,string annotation,bool training)
{
// parse the annotation
vector<double> labels;
istringstream iss(annotation);
string filename; iss >> filename; // discard.
cout << "annotation " << annotation << endl;
cout << "filename " << filename << endl;
while(iss)
{
double v; iss >> v;
labels.push_back(v);
}
// calc the name for this metadata
string metadata_filename = base_path + filename;
// load the raw data
float active_min = training?0:params::MIN_Z();
string frame_file = base_path + filename;
Mat depth = imread(frame_file,-1);
assert(!depth.empty());
CustomCamera pxc_camera(params::H_RGB_FOV,params::V_RGB_FOV, depth.cols,depth.rows);
depth.convertTo(depth,DataType<float>::type);
depth /= 10;
for(int rIter = 0; rIter < depth.rows; ++rIter)
for(int cIter = 0; cIter < depth.cols; ++cIter)
{
float & d = depth.at<float>(rIter,cIter);
if(d <= active_min)
d = inf;
}
//depth = pointCloudToDepth(depth,pxc_camera);
if(depth.empty())
{
log_once(printfpp("ICL_Video cannot load %s",metadata_filename.c_str()));
log_once(printfpp("Couldn't open %s",frame_file.c_str()));
return nullptr;
}
//Mat RGB (depth.rows,depth.cols,DataType<Vec3b>::type,Scalar(122,150,233));
Mat RGB = imageeq("",depth,false,false);
for(int rIter = 0; rIter < RGB.rows; rIter++)
for(int cIter = 0; cIter < RGB.cols; cIter++)
RGB.at<Vec3b>(rIter,cIter)[2] = 255;
// create an image
shared_ptr<ImRGBZ> im =
make_shared<ImRGBZ>(RGB,depth,metadata_filename + "image",pxc_camera);
// create the metadata object
Metadata_Simple* metadata = new Metadata_Simple(metadata_filename+".yml",true,true,false);
metadata->setIm(*im);
Point2d hand_root(labels[0 + 0*3],labels[1 + 0*3]);
suppress_flood_fill_hack(im->Z,hand_root);
metadata->setIm(*im);
Rect handBB = set_annotations(metadata,labels,im);
suppress_most_common_pixel_hack(im->Z,handBB);
metadata->setIm(*im);
log_im_decay_freq("ICL_loaded",[&]()
{
return image_datum(*metadata);
});
return metadata;
}
