bool load_table(configuration &conf, const std::string &module_name,
idx<intg>& table, intg thickness, intg noutputs)
{
std::string name = module_name;
name << "_table"; // table name
std::string name_in = name;
name_in << "_in"; // table max input
std::string name_out = name;
name_out << "_out"; // table max output
if (conf.exists(name)) // request to load table from file
{
std::string filename = conf.get_string(name.c_str());
if (!file_exists(filename))
eblerror("cannot find table file declared in variable " << name
<< ": " << filename);
table = load_matrix<intg>(filename);
eblprint("Loaded " << name << " (" << table << ") from " << filename
<< endl);
return true;
}
else if (conf.exists(name_in) && conf.exists(name_out))
{
intg in, out;
// special case, if in == "thickness", use current thickness
std::string val_in = conf.get_string(name_in);
if (!val_in.compare("thickness"))
in = thickness;
else // regular case, use string as int
in = conf.get_int(name_in.c_str());
// special case, if out == "noutputs", use current thickness
std::string val_out = conf.get_string(name_out);
if (!val_out.compare("noutputs"))
out = noutputs;
else // regular case, use string as int
out = conf.get_int(name_out.c_str());
// create table
table = full_table(in, out);
eblprint("Using a full table for " << name << ": "
<< in << " -> " << out << " (" << table << ")" << std::endl);
return true;
}
eblwarn("Failed to load table " << name << std::endl);
return false;
}
void detection_thread<T>::execute() {
try {
bool display = false;
#ifdef __GUI__
display = conf.exists_true("display")
&& conf.exists_true("display_threads");
bool mindisplay = conf.exists_true("minimal_display");
bool save_video = conf.exists_true("save_video");
bool display_states = conf.exists_true("display_states");
uint wid = 0; // window id
uint wid_states = 0; // window id
#endif
uint display_sleep = conf.try_get_uint("display_sleep", 0);
// if (!display && save_video) {
// // we still want to output images but not show them
// display = true;
// #ifdef __GUI__
// set_gui_silent();
// #endif
// }
// load network and weights in a forward-only parameter
parameter<T> theparam;
theparam.set_forward_only();
idx<ubyte> classes(1,1);
//try { // try loading classes names but do not stop upon failure
load_matrix<ubyte>(classes, conf.get_cstring("classes"));
// } catch(std::string &err) {
// merr << "warning: " << err;
// merr << std::endl;
// }
std::vector<std::string> sclasses = ubyteidx_to_stringvector(classes);
answer_module<T> *ans = create_answer<T,T,T>(conf, classes.dim(0));
uint noutputs = ans->get_nfeatures();
intg thick = -1;
module_1_1<T> *net = create_network<T>(theparam, conf, thick, noutputs,
"arch", this->_id);
// loading weights
if (conf.exists("weights")) { // manual weights
// concatenate weights if multiple ones
std::vector<std::string> w =
string_to_stringvector(conf.get_string("weights"));
mout << "Loading weights from: " << w << std::endl;
theparam.load_x(w);
// permute weights by blocks
if (conf.exists("weights_permutation")) {
std::string sblocks = conf.get_string("weights_blocks");
std::string spermut = conf.get_string("weights_permutation");
std::vector<intg> blocks = string_to_intgvector(sblocks.c_str());
std::vector<uint> permut = string_to_uintvector(spermut.c_str());
theparam.permute_x(blocks, permut);
}
} else {
if (conf.exists_true("manual_load")) { // manual load
eblwarn("\"weights\" variable not defined, loading manually "
<< "if manual_load defined");
manually_load_network(*((layers<T>*)net), conf);
} else { // random weights
int seed = dynamic_init_drand();
eblwarn("No weights to load, randomizing weights with seed " << seed);
forget_param_linear fgp(1, 0.5, seed);
net->forget(fgp);
}
}
DEBUGMEM_PRETTY("before detection");
// detector
detector<T> detect(*net, sclasses, ans, NULL, NULL, mout, merr);
init_detector(detect, conf, outdir, silent);
// keep pointer to detector
pdetect = &detect;
bootstrapping<T> boot(conf);
// when a bbox file is given, ignore the processing, load the pre-computed
// bboxes and feed them to the nms (non-maximum suppression).
bboxes boxes(bbox_all, NULL, mout, merr);
boxes.print_saving_type(); // inform user how we save boxes
bool precomputed_boxes = false;
if (conf.exists("bbox_file")) {
precomputed_boxes = true;
std::string bbfile = conf.get_string("bbox_file");
boxes.load_eblearn(bbfile);
}
bool bmask_class = false;
if (conf.exists("mask_class"))
bmask_class = detect.set_mask_class(conf.get_cstring("mask_class"));
std::string viddir = outdir;
viddir += "video/";
mkdir_full(viddir);
// gui
#ifdef __GUI__
uint display_wmax = conf.try_get_uint("display_max_width", 3000);
T display_min = (T) conf.try_get_double("display_min", -1.7);
T display_max = (T) conf.try_get_double("display_max", 1.7);
T display_in_max = (T) conf.try_get_double("display_in_max", 255);
T display_in_min = (T) conf.try_get_double("display_in_min", 0);
float display_transp = conf.try_get_float("display_bb_transparency", 0);
uint qstep1 = conf.try_get_uint("qstep1", 0);
uint qheight1 = conf.try_get_uint("qheight1", 0);
uint qwidth1 = conf.try_get_uint("qwidth1", 0);
uint qstep2 = conf.try_get_uint("qstep2", 0);
uint qheight2 = conf.try_get_uint("qheight2", 0);
uint qwidth2 = conf.try_get_uint("qwidth2", 0);
module_1_1_gui netgui;
wid_states = display_states ? new_window("network states"):0;
night_mode();
std::string title = "EBLearn detector: ";
title += _name;
if (display) {
wid = new_window(title.c_str());
mout << "displaying in window " << wid << std::endl;
night_mode();
}
float zoom = conf.try_get_float("display_zoom", 1);
bool bbox_show_conf = !conf.exists_false("bbox_show_conf");
bool bbox_show_class = !conf.exists_false("bbox_show_class");
detector_gui<T>
dgui(conf.try_get_uint("show_extracted", 0),
conf.exists_bool("queue1"), qstep1, qheight1,
qwidth1, conf.exists_bool("queue2"), qstep2, qheight2, qwidth2,
bbox_show_class, bbox_show_conf);
if (bmask_class)
dgui.set_mask_class(conf.get_cstring("mask_class"),
(T) conf.try_get_double("mask_threshold", 0));
#endif
// timing variables
timer tpass, toverall;
long ms;
// loop
toverall.start();
// we're ready
bavailable = true;
while(!this->_stop) {
// wait until a new image is made available
while (!in_updated && !_stop) {
millisleep(1);
}
tpass.restart();
if (_stop) break ;
// we got a new frame, reset new frame flag
in_updated = false; // no need to lock mutex
// check if this frame should be skipped
if (boot.skip_frame(frame_name)) {
skip_frame();
continue ;
} else if (!frame_loaded) {
uframe = load_image<ubyte>(frame_fullname);
mout << "loaded image " << frame_fullname << std::endl;
}
if (!silent) mout << "processing " << frame_name << std::endl;
// check frame is correctly allocated, if not, allocate.
if (frame.order() != uframe.order())
frame = idx<T>(uframe.get_idxdim());
else if (frame.get_idxdim() != uframe.get_idxdim())
frame.resize(uframe.get_idxdim());
// copy frame
idx_copy(uframe, frame);
// run detector
if (!display) { // fprop without display
if (precomputed_boxes) {
try {
bboxes *bb = boxes.get_group(frame_name);
idxdim d = boxes.get_group_dims(frame_name);
d.insert_dim(0, 1);
bboxes pruned;
detect.init(d);
detect.fprop_nms(*bb, pruned);
copy_bboxes(pruned); // make a copy of bounding boxes
// resize frame so that caller knows the size of the frame
idxdim framedim = frame.get_idxdim();
if (d.dim(1) == -1 || d.dim(2) == -1)
eblerror("pre-computed boxes must contain full image size, "
<< "but found: " << d);
framedim.setdim(0, d.dim(1));
framedim.setdim(1, d.dim(2));
frame.resize(framedim);
} catch(eblexception &e) {
#ifdef __NOEXCEPTIONS__
merr << "exception" << std::endl;
#else
merr << e << std::endl;
#endif
}
} else {
try {
mout << "starting processing of frame " << frame_name << std::endl;
bboxes &bb = detect.fprop(frame, frame_name.c_str(), frame_id);
copy_bboxes(bb); // make a copy of bounding boxes
} catch(ebl::eblexception &e) { // detection failed
#ifdef __NOEXCEPTIONS__
eblwarn("detection failed");
#else
eblwarn("detection failed: " << e);
#endif
clear_bboxes();
}
}
}
#ifdef __GUI__
else { // fprop and display
if (precomputed_boxes) eblerror("not implemented for nms only (TODO)");
disable_window_updates();
select_window(wid);
clear_window();
std::string title = _name;
title << ": " << frame_name;
set_window_title(title.c_str());
// clear_resize_window();
try {
if (mindisplay) {
bboxes &bb =
dgui.display(detect, frame, frame_name.c_str(), frame_id,
0, 0, zoom, display_min, display_max,
wid, _name.c_str(), display_transp);
copy_bboxes(bb); // make a copy of bounding boxes
} else {
// extract & display boxes
bboxes &bb =
dgui.display_inputs_outputs(detect, frame, frame_name.c_str(),
frame_id, 0, 0, zoom,
display_min, display_max, wid,
_name.c_str(),
display_in_min, display_in_max,
display_transp, display_wmax);
// make a copy of bounding boxes
copy_bboxes(bb);
}
} catch(ebl::eblexception &e) { // detection failed
eblwarn("detection failed: " << e);
clear_bboxes();
}
enable_window_updates();
}
if (display_states) {
dgui.display_current(detect, frame, wid_states, NULL, zoom);
select_window(wid);
}
if (save_video && display) {
std::string fname = viddir;
fname += frame_name;
save_window(fname.c_str());
if (!silent) mout << "saved " << fname << std::endl;
}
#endif
if (!silent)
mout << "processing done for frame " << frame_name << std::endl;
// bootstrapping
if (conf.exists_true("bootstrapping")) {
boot.fprop(detect, frame_name);
// add multiple scales if positives and scales are defined
if (conf.exists("gt_scales") && boot.extract_positives()) {
std::vector<double> scales =
string_to_doublevector(conf.get_cstring("gt_scales"));
for (uint s = 0; s < scales.size(); ++s) {
double f = scales[s];
// downsample input by f
detect.set_resolution(f);
detect.init(frame.get_idxdim(), frame_name.c_str());
detect.fprop(frame, frame_name.c_str(), frame_id);
boot.fprop(detect, frame_name, false, f);
}
detect.set_scaling_original();
detect.init(frame.get_idxdim(), frame_name.c_str());
}
copy_bootstrapping(boot.get_all(), boot.get_bball());
#ifdef __GUI__
// display groundtruth
if (conf.exists_true("display_bootstrapping"))
dgui.display_groundtruth(detect, frame, boot.get_gtall(),
boot.get_gtclean(), boot.get_gtrest(),
boot.get_bbpos(), boot.get_bbneg(),
boot.get_pos(), boot.get_neg(), 0, 0, zoom,
display_min, display_max);
#endif
}
total_saved = detect.get_total_saved();
ms = tpass.elapsed_milliseconds();
if (!silent) {
mout << bbs.pretty_short(detect.get_labels());
mout << "processing=" << ms << " ms ("
<< tpass.elapsed() << ")" << std::endl;
}
DEBUGMEM_PRETTY("after detection");
// switch 'updated' flag on to warn we just added new data
set_out_updated();
// display sleep
if (display_sleep > 0) {
mout << "sleeping for " << display_sleep << "ms." << std::endl;
millisleep(display_sleep);
}
if (conf.exists("save_max") &&
detect.get_total_saved() > conf.get_uint("save_max"))
break ; // limit number of detection saves
}
mout << "detection finished. Execution time: " << toverall.elapsed()<<std::endl;
// free variables
if (net) delete net;
if (ans) delete ans;
} eblcatcherror();
}
bool pascal_dataset<Tdata>::
process_objects(const std::vector<object*> &objs, int height, int width,
const std::string &image_fullname,
const std::string &image_filename,
const rect<int> *cropr) {
idx<Tdata> *img = NULL;
// process all objects
for (iobj = 0; iobj < objs.size(); ++iobj) {
object &o = *(objs[iobj]);
if (!o.ignored) { // only process non-ignored objects
if (!usepartsonly) {
// check that minimum visibility ratio is respected
if (minvisibility > 0.0 && o.visible) {
float match = o.match(*o.visible);
if (match < minvisibility) {
std::cout << "Rejecting object " << o.id << " from "
<< image_filename << " because visible ratio ("
<< match << ") is lower than "
<< "limit (" << minvisibility << ")" << std::endl;
continue ; // skip this object
}
}
// check that minimum aspect ratio is respected
if (o.height == 0) continue ; // skip this object
float ar = o.width / (float) o.height;
if (min_aspect_ratio >= 0.0 && ar < min_aspect_ratio) {
std::cout << "Rejecting object " << o.id << " from " << image_filename
<< " because aspect ratio of " << o << " (" << ar
<< ") is lower than " << "limit (" << min_aspect_ratio << ")"
<< std::endl;
continue ; // skip this object
}
if (max_aspect_ratio >= 0.0 && ar > max_aspect_ratio) {
std::cout << "Rejecting object " << o.id << " from " << image_filename
<< " because aspect ratio of " << o << " (" << ar
<< ") is bigger than " << "limit (" << max_aspect_ratio << ")"
<< std::endl;
continue ; // skip this object
}
// check that object is larger than minimum size
if (o.height < mindims.dim(0) || o.width < mindims.dim(1)) {
std::cout << "Rejecting object " << o.id << " from "
<< image_filename << " because object's size ("
<< o << ") is smaller than minimum size "
<< "(" << mindims << ")" << std::endl;
continue ; // skip this object
}
// check that bbox is not below image border distances
if (!minborders.empty()) {
if (height < 0 || width < 0) { // we don't know the image sizes
eblerror("image sizes not found in xml");
} // from now on assume all images have the same size (TODO)
int hmax = height - minborders.dim(0);
int wmax = width - minborders.dim(1);
if (o.h0 < (int) minborders.dim(0) || o.w0 < minborders.dim(1)
|| o.h0 + o.height > hmax || o.w0 + o.width > wmax) {
std::cout << "Rejecting object " << o.id << " from "
<< image_filename << " because (cropped) bbox " << o
<< " is closer to image borders (" << height << "x" << width
<< ") than allowed (" << minborders << ")" << std::endl;
continue ; // skip this object
}
}
// process image
if (included_pascal(o.name, o.difficult, o.truncated, o.occluded)) {
// load image if not already loaded
if (!img) {
// check that image exists
if (!file_exists(image_fullname)) {
eblwarn("Error: image not found " << image_fullname);
return false;
}
input_height = height;
input_width = width;
//load image
load_data(image_fullname);
}
// remove jitters that overlap with other objects
if (max_jitter_match > 0)
remove_jitter_matches(objs, iobj, max_jitter_match);
// extract object from image
process_mimage(load_img, o, o.name, o.difficult,
image_fullname, o.centroid, o.visible, cropr);
load_img.clear();
}
}
}
}
if (img) delete img;
return true;
}
