void tracking_thread<Tnet>::draw(bbox *b) {
#ifdef __GUI__
disable_window_updates();
clear_window();
draw_matrix(frame, "in");
draw_matrix(tpl, "tpl", 0, frame.dim(1));
if (b) {
ubyte red = 0, green = 0, blue = 0;
if (b->class_id == -42)
green = 255;
else
blue = 255;
draw_box(b->h0, b->w0, b->width, b->height, red, green, blue);
}
enable_window_updates();
#endif
}
int display_net(list<string>::iterator &ifname,
bool signd, bool load, list<string> *mats) {
// create network
idx<ubyte> classes(1,1);
load_matrix<ubyte>(classes, conf->get_cstring("classes"));
answer_module<T,T,T> *answer = create_answer<T,T,T>(*conf, classes.dim(0));
if (!answer) eblerror("no answer module found");
uint noutputs = answer->get_nfeatures();
parameter<T> theparam;
intg thick;
module_1_1<T> *net = create_network<T>(theparam, *conf, thick, noutputs);
cout << "Network parameters: " << theparam << endl;
vector<string> weights =
string_to_stringvector(conf->get_string("weights_file"));
// loading weights
theparam.load_x(weights);
// displaying internals
uint h0 = 0, w0 = 0;
#ifdef __GUI__
disable_window_updates();
clear_window();
if (show_filename) {
gui << at(h0, w0) << black_on_white() << ifname->c_str();
h0 += 16;
}
if (autorange) // automatic range
lg.display_internals(*net, h0, w0, zoom, (T) 0, (T) 0, maxwidth);
else // fixed range
lg.display_internals(*net, h0, w0, zoom, (T) range[0], (T) range[1],
maxwidth);
// update title
string title;
title << "matshow: " << ebl::basename(ifname->c_str());
set_window_title(title.c_str());
enable_window_updates();
#endif
return 1;
}
void detection_thread<T>::execute() {
try {
bool display = false;
#ifdef __GUI__
display = conf.exists_true("display")
&& conf.exists_true("display_threads");
bool mindisplay = conf.exists_true("minimal_display");
bool save_video = conf.exists_true("save_video");
bool display_states = conf.exists_true("display_states");
uint wid = 0; // window id
uint wid_states = 0; // window id
#endif
uint display_sleep = conf.try_get_uint("display_sleep", 0);
// if (!display && save_video) {
// // we still want to output images but not show them
// display = true;
// #ifdef __GUI__
// set_gui_silent();
// #endif
// }
// load network and weights in a forward-only parameter
parameter<T> theparam;
theparam.set_forward_only();
idx<ubyte> classes(1,1);
//try { // try loading classes names but do not stop upon failure
load_matrix<ubyte>(classes, conf.get_cstring("classes"));
// } catch(std::string &err) {
// merr << "warning: " << err;
// merr << std::endl;
// }
std::vector<std::string> sclasses = ubyteidx_to_stringvector(classes);
answer_module<T> *ans = create_answer<T,T,T>(conf, classes.dim(0));
uint noutputs = ans->get_nfeatures();
intg thick = -1;
module_1_1<T> *net = create_network<T>(theparam, conf, thick, noutputs,
"arch", this->_id);
// loading weights
if (conf.exists("weights")) { // manual weights
// concatenate weights if multiple ones
std::vector<std::string> w =
string_to_stringvector(conf.get_string("weights"));
mout << "Loading weights from: " << w << std::endl;
theparam.load_x(w);
// permute weights by blocks
if (conf.exists("weights_permutation")) {
std::string sblocks = conf.get_string("weights_blocks");
std::string spermut = conf.get_string("weights_permutation");
std::vector<intg> blocks = string_to_intgvector(sblocks.c_str());
std::vector<uint> permut = string_to_uintvector(spermut.c_str());
theparam.permute_x(blocks, permut);
}
} else {
if (conf.exists_true("manual_load")) { // manual load
eblwarn("\"weights\" variable not defined, loading manually "
<< "if manual_load defined");
manually_load_network(*((layers<T>*)net), conf);
} else { // random weights
int seed = dynamic_init_drand();
eblwarn("No weights to load, randomizing weights with seed " << seed);
forget_param_linear fgp(1, 0.5, seed);
net->forget(fgp);
}
}
DEBUGMEM_PRETTY("before detection");
// detector
detector<T> detect(*net, sclasses, ans, NULL, NULL, mout, merr);
init_detector(detect, conf, outdir, silent);
// keep pointer to detector
pdetect = &detect;
bootstrapping<T> boot(conf);
// when a bbox file is given, ignore the processing, load the pre-computed
// bboxes and feed them to the nms (non-maximum suppression).
bboxes boxes(bbox_all, NULL, mout, merr);
boxes.print_saving_type(); // inform user how we save boxes
bool precomputed_boxes = false;
if (conf.exists("bbox_file")) {
precomputed_boxes = true;
std::string bbfile = conf.get_string("bbox_file");
boxes.load_eblearn(bbfile);
}
bool bmask_class = false;
if (conf.exists("mask_class"))
bmask_class = detect.set_mask_class(conf.get_cstring("mask_class"));
std::string viddir = outdir;
viddir += "video/";
mkdir_full(viddir);
// gui
#ifdef __GUI__
uint display_wmax = conf.try_get_uint("display_max_width", 3000);
T display_min = (T) conf.try_get_double("display_min", -1.7);
T display_max = (T) conf.try_get_double("display_max", 1.7);
T display_in_max = (T) conf.try_get_double("display_in_max", 255);
T display_in_min = (T) conf.try_get_double("display_in_min", 0);
float display_transp = conf.try_get_float("display_bb_transparency", 0);
uint qstep1 = conf.try_get_uint("qstep1", 0);
uint qheight1 = conf.try_get_uint("qheight1", 0);
uint qwidth1 = conf.try_get_uint("qwidth1", 0);
uint qstep2 = conf.try_get_uint("qstep2", 0);
uint qheight2 = conf.try_get_uint("qheight2", 0);
uint qwidth2 = conf.try_get_uint("qwidth2", 0);
module_1_1_gui netgui;
wid_states = display_states ? new_window("network states"):0;
night_mode();
std::string title = "EBLearn detector: ";
title += _name;
if (display) {
wid = new_window(title.c_str());
mout << "displaying in window " << wid << std::endl;
night_mode();
}
float zoom = conf.try_get_float("display_zoom", 1);
bool bbox_show_conf = !conf.exists_false("bbox_show_conf");
bool bbox_show_class = !conf.exists_false("bbox_show_class");
detector_gui<T>
dgui(conf.try_get_uint("show_extracted", 0),
conf.exists_bool("queue1"), qstep1, qheight1,
qwidth1, conf.exists_bool("queue2"), qstep2, qheight2, qwidth2,
bbox_show_class, bbox_show_conf);
if (bmask_class)
dgui.set_mask_class(conf.get_cstring("mask_class"),
(T) conf.try_get_double("mask_threshold", 0));
#endif
// timing variables
timer tpass, toverall;
long ms;
// loop
toverall.start();
// we're ready
bavailable = true;
while(!this->_stop) {
// wait until a new image is made available
while (!in_updated && !_stop) {
millisleep(1);
}
tpass.restart();
if (_stop) break ;
// we got a new frame, reset new frame flag
in_updated = false; // no need to lock mutex
// check if this frame should be skipped
if (boot.skip_frame(frame_name)) {
skip_frame();
continue ;
} else if (!frame_loaded) {
uframe = load_image<ubyte>(frame_fullname);
mout << "loaded image " << frame_fullname << std::endl;
}
if (!silent) mout << "processing " << frame_name << std::endl;
// check frame is correctly allocated, if not, allocate.
if (frame.order() != uframe.order())
frame = idx<T>(uframe.get_idxdim());
else if (frame.get_idxdim() != uframe.get_idxdim())
frame.resize(uframe.get_idxdim());
// copy frame
idx_copy(uframe, frame);
// run detector
if (!display) { // fprop without display
if (precomputed_boxes) {
try {
bboxes *bb = boxes.get_group(frame_name);
idxdim d = boxes.get_group_dims(frame_name);
d.insert_dim(0, 1);
bboxes pruned;
detect.init(d);
detect.fprop_nms(*bb, pruned);
copy_bboxes(pruned); // make a copy of bounding boxes
// resize frame so that caller knows the size of the frame
idxdim framedim = frame.get_idxdim();
if (d.dim(1) == -1 || d.dim(2) == -1)
eblerror("pre-computed boxes must contain full image size, "
<< "but found: " << d);
framedim.setdim(0, d.dim(1));
framedim.setdim(1, d.dim(2));
frame.resize(framedim);
} catch(eblexception &e) {
#ifdef __NOEXCEPTIONS__
merr << "exception" << std::endl;
#else
merr << e << std::endl;
#endif
}
} else {
try {
mout << "starting processing of frame " << frame_name << std::endl;
bboxes &bb = detect.fprop(frame, frame_name.c_str(), frame_id);
copy_bboxes(bb); // make a copy of bounding boxes
} catch(ebl::eblexception &e) { // detection failed
#ifdef __NOEXCEPTIONS__
eblwarn("detection failed");
#else
eblwarn("detection failed: " << e);
#endif
clear_bboxes();
}
}
}
#ifdef __GUI__
else { // fprop and display
if (precomputed_boxes) eblerror("not implemented for nms only (TODO)");
disable_window_updates();
select_window(wid);
clear_window();
std::string title = _name;
title << ": " << frame_name;
set_window_title(title.c_str());
// clear_resize_window();
try {
if (mindisplay) {
bboxes &bb =
dgui.display(detect, frame, frame_name.c_str(), frame_id,
0, 0, zoom, display_min, display_max,
wid, _name.c_str(), display_transp);
copy_bboxes(bb); // make a copy of bounding boxes
} else {
// extract & display boxes
bboxes &bb =
dgui.display_inputs_outputs(detect, frame, frame_name.c_str(),
frame_id, 0, 0, zoom,
display_min, display_max, wid,
_name.c_str(),
display_in_min, display_in_max,
display_transp, display_wmax);
// make a copy of bounding boxes
copy_bboxes(bb);
}
} catch(ebl::eblexception &e) { // detection failed
eblwarn("detection failed: " << e);
clear_bboxes();
}
enable_window_updates();
}
if (display_states) {
dgui.display_current(detect, frame, wid_states, NULL, zoom);
select_window(wid);
}
if (save_video && display) {
std::string fname = viddir;
fname += frame_name;
save_window(fname.c_str());
if (!silent) mout << "saved " << fname << std::endl;
}
#endif
if (!silent)
mout << "processing done for frame " << frame_name << std::endl;
// bootstrapping
if (conf.exists_true("bootstrapping")) {
boot.fprop(detect, frame_name);
// add multiple scales if positives and scales are defined
if (conf.exists("gt_scales") && boot.extract_positives()) {
std::vector<double> scales =
string_to_doublevector(conf.get_cstring("gt_scales"));
for (uint s = 0; s < scales.size(); ++s) {
double f = scales[s];
// downsample input by f
detect.set_resolution(f);
detect.init(frame.get_idxdim(), frame_name.c_str());
detect.fprop(frame, frame_name.c_str(), frame_id);
boot.fprop(detect, frame_name, false, f);
}
detect.set_scaling_original();
detect.init(frame.get_idxdim(), frame_name.c_str());
}
copy_bootstrapping(boot.get_all(), boot.get_bball());
#ifdef __GUI__
// display groundtruth
if (conf.exists_true("display_bootstrapping"))
dgui.display_groundtruth(detect, frame, boot.get_gtall(),
boot.get_gtclean(), boot.get_gtrest(),
boot.get_bbpos(), boot.get_bbneg(),
boot.get_pos(), boot.get_neg(), 0, 0, zoom,
display_min, display_max);
#endif
}
total_saved = detect.get_total_saved();
ms = tpass.elapsed_milliseconds();
if (!silent) {
mout << bbs.pretty_short(detect.get_labels());
mout << "processing=" << ms << " ms ("
<< tpass.elapsed() << ")" << std::endl;
}
DEBUGMEM_PRETTY("after detection");
// switch 'updated' flag on to warn we just added new data
set_out_updated();
// display sleep
if (display_sleep > 0) {
mout << "sleeping for " << display_sleep << "ms." << std::endl;
millisleep(display_sleep);
}
if (conf.exists("save_max") &&
detect.get_total_saved() > conf.get_uint("save_max"))
break ; // limit number of detection saves
}
mout << "detection finished. Execution time: " << toverall.elapsed()<<std::endl;
// free variables
if (net) delete net;
if (ans) delete ans;
} eblcatcherror();
}
MAIN_QTHREAD(int, argc, char **, argv) { // macro to enable multithreaded gui
#else
int main(int argc, char **argv) { // regular main without gui
#endif
try {
// check input parameters
if ((argc != 2) && (argc != 3) ) {
cerr << "wrong number of parameters." << endl;
cerr << "usage: objdetect <config file> [directory or file]" << endl;
return -1;
}
#ifdef __LINUX__
feenableexcept(FE_DIVBYZERO | FE_INVALID); // enable float exceptions
#endif
ipp_init(1); // limit IPP (if available) to 1 core
// load configuration
configuration conf(argv[1], true, true, false);
if (!conf.exists("root2")) {
string dir;
dir << dirname(argv[1]) << "/";
cout << "Looking for trained files in: " << dir << endl;
conf.set("root2", dir.c_str());
conf.set("current_dir", dir.c_str());
}
conf.resolve();
if (conf.exists_true("show_conf")) conf.pretty();
bool color = conf.exists_bool("color");
uint norm_size = conf.get_uint("normalization_size");
Tnet threshold = (Tnet) conf.get_double("threshold");
bool display = false;
bool display_states= false;
bool mindisplay = false;
uint display_sleep = 0;
bool save_video = false;
string cam_type = conf.get_string("camera");
int height = -1;
int width = -1;
if (conf.exists("input_height")) height = conf.get_int("input_height");
if (conf.exists("input_width")) width = conf.get_int("input_width");
bool input_random = conf.exists_true("input_random");
uint npasses = 1;
char next_on_key = 0;
uint wid = 0; // window id
uint wid_states = 0; // window id
string outdir = "out_";
if (conf.exists("next_on_key")) {
next_on_key = conf.get_char("next_on_key");
cout << "Press " << next_on_key << " to process next frame." << endl;
}
outdir += tstamp();
outdir += "/";
cout << "Saving outputs to " << outdir << endl;
// load network and weights
parameter<fs(Tnet)> theparam;
idx<ubyte> classes(1,1);
try {
load_matrix<ubyte>(classes, conf.get_cstring("classes"));
} catch(string &err) { cerr << "warning: " << err << endl; }
vector<string> sclasses = ubyteidx_to_stringvector(classes);
answer_module<SFUNC2(Tnet)> *ans =
create_answer<SFUNC2(Tnet)>(conf, classes.dim(0));
uint noutputs = ans->get_nfeatures();
module_1_1<SFUNC(Tnet)> *net =
create_network<SFUNC(Tnet)>(theparam, conf, noutputs);
// loading weights
if (!conf.exists("weights")) { // manual weights
cerr << "warning: \"weights\" variable not defined, loading manually "
<< "if manual_load defined" << endl;
if (conf.exists_true("manual_load"))
manually_load_network(*((layers<SFUNC(Tnet)>*)net), conf);
} else { // multiple-file weights
// concatenate weights if multiple ones
vector<string> w =
string_to_stringvector(conf.get_string("weights"));
cout << "Loading weights from: " << w << endl;
theparam.load_x(w);
}
// detector
detector<fs(Tnet)> detect(*net, sclasses, *ans, NULL, NULL);
// multi-scaling parameters
double maxs = conf.exists("max_scale")?conf.get_double("max_scale") : 1.0;
double mins = conf.exists("min_scale")?conf.get_double("min_scale") : 1.0;
t_scaling scaling_type = SCALES_STEP;
vector<idxdim> scales;
if (conf.exists("scaling_type"))
scaling_type = (t_scaling) conf.get_uint("scaling_type");
switch (scaling_type) {
case MANUAL:
if (!conf.exists("scales"))
eblerror("expected \"scales\" variable to be defined in manual mode");
scales = string_to_idxdimvector(conf.get_cstring("scales"));
detect.set_resolutions(scales);
break ;
case ORIGINAL: detect.set_scaling_original(); break ;
case SCALES_STEP:
detect.set_resolutions(conf.get_double("scaling"), maxs, mins);
break ;
case SCALES_STEP_UP:
detect.set_resolutions(conf.get_double("scaling"), maxs, mins);
detect.set_scaling_type(scaling_type);
break ;
default:
detect.set_scaling_type(scaling_type);
}
// optimize memory usage by using only 2 buffers for entire flow
SBUF<Tnet> input(1, 1, 1), output(1, 1, 1);
if (!conf.exists_false("mem_optimization"))
detect.set_mem_optimization(input, output, true);
// zero padding
float hzpad = 0, wzpad = 0;
if (conf.exists("hzpad")) hzpad = conf.get_float("hzpad");
if (conf.exists("wzpad")) wzpad = conf.get_float("wzpad");
detect.set_zpads(hzpad, wzpad);
bool bmask_class = false;
if (conf.exists("mask_class"))
bmask_class = detect.set_mask_class(conf.get_cstring("mask_class"));
if (conf.exists("input_min")) // limit inputs size
detect.set_min_resolution(conf.get_uint("input_min"));
if (conf.exists("input_max")) // limit inputs size
detect.set_max_resolution(conf.get_uint("input_max"));
detect.set_silent();
if (conf.exists_bool("save_detections")) {
string detdir = outdir;
detdir += "detections";
detdir = detect.set_save(detdir);
if (conf.exists("save_max_per_frame"))
detect.set_save_max_per_frame(conf.get_uint("save_max_per_frame"));
}
// nms
detect.set_cluster_nms(conf.exists_true("cluster_nms"));
detect.set_scaler_mode(conf.exists_true("scaler_mode"));
if (conf.exists("nms"))
detect.set_pruning((t_pruning)conf.get_uint("nms"),
conf.exists("min_hcenter_dist") ?
conf.get_float("min_hcenter_dist") : 0.0,
conf.exists("min_wcenter_dist") ?
conf.get_float("min_wcenter_dist") : 0.0,
conf.exists("bbox_max_overlap") ?
conf.get_float("bbox_max_overlap") : 1.0,
conf.exists_true("share_parts"),
conf.exists("threshold2") ?
(Tnet) conf.get_float("threshold2") : 0.0,
conf.exists("bbox_max_center_dist") ?
conf.get_float("bbox_max_center_dist") : 0.0,
conf.exists("bbox_max_center_dist2") ?
conf.get_float("bbox_max_center_dist2") : 0.0,
conf.exists("bbox_max_wcenter_dist") ?
conf.get_float("bbox_max_wcenter_dist") : 0.0,
conf.exists("bbox_max_wcenter_dist2") ?
conf.get_float("bbox_max_wcenter_dist2") : 0.0,
conf.exists("min_wcenter_dist2") ?
conf.get_float("min_wcenter_dist2") : 0.0,
conf.exists("bbox_max_overlap2") ?
conf.get_float("bbox_max_overlap2") : 0.0,
conf.exists_true("mean_bb"),
conf.exists("same_scale_mhd") ?
conf.get_float("same_scale_mhd") : 0.0,
conf.exists("same_scale_mwd") ?
conf.get_float("same_scale_mwd") : 0.0,
conf.exists("min_scale_pred") ?
conf.get_float("min_scale_pred") : 0.0,
conf.exists("max_scale_pred") ?
conf.get_float("max_scale_pred") : 0.0
);
if (conf.exists("bbox_hfactor") && conf.exists("bbox_wfactor"))
detect.set_bbox_factors(conf.get_float("bbox_hfactor"),
conf.get_float("bbox_wfactor"),
conf.exists("bbox_woverh") ?
conf.get_float("bbox_woverh") : 1.0,
conf.exists("bbox_hfactor2") ?
conf.get_float("bbox_hfactor2") : 1.0,
conf.exists("bbox_wfactor2") ?
conf.get_float("bbox_wfactor2") : 1.0);
if (conf.exists("max_object_hratio"))
detect.set_max_object_hratio(conf.get_double("max_object_hratio"));
if (conf.exists("net_min_height") && conf.exists("net_min_width"))
detect.set_min_input(conf.get_int("net_min_height"),
conf.get_int("net_min_width"));
if (conf.exists("smoothing"))
detect.set_smoothing(conf.get_uint("smoothing"));
if (conf.exists("background_name"))
detect.set_bgclass(conf.get_cstring("background_name"));
// image search can be configured with a search pattern
const char *fpattern = IMAGE_PATTERN_MAT;
if (conf.exists("file_pattern"))
fpattern = conf.get_cstring("file_pattern");
// initialize camera (opencv, directory, shmem or video)
idx<ubyte> frame;
camera<ubyte> *cam = NULL, *cam2 = NULL;
if (!strcmp(cam_type.c_str(), "directory")) {
string dir;
if (argc >= 3) // read input dir from command line
dir = argv[2];
else if (conf.exists("input_dir"))
dir = conf.get_string("input_dir");
// given list
list<string> files;
if (conf.exists("input_list")) {
files = string_to_stringlist(conf.get_string("input_list"));
cam = new camera_directory<ubyte>(dir.c_str(), height, width,
input_random, npasses,
std::cout, std::cerr,
fpattern, &files);
} else // given directory only
cam = new camera_directory<ubyte>(dir.c_str(), height, width,
input_random, npasses,
std::cout, std::cerr,
fpattern, &files);
} else if (!strcmp(cam_type.c_str(), "opencv"))
cam = new camera_opencv<ubyte>(-1, height, width);
#ifdef __LINUX__
else if (!strcmp(cam_type.c_str(), "v4l2"))
cam = new camera_v4l2<ubyte>(conf.get_cstring("device"),
height, width,
conf.exists_true("camera_grayscale"));
#endif
else if (!strcmp(cam_type.c_str(), "shmem"))
cam = new camera_shmem<ubyte>("shared-mem", height, width);
else if (!strcmp(cam_type.c_str(), "video")) {
if (argc >= 3)
cam = new camera_video<ubyte>
(argv[2], height, width, conf.get_uint("input_video_sstep"),
conf.get_uint("input_video_max_duration"));
else eblerror("expected 2nd argument");
} else eblerror("unknown camera type");
// a camera directory may be used first, then switching to regular cam
if (conf.exists_bool("precamera"))
cam2 = new camera_directory<ubyte>(conf.get_cstring("precamdir"),
height, width);
// answer variables & initializations
vector<bbox*> bboxes;
vector<bbox*>::iterator ibboxes;
ostringstream answer_fname;
mkdir_full(outdir);
answer_fname << outdir << "bbox.txt";
// open file
ofstream fp(answer_fname.str().c_str());
if (!fp) {
cerr << "failed to open " << answer_fname.str() << endl;
eblerror("open failed");
}
// gui
#ifdef __GUI__
display = conf.exists_bool("display");
mindisplay = conf.exists_bool("minimal_display");
display_sleep = conf.get_uint("display_sleep");
display_states = conf.exists_bool("display_states");
save_video = conf.exists_bool("save_video");
uint qstep1 = 0, qheight1 = 0, qwidth1 = 0,
qheight2 = 0, qwidth2 = 0, qstep2 = 0;
if (conf.exists_bool("queue1")) {
qstep1 = conf.get_uint("qstep1");
qheight1 = conf.get_uint("qheight1");
qwidth1 = conf.get_uint("qwidth1"); }
if (conf.exists_bool("queue2")) {
qstep2 = conf.get_uint("qstep2");
qheight2 = conf.get_uint("qheight2");
qwidth2 = conf.get_uint("qwidth2"); }
module_1_1_gui netgui;
wid_states = display_states ? new_window("network states"):0;
night_mode();
wid = display ? new_window("eblearn object recognition") : 0;
night_mode();
float zoom = 1;
detector_gui<fs(Tnet)> dgui(conf.exists_bool("queue1"), qstep1, qheight1,
qwidth1, conf.exists_bool("queue2"), qstep2,
qheight2, qwidth2);
if (bmask_class)
dgui.set_mask_class(conf.get_cstring("mask_class"),
(Tnet) conf.get_double("mask_threshold"));
if (save_video) {
string viddir = outdir;
viddir += "video";
cam->start_recording(wid, viddir.c_str());
}
#endif
// timing variables
timer tpass, toverall;
long ms;
// loop
toverall.start();
while(!cam->empty()) {
// get a new frame
tpass.restart();
// if the pre-camera is defined use it until empty
if (cam2 && !cam2->empty())
frame = cam2->grab();
else // empty pre-camera, use regular camera
frame = cam->grab();
string frame_name = cam->frame_name();
// run detector
if (!display) { // fprop without display
bboxes = detect.fprop(frame, threshold, frame_name.c_str());
}
#ifdef __GUI__
else { // fprop and display
disable_window_updates();
clear_window();
if (mindisplay)
bboxes = dgui.display(detect, frame, threshold, frame_name.c_str(),
0, 0, zoom, (Tnet)0, (Tnet)255, wid);
else
bboxes =
dgui.display_inputs_outputs(detect, frame, threshold,
frame_name.c_str(), 0, 0, zoom,
(Tnet)-1.1, (Tnet)1.1, wid);
enable_window_updates();
if (display_states) {
dgui.display_current(detect, frame, wid_states);
select_window(wid);
}
if (save_video)
cam->record_frame();
}
#endif
ms = tpass.elapsed_milliseconds();
cout << "processing: " << ms << " ms." << endl;
cout << "fps: " << cam->fps() << endl;
// save bounding boxes
for (ibboxes = bboxes.begin(); ibboxes != bboxes.end(); ++ibboxes) {
fp << cam->frame_name() << " " << (*ibboxes)->class_id << " "
<< (*ibboxes)->confidence << " ";
fp << (*ibboxes)->w0 << " " << (*ibboxes)->h0 << " ";
fp << (*ibboxes)->w0 + (*ibboxes)->width << " ";
fp << (*ibboxes)->h0 + (*ibboxes)->height << endl;
}
// sleep display
if (display_sleep > 0) {
cout << "sleeping for " << display_sleep << "ms." << endl;
millisleep(display_sleep);
}
if (conf.exists("save_max") &&
detect.get_total_saved() > conf.get_uint("save_max")) {
cout << "Reached max number of detections, exiting." << endl;
break ; // limit number of detection saves
}
}
cout << "Execution time: " << toverall.elapsed_minutes() <<" mins" <<endl;
if (save_video)
cam->stop_recording(conf.exists_bool("use_original_fps") ?
cam->fps() : conf.get_uint("save_video_fps"));
// free variables
if (net) delete net;
if (cam) delete cam;
// close files
fp.close();
#ifdef __GUI__
if (!conf.exists_true("no_gui_quit")) {
cout << "Closing windows..." << endl;
quit_gui(); // close all windows
cout << "Windows closed." << endl;
}
#endif
} eblcatcherror();
return 0;
}
int display(list<string>::iterator &ifname,
bool signd, bool load, list<string> *mats) {
//cout << "displaying " << ifname->c_str() << endl;
// conf mode
if (conf)
return display_net<T>(ifname, signd, load, mats);
// mat mode
if (is_matrix(ifname->c_str())) {
idxdim d = get_matrix_dims(ifname->c_str());
if (interleaved)
d.shift_dim(0, 2);
if (save_individually || print
|| !(d.order() == 2 || (d.order() == 3 &&
(d.dim(2) == 1 || d.dim(2) == 3)))) {
// this is probably not an image, just display info and print matrix
string type;
get_matrix_type(ifname->c_str(), type);
idx<T> m = load_matrix<T>(ifname->c_str());
cout << "Matrix " << ifname->c_str() << " is of type " << type
<< " with dimensions " << d << " (min " << idx_min(m)
<< ", max " << idx_max(m) << ", mean " << idx_mean(m)
<< "):" << endl;
m.print();
if (has_multiple_matrices(ifname->c_str())) {
midx<T> ms = load_matrices<T>(ifname->c_str(), true);
// saving sub-matrices
if (save_individually) {
cout << "Saving each sub-matrix of " << *ifname << " individually..."
<< endl;
save_matrices_individually(ms, *ifname, true);
}
// printing sub-matrices
cout << "This file contains " << m << " matrices: ";
if (ms.order() == 1) {
for (intg i = 0; i < ms.dim(0); ++i) {
idx<T> tmp = ms.mget(i);
cout << tmp.info() << " ";
}
} else if (ms.order() == 2) {
for (intg i = 0; i < ms.dim(0); ++i) {
for (intg j = 0; j < ms.dim(1); ++j) {
idx<T> tmp = ms.mget(i, j);
cout << tmp.info() << " ";
}
cout << endl;
}
}
cout << endl;
} else
cout << "This is a single-matrix file." << endl;
return 0;
}
}
// image mode
int loaded = 0;
static idx<T> mat;
uint h = 0, w = 0, rowh = 0, maxh = 0;
list<string>::iterator fname = ifname;
#ifdef __GUI__
disable_window_updates();
clear_window();
if (show_filename) {
gui << at(h, w) << black_on_white() << ifname->c_str();
h += 16;
}
#endif
maxh = h;
for (uint i = 0; i < nh; ++i) {
rowh = maxh;
for (uint j = 0; j < nw; ++j) {
if (fname == mats->end())
fname = mats->begin();
try {
// if (load)
mat = load_image<T>(*fname);
if (print)
cout << *fname << ": " << mat << endl << mat.str() << endl;
// show only some channels
if (chans >= 0)
mat = mat.select(2, chans);
loaded++;
maxh = (std::max)(maxh, (uint) (rowh + mat.dim(0)));
T min = 0, max = 0;
#ifdef __GUI__
if (autorange || signd) {
if (autorange) {
min = idx_min(mat);
max = idx_max(mat);
} else if (signd) {
T matmin = idx_min(mat);
if ((double)matmin < 0) {
min = -1;
max = -1;
}
}
draw_matrix(mat, rowh, w, zoom, zoom, min, max);
} else
draw_matrix(mat, rowh, w, zoom, zoom, (T) range[0], (T) range[1]);
#endif
w += mat.dim(1) + 1;
} catch(string &err) {
ERROR_MSG(err.c_str());
}
fname++;
if (fname == ifname)
break ;
}
if (fname == ifname)
break ;
maxh++;
w = 0;
}
#ifdef __GUI__
// info
if (show_info) {
set_text_colors(0, 0, 0, 255, 255, 255, 255, 200);
gui << mat;
gui << at(15, 0) << *fname;
gui << at(29, 0) << "min: " << idx_min(mat) << " max: " << idx_max(mat);
}
// help
if (show_help) {
h = 0;
w = 0;
uint hstep = 14;
set_text_colors(0, 0, 255, 255, 255, 255, 255, 200);
gui << at(h, w) << "Controls:"; h += hstep;
set_text_colors(0, 0, 0, 255, 255, 255, 255, 200);
gui << at(h, w) << "Right/Space: next image"; h += hstep;
gui << at(h, w) << "Left/Backspace: previous image"; h += hstep;
gui << at(h, w) << "i: image info"; h += hstep;
gui << at(h, w) << "a: auto-range (use min and max as range)"; h += hstep;
gui << at(h, w) << "x/z: show more/less images on width axis"; h += hstep;
gui << at(h, w) << "y/t: show more/less images on height axis"; h += hstep;
gui << at(h, w) << "0,1,2: show channel 0, 1 or 2 only"; h += hstep;
gui << at(h, w) << "9: show alls channels"; h += hstep;
gui << at(h, w) << "h: help";
}
// update title
string title;
title << "matshow: " << ebl::basename(ifname->c_str());
set_window_title(title.c_str());
enable_window_updates();
#endif
return loaded;
}
MAIN_QTHREAD(int, argc, char **, argv) { // macro to enable multithreaded gui
#else
int main(int argc, char **argv) { // regular main without gui
#endif
try {
// check input parameters
if ((argc != 2) && (argc != 3) ) {
cerr << "warning: wrong number of parameters." << endl;
cerr << "usage: detect <config file> [directory or file]" << endl;
// return -1;
}
#ifdef __LINUX__
feenableexcept(FE_DIVBYZERO | FE_INVALID); // enable float exceptions
#endif
// load configuration
configuration conf(argv[1], true, true, false);
if (conf.exists_true("fixed_randomization"))
cout << "Using fixed seed: " << fixed_init_drand() << endl;
else
cout << "Using random seed: " << dynamic_init_drand(argc, argv) << endl;
if (!conf.exists("root2") || !conf.exists("current_dir")) {
string dir;
dir << dirname(argv[1]) << "/";
cout << "Looking for trained files in: " << dir << endl;
conf.set("root2", dir.c_str());
conf.set("current_dir", dir.c_str());
}
conf.set("run_type", "detect"); // tell conf that we are in detect mode
conf.resolve(); // manual call to resolving variable
bool silent = conf.exists_true("silent");
if (conf.exists_true("show_conf") && !silent) conf.pretty();
// output synchronization
bool sync = conf.exists_true("sync_outputs");
mutex out_mutex;
mutex_ostream mutout(std::cout, &out_mutex, "Thread M");
mutex_ostream muterr(std::cerr, &out_mutex, "Thread M");
ostream &mout = sync ? mutout : cout;
ostream &merr = sync ? muterr : cerr;
bootstrapping<t_net> boot(conf);
// output dir
string outdir = detection_thread<t_net>::get_output_directory(conf);
mout << "Saving outputs to " << outdir << endl;
// save conf to output dir
string cname = outdir;
cname << filename(argv[1]);
if (conf.write(cname.c_str()))
mout << "Wrote configuration to " << cname << endl;
// load classes of network
idx<ubyte> classes(1,1);
vector<string> sclasses;
try { // try loading classes names but do not stop upon failure
load_matrix<ubyte>(classes, conf.get_cstring("classes"));
} catch(string &err) { merr << "warning: " << err << endl; }
sclasses = ubyteidx_to_stringvector(classes);
t_bbox_saving bbsaving = bbox_none;
if (conf.exists("bbox_saving"))
bbsaving = (t_bbox_saving) conf.get_int("bbox_saving");
bboxes boxes(bbsaving, &outdir, mout, merr);
uint ipp_cores = 1;
if (conf.exists("ipp_cores")) ipp_cores = conf.get_uint("ipp_cores");
ipp_init(ipp_cores); // limit IPP (if available) to 1 core
bool save_video = conf.exists_true("save_video");
bool save_detections = conf.exists_true("save_detections");
int height = -1;
int width = -1;
if (conf.exists("input_height")) height = conf.get_int("input_height");
if (conf.exists("input_width")) width = conf.get_int("input_width");
bool input_random = conf.exists_true("input_random");
uint npasses = 1;
char next_on_key = 0;
if (conf.exists("next_on_key")) {
next_on_key = conf.get_char("next_on_key");
mout << "Press " << next_on_key << " to process next frame." << endl;
}
uint skip_frames = conf.try_get_uint("skip_frames", 0);
if (conf.exists("input_npasses"))
npasses = conf.get_uint("input_npasses");
string viddir;
if (save_video) {
viddir << outdir << "video/";
mkdir_full(viddir);
}
bool precomputed_boxes = conf.exists("bbox_file");
uint save_bbox_period = conf.try_get_uint("save_bbox_period", 500);
idxdim crop(1, 1, 1);
if (conf.exists("input_crop"))
crop = string_to_idxdim(conf.get_string("input_crop"));
string cam_type;
#ifdef __LINUX__ // default camera for linux if not defined
cam_type = "v4l2";
#endif
if (conf.exists("camera"))
cam_type = conf.get_string("camera");
// allocate threads
uint nthreads = 1;
bool updated = false;
idx<ubyte> detframe; // frame returned by detection thread
uint frame_id = 0;
svector<midx<t_net> > all_samples, samples; // extracted samples
bboxes all_bbsamples, bbsamples; // boxes corresponding to samples
if (conf.exists("nthreads"))
nthreads = (std::max)((uint) 1, conf.get_uint("nthreads"));
list<detection_thread<t_net>*> threads;
list<detection_thread<t_net>*>::iterator ithreads;
idx<uint> total_saved(nthreads);
idx_clear(total_saved);
mout << "Initializing " << nthreads << " detection threads." << endl;
for (uint i = 0; i < nthreads; ++i) {
detection_thread<t_net> *dt =
new detection_thread<t_net>(conf, &out_mutex, NULL, NULL, sync);
threads.push_back(dt);
dt->start();
}
// image search can be configured with a search pattern
const char *fpattern = IMAGE_PATTERN_MAT;
if (conf.exists("file_pattern"))
fpattern = conf.get_cstring("file_pattern");
// initialize camera (opencv, directory, shmem or video)
idx<ubyte> frame(std::max(height, 1), std::max(width, 1), 3);
camera<ubyte> *cam = NULL, *cam2 = NULL;
if (!strcmp(cam_type.c_str(), "directory")) {
string dir;
if (argc >= 3) // read input dir from command line
dir = argv[2];
else if (conf.exists("input_dir"))
dir = conf.get_string("input_dir");
// given list
list<string> files;
if (conf.exists("input_list")) {
files = string_to_stringlist(conf.get_string("input_list"));
cam = new camera_directory<ubyte>(dir.c_str(), height, width,
input_random, npasses, mout, merr,
fpattern, &files);
} else // given directory only
cam = new camera_directory<ubyte>(dir.c_str(), height, width,
input_random, npasses, mout, merr,
fpattern, &files);
} else if (!strcmp(cam_type.c_str(), "opencv"))
cam = new camera_opencv<ubyte>(-1, height, width);
#ifdef __LINUX__
else if (!strcmp(cam_type.c_str(), "v4l2"))
cam = new camera_v4l2<ubyte>(conf.get_cstring("device"),
height, width,
conf.exists_true("camera_grayscale"),
conf.exists_true("camera_rgb"));
else if (!strcmp(cam_type.c_str(), "mac"))
cam = new camera_mac<ubyte>(conf.get_cstring("device"),
height, width,
conf.exists_true("camera_grayscale"),
conf.exists_true("camera_rgb"));
else if (!strcmp(cam_type.c_str(), "mcams")) {
vector<string> devices = conf.get_all_strings("device");
cam = new camera_mcams<ubyte>(conf, devices, height, width,
conf.exists_true("camera_grayscale"),
conf.exists_true("camera_rgb"));
}
#endif
#ifdef __KINECT__
else if (!strcmp(cam_type.c_str(), "kinect"))
cam = new camera_kinect<ubyte>(height, width);
#endif
else if (!strcmp(cam_type.c_str(), "shmem"))
cam = new camera_shmem<ubyte>("shared-mem", height, width);
else if (!strcmp(cam_type.c_str(), "video")) {
if (argc >= 3)
cam = new camera_video<ubyte>
(argv[2], height, width, conf.get_uint("input_video_sstep"),
conf.get_uint("input_video_max_duration"));
else eblerror("expected 2nd argument");
} else if (!strcmp(cam_type.c_str(), "datasource")) {
cam = new camera_datasource<ubyte,int>(conf);
} else eblerror("unknown camera type, set \"camera\" in your .conf");
// a camera directory may be used first, then switching to regular cam
if (conf.exists_true("precamera"))
cam2 = new camera_directory<ubyte>(conf.get_cstring("precamdir"),
height, width, input_random,
npasses, mout, merr, fpattern);
if (conf.exists_true("camera_grayscale")) cam->set_grayscale();
if (conf.exists_true("silent")) cam->set_silent();
// answer variables & initializations
bboxes bb;
// gui
#ifdef __GUI__
bool bkey_msg = false; // display key message
bool display = conf.exists_bool("display");
bool show_parts = conf.exists_true("show_parts");
bool bbox_show_conf = !conf.exists_false("bbox_show_conf");
bool bbox_show_class = !conf.exists_false("bbox_show_class");
// mindisplay = conf.exists_bool("minimal_display");
uint display_sleep = 0;
if (conf.exists("display_sleep"))
display_sleep = conf.get_uint("display_sleep");
// display_states = conf.exists_bool("display_states");
// uint qstep1 = 0, qheight1 = 0, qwidth1 = 0,
// qheight2 = 0, qwidth2 = 0, qstep2 = 0;
// if (conf.exists_bool("queue1")) {
// qstep1 = conf.get_uint("qstep1");
// qheight1 = conf.get_uint("qheight1");
// qwidth1 = conf.get_uint("qwidth1"); }
// if (conf.exists_bool("queue2")) {
// qstep2 = conf.get_uint("qstep2");
// qheight2 = conf.get_uint("qheight2");
// qwidth2 = conf.get_uint("qwidth2"); }
// wid_states = display_states ? new_window("network states"):0;
// night_mode();
uint wid = display ? new_window("eblearn object recognition") : 0;
night_mode();
float display_transp = 0.0;
if (conf.exists("display_bb_transparency"))
display_transp = conf.get_float("display_bb_transparency");
detector_gui<t_net> dgui(conf.exists_true("show_extracted"));
#endif
// timing variables
timer tpass, toverall, tstop;
uint cnt = 0;
bool stop = false, finished = false;
// loop
toverall.start();
while (!finished) {
// check for results and send new image for each thread
uint i = 0;
finished = true;
for (ithreads = threads.begin();
ithreads != threads.end(); ++ithreads, ++i) {
// do nothing if thread is finished already
if ((*ithreads)->finished()) continue ;
finished = false; // a thread is not finished
string processed_fname;
uint processed_id = 0;
// retrieve new data if present
bool skipped = false;
updated = (*ithreads)->get_data
(bb, detframe, *(total_saved.idx_ptr() + i), processed_fname,
&processed_id, &samples, &bbsamples, &skipped);
if (skipped) cnt++; // a new skipped frame was received
// save bounding boxes
if (updated) {
idxdim d(detframe);
if (boot.activated()) bb.clear();
if (bbsaving != bbox_none) {
if (!silent)
mout << "Adding " << bb.size() << " boxes into new group: "
<< processed_fname << " with id " << processed_id << endl;
boxes.new_group(d, &processed_fname, processed_id);
boxes.add(bb, d, &processed_fname, processed_id);
if (cnt % save_bbox_period == 0) boxes.save();
// avoid sample accumulation if not using bootstrapping
if (boot.activated())
mout << "Received " << samples.size()
<< " bootstrapping samples." << endl;
}
if (conf.exists_true("bootstrapping_save")) {
all_samples.push_back_new(samples);
all_bbsamples.push_back_new(bbsamples);
}
// datasource mode, check and log answers
if (dynamic_cast<camera_datasource<ubyte,int>*>(cam)) {
camera_datasource<ubyte,int>* dscam =
(camera_datasource<ubyte,int>*) cam;
dscam->log_answers(bb);
}
cnt++;
// display processed frame
#ifdef __GUI__
if (display) {
select_window(wid);
disable_window_updates();
clear_resize_window();
set_window_title(processed_fname.c_str());
uint h = 0, w = 0;
// display frame with resulting boxes
dgui.display_minimal
(detframe, bb, ((*ithreads)->pdetect ?
(*ithreads)->pdetect->get_labels() : sclasses),
h, w, 1, 0, 255, wid, show_parts, display_transp,
bbox_show_class, bbox_show_conf, &bbsamples);
// display extracted samples
if (boot.activated()) {
dgui.display_preprocessed
(samples, bbsamples, ((*ithreads)->pdetect ?
(*ithreads)->pdetect->get_labels() : sclasses),
h, w, 1, -1, 1);
}
enable_window_updates();
if (save_video && display) {
string fname;
fname << viddir << processed_fname;
save_window(fname.c_str());
if (!silent) mout << "saved " << fname << endl;
}
}
// sleep display
if (display_sleep > 0) {
mout << "sleeping for " << display_sleep << "ms." << endl;
millisleep(display_sleep);
}
#endif
if (!silent) {
// output info
uint k = cnt, tot = cam->size() - cnt; // progress variables
if (conf.exists("save_max")) tot = conf.get_uint("save_max");
if (!silent) {
if (save_detections) {
mout << "total_saved=" << idx_sum(total_saved);
if (conf.exists("save_max")) mout << " / " << tot;
mout << endl;
}
}
if (boot.activated())
mout << "total_bootstrapping=" << all_samples.size() << endl;
mout << "remaining=" << (cam->size() - cnt)
<< " elapsed=" << toverall.elapsed();
if (cam->size() > 0)
mout << " ETA=" << toverall.eta(cnt, cam->size());
if (conf.exists("save_max") && save_detections) {
k = idx_sum(total_saved);
mout << " save_max_ETA=" << toverall.eta(k, tot);
}
mout << endl;
mout << "i=" << cnt << " processing: " << tpass.elapsed_ms()
<< " fps: " << cam->fps() << endl;
// save progress
if (!conf.exists_false("save_progress"))
job::write_progress(k, tot);
}
}
// check if ready
if ((*ithreads)->available()) {
if (stop)
(*ithreads)->ask_stop(); // stop but let thread finish
else {
// grab a new frame if available
if (cam->empty()) {
stop = true;
tstop.start(); // start countdown timer
(*ithreads)->ask_stop(); // ask this thread to stop
millisleep(50);
} else {
#ifdef __GUI__
int key = gui.pop_key_pressed();
// if thread has already received data, wait for next key
if ((*ithreads)->fed() && next_on_key) {
if ((int)next_on_key != key && (int)next_on_key != key + 32) {
if (!bkey_msg)
mout << "Press " << next_on_key
<< " to process next frame." << endl;
bkey_msg = true;
continue ; // pause until key is pressed
} else {
mout << "Key pressed (" << key
<< ") allowing next frame to process." << endl;
bkey_msg = false;
tpass.restart();
}
}
#endif
bool frame_grabbed = false;
frame_id = cam->frame_id();
// if the pre-camera is defined use it until empty
if (cam2 && !cam2->empty())
frame = cam2->grab();
else { // empty pre-camera, use regular camera
if (skip_frames > 0)
cam->skip(skip_frames); // skip frames if skip_frames > 0
if (cam->empty()) continue ;
if (precomputed_boxes && !save_video)
cam->next(); // move to next frame but without grabbing
else if (dynamic_cast<camera_directory<ubyte>*>(cam)) {
cam->grab_filename(); // just get the filename, no data
} else { // actually grab the frame
frame = cam->grab();
frame_grabbed = true;
// cropping
if (crop.nelements() > crop.order()) {
cout << "cropping frame from " << frame;
for (uint i = 0; i < crop.order(); ++i)
if (crop.dim(i) > 1)
frame = frame.narrow(i, crop.dim(i), 0);
cout << " to " << frame << endl;
}
}
}
// send new frame to this thread
string ffname = cam->frame_fullname();
string fname = cam->frame_name();
if (frame_grabbed) {
while (!(*ithreads)->set_data(frame, ffname, fname, frame_id))
millisleep(5);
} else {
while (!(*ithreads)->set_data(ffname, fname, frame_id))
millisleep(5);
}
// we just sent a new frame
tpass.restart();
}
}
}
}
if ((conf.exists("save_max") && !stop &&
idx_sum(total_saved) > conf.get_uint("save_max"))
|| (boot.activated()
&& (intg) all_samples.size() > boot.max_size())) {
mout << "Reached max number of detections, exiting." << endl;
stop = true; // limit number of detection saves
tstop.start(); // start countdown timer
}
// sleep a bit between each iteration
millisleep(5);
// check if stop countdown reached 0
if (stop && tstop.elapsed_minutes() >= 20) {
cerr << "threads did not all return 20 min after request, stopping"
<< endl;
break ; // program too long to stop, force exit
}
}
// saving boxes
if (bbsaving != bbox_none) boxes.save();
mout << "Execution time: " << toverall.elapsed() << endl;
if (save_video)
cam->stop_recording(conf.exists_bool("use_original_fps") ?
cam->fps() : conf.get_uint("save_video_fps"),
outdir.c_str());
// saving bootstrapping
if (conf.exists_true("bootstrapping_save") && boot.activated())
boot.save_dataset(all_samples, all_bbsamples, outdir, classes);
// free variables
if (cam) delete cam;
for (ithreads = threads.begin(); ithreads != threads.end(); ++ithreads) {
if (!(*ithreads)->finished())
(*ithreads)->stop(); // stop thread without waiting
delete *ithreads;
}
#ifdef __GUI__
if (!conf.exists_true("no_gui_quit") && !conf.exists("next_on_key")) {
mout << "Closing windows..." << endl;
quit_gui(); // close all windows
mout << "Windows closed." << endl;
}
#endif
job::write_finished(); // declare job finished
mout << "Detection finished." << endl;
// evaluation of bbox
if (conf.exists_true("evaluate") && conf.exists("evaluate_cmd")) {
string cmd;
cmd << "cd " << outdir << " && " << conf.get_string("evaluate_cmd");
int res = std::system(cmd.c_str());
if (res != 0)
cerr << "bbox evaluation failed with command " << cmd << endl;
}
} eblcatcherror();
return 0;
}
