void tracking_thread<Tnet>::draw(bbox *b) { #ifdef __GUI__ disable_window_updates(); clear_window(); draw_matrix(frame, "in"); draw_matrix(tpl, "tpl", 0, frame.dim(1)); if (b) { ubyte red = 0, green = 0, blue = 0; if (b->class_id == -42) green = 255; else blue = 255; draw_box(b->h0, b->w0, b->width, b->height, red, green, blue); } enable_window_updates(); #endif }
int display_net(list<string>::iterator &ifname, bool signd, bool load, list<string> *mats) { // create network idx<ubyte> classes(1,1); load_matrix<ubyte>(classes, conf->get_cstring("classes")); answer_module<T,T,T> *answer = create_answer<T,T,T>(*conf, classes.dim(0)); if (!answer) eblerror("no answer module found"); uint noutputs = answer->get_nfeatures(); parameter<T> theparam; intg thick; module_1_1<T> *net = create_network<T>(theparam, *conf, thick, noutputs); cout << "Network parameters: " << theparam << endl; vector<string> weights = string_to_stringvector(conf->get_string("weights_file")); // loading weights theparam.load_x(weights); // displaying internals uint h0 = 0, w0 = 0; #ifdef __GUI__ disable_window_updates(); clear_window(); if (show_filename) { gui << at(h0, w0) << black_on_white() << ifname->c_str(); h0 += 16; } if (autorange) // automatic range lg.display_internals(*net, h0, w0, zoom, (T) 0, (T) 0, maxwidth); else // fixed range lg.display_internals(*net, h0, w0, zoom, (T) range[0], (T) range[1], maxwidth); // update title string title; title << "matshow: " << ebl::basename(ifname->c_str()); set_window_title(title.c_str()); enable_window_updates(); #endif return 1; }
void detection_thread<T>::execute() { try { bool display = false; #ifdef __GUI__ display = conf.exists_true("display") && conf.exists_true("display_threads"); bool mindisplay = conf.exists_true("minimal_display"); bool save_video = conf.exists_true("save_video"); bool display_states = conf.exists_true("display_states"); uint wid = 0; // window id uint wid_states = 0; // window id #endif uint display_sleep = conf.try_get_uint("display_sleep", 0); // if (!display && save_video) { // // we still want to output images but not show them // display = true; // #ifdef __GUI__ // set_gui_silent(); // #endif // } // load network and weights in a forward-only parameter parameter<T> theparam; theparam.set_forward_only(); idx<ubyte> classes(1,1); //try { // try loading classes names but do not stop upon failure load_matrix<ubyte>(classes, conf.get_cstring("classes")); // } catch(std::string &err) { // merr << "warning: " << err; // merr << std::endl; // } std::vector<std::string> sclasses = ubyteidx_to_stringvector(classes); answer_module<T> *ans = create_answer<T,T,T>(conf, classes.dim(0)); uint noutputs = ans->get_nfeatures(); intg thick = -1; module_1_1<T> *net = create_network<T>(theparam, conf, thick, noutputs, "arch", this->_id); // loading weights if (conf.exists("weights")) { // manual weights // concatenate weights if multiple ones std::vector<std::string> w = string_to_stringvector(conf.get_string("weights")); mout << "Loading weights from: " << w << std::endl; theparam.load_x(w); // permute weights by blocks if (conf.exists("weights_permutation")) { std::string sblocks = conf.get_string("weights_blocks"); std::string spermut = conf.get_string("weights_permutation"); std::vector<intg> blocks = string_to_intgvector(sblocks.c_str()); std::vector<uint> permut = string_to_uintvector(spermut.c_str()); theparam.permute_x(blocks, permut); } } else { if (conf.exists_true("manual_load")) { // manual load eblwarn("\"weights\" variable not defined, loading manually " << "if manual_load defined"); manually_load_network(*((layers<T>*)net), conf); } else { // random weights int seed = dynamic_init_drand(); eblwarn("No weights to load, randomizing weights with seed " << seed); forget_param_linear fgp(1, 0.5, seed); net->forget(fgp); } } DEBUGMEM_PRETTY("before detection"); // detector detector<T> detect(*net, sclasses, ans, NULL, NULL, mout, merr); init_detector(detect, conf, outdir, silent); // keep pointer to detector pdetect = &detect; bootstrapping<T> boot(conf); // when a bbox file is given, ignore the processing, load the pre-computed // bboxes and feed them to the nms (non-maximum suppression). bboxes boxes(bbox_all, NULL, mout, merr); boxes.print_saving_type(); // inform user how we save boxes bool precomputed_boxes = false; if (conf.exists("bbox_file")) { precomputed_boxes = true; std::string bbfile = conf.get_string("bbox_file"); boxes.load_eblearn(bbfile); } bool bmask_class = false; if (conf.exists("mask_class")) bmask_class = detect.set_mask_class(conf.get_cstring("mask_class")); std::string viddir = outdir; viddir += "video/"; mkdir_full(viddir); // gui #ifdef __GUI__ uint display_wmax = conf.try_get_uint("display_max_width", 3000); T display_min = (T) conf.try_get_double("display_min", -1.7); T display_max = (T) conf.try_get_double("display_max", 1.7); T display_in_max = (T) conf.try_get_double("display_in_max", 255); T display_in_min = (T) conf.try_get_double("display_in_min", 0); float display_transp = conf.try_get_float("display_bb_transparency", 0); uint qstep1 = conf.try_get_uint("qstep1", 0); uint qheight1 = conf.try_get_uint("qheight1", 0); uint qwidth1 = conf.try_get_uint("qwidth1", 0); uint qstep2 = conf.try_get_uint("qstep2", 0); uint qheight2 = conf.try_get_uint("qheight2", 0); uint qwidth2 = conf.try_get_uint("qwidth2", 0); module_1_1_gui netgui; wid_states = display_states ? new_window("network states"):0; night_mode(); std::string title = "EBLearn detector: "; title += _name; if (display) { wid = new_window(title.c_str()); mout << "displaying in window " << wid << std::endl; night_mode(); } float zoom = conf.try_get_float("display_zoom", 1); bool bbox_show_conf = !conf.exists_false("bbox_show_conf"); bool bbox_show_class = !conf.exists_false("bbox_show_class"); detector_gui<T> dgui(conf.try_get_uint("show_extracted", 0), conf.exists_bool("queue1"), qstep1, qheight1, qwidth1, conf.exists_bool("queue2"), qstep2, qheight2, qwidth2, bbox_show_class, bbox_show_conf); if (bmask_class) dgui.set_mask_class(conf.get_cstring("mask_class"), (T) conf.try_get_double("mask_threshold", 0)); #endif // timing variables timer tpass, toverall; long ms; // loop toverall.start(); // we're ready bavailable = true; while(!this->_stop) { // wait until a new image is made available while (!in_updated && !_stop) { millisleep(1); } tpass.restart(); if (_stop) break ; // we got a new frame, reset new frame flag in_updated = false; // no need to lock mutex // check if this frame should be skipped if (boot.skip_frame(frame_name)) { skip_frame(); continue ; } else if (!frame_loaded) { uframe = load_image<ubyte>(frame_fullname); mout << "loaded image " << frame_fullname << std::endl; } if (!silent) mout << "processing " << frame_name << std::endl; // check frame is correctly allocated, if not, allocate. if (frame.order() != uframe.order()) frame = idx<T>(uframe.get_idxdim()); else if (frame.get_idxdim() != uframe.get_idxdim()) frame.resize(uframe.get_idxdim()); // copy frame idx_copy(uframe, frame); // run detector if (!display) { // fprop without display if (precomputed_boxes) { try { bboxes *bb = boxes.get_group(frame_name); idxdim d = boxes.get_group_dims(frame_name); d.insert_dim(0, 1); bboxes pruned; detect.init(d); detect.fprop_nms(*bb, pruned); copy_bboxes(pruned); // make a copy of bounding boxes // resize frame so that caller knows the size of the frame idxdim framedim = frame.get_idxdim(); if (d.dim(1) == -1 || d.dim(2) == -1) eblerror("pre-computed boxes must contain full image size, " << "but found: " << d); framedim.setdim(0, d.dim(1)); framedim.setdim(1, d.dim(2)); frame.resize(framedim); } catch(eblexception &e) { #ifdef __NOEXCEPTIONS__ merr << "exception" << std::endl; #else merr << e << std::endl; #endif } } else { try { mout << "starting processing of frame " << frame_name << std::endl; bboxes &bb = detect.fprop(frame, frame_name.c_str(), frame_id); copy_bboxes(bb); // make a copy of bounding boxes } catch(ebl::eblexception &e) { // detection failed #ifdef __NOEXCEPTIONS__ eblwarn("detection failed"); #else eblwarn("detection failed: " << e); #endif clear_bboxes(); } } } #ifdef __GUI__ else { // fprop and display if (precomputed_boxes) eblerror("not implemented for nms only (TODO)"); disable_window_updates(); select_window(wid); clear_window(); std::string title = _name; title << ": " << frame_name; set_window_title(title.c_str()); // clear_resize_window(); try { if (mindisplay) { bboxes &bb = dgui.display(detect, frame, frame_name.c_str(), frame_id, 0, 0, zoom, display_min, display_max, wid, _name.c_str(), display_transp); copy_bboxes(bb); // make a copy of bounding boxes } else { // extract & display boxes bboxes &bb = dgui.display_inputs_outputs(detect, frame, frame_name.c_str(), frame_id, 0, 0, zoom, display_min, display_max, wid, _name.c_str(), display_in_min, display_in_max, display_transp, display_wmax); // make a copy of bounding boxes copy_bboxes(bb); } } catch(ebl::eblexception &e) { // detection failed eblwarn("detection failed: " << e); clear_bboxes(); } enable_window_updates(); } if (display_states) { dgui.display_current(detect, frame, wid_states, NULL, zoom); select_window(wid); } if (save_video && display) { std::string fname = viddir; fname += frame_name; save_window(fname.c_str()); if (!silent) mout << "saved " << fname << std::endl; } #endif if (!silent) mout << "processing done for frame " << frame_name << std::endl; // bootstrapping if (conf.exists_true("bootstrapping")) { boot.fprop(detect, frame_name); // add multiple scales if positives and scales are defined if (conf.exists("gt_scales") && boot.extract_positives()) { std::vector<double> scales = string_to_doublevector(conf.get_cstring("gt_scales")); for (uint s = 0; s < scales.size(); ++s) { double f = scales[s]; // downsample input by f detect.set_resolution(f); detect.init(frame.get_idxdim(), frame_name.c_str()); detect.fprop(frame, frame_name.c_str(), frame_id); boot.fprop(detect, frame_name, false, f); } detect.set_scaling_original(); detect.init(frame.get_idxdim(), frame_name.c_str()); } copy_bootstrapping(boot.get_all(), boot.get_bball()); #ifdef __GUI__ // display groundtruth if (conf.exists_true("display_bootstrapping")) dgui.display_groundtruth(detect, frame, boot.get_gtall(), boot.get_gtclean(), boot.get_gtrest(), boot.get_bbpos(), boot.get_bbneg(), boot.get_pos(), boot.get_neg(), 0, 0, zoom, display_min, display_max); #endif } total_saved = detect.get_total_saved(); ms = tpass.elapsed_milliseconds(); if (!silent) { mout << bbs.pretty_short(detect.get_labels()); mout << "processing=" << ms << " ms (" << tpass.elapsed() << ")" << std::endl; } DEBUGMEM_PRETTY("after detection"); // switch 'updated' flag on to warn we just added new data set_out_updated(); // display sleep if (display_sleep > 0) { mout << "sleeping for " << display_sleep << "ms." << std::endl; millisleep(display_sleep); } if (conf.exists("save_max") && detect.get_total_saved() > conf.get_uint("save_max")) break ; // limit number of detection saves } mout << "detection finished. Execution time: " << toverall.elapsed()<<std::endl; // free variables if (net) delete net; if (ans) delete ans; } eblcatcherror(); }
MAIN_QTHREAD(int, argc, char **, argv) { // macro to enable multithreaded gui #else int main(int argc, char **argv) { // regular main without gui #endif try { // check input parameters if ((argc != 2) && (argc != 3) ) { cerr << "wrong number of parameters." << endl; cerr << "usage: objdetect <config file> [directory or file]" << endl; return -1; } #ifdef __LINUX__ feenableexcept(FE_DIVBYZERO | FE_INVALID); // enable float exceptions #endif ipp_init(1); // limit IPP (if available) to 1 core // load configuration configuration conf(argv[1], true, true, false); if (!conf.exists("root2")) { string dir; dir << dirname(argv[1]) << "/"; cout << "Looking for trained files in: " << dir << endl; conf.set("root2", dir.c_str()); conf.set("current_dir", dir.c_str()); } conf.resolve(); if (conf.exists_true("show_conf")) conf.pretty(); bool color = conf.exists_bool("color"); uint norm_size = conf.get_uint("normalization_size"); Tnet threshold = (Tnet) conf.get_double("threshold"); bool display = false; bool display_states= false; bool mindisplay = false; uint display_sleep = 0; bool save_video = false; string cam_type = conf.get_string("camera"); int height = -1; int width = -1; if (conf.exists("input_height")) height = conf.get_int("input_height"); if (conf.exists("input_width")) width = conf.get_int("input_width"); bool input_random = conf.exists_true("input_random"); uint npasses = 1; char next_on_key = 0; uint wid = 0; // window id uint wid_states = 0; // window id string outdir = "out_"; if (conf.exists("next_on_key")) { next_on_key = conf.get_char("next_on_key"); cout << "Press " << next_on_key << " to process next frame." << endl; } outdir += tstamp(); outdir += "/"; cout << "Saving outputs to " << outdir << endl; // load network and weights parameter<fs(Tnet)> theparam; idx<ubyte> classes(1,1); try { load_matrix<ubyte>(classes, conf.get_cstring("classes")); } catch(string &err) { cerr << "warning: " << err << endl; } vector<string> sclasses = ubyteidx_to_stringvector(classes); answer_module<SFUNC2(Tnet)> *ans = create_answer<SFUNC2(Tnet)>(conf, classes.dim(0)); uint noutputs = ans->get_nfeatures(); module_1_1<SFUNC(Tnet)> *net = create_network<SFUNC(Tnet)>(theparam, conf, noutputs); // loading weights if (!conf.exists("weights")) { // manual weights cerr << "warning: \"weights\" variable not defined, loading manually " << "if manual_load defined" << endl; if (conf.exists_true("manual_load")) manually_load_network(*((layers<SFUNC(Tnet)>*)net), conf); } else { // multiple-file weights // concatenate weights if multiple ones vector<string> w = string_to_stringvector(conf.get_string("weights")); cout << "Loading weights from: " << w << endl; theparam.load_x(w); } // detector detector<fs(Tnet)> detect(*net, sclasses, *ans, NULL, NULL); // multi-scaling parameters double maxs = conf.exists("max_scale")?conf.get_double("max_scale") : 1.0; double mins = conf.exists("min_scale")?conf.get_double("min_scale") : 1.0; t_scaling scaling_type = SCALES_STEP; vector<idxdim> scales; if (conf.exists("scaling_type")) scaling_type = (t_scaling) conf.get_uint("scaling_type"); switch (scaling_type) { case MANUAL: if (!conf.exists("scales")) eblerror("expected \"scales\" variable to be defined in manual mode"); scales = string_to_idxdimvector(conf.get_cstring("scales")); detect.set_resolutions(scales); break ; case ORIGINAL: detect.set_scaling_original(); break ; case SCALES_STEP: detect.set_resolutions(conf.get_double("scaling"), maxs, mins); break ; case SCALES_STEP_UP: detect.set_resolutions(conf.get_double("scaling"), maxs, mins); detect.set_scaling_type(scaling_type); break ; default: detect.set_scaling_type(scaling_type); } // optimize memory usage by using only 2 buffers for entire flow SBUF<Tnet> input(1, 1, 1), output(1, 1, 1); if (!conf.exists_false("mem_optimization")) detect.set_mem_optimization(input, output, true); // zero padding float hzpad = 0, wzpad = 0; if (conf.exists("hzpad")) hzpad = conf.get_float("hzpad"); if (conf.exists("wzpad")) wzpad = conf.get_float("wzpad"); detect.set_zpads(hzpad, wzpad); bool bmask_class = false; if (conf.exists("mask_class")) bmask_class = detect.set_mask_class(conf.get_cstring("mask_class")); if (conf.exists("input_min")) // limit inputs size detect.set_min_resolution(conf.get_uint("input_min")); if (conf.exists("input_max")) // limit inputs size detect.set_max_resolution(conf.get_uint("input_max")); detect.set_silent(); if (conf.exists_bool("save_detections")) { string detdir = outdir; detdir += "detections"; detdir = detect.set_save(detdir); if (conf.exists("save_max_per_frame")) detect.set_save_max_per_frame(conf.get_uint("save_max_per_frame")); } // nms detect.set_cluster_nms(conf.exists_true("cluster_nms")); detect.set_scaler_mode(conf.exists_true("scaler_mode")); if (conf.exists("nms")) detect.set_pruning((t_pruning)conf.get_uint("nms"), conf.exists("min_hcenter_dist") ? conf.get_float("min_hcenter_dist") : 0.0, conf.exists("min_wcenter_dist") ? conf.get_float("min_wcenter_dist") : 0.0, conf.exists("bbox_max_overlap") ? conf.get_float("bbox_max_overlap") : 1.0, conf.exists_true("share_parts"), conf.exists("threshold2") ? (Tnet) conf.get_float("threshold2") : 0.0, conf.exists("bbox_max_center_dist") ? conf.get_float("bbox_max_center_dist") : 0.0, conf.exists("bbox_max_center_dist2") ? conf.get_float("bbox_max_center_dist2") : 0.0, conf.exists("bbox_max_wcenter_dist") ? conf.get_float("bbox_max_wcenter_dist") : 0.0, conf.exists("bbox_max_wcenter_dist2") ? conf.get_float("bbox_max_wcenter_dist2") : 0.0, conf.exists("min_wcenter_dist2") ? conf.get_float("min_wcenter_dist2") : 0.0, conf.exists("bbox_max_overlap2") ? conf.get_float("bbox_max_overlap2") : 0.0, conf.exists_true("mean_bb"), conf.exists("same_scale_mhd") ? conf.get_float("same_scale_mhd") : 0.0, conf.exists("same_scale_mwd") ? conf.get_float("same_scale_mwd") : 0.0, conf.exists("min_scale_pred") ? conf.get_float("min_scale_pred") : 0.0, conf.exists("max_scale_pred") ? conf.get_float("max_scale_pred") : 0.0 ); if (conf.exists("bbox_hfactor") && conf.exists("bbox_wfactor")) detect.set_bbox_factors(conf.get_float("bbox_hfactor"), conf.get_float("bbox_wfactor"), conf.exists("bbox_woverh") ? conf.get_float("bbox_woverh") : 1.0, conf.exists("bbox_hfactor2") ? conf.get_float("bbox_hfactor2") : 1.0, conf.exists("bbox_wfactor2") ? conf.get_float("bbox_wfactor2") : 1.0); if (conf.exists("max_object_hratio")) detect.set_max_object_hratio(conf.get_double("max_object_hratio")); if (conf.exists("net_min_height") && conf.exists("net_min_width")) detect.set_min_input(conf.get_int("net_min_height"), conf.get_int("net_min_width")); if (conf.exists("smoothing")) detect.set_smoothing(conf.get_uint("smoothing")); if (conf.exists("background_name")) detect.set_bgclass(conf.get_cstring("background_name")); // image search can be configured with a search pattern const char *fpattern = IMAGE_PATTERN_MAT; if (conf.exists("file_pattern")) fpattern = conf.get_cstring("file_pattern"); // initialize camera (opencv, directory, shmem or video) idx<ubyte> frame; camera<ubyte> *cam = NULL, *cam2 = NULL; if (!strcmp(cam_type.c_str(), "directory")) { string dir; if (argc >= 3) // read input dir from command line dir = argv[2]; else if (conf.exists("input_dir")) dir = conf.get_string("input_dir"); // given list list<string> files; if (conf.exists("input_list")) { files = string_to_stringlist(conf.get_string("input_list")); cam = new camera_directory<ubyte>(dir.c_str(), height, width, input_random, npasses, std::cout, std::cerr, fpattern, &files); } else // given directory only cam = new camera_directory<ubyte>(dir.c_str(), height, width, input_random, npasses, std::cout, std::cerr, fpattern, &files); } else if (!strcmp(cam_type.c_str(), "opencv")) cam = new camera_opencv<ubyte>(-1, height, width); #ifdef __LINUX__ else if (!strcmp(cam_type.c_str(), "v4l2")) cam = new camera_v4l2<ubyte>(conf.get_cstring("device"), height, width, conf.exists_true("camera_grayscale")); #endif else if (!strcmp(cam_type.c_str(), "shmem")) cam = new camera_shmem<ubyte>("shared-mem", height, width); else if (!strcmp(cam_type.c_str(), "video")) { if (argc >= 3) cam = new camera_video<ubyte> (argv[2], height, width, conf.get_uint("input_video_sstep"), conf.get_uint("input_video_max_duration")); else eblerror("expected 2nd argument"); } else eblerror("unknown camera type"); // a camera directory may be used first, then switching to regular cam if (conf.exists_bool("precamera")) cam2 = new camera_directory<ubyte>(conf.get_cstring("precamdir"), height, width); // answer variables & initializations vector<bbox*> bboxes; vector<bbox*>::iterator ibboxes; ostringstream answer_fname; mkdir_full(outdir); answer_fname << outdir << "bbox.txt"; // open file ofstream fp(answer_fname.str().c_str()); if (!fp) { cerr << "failed to open " << answer_fname.str() << endl; eblerror("open failed"); } // gui #ifdef __GUI__ display = conf.exists_bool("display"); mindisplay = conf.exists_bool("minimal_display"); display_sleep = conf.get_uint("display_sleep"); display_states = conf.exists_bool("display_states"); save_video = conf.exists_bool("save_video"); uint qstep1 = 0, qheight1 = 0, qwidth1 = 0, qheight2 = 0, qwidth2 = 0, qstep2 = 0; if (conf.exists_bool("queue1")) { qstep1 = conf.get_uint("qstep1"); qheight1 = conf.get_uint("qheight1"); qwidth1 = conf.get_uint("qwidth1"); } if (conf.exists_bool("queue2")) { qstep2 = conf.get_uint("qstep2"); qheight2 = conf.get_uint("qheight2"); qwidth2 = conf.get_uint("qwidth2"); } module_1_1_gui netgui; wid_states = display_states ? new_window("network states"):0; night_mode(); wid = display ? new_window("eblearn object recognition") : 0; night_mode(); float zoom = 1; detector_gui<fs(Tnet)> dgui(conf.exists_bool("queue1"), qstep1, qheight1, qwidth1, conf.exists_bool("queue2"), qstep2, qheight2, qwidth2); if (bmask_class) dgui.set_mask_class(conf.get_cstring("mask_class"), (Tnet) conf.get_double("mask_threshold")); if (save_video) { string viddir = outdir; viddir += "video"; cam->start_recording(wid, viddir.c_str()); } #endif // timing variables timer tpass, toverall; long ms; // loop toverall.start(); while(!cam->empty()) { // get a new frame tpass.restart(); // if the pre-camera is defined use it until empty if (cam2 && !cam2->empty()) frame = cam2->grab(); else // empty pre-camera, use regular camera frame = cam->grab(); string frame_name = cam->frame_name(); // run detector if (!display) { // fprop without display bboxes = detect.fprop(frame, threshold, frame_name.c_str()); } #ifdef __GUI__ else { // fprop and display disable_window_updates(); clear_window(); if (mindisplay) bboxes = dgui.display(detect, frame, threshold, frame_name.c_str(), 0, 0, zoom, (Tnet)0, (Tnet)255, wid); else bboxes = dgui.display_inputs_outputs(detect, frame, threshold, frame_name.c_str(), 0, 0, zoom, (Tnet)-1.1, (Tnet)1.1, wid); enable_window_updates(); if (display_states) { dgui.display_current(detect, frame, wid_states); select_window(wid); } if (save_video) cam->record_frame(); } #endif ms = tpass.elapsed_milliseconds(); cout << "processing: " << ms << " ms." << endl; cout << "fps: " << cam->fps() << endl; // save bounding boxes for (ibboxes = bboxes.begin(); ibboxes != bboxes.end(); ++ibboxes) { fp << cam->frame_name() << " " << (*ibboxes)->class_id << " " << (*ibboxes)->confidence << " "; fp << (*ibboxes)->w0 << " " << (*ibboxes)->h0 << " "; fp << (*ibboxes)->w0 + (*ibboxes)->width << " "; fp << (*ibboxes)->h0 + (*ibboxes)->height << endl; } // sleep display if (display_sleep > 0) { cout << "sleeping for " << display_sleep << "ms." << endl; millisleep(display_sleep); } if (conf.exists("save_max") && detect.get_total_saved() > conf.get_uint("save_max")) { cout << "Reached max number of detections, exiting." << endl; break ; // limit number of detection saves } } cout << "Execution time: " << toverall.elapsed_minutes() <<" mins" <<endl; if (save_video) cam->stop_recording(conf.exists_bool("use_original_fps") ? cam->fps() : conf.get_uint("save_video_fps")); // free variables if (net) delete net; if (cam) delete cam; // close files fp.close(); #ifdef __GUI__ if (!conf.exists_true("no_gui_quit")) { cout << "Closing windows..." << endl; quit_gui(); // close all windows cout << "Windows closed." << endl; } #endif } eblcatcherror(); return 0; }
int display(list<string>::iterator &ifname, bool signd, bool load, list<string> *mats) { //cout << "displaying " << ifname->c_str() << endl; // conf mode if (conf) return display_net<T>(ifname, signd, load, mats); // mat mode if (is_matrix(ifname->c_str())) { idxdim d = get_matrix_dims(ifname->c_str()); if (interleaved) d.shift_dim(0, 2); if (save_individually || print || !(d.order() == 2 || (d.order() == 3 && (d.dim(2) == 1 || d.dim(2) == 3)))) { // this is probably not an image, just display info and print matrix string type; get_matrix_type(ifname->c_str(), type); idx<T> m = load_matrix<T>(ifname->c_str()); cout << "Matrix " << ifname->c_str() << " is of type " << type << " with dimensions " << d << " (min " << idx_min(m) << ", max " << idx_max(m) << ", mean " << idx_mean(m) << "):" << endl; m.print(); if (has_multiple_matrices(ifname->c_str())) { midx<T> ms = load_matrices<T>(ifname->c_str(), true); // saving sub-matrices if (save_individually) { cout << "Saving each sub-matrix of " << *ifname << " individually..." << endl; save_matrices_individually(ms, *ifname, true); } // printing sub-matrices cout << "This file contains " << m << " matrices: "; if (ms.order() == 1) { for (intg i = 0; i < ms.dim(0); ++i) { idx<T> tmp = ms.mget(i); cout << tmp.info() << " "; } } else if (ms.order() == 2) { for (intg i = 0; i < ms.dim(0); ++i) { for (intg j = 0; j < ms.dim(1); ++j) { idx<T> tmp = ms.mget(i, j); cout << tmp.info() << " "; } cout << endl; } } cout << endl; } else cout << "This is a single-matrix file." << endl; return 0; } } // image mode int loaded = 0; static idx<T> mat; uint h = 0, w = 0, rowh = 0, maxh = 0; list<string>::iterator fname = ifname; #ifdef __GUI__ disable_window_updates(); clear_window(); if (show_filename) { gui << at(h, w) << black_on_white() << ifname->c_str(); h += 16; } #endif maxh = h; for (uint i = 0; i < nh; ++i) { rowh = maxh; for (uint j = 0; j < nw; ++j) { if (fname == mats->end()) fname = mats->begin(); try { // if (load) mat = load_image<T>(*fname); if (print) cout << *fname << ": " << mat << endl << mat.str() << endl; // show only some channels if (chans >= 0) mat = mat.select(2, chans); loaded++; maxh = (std::max)(maxh, (uint) (rowh + mat.dim(0))); T min = 0, max = 0; #ifdef __GUI__ if (autorange || signd) { if (autorange) { min = idx_min(mat); max = idx_max(mat); } else if (signd) { T matmin = idx_min(mat); if ((double)matmin < 0) { min = -1; max = -1; } } draw_matrix(mat, rowh, w, zoom, zoom, min, max); } else draw_matrix(mat, rowh, w, zoom, zoom, (T) range[0], (T) range[1]); #endif w += mat.dim(1) + 1; } catch(string &err) { ERROR_MSG(err.c_str()); } fname++; if (fname == ifname) break ; } if (fname == ifname) break ; maxh++; w = 0; } #ifdef __GUI__ // info if (show_info) { set_text_colors(0, 0, 0, 255, 255, 255, 255, 200); gui << mat; gui << at(15, 0) << *fname; gui << at(29, 0) << "min: " << idx_min(mat) << " max: " << idx_max(mat); } // help if (show_help) { h = 0; w = 0; uint hstep = 14; set_text_colors(0, 0, 255, 255, 255, 255, 255, 200); gui << at(h, w) << "Controls:"; h += hstep; set_text_colors(0, 0, 0, 255, 255, 255, 255, 200); gui << at(h, w) << "Right/Space: next image"; h += hstep; gui << at(h, w) << "Left/Backspace: previous image"; h += hstep; gui << at(h, w) << "i: image info"; h += hstep; gui << at(h, w) << "a: auto-range (use min and max as range)"; h += hstep; gui << at(h, w) << "x/z: show more/less images on width axis"; h += hstep; gui << at(h, w) << "y/t: show more/less images on height axis"; h += hstep; gui << at(h, w) << "0,1,2: show channel 0, 1 or 2 only"; h += hstep; gui << at(h, w) << "9: show alls channels"; h += hstep; gui << at(h, w) << "h: help"; } // update title string title; title << "matshow: " << ebl::basename(ifname->c_str()); set_window_title(title.c_str()); enable_window_updates(); #endif return loaded; }
MAIN_QTHREAD(int, argc, char **, argv) { // macro to enable multithreaded gui #else int main(int argc, char **argv) { // regular main without gui #endif try { // check input parameters if ((argc != 2) && (argc != 3) ) { cerr << "warning: wrong number of parameters." << endl; cerr << "usage: detect <config file> [directory or file]" << endl; // return -1; } #ifdef __LINUX__ feenableexcept(FE_DIVBYZERO | FE_INVALID); // enable float exceptions #endif // load configuration configuration conf(argv[1], true, true, false); if (conf.exists_true("fixed_randomization")) cout << "Using fixed seed: " << fixed_init_drand() << endl; else cout << "Using random seed: " << dynamic_init_drand(argc, argv) << endl; if (!conf.exists("root2") || !conf.exists("current_dir")) { string dir; dir << dirname(argv[1]) << "/"; cout << "Looking for trained files in: " << dir << endl; conf.set("root2", dir.c_str()); conf.set("current_dir", dir.c_str()); } conf.set("run_type", "detect"); // tell conf that we are in detect mode conf.resolve(); // manual call to resolving variable bool silent = conf.exists_true("silent"); if (conf.exists_true("show_conf") && !silent) conf.pretty(); // output synchronization bool sync = conf.exists_true("sync_outputs"); mutex out_mutex; mutex_ostream mutout(std::cout, &out_mutex, "Thread M"); mutex_ostream muterr(std::cerr, &out_mutex, "Thread M"); ostream &mout = sync ? mutout : cout; ostream &merr = sync ? muterr : cerr; bootstrapping<t_net> boot(conf); // output dir string outdir = detection_thread<t_net>::get_output_directory(conf); mout << "Saving outputs to " << outdir << endl; // save conf to output dir string cname = outdir; cname << filename(argv[1]); if (conf.write(cname.c_str())) mout << "Wrote configuration to " << cname << endl; // load classes of network idx<ubyte> classes(1,1); vector<string> sclasses; try { // try loading classes names but do not stop upon failure load_matrix<ubyte>(classes, conf.get_cstring("classes")); } catch(string &err) { merr << "warning: " << err << endl; } sclasses = ubyteidx_to_stringvector(classes); t_bbox_saving bbsaving = bbox_none; if (conf.exists("bbox_saving")) bbsaving = (t_bbox_saving) conf.get_int("bbox_saving"); bboxes boxes(bbsaving, &outdir, mout, merr); uint ipp_cores = 1; if (conf.exists("ipp_cores")) ipp_cores = conf.get_uint("ipp_cores"); ipp_init(ipp_cores); // limit IPP (if available) to 1 core bool save_video = conf.exists_true("save_video"); bool save_detections = conf.exists_true("save_detections"); int height = -1; int width = -1; if (conf.exists("input_height")) height = conf.get_int("input_height"); if (conf.exists("input_width")) width = conf.get_int("input_width"); bool input_random = conf.exists_true("input_random"); uint npasses = 1; char next_on_key = 0; if (conf.exists("next_on_key")) { next_on_key = conf.get_char("next_on_key"); mout << "Press " << next_on_key << " to process next frame." << endl; } uint skip_frames = conf.try_get_uint("skip_frames", 0); if (conf.exists("input_npasses")) npasses = conf.get_uint("input_npasses"); string viddir; if (save_video) { viddir << outdir << "video/"; mkdir_full(viddir); } bool precomputed_boxes = conf.exists("bbox_file"); uint save_bbox_period = conf.try_get_uint("save_bbox_period", 500); idxdim crop(1, 1, 1); if (conf.exists("input_crop")) crop = string_to_idxdim(conf.get_string("input_crop")); string cam_type; #ifdef __LINUX__ // default camera for linux if not defined cam_type = "v4l2"; #endif if (conf.exists("camera")) cam_type = conf.get_string("camera"); // allocate threads uint nthreads = 1; bool updated = false; idx<ubyte> detframe; // frame returned by detection thread uint frame_id = 0; svector<midx<t_net> > all_samples, samples; // extracted samples bboxes all_bbsamples, bbsamples; // boxes corresponding to samples if (conf.exists("nthreads")) nthreads = (std::max)((uint) 1, conf.get_uint("nthreads")); list<detection_thread<t_net>*> threads; list<detection_thread<t_net>*>::iterator ithreads; idx<uint> total_saved(nthreads); idx_clear(total_saved); mout << "Initializing " << nthreads << " detection threads." << endl; for (uint i = 0; i < nthreads; ++i) { detection_thread<t_net> *dt = new detection_thread<t_net>(conf, &out_mutex, NULL, NULL, sync); threads.push_back(dt); dt->start(); } // image search can be configured with a search pattern const char *fpattern = IMAGE_PATTERN_MAT; if (conf.exists("file_pattern")) fpattern = conf.get_cstring("file_pattern"); // initialize camera (opencv, directory, shmem or video) idx<ubyte> frame(std::max(height, 1), std::max(width, 1), 3); camera<ubyte> *cam = NULL, *cam2 = NULL; if (!strcmp(cam_type.c_str(), "directory")) { string dir; if (argc >= 3) // read input dir from command line dir = argv[2]; else if (conf.exists("input_dir")) dir = conf.get_string("input_dir"); // given list list<string> files; if (conf.exists("input_list")) { files = string_to_stringlist(conf.get_string("input_list")); cam = new camera_directory<ubyte>(dir.c_str(), height, width, input_random, npasses, mout, merr, fpattern, &files); } else // given directory only cam = new camera_directory<ubyte>(dir.c_str(), height, width, input_random, npasses, mout, merr, fpattern, &files); } else if (!strcmp(cam_type.c_str(), "opencv")) cam = new camera_opencv<ubyte>(-1, height, width); #ifdef __LINUX__ else if (!strcmp(cam_type.c_str(), "v4l2")) cam = new camera_v4l2<ubyte>(conf.get_cstring("device"), height, width, conf.exists_true("camera_grayscale"), conf.exists_true("camera_rgb")); else if (!strcmp(cam_type.c_str(), "mac")) cam = new camera_mac<ubyte>(conf.get_cstring("device"), height, width, conf.exists_true("camera_grayscale"), conf.exists_true("camera_rgb")); else if (!strcmp(cam_type.c_str(), "mcams")) { vector<string> devices = conf.get_all_strings("device"); cam = new camera_mcams<ubyte>(conf, devices, height, width, conf.exists_true("camera_grayscale"), conf.exists_true("camera_rgb")); } #endif #ifdef __KINECT__ else if (!strcmp(cam_type.c_str(), "kinect")) cam = new camera_kinect<ubyte>(height, width); #endif else if (!strcmp(cam_type.c_str(), "shmem")) cam = new camera_shmem<ubyte>("shared-mem", height, width); else if (!strcmp(cam_type.c_str(), "video")) { if (argc >= 3) cam = new camera_video<ubyte> (argv[2], height, width, conf.get_uint("input_video_sstep"), conf.get_uint("input_video_max_duration")); else eblerror("expected 2nd argument"); } else if (!strcmp(cam_type.c_str(), "datasource")) { cam = new camera_datasource<ubyte,int>(conf); } else eblerror("unknown camera type, set \"camera\" in your .conf"); // a camera directory may be used first, then switching to regular cam if (conf.exists_true("precamera")) cam2 = new camera_directory<ubyte>(conf.get_cstring("precamdir"), height, width, input_random, npasses, mout, merr, fpattern); if (conf.exists_true("camera_grayscale")) cam->set_grayscale(); if (conf.exists_true("silent")) cam->set_silent(); // answer variables & initializations bboxes bb; // gui #ifdef __GUI__ bool bkey_msg = false; // display key message bool display = conf.exists_bool("display"); bool show_parts = conf.exists_true("show_parts"); bool bbox_show_conf = !conf.exists_false("bbox_show_conf"); bool bbox_show_class = !conf.exists_false("bbox_show_class"); // mindisplay = conf.exists_bool("minimal_display"); uint display_sleep = 0; if (conf.exists("display_sleep")) display_sleep = conf.get_uint("display_sleep"); // display_states = conf.exists_bool("display_states"); // uint qstep1 = 0, qheight1 = 0, qwidth1 = 0, // qheight2 = 0, qwidth2 = 0, qstep2 = 0; // if (conf.exists_bool("queue1")) { // qstep1 = conf.get_uint("qstep1"); // qheight1 = conf.get_uint("qheight1"); // qwidth1 = conf.get_uint("qwidth1"); } // if (conf.exists_bool("queue2")) { // qstep2 = conf.get_uint("qstep2"); // qheight2 = conf.get_uint("qheight2"); // qwidth2 = conf.get_uint("qwidth2"); } // wid_states = display_states ? new_window("network states"):0; // night_mode(); uint wid = display ? new_window("eblearn object recognition") : 0; night_mode(); float display_transp = 0.0; if (conf.exists("display_bb_transparency")) display_transp = conf.get_float("display_bb_transparency"); detector_gui<t_net> dgui(conf.exists_true("show_extracted")); #endif // timing variables timer tpass, toverall, tstop; uint cnt = 0; bool stop = false, finished = false; // loop toverall.start(); while (!finished) { // check for results and send new image for each thread uint i = 0; finished = true; for (ithreads = threads.begin(); ithreads != threads.end(); ++ithreads, ++i) { // do nothing if thread is finished already if ((*ithreads)->finished()) continue ; finished = false; // a thread is not finished string processed_fname; uint processed_id = 0; // retrieve new data if present bool skipped = false; updated = (*ithreads)->get_data (bb, detframe, *(total_saved.idx_ptr() + i), processed_fname, &processed_id, &samples, &bbsamples, &skipped); if (skipped) cnt++; // a new skipped frame was received // save bounding boxes if (updated) { idxdim d(detframe); if (boot.activated()) bb.clear(); if (bbsaving != bbox_none) { if (!silent) mout << "Adding " << bb.size() << " boxes into new group: " << processed_fname << " with id " << processed_id << endl; boxes.new_group(d, &processed_fname, processed_id); boxes.add(bb, d, &processed_fname, processed_id); if (cnt % save_bbox_period == 0) boxes.save(); // avoid sample accumulation if not using bootstrapping if (boot.activated()) mout << "Received " << samples.size() << " bootstrapping samples." << endl; } if (conf.exists_true("bootstrapping_save")) { all_samples.push_back_new(samples); all_bbsamples.push_back_new(bbsamples); } // datasource mode, check and log answers if (dynamic_cast<camera_datasource<ubyte,int>*>(cam)) { camera_datasource<ubyte,int>* dscam = (camera_datasource<ubyte,int>*) cam; dscam->log_answers(bb); } cnt++; // display processed frame #ifdef __GUI__ if (display) { select_window(wid); disable_window_updates(); clear_resize_window(); set_window_title(processed_fname.c_str()); uint h = 0, w = 0; // display frame with resulting boxes dgui.display_minimal (detframe, bb, ((*ithreads)->pdetect ? (*ithreads)->pdetect->get_labels() : sclasses), h, w, 1, 0, 255, wid, show_parts, display_transp, bbox_show_class, bbox_show_conf, &bbsamples); // display extracted samples if (boot.activated()) { dgui.display_preprocessed (samples, bbsamples, ((*ithreads)->pdetect ? (*ithreads)->pdetect->get_labels() : sclasses), h, w, 1, -1, 1); } enable_window_updates(); if (save_video && display) { string fname; fname << viddir << processed_fname; save_window(fname.c_str()); if (!silent) mout << "saved " << fname << endl; } } // sleep display if (display_sleep > 0) { mout << "sleeping for " << display_sleep << "ms." << endl; millisleep(display_sleep); } #endif if (!silent) { // output info uint k = cnt, tot = cam->size() - cnt; // progress variables if (conf.exists("save_max")) tot = conf.get_uint("save_max"); if (!silent) { if (save_detections) { mout << "total_saved=" << idx_sum(total_saved); if (conf.exists("save_max")) mout << " / " << tot; mout << endl; } } if (boot.activated()) mout << "total_bootstrapping=" << all_samples.size() << endl; mout << "remaining=" << (cam->size() - cnt) << " elapsed=" << toverall.elapsed(); if (cam->size() > 0) mout << " ETA=" << toverall.eta(cnt, cam->size()); if (conf.exists("save_max") && save_detections) { k = idx_sum(total_saved); mout << " save_max_ETA=" << toverall.eta(k, tot); } mout << endl; mout << "i=" << cnt << " processing: " << tpass.elapsed_ms() << " fps: " << cam->fps() << endl; // save progress if (!conf.exists_false("save_progress")) job::write_progress(k, tot); } } // check if ready if ((*ithreads)->available()) { if (stop) (*ithreads)->ask_stop(); // stop but let thread finish else { // grab a new frame if available if (cam->empty()) { stop = true; tstop.start(); // start countdown timer (*ithreads)->ask_stop(); // ask this thread to stop millisleep(50); } else { #ifdef __GUI__ int key = gui.pop_key_pressed(); // if thread has already received data, wait for next key if ((*ithreads)->fed() && next_on_key) { if ((int)next_on_key != key && (int)next_on_key != key + 32) { if (!bkey_msg) mout << "Press " << next_on_key << " to process next frame." << endl; bkey_msg = true; continue ; // pause until key is pressed } else { mout << "Key pressed (" << key << ") allowing next frame to process." << endl; bkey_msg = false; tpass.restart(); } } #endif bool frame_grabbed = false; frame_id = cam->frame_id(); // if the pre-camera is defined use it until empty if (cam2 && !cam2->empty()) frame = cam2->grab(); else { // empty pre-camera, use regular camera if (skip_frames > 0) cam->skip(skip_frames); // skip frames if skip_frames > 0 if (cam->empty()) continue ; if (precomputed_boxes && !save_video) cam->next(); // move to next frame but without grabbing else if (dynamic_cast<camera_directory<ubyte>*>(cam)) { cam->grab_filename(); // just get the filename, no data } else { // actually grab the frame frame = cam->grab(); frame_grabbed = true; // cropping if (crop.nelements() > crop.order()) { cout << "cropping frame from " << frame; for (uint i = 0; i < crop.order(); ++i) if (crop.dim(i) > 1) frame = frame.narrow(i, crop.dim(i), 0); cout << " to " << frame << endl; } } } // send new frame to this thread string ffname = cam->frame_fullname(); string fname = cam->frame_name(); if (frame_grabbed) { while (!(*ithreads)->set_data(frame, ffname, fname, frame_id)) millisleep(5); } else { while (!(*ithreads)->set_data(ffname, fname, frame_id)) millisleep(5); } // we just sent a new frame tpass.restart(); } } } } if ((conf.exists("save_max") && !stop && idx_sum(total_saved) > conf.get_uint("save_max")) || (boot.activated() && (intg) all_samples.size() > boot.max_size())) { mout << "Reached max number of detections, exiting." << endl; stop = true; // limit number of detection saves tstop.start(); // start countdown timer } // sleep a bit between each iteration millisleep(5); // check if stop countdown reached 0 if (stop && tstop.elapsed_minutes() >= 20) { cerr << "threads did not all return 20 min after request, stopping" << endl; break ; // program too long to stop, force exit } } // saving boxes if (bbsaving != bbox_none) boxes.save(); mout << "Execution time: " << toverall.elapsed() << endl; if (save_video) cam->stop_recording(conf.exists_bool("use_original_fps") ? cam->fps() : conf.get_uint("save_video_fps"), outdir.c_str()); // saving bootstrapping if (conf.exists_true("bootstrapping_save") && boot.activated()) boot.save_dataset(all_samples, all_bbsamples, outdir, classes); // free variables if (cam) delete cam; for (ithreads = threads.begin(); ithreads != threads.end(); ++ithreads) { if (!(*ithreads)->finished()) (*ithreads)->stop(); // stop thread without waiting delete *ithreads; } #ifdef __GUI__ if (!conf.exists_true("no_gui_quit") && !conf.exists("next_on_key")) { mout << "Closing windows..." << endl; quit_gui(); // close all windows mout << "Windows closed." << endl; } #endif job::write_finished(); // declare job finished mout << "Detection finished." << endl; // evaluation of bbox if (conf.exists_true("evaluate") && conf.exists("evaluate_cmd")) { string cmd; cmd << "cd " << outdir << " && " << conf.get_string("evaluate_cmd"); int res = std::system(cmd.c_str()); if (res != 0) cerr << "bbox evaluation failed with command " << cmd << endl; } } eblcatcherror(); return 0; }