Exemplo n.º 1
0
  void tracking_thread<Tnet>::execute() { 
    try {
      // configuration
      string    cam_type   = conf.get_string("camera");
      int       height     = conf.get_int("input_height");
      int       width      = conf.get_int("input_width");
      bool      save_video = conf.exists_bool("save_video");
      bool      display    = conf.exists_bool("tracking_display");
      // initialize camera (opencv, directory, shmem or video)
      camera<ubyte> *cam = NULL;
      if (!strcmp(cam_type.c_str(), "directory")) {
	if (arg2) cam = new camera_directory<ubyte>(arg2, height,width);
	else eblerror("expected 2nd argument");
      } else if (!strcmp(cam_type.c_str(), "opencv"))
	cam = new camera_opencv<ubyte>(-1, height, width);
#ifdef __LINUX__
      else if (!strcmp(cam_type.c_str(), "v4l2"))
	cam = new camera_v4l2<ubyte>(conf.get_cstring("device"), height, width);
#endif
      else if (!strcmp(cam_type.c_str(), "shmem"))
	cam = new camera_shmem<ubyte>("shared-mem", height, width);
      else if (!strcmp(cam_type.c_str(), "video")) {
	if (arg2)
	  cam = new camera_video<ubyte>
	    (arg2, height, width, conf.get_uint("input_video_sstep"),
	     conf.get_uint("input_video_max_duration"));
	else eblerror("expected 2nd argument");
      } else eblerror("unknown camera type");
      if (save_video)
	cam->start_recording();
      // other variables
      bool updated = false;
      bbox *b = new bbox;
      vector<bbox*> bb;
#ifdef __OPENCV__
      IplImage *iplframe = NULL;
      IplImage *ipltpl = NULL;
#endif
      // main loop
      while(!cam->empty()) {
	try {
	  frame = cam->grab();
	  // send new frame to vision_thread and check if new data was output
	  dt.set_data(frame);
	  updated = dt.get_data(bb, detframe);
	  // update target position if vision thread ready
	  if (updated) {
	    // find bbox with max confidence
	    if (bb.size() > 0) {
	      double maxconf = -5.0;
	      uint maxi = 0;
	      for (uint i = 0; i < bb.size(); ++i)
		if (bb[i]->confidence > maxconf) {
		  maxconf = bb[i]->confidence;
		  maxi = i;
		}
	      //	      cout << "found bbox" << endl;
	      b = bb[maxi];
	      if (b) {
		// update template
		idx<ubyte> tmptpl = detframe.narrow(0, b->height, b->h0);
		tmptpl = tmptpl.narrow(1, b->width, b->w0);
		tpl = idx<ubyte>(tmptpl.get_idxdim());
		idx_copy(tmptpl, tpl);
	      }
	    }
	  } else { // tracking (only when we didnt just receive an update)
#ifdef __OPENCV__
	    iplframe = idx_to_iplptr(frame);
	    ipltpl = idx_to_iplptr(tpl);
	    CvPoint minloc, maxloc;
	    double minval, maxval;
	    // vector<CvPoint> found;
	    // vector<double> confidences;
	    // FastMatchTemplate(*iplframe, *ipltpl, &found, &confidences, 1, false,
	    // 			5, 1, 15);
	    //      CvRect searchRoi;
	    //      cvSetImageROI( searchImage, searchRoi );
	    // vector<CvPoint>::iterator p = found.begin();
	    // vector<double>::iterator c = confidences.begin();
	    // for ( ; p != found.end() && c != confidences.end(); ++p, ++c) {
	    // 	gui << at(p->x, p->y) << *c;
	    // }
	    IplImage *tm = cvCreateImage(cvSize(frame.dim(1)  - tpl.dim(1)  + 1,
						frame.dim(0) - tpl.dim(0) + 1 ),
					 IPL_DEPTH_32F, 1 );
	    cvMatchTemplate(iplframe, ipltpl, tm, CV_TM_SQDIFF_NORMED);
	    cvMinMaxLoc(tm, &minval, &maxval, &minloc, &maxloc, 0);
	    //	    gui << at(minloc.y, minloc.x) << "M";
	    b->class_id = -42; // tell that this bbox is result of tracking
	    b->h0 = minloc.y;
	    b->w0 = minloc.x;
	    //	    cout << "maxloc h: " << maxloc.y << " w: " << maxloc.x << " minloc h:" << minloc.y << " w: " << minloc.x << endl;
#endif
	  }
	  // make a copy of found bounding boxes
	  copy_bboxes(bb);
	  // switch 'updated' flag on to warn we just added new data
	  set_out_updated();
	  // display
	  if (display)
	    draw(b);
	} catch (string &err) {
	  cerr << err << endl;
	}
      }
      if (save_video)
	cam->stop_recording(conf.exists_bool("use_original_fps") ?
			    cam->fps() : conf.get_uint("save_video_fps"));
      if (cam) delete cam;
    } catch(string &err) { eblerror(err.c_str()); }
  }
Exemplo n.º 2
0
void detection_thread<T>::execute() {
  try {
    bool	display        = false;
#ifdef __GUI__
    display = conf.exists_true("display")
        && conf.exists_true("display_threads");
    bool	mindisplay     = conf.exists_true("minimal_display");
    bool       save_video     = conf.exists_true("save_video");
    bool	display_states = conf.exists_true("display_states");
    uint       wid	       = 0;	// window id
    uint       wid_states     = 0;	// window id
#endif
    uint	display_sleep  = conf.try_get_uint("display_sleep", 0);
    //      if (!display && save_video) {
    //        // we still want to output images but not show them
    //        display = true;
    // #ifdef __GUI__
    //        set_gui_silent();
    // #endif
    //      }
    // load network and weights in a forward-only parameter
    parameter<T> theparam;
    theparam.set_forward_only();
    idx<ubyte> classes(1,1);
    //try { // try loading classes names but do not stop upon failure
    load_matrix<ubyte>(classes, conf.get_cstring("classes"));
    // } catch(std::string &err) {
    //   merr << "warning: " << err;
    //   merr << std::endl;
    // }
    std::vector<std::string> sclasses = ubyteidx_to_stringvector(classes);
    answer_module<T> *ans = create_answer<T,T,T>(conf, classes.dim(0));
    uint noutputs = ans->get_nfeatures();
    intg thick = -1;
    module_1_1<T> *net = create_network<T>(theparam, conf, thick, noutputs,
                                           "arch", this->_id);
    // loading weights
    if (conf.exists("weights")) { // manual weights
      // concatenate weights if multiple ones
      std::vector<std::string> w =
          string_to_stringvector(conf.get_string("weights"));
      mout << "Loading weights from: " << w << std::endl;
      theparam.load_x(w);
      // permute weights by blocks
      if (conf.exists("weights_permutation")) {
        std::string sblocks = conf.get_string("weights_blocks");
        std::string spermut = conf.get_string("weights_permutation");
        std::vector<intg> blocks = string_to_intgvector(sblocks.c_str());
        std::vector<uint> permut = string_to_uintvector(spermut.c_str());
        theparam.permute_x(blocks, permut);
      }
    } else {
      if (conf.exists_true("manual_load")) { // manual load
        eblwarn("\"weights\" variable not defined, loading manually "
                << "if manual_load defined");
        manually_load_network(*((layers<T>*)net), conf);
      } else { // random weights
        int seed = dynamic_init_drand();
        eblwarn("No weights to load, randomizing weights with seed " << seed);
        forget_param_linear fgp(1, 0.5, seed);
        net->forget(fgp);
      }
    }
    DEBUGMEM_PRETTY("before detection");
    // detector
    detector<T> detect(*net, sclasses, ans, NULL, NULL, mout, merr);
    init_detector(detect, conf, outdir, silent);
    // keep pointer to detector
    pdetect = &detect;
    bootstrapping<T> boot(conf);

    // when a bbox file is given, ignore the processing, load the pre-computed
    // bboxes and feed them to the nms (non-maximum suppression).
    bboxes boxes(bbox_all, NULL, mout, merr);
    boxes.print_saving_type(); // inform user how we save boxes
    bool precomputed_boxes = false;
    if (conf.exists("bbox_file")) {
      precomputed_boxes = true;
      std::string bbfile = conf.get_string("bbox_file");
      boxes.load_eblearn(bbfile);
    }
    bool bmask_class = false;
    if (conf.exists("mask_class"))
      bmask_class = detect.set_mask_class(conf.get_cstring("mask_class"));

    std::string viddir = outdir;
    viddir += "video/";
    mkdir_full(viddir);
    // gui
#ifdef __GUI__
    uint display_wmax = conf.try_get_uint("display_max_width", 3000);
    T display_min = (T) conf.try_get_double("display_min", -1.7);
    T display_max = (T) conf.try_get_double("display_max", 1.7);
    T display_in_max = (T) conf.try_get_double("display_in_max", 255);
    T display_in_min = (T) conf.try_get_double("display_in_min", 0);
    float display_transp = conf.try_get_float("display_bb_transparency", 0);
    uint qstep1 = conf.try_get_uint("qstep1", 0);
    uint qheight1 = conf.try_get_uint("qheight1", 0);
    uint qwidth1 = conf.try_get_uint("qwidth1", 0);
    uint qstep2 = conf.try_get_uint("qstep2", 0);
    uint qheight2 = conf.try_get_uint("qheight2", 0);
    uint qwidth2 = conf.try_get_uint("qwidth2", 0);
    module_1_1_gui	netgui;
    wid_states  = display_states ? new_window("network states"):0;
    night_mode();
    std::string title = "EBLearn detector: ";
    title += _name;
    if (display) {
      wid = new_window(title.c_str());
      mout << "displaying in window " << wid << std::endl;
      night_mode();
    }
    float zoom = conf.try_get_float("display_zoom", 1);
    bool bbox_show_conf = !conf.exists_false("bbox_show_conf");
    bool bbox_show_class = !conf.exists_false("bbox_show_class");
    detector_gui<T>
        dgui(conf.try_get_uint("show_extracted", 0),
             conf.exists_bool("queue1"), qstep1, qheight1,
             qwidth1, conf.exists_bool("queue2"), qstep2, qheight2, qwidth2,
             bbox_show_class, bbox_show_conf);
    if (bmask_class)
      dgui.set_mask_class(conf.get_cstring("mask_class"),
                          (T) conf.try_get_double("mask_threshold", 0));
#endif
    // timing variables
    timer tpass, toverall;
    long ms;
    // loop
    toverall.start();
    // we're ready
    bavailable = true;
    while(!this->_stop) {
      // wait until a new image is made available
      while (!in_updated && !_stop) {
        millisleep(1);
      }
      tpass.restart();
      if (_stop) break ;
      // we got a new frame, reset new frame flag
      in_updated = false; // no need to lock mutex
      // check if this frame should be skipped
      if (boot.skip_frame(frame_name)) {
        skip_frame();
        continue ;
      } else if (!frame_loaded) {
        uframe = load_image<ubyte>(frame_fullname);
        mout << "loaded image " << frame_fullname << std::endl;
      }
      if (!silent) mout << "processing " << frame_name << std::endl;
      // check frame is correctly allocated, if not, allocate.
      if (frame.order() != uframe.order())
        frame = idx<T>(uframe.get_idxdim());
      else if (frame.get_idxdim() != uframe.get_idxdim())
        frame.resize(uframe.get_idxdim());
      // copy frame
      idx_copy(uframe, frame);
      // run detector
      if (!display) { // fprop without display
        if (precomputed_boxes) {
          try {
            bboxes *bb = boxes.get_group(frame_name);
            idxdim d = boxes.get_group_dims(frame_name);
            d.insert_dim(0, 1);
            bboxes pruned;
            detect.init(d);
            detect.fprop_nms(*bb, pruned);
            copy_bboxes(pruned); // make a copy of bounding boxes
            // resize frame so that caller knows the size of the frame
            idxdim framedim = frame.get_idxdim();
            if (d.dim(1) == -1 || d.dim(2) == -1)
              eblerror("pre-computed boxes must contain full image size, "
                       << "but found: " << d);
            framedim.setdim(0, d.dim(1));
            framedim.setdim(1, d.dim(2));
            frame.resize(framedim);
          } catch(eblexception &e) {
#ifdef __NOEXCEPTIONS__
            merr << "exception" << std::endl;
#else
            merr << e << std::endl;
#endif
          }
        } else {
          try {
            mout << "starting processing of frame " << frame_name << std::endl;
            bboxes &bb = detect.fprop(frame, frame_name.c_str(), frame_id);
            copy_bboxes(bb); // make a copy of bounding boxes
          } catch(ebl::eblexception &e) { // detection failed
#ifdef __NOEXCEPTIONS__
            eblwarn("detection failed");
#else
            eblwarn("detection failed: " << e);
#endif
            clear_bboxes();
          }
        }
      }
#ifdef __GUI__
      else { // fprop and display
        if (precomputed_boxes) eblerror("not implemented for nms only (TODO)");
        disable_window_updates();
        select_window(wid);
        clear_window();
        std::string title = _name;
        title << ": " << frame_name;
        set_window_title(title.c_str());
        //	 clear_resize_window();
        try {
          if (mindisplay) {
            bboxes &bb =
                dgui.display(detect, frame, frame_name.c_str(), frame_id,
                             0, 0, zoom, display_min, display_max,
                             wid, _name.c_str(), display_transp);
            copy_bboxes(bb); // make a copy of bounding boxes
          } else {
            // extract & display boxes
            bboxes &bb =
                dgui.display_inputs_outputs(detect, frame, frame_name.c_str(),
                                            frame_id, 0, 0, zoom,
                                            display_min, display_max, wid,
                                            _name.c_str(),
                                            display_in_min, display_in_max,
                                            display_transp, display_wmax);
            // make a copy of bounding boxes
            copy_bboxes(bb);
          }
        } catch(ebl::eblexception &e) { // detection failed
          eblwarn("detection failed: " << e);
          clear_bboxes();
        }
        enable_window_updates();
      }
      if (display_states) {
        dgui.display_current(detect, frame, wid_states, NULL, zoom);
        select_window(wid);
      }
      if (save_video && display) {
        std::string fname = viddir;
        fname += frame_name;
        save_window(fname.c_str());
        if (!silent) mout << "saved " << fname << std::endl;
      }
#endif
      if (!silent)
        mout << "processing done for frame " << frame_name << std::endl;
      // bootstrapping
      if (conf.exists_true("bootstrapping")) {
        boot.fprop(detect, frame_name);
        // add multiple scales if positives and scales are defined
        if (conf.exists("gt_scales") && boot.extract_positives()) {
          std::vector<double> scales =
              string_to_doublevector(conf.get_cstring("gt_scales"));
          for (uint s = 0; s < scales.size(); ++s) {
            double f = scales[s];
            // downsample input by f
            detect.set_resolution(f);
            detect.init(frame.get_idxdim(), frame_name.c_str());
            detect.fprop(frame, frame_name.c_str(), frame_id);
            boot.fprop(detect, frame_name, false, f);
          }
          detect.set_scaling_original();
          detect.init(frame.get_idxdim(), frame_name.c_str());
        }
        copy_bootstrapping(boot.get_all(), boot.get_bball());
#ifdef __GUI__
        // display groundtruth
        if (conf.exists_true("display_bootstrapping"))
          dgui.display_groundtruth(detect, frame, boot.get_gtall(),
                                   boot.get_gtclean(), boot.get_gtrest(),
                                   boot.get_bbpos(), boot.get_bbneg(),
                                   boot.get_pos(), boot.get_neg(), 0, 0, zoom,
                                   display_min, display_max);
#endif
      }
      total_saved = detect.get_total_saved();
      ms = tpass.elapsed_milliseconds();
      if (!silent) {
        mout << bbs.pretty_short(detect.get_labels());
        mout << "processing=" << ms << " ms ("
             << tpass.elapsed() << ")" << std::endl;
      }
      DEBUGMEM_PRETTY("after detection");
      // switch 'updated' flag on to warn we just added new data
      set_out_updated();
      // display sleep
      if (display_sleep > 0) {
        mout << "sleeping for " << display_sleep << "ms." << std::endl;
        millisleep(display_sleep);
      }
      if (conf.exists("save_max") &&
          detect.get_total_saved() > conf.get_uint("save_max"))
        break ; // limit number of detection saves
    }
    mout << "detection finished. Execution time: " << toverall.elapsed()<<std::endl;
    // free variables
    if (net) delete net;
    if (ans) delete ans;
  } eblcatcherror();
}