void VideoReaderUnit::ProcessFrame(FrameSetPtr input, list<FrameSetPtr>* output) {
  if (!used_as_root_) {
    input->push_back(shared_ptr<VideoFrame>(ReadNextFrame()));
    output->push_back(input);
    ++frame_num_;
  }
}
예제 #2
0
void VideoPipelineStats::ProcessFrame(FrameSetPtr input, list<FrameSetPtr>* output) {
  const int bins = sinks_.size();
  const int border = 40;
  const float scale = (options_.frame_height - border) /
    (options_.max_queue_height * 1.3);

  // Create a new frameset and video-frame.
  shared_ptr<VideoFrame> curr_frame(new VideoFrame(options_.frame_width,
                                                   options_.frame_height,
                                                   3,
                                                   frame_width_step_));

  cv::Mat image;
  curr_frame->MatView(&image);
  image.setTo(180);

  // Draw histogram.
  const int spacing = image.cols / (bins + 2);
  for (int i = 0; i < bins; ++i) {
    // Locations.
    int val = sinks_[i].first->GetQueueSize();
    int col = (i + 1) * spacing;
    int row = options_.frame_height - border - (val * scale);
    cv::Point pt1(col - spacing / 3, row);
    cv::Point pt2(col + spacing / 3, image.rows - border);
    // Bar plot.
    cv::rectangle(image, pt1, pt2, CV_RGB(0, 0, 200), -2);   // Neg. last arg for filled.

    // Value text.
    cv::Point txt_pt(col - spacing / 3, options_.frame_height - border / 3 - 10);
    cv::putText(image,
                base::StringPrintf("%02d", val),
                txt_pt,
                cv::FONT_HERSHEY_SIMPLEX,
                0.5,
                CV_RGB(0, 0, 0),
                2);

    // Name text.
    txt_pt.y += 15;
    cv::putText(image,
                sinks_[i].second,
                txt_pt,
                cv::FONT_HERSHEY_SIMPLEX,
                0.4,
                CV_RGB(0, 0, 0),
                1);

    // Fps text.
    txt_pt.y = border / 3;
    cv::putText(image,
                base::StringPrintf("%3.1f", sinks_[i].first->MinTreeRate()),
                txt_pt,
                cv::FONT_HERSHEY_SIMPLEX,
                0.4,
                CV_RGB(0, 0, 0),
                1);

  }

  // Print running time.
  boost::posix_time::ptime curr_time = boost::posix_time::microsec_clock::local_time();
  float secs_passed = boost::posix_time::time_period(
      start_time_, curr_time).length().total_milliseconds() * 1.e-3f;
  cv::putText(image,
              base::StringPrintf("up: %5.1f", secs_passed),
              cv::Point(options_.frame_width - 75, border / 3),
              cv::FONT_HERSHEY_SIMPLEX,
              0.4,
              CV_RGB(0, 0, 0),
              1);

  // Pass to output.
  input->push_back(curr_frame);
  output->push_back(input);
}
  void OpticalFlowUnit::KeyFrameTrack(FrameSetPtr input, const FrameSet& feedback_input,
                                      list<FrameSetPtr>* output) {
    // Get luminance IplImage.
    const VideoFrame* frame = dynamic_cast<const VideoFrame*>(input->at(video_stream_idx_).get());
    ASSERT_LOG(frame);

    IplImage image;
    frame->ImageView(&image);

    ASSERT_LOG(image.nChannels == 1);

    const int retrack_dist_ = 1000;

    bool reset_key_frame = false;
    if (key_frame_stream_idx_ >= 0 && processed_frames_ > 0) {
      const ValueFrame<bool>* key_frame =
          dynamic_cast<ValueFrame<bool>*>(feedback_input.at(key_frame_stream_idx_).get());

      reset_key_frame = key_frame->value();
    }

    // We set a key-frame is reset_key_frame is already set
    // or frame_num_ frames have passed.
    if (processed_frames_ - last_key_frame_ >= frame_num_) {
      reset_key_frame = true;
    }

    const int frames_to_track = 1;

    int frame_size = 0;           // Size of optical flow frame.
    if (processed_frames_ > 0) {
      // Key-frame: Track all frames until last key-frame.
      if (reset_key_frame)
        frame_size = processed_frames_ - last_key_frame_;
      else if (processed_frames_ - last_key_frame_ <= frames_to_track) {
         // Second frame. Prev. frame and key-frame are identical.
        frame_size = processed_frames_ - last_key_frame_;
      }
      else if ((processed_frames_ - last_key_frame_) % retrack_dist_ == 0)
        frame_size = processed_frames_ - last_key_frame_;
      else
        frame_size = frames_to_track + 1;   // plus key-frame track.
    }

    shared_ptr<OpticalFlowFrame> off(new OpticalFlowFrame(frame_size, reset_key_frame));

    int num_features = max_features_;

    // Track w.r.t. previous frame
    if (processed_frames_ > 0) {
      // Extract features in current image.
      cvGoodFeaturesToTrack(&image, eig_image_.get(), tmp_image_.get(),
                            cur_features_.get(), &num_features, .005, 2, 0);

      CvTermCriteria term_criteria = cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS,
                                                    50, .02);
      int flow_flags = 0;

      // Track w.r.t. previous frame.
      // If key-frame, track w.r.t to all previous frames until last key-frame.
      int tracked_frames = std::min(frame_size, frames_to_track); //1;
      if (reset_key_frame)
        tracked_frames = processed_frames_ - last_key_frame_;
      else {
        // Do a dense track from time to time to rid out false matches.
        if ((processed_frames_ - last_key_frame_) % retrack_dist_ == 0)
          tracked_frames = processed_frames_ - last_key_frame_;
      }


      for (int t = 0; t < tracked_frames; ++t) {
        ASSURE_LOG(processed_frames_ - t - 1 >= 0);
        cvCalcOpticalFlowPyrLK(&image,
                               prev_image_[t].get(),
                               cur_pyramid_.get(),
                               prev_pyramid_[t].get(),
                               cur_features_.get(),
                               prev_features_.get(),
                               num_features,
                               cvSize(10, 10),
                               3,
                               flow_status_.get(),
                               flow_track_error_.get(),
                               term_criteria, flow_flags);

        flow_flags |= CV_LKFLOW_INITIAL_GUESSES;
        flow_flags |= CV_LKFLOW_PYR_A_READY;
        float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
                                        frame_height_ * frame_height_));

        off->AddFeatures(prev_features_.get(), cur_features_.get(),
                         flow_status_.get(), num_features, frame_diam * .05,
                         t, -t - 1, 0.f);
      }

      if (reset_key_frame) {
        // Is key-frame --> reinitialize.
        last_key_frame_ = processed_frames_;
      // Test for sufficient data and no retrack happens.
      } else if (processed_frames_ - last_key_frame_ > frames_to_track &&
                 (processed_frames_ - last_key_frame_) % retrack_dist_ != 0) {
        // Track w.r.t. prev. key-frame.
        const DataFrame* seg_frame = feedback_input[seg_stream_idx_].get();
        Segment::SegmentationDesc desc;
        desc.ParseFromArray(seg_frame->data(), seg_frame->size());

        // Expect per-frame segmentation.
        ASSURE_LOG(desc.hierarchy_size() > 1)
            << "Hierarchy expected to be present in every frame.";

        const RegionFlowFrame* flow_frame =
            dynamic_cast<const RegionFlowFrame*>(feedback_input[region_flow_idx_].get());

        // Get region_flow w.r.t prev. key_frame, which is always the last frame in region_flow.
        vector<RegionFlowFrame::RegionFlow> region_flow;
        flow_frame->FillVector(&region_flow, flow_frame->MatchedFrames() - 1);
        ASSURE_LOG(flow_frame->FrameID(flow_frame->MatchedFrames() - 1) + processed_frames_ - 1
                   == last_key_frame_);

        int level = flow_frame->Level();

        Segment::SegmentationDescToIdImage(id_img_.get(),
                                           frame_width_ * sizeof(int),    // width_step
                                           frame_width_,
                                           frame_height_,
                                           level,
                                           desc,
                                           0);  // no hierarchy.

        // Initialize prev. feature locations with guess from region flow.
        int tracked_feature = 0;

        // Buffer region vector lookups.
        // Neighbor processing later on, can be costly if executed for each feature indepently.
        hash_map<int, CvPoint2D32f> region_vec;

        for (int i = 0; i < num_features; ++i) {
          int x = prev_features_[i].x + 0.5;
          int y = prev_features_[i].y + 0.5;

          x = std::max(0, std::min(x, frame_width_ - 1));
          y = std::max(0, std::min(y, frame_height_ - 1));

          int region_id = id_img_[y * frame_width_ + x];

          // Buffered?
          hash_map<int, CvPoint2D32f>::const_iterator look_up_loc = region_vec.find(region_id);
          if (look_up_loc != region_vec.end() &&
              look_up_loc->first == region_id) {
            prev_features_[i] = prev_features_[i] + look_up_loc->second;
            ++tracked_feature;
            continue;
          }

          RegionFlowFrame::RegionFlow rf_to_find;
          rf_to_find.region_id = region_id;
          vector<RegionFlowFrame::RegionFlow>::const_iterator rf_loc =
          std::lower_bound(region_flow.begin(), region_flow.end(), rf_to_find);

          if(rf_loc != region_flow.end() &&
             rf_loc->region_id == region_id) {
            prev_features_[i] = prev_features_[i] + rf_loc->mean_vector;
            region_vec[region_id] = rf_loc->mean_vector;
            ++tracked_feature;
          } else {
            // Poll the neighbors recursively.
            int neighbor_matches = 0;
            CvPoint2D32f neighbor_vec = cvPoint2D32f(0, 0);

            const int max_rounds = 2;
            vector<int> to_visit(1, region_id);
            vector<int> visited;

            for (int round = 0; round < max_rounds; ++round) {
              // Add all neighbors of all elements in to_visit that are not in visited to to_visit.
              vector<int> new_to_visit;

              for (vector<int>::const_iterator v = to_visit.begin(); v != to_visit.end(); ++v) {
                const Segment::CompoundRegion& comp_region =
                    GetCompoundRegionFromId(*v, desc.hierarchy(level));

                for (int j = 0,
                     n_sz = comp_region.neighbor_id_size();
                     j < n_sz;
                     ++j) {
                  int n_id = comp_region.neighbor_id(j);
                  vector<int>::const_iterator visit_loc = std::lower_bound(visited.begin(),
                                                                             visited.end(),
                                                                             n_id);
                  if (visit_loc == visited.end() ||
                      *visit_loc != n_id)
                    new_to_visit.push_back(n_id);
                }
              }

              // Remove duplicates from new_to_vist.
              std::sort(new_to_visit.begin(), new_to_visit.end());
              vector<int>::iterator new_to_visit_end = std::unique(new_to_visit.begin(),
                                                                   new_to_visit.end());

              // Process all members in new_to_visit.
              for (vector<int>::const_iterator v = new_to_visit.begin();
                   v != new_to_visit_end;
                   ++v) {
                RegionFlowFrame::RegionFlow rf_to_find;
                rf_to_find.region_id = *v;

                vector<RegionFlowFrame::RegionFlow>::const_iterator rf_loc =
                std::lower_bound(region_flow.begin(), region_flow.end(), rf_to_find);

                if(rf_loc != region_flow.end() &&
                   rf_loc->region_id == rf_to_find.region_id) {
                  ++neighbor_matches;
                  neighbor_vec = neighbor_vec + rf_loc->mean_vector;
                }
              }

              if (neighbor_matches > 0) {
                ++tracked_feature;
                prev_features_[i] = prev_features_[i] + neighbor_vec * (1.0f / neighbor_matches);
                region_vec[region_id] = neighbor_vec * (1.0f / neighbor_matches);
                break;
              }

              // Setup next round.
              visited.insert(visited.end(), to_visit.begin(), to_visit.end());
              to_visit.swap(new_to_visit);
              visited.insert(visited.end(), to_visit.begin(), to_visit.end());
            }
          }  // end polling neighbors.
        } // end for loop over features.

        std::cout << "Tracked feature ratio: " << (float)tracked_feature / num_features << "\n";

        // Compute flow to key frame.
        int t = processed_frames_ - last_key_frame_ - 1;
        flow_flags |= CV_LKFLOW_INITIAL_GUESSES;
        flow_flags |= CV_LKFLOW_PYR_A_READY;

        cvCalcOpticalFlowPyrLK(&image,
                               prev_image_[t].get(),
                               cur_pyramid_.get(),
                               prev_pyramid_[t].get(),
                               cur_features_.get(),
                               prev_features_.get(),
                               num_features,
                               cvSize(10, 10),
                               3,
                               flow_status_.get(),
                               flow_track_error_.get(),
                               term_criteria, flow_flags);

        float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
                                        frame_height_ * frame_height_));

        // Add to end.
        off->AddFeatures(prev_features_.get(), cur_features_.get(),
                         flow_status_.get(), num_features, frame_diam * .05,
                         frame_size - 1, -t - 1, 0.f);

      }
    }

    input->push_back(off);
    prev_image_.push_front(cvCloneImageShared(&image));

    ++processed_frames_;
    output->push_back(input);
  }
  void OpticalFlowUnit::PreviousFrameTrack(FrameSetPtr input,
                                           list<FrameSetPtr>* output,
                                           const vector<CvPoint2D32f>* polygon) {
    // Get luminance IplImage.
    const VideoFrame* frame =
        dynamic_cast<const VideoFrame*>(input->at(video_stream_idx_).get());

    ASSERT_LOG(frame);

    IplImage image;
    frame->ImageView(&image);

    ASSERT_LOG(image.nChannels == 1);

    // First match always direct predecessor.
    int num_matches = processed_frames_ > 0 ? 1 : 0;
    if (processed_frames_ > 0) {
      // For t = num_matches - 1, we need have
      // processed_frames_ - 1 - t * matching_stride_ >= 0
      // t * matching_stride <= processed_frames_ - 1
      // num_matches - 1  <= (processed_frames_ - 1) / matching_stride
      // num_matches <= (processed_frames_ - 1) / matching_stride + 1
      num_matches += std::min(frame_num_ - 1,
                             (processed_frames_ - 1) / matching_stride_);
    }

    shared_ptr<OpticalFlowFrame> off(new OpticalFlowFrame(num_matches, false));

    int num_features = max_features_;
    const float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
                                          frame_height_ * frame_height_));

    // Extract features in current image.
    // Change param to .05 for fewer features and more speed.
    cvGoodFeaturesToTrack(&image,
                          eig_image_.get(),
                          tmp_image_.get(),
                          cur_features_.get(),
                          &num_features,
                          .005,
                          5,
                          0);

    if (polygon) {
      // Reject all features out of bounds.
      vector<CvPoint2D32f> outline(*polygon);
      for (int i = 0; i < outline.size(); ++i) {
        outline[i].y = frame_height_ - 1 - outline[i].y;
      }
      outline.push_back(outline[0]);

      // Use prev. features as tmp. buffer
      int p = 0;
      for (int i = 0; i < num_features; ++i) {
        CvPoint2D32f pt = cur_features_[i];
        pt.y = frame_height_ - 1 - pt.y;
        if (WithinConvexPoly(pt, outline, 0))
          prev_features_[p++] = cur_features_[i];
      }

      cur_features_.swap(prev_features_);
      num_features = p;
    }

    CvTermCriteria term_criteria = cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS,
                                                  50, .02);
    int flow_flags = 0;

    for (int t = 0; t < off->MatchedFrames(); ++t ) {
      int prev_frame = processed_frames_ - 1 - t * matching_stride_;
      ASSERT_LOG(prev_frame >= 0);

      // Find correspond features in previous image for each extracted
      // feature in current frame.
      cvCalcOpticalFlowPyrLK(&image,
                             prev_image_[t * matching_stride_].get(),
                             cur_pyramid_.get(),
                             prev_pyramid_[0].get(),  // just one pyramid here.
                             cur_features_.get(),
                             prev_features_.get(),
                             num_features,
                             cvSize(10, 10),
                             3,
                             flow_status_.get(),
                             flow_track_error_.get(),
                             term_criteria,
                             flow_flags);

      flow_flags |= CV_LKFLOW_PYR_A_READY;
      flow_flags |= CV_LKFLOW_INITIAL_GUESSES;

      off->AddFeatures(prev_features_.get(),
                       cur_features_.get(),
                       flow_status_.get(),
                       num_features,
                       frame_diam * .05,
                       t,
                       -1 - t * matching_stride_,
                       0.f);
    }

    input->push_back(off);
    prev_image_.push_front(cvCloneImageShared(&image));

    ++processed_frames_;

    output->push_back(input);
  }