void VideoReaderUnit::ProcessFrame(FrameSetPtr input, list<FrameSetPtr>* output) {
if (!used_as_root_) {
input->push_back(shared_ptr<VideoFrame>(ReadNextFrame()));
output->push_back(input);
++frame_num_;
}
}
void VideoPipelineStats::ProcessFrame(FrameSetPtr input, list<FrameSetPtr>* output) {
const int bins = sinks_.size();
const int border = 40;
const float scale = (options_.frame_height - border) /
(options_.max_queue_height * 1.3);
// Create a new frameset and video-frame.
shared_ptr<VideoFrame> curr_frame(new VideoFrame(options_.frame_width,
options_.frame_height,
3,
frame_width_step_));
cv::Mat image;
curr_frame->MatView(&image);
image.setTo(180);
// Draw histogram.
const int spacing = image.cols / (bins + 2);
for (int i = 0; i < bins; ++i) {
// Locations.
int val = sinks_[i].first->GetQueueSize();
int col = (i + 1) * spacing;
int row = options_.frame_height - border - (val * scale);
cv::Point pt1(col - spacing / 3, row);
cv::Point pt2(col + spacing / 3, image.rows - border);
// Bar plot.
cv::rectangle(image, pt1, pt2, CV_RGB(0, 0, 200), -2); // Neg. last arg for filled.
// Value text.
cv::Point txt_pt(col - spacing / 3, options_.frame_height - border / 3 - 10);
cv::putText(image,
base::StringPrintf("%02d", val),
txt_pt,
cv::FONT_HERSHEY_SIMPLEX,
0.5,
CV_RGB(0, 0, 0),
2);
// Name text.
txt_pt.y += 15;
cv::putText(image,
sinks_[i].second,
txt_pt,
cv::FONT_HERSHEY_SIMPLEX,
0.4,
CV_RGB(0, 0, 0),
1);
// Fps text.
txt_pt.y = border / 3;
cv::putText(image,
base::StringPrintf("%3.1f", sinks_[i].first->MinTreeRate()),
txt_pt,
cv::FONT_HERSHEY_SIMPLEX,
0.4,
CV_RGB(0, 0, 0),
1);
}
// Print running time.
boost::posix_time::ptime curr_time = boost::posix_time::microsec_clock::local_time();
float secs_passed = boost::posix_time::time_period(
start_time_, curr_time).length().total_milliseconds() * 1.e-3f;
cv::putText(image,
base::StringPrintf("up: %5.1f", secs_passed),
cv::Point(options_.frame_width - 75, border / 3),
cv::FONT_HERSHEY_SIMPLEX,
0.4,
CV_RGB(0, 0, 0),
1);
// Pass to output.
input->push_back(curr_frame);
output->push_back(input);
}
void OpticalFlowUnit::KeyFrameTrack(FrameSetPtr input, const FrameSet& feedback_input,
list<FrameSetPtr>* output) {
// Get luminance IplImage.
const VideoFrame* frame = dynamic_cast<const VideoFrame*>(input->at(video_stream_idx_).get());
ASSERT_LOG(frame);
IplImage image;
frame->ImageView(&image);
ASSERT_LOG(image.nChannels == 1);
const int retrack_dist_ = 1000;
bool reset_key_frame = false;
if (key_frame_stream_idx_ >= 0 && processed_frames_ > 0) {
const ValueFrame<bool>* key_frame =
dynamic_cast<ValueFrame<bool>*>(feedback_input.at(key_frame_stream_idx_).get());
reset_key_frame = key_frame->value();
}
// We set a key-frame is reset_key_frame is already set
// or frame_num_ frames have passed.
if (processed_frames_ - last_key_frame_ >= frame_num_) {
reset_key_frame = true;
}
const int frames_to_track = 1;
int frame_size = 0; // Size of optical flow frame.
if (processed_frames_ > 0) {
// Key-frame: Track all frames until last key-frame.
if (reset_key_frame)
frame_size = processed_frames_ - last_key_frame_;
else if (processed_frames_ - last_key_frame_ <= frames_to_track) {
// Second frame. Prev. frame and key-frame are identical.
frame_size = processed_frames_ - last_key_frame_;
}
else if ((processed_frames_ - last_key_frame_) % retrack_dist_ == 0)
frame_size = processed_frames_ - last_key_frame_;
else
frame_size = frames_to_track + 1; // plus key-frame track.
}
shared_ptr<OpticalFlowFrame> off(new OpticalFlowFrame(frame_size, reset_key_frame));
int num_features = max_features_;
// Track w.r.t. previous frame
if (processed_frames_ > 0) {
// Extract features in current image.
cvGoodFeaturesToTrack(&image, eig_image_.get(), tmp_image_.get(),
cur_features_.get(), &num_features, .005, 2, 0);
CvTermCriteria term_criteria = cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS,
50, .02);
int flow_flags = 0;
// Track w.r.t. previous frame.
// If key-frame, track w.r.t to all previous frames until last key-frame.
int tracked_frames = std::min(frame_size, frames_to_track); //1;
if (reset_key_frame)
tracked_frames = processed_frames_ - last_key_frame_;
else {
// Do a dense track from time to time to rid out false matches.
if ((processed_frames_ - last_key_frame_) % retrack_dist_ == 0)
tracked_frames = processed_frames_ - last_key_frame_;
}
for (int t = 0; t < tracked_frames; ++t) {
ASSURE_LOG(processed_frames_ - t - 1 >= 0);
cvCalcOpticalFlowPyrLK(&image,
prev_image_[t].get(),
cur_pyramid_.get(),
prev_pyramid_[t].get(),
cur_features_.get(),
prev_features_.get(),
num_features,
cvSize(10, 10),
3,
flow_status_.get(),
flow_track_error_.get(),
term_criteria, flow_flags);
flow_flags |= CV_LKFLOW_INITIAL_GUESSES;
flow_flags |= CV_LKFLOW_PYR_A_READY;
float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
frame_height_ * frame_height_));
off->AddFeatures(prev_features_.get(), cur_features_.get(),
flow_status_.get(), num_features, frame_diam * .05,
t, -t - 1, 0.f);
}
if (reset_key_frame) {
// Is key-frame --> reinitialize.
last_key_frame_ = processed_frames_;
// Test for sufficient data and no retrack happens.
} else if (processed_frames_ - last_key_frame_ > frames_to_track &&
(processed_frames_ - last_key_frame_) % retrack_dist_ != 0) {
// Track w.r.t. prev. key-frame.
const DataFrame* seg_frame = feedback_input[seg_stream_idx_].get();
Segment::SegmentationDesc desc;
desc.ParseFromArray(seg_frame->data(), seg_frame->size());
// Expect per-frame segmentation.
ASSURE_LOG(desc.hierarchy_size() > 1)
<< "Hierarchy expected to be present in every frame.";
const RegionFlowFrame* flow_frame =
dynamic_cast<const RegionFlowFrame*>(feedback_input[region_flow_idx_].get());
// Get region_flow w.r.t prev. key_frame, which is always the last frame in region_flow.
vector<RegionFlowFrame::RegionFlow> region_flow;
flow_frame->FillVector(®ion_flow, flow_frame->MatchedFrames() - 1);
ASSURE_LOG(flow_frame->FrameID(flow_frame->MatchedFrames() - 1) + processed_frames_ - 1
== last_key_frame_);
int level = flow_frame->Level();
Segment::SegmentationDescToIdImage(id_img_.get(),
frame_width_ * sizeof(int), // width_step
frame_width_,
frame_height_,
level,
desc,
0); // no hierarchy.
// Initialize prev. feature locations with guess from region flow.
int tracked_feature = 0;
// Buffer region vector lookups.
// Neighbor processing later on, can be costly if executed for each feature indepently.
hash_map<int, CvPoint2D32f> region_vec;
for (int i = 0; i < num_features; ++i) {
int x = prev_features_[i].x + 0.5;
int y = prev_features_[i].y + 0.5;
x = std::max(0, std::min(x, frame_width_ - 1));
y = std::max(0, std::min(y, frame_height_ - 1));
int region_id = id_img_[y * frame_width_ + x];
// Buffered?
hash_map<int, CvPoint2D32f>::const_iterator look_up_loc = region_vec.find(region_id);
if (look_up_loc != region_vec.end() &&
look_up_loc->first == region_id) {
prev_features_[i] = prev_features_[i] + look_up_loc->second;
++tracked_feature;
continue;
}
RegionFlowFrame::RegionFlow rf_to_find;
rf_to_find.region_id = region_id;
vector<RegionFlowFrame::RegionFlow>::const_iterator rf_loc =
std::lower_bound(region_flow.begin(), region_flow.end(), rf_to_find);
if(rf_loc != region_flow.end() &&
rf_loc->region_id == region_id) {
prev_features_[i] = prev_features_[i] + rf_loc->mean_vector;
region_vec[region_id] = rf_loc->mean_vector;
++tracked_feature;
} else {
// Poll the neighbors recursively.
int neighbor_matches = 0;
CvPoint2D32f neighbor_vec = cvPoint2D32f(0, 0);
const int max_rounds = 2;
vector<int> to_visit(1, region_id);
vector<int> visited;
for (int round = 0; round < max_rounds; ++round) {
// Add all neighbors of all elements in to_visit that are not in visited to to_visit.
vector<int> new_to_visit;
for (vector<int>::const_iterator v = to_visit.begin(); v != to_visit.end(); ++v) {
const Segment::CompoundRegion& comp_region =
GetCompoundRegionFromId(*v, desc.hierarchy(level));
for (int j = 0,
n_sz = comp_region.neighbor_id_size();
j < n_sz;
++j) {
int n_id = comp_region.neighbor_id(j);
vector<int>::const_iterator visit_loc = std::lower_bound(visited.begin(),
visited.end(),
n_id);
if (visit_loc == visited.end() ||
*visit_loc != n_id)
new_to_visit.push_back(n_id);
}
}
// Remove duplicates from new_to_vist.
std::sort(new_to_visit.begin(), new_to_visit.end());
vector<int>::iterator new_to_visit_end = std::unique(new_to_visit.begin(),
new_to_visit.end());
// Process all members in new_to_visit.
for (vector<int>::const_iterator v = new_to_visit.begin();
v != new_to_visit_end;
++v) {
RegionFlowFrame::RegionFlow rf_to_find;
rf_to_find.region_id = *v;
vector<RegionFlowFrame::RegionFlow>::const_iterator rf_loc =
std::lower_bound(region_flow.begin(), region_flow.end(), rf_to_find);
if(rf_loc != region_flow.end() &&
rf_loc->region_id == rf_to_find.region_id) {
++neighbor_matches;
neighbor_vec = neighbor_vec + rf_loc->mean_vector;
}
}
if (neighbor_matches > 0) {
++tracked_feature;
prev_features_[i] = prev_features_[i] + neighbor_vec * (1.0f / neighbor_matches);
region_vec[region_id] = neighbor_vec * (1.0f / neighbor_matches);
break;
}
// Setup next round.
visited.insert(visited.end(), to_visit.begin(), to_visit.end());
to_visit.swap(new_to_visit);
visited.insert(visited.end(), to_visit.begin(), to_visit.end());
}
} // end polling neighbors.
} // end for loop over features.
std::cout << "Tracked feature ratio: " << (float)tracked_feature / num_features << "\n";
// Compute flow to key frame.
int t = processed_frames_ - last_key_frame_ - 1;
flow_flags |= CV_LKFLOW_INITIAL_GUESSES;
flow_flags |= CV_LKFLOW_PYR_A_READY;
cvCalcOpticalFlowPyrLK(&image,
prev_image_[t].get(),
cur_pyramid_.get(),
prev_pyramid_[t].get(),
cur_features_.get(),
prev_features_.get(),
num_features,
cvSize(10, 10),
3,
flow_status_.get(),
flow_track_error_.get(),
term_criteria, flow_flags);
float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
frame_height_ * frame_height_));
// Add to end.
off->AddFeatures(prev_features_.get(), cur_features_.get(),
flow_status_.get(), num_features, frame_diam * .05,
frame_size - 1, -t - 1, 0.f);
}
}
input->push_back(off);
prev_image_.push_front(cvCloneImageShared(&image));
++processed_frames_;
output->push_back(input);
}
void OpticalFlowUnit::PreviousFrameTrack(FrameSetPtr input,
list<FrameSetPtr>* output,
const vector<CvPoint2D32f>* polygon) {
// Get luminance IplImage.
const VideoFrame* frame =
dynamic_cast<const VideoFrame*>(input->at(video_stream_idx_).get());
ASSERT_LOG(frame);
IplImage image;
frame->ImageView(&image);
ASSERT_LOG(image.nChannels == 1);
// First match always direct predecessor.
int num_matches = processed_frames_ > 0 ? 1 : 0;
if (processed_frames_ > 0) {
// For t = num_matches - 1, we need have
// processed_frames_ - 1 - t * matching_stride_ >= 0
// t * matching_stride <= processed_frames_ - 1
// num_matches - 1 <= (processed_frames_ - 1) / matching_stride
// num_matches <= (processed_frames_ - 1) / matching_stride + 1
num_matches += std::min(frame_num_ - 1,
(processed_frames_ - 1) / matching_stride_);
}
shared_ptr<OpticalFlowFrame> off(new OpticalFlowFrame(num_matches, false));
int num_features = max_features_;
const float frame_diam = sqrt((float)(frame_width_ * frame_width_ +
frame_height_ * frame_height_));
// Extract features in current image.
// Change param to .05 for fewer features and more speed.
cvGoodFeaturesToTrack(&image,
eig_image_.get(),
tmp_image_.get(),
cur_features_.get(),
&num_features,
.005,
5,
0);
if (polygon) {
// Reject all features out of bounds.
vector<CvPoint2D32f> outline(*polygon);
for (int i = 0; i < outline.size(); ++i) {
outline[i].y = frame_height_ - 1 - outline[i].y;
}
outline.push_back(outline[0]);
// Use prev. features as tmp. buffer
int p = 0;
for (int i = 0; i < num_features; ++i) {
CvPoint2D32f pt = cur_features_[i];
pt.y = frame_height_ - 1 - pt.y;
if (WithinConvexPoly(pt, outline, 0))
prev_features_[p++] = cur_features_[i];
}
cur_features_.swap(prev_features_);
num_features = p;
}
CvTermCriteria term_criteria = cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS,
50, .02);
int flow_flags = 0;
for (int t = 0; t < off->MatchedFrames(); ++t ) {
int prev_frame = processed_frames_ - 1 - t * matching_stride_;
ASSERT_LOG(prev_frame >= 0);
// Find correspond features in previous image for each extracted
// feature in current frame.
cvCalcOpticalFlowPyrLK(&image,
prev_image_[t * matching_stride_].get(),
cur_pyramid_.get(),
prev_pyramid_[0].get(), // just one pyramid here.
cur_features_.get(),
prev_features_.get(),
num_features,
cvSize(10, 10),
3,
flow_status_.get(),
flow_track_error_.get(),
term_criteria,
flow_flags);
flow_flags |= CV_LKFLOW_PYR_A_READY;
flow_flags |= CV_LKFLOW_INITIAL_GUESSES;
off->AddFeatures(prev_features_.get(),
cur_features_.get(),
flow_status_.get(),
num_features,
frame_diam * .05,
t,
-1 - t * matching_stride_,
0.f);
}
input->push_back(off);
prev_image_.push_front(cvCloneImageShared(&image));
++processed_frames_;
output->push_back(input);
}