virtual void test_by_pict()
{
Mat frame1 = readImage("optflow/RubberWhale1.png", IMREAD_GRAYSCALE);
UMat usrc;
frame1.copyTo(usrc);
int histSize = randomInt(3, 29);
float hue_range[] = { 0, 180 };
const float* ranges1 = { hue_range };
Mat hist1;
//compute histogram
calcHist(&frame1, 1, 0, Mat(), hist1, 1, &histSize, &ranges1, true, false);
normalize(hist1, hist1, 0, 255, NORM_MINMAX, -1, Mat());
Mat dst1;
UMat udst1, src, uhist1;
hist1.copyTo(uhist1);
std::vector<UMat> uims;
uims.push_back(usrc);
std::vector<float> urngs;
urngs.push_back(0);
urngs.push_back(180);
std::vector<int> chs;
chs.push_back(0);
OCL_OFF(calcBackProject(&frame1, 1, 0, hist1, dst1, &ranges1, 1, true));
OCL_ON(calcBackProject(uims, chs, uhist1, udst1, urngs, 1.0));
EXPECT_MAT_NEAR(dst1, udst1, 0.0);
}
Mat contentfinder::find(const Mat& image) {
Mat result;
hranges[0]= 0.0; // range [0,255]
hranges[1]= 255.0;
channels[0]= 0; // the three channels
channels[1]= 1;
channels[2]= 2;
calcBackProject(&image,
1, // one image
channels, // vector specifying what histogram dimensions belong to what image channels
histogram, // the histogram we are using
result, // the resulting back projection image
ranges, // the range of values, for each dimension
255.0 // the scaling factor is chosen such that a histogram value of 1 maps to 255
);
// Threshold back projection to obtain a binary image
if (threshold>0.0)
cv::threshold(result, result, 255*threshold, 255, THRESH_BINARY);
return result;
}
Mat calcProbability()
{
Mat hue,backproj;
hue.create(hsv.size(), hsv.depth());
mixChannels(&hsv, 1, &hue, 1, ch, 1);
calcBackProject(&hue, 1, 0, hist, backproj, (const float**)phranges);
return backproj;
}
void ModalityColor3DHistogram::probability(Image& image, Mat& p) {
Mat arrays[] = {image.get(colorspace)};
calcBackProject(arrays, 1, channels, model, tmp_p, ranges, 1, true);
tmp_p.convertTo(p, CV_32F, 1.0 / sum(tmp_p)[0]);
}
void test_by_pict()
{
Mat frame1 = readImage("optflow/RubberWhale1.png", IMREAD_GRAYSCALE);
UMat usrc;
frame1.copyTo(usrc);
int histSize = randomInt(3, 29);
float hue_range[] = { 0, 180 };
const float* ranges1 = { hue_range };
Mat hist1;
//compute histogram
calcHist(&frame1, 1, 0, Mat(), hist1, 1, &histSize, &ranges1, true, false);
normalize(hist1, hist1, 0, 255, NORM_MINMAX, -1, Mat());
Mat dst1;
UMat udst1, src, uhist1;
hist1.copyTo(uhist1);
std::vector<UMat> uims;
uims.push_back(usrc);
std::vector<float> urngs;
urngs.push_back(0);
urngs.push_back(180);
std::vector<int> chs;
chs.push_back(0);
OCL_OFF(calcBackProject(&frame1, 1, 0, hist1, dst1, &ranges1, 1, true));
OCL_ON(calcBackProject(uims, chs, uhist1, udst1, urngs, 1.0));
if (cv::ocl::useOpenCL() && cv::ocl::Device::getDefault().isAMD())
{
Size dstSize = dst1.size();
int nDiffs = (int)(0.03f*dstSize.height*dstSize.width);
//check if the dst mats are the same except 3% difference
EXPECT_MAT_N_DIFF(dst1, udst1, nDiffs);
}
else
{
EXPECT_MAT_NEAR(dst1, udst1, 0.0);
}
}
void HDcolorModel::detect(Mat & _rgbImg, Mat & _depthImg, OutputArray _probImg) {
if(detectorInit == false)
return; // ??? do what
Mat hsvImg = _rgbImg;
//cvtColor(_rgbImg, hsvImg, CV_BGR2HSV);
vector<Mat> channel;
split(hsvImg, channel);
const float* range[] = {histRange[0], histRange[1], histRange[2], histRange[3]};
calcBackProject(&channel[0], 1, 0, hist[0], backPro[0], range);
calcBackProject(&channel[1], 1, 0, hist[1], backPro[1], range+1);
calcBackProject(&channel[2], 1, 0, hist[2], backPro[2], range+2);
if(useDepth)
calcBackProject(&_depthImg, 1, 0, hist[3], backPro[3], range+3);
multiply(backPro[0], backPro[1], _probImg, 1./255.0);
multiply(backPro[2], _probImg, _probImg, 1./255.0);
if(useDepth)
multiply(backPro[3], _probImg, _probImg, 1, CV_8UC1);
}
Mat contentfinder::find(const Mat &image, float minValue, float maxValue, int *channels, int dim){
Mat result;
hranges[0]=minValue;
hranges[1]=maxValue;
for (int i=0;i<dim;i++){
this->channels[i]=channels[i];
calcBackProject(&image,1,channels,histogram,result,ranges,255.0);
if(threshold>0.0){
cv::threshold(result,result,255*threshold,255,THRESH_BINARY);
}
}
return result;
}
void AppTemplate::calcBP(const Mat* frame_set, Mat& occ_map,Rect ROI)//*******************
{
confidence_map=Mat::zeros(ROI.height,ROI.width,CV_8UC1);
Rect frame_win(0,0,frame_set[0].cols,frame_set[0].rows);
Rect roi=frame_win & ROI;//the rest of the win will be filled with zero
//CAUTION: cannot generalize to other structure of feature channels
Mat roi_set[]={Mat(frame_set[0],roi), Mat(frame_set[1],roi),Mat(frame_set[2],roi)};
Mat roi_backproj(confidence_map,roi-Point(ROI.x, ROI.y));
Mat roi_mask(occ_map,roi);//occ_map: 1 for no occupancy, 0 for occupancy
calcBackProject(roi_set,3,channels,hist,roi_backproj,hRange);
roi_backproj.setTo(Scalar(0.0),roi_mask);
confidence_map.convertTo(confidence_map,CV_32FC1);//[0,255]
}
void FindObjectMain::process_camshift()
{
// Some user defined parameters
int vmin = config.vmin;
int vmax = config.vmax;
int smin = config.smin;
float hranges[] = { 0, 180 };
const float* phranges = hranges;
// Create aligned, RGB images
if(!object_image)
{
object_image = cvCreateImage(
cvSize(object_image_w, object_image_h),
8,
3);
}
if(!scene_image)
{
scene_image = cvCreateImage(
cvSize(scene_image_w, scene_image_h),
8,
3);
}
// Temporary row pointers
unsigned char **object_rows = new unsigned char*[object_image_h];
unsigned char **scene_rows = new unsigned char*[scene_image_h];
for(int i = 0; i < object_image_h; i++)
{
object_rows[i] = (unsigned char*)(object_image->imageData + i * object_image_w * 3);
}
for(int i = 0; i < scene_image_h; i++)
{
scene_rows[i] = (unsigned char*)(scene_image->imageData + i * scene_image_w * 3);
}
// Transfer object & scene to RGB images for OpenCV
if(!prev_object) prev_object = new unsigned char[object_image_w * object_image_h * 3];
// Back up old object image
memcpy(prev_object, object_image->imageData, object_image_w * object_image_h * 3);
BC_CModels::transfer(object_rows,
get_input(object_layer)->get_rows(),
0,
0,
0,
0,
0,
0,
object_x1,
object_y1,
object_w,
object_h,
0,
0,
object_w,
object_h,
get_input(object_layer)->get_color_model(),
BC_RGB888,
0,
0,
0);
BC_CModels::transfer(scene_rows,
get_input(scene_layer)->get_rows(),
0,
0,
0,
0,
0,
0,
scene_x1,
scene_y1,
scene_w,
scene_h,
0,
0,
scene_w,
scene_h,
get_input(scene_layer)->get_color_model(),
BC_RGB888,
0,
0,
0);
delete [] object_rows;
delete [] scene_rows;
// from camshiftdemo.cpp
// Compute new object
if(memcmp(prev_object,
object_image->imageData,
object_image_w * object_image_h * 3) ||
!hist.dims)
{
Mat image(object_image);
Mat hsv, hue, mask;
cvtColor(image, hsv, CV_RGB2HSV);
int _vmin = vmin, _vmax = vmax;
//printf("FindObjectMain::process_camshift %d\n", __LINE__);
inRange(hsv,
Scalar(0, smin, MIN(_vmin,_vmax)),
Scalar(180, 256, MAX(_vmin, _vmax)),
mask);
int ch[] = { 0, 0 };
hue.create(hsv.size(), hsv.depth());
mixChannels(&hsv, 1, &hue, 1, ch, 1);
Rect selection = Rect(0, 0, object_w, object_h);
trackWindow = selection;
int hsize = 16;
Mat roi(hue, selection), maskroi(mask, selection);
calcHist(&roi, 1, 0, maskroi, hist, 1, &hsize, &phranges);
normalize(hist, hist, 0, 255, CV_MINMAX);
}
// compute scene
Mat image(scene_image);
Mat hsv, hue, mask, backproj;
cvtColor(image, hsv, CV_RGB2HSV);
int _vmin = vmin, _vmax = vmax;
inRange(hsv,
Scalar(0, smin, MIN(_vmin,_vmax)),
Scalar(180, 256, MAX(_vmin, _vmax)),
mask);
int ch[] = {0, 0};
hue.create(hsv.size(), hsv.depth());
mixChannels(&hsv, 1, &hue, 1, ch, 1);
//printf("FindObjectMain::process_camshift %d %d %d\n", __LINE__, hist.dims, hist.size[1]);
RotatedRect trackBox = RotatedRect(
Point2f((object_x1 + object_x2) / 2, (object_y1 + object_y2) / 2),
Size2f(object_w, object_h),
0);
trackWindow = Rect(0,
0,
scene_w,
scene_h);
if(hist.dims > 0)
{
calcBackProject(&hue, 1, 0, hist, backproj, &phranges);
backproj &= mask;
//printf("FindObjectMain::process_camshift %d\n", __LINE__);
// if(trackWindow.width <= 0 ||
// trackWindow.height <= 0)
// {
// trackWindow.width = object_w;
// trackWindow.height = object_h;
// }
trackBox = CamShift(backproj,
trackWindow,
TermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ));
//printf("FindObjectMain::process_camshift %d\n", __LINE__);
// if( trackWindow.area() <= 1 )
// {
// int cols = backproj.cols;
// int rows = backproj.rows;
// int r = (MIN(cols, rows) + 5) / 6;
// trackWindow = Rect(trackWindow.x - r, trackWindow.y - r,
// trackWindow.x + r, trackWindow.y + r) &
// Rect(0, 0, cols, rows);
// }
}
// printf("FindObjectMain::process_camshift %d %d %d %d %d\n",
// __LINE__,
// trackWindow.x,
// trackWindow.y,
// trackWindow.width,
// trackWindow.height);
// Draw mask over scene
if(config.draw_keypoints)
{
for(int i = 0; i < scene_h; i++)
{
switch(get_input(scene_layer)->get_color_model())
{
case BC_YUV888:
{
unsigned char *input = backproj.data + i * scene_image_w;
unsigned char *output = get_input(scene_layer)->get_rows()[i + scene_y1] + scene_x1 * 3;
for(int j = 0; j < scene_w; j++)
{
output[0] = *input;
output[1] = 0x80;
output[2] = 0x80;
output += 3;
input++;
}
break;
}
}
}
}
// Get object outline in the scene layer
// printf("FindObjectMain::process_camshift %d %d %d %d %d %d\n",
// __LINE__,
// (int)trackBox.center.x,
// (int)trackBox.center.y,
// (int)trackBox.size.width,
// (int)trackBox.size.height,
// (int)trackBox.angle);
double angle = trackBox.angle * 2 * M_PI / 360;
double angle1 = atan2(-(double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
double angle2 = atan2(-(double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
double angle3 = atan2((double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
double angle4 = atan2((double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
double radius = sqrt(SQR(trackBox.size.height / 2) + SQR(trackBox.size.width / 2));
border_x1 = (int)(trackBox.center.x + cos(angle1) * radius) + scene_x1;
border_y1 = (int)(trackBox.center.y + sin(angle1) * radius) + scene_y1;
border_x2 = (int)(trackBox.center.x + cos(angle2) * radius) + scene_x1;
border_y2 = (int)(trackBox.center.y + sin(angle2) * radius) + scene_y1;
border_x3 = (int)(trackBox.center.x + cos(angle3) * radius) + scene_x1;
border_y3 = (int)(trackBox.center.y + sin(angle3) * radius) + scene_y1;
border_x4 = (int)(trackBox.center.x + cos(angle4) * radius) + scene_x1;
border_y4 = (int)(trackBox.center.y + sin(angle4) * radius) + scene_y1;
}
AppTemplate::AppTemplate(const Mat* frame_set, const Rect iniWin,int ID)
:ID(ID)//bgr,hsv,lab
{
//get roi out of frame set
Rect body_win=scaleWin(iniWin,1/TRACKING_TO_BODYSIZE_RATIO);
Rect roi_win(body_win.x-body_win.width,body_win.y-body_win.width,3*body_win.width,2*body_win.width+body_win.height);
body_win= body_win&Rect(0,0,frame_set[0].cols,frame_set[0].rows);
roi_win=roi_win&Rect(0,0,frame_set[0].cols,frame_set[0].rows);
Mat roi_set[]={Mat(frame_set[0],roi_win),Mat(frame_set[1],roi_win),Mat(frame_set[2],roi_win)};
Rect iniWin_roi=iniWin-Point(roi_win.x,roi_win.y);
//scores for each channel
list<ChannelScore> channel_score;
Mat mask_roi(roi_set[0].rows,roi_set[0].cols,CV_8UC1,Scalar(0));
rectangle(mask_roi,iniWin_roi,Scalar(255),-1);
Mat inv_mask_roi(roi_set[0].rows,roi_set[0].cols,CV_8UC1,Scalar(255));
rectangle(inv_mask_roi,body_win-Point(roi_win.x,roi_win.y),Scalar(0),-1);
//calculate score for each channel
Mat temp_hist;
Mat temp_bp;
int hist_size[]={BIN_NUMBER};
for (int i=0;i<9;i++)
{
float range1[]={0,255};
if (i==3)
{
range1[1]=179;
}
const float* hist_range[]={range1};
calcHist(roi_set,3,&i,inv_mask_roi,temp_hist,1,hist_size,hist_range);
normalize(temp_hist,temp_hist,255,0.0,NORM_L1);//scale to 255 for display
calcBackProject(roi_set,3,&i,temp_hist,temp_bp,hist_range);
int c[]={0};
int hs[]={BIN_NUMBER};
float hr[]={0,255};
const float* hrr[]={hr};
Mat hist_fore;
Mat hist_back;
calcHist(&temp_bp,1,c,mask_roi,hist_fore,1,hs,hrr);
calcHist(&temp_bp,1,c,inv_mask_roi,hist_back,1,hs,hrr);
normalize(hist_fore,hist_fore,1.0,0.0,NORM_L1);
normalize(hist_back,hist_back,1.0,0.0,NORM_L1);
//deal with gray image to get rid of #IND
double score=getVR(hist_back,hist_fore);
score=score==score ? score:0;
channel_score.push_back(ChannelScore(i,score));
}
//choose the 2 highest scored channels
channel_score.sort(compareChannel);
channels[0]=channel_score.back().idx;
channel_score.pop_back();
channels[1]=channel_score.back().idx;
//using 2 best channel to calculate histogram
for (int i=0;i<2;++i)
{
_hRang[i][0]=0;
if (channels[i]==3)
_hRang[i][1]=179;
else
_hRang[i][1]=255;
hRange[i]=_hRang[i];
}
calcHist(roi_set,3,channels,inv_mask_roi,temp_hist,2,hSize,hRange);
normalize(temp_hist,temp_hist,255,0,NORM_L1);
Mat final_mask;//mask for sampling
calcBackProject(roi_set,3,channels,temp_hist,final_mask,hRange);
threshold(final_mask,final_mask,5,255,CV_THRESH_BINARY_INV);
final_mask=min(final_mask,mask_roi);
//choose the best two feature space for foreground****************
Mat hist_fore,hist_back;
channel_score.clear();
double sum_score=0;
for (int i=0;i<9;i++)
{
float range1[]={0,255};
if (i==3)
{
range1[1]=179;
}
const float* hist_range[]={range1};
Mat temp_hist_neg;
calcHist(roi_set,3,&i,final_mask,temp_hist,1,hist_size,hist_range);
normalize(temp_hist,temp_hist,255,0,NORM_L1);
calcHist(roi_set,3,&i,inv_mask_roi,temp_hist_neg,1,hist_size,hist_range);
normalize(temp_hist_neg,temp_hist_neg,255,0,NORM_L1);
log(temp_hist,temp_hist);
log(temp_hist_neg,temp_hist_neg);
temp_hist=temp_hist-temp_hist_neg;
threshold(temp_hist,temp_hist,0,255,CV_THRESH_TOZERO);
normalize(temp_hist,temp_hist,255,0.0,NORM_L1);//scale to 255 for display
calcBackProject(roi_set,3,&i,temp_hist,temp_bp,hist_range);
int c[]={0};
int hs[]={BIN_NUMBER};
float hr[]={0,255};
const float* hrr[]={hr};
calcHist(&temp_bp,1,c,final_mask,hist_fore,1,hs,hrr);
calcHist(&temp_bp,1,c,inv_mask_roi,hist_back,1,hs,hrr);
normalize(hist_fore,hist_fore,1.0,0.0,NORM_L1);
normalize(hist_back,hist_back,1.0,0.0,NORM_L1);
double score=getVR(hist_back,hist_fore);
score=score==score ? score:0;
channel_score.push_back(ChannelScore(i,score));
sum_score+=exp(score);
}
channel_score.sort(compareChannel);
channels[0]=channel_score.back().idx;
channel_score.pop_back();
channels[1]=channel_score.back().idx;
for (int i=0;i<2;++i)
{
_hRang[i][0]=0;
if (channels[i]==3)
_hRang[i][1]=179;
else
_hRang[i][1]=255;
hRange[i]=_hRang[i];
}
calcHist(roi_set,3,channels,final_mask,hist,2,hSize,hRange);///////////////////
normalize(hist,hist,255,0,NORM_L1);
//recover the shift_vector
Mat backPro;
calcBackProject(roi_set,3,channels,hist,backPro,hRange);
iniWin_roi=iniWin-Point(roi_win.x,roi_win.y);
Point2f origin_point_roi((float)(iniWin_roi.x+0.5*iniWin_roi.width),(float)(iniWin_roi.y+0.5*iniWin_roi.height));
meanShift(backPro,iniWin_roi,TermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ));
Point2f shift_point_roi((float)(iniWin_roi.x+0.5*iniWin_roi.width),(float)(iniWin_roi.y+0.5*iniWin_roi.height));
shift_vector=(shift_point_roi-origin_point_roi)*(1/(float)iniWin.width);
}
void MeanShiftDemo( VideoCapture& video, Rect& starting_position, int starting_frame_number, int end_frame)
{
bool half_size = true;
video.set(CV_CAP_PROP_POS_FRAMES,starting_frame_number);
Mat current_frame, hls_image;
std::vector<cv::Mat> hls_planes(3);
video >> current_frame;
Rect current_position(starting_position);
if (half_size)
{
resize(current_frame, current_frame, Size( current_frame.cols/2, current_frame.rows/2 ));
current_position.height /= 2;
current_position.width /= 2;
current_position.x /= 2;
current_position.y /= 2;
}
cvtColor(current_frame, hls_image, CV_BGR2HLS);
split(hls_image,hls_planes);
int chosen_channel = 0; // Hue channel
Mat image1ROI = hls_planes[chosen_channel](current_position);
float channel_range[2] = { 0.0, 255.0 };
int channel_numbers[1] = { 0 };
int number_bins[1] = { 32 };
MatND histogram[1];
const float* channel_ranges = channel_range;
calcHist(&(image1ROI), 1, channel_numbers, Mat(), histogram[0], 1 , number_bins, &channel_ranges);
normalize(histogram[0],histogram[0],1.0);
rectangle(current_frame,current_position,Scalar(0,255,0),2);
Mat starting_frame = current_frame.clone();
int frame_number = starting_frame_number;
while (!current_frame.empty() && (frame_number < end_frame))
{
// Calculate back projection
Mat back_projection_probabilities;
calcBackProject(&(hls_planes[chosen_channel]),1,channel_numbers,*histogram,back_projection_probabilities,&channel_ranges,255.0);
// Remove low saturation points from consideration
Mat saturation_mask;
inRange( hls_image, Scalar(0,10,50,0),Scalar(180,256,256,0), saturation_mask );
bitwise_and( back_projection_probabilities, back_projection_probabilities,back_projection_probabilities, saturation_mask );
// Mean shift
TermCriteria criteria(cv::TermCriteria::MAX_ITER,5,0.01);
meanShift(back_projection_probabilities,current_position,criteria);
// Output to screen
rectangle(current_frame,current_position,Scalar(0,255,0),2);
Mat chosen_channel_image, back_projection_image;
cvtColor(hls_planes[chosen_channel], chosen_channel_image, CV_GRAY2BGR);
cvtColor(back_projection_probabilities, back_projection_image, CV_GRAY2BGR);
Mat row1_output = JoinImagesHorizontally( starting_frame, "Starting position", chosen_channel_image, "Chosen channel (Hue)", 4 );
Mat row2_output = JoinImagesHorizontally( back_projection_image, "Back projection", current_frame, "Current position", 4 );
Mat mean_shift_output = JoinImagesVertically(row1_output,"",row2_output,"", 4);
imshow("Mean Shift Tracking", mean_shift_output );
// Advance to next frame
video >> current_frame;
if (half_size)
resize(current_frame, current_frame, Size( current_frame.cols/2, current_frame.rows/2 ));
cvtColor(current_frame, hls_image, CV_BGR2HLS);
split(hls_image,hls_planes);
frame_number++;
cvWaitKey(1000);
}
char c = cvWaitKey();
cvDestroyAllWindows();
}
