void
PeriodicPredictor::calculate_autocorrelation(std::vector<double> *container) {
std::vector<double> samples_copy;
std::deque<double>::iterator it;
for (it = this->samples.begin(); it != this->samples.end(); it++) {
samples_copy.push_back(*it);
}
for (int32_t offset = 0; offset <= this->sample_size/2; offset++) {
std::vector<double> container1(samples_copy.begin(),
samples_copy.end() - offset);
std::vector<double> container2(samples_copy.begin() + offset,
samples_copy.end());
double variance1 = calculate_variance(container1);
double variance2 = calculate_variance(container2);
double covariance = calculate_covariance(container1, container2);
double correlation = (variance1 == 0 || variance2 == 0) ?
1.0 : covariance/(variance1*variance2);
container->push_back(correlation);
}
}
vector<Mat_<double> > FernCascade::Train(const vector<Mat_<uchar> >& images,
const vector<Mat_<double> >& current_shapes,
const vector<Mat_<double> >& ground_truth_shapes,
const vector<BoundingBox> & bounding_box,
const Mat_<double>& mean_shape,
int second_level_num,
int candidate_pixel_num,
int fern_pixel_num){
Mat_<double> candidate_pixel_locations(candidate_pixel_num,2);
Mat_<int> nearest_landmark_index(candidate_pixel_num,1);
vector<Mat_<double> > regression_targets;
RNG random_generator(getTickCount());
second_level_num_ = second_level_num;
// calculate regression targets: the difference between ground truth shapes and current shapes
// candidate_pixel_locations: the locations of candidate pixels, indexed relative to its nearest landmark on mean shape
regression_targets.resize(current_shapes.size());
for(int i = 0;i < current_shapes.size();i++){
regression_targets[i] = ProjectShape(ground_truth_shapes[i],bounding_box[i])
- ProjectShape(current_shapes[i],bounding_box[i]);
Mat_<double> rotation;
double scale;
SimilarityTransform(mean_shape,ProjectShape(current_shapes[i],bounding_box[i]),rotation,scale);
transpose(rotation,rotation);
regression_targets[i] = scale * regression_targets[i] * rotation;
}
// get candidate pixel locations, please refer to 'shape-indexed features'
for(int i = 0;i < candidate_pixel_num;i++){
double x = random_generator.uniform(-1.0,1.0);
double y = random_generator.uniform(-1.0,1.0);
if(x*x + y*y > 1.0){
i--;
continue;
}
// find nearest landmark index
double min_dist = 1e10;
int min_index = 0;
for(int j = 0;j < mean_shape.rows;j++){
double temp = pow(mean_shape(j,0)-x,2.0) + pow(mean_shape(j,1)-y,2.0);
if(temp < min_dist){
min_dist = temp;
min_index = j;
}
}
candidate_pixel_locations(i,0) = x - mean_shape(min_index,0);
candidate_pixel_locations(i,1) = y - mean_shape(min_index,1);
nearest_landmark_index(i) = min_index;
}
// get densities of candidate pixels for each image
// for densities: each row is the pixel densities at each candidate pixels for an image
// Mat_<double> densities(images.size(), candidate_pixel_num);
vector<vector<double> > densities;
densities.resize(candidate_pixel_num);
for(int i = 0;i < images.size();i++){
Mat_<double> rotation;
double scale;
Mat_<double> temp = ProjectShape(current_shapes[i],bounding_box[i]);
SimilarityTransform(temp,mean_shape,rotation,scale);
for(int j = 0;j < candidate_pixel_num;j++){
double project_x = rotation(0,0) * candidate_pixel_locations(j,0) + rotation(0,1) * candidate_pixel_locations(j,1);
double project_y = rotation(1,0) * candidate_pixel_locations(j,0) + rotation(1,1) * candidate_pixel_locations(j,1);
project_x = scale * project_x * bounding_box[i].width / 2.0;
project_y = scale * project_y * bounding_box[i].height / 2.0;
int index = nearest_landmark_index(j);
int real_x = project_x + current_shapes[i](index,0);
int real_y = project_y + current_shapes[i](index,1);
real_x = std::max(0.0,std::min((double)real_x,images[i].cols-1.0));
real_y = std::max(0.0,std::min((double)real_y,images[i].rows-1.0));
densities[j].push_back((int)images[i](real_y,real_x));
}
}
// calculate the covariance between densities at each candidate pixels
Mat_<double> covariance(candidate_pixel_num,candidate_pixel_num);
Mat_<double> mean;
for(int i = 0;i < candidate_pixel_num;i++){
for(int j = i;j< candidate_pixel_num;j++){
double correlation_result = calculate_covariance(densities[i],densities[j]);
covariance(i,j) = correlation_result;
covariance(j,i) = correlation_result;
}
}
// train ferns
vector<Mat_<double> > prediction;
prediction.resize(regression_targets.size());
for(int i = 0;i < regression_targets.size();i++){
prediction[i] = Mat::zeros(mean_shape.rows,2,CV_64FC1);
}
ferns_.resize(second_level_num);
for(int i = 0;i < second_level_num;i++){
cout<<"Training ferns: "<<i+1<<" out of "<<second_level_num<<endl;
vector<Mat_<double> > temp = ferns_[i].Train(densities,covariance,candidate_pixel_locations,nearest_landmark_index,regression_targets,fern_pixel_num);
// update regression targets
for(int j = 0;j < temp.size();j++){
prediction[j] = prediction[j] + temp[j];
regression_targets[j] = regression_targets[j] - temp[j];
}
}
for(int i = 0;i < prediction.size();i++){
Mat_<double> rotation;
double scale;
SimilarityTransform(ProjectShape(current_shapes[i],bounding_box[i]),mean_shape,rotation,scale);
transpose(rotation,rotation);
prediction[i] = scale * prediction[i] * rotation;
}
return prediction;
}
/**
* Train a fern cascade.
* @param images training images in gray scale
* @param normalize_matrix similarity matrix
* @param target_shapes target shapes of each face image
* @param mean_shape mean shape
* @param second_level_num level number for second level regression
* @param current_shapes current shapes of training images
* @param pixel_pair_num number of pair of pixels to be selected
* @param normalized_targets (target - current) * normalize_matrix
*/
void FernCascade::train(const vector<Mat_<uchar> >& images,
const vector<Mat_<double> >& target_shapes,
int second_level_num,
vector<Mat_<double> >& current_shapes,
int pixel_pair_num,
vector<Mat_<double> >& normalized_targets,
int pixel_pair_in_fern,
const Mat_<double>& mean_shape,
const vector<Bbox>& target_bounding_box){
second_level_num_ = second_level_num;
// coordinates of selected pixels
Mat_<double> pixel_coordinates(pixel_pair_num,2);
Mat_<int> nearest_keypoint_index(pixel_pair_num,1);
RNG random_generator(getTickCount());
int landmark_num = current_shapes[0].rows;
int training_num = images.size();
int image_width = images[0].cols;
int image_height = images[0].rows;
/* vector<Mat_<double> > normalized_curr_shape; */
// get bounding box of target shapes
vector<Mat_<double> > normalized_shapes;
/* vector<Mat_<double> > normalized_ground_truth; */
normalized_shapes = project_shape(current_shapes,target_bounding_box);
/* normalized_ground_truth = project_shape(target_shapes,target_bounding_box); */
/*
vector<SimilarityTransform> curr_to_mean_shape;
vector<SimilarityTransform> ground_to_mean_shape;
curr_to_mean_shape = get_similarity_transform(mean_shape,normalized_shapes);
ground_to_mean_shape = get_similarity_transform(mean_shape,normalized_ground_truth);
vector<Mat_<double> > curr_project_to_mean;
vector<Mat_<double> > ground_project_to_mean;
for(int i = 0;i < current_shapes.size();i++){
ipd// get normalized_targets
Mat_<double> temp = curr_to_mean_shape[i].apply_similarity_transform(normalized_shapes[i]);
curr_project_to_mean.push_back(temp.clone());
temp = ground_to_mean_shape[i].apply_similarity_transform(normalized_ground_truth[i]);
ground_to_mean_shape.push_back(temp.clone());
}i
*/
// calculate normalized targets
normalized_targets = inverse_shape(current_shapes,target_bounding_box);
normalized_targets = compose_shape(normalized_targets,target_shapes,target_bounding_box);
for(int i = 0;i < normalized_targets.size();i++){
Mat_<double> rotation;
Mat_<double> translation;
double scale;
translate_scale_rotate(mean_shape,normalized_shapes[i],translation,scale,rotation);
transpose(rotation,rotation);
normalized_targets[i] = scale * normalized_targets[i] * rotation;
}
// normalized_targets.clear();
// for(int i = 0;i < curr_project_to_mean.size();i++){
// normalized_targets.push_back(ground_project_to_mean[i] - curr_project_to_mean[i]);
// }
// generate feature pixel location
for(int i = 0;i < pixel_pair_num;i++){
double x = random_generator.uniform(-1.0,1.0);
double y = random_generator.uniform(-1.0,1.0);
if(x*x + y*y > 1){
i--;
continue;
}
// get its nearest landmark
double min_dist = 1e10;
int min_index = 0;
for(int j = 0;j < landmark_num;j++){
double temp = pow(mean_shape(j,0) - x,2.0) + pow(mean_shape(j,1) - y,2.0);
if(temp < min_dist){
min_dist = temp;
min_index = j;
}
}
nearest_keypoint_index(i) = min_index;
pixel_coordinates(i,0) = x - mean_shape(min_index,0);
pixel_coordinates(i,1) = y - mean_shape(min_index,1);
}
// get feature pixel location for each image
// for pixel_density, each vector in it stores the pixel value for each image on the corresponding pixel locations
vector<vector<double> > pixel_density;
pixel_density.resize(pixel_pair_num);
for(int i = 0;i < normalized_shapes.size();i++){
// similarity transform from normalized_shapes to mean shape
Mat_<double> rotation(2,2);
Mat_<double> translation(landmark_num,2);
double scale = 0;
translate_scale_rotate(normalized_shapes[i],mean_shape,translation,scale,rotation);
for(int j = 0;j < pixel_pair_num;j++){
double x = pixel_coordinates(j,0);
double y = pixel_coordinates(j,1);
double project_x = rotation(0,0) * x + rotation(0,1) * y;
double project_y = rotation(1,0) * x + rotation(1,1) * y;
project_x = project_x * scale;
project_y = project_y * scale;
// resize according to bounding_box
project_x = project_x * target_bounding_box[i].width/2.0;
project_y = project_y * target_bounding_box[i].height/2.0;
int index = nearest_keypoint_index(j);
int real_x = project_x + current_shapes[i](index,0);
int real_y = project_y + current_shapes[i](index,1);
if(real_x < 0){
real_x = 0;
}
if(real_y < 0){
real_y = 0;
}
if(real_x > images[i].cols-1){
real_x = images[i].cols-1;
}
if(real_y > images[i].rows - 1){
real_y = images[i].rows - 1;
}
pixel_density[j].push_back(int(images[i](real_y,real_x)));
}
}
// calculate the correlation between pixels
Mat_<double> correlation(pixel_pair_num,pixel_pair_num);
for(int i = 0;i < pixel_pair_num;i++){
for(int j = i;j< pixel_pair_num;j++){
double correlation_result = calculate_covariance(pixel_density[i],pixel_density[j]);
correlation(i,j) = correlation_result;
correlation(j,i) = correlation_result;
}
}
// train ferns
primary_fern_.resize(second_level_num);
// predications for each shape
vector<Mat_<double> > prediction;
prediction.resize(current_shapes.size());
for(int i = 0;i < current_shapes.size();i++){
prediction[i] = Mat::zeros(landmark_num,2,CV_64FC1);
}
for(int i = 0;i < second_level_num;i++){
cout<<"Training fern "<<i<<endl;
primary_fern_[i].train(pixel_density,correlation,pixel_coordinates,nearest_keypoint_index, current_shapes,pixel_pair_in_fern,normalized_targets,
prediction);
}
for(int i = 0;i < prediction.size();i++){
Mat_<double> rotation;
Mat_<double> translation;
double scale;
translate_scale_rotate(normalized_shapes[i],mean_shape,translation,scale,rotation);
transpose(rotation,rotation);
prediction[i] = scale * prediction[i] * rotation;
}
current_shapes = compose_shape(prediction, current_shapes, target_bounding_box);
current_shapes = reproject_shape(current_shapes, target_bounding_box);
/* Mat_<uchar> test_image_1 = images[10].clone(); */
// for(int i = 0;i < landmark_num;i++){
// circle(test_image_1,Point2d(current_shapes[10](i,0),current_shapes[10](i,1)),3,Scalar(255,0,0),-1,8,0);
// }
// imshow("result",test_image_1);
/* waitKey(0); */
}
