void Optimizer::ShowSegmentPoints(InputPtr input, ModelPtr model, Parameter* para) const
{
int num = 0;
model->EnableIterator(ModelImage::NOSE);
num = model->Size(ModelImage::NOSE);
for (int i = 0; i < num; ++i)
{
ivec2 pos = para->ComputePosition(i);
vec3 rgb = para->ComputeColor(i);
//vec3 rgb = input_image->GetColor(Point(pos[0], pos[1]));
//Vec3b color(static_cast<uchar>(rgb[2]), static_cast<uchar>(rgb[1]), static_cast<uchar>(rgb[0]));
Vec3b color(0, 0, 255);
model->WriteColor(pos[0], pos[1], color);
}
model->EnableIterator(ModelImage::EYE);
num = model->Size(ModelImage::EYE);
for (int i = 0; i < num; ++i)
{
ivec2 pos = para->ComputePosition(i);
vec3 rgb = para->ComputeColor(i);
//vec3 rgb = input_image->GetColor(Point(pos[0], pos[1]));
//Vec3b color(static_cast<uchar>(rgb[2]), static_cast<uchar>(rgb[1]), static_cast<uchar>(rgb[0]));
Vec3b color(0, 255, 0);
model->WriteColor(pos[0], pos[1], color);
}
model->EnableIterator(ModelImage::MOUTH);
num = model->Size(ModelImage::MOUTH);
for (int i = 0; i < num; ++i)
{
ivec2 pos = para->ComputePosition(i);
vec3 rgb = para->ComputeColor(i);
//vec3 rgb = input_image->GetColor(Point(pos[0], pos[1]));
//Vec3b color(static_cast<uchar>(rgb[2]), static_cast<uchar>(rgb[1]), static_cast<uchar>(rgb[0]));
Vec3b color(0, 255, 255);
model->WriteColor(pos[0], pos[1], color);
}
model->EnableIterator(ModelImage::REST);
num = model->Size(ModelImage::REST);
for (int i = 0; i < num; ++i)
{
ivec2 pos = para->ComputePosition(i);
vec3 rgb = para->ComputeColor(i);
//vec3 rgb = input_image->GetColor(Point(pos[0], pos[1]));
//Vec3b color(static_cast<uchar>(rgb[2]), static_cast<uchar>(rgb[1]), static_cast<uchar>(rgb[0]));
Vec3b color(255, 0, 0);
model->WriteColor(pos[0], pos[1], color);
}
}
void Optimizer::ShowPoints(mat& alpha,
mat& beta,
mat& rho,
mat& lamda,
InputPtr input,
ModelPtr model,
MeshPtr mesh,
ShapePtr shape,
TexturePtr texture,
int point_type)
{
mat alpha_gradient(PrincipalNum, 1);
alpha_gradient.fill(0);
mat beta_gradient(PrincipalNum, 1);
beta_gradient.fill(0);
mat rho_gradient(RhoNum, 1);
rho_gradient.fill(0);
mat lamda_gradient(LamdaNum, 1);
lamda_gradient.fill(0);
Alpha alpha_para(alpha, input, model, mesh, shape, texture);
Beta beta_para(beta, model, mesh, shape, texture);
Rho rho_para(rho, input, model, mesh, shape, texture);
Lamda lamda_para(lamda, model, mesh, shape, texture);
switch (point_type)
{
case RANDOM:
model->InitialRandomGenerator();
GenerateRandomPoints(model, 40);
ShowRandomPoints(input, model, &alpha_para);
break;
case VISIBLE_TRIANGLE:
model->EnableIterator();
ShowVisiblePoints(input, model, &alpha_para);
break;
case SHADOW_TRIANGLE:
model->EnableIterator();
ShowShadowPoints(input, model, &alpha_para);
break;
case SEGMENT:
ShowSegmentPoints(input, model, &alpha_para);
break;
default:
break;
}
model->Show();
}
void Optimizer::FitSegments(mat& alpha,
mat& beta,
mat& rho,
mat& lamda,
InputPtr input,
ModelPtr model,
MeshPtr mesh,
ShapePtr shape,
TexturePtr texture)
{
mat eye_alpha = alpha;
mat eye_beta = beta;
mat nose_alpha = alpha;
mat nose_beta = beta;
mat mouth_alpha = alpha;
mat mouth_beta = beta;
mat rest_alpha = alpha;
mat rest_beta = beta;
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
TwoPassZbuffer(rho, lamda, mesh, model, true); // cal segment two pass z-buffer
for (int c = 0; c < 3; ++c)
{
model->EnableIterator(ModelImage::NOSE);
FitNose(nose_alpha, nose_beta, rho, lamda, input, model, mesh, shape, texture);
model->EnableIterator(ModelImage::EYE);
FitEye(eye_alpha, eye_beta, rho, lamda, input, model, mesh, shape, texture);
model->EnableIterator(ModelImage::MOUTH);
FitMouth(mouth_alpha, mouth_beta, rho, lamda, input, model, mesh, shape, texture);
model->EnableIterator(ModelImage::REST);
FitRest(rest_alpha, rest_beta, rho, lamda, input, model, mesh, shape, texture);
}
mat all_alpha(PrincipalNum,SegmentsNum);
all_alpha.cols(0, 0) = nose_alpha;
all_alpha.cols(1, 1) = eye_alpha;
all_alpha.cols(2, 2) = mouth_alpha;
all_alpha.cols(3, 3) = rest_alpha;
mat all_beta(PrincipalNum, SegmentsNum);
all_beta.cols(0, 0) = nose_beta;
all_beta.cols(1, 1) = eye_beta;
all_beta.cols(2, 2) = mouth_beta;
all_beta.cols(3, 3) = rest_beta;
mesh = face3d_model.Construction(all_alpha, all_beta);
mesh->Blend();
rho.quiet_save("rho_para", arma_binary);
lamda.quiet_save("lamda_para", arma_binary);
}
void Optimizer::FitAll(mat& alpha,
mat& beta,
mat& rho,
mat& lamda,
InputPtr input,
ModelPtr model,
MeshPtr mesh,
ShapePtr shape,
TexturePtr texture)
{
mat alpha_gradient = mat(PrincipalNum, 1, fill::zeros);
mat alpha_hessian_inv = mat(PrincipalNum, PrincipalNum, fill::zeros);
mat beta_gradient = mat(PrincipalNum, 1, fill::zeros);
mat beta_hessian_inv = mat(PrincipalNum, PrincipalNum, fill::zeros);
mat rho_gradient = mat(RhoNum, 1, fill::zeros);
mat rho_hessian_inv = mat(RhoNum, RhoNum, fill::zeros);
mat lamda_gradient = mat(LamdaNum, 1, fill::zeros);
mat lamda_hessian_inv = mat(LamdaNum, LamdaNum, fill::zeros);
vec step(RhoNum, 1);
step.rows(0, 2).fill(0.00000006);
step.rows(3, 5).fill(0.0002);
step.rows(6, 6).fill(2.0);
mat step_mat = diagmat(step);
SetStartingValue(rho, lamda);
// 2500 1000 700 500 300 200
array<double, 6> weight_intensity = { { 1.0 / 1000, 1.0 / 900, 1.0 / 800, 1.0 / 700, 1.0 / 500, 1.0 / 300 } };
array<int, 6> para_num = { { 10, 15, 25, 35, 55, 99 } }; // 20 40
array<int, 6> iterations = { { 1000, 1000, 1000, 1000, 1000, 1000 } };
//array<double, 4> weight_intensity = { { 1.0 / 900, 1.0 / 700, 1.0 / 500, 1.0 / 400 } };
//array<int, 4> para_num = { { 10, 20, 40, 99 } }; // 20 40
//array<int, 4> iterations = { { 1000, 1000, 800, 600 } };
int counter = 0;
for (int c = 0; c < 6; ++c)
{
for (int l = 0; l < iterations[c]; ++l)
{
if (counter == 0 || counter % 1000 == 0)
{
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
TwoPassZbuffer(rho, lamda, mesh, model);
model->EnableIterator();
model->InitialRandomGenerator();
//GenerateRandomPoints(model, HessianRandomNum);
//for (int i = 0; i < para_num[c]; ++i)
//{
// double variance = shape->GetVariance(i);
// mat alpha1 = alpha;
// alpha1[i] -= H;
// Alpha alpha_para1(alpha1, input, model, mesh, shape, texture);
// double first_derivative1 = weight_feature[c] * ComputeLandmarkGradient(input, &alpha_para1, i) +
// weight_intensity[c] * ComputeIntensityGradient(input, &alpha_para1, i);
// mat alpha2 = alpha;
// alpha2[i] += H;
// Alpha alpha_para2(alpha2, input, model, mesh, shape, texture);
// double first_derivative2 = weight_feature[c] * ComputeLandmarkGradient(input, &alpha_para2, i) +
// weight_intensity[c] * ComputeIntensityGradient(input, &alpha_para2, i);
// double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
// alpha_hessian_inv(i, i) = 1 / (second_derivative +2 / variance);
//}
/* for (int i = 0; i < para_num[c]; ++i)
{
double variance = texture->GetVariance(i);
mat beta1 = beta;
beta1[i] -= H;
Beta beta_para1(beta1, model, mesh, shape, texture);
double first_derivative1 = weight_intensity[c] * ComputeIntensityGradient(input, &beta_para1, i);
mat beta2 = beta;
beta2[i] += H;
Beta beta_para2(beta2, model, mesh, shape, texture);
double first_derivative2 = weight_intensity[c] * ComputeIntensityGradient(input, &beta_para2, i);
double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
beta_hessian_inv(i, i) = 1 / (second_derivative + 2 / variance);
}*/
//for (int i = 0; i < RhoNum; ++i)
//{
// double variance = GetRhoVariance(i);
// mat rho1 = rho;
// rho1[i] -= H;
// Rho rho_para1(rho1, input, model, mesh, shape, texture);
// double first_derivative1 =/* weight_feature[c] * ComputeLandmarkGradient(input, &rho_para1, i) +*/
// weight_intensity[c] * ComputeIntensityGradient(input, &rho_para1, i);
// mat rho2 = rho;
// rho2[i] += H;
// Rho rho_para2(rho2, input, model, mesh, shape, texture);
// double first_derivative2 = /*weight_feature[c] * ComputeLandmarkGradient(input, &rho_para2, i) +*/
// weight_intensity[c] * ComputeIntensityGradient(input, &rho_para2, i);
// double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
// rho_hessian_inv(i, i) = 1 / (second_derivative + 2 / variance);
//}
//for (int i = 0; i < LamdaNum - 7; ++i)
//{
// double variance = GetLamdaVariance(i);
// mat lamda1 = lamda;
// lamda1[i] -= H;
// Lamda lamda_para1(lamda1, model, mesh, shape, texture);
// double first_derivative1 = weight_intensity[c] * ComputeIntensityGradient(input, &lamda_para1, i);
// mat lamda2 = lamda;
// lamda2[i] += H;
// Lamda lamda_para2(lamda2, model, mesh, shape, texture);
// double first_derivative2 = weight_intensity[c] * ComputeIntensityGradient(input, &lamda_para2, i);
// double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
// lamda_hessian_inv(i, i) = 1 / (second_derivative + 2 / variance);
//}
}
Alpha alpha_para(alpha, input, model, mesh, shape, texture);
Beta beta_para(beta, model, mesh, shape, texture);
Rho rho_para(rho, input, model, mesh, shape, texture);
Lamda lamda_para(lamda, model, mesh, shape, texture);
GenerateRandomPoints(model, GradientRandomNum);
//double function_value = 0;
//function_value = ComputeCost(input, model, alpha_para);
//ofstream cost;
//cost.open("all_cost", ios::app);
//cost << function_value << "\n";
//cost.close();
for (int i = 0; i < para_num[c]; ++i)
{
double variance = shape->GetVariance(i);
alpha_gradient[i] = /*weight_feature[c] * ComputeLandmarkGradient(input, &alpha_para, i) +*/
weight_intensity[c] * ComputeIntensityGradient(input, &alpha_para, i) +2 * alpha[i] / variance;
}
for (int i = 0; i < para_num[c]; ++i)
{
double variance = texture->GetVariance(i);
beta_gradient[i] = weight_intensity[c] * ComputeIntensityGradient(input, &beta_para, i) + 2 * beta[i] / variance;
}
for (int i = 0; i < RhoNum; ++i)
{
//double variance = GetRhoVariance(i);
//double mean = GetRhoMean(i);
rho_gradient[i] = /*weight_feature[c] * ComputeLandmarkGradient(input, &rho_para, i) +*/
weight_intensity[c] * ComputeIntensityGradient(input, &rho_para, i);// +2 * (rho[i] - mean) / variance;
}
for (int i = 0; i < LamdaNum; ++i)
{
double variance = GetLamdaVariance(i);
double mean = GetLamdaMean(i);
lamda_gradient[i] = weight_intensity[c] * ComputeIntensityGradient(input, &lamda_para, i) +2 * (lamda[i] - mean) / variance;
}
//alpha -= 0.00003*alpha_hessian_inv*alpha_gradient; // 0.01 0.00003
//beta -= 0.01*beta_hessian_inv*beta_gradient;
//lamda -= 0.03*lamda_hessian_inv*lamda_gradient;
//rho -= 0.0001*rho_hessian_inv*rho_gradient;
alpha -= alpha_para.Step*alpha_gradient;
beta -= beta_para.Step*beta_gradient;
rho -= step_mat*rho_gradient; // gradient descent
lamda -= lamda_para.Step*lamda_gradient;
++counter;
}
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
VisualizeResult(rho, lamda, input, mesh, "segmented", c);
}
}
void Optimizer::FitFirstPrincipalComponents(mat& alpha,
mat& beta,
mat& rho,
mat& lamda,
InputPtr input,
ModelPtr model,
MeshPtr mesh,
ShapePtr shape,
TexturePtr texture,
const cv::Mat& ready)
{
mat alpha_gradient = mat(PrincipalNum, 1, fill::zeros);
mat alpha_hessian_inv = mat(PrincipalNum, PrincipalNum, fill::zeros);
mat beta_gradient = mat(PrincipalNum, 1, fill::zeros);
mat rho_gradient = mat(RhoNum, 1, fill::zeros);
mat lamda_gradient = mat(LamdaNum, 1, fill::zeros);
vec step(RhoNum, 1);
step.rows(0, 2).fill(0.00000006);
step.rows(3, 5).fill(0.0002);
step.rows(6, 6).fill(2.0);
mat step_mat = diagmat(step);
// down sampled version
cv::Mat down_sampled;
pyrDown(ready, down_sampled, Size(IMAGE_WIDTH / 2, IMAGE_HEIGHT / 2));
input = make_shared<InputImage>(down_sampled);
model->SetSize(IMAGE_WIDTH / 2, IMAGE_HEIGHT / 2);
rho[FOCAL] /= 2;
//SetStartingValue(rho, lamda);
double para_num = 10;
double weight = 1.0 / 900;
int counter = 0;
while (counter < 1000)
{
Alpha alpha_para(alpha, input, model, mesh, shape, texture);
Beta beta_para(beta, model, mesh, shape, texture);
Rho rho_para(rho, input, model, mesh, shape, texture);
Lamda lamda_para(lamda, model, mesh, shape, texture);
if (counter == 0 || counter % 1000 == 0)
{
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
TwoPassZbuffer(rho, lamda, mesh, model);
model->EnableIterator();
model->InitialRandomGenerator();
//GenerateRandomPoints(model, HessianRandomNum);
//for (int i = 0; i < para_num; ++i)
//{
// double variance = shape->GetVariance(i);
// mat alpha1 = alpha;
// alpha1[i] -= H;
// Alpha alpha_para1(alpha1, input, model, mesh, shape, texture);
// double first_derivative1 = weight * ComputeIntensityGradient(input, &alpha_para1, i) + 2 * alpha1[i] / variance;
// mat alpha2 = alpha;
// alpha2[i] += H;
// Alpha alpha_para2(alpha2, input, model, mesh, shape, texture);
// double first_derivative2 = weight * ComputeIntensityGradient(input, &alpha_para2, i) + 2 * alpha2[i] / variance;
// double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
// alpha_hessian_inv(i, i) = 1 / (second_derivative + 2 / variance);
//}
}
// Generate random points
GenerateRandomPoints(model, GradientRandomNum);
double function_value = 0;
function_value = ComputeCost(input, model, alpha_para);
ofstream cost;
cost.open("first_cost", ios::app);
cost << function_value << "\n";
cost.close();
for (int i = 0; i < para_num; ++i)
{
double variance = shape->GetVariance(i);
alpha_gradient[i] = weight * ComputeIntensityGradient(input, &alpha_para, i) + 2 * alpha[i] / variance;
}
//for (int i = 0; i < 10; ++i)
//{
// double variance = texture->GetVariance(i);
// beta_gradient[i] = weight *ComputeIntensityGradient(input, &beta_para, i) + 2 * beta[i] / variance;
//}
//for (int i = 0; i < RhoNum; ++i)
//{
// double first_derivative1 = 1.0 / 1000*ComputeIntensityGradient(input, &rho_para, i);
// //double variance = rho_para.GetVariance(i);
// //double mean = rho_para.GetMean(i);
// rho_gradient[i] = first_derivative1;// +2 * (rho[i] - mean) / variance;
//}
//for (int i = 0; i < LamdaNum-7; ++i)
//{
// double first_derivative1 = 1.0 / 1000 * ComputeIntensityGradient(input, &lamda_para, i);
// //double variance = lamda_para.GetVariance(i);
// //double mean = lamda_para.GetMean(i);
// lamda_gradient[i] = first_derivative1; //+2 * (lamda[i] - mean) / variance;
//}
alpha -= alpha_para.Step*alpha_gradient;
//beta -= beta_para.Step*beta_gradient;
//rho -= step_mat*rho_gradient;
//lamda -= lamda_para.Step*lamda_gradient;
//alpha -= 0.00008*alpha_hessian_inv*alpha_gradient;
++counter;
}
// restore original version
input= make_shared<InputImage>(ready);
model->SetSize(IMAGE_WIDTH, IMAGE_HEIGHT);
rho[FOCAL] *= 2;
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
VisualizeResult(rho, lamda, input, mesh, "first");
}
void Optimizer::FitIllumination(mat& alpha,
mat& beta,
mat& rho,
mat& lamda,
InputPtr input,
ModelPtr model,
MeshPtr mesh,
ShapePtr shape,
TexturePtr texture,
const cv::Mat& ready)
{
mat lamda_gradient=mat(LamdaNum, 1,fill::zeros);
mat lamda_hessian_inv = mat(LamdaNum, LamdaNum, fill::zeros);
Face3dModel face3d_model(shape, texture);
mesh = face3d_model.Construction(alpha, beta);
//VisualizeResult(rho, lamda, input, mesh, 0);
TwoPassZbuffer(rho, lamda, mesh, model);
model->EnableIterator();
model->InitialRandomGenerator();
// down sampled version
cv::Mat down_sampled;
pyrDown(ready, down_sampled, Size(IMAGE_WIDTH / 2, IMAGE_HEIGHT / 2));
input = make_shared<InputImage>(down_sampled);
model->SetSize(IMAGE_WIDTH / 2, IMAGE_HEIGHT / 2);
rho[FOCAL] /= 2;
double weight = 1.0 / 1000;
int counter = 0;
while (counter < 500)
{
Alpha alpha_para(alpha, input, model, mesh, shape, texture);
Beta beta_para(beta, model, mesh, shape, texture);
Rho rho_para(rho, input, model, mesh, shape, texture);
Lamda lamda_para(lamda, model, mesh, shape, texture);
// Generate random points
//if (counter == 0 || counter % 1000 == 0)
//{
// GenerateRandomPoints(model, HessianRandomNum);
// for (int i = 0; i < LamdaNum - 7; ++i)
// {
// mat lamda1 = lamda;
// lamda1[i] -= H;
// Lamda lamda_para1(lamda1, model, mesh, shape, texture);
// double first_derivative1 = weight*ComputeIntensityGradient(input, &lamda_para1, i);
// mat lamda2 = lamda;
// lamda2[i] += H;
// Lamda lamda_para2(lamda2, model, mesh, shape, texture);
// double first_derivative2 = weight* ComputeIntensityGradient(input, &lamda_para2, i);
// double second_derivative = (first_derivative2 - first_derivative1) / (2 * H);
// lamda_hessian_inv(i, i) = 1 / second_derivative;
// }
//}
GenerateRandomPoints(model, GradientRandomNum);
//double function_value = 0;
//function_value = ComputeCost(input, model, alpha_para);
//ofstream cost;
//cost.open("illumination_cost", ios::app);
//cost << function_value << "\n";
//cost.close();
for (int i = 0; i < LamdaNum; ++i)
{
lamda_gradient[i] = weight*ComputeIntensityGradient(input, &lamda_para, i);
}
lamda -= lamda_para.Step*lamda_gradient;
//lamda -= 0.5*lamda_hessian_inv*lamda_gradient;
++counter;
}
// restore original version
input= make_shared<InputImage>(ready);
model->SetSize(IMAGE_WIDTH, IMAGE_HEIGHT);
rho[FOCAL] *= 2;
VisualizeResult(rho, lamda, input, mesh, "illumination");
}