//============================================================================ void AAM_TDM::Train(const file_lists& pts_files, const file_lists& img_files, const AAM_PAW& m_warp, double texture_percentage /* = 0.975 */, bool registration /* = true */) { int nPoints = m_warp.nPoints(); int nPixels = m_warp.nPix()*3; int nSamples = pts_files.size(); CvMat *AllTextures = cvCreateMat(nSamples, nPixels, CV_64FC1); CvMat * matshape = cvCreateMat(1, nPoints*2, CV_64FC1); for(int i = 0; i < nSamples; i++) { IplImage* image = cvLoadImage(img_files[i].c_str(), -1); AAM_Shape trueshape; if(!trueshape.ReadAnnotations(pts_files[i])) trueshape.ScaleXY(image->width, image->height); trueshape.Point2Mat(matshape); AAM_Common::CheckShape(matshape, image->width, image->height); CvMat t; cvGetRow(AllTextures, &t, i); m_warp.CalcWarpTexture(matshape, image, &t); cvReleaseImage(&image); } cvReleaseMat(&matshape); // align texture so as to minimize the lighting variation AAM_TDM::AlignTextures(AllTextures); //now do pca DoPCA(AllTextures, texture_percentage); if(registration) SaveSeriesTemplate(AllTextures, m_warp); cvReleaseMat(&AllTextures); }
//============================================================================ void AAM_IC::Train(const file_lists& pts_files, const file_lists& img_files, double scale /* = 1.0 */, double shape_percentage /* = 0.975 */, double texture_percentage /* = 0.975 */) { if(pts_files.size() != img_files.size()) { fprintf(stderr, "ERROE(%s, %d): #Shapes != #Images\n", __FILE__, __LINE__); exit(0); } printf("################################################\n"); printf("Build Inverse Compositional Image Alignmennt Model...\n"); std::vector<AAM_Shape> AllShapes; for(int ii = 0; ii < pts_files.size(); ii++) { AAM_Shape Shape; bool flag = Shape.ReadAnnotations(pts_files[ii]); if(!flag) { IplImage* image = cvLoadImage(img_files[ii].c_str(), -1); Shape.ScaleXY(image->width, image->height); cvReleaseImage(&image); } AllShapes.push_back(Shape); } //building shape and texture distribution model printf("Build point distribution model...\n"); __shape.Train(AllShapes, scale, shape_percentage); printf("Build warp information of mean shape mesh..."); __Points = cvCreateMat (1, __shape.nPoints(), CV_32FC2); __Storage = cvCreateMemStorage(0); double sp = 1.0; //if(__shape.GetMeanShape().GetWidth() > 48) // sp = 48/__shape.GetMeanShape().GetWidth(); __paw.Train(__shape.GetMeanShape()*sp, __Points, __Storage); printf("[%d by %d, triangles #%d, pixels #%d*3]\n", __paw.Width(), __paw.Height(), __paw.nTri(), __paw.nPix()); printf("Build texture distribution model...\n"); __texture.Train(pts_files, img_files, __paw, texture_percentage, true); //calculate gradient of texture printf("Calculating texture gradient...\n"); CvMat* dTx = cvCreateMat(1, __texture.nPixels(), CV_64FC1); CvMat* dTy = cvCreateMat(1, __texture.nPixels(), CV_64FC1); CalcTexGrad(__texture.GetMean(), dTx, dTy); // save gradient image mkdir("Modes"); __paw.SaveWarpTextureToImage("Modes/dTx.jpg", dTx); __paw.SaveWarpTextureToImage("Modes/dTy.jpg", dTy); //calculate warp Jacobian at base shape printf("Calculating warp Jacobian...\n"); CvMat* Jx = cvCreateMat(__paw.nPix(), __shape.nModes()+4, CV_64FC1); CvMat* Jy = cvCreateMat(__paw.nPix(), __shape.nModes()+4, CV_64FC1); CalcWarpJacobian(Jx,Jy); //calculate modified steepest descent image printf("Calculating steepest descent images...\n"); CvMat* SD = cvCreateMat(__shape.nModes()+4, __texture.nPixels(), CV_64FC1); CalcModifiedSD(SD, dTx, dTy, Jx, Jy); //calculate inverse Hessian matrix printf("Calculating Hessian inverse matrix...\n"); CvMat* H = cvCreateMat(__shape.nModes()+4, __shape.nModes()+4, CV_64FC1); CalcHessian(H, SD); //calculate update matrix (multiply inverse Hessian by modified steepest descent image) __G = cvCreateMat(__shape.nModes()+4, __texture.nPixels(), CV_64FC1); cvMatMul(H, SD, __G); //release cvReleaseMat(&Jx); cvReleaseMat(&Jy); cvReleaseMat(&dTx); cvReleaseMat(&dTy); cvReleaseMat(&SD); cvReleaseMat(&H); //alocate memory for on-line fitting stuff __update_s0 = cvCreateMat(1, __shape.nPoints()*2, CV_64FC1); __inv_pq = cvCreateMat(1, __shape.nModes()+4, CV_64FC1); __warp_t = cvCreateMat(1, __texture.nPixels(), CV_64FC1); __error_t = cvCreateMat(1, __texture.nPixels(), CV_64FC1); __search_pq = cvCreateMat(1, __shape.nModes()+4, CV_64FC1); __delta_pq = cvCreateMat(1, __shape.nModes()+4, CV_64FC1); __current_s = cvCreateMat(1, __shape.nPoints()*2, CV_64FC1); __update_s = cvCreateMat(1, __shape.nPoints()*2, CV_64FC1); __lamda = cvCreateMat(1, __texture.nModes(), CV_64FC1); printf("################################################\n\n"); }
//============================================================================ void AAM_CAM::Train(const file_lists& pts_files, const file_lists& img_files, double scale /* = 1.0 */, double shape_percentage /* = 0.975 */, double texture_percentage /* = 0.975 */, double appearance_percentage /* = 0.975 */) { //building shape and texture distribution model std::vector<AAM_Shape> AllShapes; for(int ii = 0; ii < pts_files.size(); ii++) { AAM_Shape Shape; bool flag = Shape.ReadAnnotations(pts_files[ii]); if(!flag) { IplImage* image = cvLoadImage(img_files[ii].c_str(), -1); Shape.ScaleXY(image->width, image->height); cvReleaseImage(&image); } AllShapes.push_back(Shape); } printf("Build point distribution model...\n"); __shape.Train(AllShapes, scale, shape_percentage); printf("Build warp information of mean shape mesh..."); __Points = cvCreateMat (1, __shape.nPoints(), CV_32FC2); __Storage = cvCreateMemStorage(0); AAM_Shape refShape = __shape.__AAMRefShape/* * scale */; //if(refShape.GetWidth() > 50) // refShape.Scale(50/refShape.GetWidth()); __paw.Train(refShape, __Points, __Storage); printf("[%d by %d, %d triangles, %d*3 pixels]\n", __paw.Width(), __paw.Height(), __paw.nTri(), __paw.nPix()); printf("Build texture distribution model...\n"); __texture.Train(pts_files, img_files, __paw, texture_percentage, true); __pq = cvCreateMat(1, __shape.nModes()+4, CV_64FC1); printf("Build combined appearance model...\n"); int nsamples = pts_files.size(); int npointsby2 = __shape.nPoints()*2; int npixels = __texture.nPixels(); int nfeatures = __shape.nModes() + __texture.nModes(); CvMat* AllAppearances = cvCreateMat(nsamples, nfeatures, CV_64FC1); CvMat* s = cvCreateMat(1, npointsby2, CV_64FC1); CvMat* t = cvCreateMat(1, npixels, CV_64FC1); __MeanS = cvCreateMat(1, npointsby2, CV_64FC1); __MeanG = cvCreateMat(1, npixels, CV_64FC1); cvCopy(__shape.GetMean(), __MeanS); cvCopy(__texture.GetMean(), __MeanG); //calculate ratio of shape to appearance CvScalar Sum1 = cvSum(__shape.__ShapesEigenValues); CvScalar Sum2 = cvSum(__texture.__TextureEigenValues); __WeightsS2T = sqrt(Sum2.val[0] / Sum1.val[0]); printf("Combine shape and texture parameters...\n"); for(int i = 0; i < nsamples; i++) { //Get Shape and Texture respectively IplImage* image = cvLoadImage(img_files[i].c_str(), -1); AAM_Shape Shape; if(!Shape.ReadAnnotations(pts_files[i])) Shape.ScaleXY(image->width, image->height); Shape.Point2Mat(s); AAM_Common::CheckShape(s, image->width, image->height); __paw.CalcWarpTexture(s, image, t); __texture.NormalizeTexture(__MeanG, t); //combine shape and texture parameters CvMat OneAppearance; cvGetRow(AllAppearances, &OneAppearance, i); ShapeTexture2Combined(s, t, &OneAppearance); cvReleaseImage(&image); } //Do PCA of appearances DoPCA(AllAppearances, appearance_percentage); int np = __AppearanceEigenVectors->rows; printf("Extracting the shape and texture part of the combined eigen vectors..\n"); // extract the shape part of the combined eigen vectors CvMat Ps; cvGetCols(__AppearanceEigenVectors, &Ps, 0, __shape.nModes()); __Qs = cvCreateMat(np, npointsby2, CV_64FC1); cvMatMul(&Ps, __shape.GetBases(), __Qs); cvConvertScale(__Qs, __Qs, 1.0/__WeightsS2T); // extract the texture part of the combined eigen vectors CvMat Pg; cvGetCols(__AppearanceEigenVectors, &Pg, __shape.nModes(), nfeatures); __Qg = cvCreateMat(np, npixels, CV_64FC1); cvMatMul(&Pg, __texture.GetBases(), __Qg); __a = cvCreateMat(1, __AppearanceEigenVectors->cols, CV_64FC1); }
//============================================================================ void AAM_Basic::CalcJacobianMatrix(const file_lists& pts_files, const file_lists& img_files, double disp_scale /* = 0.2 */, double disp_angle /* = 20 */, double disp_trans /* = 5.0 */, double disp_std /* = 1.0 */, int nExp /* = 30 */) { CvMat* J = cvCreateMat(__cam.nModes()+4, __cam.__texture.nPixels(), CV_64FC1); CvMat* d = cvCreateMat(1, __cam.nModes()+4, CV_64FC1); CvMat* o = cvCreateMat(1, __cam.nModes()+4, CV_64FC1); CvMat* oo = cvCreateMat(1, __cam.nModes()+4, CV_64FC1); CvMat* t = cvCreateMat(1, __cam.__texture.nPixels(), CV_64FC1); CvMat* t_m = cvCreateMat(1, __cam.__texture.nPixels(), CV_64FC1); CvMat* t_s = cvCreateMat(1, __cam.__texture.nPixels(), CV_64FC1); CvMat* t1 = cvCreateMat(1, __cam.__texture.nPixels(), CV_64FC1); CvMat* t2 = cvCreateMat(1, __cam.__texture.nPixels(), CV_64FC1); CvMat* u = cvCreateMat(1, __cam.nModes()+4, CV_64FC1); CvMat* c = cvCreateMat(1, __cam.nModes(), CV_64FC1); CvMat* s = cvCreateMat(1, __cam.__shape.nPoints()*2, CV_64FC1); CvMat* q = cvCreateMat(1, 4, CV_64FC1); CvMat* p = cvCreateMat(1, __cam.__shape.nModes(),CV_64FC1); CvMat* lamda = cvCreateMat(1, __cam.__texture.nModes(), CV_64FC1); double theta = disp_angle * CV_PI / 180; double aa = MAX(fabs(disp_scale*cos(theta)), fabs(disp_scale*sin(theta))); cvmSet(d,0,0,aa); cvmSet(d,0,1,aa); cvmSet(d,0,2,disp_trans); cvmSet(d,0,3,disp_trans); for(int nmode = 0; nmode < __cam.nModes(); nmode++) cvmSet(d,0,4+nmode,disp_std*sqrt(__cam.Var(nmode))); srand(unsigned(time(0))); cvSetZero(u);cvSetZero(J); for(int i = 0; i < pts_files.size(); i++) { IplImage* image = cvLoadImage(img_files[i].c_str(), -1); AAM_Shape Shape; if(!Shape.ReadAnnotations(pts_files[i])) Shape.ScaleXY(image->width, image->height); Shape.Point2Mat(s); //calculate current texture vector __cam.__paw.CalcWarpTexture(s, image, t); __cam.__texture.NormalizeTexture(__cam.__MeanG, t); //calculate appearance parameters __cam.__shape.CalcParams(s, p, q); __cam.__texture.CalcParams(t, lamda); __cam.CalcParams(c, p, lamda); //update appearance and pose parameters CvMat subo; cvGetCols(o, &subo, 0, 4); cvCopy(q, &subo); cvGetCols(o, &subo, 4, 4+__cam.nModes()); cvCopy(c, &subo); //get optimal EstResidual EstResidual(image, o, s, t_m, t_s, t1); for(int j = 0; j < nExp; j++) { printf("Pertubing (%d/%d) for image (%d/%d)...\r", j, nExp, i, pts_files.size()); for(int l = 0; l < 4+__cam.nModes(); l++) { double D = cvmGet(d,0,l); double v = rand_in_between(-D, D); cvCopy(o, oo); CV_MAT_ELEM(*oo,double,0,l) += v; EstResidual(image, oo, s, t_m, t_s, t2); cvSub(t1, t2, t2); cvConvertScale(t2, t2, 1.0/v); //accumulate into l-th row CvMat Jl; cvGetRow(J, &Jl, l); cvAdd(&Jl, t2, &Jl); CV_MAT_ELEM(*u, double, 0, l) += 1.0; } } cvReleaseImage(&image); } //normalize for(int l = 0; l < __cam.nModes()+4; l++) { CvMat Jl; cvGetRow(J, &Jl, l); cvConvertScale(&Jl, &Jl, 1.0/cvmGet(u,0,l)); } CvMat* JtJ = cvCreateMat(__cam.nModes()+4, __cam.nModes()+4, CV_64FC1); CvMat* InvJtJ = cvCreateMat(__cam.nModes()+4, __cam.nModes()+4, CV_64FC1); cvGEMM(J, J, 1, NULL, 0, JtJ, CV_GEMM_B_T); cvInvert(JtJ, InvJtJ, CV_SVD); cvMatMul(InvJtJ, J, __G); cvReleaseMat(&J); cvReleaseMat(&d); cvReleaseMat(&o); cvReleaseMat(&oo); cvReleaseMat(&t); cvReleaseMat(&t_s); cvReleaseMat(&t_m); cvReleaseMat(&t1); cvReleaseMat(&t2); cvReleaseMat(&u); cvReleaseMat(&c); cvReleaseMat(&s); cvReleaseMat(&q); cvReleaseMat(&p); cvReleaseMat(&lamda); cvReleaseMat(&JtJ); cvReleaseMat(&InvJtJ); }