/**
* @author Yao Wei
* @brief CMU Inverse Compositional !!
* @param - matDeltaP Input -- deltap
* @param - matDeltaQ Input -- deltaq
* @param - s Input -- the shape
* @param - estShape Output -- newly estimated shape by Inverse compositional
*/
void VO_FittingAAMInverseIA::VO_CMUInverseCompositional(const Mat_<float>& matDeltaP,
const Mat_<float>& matDeltaQ,
const VO_Shape& s,
VO_Shape& estShape)
{
VO_Shape S0;
this->VO_PParamQParam2ModelAlignedShape( matDeltaP, matDeltaQ, S0);
// cvConvertScale(dpq, __inv_pq, -1);
// __shape.CalcShape(__inv_pq, __update_s0); // __update_s0 = N.W(s0, -delta_p, -delta_q)
//Secondly: Composing the Incremental Warp with the Current Warp Estimate.
Point2f res, tmp;
int count = 0;
vector<unsigned int> vertexIdxes;
for(unsigned int i = 0; i < this->m_VOAAMInverseIA->m_iNbOfPoints; i++)
{
res.x = 0.0; res.y = 0.0;
count = 0;
//The only problem with this approach is which triangle do we use?
//In general there will be several triangles that share the i-th vertex.
for(unsigned j = 0; j < this->m_VOAAMInverseIA->m_iNbOfTriangles; j++) // see Figure (11)
{
if ( this->m_vTriangle2D[j].HasNode(i) )
{
vertexIdxes = this->m_vTriangle2D[j].GetVertexIndexes();
VO_WarpingPoint::WarpOnePoint( S0.GetA2DPoint(i),
this->m_vTriangle2D[j],
tmp,
s.GetA2DPoint(vertexIdxes[0]),
s.GetA2DPoint(vertexIdxes[1]),
s.GetA2DPoint(vertexIdxes[2]) );
res.x += tmp.x;
res.y += tmp.y;
count++;
}
}
// average the result so as to smooth the warp at each vertex
if(count == 0)
cerr << "There must be something wrong when CMU Inverse Compositional !" << endl;
res.x /= count;
res.y /= count;
estShape.SetA2DPoint(res, i);
}
}
/**
* @author JIA Pei
* @version 2010-06-07
* @brief Constrain all points respetively
* @param ioShape Input and Output - the input and output shape
*/
void VO_Point2DDistributionModel::VO_ConstrainAllPoints(VO_Shape& ioShape)
{
unsigned int NbOfPoints = ioShape.GetNbOfPoints();
Point2f pt;
for(unsigned int i = 0; i < NbOfPoints; i++)
{
pt = ioShape.GetA2DPoint(i);
VO_Point2DDistributionModel::VO_ConstrainSinglePoint( pt, this->m_VONormalizedEllipses[i] );
ioShape.SetA2DPoint(pt, i);
}
}
/**
* @author JIA Pei
* @version 2010-02-07
* @brief Write all annotation data in VO_Shape to a file
* @param filename output parameter, which .pts annotation file to write
* @param iAAMShape input parameter, save annotation data from AAM shape data structure
*/
void CAnnotationDBIO::WritePTS( const std::string &filename,
const VO_Shape& iAAMShape)
{
std::fstream fp;
fp.open(filename.c_str (), std::ios::out);
std::string temp, oneLine;
std::stringstream ss;
float tempFloat = 0.0f;
unsigned int NbOfPoints = iAAMShape.GetNbOfPoints();
fp << "version: 1" << std::endl
<< "n_points: " << NbOfPoints << std::endl
<< "{" << std::endl;
for (unsigned int i = 0; i < NbOfPoints; i++)
{
fp << iAAMShape.GetA2DPoint(i).x << " " << iAAMShape.GetA2DPoint(i).y << std::endl;
}
fp << "}" << std::endl << std::endl;
fp.close ();
}
/**
* @author JIA Pei
* @version 2010-02-22
* @brief Build wavelet for key points
* @param iImg Input -- the concerned image
* @param theShape Input -- the concerned shape
* @param ptIdx Input -- which point?
* @param imgSize Input -- the image size
* @param mtd Input -- LTC method
* @param shiftX Input -- shift in X direction
* @param shiftY Input -- shift in Y direction
* @return cv::Mat_<float> Output -- the extracted LTC
*/
void VO_ASMLTCs::VO_LoadLTC4OneAnnotatedPoint( const cv::Mat& iImg,
const VO_Shape& theShape,
unsigned int ptIdx,
cv::Size imgSize,
VO_Features* vofeatures,
int shiftX,
int shiftY)
{
cv::Point2f pt = theShape.GetA2DPoint(ptIdx);
pt.x += shiftX;
pt.y += shiftY;
cv::Rect rect = VO_ASMLTCs::VO_CalcImagePatchRect(iImg, pt, imgSize);
cv::Mat imgPatch = iImg(rect);
vofeatures->VO_GenerateAllFeatures(imgPatch);
}
/**
* @author JIA Pei
* @version 2010-05-07
* @brief draw a point on the image
* @param iShape Input -- the input shape
* @param iAAMModel Input -- the model
* @param ioImg Input and Output -- the image
* @return void
*/
void VO_Fitting2DSM::VO_DrawMesh(const VO_Shape& iShape, const VO_AXM* iModel, cv::Mat& ioImg)
{
cv::Point iorg,idst;
std::vector<VO_Edge> edges = iModel->GetEdge();
unsigned int NbOfEdges = iModel->GetNbOfEdges();
for (unsigned int i = 0; i < NbOfEdges; i++)
{
iorg = cvPointFrom32f( iShape.GetA2DPoint( edges[i].GetIndex1() ) );
idst = cvPointFrom32f( iShape.GetA2DPoint( edges[i].GetIndex2() ) );
// Edge
cv::line( ioImg, iorg, idst, colors[8], 1, 0, 0 );
// Key points
cv::circle( ioImg, iorg, 2, colors[0], -1, 8, 0 );
cv::circle( ioImg, idst, 2, colors[0], -1, 8, 0 );
}
}
/**
* @author JIA Pei
* @version 2010-02-22
* @brief Build wavelet for key points
* @param iImg Input -- the concerned image
* @param theShape Input -- the concerned shape
* @param ptIdx Input -- which point?
* @param imgSize Input -- the image size
* @param mtd Input -- LTC method
* @return cv::Mat_<float> Output -- the extracted LTC
*/
cv::Mat_<float> VO_AFM::VO_LoadLTC4OneAnnotatedPoint(const cv::Mat& iImg,
const VO_Shape& theShape,
unsigned int ptIdx,
cv::Size imgSize,
unsigned int mtd)
{
cv::Mat_<float> resLTC;
cv::Point2f pt = theShape.GetA2DPoint(ptIdx);
cv::Rect rect = this->VO_CalcImagePatchRect(iImg, pt, imgSize);
cv::Mat imgPatch = iImg(rect);
switch(mtd)
{
case VO_Features::LBP:
{
// initialize the image before wavelet transform
for(unsigned int i = 0; i < rect.height; ++i)
{
for(unsigned int j = 0; j < rect.width; ++j)
{
}
}
bool showWaveletImage = true;
if(showWaveletImage)
{
imwrite("originalImage.jpg", imgPatch);
// this->VO_HardSaveWaveletSingleChannelImage("waveletImage.jpg", waveParamsGray, imgSize);
// this->VO_HardSaveWaveletSingleChannelImage("inverseWaveletImage.jpg", waveParamsGray, imgSize);
}
}
default:
break;
}
return resLTC;
}
/**
* @author YAO Wei, JIA Pei
* @version 2010-05-20
* @brief Find the best offset for one point
* @param asmmodel Input - the ASM model
* @param iImg Input - image to be fitted
* @param ioShape Input and output - the input and output shape
* @param iShapeInfo Input - the shape information
* @param iMean Input - mean profile
* @param iCovInverse Input - covariance inverse
* @param Lev Input - current pyramid level
* @param offSetTolerance Input - offset tolerance, which is used to determine whether this point is convergede or not
* @param profdim Input - specify the dimension that is going to be used when updating shape.
* Sometimes, the trained data is of 4D profiles, but the user may only use 1D to test.
* @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps.
*/
int VO_FittingASMNDProfiles::UpdateShape( const VO_ASMNDProfiles* asmmodel,
const cv::Mat& iImg,
VO_Shape& ioShape,
const std::vector<VO_Shape2DInfo>& iShapeInfo,
const std::vector< VO_Profile >& iMean,
const std::vector< std::vector< cv::Mat_<float> > >& iCovInverse,
unsigned int offSetTolerance,
unsigned int profdim)
{
int nGoodLandmarks = 0;
std::vector<int> nBestOffset(profdim, 0);
unsigned int NbOfPoints = ioShape.GetNbOfPoints();
unsigned int NbOfShapeDim = ioShape.GetNbOfDim();
unsigned int ProfileLength = iMean[0].GetProfileLength();
//std::vector<float> dists(NbOfPoints, 0.0f);
cv::Point2f pt;
// Take care of the 1st direction first.
for (unsigned int i = 0; i < NbOfPoints; i++)
{
/////////////////////////////////////////////////////////////////////////////
///Calculate profile norm direction//////////////////////////////////////////
/** Here, this is not compatible with 3D */
cv::Point2f PrevPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetFrom() );
cv::Point2f ThisPoint = ioShape.GetA2DPoint ( i );
cv::Point2f NextPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetTo() );
float deltaX, deltaY;
float normX, normY;
float sqrtsum;
float bestXOffset, bestYOffset;
// left side (connected from side)
deltaX = ThisPoint.x - PrevPoint.x;
deltaY = ThisPoint.y - PrevPoint.y;
sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize
// Firstly, normX normY record left side norm.
normX = -deltaY;
normY = deltaX;
// right side (connected to side)
deltaX = NextPoint.x - ThisPoint.x;
deltaY = NextPoint.y - ThisPoint.y;
sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize
// Secondly, normX normY will average both left side and right side norm.
normX += -deltaY;
normY += deltaX;
// Average left right side
sqrtsum = sqrt ( normX*normX + normY*normY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
normX /= sqrtsum;
normY /= sqrtsum; // Final Normalize
/////////////////////////////////////////////////////////////////////////////
nBestOffset[0] = VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( iImg,
ThisPoint,
iMean[i].Get1DimProfile(0),
iCovInverse[i][0],
ProfileLength,
offSetTolerance,
normX,
normY);
// set OutShape(iPoint) to best offset from current position
// one dimensional profile: must move point along the whisker
bestXOffset = nBestOffset[0] * normX;
bestYOffset = nBestOffset[0] * normY;
pt.x = ThisPoint.x + bestXOffset;
pt.y = ThisPoint.y + bestYOffset;
ioShape.SetA2DPoint(pt, i);
//dists[i] = sqrt( pow( (double)bestXOffset, 2.0) + pow( (double)bestYOffset, 2.0) );
//if (abs(nBestOffset[0]) <= offSetTolerance/2)
if(profdim == 1)
{
if (abs(nBestOffset[0]) <= 1)
nGoodLandmarks++;
}
}
// Originality from JIA Pei!! Now, take care of the 2nd direction now.
if(profdim == 2)
{
for (unsigned int i = 0; i < NbOfPoints; i++)
{
/////////////////////////////////////////////////////////////////////////////
///Calculate profile norm direction//////////////////////////////////////////
/** Here, this is not compatible with 3D */
cv::Point2f PrevPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetFrom() );
cv::Point2f ThisPoint = ioShape.GetA2DPoint ( i );
cv::Point2f NextPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetTo() );
float deltaX, deltaY;
float normX, normY;
float tangentX, tangentY;
float sqrtsum;
float bestXOffset, bestYOffset;
// left side (connected from side)
deltaX = ThisPoint.x - PrevPoint.x;
deltaY = ThisPoint.y - PrevPoint.y;
sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize
// Firstly, normX normY record left side norm.
normX = -deltaY;
normY = deltaX;
// right side (connected to side)
deltaX = NextPoint.x - ThisPoint.x;
deltaY = NextPoint.y - ThisPoint.y;
sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize
// Secondly, normX normY will average both left side and right side norm.
normX += -deltaY;
normY += deltaX;
// Average left right side
sqrtsum = sqrt ( normX*normX + normY*normY );
if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f;
normX /= sqrtsum;
normY /= sqrtsum; // Final Normalize
tangentX = -normY;
tangentY = normX; // Final tangent
/////////////////////////////////////////////////////////////////////////////
nBestOffset[1] = VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( iImg,
ThisPoint,
iMean[i].Get1DimProfile(1),
iCovInverse[i][1],
ProfileLength,
1, // in tangent direction, offset = 1
tangentX,
tangentY);
// set OutShape(iPoint) to best offset from current position
// one dimensional profile: must move point along the whisker
bestXOffset = nBestOffset[1] * tangentX;
bestYOffset = nBestOffset[1] * tangentY;
pt.x = ThisPoint.x + bestXOffset;
pt.y = ThisPoint.y + bestYOffset;
ioShape.SetA2DPoint(pt, i);
//dists[i] += sqrt( pow((double)bestXOffset, 2.0) + pow((double)bestYOffset, 2.0) );
//if (abs(nBestOffset) <= offSetTolerance/2)
if (abs(nBestOffset[0]) <= 1 && abs(nBestOffset[1]) <= 1)
nGoodLandmarks++;
}
}
return nGoodLandmarks;
}
/**
* @brief Calculate some key points on the face
* @param oPoint output point list
* @param iShape input shape
* @param iFaceParts inut faceparts
* @param ptType input point type
* @return void
*/
void VO_KeyPoint::CalcFaceKeyPoint( cv::Point2f& oPoint,
const VO_Shape& iShape,
const VO_FaceParts& iFaceParts,
unsigned int ptType)
{
std::vector<unsigned int> facePartsPoints;
VO_Shape subiShape;
// Very very very very important.
// Explained by JIA Pei.
// "resize()" is just for resize;
// it doesn't always set what's already inside the the std::vector to "0"
// Therefore, clear() is a must before resize().
switch(ptType)
{
case CENTEROFGRAVITY:
if (iShape.GetNbOfPoints() > 0)
oPoint = iShape.GetA2DPoint( VO_Shape::CENTER);
break;
case LEFTEYELEFTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST);
}
}
break;
case LEFTEYERIGHTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST);
}
}
break;
case LEFTEYECENTER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER);
}
}
break;
case RIGHTEYELEFTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST);
}
}
break;
case RIGHTEYERIGHTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST);
}
}
break;
case RIGHTEYECENTER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER);
}
}
break;
case NOSETIPKEY:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSETIP).GetIndexes(); // Just one point
if (facePartsPoints.size() == 1)
oPoint = iShape.GetA2DPoint(facePartsPoints[0]);
}
break;
case NOSTRILLEFT:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST);
}
}
break;
case NOSTRILRIGHT:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST);
}
}
break;
case NOSECENTER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER);
}
}
break;
case MOUTHLEFTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST);
}
}
break;
case MOUTHRIGHTCORNER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST);
}
}
break;
case MOUTHCENTER:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes();
if (facePartsPoints.size() > 0)
{
subiShape = iShape.GetSubShape(facePartsPoints);
oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER);
}
}
break;
case EARLOBELEFT:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEAR).GetIndexes();
if (facePartsPoints.size() > 0)
{
}
}
break;
case EARLOBERIGHT:
{
facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEAR).GetIndexes();
if (facePartsPoints.size() > 0)
{
}
}
break;
}
}
