void PrevAndNextLandmarks(
int& prev, // out
int& next, // out
int ipoint, // in
const Shape& shape) // in
{
const int npoints = shape.rows;
const LANDMARK_INFO* const info = LANDMARK_INFO_TAB;
CV_Assert(NELEMS(LANDMARK_INFO_TAB) == npoints);
CV_Assert(ipoint >= 0 && ipoint < npoints);
prev = info[ipoint].prev;
if (prev < 0) // not specified in table?
prev = (ipoint + npoints - 1) % npoints;
next = info[ipoint].next;
if (next < 0)
next = (ipoint + 1) % npoints;
CV_Assert(prev >= 0);
CV_Assert(next >= 0);
CV_Assert(prev < int(shape.rows));
CV_Assert(next < int(shape.rows));
CV_Assert(prev != next);
CV_Assert(PointUsed(shape, prev));
CV_Assert(PointUsed(shape, next));
}
static void InterPoint( // interpolate a point from two nearby oldshape points
Shape& shape, // io
const Shape& oldshape, // in
int i, // in: shape point
double ratio, // in: interpolation ratio, 0 to 1
int i1, // in: oldshape point 1
int i2) // in: oldshape point 2
{
if (!PointUsed(oldshape, i1) && !PointUsed(oldshape, i2))
{
shape(i, IX) = 0;
shape(i, IY) = 0;
}
else if (!PointUsed(oldshape, i1))
{
shape(i, IX) = oldshape(i2, IX) + 1; // +1 is arb, to disambiguate point
shape(i, IY) = oldshape(i2, IY) + 1;
}
else if (!PointUsed(oldshape, i2))
{
shape(i, IX) = oldshape(i1, IX) + 1;
shape(i, IY) = oldshape(i1, IY) + 1;
}
else
{
CV_Assert(ratio >= 0 && ratio <= 1);
shape(i, IX) = ratio * oldshape(i1, IX) + (1-ratio) * oldshape(i2, IX);
shape(i, IY) = ratio * oldshape(i1, IY) + (1-ratio) * oldshape(i2, IY);
}
}
static bool HaveCanonical5Points(
const Shape& pinned) // in: pinned landmarks
{
return PointUsed(pinned, L_LEyeOuter) &&
PointUsed(pinned, L_REyeOuter) &&
PointUsed(pinned, L_CNoseTip) &&
PointUsed(pinned, L_LMouthCorner) &&
PointUsed(pinned, L_RMouthCorner);
}
static MAT CalcShapeCov(
const vec_Shape& shapes, // in
const Shape& meanshape) // in
{
int npoints = meanshape.rows;
int i, j;
MAT cov(2 * npoints, 2 * npoints, 0.); // covariance of every x and y coord
MAT nused(npoints, npoints, 0.);
// Because of possible unused points, we have to iterate by
// hand (we can't use standard matrix multiplies).
for (int ishape = 0; ishape < NSIZE(shapes); ishape++)
{
Shape shape(shapes[ishape]);
for (i = 0; i < npoints; i++)
if (PointUsed(shape, i))
for (j = 0; j < npoints; j++)
if (PointUsed(shape, j))
{
cov(2*i, 2*j) += // x * x
(shape(i, IX) - meanshape(i, IX)) *
(shape(j, IX) - meanshape(j, IX));
cov(2*i+1, 2*j) += // y * x
(shape(i, IY) - meanshape(i, IY)) *
(shape(j, IX) - meanshape(j, IX));
cov(2*i, 2*j+1) += // x * y
(shape(i, IX) - meanshape(i, IX)) *
(shape(j, IY) - meanshape(j, IY));
cov(2*i+1, 2*j+1) += // y * y
(shape(i, IY) - meanshape(i, IY)) *
(shape(j, IY) - meanshape(j, IY));
nused(i, j)++;
}
}
for (i = 0; i < npoints; i++)
for (j = 0; j < npoints; j++)
{
const double n = nused(i, j);
if (n < 3) // 3 is somewhat arb
Err("Cannot calculate covariance of %g shape%s (need more shapes)",
n, plural(int(n)));
cov(2*i, 2*j) /= n;
cov(2*i+1, 2*j) /= n;
cov(2*i, 2*j+1) /= n;
cov(2*i+1, 2*j+1) /= n;
}
return cov;
}
static double CanonicalEyeMouthDist( // return 0 if pupils and mouth not avail
const Shape& shape17) // in
{
if (!PointUsed(shape17, L17_LPupil) ||
!PointUsed(shape17, L17_RPupil) ||
!PointUsed(shape17, L17_CBotOfBotLip))
{
return 0; // note return
}
return PointDist(
MeanPoint(shape17, L17_LPupil, L17_RPupil, IX), // eye mid point
MeanPoint(shape17, L17_LPupil, L17_RPupil, IY),
shape17(L17_CBotOfBotLip, IX), // bot of bot lip
shape17(L17_CBotOfBotLip, IY));
}
static Shape AlignMeanShapeToBothEyesNoMouth(
const DetPar& detpar, // in
const Shape& meanshape) // in
{
if (trace_g)
lprintf("AlignToBothEyesNoMouth ");
CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0));
CV_Assert(Valid(detpar.lex));
CV_Assert(Valid(detpar.rex));
Shape meanline(2, 2), detline(2, 2); // line from eye to eye
meanline(0, IX) = meanshape(L_LPupil, IX); // left eye
meanline(0, IY) = meanshape(L_LPupil, IY);
meanline(1, IX) = meanshape(L_RPupil, IX); // right eye
meanline(1, IY) = meanshape(L_RPupil, IY);
detline(0, IX) = detpar.lex; // left eye
detline(0, IY) = detpar.ley;
detline(1, IX) = detpar.rex; // right eye
detline(1, IY) = detpar.rey;
return AlignShape(meanshape, AlignmentMat(meanline, detline));
}
static Shape AlignMeanShapeToRightEyeAndMouth(
const DetPar& detpar, // in
const Shape& meanshape) // in
{
if (trace_g)
lprintf("AlignToRightEyeAndMouth ");
CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0));
CV_Assert(!Valid(detpar.lex)); // left eye invalid? (else why are we here?)
CV_Assert(Valid(detpar.rex)); // right eye valid?
CV_Assert(Valid(detpar.mouthx)); // mouth valid?
const double x_meanmouth =
(meanshape(L_CTopOfTopLip, IX) + meanshape(L_CBotOfBotLip, IX)) / 2;
const double y_meanmouth =
(meanshape(L_CTopOfTopLip, IY) + meanshape(L_CBotOfBotLip, IY)) / 2;
Shape meanline(2, 2), detline(2, 2); // line from eye to mouth
meanline(0, IX) = meanshape(L_RPupil, IX); // right eye
meanline(0, IY) = meanshape(L_RPupil, IY);
meanline(1, IX) = x_meanmouth; // mouth
meanline(1, IY) = y_meanmouth;
detline(0, IX) = detpar.rex; // right eye
detline(0, IY) = detpar.rey;
detline(1, IX) = detpar.mouthx; // mouth
detline(1, IY) = detpar.mouthy;
return AlignShape(meanshape, AlignmentMat(meanline, detline));
}
void Fm29(
double& fm29, // out: FM29 measure of fitness
int& iworst, // out: index of point with worse fit
const Shape& shape, // in
const Shape& refshape) // in
{
if (shape.rows != 77)
Err("Fitness measure FM29 can be used only on shapes with 77 points "
"(your shape has %d points)", shape.rows);
if (refshape.rows != 77)
Err("Fitness measure FM29 can be used only on shapes with 77 points "
"(your reference shape has %d points)", refshape.rows);
fm29 = 0;
iworst = -1;
double worst = -1;
double weight = 0;
for (int i = 0; i < shape.rows; i++)
if (FITPARAMS[i].xres && // point is used for FM29?
PointUsed(refshape, i)) // point present in ref shape?
{
CV_Assert(PointUsed(shape, i));
CV_Assert(FITPARAMS[i].yres);
const double pointfit =
SQ((shape(i, IX) - refshape(i, IX)) / FITPARAMS[i].xres) +
SQ((shape(i, IY) - refshape(i, IY)) / FITPARAMS[i].yres);
fm29 += pointfit;
const double pointweight =
1 / SQ(FITPARAMS[i].xres) +
1 / SQ(FITPARAMS[i].yres);
weight += pointweight;
if (pointfit > worst)
{
worst = pointfit;
iworst = i;
}
}
CV_Assert(weight > 0);
// multiply by 2 so same as mean euclidean dist when all xres = yres = 1
fm29 = sqrt(fm29 * 2 / weight) / EyeMouthDist(refshape);
}
static Shape AlignMeanShapeToBothEyesEstMouth(
const DetPar& detpar, // in
const Shape& meanshape) // in
{
// .48 was tested to give slightly better worse case results than .50
static double EYEMOUTH_TO_FACERECT_RATIO = .48;
if (trace_g)
lprintf("AlignToBothEyesNoMouth(EstMouth) ");
CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0));
CV_Assert(Valid(detpar.lex));
CV_Assert(Valid(detpar.rex));
// estimate the mouth's position
double x_eyemid = 0;
switch (detpar.eyaw)
{
case EYAW00: // mid point
x_eyemid = .50 * detpar.lex + .50 * detpar.rex;
break;
// TODO The constants below have not been empirically optimized.
case EYAW_45: // closer to left eye
x_eyemid = .30 * detpar.lex + .70 * detpar.rex;
break;
case EYAW_22: // closer to left eye
x_eyemid = .30 * detpar.lex + .70 * detpar.rex;
break;
case EYAW22: // closer to right eye
x_eyemid = .30 * detpar.lex + .70 * detpar.rex;
break;
case EYAW45: // closer to right eye
x_eyemid = .30 * detpar.lex + .70 * detpar.rex;
break;
default:
Err("AlignMeanShapeToBothEyesEstMouth: Invalid eyaw %d", detpar.eyaw);
break;
}
const double y_eyemid = (detpar.ley + detpar.rey) / 2;
Shape mean_tri(3, 2), det_tri(3, 2); // triangle of eyes and mouth
mean_tri(0, IX) = meanshape(L_LPupil, IX); // left eye
mean_tri(0, IY) = meanshape(L_LPupil, IY);
mean_tri(1, IX) = meanshape(L_RPupil, IX); // right eye
mean_tri(1, IY) = meanshape(L_RPupil, IY);
mean_tri(2, IX) = meanshape(L_CBotOfBotLip, IX); // mouth
mean_tri(2, IY) = meanshape(L_CBotOfBotLip, IY);
det_tri(0, IX) = detpar.lex; // left eye
det_tri(0, IY) = detpar.ley;
det_tri(1, IX) = detpar.rex; // right eye
det_tri(1, IY) = detpar.rey;
det_tri(2, IX) = x_eyemid; // mouth
det_tri(2, IY) = y_eyemid + EYEMOUTH_TO_FACERECT_RATIO * detpar.width;
return AlignShape(meanshape, AlignmentMat(mean_tri, det_tri));
}
static void FlipPoint(
Shape& shape, // io
const Shape& oldshape, // in
int inew, // in
int iold, // in
int imgwidth) // in
{
if (!PointUsed(oldshape, iold))
shape(inew, IX) = shape(inew, IY) = 0;
else
{
shape(inew, IX) = imgwidth - oldshape(iold, IX) - 1;
shape(inew, IY) = oldshape(iold, IY);
if (!PointUsed(shape, inew)) // falsely marked unused after conversion?
shape(inew, IX) = XJITTER; // adjust so not marked as unused
}
}
double EyeAngle( // eye angle in degrees, INVALID if eye angle not available
const Shape& shape) // in
{
double angle = INVALID;
const Shape shape17(Shape17OrEmpty(shape));
if (shape17.rows && // converted shape to a Shape17 successfully?
Valid(shape17(L17_LPupil, IX)) && Valid(shape17(L17_RPupil, IX)) &&
PointUsed(shape17, L17_LPupil) && PointUsed(shape17, L17_RPupil))
{
angle = RadsToDegrees(
-atan2(shape17(L17_RPupil, IY) - shape17(L17_LPupil, IY),
shape17(L17_RPupil, IX) - shape17(L17_LPupil, IX)));
}
return angle;
}
int NbrUsedPoints( // return the number of used points in shape
const Shape& shape) // in
{
int nused = 0;
for (int ipoint = 0; ipoint < shape.rows; ipoint++)
if (PointUsed(shape, ipoint))
nused++;
return nused;
}
static void InitDetParEyeMouthFromShape( // fill in eye and mouth fields of detpar
DetPar& detpar,
Shape& shape)
{
if (PointUsed(shape, L_LPupil))
{
detpar.lex = shape(L_LPupil, IX);
detpar.ley = shape(L_LPupil, IY);
}
if (PointUsed(shape, L_RPupil))
{
detpar.rex = shape(L_RPupil, IX);
detpar.rey = shape(L_RPupil, IY);
}
if (PointUsed(shape, L_CBotOfBotLip))
{
detpar.mouthx = shape(L_CBotOfBotLip, IX);
detpar.mouthy = shape(L_CBotOfBotLip, IY);
}
}
static int TabPoint( // return first used point in tab, -1 if none
const int* tab, // in
int ntab, // in
const Shape& shape) // in
{
for (int i = 0; i < ntab; i++)
if (PointUsed(shape, tab[i]))
return tab[i]; // note return
return -1;
}
static DetPar PseudoDetParFromStartShape(
const Shape& startshape,
double rot,
double yaw,
int nmods)
{
const double lex = startshape(L_LPupil, IX); // left eye
const double ley = startshape(L_LPupil, IY);
const double rex = startshape(L_RPupil, IX); // right eye
const double rey = startshape(L_RPupil, IY);
const double mouthx = startshape(L_CBotOfBotLip, IX); // mouth
const double mouthy = startshape(L_CBotOfBotLip, IY);
CV_Assert(PointUsed(lex, ley));
CV_Assert(PointUsed(rex, rey));
CV_Assert(PointUsed(mouthx, mouthy));
const double xeye = (lex + rex) / 2; // midpoint of eyes
const double yeye = (ley + rey) / 2;
const double eyemouth = PointDist(xeye, yeye, mouthx, mouthy);
DetPar detpar;
detpar.x = .7 * xeye + .3 * mouthx;
detpar.y = .7 * yeye + .3 * mouthy;
detpar.width = 2.0 * eyemouth;
detpar.height = 2.0 * eyemouth;
detpar.lex = lex;
detpar.ley = ley;
detpar.rex = rex;
detpar.rey = rey;
detpar.mouthx = mouthx;
detpar.mouthy = mouthy;
detpar.rot = rot;
detpar.eyaw = DegreesAsEyaw(yaw, nmods); // determines what ASM model to use
detpar.yaw = yaw;
return detpar;
}
void Mod::SuggestShape_( // args same as non OpenMP version, see below
Shape& shape, // io
int ilev, // in
const Image& img, // in
const Shape& pinned) // in
const
{
static bool firsttime = true;
int ncatch = 0;
const Shape inshape(shape.clone());
// Call the search function DescSearch_ concurrently for multiple points.
// Note that dynamic OpenMP scheduling is faster here than static,
// because the time through the loop varies widely (mainly because
// classic descriptors are faster than HATs).
#pragma omp parallel for schedule(dynamic)
for (int ipoint = 0; ipoint < shape.rows; ipoint++)
if (pinned.rows == 0 || !PointUsed(pinned, ipoint)) // skip point if pinned
{
// You are not allowed to jump out of an OpenMP for loop. Thus
// we need this try block, to subsume the global try blocks in
// stasm_lib.cpp. Without this try, a call to Err would cause
// a jump to the global catch.
try
{
if (firsttime && omp_get_thread_num() == 0)
{
firsttime = false;
logprintf("[nthreads %d]", omp_get_num_threads());
}
descmods_[ilev][ipoint]->
DescSearch_(shape(ipoint, IX), shape(ipoint, IY),
img, inshape, ilev, ipoint);
}
catch(...)
{
ncatch++; // a call was made to Err or a CV_Assert failed
}
}
if (ncatch)
{
if (ncatch > 1)
lprintf_always("\nMultiple errors, only the first will be printed\n");
// does not matter what we throw, will be caught by global catch
throw "SuggestShape_";
}
}
static Shape PinMeanShape( // align mean shape to the pinned points
const Shape& pinned, // in: at least two of these points must be set
const Shape& meanshape) // in
{
CV_Assert(pinned.rows == meanshape.rows);
int ipoint, nused = 0; // number of points used in pinned
for (ipoint = 0; ipoint < meanshape.rows; ipoint++)
if (PointUsed(pinned, ipoint))
nused++;
if (nused < 2)
Err("Need at least two pinned landmarks");
// Create an anchor shape (the pinned landmarks) and an alignment shape (the
// points in meanshape that correspond to those pinned landmarks). Do that by
// copying the used points in pinned to pinned_used, and the corresponding
// points in meanshape to meanused.
Shape pinned_used(nused, 2), mean_used(nused, 2);
int i = 0;
for (ipoint = 0; ipoint < meanshape.rows; ipoint++)
if (PointUsed(pinned, ipoint))
{
pinned_used(i, IX) = pinned(ipoint, IX);
pinned_used(i, IY) = pinned(ipoint, IY);
mean_used(i, IX) = meanshape(ipoint, IX);
mean_used(i, IY) = meanshape(ipoint, IY);
i++;
}
CV_Assert(i == nused);
// transform meanshape to pose generated by aligning mean_used to pinned_used
Shape TransformedShape(
AlignShape(meanshape, AlignmentMat(mean_used, pinned_used)));
return JitterPointsAt00(TransformedShape);
}
static void ShowPercentagePointsMissing(
int nshapes_with_missing_points, // in
const vec_Shape& shapes) // in
{
const int nshapes = NSIZE(shapes);
const int npoints = shapes[0].rows;
int ipoint_min = -1, min = nshapes;
lprintf("%d (%.0f%%) of %d shapes have all %d points\n",
nshapes - nshapes_with_missing_points,
100 * double(nshapes - nshapes_with_missing_points) / nshapes,
nshapes, npoints);
lprintf("Percentage of points unused over all shapes:\n");
lprintf(" 0 1 2 3 4 5 "
"6 7 8 9");
for (int ipoint = 0; ipoint < npoints; ipoint++)
{
if (ipoint % 10 == 0)
lprintf("\n %2d ", ipoint);
int nused = 0; // for current point, nbr used over all shapes
for (int ishape = 0; ishape < nshapes; ishape++)
if (PointUsed(shapes[ishape], ipoint))
nused++;
if (nused == nshapes) // all used?
lprintf(" . ");
else
lprintf(" %6.2f", 100. * (nshapes - nused) / nshapes);
if (nused < min)
{
min = nused;
ipoint_min = ipoint;
}
}
lprintf("\n");
if (min == 0)
Err("Point %d is unused in all shapes",
ipoint_min, 100. * min/nshapes);
else if (min < .333 * nshapes) // .333 is arb
{
// error message is issued because we need more used points
// to build a shape model, even when imputing points
Err("Point %d is unused in %.0f%% of the shapes ",
ipoint_min, 100. * (nshapes - min) / double(nshapes));
}
}
static Shape AlignMeanShapeToBothEyesMouth(
const DetPar& detpar, // in
const Shape& meanshape) // in
{
if (trace_g)
lprintf("AlignToBothEyesMouth ");
CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0));
CV_Assert(Valid(detpar.mouthx));
CV_Assert(Valid(detpar.lex));
CV_Assert(Valid(detpar.rex));
Shape mean_tri(3, 2), det_tri(3, 2); // triangle of eyes and mouth
const double x_meanmouth =
(meanshape(L_CTopOfTopLip, IX) + meanshape(L_CBotOfBotLip, IX)) / 2.;
const double y_meanmouth =
(meanshape(L_CTopOfTopLip, IY) + meanshape(L_CBotOfBotLip, IY)) / 2.;
const Shape shape17(Shape17(meanshape));
mean_tri(0, IX) = shape17(L17_LPupil, IX); // left eye
mean_tri(0, IY) = shape17(L17_LPupil, IY);
mean_tri(1, IX) = shape17(L17_RPupil, IX); // right eye
mean_tri(1, IY) = shape17(L17_RPupil, IY);
mean_tri(2, IX) = x_meanmouth; // mouth
mean_tri(2, IY) = y_meanmouth;
det_tri(0, IX) = detpar.lex; // left eye
det_tri(0, IY) = detpar.ley;
det_tri(1, IX) = detpar.rex; // right eye
det_tri(1, IY) = detpar.rey;
det_tri(2, IX) = detpar.mouthx; // mouth
det_tri(2, IY) = detpar.mouthy;
return TransformShape(meanshape, AlignmentMat(mean_tri, det_tri));
}
void Mod::SuggestShape_( // estimate shape by matching descr at each point
Shape& shape, // io: points will be moved for best descriptor matches
int ilev, // in: pyramid level (0 is full size)
const Image& img, // in: image scaled to this pyramid level
const Shape& pinned) // in: if no rows then no pinned landmarks, else
// points except those equal to 0,0 are pinned
const
{
const Shape inshape(shape.clone());
for (int ipoint = 0; ipoint < shape.rows; ipoint++)
if (pinned.rows == 0 || !PointUsed(pinned, ipoint)) // skip point if pinned
{
// Call the ClassicDescMod or HatDescMod search function
// to update the current point in shape (ipoint).
// For the yaw00 model, yaw00.mh:YAW00_DESCMODS defines which
// descriptor model is used for each point.
descmods_[ilev][ipoint]->
DescSearch_(shape(ipoint, IX), shape(ipoint, IY),
img, inshape, ilev, ipoint);
}
}
static void main1(int argc, const char** argv)
{
print_g = true; // want to be able to see lprintfs
if (argc != 2)
Err("Usage: shapetostasm31 file.shape");
ShapeFile sh; // contents of the shape file
sh.Open_(argv[1]);
if (sh.shapes_[0].rows == 77)
lprintf("Converting 77 point to 76 point shapes\n");
char newpath[SLEN];
sprintf(newpath, "%s_stasm31.shape", Base(argv[1]));
lprintf("Generating %s ", newpath);
FILE* file = fopen(newpath, "wb");
if (!file)
Err("Cannot open %s for writing", newpath);
Fprintf(file, "ss %s\n\n", newpath);
Fprintf(file, "Directories %s\n\n", sh.dirs_);
Pacifier pacifier(sh.nshapes_);
for (int ishape = 0; ishape < sh.nshapes_; ishape++)
{
// we need the image width and height to convert the coords
const char* imgpath = PathGivenDirs(sh.bases_[ishape], sh.dirs_, sh.shapepath_);
cv::Mat_<unsigned char> img(cv::imread(imgpath, CV_LOAD_IMAGE_GRAYSCALE));
if (!img.data)
Err("Cannot load %s", imgpath);
Shape shape;
if (sh.shapes_[0].rows == 77)
shape = (ConvertShape(sh.shapes_[ishape], 76)); // convert 76 point shape
else
shape = sh.shapes_[ishape].clone();
CV_Assert(shape.rows);
Fprintf(file, "\"%4.4x %s\"\n",
NewBits(sh.bits_[ishape]), sh.bases_[ishape].c_str());
Fprintf(file, "{ %d %d\n", shape.rows, shape.cols);
const int cols2 = img.cols / 2;
const int rows2 = img.rows / 2;
for (int i = 0; i < shape.rows; i++)
{
if (!PointUsed(shape, i))
Fprintf(file, "0 0\n");
else
{
double oldx = shape(i, IX) - cols2;
double oldy = rows2 - shape(i, IY) - 1;
if (!PointUsed(oldx, oldy))
oldx = XJITTER;
Print(file, oldx, " ");
Print(file, oldy, "\n");
}
}
Fprintf(file, "}\n");
pacifier.Print_(ishape);
}
pacifier.End_();
fclose(file);
lprintf("\n");
}
