void PrevAndNextLandmarks( int& prev, // out int& next, // out int ipoint, // in const Shape& shape) // in { const int npoints = shape.rows; const LANDMARK_INFO* const info = LANDMARK_INFO_TAB; CV_Assert(NELEMS(LANDMARK_INFO_TAB) == npoints); CV_Assert(ipoint >= 0 && ipoint < npoints); prev = info[ipoint].prev; if (prev < 0) // not specified in table? prev = (ipoint + npoints - 1) % npoints; next = info[ipoint].next; if (next < 0) next = (ipoint + 1) % npoints; CV_Assert(prev >= 0); CV_Assert(next >= 0); CV_Assert(prev < int(shape.rows)); CV_Assert(next < int(shape.rows)); CV_Assert(prev != next); CV_Assert(PointUsed(shape, prev)); CV_Assert(PointUsed(shape, next)); }
static void InterPoint( // interpolate a point from two nearby oldshape points Shape& shape, // io const Shape& oldshape, // in int i, // in: shape point double ratio, // in: interpolation ratio, 0 to 1 int i1, // in: oldshape point 1 int i2) // in: oldshape point 2 { if (!PointUsed(oldshape, i1) && !PointUsed(oldshape, i2)) { shape(i, IX) = 0; shape(i, IY) = 0; } else if (!PointUsed(oldshape, i1)) { shape(i, IX) = oldshape(i2, IX) + 1; // +1 is arb, to disambiguate point shape(i, IY) = oldshape(i2, IY) + 1; } else if (!PointUsed(oldshape, i2)) { shape(i, IX) = oldshape(i1, IX) + 1; shape(i, IY) = oldshape(i1, IY) + 1; } else { CV_Assert(ratio >= 0 && ratio <= 1); shape(i, IX) = ratio * oldshape(i1, IX) + (1-ratio) * oldshape(i2, IX); shape(i, IY) = ratio * oldshape(i1, IY) + (1-ratio) * oldshape(i2, IY); } }
static bool HaveCanonical5Points( const Shape& pinned) // in: pinned landmarks { return PointUsed(pinned, L_LEyeOuter) && PointUsed(pinned, L_REyeOuter) && PointUsed(pinned, L_CNoseTip) && PointUsed(pinned, L_LMouthCorner) && PointUsed(pinned, L_RMouthCorner); }
static MAT CalcShapeCov( const vec_Shape& shapes, // in const Shape& meanshape) // in { int npoints = meanshape.rows; int i, j; MAT cov(2 * npoints, 2 * npoints, 0.); // covariance of every x and y coord MAT nused(npoints, npoints, 0.); // Because of possible unused points, we have to iterate by // hand (we can't use standard matrix multiplies). for (int ishape = 0; ishape < NSIZE(shapes); ishape++) { Shape shape(shapes[ishape]); for (i = 0; i < npoints; i++) if (PointUsed(shape, i)) for (j = 0; j < npoints; j++) if (PointUsed(shape, j)) { cov(2*i, 2*j) += // x * x (shape(i, IX) - meanshape(i, IX)) * (shape(j, IX) - meanshape(j, IX)); cov(2*i+1, 2*j) += // y * x (shape(i, IY) - meanshape(i, IY)) * (shape(j, IX) - meanshape(j, IX)); cov(2*i, 2*j+1) += // x * y (shape(i, IX) - meanshape(i, IX)) * (shape(j, IY) - meanshape(j, IY)); cov(2*i+1, 2*j+1) += // y * y (shape(i, IY) - meanshape(i, IY)) * (shape(j, IY) - meanshape(j, IY)); nused(i, j)++; } } for (i = 0; i < npoints; i++) for (j = 0; j < npoints; j++) { const double n = nused(i, j); if (n < 3) // 3 is somewhat arb Err("Cannot calculate covariance of %g shape%s (need more shapes)", n, plural(int(n))); cov(2*i, 2*j) /= n; cov(2*i+1, 2*j) /= n; cov(2*i, 2*j+1) /= n; cov(2*i+1, 2*j+1) /= n; } return cov; }
static double CanonicalEyeMouthDist( // return 0 if pupils and mouth not avail const Shape& shape17) // in { if (!PointUsed(shape17, L17_LPupil) || !PointUsed(shape17, L17_RPupil) || !PointUsed(shape17, L17_CBotOfBotLip)) { return 0; // note return } return PointDist( MeanPoint(shape17, L17_LPupil, L17_RPupil, IX), // eye mid point MeanPoint(shape17, L17_LPupil, L17_RPupil, IY), shape17(L17_CBotOfBotLip, IX), // bot of bot lip shape17(L17_CBotOfBotLip, IY)); }
static Shape AlignMeanShapeToBothEyesNoMouth( const DetPar& detpar, // in const Shape& meanshape) // in { if (trace_g) lprintf("AlignToBothEyesNoMouth "); CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0)); CV_Assert(Valid(detpar.lex)); CV_Assert(Valid(detpar.rex)); Shape meanline(2, 2), detline(2, 2); // line from eye to eye meanline(0, IX) = meanshape(L_LPupil, IX); // left eye meanline(0, IY) = meanshape(L_LPupil, IY); meanline(1, IX) = meanshape(L_RPupil, IX); // right eye meanline(1, IY) = meanshape(L_RPupil, IY); detline(0, IX) = detpar.lex; // left eye detline(0, IY) = detpar.ley; detline(1, IX) = detpar.rex; // right eye detline(1, IY) = detpar.rey; return AlignShape(meanshape, AlignmentMat(meanline, detline)); }
static Shape AlignMeanShapeToRightEyeAndMouth( const DetPar& detpar, // in const Shape& meanshape) // in { if (trace_g) lprintf("AlignToRightEyeAndMouth "); CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0)); CV_Assert(!Valid(detpar.lex)); // left eye invalid? (else why are we here?) CV_Assert(Valid(detpar.rex)); // right eye valid? CV_Assert(Valid(detpar.mouthx)); // mouth valid? const double x_meanmouth = (meanshape(L_CTopOfTopLip, IX) + meanshape(L_CBotOfBotLip, IX)) / 2; const double y_meanmouth = (meanshape(L_CTopOfTopLip, IY) + meanshape(L_CBotOfBotLip, IY)) / 2; Shape meanline(2, 2), detline(2, 2); // line from eye to mouth meanline(0, IX) = meanshape(L_RPupil, IX); // right eye meanline(0, IY) = meanshape(L_RPupil, IY); meanline(1, IX) = x_meanmouth; // mouth meanline(1, IY) = y_meanmouth; detline(0, IX) = detpar.rex; // right eye detline(0, IY) = detpar.rey; detline(1, IX) = detpar.mouthx; // mouth detline(1, IY) = detpar.mouthy; return AlignShape(meanshape, AlignmentMat(meanline, detline)); }
void Fm29( double& fm29, // out: FM29 measure of fitness int& iworst, // out: index of point with worse fit const Shape& shape, // in const Shape& refshape) // in { if (shape.rows != 77) Err("Fitness measure FM29 can be used only on shapes with 77 points " "(your shape has %d points)", shape.rows); if (refshape.rows != 77) Err("Fitness measure FM29 can be used only on shapes with 77 points " "(your reference shape has %d points)", refshape.rows); fm29 = 0; iworst = -1; double worst = -1; double weight = 0; for (int i = 0; i < shape.rows; i++) if (FITPARAMS[i].xres && // point is used for FM29? PointUsed(refshape, i)) // point present in ref shape? { CV_Assert(PointUsed(shape, i)); CV_Assert(FITPARAMS[i].yres); const double pointfit = SQ((shape(i, IX) - refshape(i, IX)) / FITPARAMS[i].xres) + SQ((shape(i, IY) - refshape(i, IY)) / FITPARAMS[i].yres); fm29 += pointfit; const double pointweight = 1 / SQ(FITPARAMS[i].xres) + 1 / SQ(FITPARAMS[i].yres); weight += pointweight; if (pointfit > worst) { worst = pointfit; iworst = i; } } CV_Assert(weight > 0); // multiply by 2 so same as mean euclidean dist when all xres = yres = 1 fm29 = sqrt(fm29 * 2 / weight) / EyeMouthDist(refshape); }
static Shape AlignMeanShapeToBothEyesEstMouth( const DetPar& detpar, // in const Shape& meanshape) // in { // .48 was tested to give slightly better worse case results than .50 static double EYEMOUTH_TO_FACERECT_RATIO = .48; if (trace_g) lprintf("AlignToBothEyesNoMouth(EstMouth) "); CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0)); CV_Assert(Valid(detpar.lex)); CV_Assert(Valid(detpar.rex)); // estimate the mouth's position double x_eyemid = 0; switch (detpar.eyaw) { case EYAW00: // mid point x_eyemid = .50 * detpar.lex + .50 * detpar.rex; break; // TODO The constants below have not been empirically optimized. case EYAW_45: // closer to left eye x_eyemid = .30 * detpar.lex + .70 * detpar.rex; break; case EYAW_22: // closer to left eye x_eyemid = .30 * detpar.lex + .70 * detpar.rex; break; case EYAW22: // closer to right eye x_eyemid = .30 * detpar.lex + .70 * detpar.rex; break; case EYAW45: // closer to right eye x_eyemid = .30 * detpar.lex + .70 * detpar.rex; break; default: Err("AlignMeanShapeToBothEyesEstMouth: Invalid eyaw %d", detpar.eyaw); break; } const double y_eyemid = (detpar.ley + detpar.rey) / 2; Shape mean_tri(3, 2), det_tri(3, 2); // triangle of eyes and mouth mean_tri(0, IX) = meanshape(L_LPupil, IX); // left eye mean_tri(0, IY) = meanshape(L_LPupil, IY); mean_tri(1, IX) = meanshape(L_RPupil, IX); // right eye mean_tri(1, IY) = meanshape(L_RPupil, IY); mean_tri(2, IX) = meanshape(L_CBotOfBotLip, IX); // mouth mean_tri(2, IY) = meanshape(L_CBotOfBotLip, IY); det_tri(0, IX) = detpar.lex; // left eye det_tri(0, IY) = detpar.ley; det_tri(1, IX) = detpar.rex; // right eye det_tri(1, IY) = detpar.rey; det_tri(2, IX) = x_eyemid; // mouth det_tri(2, IY) = y_eyemid + EYEMOUTH_TO_FACERECT_RATIO * detpar.width; return AlignShape(meanshape, AlignmentMat(mean_tri, det_tri)); }
static void FlipPoint( Shape& shape, // io const Shape& oldshape, // in int inew, // in int iold, // in int imgwidth) // in { if (!PointUsed(oldshape, iold)) shape(inew, IX) = shape(inew, IY) = 0; else { shape(inew, IX) = imgwidth - oldshape(iold, IX) - 1; shape(inew, IY) = oldshape(iold, IY); if (!PointUsed(shape, inew)) // falsely marked unused after conversion? shape(inew, IX) = XJITTER; // adjust so not marked as unused } }
double EyeAngle( // eye angle in degrees, INVALID if eye angle not available const Shape& shape) // in { double angle = INVALID; const Shape shape17(Shape17OrEmpty(shape)); if (shape17.rows && // converted shape to a Shape17 successfully? Valid(shape17(L17_LPupil, IX)) && Valid(shape17(L17_RPupil, IX)) && PointUsed(shape17, L17_LPupil) && PointUsed(shape17, L17_RPupil)) { angle = RadsToDegrees( -atan2(shape17(L17_RPupil, IY) - shape17(L17_LPupil, IY), shape17(L17_RPupil, IX) - shape17(L17_LPupil, IX))); } return angle; }
int NbrUsedPoints( // return the number of used points in shape const Shape& shape) // in { int nused = 0; for (int ipoint = 0; ipoint < shape.rows; ipoint++) if (PointUsed(shape, ipoint)) nused++; return nused; }
static void InitDetParEyeMouthFromShape( // fill in eye and mouth fields of detpar DetPar& detpar, Shape& shape) { if (PointUsed(shape, L_LPupil)) { detpar.lex = shape(L_LPupil, IX); detpar.ley = shape(L_LPupil, IY); } if (PointUsed(shape, L_RPupil)) { detpar.rex = shape(L_RPupil, IX); detpar.rey = shape(L_RPupil, IY); } if (PointUsed(shape, L_CBotOfBotLip)) { detpar.mouthx = shape(L_CBotOfBotLip, IX); detpar.mouthy = shape(L_CBotOfBotLip, IY); } }
static int TabPoint( // return first used point in tab, -1 if none const int* tab, // in int ntab, // in const Shape& shape) // in { for (int i = 0; i < ntab; i++) if (PointUsed(shape, tab[i])) return tab[i]; // note return return -1; }
static DetPar PseudoDetParFromStartShape( const Shape& startshape, double rot, double yaw, int nmods) { const double lex = startshape(L_LPupil, IX); // left eye const double ley = startshape(L_LPupil, IY); const double rex = startshape(L_RPupil, IX); // right eye const double rey = startshape(L_RPupil, IY); const double mouthx = startshape(L_CBotOfBotLip, IX); // mouth const double mouthy = startshape(L_CBotOfBotLip, IY); CV_Assert(PointUsed(lex, ley)); CV_Assert(PointUsed(rex, rey)); CV_Assert(PointUsed(mouthx, mouthy)); const double xeye = (lex + rex) / 2; // midpoint of eyes const double yeye = (ley + rey) / 2; const double eyemouth = PointDist(xeye, yeye, mouthx, mouthy); DetPar detpar; detpar.x = .7 * xeye + .3 * mouthx; detpar.y = .7 * yeye + .3 * mouthy; detpar.width = 2.0 * eyemouth; detpar.height = 2.0 * eyemouth; detpar.lex = lex; detpar.ley = ley; detpar.rex = rex; detpar.rey = rey; detpar.mouthx = mouthx; detpar.mouthy = mouthy; detpar.rot = rot; detpar.eyaw = DegreesAsEyaw(yaw, nmods); // determines what ASM model to use detpar.yaw = yaw; return detpar; }
void Mod::SuggestShape_( // args same as non OpenMP version, see below Shape& shape, // io int ilev, // in const Image& img, // in const Shape& pinned) // in const { static bool firsttime = true; int ncatch = 0; const Shape inshape(shape.clone()); // Call the search function DescSearch_ concurrently for multiple points. // Note that dynamic OpenMP scheduling is faster here than static, // because the time through the loop varies widely (mainly because // classic descriptors are faster than HATs). #pragma omp parallel for schedule(dynamic) for (int ipoint = 0; ipoint < shape.rows; ipoint++) if (pinned.rows == 0 || !PointUsed(pinned, ipoint)) // skip point if pinned { // You are not allowed to jump out of an OpenMP for loop. Thus // we need this try block, to subsume the global try blocks in // stasm_lib.cpp. Without this try, a call to Err would cause // a jump to the global catch. try { if (firsttime && omp_get_thread_num() == 0) { firsttime = false; logprintf("[nthreads %d]", omp_get_num_threads()); } descmods_[ilev][ipoint]-> DescSearch_(shape(ipoint, IX), shape(ipoint, IY), img, inshape, ilev, ipoint); } catch(...) { ncatch++; // a call was made to Err or a CV_Assert failed } } if (ncatch) { if (ncatch > 1) lprintf_always("\nMultiple errors, only the first will be printed\n"); // does not matter what we throw, will be caught by global catch throw "SuggestShape_"; } }
static Shape PinMeanShape( // align mean shape to the pinned points const Shape& pinned, // in: at least two of these points must be set const Shape& meanshape) // in { CV_Assert(pinned.rows == meanshape.rows); int ipoint, nused = 0; // number of points used in pinned for (ipoint = 0; ipoint < meanshape.rows; ipoint++) if (PointUsed(pinned, ipoint)) nused++; if (nused < 2) Err("Need at least two pinned landmarks"); // Create an anchor shape (the pinned landmarks) and an alignment shape (the // points in meanshape that correspond to those pinned landmarks). Do that by // copying the used points in pinned to pinned_used, and the corresponding // points in meanshape to meanused. Shape pinned_used(nused, 2), mean_used(nused, 2); int i = 0; for (ipoint = 0; ipoint < meanshape.rows; ipoint++) if (PointUsed(pinned, ipoint)) { pinned_used(i, IX) = pinned(ipoint, IX); pinned_used(i, IY) = pinned(ipoint, IY); mean_used(i, IX) = meanshape(ipoint, IX); mean_used(i, IY) = meanshape(ipoint, IY); i++; } CV_Assert(i == nused); // transform meanshape to pose generated by aligning mean_used to pinned_used Shape TransformedShape( AlignShape(meanshape, AlignmentMat(mean_used, pinned_used))); return JitterPointsAt00(TransformedShape); }
static void ShowPercentagePointsMissing( int nshapes_with_missing_points, // in const vec_Shape& shapes) // in { const int nshapes = NSIZE(shapes); const int npoints = shapes[0].rows; int ipoint_min = -1, min = nshapes; lprintf("%d (%.0f%%) of %d shapes have all %d points\n", nshapes - nshapes_with_missing_points, 100 * double(nshapes - nshapes_with_missing_points) / nshapes, nshapes, npoints); lprintf("Percentage of points unused over all shapes:\n"); lprintf(" 0 1 2 3 4 5 " "6 7 8 9"); for (int ipoint = 0; ipoint < npoints; ipoint++) { if (ipoint % 10 == 0) lprintf("\n %2d ", ipoint); int nused = 0; // for current point, nbr used over all shapes for (int ishape = 0; ishape < nshapes; ishape++) if (PointUsed(shapes[ishape], ipoint)) nused++; if (nused == nshapes) // all used? lprintf(" . "); else lprintf(" %6.2f", 100. * (nshapes - nused) / nshapes); if (nused < min) { min = nused; ipoint_min = ipoint; } } lprintf("\n"); if (min == 0) Err("Point %d is unused in all shapes", ipoint_min, 100. * min/nshapes); else if (min < .333 * nshapes) // .333 is arb { // error message is issued because we need more used points // to build a shape model, even when imputing points Err("Point %d is unused in %.0f%% of the shapes ", ipoint_min, 100. * (nshapes - min) / double(nshapes)); } }
static Shape AlignMeanShapeToBothEyesMouth( const DetPar& detpar, // in const Shape& meanshape) // in { if (trace_g) lprintf("AlignToBothEyesMouth "); CV_Assert(NSIZE(meanshape) > 0 && PointUsed(meanshape, 0)); CV_Assert(Valid(detpar.mouthx)); CV_Assert(Valid(detpar.lex)); CV_Assert(Valid(detpar.rex)); Shape mean_tri(3, 2), det_tri(3, 2); // triangle of eyes and mouth const double x_meanmouth = (meanshape(L_CTopOfTopLip, IX) + meanshape(L_CBotOfBotLip, IX)) / 2.; const double y_meanmouth = (meanshape(L_CTopOfTopLip, IY) + meanshape(L_CBotOfBotLip, IY)) / 2.; const Shape shape17(Shape17(meanshape)); mean_tri(0, IX) = shape17(L17_LPupil, IX); // left eye mean_tri(0, IY) = shape17(L17_LPupil, IY); mean_tri(1, IX) = shape17(L17_RPupil, IX); // right eye mean_tri(1, IY) = shape17(L17_RPupil, IY); mean_tri(2, IX) = x_meanmouth; // mouth mean_tri(2, IY) = y_meanmouth; det_tri(0, IX) = detpar.lex; // left eye det_tri(0, IY) = detpar.ley; det_tri(1, IX) = detpar.rex; // right eye det_tri(1, IY) = detpar.rey; det_tri(2, IX) = detpar.mouthx; // mouth det_tri(2, IY) = detpar.mouthy; return TransformShape(meanshape, AlignmentMat(mean_tri, det_tri)); }
void Mod::SuggestShape_( // estimate shape by matching descr at each point Shape& shape, // io: points will be moved for best descriptor matches int ilev, // in: pyramid level (0 is full size) const Image& img, // in: image scaled to this pyramid level const Shape& pinned) // in: if no rows then no pinned landmarks, else // points except those equal to 0,0 are pinned const { const Shape inshape(shape.clone()); for (int ipoint = 0; ipoint < shape.rows; ipoint++) if (pinned.rows == 0 || !PointUsed(pinned, ipoint)) // skip point if pinned { // Call the ClassicDescMod or HatDescMod search function // to update the current point in shape (ipoint). // For the yaw00 model, yaw00.mh:YAW00_DESCMODS defines which // descriptor model is used for each point. descmods_[ilev][ipoint]-> DescSearch_(shape(ipoint, IX), shape(ipoint, IY), img, inshape, ilev, ipoint); } }
static void main1(int argc, const char** argv) { print_g = true; // want to be able to see lprintfs if (argc != 2) Err("Usage: shapetostasm31 file.shape"); ShapeFile sh; // contents of the shape file sh.Open_(argv[1]); if (sh.shapes_[0].rows == 77) lprintf("Converting 77 point to 76 point shapes\n"); char newpath[SLEN]; sprintf(newpath, "%s_stasm31.shape", Base(argv[1])); lprintf("Generating %s ", newpath); FILE* file = fopen(newpath, "wb"); if (!file) Err("Cannot open %s for writing", newpath); Fprintf(file, "ss %s\n\n", newpath); Fprintf(file, "Directories %s\n\n", sh.dirs_); Pacifier pacifier(sh.nshapes_); for (int ishape = 0; ishape < sh.nshapes_; ishape++) { // we need the image width and height to convert the coords const char* imgpath = PathGivenDirs(sh.bases_[ishape], sh.dirs_, sh.shapepath_); cv::Mat_<unsigned char> img(cv::imread(imgpath, CV_LOAD_IMAGE_GRAYSCALE)); if (!img.data) Err("Cannot load %s", imgpath); Shape shape; if (sh.shapes_[0].rows == 77) shape = (ConvertShape(sh.shapes_[ishape], 76)); // convert 76 point shape else shape = sh.shapes_[ishape].clone(); CV_Assert(shape.rows); Fprintf(file, "\"%4.4x %s\"\n", NewBits(sh.bits_[ishape]), sh.bases_[ishape].c_str()); Fprintf(file, "{ %d %d\n", shape.rows, shape.cols); const int cols2 = img.cols / 2; const int rows2 = img.rows / 2; for (int i = 0; i < shape.rows; i++) { if (!PointUsed(shape, i)) Fprintf(file, "0 0\n"); else { double oldx = shape(i, IX) - cols2; double oldy = rows2 - shape(i, IY) - 1; if (!PointUsed(oldx, oldy)) oldx = XJITTER; Print(file, oldx, " "); Print(file, oldy, "\n"); } } Fprintf(file, "}\n"); pacifier.Print_(ishape); } pacifier.End_(); fclose(file); lprintf("\n"); }