/*!
* dewarpPopulateFullRes()
*
* Input: dew
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) If the full resolution vertical (and, optionally horizontal)
* disparity arrays do not exist, they are built from the
* subsampled ones.
*/
l_int32
dewarpPopulateFullRes(L_DEWARP *dew)
{
PROCNAME("dewarpPopulateFullRes");
if (!dew)
return ERROR_INT("dew not defined", procName, 1);
if (!dew->sampvdispar)
return ERROR_INT("no sampled vert disparity", procName, 1);
if (!dew->fullvdispar)
dew->fullvdispar = fpixScaleByInteger(dew->sampvdispar, dew->sampling);
if (!dew->fullhdispar && dew->samphdispar)
dew->fullhdispar = fpixScaleByInteger(dew->samphdispar, dew->sampling);
return 0;
}
/*!
* dewarpBuildModel()
*
* Input: dew
* debugflag (1 for debugging output)
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) This is the basic function that builds the vertical
* disparity array, which allows determination of the
* src pixel in the input image corresponding to each
* dest pixel in the dewarped image.
* (2) The method is as follows:
* * Estimate the centers of all the long textlines and
* fit a LS quadratic to each one. This smooths the curves.
* * Sample each curve at a regular interval, find the y-value
* of the flat point on each curve, and subtract the sampled
* curve value from this value. This is the vertical
* disparity.
* * Fit a LS quadratic to each set of vertically aligned
* disparity samples. This smooths the disparity values
* in the vertical direction. Then resample at the same
* regular interval, We now have a regular grid of smoothed
* vertical disparity valuels.
* * Interpolate this grid to get a full resolution disparity
* map. This can be applied directly to the src image
* pixels to dewarp the image in the vertical direction,
* making all textlines horizontal.
*/
l_int32
dewarpBuildModel(L_DEWARP *dew,
l_int32 debugflag)
{
char *tempname;
l_int32 i, j, nlines, nx, ny, sampling;
l_float32 c0, c1, c2, x, y, flaty, val;
l_float32 *faflats;
NUMA *nax, *nafit, *nacurve, *nacurves, *naflat, *naflats, *naflatsi;
PIX *pixs, *pixt1, *pixt2;
PTA *pta, *ptad;
PTAA *ptaa1, *ptaa2, *ptaa3, *ptaa4, *ptaa5, *ptaa6, *ptaa7;
FPIX *fpix1, *fpix2, *fpix3;
PROCNAME("dewarpBuildModel");
if (!dew)
return ERROR_INT("dew not defined", procName, 1);
pixs = dew->pixs;
if (debugflag) {
pixDisplayWithTitle(pixs, 0, 0, "pixs", 1);
pixWriteTempfile("/tmp", "pixs.png", pixs, IFF_PNG, NULL);
}
/* Make initial estimate of centers of textlines */
ptaa1 = pixGetTextlineCenters(pixs, DEBUG_TEXTLINE_CENTERS);
if (debugflag) {
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa1);
pixWriteTempfile("/tmp", "lines1.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
}
/* Remove all lines that are not near the length
* of the longest line. */
ptaa2 = ptaaRemoveShortLines(pixs, ptaa1, 0.8, DEBUG_SHORT_LINES);
if (debugflag) {
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa2);
pixWriteTempfile("/tmp", "lines2.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
}
nlines = ptaaGetCount(ptaa2);
if (nlines < dew->minlines)
return ERROR_INT("insufficient lines to build model", procName, 1);
/* Do quadratic fit to smooth each line. A single quadratic
* over the entire width of the line appears to be sufficient.
* Quartics tend to overfit to noise. Each line is thus
* represented by three coefficients: c2 * x^2 + c1 * x + c0.
* Using the coefficients, sample each fitted curve uniformly
* across the full width of the image. */
sampling = dew->sampling;
nx = dew->nx;
ny = dew->ny;
ptaa3 = ptaaCreate(nlines);
nacurve = numaCreate(nlines); /* stores curvature coeff c2 */
for (i = 0; i < nlines; i++) { /* for each line */
pta = ptaaGetPta(ptaa2, i, L_CLONE);
ptaGetQuadraticLSF(pta, &c2, &c1, &c0, NULL);
numaAddNumber(nacurve, c2);
ptad = ptaCreate(nx);
for (j = 0; j < nx; j++) { /* uniformly sampled in x */
x = j * sampling;
applyQuadraticFit(c2, c1, c0, x, &y);
ptaAddPt(ptad, x, y);
}
ptaaAddPta(ptaa3, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
ptaa4 = ptaaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa2, i, L_CLONE);
ptaGetArrays(pta, &nax, NULL);
ptaGetQuadraticLSF(pta, NULL, NULL, NULL, &nafit);
ptad = ptaCreateFromNuma(nax, nafit);
ptaaAddPta(ptaa4, ptad, L_INSERT);
ptaDestroy(&pta);
numaDestroy(&nax);
numaDestroy(&nafit);
}
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa4);
pixWriteTempfile("/tmp", "lines3.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
ptaaDestroy(&ptaa4);
}
/* Find and save the flat points in each curve. */
naflat = numaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa3, i, L_CLONE);
numaGetFValue(nacurve, i, &c2);
if (c2 <= 0) /* flat point at bottom; max value of y in curve */
ptaGetRange(pta, NULL, NULL, NULL, &flaty);
else /* flat point at top; min value of y in curve */
ptaGetRange(pta, NULL, NULL, &flaty, NULL);
numaAddNumber(naflat, flaty);
ptaDestroy(&pta);
}
/* Sort the lines in ptaa3 by their position */
naflatsi = numaGetSortIndex(naflat, L_SORT_INCREASING);
naflats = numaSortByIndex(naflat, naflatsi);
nacurves = numaSortByIndex(nacurve, naflatsi);
dew->naflats = naflats;
dew->nacurves = nacurves;
ptaa4 = ptaaSortByIndex(ptaa3, naflatsi);
numaDestroy(&naflat);
numaDestroy(&nacurve);
numaDestroy(&naflatsi);
if (debugflag) {
tempname = genTempFilename("/tmp", "naflats.na", 0);
numaWrite(tempname, naflats);
FREE(tempname);
}
/* Convert the sampled points in ptaa3 to a sampled disparity with
* with respect to the flat point in the curve. */
ptaa5 = ptaaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa4, i, L_CLONE);
numaGetFValue(naflats, i, &flaty);
ptad = ptaCreate(nx);
for (j = 0; j < nx; j++) {
ptaGetPt(pta, j, &x, &y);
ptaAddPt(ptad, x, flaty - y);
}
ptaaAddPta(ptaa5, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa5.ptaa", 0);
ptaaWrite(tempname, ptaa5, 0);
FREE(tempname);
}
/* Generate a ptaa taking vertical 'columns' from ptaa5.
* We want to fit the vertical disparity on the column to the
* vertical position of the line, which we call 'y' here and
* obtain from naflats. */
ptaa6 = ptaaCreate(nx);
faflats = numaGetFArray(naflats, L_NOCOPY);
for (j = 0; j < nx; j++) {
pta = ptaCreate(nlines);
for (i = 0; i < nlines; i++) {
y = faflats[i];
ptaaGetPt(ptaa5, i, j, NULL, &val); /* disparity value */
ptaAddPt(pta, y, val);
}
ptaaAddPta(ptaa6, pta, L_INSERT);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa6.ptaa", 0);
ptaaWrite(tempname, ptaa6, 0);
FREE(tempname);
}
/* Do quadratic fit vertically on a subset of pixel columns
* for the vertical displacement, which identifies the
* src pixel(s) for each dest pixel. Sample the displacement
* on a regular grid in the vertical direction. */
ptaa7 = ptaaCreate(nx); /* uniformly sampled across full height of image */
for (j = 0; j < nx; j++) { /* for each column */
pta = ptaaGetPta(ptaa6, j, L_CLONE);
ptaGetQuadraticLSF(pta, &c2, &c1, &c0, NULL);
ptad = ptaCreate(ny);
for (i = 0; i < ny; i++) { /* uniformly sampled in y */
y = i * sampling;
applyQuadraticFit(c2, c1, c0, y, &val);
ptaAddPt(ptad, y, val);
}
ptaaAddPta(ptaa7, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa7.ptaa", 0);
ptaaWrite(tempname, ptaa7, 0);
FREE(tempname);
}
/* Save the result in a fpix at the specified subsampling */
fpix1 = fpixCreate(nx, ny);
for (i = 0; i < ny; i++) {
for (j = 0; j < nx; j++) {
ptaaGetPt(ptaa7, j, i, NULL, &val);
fpixSetPixel(fpix1, j, i, val);
}
}
dew->sampvdispar = fpix1;
/* Generate a full res fpix for vertical dewarping. We require that
* the size of this fpix is at least as big as the input image. */
fpix2 = fpixScaleByInteger(fpix1, sampling);
dew->fullvdispar = fpix2;
if (debugflag) {
pixt1 = fpixRenderContours(fpix2, -2., 2.0, 0.2);
pixWriteTempfile("/tmp", "vert-contours.png", pixt1, IFF_PNG, NULL);
pixDisplay(pixt1, 1000, 0);
pixDestroy(&pixt1);
}
/* Generate full res and sampled fpix for horizontal dewarping. This
* works to the extent that the line curvature is due to bending
* out of the plane normal to the camera, and not wide-angle
* "fishbowl" distortion. Also generate the sampled horizontal
* disparity array. */
if (dew->applyhoriz) {
fpix3 = fpixBuildHorizontalDisparity(fpix2, 0, &dew->extraw);
dew->fullhdispar = fpix3;
dew->samphdispar = fpixSampledDisparity(fpix3, dew->sampling);
if (debugflag) {
pixt1 = fpixRenderContours(fpix3, -2., 2.0, 0.2);
pixWriteTempfile("/tmp", "horiz-contours.png", pixt1,
IFF_PNG, NULL);
pixDisplay(pixt1, 1000, 0);
pixDestroy(&pixt1);
}
}
dew->success = 1;
ptaaDestroy(&ptaa1);
ptaaDestroy(&ptaa2);
ptaaDestroy(&ptaa3);
ptaaDestroy(&ptaa4);
ptaaDestroy(&ptaa5);
ptaaDestroy(&ptaa6);
ptaaDestroy(&ptaa7);
return 0;
}
/*!
* dewarpPopulateFullRes()
*
* Input: dew
* pix (<optional>, to give size of actual image)
* x, y (origin for generation of disparity arrays)
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) If the full resolution vertical and horizontal disparity
* arrays do not exist, they are built from the subsampled ones.
* (2) If pixs is not given, the size of the arrays is determined
* by the original image from which the sampled version was
* generated. Any values of (x,y) are ignored.
* (3) If pixs is given, the full resolution disparity arrays must
* be large enough to accommodate it.
* (a) If the arrays do not exist, the value of (x,y) determines
* the origin of the full resolution arrays without extension,
* relative to pixs. Thus, (x,y) gives the amount of
* slope extension in (left, top). The (right, bottom)
* extension is then determined by the size of pixs and
* (x,y); the values should never be < 0.
* (b) If the arrays exist and pixs is too large, the existing
* full res arrays are destroyed and new ones are made,
* again using (x,y) to determine the extension in the
* four directions.
*/
l_int32
dewarpPopulateFullRes(L_DEWARP *dew,
PIX *pix,
l_int32 x,
l_int32 y)
{
l_int32 width, height, fw, fh, deltaw, deltah, redfactor;
FPIX *fpixt1, *fpixt2;
PROCNAME("dewarpPopulateFullRes");
if (!dew)
return ERROR_INT("dew not defined", procName, 1);
if (!dew->sampvdispar)
return ERROR_INT("no sampled vert disparity", procName, 1);
if (x < 0) x = 0;
if (y < 0) y = 0;
/* Establish the target size for the full res arrays */
if (pix)
pixGetDimensions(pix, &width, &height, NULL);
else {
width = dew->w;
height = dew->h;
}
/* Destroy the existing arrays if they are too small */
if (dew->fullvdispar) {
fpixGetDimensions(dew->fullvdispar, &fw, &fh);
if (width > fw || height > fw)
fpixDestroy(&dew->fullvdispar);
}
if (dew->fullhdispar) {
fpixGetDimensions(dew->fullhdispar, &fw, &fh);
if (width > fw || height > fw)
fpixDestroy(&dew->fullhdispar);
}
/* Find the required width and height expansion deltas */
deltaw = width - dew->sampling * (dew->nx - 1) + 2;
deltah = height - dew->sampling * (dew->ny - 1) + 2;
redfactor = dew->redfactor;
deltaw = redfactor * L_MAX(0, deltaw);
deltah = redfactor * L_MAX(0, deltah);
/* Generate the full res vertical array if it doesn't exist,
* extending it as required to make it big enough. Use x,y
* to determine the amounts on each side. */
if (!dew->fullvdispar) {
fpixt1 = fpixCopy(NULL, dew->sampvdispar);
if (redfactor == 2)
fpixAddMultConstant(fpixt1, 0.0, (l_float32)redfactor);
fpixt2 = fpixScaleByInteger(fpixt1, dew->sampling * redfactor);
fpixDestroy(&fpixt1);
if (deltah == 0 && deltaw == 0) {
dew->fullvdispar = fpixt2;
}
else {
dew->fullvdispar = fpixAddSlopeBorder(fpixt2, x, deltaw - x,
y, deltah - y);
fpixDestroy(&fpixt2);
}
}
/* Similarly, generate the full res horizontal array if it
* doesn't exist. Do this even if useboth == 0. */
if (!dew->fullhdispar && dew->samphdispar) {
fpixt1 = fpixCopy(NULL, dew->samphdispar);
if (redfactor == 2)
fpixAddMultConstant(fpixt1, 0.0, (l_float32)redfactor);
fpixt2 = fpixScaleByInteger(fpixt1, dew->sampling * redfactor);
fpixDestroy(&fpixt1);
if (deltah == 0 && deltaw == 0) {
dew->fullhdispar = fpixt2;
}
else {
dew->fullhdispar = fpixAddSlopeBorder(fpixt2, x, deltaw - x,
y, deltah - y);
fpixDestroy(&fpixt2);
}
}
return 0;
}
l_int32 main(int argc,
char **argv)
{
l_int32 i, n;
l_float32 a, b, c;
L_DEWARP *dew, *dew2;
DPIX *dpix1, *dpix2, *dpix3;
FPIX *fpix1, *fpix2, *fpix3;
NUMA *nax, *nafit;
PIX *pixs, *pixn, *pixg, *pixb, *pixt1, *pixt2;
PIX *pixs2, *pixn2, *pixg2, *pixb2;
PTA *pta, *ptad;
PTAA *ptaa1, *ptaa2;
L_REGPARAMS *rp;
if (regTestSetup(argc, argv, &rp))
return 1;
pixs = pixRead("1555-7.jpg");
/* Normalize for varying background and binarize */
pixn = pixBackgroundNormSimple(pixs, NULL, NULL);
pixg = pixConvertRGBToGray(pixn, 0.5, 0.3, 0.2);
pixb = pixThresholdToBinary(pixg, 130);
pixDestroy(&pixn);
pixDestroy(&pixg);
regTestWritePixAndCheck(rp, pixb, IFF_PNG); /* 0 */
pixDisplayWithTitle(pixb, 0, 0, "binarized input", rp->display);
/* Get the textline centers */
ptaa1 = pixGetTextlineCenters(pixb, 0);
pixt1 = pixCreateTemplate(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa1);
regTestWritePixAndCheck(rp, pixt2, IFF_PNG); /* 1 */
pixDisplayWithTitle(pixt2, 0, 500, "textline centers", rp->display);
pixDestroy(&pixt1);
/* Remove short lines */
ptaa2 = ptaaRemoveShortLines(pixb, ptaa1, 0.8, 0);
/* Fit to quadratic */
n = ptaaGetCount(ptaa2);
for (i = 0; i < n; i++) {
pta = ptaaGetPta(ptaa2, i, L_CLONE);
ptaGetArrays(pta, &nax, NULL);
ptaGetQuadraticLSF(pta, &a, &b, &c, &nafit);
ptad = ptaCreateFromNuma(nax, nafit);
pixDisplayPta(pixt2, pixt2, ptad);
ptaDestroy(&pta);
ptaDestroy(&ptad);
numaDestroy(&nax);
numaDestroy(&nafit);
}
regTestWritePixAndCheck(rp, pixt2, IFF_PNG); /* 2 */
pixDisplayWithTitle(pixt2, 300, 500, "fitted lines superimposed",
rp->display);
ptaaDestroy(&ptaa1);
ptaaDestroy(&ptaa2);
pixDestroy(&pixt2);
/* Run with only vertical disparity correction */
if ((dew = dewarpCreate(pixb, 7, 30, 15, 0)) == NULL)
return ERROR_INT("\n\n\n FAILURE !!! \n\n\n", rp->testname, 1);
dewarpBuildModel(dew, 0);
dewarpApplyDisparity(dew, pixb, 0);
regTestWritePixAndCheck(rp, dew->pixd, IFF_PNG); /* 3 */
pixDisplayWithTitle(dew->pixd, 400, 0, "fixed for vert disparity",
rp->display);
dewarpDestroy(&dew);
/* Run with both vertical and horizontal disparity correction */
if ((dew = dewarpCreate(pixb, 7, 30, 15, 1)) == NULL)
return ERROR_INT("\n\n\n FAILURE !!! \n\n\n", rp->testname, 1);
dewarpBuildModel(dew, 0);
dewarpApplyDisparity(dew, pixb, 0);
regTestWritePixAndCheck(rp, dew->pixd, IFF_PNG); /* 4 */
pixDisplayWithTitle(dew->pixd, 800, 0, "fixed for both disparities",
rp->display);
/* Read another image, normalize background and binarize */
pixs2 = pixRead("1555-3.jpg");
pixn2 = pixBackgroundNormSimple(pixs2, NULL, NULL);
pixg2 = pixConvertRGBToGray(pixn2, 0.5, 0.3, 0.2);
pixb2 = pixThresholdToBinary(pixg2, 130);
pixDestroy(&pixn2);
pixDestroy(&pixg2);
regTestWritePixAndCheck(rp, pixb, IFF_PNG); /* 5 */
pixDisplayWithTitle(pixb, 0, 400, "binarized input (2)", rp->display);
/* Minimize and re-apply previous disparity to this image */
dewarpMinimize(dew);
dewarpApplyDisparity(dew, pixb2, 0);
regTestWritePixAndCheck(rp, dew->pixd, IFF_PNG); /* 6 */
pixDisplayWithTitle(dew->pixd, 400, 400, "fixed (2) for both disparities",
rp->display);
/* Write and read back minimized dewarp struct */
dewarpWrite("/tmp/dewarp.7.dew", dew);
regTestCheckFile(rp, "/tmp/dewarp.7.dew"); /* 7 */
dew2 = dewarpRead("/tmp/dewarp.7.dew");
dewarpWrite("/tmp/dewarp.8.dew", dew2);
regTestCheckFile(rp, "/tmp/dewarp.8.dew"); /* 8 */
regTestCompareFiles(rp, 7, 8); /* 9 */
/* Apply dew2 to pixb2 */
dewarpApplyDisparity(dew2, pixb2, 0);
regTestWritePixAndCheck(rp, dew2->pixd, IFF_PNG); /* 10 */
pixDisplayWithTitle(dew->pixd, 800, 400, "fixed (3) for both disparities",
rp->display);
/* Minimize, repopulate disparity arrays, and apply again */
dewarpMinimize(dew2);
dewarpApplyDisparity(dew2, pixb2, 0);
regTestWritePixAndCheck(rp, dew2->pixd, IFF_PNG); /* 11 */
regTestCompareFiles(rp, 10, 11); /* 12 */
pixDisplayWithTitle(dew->pixd, 900, 400, "fixed (4) for both disparities",
rp->display);
/* Test a few of the fpix functions */
fpix1 = fpixClone(dew->sampvdispar);
fpixWrite("/tmp/sampv.13.fpix", fpix1);
regTestCheckFile(rp, "/tmp/sampv.13.fpix"); /* 13 */
fpix2 = fpixRead("/tmp/sampv.13.fpix");
fpixWrite("/tmp/sampv.14.fpix", fpix2);
regTestCheckFile(rp, "/tmp/sampv.14.fpix"); /* 14 */
regTestCompareFiles(rp, 13, 14); /* 15 */
fpix3 = fpixScaleByInteger(fpix2, 30);
pixt1 = fpixRenderContours(fpix3, -2., 2.0, 0.2);
regTestWritePixAndCheck(rp, pixt1, IFF_PNG); /* 16 */
pixDisplayWithTitle(pixt1, 0, 800, "v. disparity contours", rp->display);
fpixDestroy(&fpix1);
fpixDestroy(&fpix2);
fpixDestroy(&fpix3);
pixDestroy(&pixt1);
/* Test a few of the dpix functions */
dpix1 = fpixConvertToDPix(dew->sampvdispar);
dpixWrite("/tmp/sampv.17.dpix", dpix1);
regTestCheckFile(rp, "/tmp/sampv.17.dpix"); /* 17 */
dpix2 = dpixRead("/tmp/sampv.17.dpix");
dpixWrite("/tmp/sampv.18.dpix", dpix2);
regTestCheckFile(rp, "/tmp/sampv.18.dpix"); /* 18 */
regTestCompareFiles(rp, 17, 18); /* 19 */
dpix3 = dpixScaleByInteger(dpix2, 30);
fpix3 = dpixConvertToFPix(dpix3);
pixt1 = fpixRenderContours(fpix3, -2., 2.0, 0.2);
regTestWritePixAndCheck(rp, pixt1, IFF_PNG); /* 20 */
pixDisplayWithTitle(pixt1, 400, 800, "v. disparity contours", rp->display);
regTestCompareFiles(rp, 16, 20); /* 21 */
dpixDestroy(&dpix1);
dpixDestroy(&dpix2);
dpixDestroy(&dpix3);
fpixDestroy(&fpix3);
pixDestroy(&pixt1);
dewarpDestroy(&dew);
dewarpDestroy(&dew2);
pixDestroy(&pixs);
pixDestroy(&pixb);
pixDestroy(&pixs2);
pixDestroy(&pixb2);
regTestCleanup(rp);
return 0;
}
