/*!
* fpixSampledDisparity()
*
* Input: fpixs (full resolution disparity model)
* sampling (sampling factor)
* Return: fpixd (sampled disparity model), or null on error
*
* Notes:
* (1) This converts full to sampled disparity.
* (2) The input array is sampled at the right and top edges, and
* at every @sampling pixels horizontally and vertically.
* (3) The sampled array may not extend to the right and bottom
* pixels in fpixs. This will occur if fpixs was generated
* with slope extension because the image on that page was
* larger than normal. This is fine, because in use the
* sampled array will be interpolated back to full resolution
* and then extended as required. So the operations of
* sampling and interpolation will be idempotent.
* (4) There must be at least 3 sampled points horizontally and
* vertically.
*/
static FPIX *
fpixSampledDisparity(FPIX *fpixs,
l_int32 sampling)
{
l_int32 w, h, wd, hd, i, j, is, js;
l_float32 val;
FPIX *fpixd;
PROCNAME("fpixSampledDisparity");
if (!fpixs)
return (FPIX *)ERROR_PTR("fpixs not defined", procName, NULL);
if (sampling < 1)
return (FPIX *)ERROR_PTR("sampling < 1", procName, NULL);
fpixGetDimensions(fpixs, &w, &h);
wd = 1 + (w + sampling - 2) / sampling;
hd = 1 + (h + sampling - 2) / sampling;
if (wd < 3 || hd < 3)
return (FPIX *)ERROR_PTR("wd < 3 or hd < 3", procName, NULL);
fpixd = fpixCreate(wd, hd);
for (i = 0; i < hd; i++) {
is = sampling * i;
if (is >= h) continue;
for (j = 0; j < wd; j++) {
js = sampling * j;
if (js >= w) continue;
fpixGetPixel(fpixs, js, is, &val);
fpixSetPixel(fpixd, j, i, val);
}
}
return fpixd;
}
/*!
* fpixSampledDisparity()
*
* Input: fpixs (full resolution disparity model)
* sampling (sampling factor)
* Return: fpixd (sampled disparity model), or null on error
*
* Notes:
* (1) The input array is sampled at the right and top edges, and
* at every @sampling pixels horizontally and vertically.
* (2) The sampled array is constructed large enough to (a) cover fpixs
* and (b) have the sampled grid on all boundary pixels in fpixd.
* Having sampled pixels around the boundary simplifies interpolation.
* (3) There must be at least 3 sampled points horizontally and
* vertically.
*/
FPIX *
fpixSampledDisparity(FPIX *fpixs,
l_int32 sampling)
{
l_int32 w, h, wd, hd, i, j, is, js;
l_float32 val;
l_float32 *array;
FPIX *fpixd;
PROCNAME("fpixSampledDisparity");
if (!fpixs)
return (FPIX *)ERROR_PTR("fpixs not defined", procName, NULL);
if (sampling < 1)
return (FPIX *)ERROR_PTR("sampling < 1", procName, NULL);
fpixGetDimensions(fpixs, &w, &h);
wd = 1 + (w + sampling - 2) / sampling;
hd = 1 + (h + sampling - 2) / sampling;
if (wd < 3 || hd < 3)
return (FPIX *)ERROR_PTR("wd < 3 or hd < 3", procName, NULL);
if ((array = (l_float32 *)CALLOC(w, sizeof(l_float32))) == NULL)
return (FPIX *)ERROR_PTR("calloc fail for array", procName, NULL);
fpixd = fpixCreate(wd, hd);
for (i = 0; i < hd; i++) {
is = sampling * i;
if (is >= h) continue;
for (j = 0; j < wd; j++) {
js = sampling * j;
if (js >= w) continue;
fpixGetPixel(fpixs, js, is, &val);
fpixSetPixel(fpixd, j, i, val);
array[j] = val;
}
/* Linear extrapolation to right-hand end point */
fpixSetPixel(fpixd, wd - 1, i, 2 * array[wd - 1] - array[wd - 2]);
}
/* Replicate value to bottom set */
for (j = 0; j < wd; j++) {
fpixGetPixel(fpixd, j, hd - 2, &val);
fpixSetPixel(fpixd, j, hd - 1, val);
}
FREE(array);
return fpixd;
}
/*!
* \brief pixQuadtreeMean()
*
* \param[in] pixs 8 bpp, no colormap
* \param[in] nlevels in quadtree; max allowed depends on image size
* \param[in] *pix_ma input mean accumulator; can be null
* \param[out] *pfpixa mean values in quadtree
* \return 0 if OK, 1 on error
*
* <pre>
* Notes:
* (1) The returned fpixa has %nlevels of fpix, each containing
* the mean values at its level. Level 0 has a
* single value; level 1 has 4 values; level 2 has 16; etc.
* </pre>
*/
l_int32
pixQuadtreeMean(PIX *pixs,
l_int32 nlevels,
PIX *pix_ma,
FPIXA **pfpixa)
{
l_int32 i, j, w, h, size, n;
l_float32 val;
BOX *box;
BOXA *boxa;
BOXAA *baa;
FPIX *fpix;
PIX *pix_mac;
PROCNAME("pixQuadtreeMean");
if (!pfpixa)
return ERROR_INT("&fpixa not defined", procName, 1);
*pfpixa = NULL;
if (!pixs || pixGetDepth(pixs) != 8)
return ERROR_INT("pixs not defined or not 8 bpp", procName, 1);
pixGetDimensions(pixs, &w, &h, NULL);
if (nlevels > quadtreeMaxLevels(w, h))
return ERROR_INT("nlevels too large for image", procName, 1);
if (!pix_ma)
pix_mac = pixBlockconvAccum(pixs);
else
pix_mac = pixClone(pix_ma);
if (!pix_mac)
return ERROR_INT("pix_mac not made", procName, 1);
if ((baa = boxaaQuadtreeRegions(w, h, nlevels)) == NULL) {
pixDestroy(&pix_mac);
return ERROR_INT("baa not made", procName, 1);
}
*pfpixa = fpixaCreate(nlevels);
for (i = 0; i < nlevels; i++) {
boxa = boxaaGetBoxa(baa, i, L_CLONE);
size = 1 << i;
n = boxaGetCount(boxa); /* n == size * size */
fpix = fpixCreate(size, size);
for (j = 0; j < n; j++) {
box = boxaGetBox(boxa, j, L_CLONE);
pixMeanInRectangle(pixs, box, pix_mac, &val);
fpixSetPixel(fpix, j % size, j / size, val);
boxDestroy(&box);
}
fpixaAddFPix(*pfpixa, fpix, L_INSERT);
boxaDestroy(&boxa);
}
pixDestroy(&pix_mac);
boxaaDestroy(&baa);
return 0;
}
/*!
* pixQuadtreeVariance()
*
* Input: pixs (8 bpp, no colormap)
* nlevels (in quadtree)
* *pix_ma (input mean accumulator; can be null)
* *dpix_msa (input mean square accumulator; can be null)
* *pfpixa_v (<optional return> variance values in quadtree)
* *pfpixa_rv (<optional return> root variance values in quadtree)
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) The returned fpixav and fpixarv have @nlevels of fpix,
* each containing at the respective levels the variance
* and root variance values.
*/
l_int32
pixQuadtreeVariance(PIX *pixs,
l_int32 nlevels,
PIX *pix_ma,
DPIX *dpix_msa,
FPIXA **pfpixa_v,
FPIXA **pfpixa_rv) {
l_int32 i, j, w, h, size, n;
l_float32 var, rvar;
BOX *box;
BOXA *boxa;
BOXAA *baa;
FPIX *fpixv, *fpixrv;
PIX *pix_mac; /* copy of mean accumulator */
DPIX *dpix_msac; /* msa clone */
PROCNAME("pixQuadtreeVariance");
if (!pfpixa_v && !pfpixa_rv)
return ERROR_INT("neither &fpixav nor &fpixarv defined", procName, 1);
if (pfpixa_v) *pfpixa_v = NULL;
if (pfpixa_rv) *pfpixa_rv = NULL;
if (!pixs || pixGetDepth(pixs) != 8)
return ERROR_INT("pixs not defined or not 8 bpp", procName, 1);
pixGetDimensions(pixs, &w, &h, NULL);
if (nlevels > quadtreeMaxLevels(w, h))
return ERROR_INT("nlevels too large for image", procName, 1);
if (!pix_ma)
pix_mac = pixBlockconvAccum(pixs);
else
pix_mac = pixClone(pix_ma);
if (!pix_mac)
return ERROR_INT("pix_mac not made", procName, 1);
if (!dpix_msa)
dpix_msac = pixMeanSquareAccum(pixs);
else
dpix_msac = dpixClone(dpix_msa);
if (!dpix_msac)
return ERROR_INT("dpix_msac not made", procName, 1);
if ((baa = boxaaQuadtreeRegions(w, h, nlevels)) == NULL) {
pixDestroy(&pix_mac);
dpixDestroy(&dpix_msac);
return ERROR_INT("baa not made", procName, 1);
}
if (pfpixa_v) *pfpixa_v = fpixaCreate(nlevels);
if (pfpixa_rv) *pfpixa_rv = fpixaCreate(nlevels);
for (i = 0; i < nlevels; i++) {
boxa = boxaaGetBoxa(baa, i, L_CLONE);
size = 1 << i;
n = boxaGetCount(boxa); /* n == size * size */
if (pfpixa_v) fpixv = fpixCreate(size, size);
if (pfpixa_rv) fpixrv = fpixCreate(size, size);
for (j = 0; j < n; j++) {
box = boxaGetBox(boxa, j, L_CLONE);
pixVarianceInRectangle(pixs, box, pix_mac, dpix_msac, &var, &rvar);
if (pfpixa_v) fpixSetPixel(fpixv, j % size, j / size, var);
if (pfpixa_rv) fpixSetPixel(fpixrv, j % size, j / size, rvar);
boxDestroy(&box);
}
if (pfpixa_v) fpixaAddFPix(*pfpixa_v, fpixv, L_INSERT);
if (pfpixa_rv) fpixaAddFPix(*pfpixa_rv, fpixrv, L_INSERT);
boxaDestroy(&boxa);
}
pixDestroy(&pix_mac);
dpixDestroy(&dpix_msac);
boxaaDestroy(&baa);
return 0;
}
/*!
* dewarpBuildModel()
*
* Input: dew
* debugflag (1 for debugging output)
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) This is the basic function that builds the vertical
* disparity array, which allows determination of the
* src pixel in the input image corresponding to each
* dest pixel in the dewarped image.
* (2) The method is as follows:
* * Estimate the centers of all the long textlines and
* fit a LS quadratic to each one. This smooths the curves.
* * Sample each curve at a regular interval, find the y-value
* of the flat point on each curve, and subtract the sampled
* curve value from this value. This is the vertical
* disparity.
* * Fit a LS quadratic to each set of vertically aligned
* disparity samples. This smooths the disparity values
* in the vertical direction. Then resample at the same
* regular interval, We now have a regular grid of smoothed
* vertical disparity valuels.
* * Interpolate this grid to get a full resolution disparity
* map. This can be applied directly to the src image
* pixels to dewarp the image in the vertical direction,
* making all textlines horizontal.
*/
l_int32
dewarpBuildModel(L_DEWARP *dew,
l_int32 debugflag)
{
char *tempname;
l_int32 i, j, nlines, nx, ny, sampling;
l_float32 c0, c1, c2, x, y, flaty, val;
l_float32 *faflats;
NUMA *nax, *nafit, *nacurve, *nacurves, *naflat, *naflats, *naflatsi;
PIX *pixs, *pixt1, *pixt2;
PTA *pta, *ptad;
PTAA *ptaa1, *ptaa2, *ptaa3, *ptaa4, *ptaa5, *ptaa6, *ptaa7;
FPIX *fpix1, *fpix2, *fpix3;
PROCNAME("dewarpBuildModel");
if (!dew)
return ERROR_INT("dew not defined", procName, 1);
pixs = dew->pixs;
if (debugflag) {
pixDisplayWithTitle(pixs, 0, 0, "pixs", 1);
pixWriteTempfile("/tmp", "pixs.png", pixs, IFF_PNG, NULL);
}
/* Make initial estimate of centers of textlines */
ptaa1 = pixGetTextlineCenters(pixs, DEBUG_TEXTLINE_CENTERS);
if (debugflag) {
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa1);
pixWriteTempfile("/tmp", "lines1.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
}
/* Remove all lines that are not near the length
* of the longest line. */
ptaa2 = ptaaRemoveShortLines(pixs, ptaa1, 0.8, DEBUG_SHORT_LINES);
if (debugflag) {
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa2);
pixWriteTempfile("/tmp", "lines2.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
}
nlines = ptaaGetCount(ptaa2);
if (nlines < dew->minlines)
return ERROR_INT("insufficient lines to build model", procName, 1);
/* Do quadratic fit to smooth each line. A single quadratic
* over the entire width of the line appears to be sufficient.
* Quartics tend to overfit to noise. Each line is thus
* represented by three coefficients: c2 * x^2 + c1 * x + c0.
* Using the coefficients, sample each fitted curve uniformly
* across the full width of the image. */
sampling = dew->sampling;
nx = dew->nx;
ny = dew->ny;
ptaa3 = ptaaCreate(nlines);
nacurve = numaCreate(nlines); /* stores curvature coeff c2 */
for (i = 0; i < nlines; i++) { /* for each line */
pta = ptaaGetPta(ptaa2, i, L_CLONE);
ptaGetQuadraticLSF(pta, &c2, &c1, &c0, NULL);
numaAddNumber(nacurve, c2);
ptad = ptaCreate(nx);
for (j = 0; j < nx; j++) { /* uniformly sampled in x */
x = j * sampling;
applyQuadraticFit(c2, c1, c0, x, &y);
ptaAddPt(ptad, x, y);
}
ptaaAddPta(ptaa3, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
ptaa4 = ptaaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa2, i, L_CLONE);
ptaGetArrays(pta, &nax, NULL);
ptaGetQuadraticLSF(pta, NULL, NULL, NULL, &nafit);
ptad = ptaCreateFromNuma(nax, nafit);
ptaaAddPta(ptaa4, ptad, L_INSERT);
ptaDestroy(&pta);
numaDestroy(&nax);
numaDestroy(&nafit);
}
pixt1 = pixConvertTo32(pixs);
pixt2 = pixDisplayPtaa(pixt1, ptaa4);
pixWriteTempfile("/tmp", "lines3.png", pixt2, IFF_PNG, NULL);
pixDestroy(&pixt1);
pixDestroy(&pixt2);
ptaaDestroy(&ptaa4);
}
/* Find and save the flat points in each curve. */
naflat = numaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa3, i, L_CLONE);
numaGetFValue(nacurve, i, &c2);
if (c2 <= 0) /* flat point at bottom; max value of y in curve */
ptaGetRange(pta, NULL, NULL, NULL, &flaty);
else /* flat point at top; min value of y in curve */
ptaGetRange(pta, NULL, NULL, &flaty, NULL);
numaAddNumber(naflat, flaty);
ptaDestroy(&pta);
}
/* Sort the lines in ptaa3 by their position */
naflatsi = numaGetSortIndex(naflat, L_SORT_INCREASING);
naflats = numaSortByIndex(naflat, naflatsi);
nacurves = numaSortByIndex(nacurve, naflatsi);
dew->naflats = naflats;
dew->nacurves = nacurves;
ptaa4 = ptaaSortByIndex(ptaa3, naflatsi);
numaDestroy(&naflat);
numaDestroy(&nacurve);
numaDestroy(&naflatsi);
if (debugflag) {
tempname = genTempFilename("/tmp", "naflats.na", 0);
numaWrite(tempname, naflats);
FREE(tempname);
}
/* Convert the sampled points in ptaa3 to a sampled disparity with
* with respect to the flat point in the curve. */
ptaa5 = ptaaCreate(nlines);
for (i = 0; i < nlines; i++) {
pta = ptaaGetPta(ptaa4, i, L_CLONE);
numaGetFValue(naflats, i, &flaty);
ptad = ptaCreate(nx);
for (j = 0; j < nx; j++) {
ptaGetPt(pta, j, &x, &y);
ptaAddPt(ptad, x, flaty - y);
}
ptaaAddPta(ptaa5, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa5.ptaa", 0);
ptaaWrite(tempname, ptaa5, 0);
FREE(tempname);
}
/* Generate a ptaa taking vertical 'columns' from ptaa5.
* We want to fit the vertical disparity on the column to the
* vertical position of the line, which we call 'y' here and
* obtain from naflats. */
ptaa6 = ptaaCreate(nx);
faflats = numaGetFArray(naflats, L_NOCOPY);
for (j = 0; j < nx; j++) {
pta = ptaCreate(nlines);
for (i = 0; i < nlines; i++) {
y = faflats[i];
ptaaGetPt(ptaa5, i, j, NULL, &val); /* disparity value */
ptaAddPt(pta, y, val);
}
ptaaAddPta(ptaa6, pta, L_INSERT);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa6.ptaa", 0);
ptaaWrite(tempname, ptaa6, 0);
FREE(tempname);
}
/* Do quadratic fit vertically on a subset of pixel columns
* for the vertical displacement, which identifies the
* src pixel(s) for each dest pixel. Sample the displacement
* on a regular grid in the vertical direction. */
ptaa7 = ptaaCreate(nx); /* uniformly sampled across full height of image */
for (j = 0; j < nx; j++) { /* for each column */
pta = ptaaGetPta(ptaa6, j, L_CLONE);
ptaGetQuadraticLSF(pta, &c2, &c1, &c0, NULL);
ptad = ptaCreate(ny);
for (i = 0; i < ny; i++) { /* uniformly sampled in y */
y = i * sampling;
applyQuadraticFit(c2, c1, c0, y, &val);
ptaAddPt(ptad, y, val);
}
ptaaAddPta(ptaa7, ptad, L_INSERT);
ptaDestroy(&pta);
}
if (debugflag) {
tempname = genTempFilename("/tmp", "ptaa7.ptaa", 0);
ptaaWrite(tempname, ptaa7, 0);
FREE(tempname);
}
/* Save the result in a fpix at the specified subsampling */
fpix1 = fpixCreate(nx, ny);
for (i = 0; i < ny; i++) {
for (j = 0; j < nx; j++) {
ptaaGetPt(ptaa7, j, i, NULL, &val);
fpixSetPixel(fpix1, j, i, val);
}
}
dew->sampvdispar = fpix1;
/* Generate a full res fpix for vertical dewarping. We require that
* the size of this fpix is at least as big as the input image. */
fpix2 = fpixScaleByInteger(fpix1, sampling);
dew->fullvdispar = fpix2;
if (debugflag) {
pixt1 = fpixRenderContours(fpix2, -2., 2.0, 0.2);
pixWriteTempfile("/tmp", "vert-contours.png", pixt1, IFF_PNG, NULL);
pixDisplay(pixt1, 1000, 0);
pixDestroy(&pixt1);
}
/* Generate full res and sampled fpix for horizontal dewarping. This
* works to the extent that the line curvature is due to bending
* out of the plane normal to the camera, and not wide-angle
* "fishbowl" distortion. Also generate the sampled horizontal
* disparity array. */
if (dew->applyhoriz) {
fpix3 = fpixBuildHorizontalDisparity(fpix2, 0, &dew->extraw);
dew->fullhdispar = fpix3;
dew->samphdispar = fpixSampledDisparity(fpix3, dew->sampling);
if (debugflag) {
pixt1 = fpixRenderContours(fpix3, -2., 2.0, 0.2);
pixWriteTempfile("/tmp", "horiz-contours.png", pixt1,
IFF_PNG, NULL);
pixDisplay(pixt1, 1000, 0);
pixDestroy(&pixt1);
}
}
dew->success = 1;
ptaaDestroy(&ptaa1);
ptaaDestroy(&ptaa2);
ptaaDestroy(&ptaa3);
ptaaDestroy(&ptaa4);
ptaaDestroy(&ptaa5);
ptaaDestroy(&ptaa6);
ptaaDestroy(&ptaa7);
return 0;
}
