/**
*
* Downsample an image
*
**/
static void zoom_out(
const float *I, // input image
float *Iout, // output image
const int nx, // image width
const int ny, // image height
const float factor // zoom factor between 0 and 1
)
{
// temporary working image
float *Is = xmalloc(nx * ny * sizeof(float));
for(int i = 0; i < nx * ny; i++)
Is[i] = I[i];
// compute the size of the zoomed image
int nxx, nyy;
zoom_size(nx, ny, &nxx, &nyy, factor);
// compute the Gaussian sigma for smoothing
const float sigma = ZOOM_SIGMA_ZERO * sqrt(1.0/(factor*factor) - 1.0);
// pre-smooth the image
gaussian_smoothing_in_place(Is, nx, ny, sigma);
// re-sample the image using bicubic interpolation
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (int i1 = 0; i1 < nyy; i1++)
for (int j1 = 0; j1 < nxx; j1++)
{
const float i2 = (float) i1 / factor;
const float j2 = (float) j1 / factor;
float g = bicubic_interpolation_at(Is, j2, i2, nx, ny, false);
Iout[i1 * nxx + j1] = g;
}
free(Is);
}
/**
*
* Procedure to handle the pyramidal approach.
* This procedure relies on the previous functions to calculate
* large optical flow fields using a pyramidal scheme.
*
*/
void horn_schunck_pyramidal(
const float *I1, // source image
const float *I2, // target image
float *u, // x component of optical flow
float *v, // y component of optical flow
const int nx, // image width
const int ny, // image height
const float alpha, // smoothing weight
const int nscales, // number of scales
const float zfactor, // zoom factor
const int warps, // number of warpings per scale
const float TOL, // stopping criterion threshold
const int maxiter, // maximum number of iterations
const bool verbose // switch on messages
)
{
if (verbose) fprintf(stderr, "Multiscale Horn-Schunck of a %dx%d pair"
"\n\ta=%g ns=%d zf=%g nw=%d eps=%g mi=%d\n", nx, ny,
alpha, nscales, zfactor, warps, TOL, maxiter);
int size = nx * ny;
float *I1s[nscales];
float *I2s[nscales];
float *us[nscales];
float *vs[nscales];
int nxx[nscales];
int nyy[nscales];
I1s[0] = xmalloc(size * sizeof(float));
I2s[0] = xmalloc(size * sizeof(float));
// normalize the finest scale images between 0 and 255
image_normalization(I1, I2, I1s[0], I2s[0], size);
// presmoothing the finest scale images
gaussian(I1s[0], nx, ny, INPUT_PRESMOOTHING_SIGMA);
gaussian(I2s[0], nx, ny, INPUT_PRESMOOTHING_SIGMA);
us[0] = u;
vs[0] = v;
nxx[0] = nx;
nyy[0] = ny;
// create the scales
for(int s = 1; s < nscales; s++)
{
zoom_size(nxx[s-1], nyy[s-1], nxx+s, nyy+s, zfactor);
const int sizes = nxx[s] * nyy[s];
I1s[s] = xmalloc(sizes * sizeof(float));
I2s[s] = xmalloc(sizes * sizeof(float));
us[s] = xmalloc(sizes * sizeof(float));
vs[s] = xmalloc(sizes * sizeof(float));
// compute the zoom from the previous finer scale
zoom_out(I1s[s-1], I1s[s], nxx[s-1], nyy[s-1], zfactor);
zoom_out(I2s[s-1], I2s[s], nxx[s-1], nyy[s-1], zfactor);
}
// initialize the flow
for (int i = 0; i < nxx[nscales-1] * nyy[nscales-1]; i++)
{
us[nscales-1][i] = 0;
vs[nscales-1][i] = 0;
}
// pyramidal approximation to the optic flow
for(int s = nscales-1; s >= 0; s--)
{
if(verbose)
fprintf(stderr, "Scale: %d %dx%d\n", s, nxx[s], nyy[s]);
// compute the optical flow at this scale
horn_schunck_optical_flow(
I1s[s], I2s[s], us[s], vs[s], nxx[s], nyy[s],
alpha, warps, TOL, maxiter, verbose
);
// if this was the last scale, finish now
if (!s) break;
// otherwise, upsample the optical flow
// zoom the optic flow for the next finer scale
zoom_in(us[s], us[s-1], nxx[s], nyy[s], nxx[s-1], nyy[s-1]);
zoom_in(vs[s], vs[s-1], nxx[s], nyy[s], nxx[s-1], nyy[s-1]);
// scale the optic flow with the appropriate zoom factor
for(int i = 0; i < nxx[s-1] * nyy[s-1]; i++)
{
us[s-1][i] *= 1.0 / zfactor;
vs[s-1][i] *= 1.0 / zfactor;
}
}
// free the allocated memory
free(I1s[0]);
free(I2s[0]);
for(int i = 1; i < nscales; i++)
{
free(I1s[i]);
free(I2s[i]);
free(us[i]);
free(vs[i]);
}
}
/**
* calcMSCLG_OF
*
* Main Multiscale CLG-optical flow (CLG-OF) computation function.
*
* Parameters:
*
* image1 Pointer to the first image (the "previous" time frame).
* image2 Pointer to the second image (the "current" time frame).
* uOut Pointer to the horizontal component of the CLG-OF solution.
* vOut Pointer to the vertical component of the CLG-OF solution.
* nRows Number of image rows (same for the CLG-OF vector field).
* nCols Number of image columns (same for the CLG-OF vector field).
* iterations Number of iterations for iterative solution.
* alpha Global smoothing coefficient of the CLG-OF.
* rho Local spatio-temporal smoothing coefficient of the CLG-OF.
* sigma Standard deviation for an optional gaussian smoothing kernel,
* applied to the input images prior to the CLG-OF computation.
* wFactor SOR relaxation factor, between 0 and 2.
* nScales Total number of scales (at least 1).
* scaleFactor Downsampling factor, between 0.5 and 1.0.
* coupledMode Iteration type. 1->Pointwise-Coupled Gauss-Seidel, 0->SOR.
* verbose Display/hide messages to the stdout.
*
*/
int calcMSCLG_OF(double* image1,
double* image2,
double* uOut,
double* vOut,
int nRows,
int nCols,
int iterations,
double alpha,
double rho,
double sigma,
double wFactor,
int nScales,
const double scaleFactor,
int coupledMode,
int verbose) {
if (verbose) {
printf("calcMSCLG_OF\n");
printf(" parameters: nScales=%d, scaleFactor=%f\n",
nScales, scaleFactor);
}
// Variables declaration.
int s, i;
double *image2warped;
int size = nCols * nRows;
double *I1s[nScales];
double *I2s[nScales];
double *us[nScales];
double *vs[nScales];
int nxx[nScales];
int nyy[nScales];
I1s[0] = image1;
I2s[0] = image2;
// Pre-smoothing the finest scale images.
int filterSize = (int) 2*(DEFAULT_GAUSSIAN_WINDOW_SIZE * sigma) + 1;
if (filterSize >= MIN_GAUSSIAN_SIZE) {
printf(" apply gaussian smoothing for each frame\n");
gaussian(I1s[0], nCols, nRows, sigma);
gaussian(I2s[0], nCols, nRows, sigma);
} else {
printf(" no gaussian smoothing was applied to input frames\n");
}
us[0] = uOut;
vs[0] = vOut;
nxx[0] = nCols;
nyy[0] = nRows;
// Create the scales.
for (s=1; s<nScales; s++) {
if (verbose)
printf(" scales %d init\n", s);
zoom_size(nxx[s-1], nyy[s-1], nxx+s, nyy+s, scaleFactor);
const int sizes = nxx[s] * nyy[s];
I1s[s] = (double *) xmalloc(sizes * sizeof(double));
I2s[s] = (double *) xmalloc(sizes * sizeof(double));
us[s] = (double *) xmalloc(sizes * sizeof(double));
vs[s] = (double *) xmalloc(sizes * sizeof(double));
// Compute the zoom from the previous finer scale.
zoom_out(I1s[s-1], I1s[s], nxx[s-1], nyy[s-1], scaleFactor);
zoom_out(I2s[s-1], I2s[s], nxx[s-1], nyy[s-1], scaleFactor);
}
// Initialize the OF arrays.
for (i=0; i < nxx[nScales-1] * nyy[nScales-1]; i++) {
us[nScales-1][i] = 0.0;
vs[nScales-1][i] = 0.0;
}
// Pyramidal approximation to the OF.
for (s=nScales-1; s>=0; s--) {
if (verbose)
printf("Scale: %d %dx%d, nScales=%d\n",
s, nxx[s], nyy[s], nScales);
image2warped = (double *) xmalloc(nxx[s] * nyy[s] * sizeof(double));
// Warp the second image before computing the derivatives.
bicubic_interpolation_warp(I2s[s], us[s], vs[s], image2warped,
nxx[s], nyy[s], false);
// Compute the optical flow.
calcCLG_OF(I1s[s], image2warped, us[s], vs[s], nyy[s], nxx[s],
iterations, alpha, rho, wFactor, verbose, coupledMode);
free(image2warped);
// If this was the last scale, finish now.
if (!s) break;
// Otherwise, upsample the optical flow.
// Zoom the OF for the next finer scale.
zoom_in(us[s], us[s-1], nxx[s], nyy[s], nxx[s-1], nyy[s-1]);
zoom_in(vs[s], vs[s-1], nxx[s], nyy[s], nxx[s-1], nyy[s-1]);
// Scale the OF with the appropriate zoom factor.
for (i=0; i < nxx[s-1] * nyy[s-1]; i++) {
us[s-1][i] *= 1.0 / scaleFactor;
vs[s-1][i] *= 1.0 / scaleFactor;
}
}
// Free the allocated memory.
for (i=1; i<nScales; i++) {
if (verbose)
printf("Free Scale: %d %dx%d\n", s, nxx[s], nyy[s]);
free(I1s[i]);
free(I2s[i]);
free(us[i]);
free(vs[i]);
}
if (verbose)
printf("calcMSCLG_OF: done\n");
return 1;
}
/**
*
* Function to compute the optical flow using multiple scales
*
**/
void Dual_TVL1_optic_flow_multiscale(
float *I0, // source image
float *I1, // target image
float *u1, // x component of the optical flow
float *u2, // y component of the optical flow
const int nxx, // image width
const int nyy, // image height
const float tau, // time step
const float lambda, // weight parameter for the data term
const float theta, // weight parameter for (u - v)²
const int nscales, // number of scales
const float zfactor, // factor for building the image piramid
const int warps, // number of warpings per scale
const float epsilon, // tolerance for numerical convergence
const bool verbose // enable/disable the verbose mode
)
{
int size = nxx * nyy;
// allocate memory for the pyramid structure
float **I0s = xmalloc(nscales * sizeof(float*));
float **I1s = xmalloc(nscales * sizeof(float*));
float **u1s = xmalloc(nscales * sizeof(float*));
float **u2s = xmalloc(nscales * sizeof(float*));
int *nx = xmalloc(nscales * sizeof(int));
int *ny = xmalloc(nscales * sizeof(int));
I0s[0] = xmalloc(size*sizeof(float));
I1s[0] = xmalloc(size*sizeof(float));
u1s[0] = u1;
u2s[0] = u2;
nx [0] = nxx;
ny [0] = nyy;
// normalize the images between 0 and 255
image_normalization(I0, I1, I0s[0], I1s[0], size);
// pre-smooth the original images
gaussian(I0s[0], nx[0], ny[0], PRESMOOTHING_SIGMA);
gaussian(I1s[0], nx[0], ny[0], PRESMOOTHING_SIGMA);
// create the scales
for (int s = 1; s < nscales; s++)
{
zoom_size(nx[s-1], ny[s-1], &nx[s], &ny[s], zfactor);
const int sizes = nx[s] * ny[s];
// allocate memory
I0s[s] = xmalloc(sizes*sizeof(float));
I1s[s] = xmalloc(sizes*sizeof(float));
u1s[s] = xmalloc(sizes*sizeof(float));
u2s[s] = xmalloc(sizes*sizeof(float));
// zoom in the images to create the pyramidal structure
zoom_out(I0s[s-1], I0s[s], nx[s-1], ny[s-1], zfactor);
zoom_out(I1s[s-1], I1s[s], nx[s-1], ny[s-1], zfactor);
}
// initialize the flow at the coarsest scale
for (int i = 0; i < nx[nscales-1] * ny[nscales-1]; i++)
u1s[nscales-1][i] = u2s[nscales-1][i] = 0.0;
// pyramidal structure for computing the optical flow
for (int s = nscales-1; s >= 0; s--)
{
if (verbose)
fprintf(stderr, "Scale %d: %dx%d\n", s, nx[s], ny[s]);
// compute the optical flow at the current scale
Dual_TVL1_optic_flow(
I0s[s], I1s[s], u1s[s], u2s[s], nx[s], ny[s],
tau, lambda, theta, warps, epsilon, verbose
);
// if this was the last scale, finish now
if (!s) break;
// otherwise, upsample the optical flow
// zoom the optical flow for the next finer scale
zoom_in(u1s[s], u1s[s-1], nx[s], ny[s], nx[s-1], ny[s-1]);
zoom_in(u2s[s], u2s[s-1], nx[s], ny[s], nx[s-1], ny[s-1]);
// scale the optical flow with the appropriate zoom factor
for (int i = 0; i < nx[s-1] * ny[s-1]; i++)
{
u1s[s-1][i] *= (float) 1.0 / zfactor;
u2s[s-1][i] *= (float) 1.0 / zfactor;
}
}
// delete allocated memory
for (int i = 1; i < nscales; i++)
{
free(I0s[i]);
free(I1s[i]);
free(u1s[i]);
free(u2s[i]);
}
free(I0s[0]);
free(I1s[0]);
free(I0s);
free(I1s);
free(u1s);
free(u2s);
free(nx);
free(ny);
}