void SimilarityContent::addImage(VImage *vim) {
color_image_t* im = makeGistImage(vim);
float* desc = color_gist_scaletab(
im, NBLOCKS, N_SCALE, ORIENTATIONS_PER_SCALE);
color_image_delete(im);
descriptions[vim] = desc;
}
/* resize a color image with bilinear interpolation */
color_image_t *color_image_resize_bilinear(const color_image_t *src, const float scale){
const int width = src->width, height = src->height;
const int newwidth = (int) (1.5f + (width-1) / scale); // 0.5f for rounding instead of flooring, and the remaining comes from scale = (dst-1)/(src-1)
const int newheight = (int) (1.5f + (height-1) / scale);
color_image_t *dst = color_image_new(newwidth,newheight);
if(height*newwidth < width*newheight){
color_image_t *tmp = color_image_new(newwidth,height);
color_image_resize_horiz(tmp,src);
color_image_resize_vert(dst,tmp);
color_image_delete(tmp);
}else{
color_image_t *tmp = color_image_new(width,newheight);
color_image_resize_vert(tmp,src);
color_image_resize_horiz(dst,tmp);
color_image_delete(tmp);
}
return dst;
}
int main(int argc,char **args) {
const char *infilename="/dev/stdin";
int nblocks=4;
int n_scale=3;
int orientations_per_scale[50]={8,8,4};
while(*++args) {
const char *a=*args;
if(!strcmp(a,"-h")) usage();
else if(!strcmp(a,"-nblocks")) {
if(!sscanf(*++args,"%d",&nblocks)) {
fprintf(stderr,"could not parse %s argument",a);
usage();
}
} else if(!strcmp(a,"-orientationsPerScale")) {
char *c;
n_scale=0;
for(c=strtok(*++args,",");c;c=strtok(NULL,",")) {
if(!sscanf(c,"%d",&orientations_per_scale[n_scale++])) {
fprintf(stderr,"could not parse %s argument",a);
usage();
}
}
} else {
infilename=a;
}
}
color_image_t *im=load_ppm(infilename);
float *desc = color_gist_scaletab(im,nblocks,n_scale,orientations_per_scale);
int i;
int descsize=0;
/* compute descriptor size */
for(i=0;i<n_scale;i++)
descsize+=nblocks*nblocks*orientations_per_scale[i];
descsize*=3; /* color */
/* print descriptor */
for(i=0;i<descsize;i++)
printf("%.4f ",desc[i]);
printf("\n");
free(desc);
color_image_delete(im);
return 0;
}
/* delete the structure of a pyramid of color images and all the color images in it*/
void color_image_pyramid_delete(color_image_pyramid_t *pyr){
if(pyr==NULL){
return;
}
int i;
for(i=0 ; i<pyr->size ; i++){
color_image_delete(pyr->images[i]);
}
free(pyr->images);
free(pyr);
}
/* Compute a refinement of the optical flow (wx and wy are modified) between im1 and im2 */
void variational(image_t *wx, image_t *wy, const color_image_t *im1, const color_image_t *im2, variational_params_t *params){
// Check parameters
if(!params){
params = (variational_params_t*) malloc(sizeof(variational_params_t));
if(!params){
fprintf(stderr,"error: not enough memory\n");
exit(1);
}
variational_params_default(params);
}
// initialize global variables
half_alpha = 0.5f*params->alpha;
half_gamma_over3 = params->gamma*0.5f/3.0f;
half_delta_over3 = params->delta*0.5f/3.0f;
float deriv_filter[3] = {0.0f, -8.0f/12.0f, 1.0f/12.0f};
deriv = convolution_new(2, deriv_filter, 0);
float deriv_filter_flow[2] = {0.0f, -0.5f};
deriv_flow = convolution_new(1, deriv_filter_flow, 0);
// presmooth images
int width = im1->width, height = im1->height, filter_size;
color_image_t *smooth_im1 = color_image_new(width, height), *smooth_im2 = color_image_new(width, height);
float *presmooth_filter = gaussian_filter(params->sigma, &filter_size);
convolution_t *presmoothing = convolution_new(filter_size, presmooth_filter, 1);
color_image_convolve_hv(smooth_im1, im1, presmoothing, presmoothing);
color_image_convolve_hv(smooth_im2, im2, presmoothing, presmoothing);
convolution_delete(presmoothing);
free(presmooth_filter);
compute_one_level(wx, wy, smooth_im1, smooth_im2, params);
// free memory
color_image_delete(smooth_im1);
color_image_delete(smooth_im2);
convolution_delete(deriv);
convolution_delete(deriv_flow);
}
void color_gist_scaletab_wrap(uint8_t *data, int height, int width, int nblocks, int n_scale, const int *orientations_per_scale, float *desc, int desc_size) {
color_image_t *im=color_image_new(width, height);
int i, size = height * width;
for (i = 0; i < size; ++i) {
im->c1[i] = *(data++);
im->c2[i] = *(data++);
im->c3[i] = *(data++);
}
float *desc_out = color_gist_scaletab(im, nblocks, n_scale, orientations_per_scale);
memcpy(desc, desc_out, desc_size * sizeof(float));
free(desc_out);
color_image_delete(im);
}
void color_gist_scaletab_wrap(unsigned char *data, int height, int width, int nblocks, int n_scale, const int *orientations_per_scale, float *desc, int desc_size) {
float *desc_out;
color_image_t *im=color_image_new(width, height);
int i, size = height * width;
// Not only copies to data structure but also switches BGR -> RGB
for (i = 0; i < size; ++i) {
im->c3[i] = *(data++);
im->c2[i] = *(data++);
im->c1[i] = *(data++);
}
desc_out = color_gist_scaletab(im, nblocks, n_scale, orientations_per_scale);
if (desc_out != NULL)
memcpy(desc, desc_out, desc_size * sizeof(float));
free(desc_out);
color_image_delete(im);
}
/* create a pyramid of color images using a given scale factor, stopping when one dimension reach min_size and with applying a gaussian smoothing of standard deviation spyr (no smoothing if 0) */
color_image_pyramid_t *color_image_pyramid_create(const color_image_t *src, const float scale_factor, const int min_size, const float spyr){
const int nb_max_scale = 1000;
// allocate structure
color_image_pyramid_t *pyramid = color_image_pyramid_new();
pyramid->min_size = min_size;
pyramid->scale_factor = scale_factor;
convolution_t *conv = NULL;
if(spyr>0.0f){
int fsize;
float *filter_coef = gaussian_filter(spyr, &fsize);
conv = convolution_new(fsize, filter_coef, 1);
free(filter_coef);
}
color_image_pyramid_set_size(pyramid, nb_max_scale);
pyramid->images[0] = color_image_cpy(src);
int i;
for( i=1 ; i<nb_max_scale ; i++){
const int oldwidth = pyramid->images[i-1]->width, oldheight = pyramid->images[i-1]->height;
const int newwidth = (int) (1.5f + (oldwidth-1) / scale_factor);
const int newheight = (int) (1.5f + (oldheight-1) / scale_factor);
if( newwidth <= min_size || newheight <= min_size){
color_image_pyramid_set_size(pyramid, i);
break;
}
if(spyr>0.0f){
color_image_t* tmp = color_image_new(oldwidth, oldheight);
color_image_convolve_hv(tmp,pyramid->images[i-1], conv, conv);
pyramid->images[i]= color_image_resize_bilinear(tmp, scale_factor);
color_image_delete(tmp);
}else{
pyramid->images[i] = color_image_resize_bilinear(pyramid->images[i-1], scale_factor);
}
}
if(spyr>0.0f){
convolution_delete(conv);
}
return pyramid;
}
float *color_gist_scaletab(color_image_t *src, int w, int n_scale, const int *n_orientation)
{
int i;
if(src->width < 8 || src->height < 8)
{
fprintf(stderr, "Error: color_gist_scaletab() - Image not big enough !\n");
return NULL;
}
int numberBlocks = w;
int tot_oris=0;
for(i=0;i<n_scale;i++) tot_oris+=n_orientation[i];
color_image_t *img = color_image_cpy(src);
image_list_t *G = create_gabor(n_scale, n_orientation, img->width, img->height);
color_prefilt(img, 4);
float *g = color_gist_gabor(img, numberBlocks, G);
for(i = 0; i < tot_oris*w*w*3; i++)
{
if(!finite(g[i]))
{
fprintf(stderr, "Error: color_gist_scaletab() - descriptor not valid (nan or inf)\n");
free(g); g=NULL;
break;
}
}
image_list_delete(G);
color_image_delete(img);
return g;
}
/* compute image first and second order spatio-temporal derivatives of a color image */
void get_derivatives(const color_image_t *im1, const color_image_t *im2, const convolution_t *deriv,
color_image_t *dx, color_image_t *dy, color_image_t *dt,
color_image_t *dxx, color_image_t *dxy, color_image_t *dyy, color_image_t *dxt, color_image_t *dyt) {
// derivatives are computed on the mean of the first image and the warped second image
color_image_t *tmp_im2 = color_image_new(im2->width,im2->height);
v4sf *tmp_im2p = (v4sf*) tmp_im2->c1, *dtp = (v4sf*) dt->c1, *im1p = (v4sf*) im1->c1, *im2p = (v4sf*) im2->c1;
const v4sf half = {0.5f,0.5f,0.5f,0.5f};
int i=0;
for(i=0 ; i<3*im1->height*im1->stride/4 ; i++){
*tmp_im2p = half * ( (*im2p) + (*im1p) );
*dtp = (*im2p)-(*im1p);
dtp+=1; im1p+=1; im2p+=1; tmp_im2p+=1;
}
// compute all other derivatives
color_image_convolve_hv(dx, tmp_im2, deriv, NULL);
color_image_convolve_hv(dy, tmp_im2, NULL, deriv);
color_image_convolve_hv(dxx, dx, deriv, NULL);
color_image_convolve_hv(dxy, dx, NULL, deriv);
color_image_convolve_hv(dyy, dy, NULL, deriv);
color_image_convolve_hv(dxt, dt, deriv, NULL);
color_image_convolve_hv(dyt, dt, NULL, deriv);
// free memory
color_image_delete(tmp_im2);
}
/* perform flow computation at one level of the pyramid */
void compute_one_level(image_t *wx, image_t *wy, color_image_t *im1, color_image_t *im2, const variational_params_t *params){
const int width = wx->width, height = wx->height, stride=wx->stride;
image_t *du = image_new(width,height), *dv = image_new(width,height), // the flow increment
*mask = image_new(width,height), // mask containing 0 if a point goes outside image boundary, 1 otherwise
*smooth_horiz = image_new(width,height), *smooth_vert = image_new(width,height), // horiz: (i,j) contains the diffusivity coeff from (i,j) to (i+1,j)
*uu = image_new(width,height), *vv = image_new(width,height), // flow plus flow increment
*a11 = image_new(width,height), *a12 = image_new(width,height), *a22 = image_new(width,height), // system matrix A of Ax=b for each pixel
*b1 = image_new(width,height), *b2 = image_new(width,height); // system matrix b of Ax=b for each pixel
color_image_t *w_im2 = color_image_new(width,height), // warped second image
*Ix = color_image_new(width,height), *Iy = color_image_new(width,height), *Iz = color_image_new(width,height), // first order derivatives
*Ixx = color_image_new(width,height), *Ixy = color_image_new(width,height), *Iyy = color_image_new(width,height), *Ixz = color_image_new(width,height), *Iyz = color_image_new(width,height); // second order derivatives
image_t *dpsis_weight = compute_dpsis_weight(im1, 5.0f, deriv);
int i_outer_iteration;
for(i_outer_iteration = 0 ; i_outer_iteration < params->niter_outer ; i_outer_iteration++){
int i_inner_iteration;
// warp second image
image_warp(w_im2, mask, im2, wx, wy);
// compute derivatives
get_derivatives(im1, w_im2, deriv, Ix, Iy, Iz, Ixx, Ixy, Iyy, Ixz, Iyz);
// erase du and dv
image_erase(du);
image_erase(dv);
// initialize uu and vv
memcpy(uu->data,wx->data,wx->stride*wx->height*sizeof(float));
memcpy(vv->data,wy->data,wy->stride*wy->height*sizeof(float));
// inner fixed point iterations
for(i_inner_iteration = 0 ; i_inner_iteration < params->niter_inner ; i_inner_iteration++){
// compute robust function and system
compute_smoothness(smooth_horiz, smooth_vert, uu, vv, dpsis_weight, deriv_flow, half_alpha );
compute_data_and_match(a11, a12, a22, b1, b2, mask, du, dv, Ix, Iy, Iz, Ixx, Ixy, Iyy, Ixz, Iyz, half_delta_over3, half_gamma_over3);
sub_laplacian(b1, wx, smooth_horiz, smooth_vert);
sub_laplacian(b2, wy, smooth_horiz, smooth_vert);
// solve system
sor_coupled(du, dv, a11, a12, a22, b1, b2, smooth_horiz, smooth_vert, params->niter_solver, params->sor_omega);
// update flow plus flow increment
int i;
v4sf *uup = (v4sf*) uu->data, *vvp = (v4sf*) vv->data, *wxp = (v4sf*) wx->data, *wyp = (v4sf*) wy->data, *dup = (v4sf*) du->data, *dvp = (v4sf*) dv->data;
for( i=0 ; i<height*stride/4 ; i++){
(*uup) = (*wxp) + (*dup);
(*vvp) = (*wyp) + (*dvp);
uup+=1; vvp+=1; wxp+=1; wyp+=1;dup+=1;dvp+=1;
}
}
// add flow increment to current flow
memcpy(wx->data,uu->data,uu->stride*uu->height*sizeof(float));
memcpy(wy->data,vv->data,vv->stride*vv->height*sizeof(float));
}
// free memory
image_delete(du); image_delete(dv);
image_delete(mask);
image_delete(smooth_horiz); image_delete(smooth_vert);
image_delete(uu); image_delete(vv);
image_delete(a11); image_delete(a12); image_delete(a22);
image_delete(b1); image_delete(b2);
image_delete(dpsis_weight);
color_image_delete(w_im2);
color_image_delete(Ix); color_image_delete(Iy); color_image_delete(Iz);
color_image_delete(Ixx); color_image_delete(Ixy); color_image_delete(Iyy); color_image_delete(Ixz); color_image_delete(Iyz);
}
static PyObject* gist_extract(PyObject *self, PyObject *args)
{
int nblocks=4;
int n_scale=3;
int orientations_per_scale[50]={8,8,4};
PyArrayObject *image, *descriptor;
if (!PyArg_ParseTuple(args, "O", &image))
{
return NULL;
}
if (PyArray_TYPE(image) != NPY_UINT8) {
PyErr_SetString(PyExc_TypeError, "type of image must be uint8");
return NULL;
}
if (PyArray_NDIM(image) != 3) {
PyErr_SetString(PyExc_TypeError, "dimensions of image must be 3.");
return NULL;
}
npy_intp *dims_image = PyArray_DIMS(image);
const int w = (int) *(dims_image+1);
const int h = (int) *(dims_image);
// Read image to color_image_t structure
color_image_t *im=color_image_new(w,h);
for (int y=0, i=0 ; y<h ; ++y) {
for (int x=0 ; x<w ; ++x, ++i) {
im->c1[i] = *(unsigned char *)PyArray_GETPTR3(image, y, x, 0);
im->c2[i] = *(unsigned char *)PyArray_GETPTR3(image, y, x, 1);
im->c3[i] = *(unsigned char *)PyArray_GETPTR3(image, y, x, 2);
}
}
// Extract descriptor
float *desc=color_gist_scaletab(im,nblocks,n_scale,orientations_per_scale);
int descsize=0;
/* compute descriptor size */
for(int i=0;i<n_scale;i++)
descsize+=nblocks*nblocks*orientations_per_scale[i];
descsize*=3; /* color */
// Allocate output
npy_intp dim_desc[1] = {descsize};
descriptor = (PyArrayObject *) PyArray_SimpleNew(1, dim_desc, NPY_FLOAT);
// Set val
for (int i=0 ; i<descsize ; ++i) {
*(float *)PyArray_GETPTR1(descriptor, i) = desc[i];
}
// Release memory
color_image_delete(im);
free(desc);
return PyArray_Return(descriptor);
}
int main(int argc, char ** argv){
image_t *match_x = NULL, *match_y = NULL, *match_z = NULL;
// load images
if(argc < 4){
fprintf(stderr,"Wrong command, require at least 3 arguments.\n\n");
usage();
exit(1);
}
color_image_t *im1 = color_image_load(argv[1]), *im2 = color_image_load(argv[2]);
if(im1->width != im2->width || im1->height != im2->height){
fprintf(stderr,"Image dimensions does not match\n");
exit(1);
}
// set params to default
optical_flow_params_t* params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t));
if(!params){
fprintf(stderr,"error deepflow(): not enough memory\n");
exit(1);
}
optical_flow_params_default(params);
// parse options
int current_arg = 4;
while(1){
if( current_arg >= argc) break;
if(!strcmp(argv[current_arg],"-h") || !strcmp(argv[current_arg],"--help") ){
usage();
exit(1);
}else if(!strcmp(argv[current_arg],"-a") || !strcmp(argv[current_arg],"-alpha") ){
current_arg++;
if(current_arg >= argc) require_argument("alpha");
float alpha = atof(argv[current_arg++]);
if(alpha<0){
fprintf(stderr,"Alpha argument cannot be negative\n");
exit(1);
}
params->alpha = alpha;
}else if(!strcmp(argv[current_arg],"-b") || !strcmp(argv[current_arg],"-beta") ){
current_arg++;
if(current_arg >= argc) require_argument("beta");
float beta = atof(argv[current_arg++]);
if(beta<0){
fprintf(stderr,"Beta argument cannot be negative\n");
exit(1);
}
params->beta = beta;
}else if(!strcmp(argv[current_arg],"-g") || !strcmp(argv[current_arg],"-gamma") ){
current_arg++;
if(current_arg >= argc) require_argument("gamma");
float gamma = atof(argv[current_arg++]);
if(gamma<0){
fprintf(stderr,"Gamma argument cannot be negative\n");
exit(1);
}
params->gamma = gamma;
}else if(!strcmp(argv[current_arg],"-d") || !strcmp(argv[current_arg],"-delta") ){
current_arg++;
if(current_arg >= argc) require_argument("delta");
float delta = atof(argv[current_arg++]);
if(delta<0) {
fprintf(stderr,"Delta argument cannot be negative\n");
exit(1);
}
params->delta = delta;
}else if(!strcmp(argv[current_arg],"-s") || !strcmp(argv[current_arg],"-sigma") ){
current_arg++;
if(current_arg >= argc) require_argument("sigma");
float sigma = atof(argv[current_arg++]);
if(sigma<0){
fprintf(stderr,"Sigma argument is negative\n");
exit(1);
}
params->sigma = sigma;
}else if(!strcmp(argv[current_arg],"-bk")) {
current_arg++;
if(current_arg >= argc) require_argument("bk");
float betak = atof(argv[current_arg++]);
if(betak<0.0f){
fprintf(stderr,"Bk argument must be positive\n");
exit(1);
}
params->bk = betak;
}else if(!strcmp(argv[current_arg],"-e") || !strcmp(argv[current_arg],"-eta") ){
current_arg++;
if(current_arg >= argc) require_argument("eta");
float eta = atof(argv[current_arg++]);
if(eta<0.25 || eta>0.98){
fprintf(stderr,"Eta argument has to be between 0.25 and 0.98\n");
exit(1);
}
params->eta = eta;
}else if( !strcmp(argv[current_arg],"-minsize") ){
current_arg++;
if(current_arg >= argc) require_argument("minsize");
int minsize = atoi(argv[current_arg++]);
if(minsize < 10){
fprintf(stderr,"Minsize argument has to be higher than 10\n");
exit(1);
}
params->min_size = minsize;
}else if(!strcmp(argv[current_arg],"-inner") ){
current_arg++;
if(current_arg >= argc) require_argument("inner");
int inner = atoi(argv[current_arg++]);
if(inner<=0){
fprintf(stderr,"Inner argument must be strictly positive\n");
exit(1);
}
params->n_inner_iteration = inner;
}else if(!strcmp(argv[current_arg],"-iter") ){
current_arg++;
if(current_arg >= argc) require_argument("iter");
int iter = atoi(argv[current_arg++]);
if(iter<=0){
fprintf(stderr,"Iter argument must be strictly positive\n");
exit(1);
}
params->n_solver_iteration = iter;
}else if( !strcmp(argv[current_arg],"-match") || !strcmp(argv[current_arg],"-matchf")){
int wm = im1->width, hm = im1->height;
if( !strcmp(argv[current_arg++],"-match") ){
wm = 512;
hm = 256;
}
image_delete(match_x); image_delete(match_y); image_delete(match_z);
match_x = image_new(wm, hm); match_y = image_new(wm, hm); match_z = image_new(wm, hm);
image_erase(match_x); image_erase(match_y); image_erase(match_z);
FILE *fid = stdin;
if( current_arg<argc && argv[current_arg][0] != '-'){
fid = fopen(argv[current_arg++], "r");
if(fid==NULL){
fprintf(stderr, "Cannot read matches from file %s", argv[current_arg-1]);
exit(1);
}
}
int x1, x2, y1, y2;
float score;
while(!feof(fid) && fscanf(fid, "%d %d %d %d %f\n", &x1, &y1, &x2, &y2, &score)==5){
if( x1<0 || y1<0 || x2<0 || y2<0 || x1>=wm || y1>=hm || x2>=wm || y2>=hm){
fprintf(stderr, "Error while reading matches %d %d -> %d %d, out of bounds\n", x1, y1, x2, y2);
exit(1);
}
match_x->data[ y1*match_x->stride+x1 ] = (float) (x2-x1);
match_y->data[ y1*match_x->stride+x1 ] = (float) (y2-y1);
match_z->data[ y1*match_x->stride+x1 ] = score;
}
}else if ( !strcmp(argv[current_arg],"-sintel") ){
current_arg++;
optical_flow_params_sintel(params);
}else if ( !strcmp(argv[current_arg],"-middlebury") ){
current_arg++;
optical_flow_params_middlebury(params);
}else if ( !strcmp(argv[current_arg],"-kitti") ){
current_arg++;
optical_flow_params_kitti(params);
}else{
if(argv[current_arg][0] == '-'){
fprintf(stderr,"Unknow options %s\n",argv[current_arg]);
}else{
fprintf(stderr,"Error while reading options, %s\n",argv[current_arg]);
}
exit(1);
}
}
image_t *wx = image_new(im1->width,im1->height), *wy = image_new(im1->width,im1->height);
optical_flow(wx, wy, im1, im2, params, match_x, match_y, match_z);
writeFlowFile(argv[3], wx, wy);
image_delete(wx);
image_delete(wy);
image_delete(match_x); image_delete(match_y); image_delete(match_z);
color_image_delete(im1); color_image_delete(im2);
free(params);
return 0;
}
static void color_prefilt(color_image_t *src, int fc)
{
fftw_lock();
int i, j;
/* Log */
for(j = 0; j < src->height; j++)
{
for(i = 0; i < src->width; i++)
{
src->c1[j*src->width+i] = log(src->c1[j*src->width+i]+1.0f);
src->c2[j*src->width+i] = log(src->c2[j*src->width+i]+1.0f);
src->c3[j*src->width+i] = log(src->c3[j*src->width+i]+1.0f);
}
}
color_image_t *img_pad = color_image_add_padding(src, 5);
/* Get sizes */
int width = img_pad->width;
int height = img_pad->height;
/* Alloc memory */
float *fx = (float *) fftwf_malloc(width*height*sizeof(float));
float *fy = (float *) fftwf_malloc(width*height*sizeof(float));
float *gfc = (float *) fftwf_malloc(width*height*sizeof(float));
fftwf_complex *ina1 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *ina2 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *ina3 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *inb1 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *inb2 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *inb3 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *out1 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *out2 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
fftwf_complex *out3 = (fftwf_complex *) fftwf_malloc(width*height*sizeof(fftwf_complex));
/* Build whitening filter */
float s1 = fc/sqrt(log(2));
for(j = 0; j < height; j++)
{
for(i = 0; i < width; i++)
{
ina1[j*width + i][0] = img_pad->c1[j*width+i];
ina2[j*width + i][0] = img_pad->c2[j*width+i];
ina3[j*width + i][0] = img_pad->c3[j*width+i];
ina1[j*width + i][1] = 0.0f;
ina2[j*width + i][1] = 0.0f;
ina3[j*width + i][1] = 0.0f;
fx[j*width + i] = (float) i - width/2.0f;
fy[j*width + i] = (float) j - height/2.0f;
gfc[j*width + i] = exp(-(fx[j*width + i]*fx[j*width + i] + fy[j*width + i]*fy[j*width + i]) / (s1*s1));
}
}
fftshift(gfc, width, height);
/* FFT */
fftwf_plan fft11 = fftwf_plan_dft_2d(width, height, ina1, out1, FFTW_FORWARD, FFTW_ESTIMATE);
fftwf_plan fft12 = fftwf_plan_dft_2d(width, height, ina2, out2, FFTW_FORWARD, FFTW_ESTIMATE);
fftwf_plan fft13 = fftwf_plan_dft_2d(width, height, ina3, out3, FFTW_FORWARD, FFTW_ESTIMATE);
fftw_unlock();
fftwf_execute(fft11);
fftwf_execute(fft12);
fftwf_execute(fft13);
fftw_lock();
/* Apply whitening filter */
for(j = 0; j < height; j++)
{
for(i = 0; i < width; i++)
{
out1[j*width+i][0] *= gfc[j*width + i];
out2[j*width+i][0] *= gfc[j*width + i];
out3[j*width+i][0] *= gfc[j*width + i];
out1[j*width+i][1] *= gfc[j*width + i];
out2[j*width+i][1] *= gfc[j*width + i];
out3[j*width+i][1] *= gfc[j*width + i];
}
}
/* IFFT */
fftwf_plan ifft11 = fftwf_plan_dft_2d(width, height, out1, inb1, FFTW_BACKWARD, FFTW_ESTIMATE);
fftwf_plan ifft12 = fftwf_plan_dft_2d(width, height, out2, inb2, FFTW_BACKWARD, FFTW_ESTIMATE);
fftwf_plan ifft13 = fftwf_plan_dft_2d(width, height, out3, inb3, FFTW_BACKWARD, FFTW_ESTIMATE);
fftw_unlock();
fftwf_execute(ifft11);
fftwf_execute(ifft12);
fftwf_execute(ifft13);
fftw_lock();
/* Local contrast normalisation */
for(j = 0; j < height; j++)
{
for(i = 0; i < width; i++)
{
img_pad->c1[j*width+i] -= inb1[j*width+i][0] / (width*height);
img_pad->c2[j*width+i] -= inb2[j*width+i][0] / (width*height);
img_pad->c3[j*width+i] -= inb3[j*width+i][0] / (width*height);
float mean = (img_pad->c1[j*width+i] + img_pad->c2[j*width+i] + img_pad->c3[j*width+i])/3.0f;
ina1[j*width+i][0] = mean*mean;
ina1[j*width+i][1] = 0.0f;
}
}
/* FFT */
fftwf_plan fft21 = fftwf_plan_dft_2d(width, height, ina1, out1, FFTW_FORWARD, FFTW_ESTIMATE);
fftw_unlock();
fftwf_execute(fft21);
fftw_lock();
/* Apply contrast normalisation filter */
for(j = 0; j < height; j++)
{
for(i = 0; i < width; i++)
{
out1[j*width+i][0] *= gfc[j*width + i];
out1[j*width+i][1] *= gfc[j*width + i];
}
}
/* IFFT */
fftwf_plan ifft2 = fftwf_plan_dft_2d(width, height, out1, inb1, FFTW_BACKWARD, FFTW_ESTIMATE);
fftw_unlock();
fftwf_execute(ifft2);
fftw_lock();
/* Get result from contrast normalisation filter */
for(j = 0; j < height; j++)
{
for(i = 0; i < width; i++)
{
float val = sqrt(sqrt(inb1[j*width+i][0]*inb1[j*width+i][0]+inb1[j*width+i][1]*inb1[j*width+i][1]) / (width*height));
img_pad->c1[j*width+i] /= (0.2f+val);
img_pad->c2[j*width+i] /= (0.2f+val);
img_pad->c3[j*width+i] /= (0.2f+val);
}
}
color_image_rem_padding(src, img_pad, 5);
/* Free */
fftwf_destroy_plan(fft11);
fftwf_destroy_plan(fft12);
fftwf_destroy_plan(fft13);
fftwf_destroy_plan(ifft11);
fftwf_destroy_plan(ifft12);
fftwf_destroy_plan(ifft13);
fftwf_destroy_plan(fft21);
fftwf_destroy_plan(ifft2);
color_image_delete(img_pad);
fftwf_free(ina1);
fftwf_free(ina2);
fftwf_free(ina3);
fftwf_free(inb1);
fftwf_free(inb2);
fftwf_free(inb3);
fftwf_free(out1);
fftwf_free(out2);
fftwf_free(out3);
fftwf_free(fx);
fftwf_free(fy);
fftwf_free(gfc);
fftw_unlock();
}
int main(int argc, char **argv){
if( argc<6){
if(argc>1) fprintf(stderr,"Error, not enough arguments\n");
usage();
exit(1);
}
// read arguments
color_image_t *im1 = color_image_load(argv[1]);
color_image_t *im2 = color_image_load(argv[2]);
float_image edges = read_edges(argv[3], im1->width, im1->height);
float_image matches = read_matches(argv[4]);
const char *outputfile = argv[5];
// prepare variables
epic_params_t epic_params;
epic_params_default(&epic_params);
variational_params_t flow_params;
variational_params_default(&flow_params);
image_t *wx = image_new(im1->width, im1->height), *wy = image_new(im1->width, im1->height);
// read optional arguments
#define isarg(key) !strcmp(a,key)
int current_arg = 6;
while(current_arg < argc ){
const char* a = argv[current_arg++];
if( isarg("-h") || isarg("-help") )
usage();
else if( isarg("-nw") )
strcpy(epic_params.method, "NW");
else if( isarg("-p") || isarg("-prefnn") )
epic_params.pref_nn = atoi(argv[current_arg++]);
else if( isarg("-n") || isarg("-nn") )
epic_params.nn = atoi(argv[current_arg++]);
else if( isarg("-k") )
epic_params.coef_kernel = atof(argv[current_arg++]);
else if( isarg("-i") || isarg("-iter") )
flow_params.niter_outer = atoi(argv[current_arg++]);
else if( isarg("-a") || isarg("-alpha") )
flow_params.alpha= atof(argv[current_arg++]);
else if( isarg("-g") || isarg("-gamma") )
flow_params.gamma= atof(argv[current_arg++]);
else if( isarg("-d") || isarg("-delta") )
flow_params.delta= atof(argv[current_arg++]);
else if( isarg("-s") || isarg("-sigma") )
flow_params.sigma= atof(argv[current_arg++]);
else if( isarg("-sintel") ){
epic_params.pref_nn= 25;
epic_params.nn= 160;
epic_params.coef_kernel = 1.1f;
flow_params.niter_outer = 5;
flow_params.alpha = 1.0f;
flow_params.gamma = 0.72f;
flow_params.delta = 0.0f;
flow_params.sigma = 1.1f;
}
else if( isarg("-kitti") ){
epic_params.pref_nn= 25;
epic_params.nn= 160;
epic_params.coef_kernel = 1.1f;
flow_params.niter_outer = 2;
flow_params.alpha = 1.0f;
flow_params.gamma = 0.77f;
flow_params.delta = 0.0f;
flow_params.sigma = 1.7f;
}
else if( isarg("-middlebury") ){
epic_params.pref_nn= 15;
epic_params.nn= 65;
epic_params.coef_kernel = 0.2f;
flow_params.niter_outer = 25;
flow_params.alpha = 1.0f;
flow_params.gamma = 0.72f;
flow_params.delta = 0.0f;
flow_params.sigma = 1.1f;
}
else{
fprintf(stderr, "unknown argument %s", a);
usage();
exit(1);
}
}
// compute interpolation and energy minimization
color_image_t *imlab = rgb_to_lab(im1);
epic(wx, wy, imlab, &matches, &edges, &epic_params, 1);
// energy minimization
variational(wx, wy, im1, im2, &flow_params);
// write output file and free memory
writeFlowFile(outputfile, wx, wy);
color_image_delete(im1);
color_image_delete(imlab);
color_image_delete(im2);
free(matches.pixels);
free(edges.pixels);
image_delete(wx);
image_delete(wy);
return 0;
}
void mexFunction( int nl, mxArray *pl[], int nr, const mxArray *pr[] ) {
if( nr==0 ){
usage(MATLAB_OPTIONS);
return;
}
if ( nl != 1){
usage(MATLAB_OPTIONS);
mexErrMsgTxt("error: returns one output");
return;
}
if( nr < 2 || nr > 4){
usage(MATLAB_OPTIONS);
mexErrMsgTxt("error: takes two to four inputs");
return;
}
// The code is originally written for C-order arrays.
// We thus transpose all arrays in this mex-function which is not efficient...
const int *pDims;
if( mxGetNumberOfDimensions(pr[0]) != 3 ) mexErrMsgTxt("input images must have 3 dimensions");
if( !mxIsClass(pr[0], "single") ) mexErrMsgTxt("input images must be single");
pDims = mxGetDimensions(pr[0]);
if( pDims[2]!=3 ) mexErrMsgTxt("input images must have 3 channels");
const int h = pDims[0], w = pDims[1];
color_image_t *im1 = input3darray_to_color_image( pr[0] );
if( mxGetNumberOfDimensions(pr[1]) != 3 ) mexErrMsgTxt("input images must have 3 dimensions");
if( !mxIsClass(pr[1], "single") ) mexErrMsgTxt("input images must be single");
pDims = mxGetDimensions(pr[1]);
if( pDims[0]!=h || pDims[1]!=w || pDims[2]!=3) mexErrMsgTxt( "input images must have the same size" );
color_image_t *im2 = input3darray_to_color_image( pr[1] );
image_t *match_x = NULL, *match_y = NULL, *match_z = NULL;
if( nr>2 && !mxIsEmpty(pr[2]) ){
if( mxGetNumberOfDimensions(pr[2]) != 2 ) mexErrMsgTxt("input matches must be a 2d-matrix");
if( !mxIsClass(pr[2], "single")) mexErrMsgTxt("input matches must be single");
pDims = mxGetDimensions(pr[1]);
if( pDims[1]<4) mexErrMsgTxt( "input matches must have at least 4 columns: x1 y1 x2 y2" );
match_x = image_new(w, h); match_y = image_new(w, h); match_z = image_new(w, h);
input2darray_to_matches( match_x, match_y, match_z, pr[2]);
}
// set params to default
optical_flow_params_t* params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t));
if(!params){
fprintf(stderr,"error deepflow2(): not enough memory\n");
exit(1);
}
optical_flow_params_default(params);
// read options
if( nr > 3 ){
char *options = mxArrayToString(pr[3]);
if( !options ) mexErrMsgTxt("Fourth parameter must be a string");
int argc=0;
char* argv[256];
argv[argc]=strtok(options," ");
while(argv[argc]!=NULL)
{
argv[++argc]=strtok(NULL," ");
}
parse_options(params, argc, argv, MATLAB_OPTIONS, w, h);
}
image_t *wx = image_new(im1->width,im1->height), *wy = image_new(im1->width,im1->height);
optical_flow(wx, wy, im1, im2, params, match_x, match_y, match_z);
int dims[3] = {h,w,2};
pl[0] = mxCreateNumericArray(3, dims, mxSINGLE_CLASS, mxREAL);
flow_to_output3darray(wx, wy, pl[0]);
image_delete(wx);
image_delete(wy);
image_delete(match_x); image_delete(match_y); image_delete(match_z);
color_image_delete(im1); color_image_delete(im2);
free(params);
}
/* compute the saliency of a given image */
image_t* saliency(const color_image_t *im, float sigma_image, float sigma_matrix ){
int width = im->width, height = im->height, filter_size;
// smooth image
color_image_t *sim = color_image_new(width, height);
float *presmooth_filter = gaussian_filter(sigma_image, &filter_size);
convolution_t *presmoothing = convolution_new(filter_size, presmooth_filter, 1);
color_image_convolve_hv(sim, im, presmoothing, presmoothing);
convolution_delete(presmoothing);
free(presmooth_filter);
// compute derivatives
float deriv_filter[2] = {0.0f, -0.5f};
convolution_t *deriv = convolution_new(1, deriv_filter, 0);
color_image_t *imx = color_image_new(width, height), *imy = color_image_new(width, height);
color_image_convolve_hv(imx, sim, deriv, NULL);
color_image_convolve_hv(imy, sim, NULL, deriv);
convolution_delete(deriv);
// compute autocorrelation matrix
image_t *imxx = image_new(width, height), *imxy = image_new(width, height), *imyy = image_new(width, height);
v4sf *imx1p = (v4sf*) imx->c1, *imx2p = (v4sf*) imx->c2, *imx3p = (v4sf*) imx->c3, *imy1p = (v4sf*) imy->c1, *imy2p = (v4sf*) imy->c2, *imy3p = (v4sf*) imy->c3,
*imxxp = (v4sf*) imxx->data, *imxyp = (v4sf*) imxy->data, *imyyp = (v4sf*) imyy->data;
int i;
for(i = 0 ; i<height*im->stride/4 ; i++){
*imxxp = (*imx1p)*(*imx1p) + (*imx2p)*(*imx2p) + (*imx3p)*(*imx3p);
*imxyp = (*imx1p)*(*imy1p) + (*imx2p)*(*imy2p) + (*imx3p)*(*imy3p);
*imyyp = (*imy1p)*(*imy1p) + (*imy2p)*(*imy2p) + (*imy3p)*(*imy3p);
imxxp+=1; imxyp+=1; imyyp+=1;
imx1p+=1; imx2p+=1; imx3p+=1;
imy1p+=1; imy2p+=1; imy3p+=1;
}
// integrate autocorrelation matrix
float *smooth_filter = gaussian_filter(sigma_matrix, &filter_size);
convolution_t *smoothing = convolution_new(filter_size, smooth_filter, 1);
image_t *tmp = image_new(width, height);
convolve_horiz(tmp, imxx, smoothing);
convolve_vert(imxx, tmp, smoothing);
convolve_horiz(tmp, imxy, smoothing);
convolve_vert(imxy, tmp, smoothing);
convolve_horiz(tmp, imyy, smoothing);
convolve_vert(imyy, tmp, smoothing);
convolution_delete(smoothing);
free(smooth_filter);
// compute smallest eigenvalue
v4sf vzeros = {0.0f,0.0f,0.0f,0.0f};
v4sf vhalf = {0.5f,0.5f,0.5f,0.5f};
v4sf *tmpp = (v4sf*) tmp->data;
imxxp = (v4sf*) imxx->data; imxyp = (v4sf*) imxy->data; imyyp = (v4sf*) imyy->data;
for(i = 0 ; i<height*im->stride/4 ; i++){
(*tmpp) = vhalf*( (*imxxp)+(*imyyp) ) ;
(*tmpp) = __builtin_ia32_sqrtps(__builtin_ia32_maxps(vzeros, (*tmpp) - __builtin_ia32_sqrtps(__builtin_ia32_maxps(vzeros, (*tmpp)*(*tmpp) + (*imxyp)*(*imxyp) - (*imxxp)*(*imyyp) ) )));
tmpp+=1; imxyp+=1; imxxp+=1; imyyp+=1;
}
image_delete(imxx); image_delete(imxy); image_delete(imyy);
color_image_delete(imx); color_image_delete(imy);
color_image_delete(sim);
return tmp;
}
