static void
_vl_dsift_alloc_buffers (VlDsiftFilter* self)
{
_vl_dsift_update_buffers (self) ;
{
int numFrameAlloc = vl_dsift_get_keypoint_num (self) ;
int numBinAlloc = vl_dsift_get_descriptor_size (self) ;
int numGradAlloc = self->geom.numBinT ;
/* see if we need to update the buffers */
if (numBinAlloc != self->numBinAlloc ||
numGradAlloc != self->numGradAlloc ||
numFrameAlloc != self->numFrameAlloc) {
int t ;
_vl_dsift_free_buffers(self) ;
self->frames = vl_malloc(sizeof(VlDsiftKeypoint) * numFrameAlloc) ;
self->descrs = vl_malloc(sizeof(float) * numBinAlloc * numFrameAlloc) ;
self->grads = vl_malloc(sizeof(float*) * numGradAlloc) ;
for (t = 0 ; t < numGradAlloc ; ++t) {
self->grads[t] =
vl_malloc(sizeof(float) * self->imWidth * self->imHeight) ;
}
self->numBinAlloc = numBinAlloc ;
self->numGradAlloc = numGradAlloc ;
self->numFrameAlloc = numFrameAlloc ;
}
}
}
VL_INLINE
void _vl_dsift_with_flat_window (VlDsiftFilter* self)
{
int binx, biny, bint ;
int framex, framey ;
/* for each orientation bin */
for (bint = 0 ; bint < self->geom.numBinT ; ++bint) {
vl_imconvcoltri_vf (self->convTmp1, self->imHeight,
self->grads [bint], self->imWidth, self->imHeight,
self->imWidth,
self->geom.binSizeY - 1, /* filt size */
1, /* subsampling step */
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
vl_imconvcoltri_vf (self->convTmp2, self->imWidth,
self->convTmp1, self->imHeight, self->imWidth,
self->imHeight,
self->geom.binSizeX - 1,
1,
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
for (biny = 0 ; biny < self->geom.numBinY ; ++biny) {
for (binx = 0 ; binx < self->geom.numBinX ; ++binx) {
float *dst = self->descrs
+ bint
+ binx * self->geom.numBinT
+ biny * (self->geom.numBinX * self->geom.numBinT) ;
float *src = self->convTmp2 ;
int frameSizeX = self->geom.binSizeX * (self->geom.numBinX - 1) + 1 ;
int frameSizeY = self->geom.binSizeY * (self->geom.numBinY - 1) + 1 ;
int descrSize = vl_dsift_get_descriptor_size (self) ;
for (framey = self->boundMinY ;
framey <= self->boundMaxY - frameSizeY + 1 ;
framey += self->stepY) {
for (framex = self->boundMinX ;
framex <= self->boundMaxX - frameSizeX + 1 ;
framex += self->stepX) {
*dst = src [(framex + binx * self->geom.binSizeX) * 1 +
(framey + biny * self->geom.binSizeY) * self->imWidth] ;
dst += descrSize ;
} /* framex */
} /* framey */
} /* binx */
} /* biny */
} /* bint */
}
void vl_dsift_process (VlDsiftFilter* self, float const* im)
{
int t, x, y ;
/* update buffers */
_vl_dsift_alloc_buffers (self) ;
/* clear integral images */
for (t = 0 ; t < self->geom.numBinT ; ++t)
memset (self->grads[t], 0,
sizeof(float) * self->imWidth * self->imHeight) ;
#undef at
#define at(x,y) (im[(y)*self->imWidth+(x)])
/* Compute gradients, their norm, and their angle */
for (y = 0 ; y < self->imHeight ; ++ y) {
for (x = 0 ; x < self->imWidth ; ++ x) {
float gx, gy ;
float angle, mod, nt, rbint ;
int bint ;
/* y derivative */
if (y == 0) {
gy = at(x,y+1) - at(x,y) ;
} else if (y == self->imHeight - 1) {
gy = at(x,y) - at(x,y-1) ;
} else {
gy = 0.5F * (at(x,y+1) - at(x,y-1)) ;
}
/* x derivative */
if (x == 0) {
gx = at(x+1,y) - at(x,y) ;
} else if (x == self->imWidth - 1) {
gx = at(x,y) - at(x-1,y) ;
} else {
gx = 0.5F * (at(x+1,y) - at(x-1,y)) ;
}
/* angle and modulus */
angle = vl_fast_atan2_f (gy,gx) ;
mod = vl_fast_sqrt_f (gx*gx + gy*gy) ;
/* quantize angle */
nt = vl_mod_2pi_f (angle) * (self->geom.numBinT / (2*VL_PI)) ;
bint = (int) vl_floor_f (nt) ;
rbint = nt - bint ;
/* write it back */
self->grads [(bint ) % self->geom.numBinT][x + y * self->imWidth] = (1 - rbint) * mod ;
self->grads [(bint + 1) % self->geom.numBinT][x + y * self->imWidth] = ( rbint) * mod ;
}
}
if (self->useFlatWindow) {
_vl_dsift_with_flat_window(self) ;
} else {
_vl_dsift_with_gaussian_window(self) ;
}
{
VlDsiftKeypoint* frameIter = self->frames ;
float * descrIter = self->descrs ;
int framex, framey, bint ;
int frameSizeX = self->geom.binSizeX * (self->geom.numBinX - 1) + 1 ;
int frameSizeY = self->geom.binSizeY * (self->geom.numBinY - 1) + 1 ;
int descrSize = vl_dsift_get_descriptor_size (self) ;
float deltaCenterX = 0.5F * self->geom.binSizeX * (self->geom.numBinX - 1) ;
float deltaCenterY = 0.5F * self->geom.binSizeY * (self->geom.numBinY - 1) ;
float normConstant = frameSizeX * frameSizeY ;
for (framey = self->boundMinY ;
framey <= self->boundMaxY - frameSizeY + 1 ;
framey += self->stepY) {
for (framex = self->boundMinX ;
framex <= self->boundMaxX - frameSizeX + 1 ;
framex += self->stepX) {
frameIter->x = framex + deltaCenterX ;
frameIter->y = framey + deltaCenterY ;
/* mass */
{
float mass = 0 ;
for (bint = 0 ; bint < descrSize ; ++ bint)
mass += descrIter[bint] ;
mass /= normConstant ;
frameIter->norm = mass ;
}
/* L2 normalize */
_vl_dsift_normalize_histogram (descrIter, descrIter + descrSize) ;
/* clamp */
for(bint = 0 ; bint < descrSize ; ++ bint)
if (descrIter[bint] > 0.2F) descrIter[bint] = 0.2F ;
/* L2 normalize */
_vl_dsift_normalize_histogram (descrIter, descrIter + descrSize) ;
frameIter ++ ;
descrIter += descrSize ;
} /* for framex */
} /* for framey */
}
}
VL_INLINE void
_vl_dsift_with_flat_window (VlDsiftFilter* self)
{
int binx, biny, bint ;
int framex, framey ;
/* for each orientation bin */
for (bint = 0 ; bint < self->geom.numBinT ; ++bint) {
vl_imconvcoltri_f (self->convTmp1, self->imHeight,
self->grads [bint], self->imWidth, self->imHeight,
self->imWidth,
self->geom.binSizeY, /* filt size */
1, /* subsampling step */
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
vl_imconvcoltri_f (self->convTmp2, self->imWidth,
self->convTmp1, self->imHeight, self->imWidth,
self->imHeight,
self->geom.binSizeX,
1,
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
for (biny = 0 ; biny < self->geom.numBinY ; ++biny) {
/*
This fast version of DSIFT does not use a proper Gaussian
weighting scheme for the gradiens that are accumulated on the
spatial bins. Instead each spatial bins is accumulated based on
the triangular kernel only, equivalent to bilinear interpolation
plus a flat, rather than Gaussian, window. Eventually, however,
the magnitude of the spatial bins in the SIFT descriptor is
reweighted by the average of the Gaussian window on each bin.
*/
float wy = _vl_dsift_get_bin_window_mean
(self->geom.binSizeY, self->geom.numBinY, biny,
self->windowSize) ;
/* The convolution functions vl_imconvcoltri_* convolve by a
* triangular kernel with unit integral. Instead for SIFT the
* triangular kernel should have unit height. This is
* compensated for by multiplying by the bin size:
*/
wy *= self->geom.binSizeY ;
for (binx = 0 ; binx < self->geom.numBinX ; ++binx) {
float w ;
float wx = _vl_dsift_get_bin_window_mean (self->geom.binSizeX,
self->geom.numBinX,
binx,
self->windowSize) ;
float *dst = self->descrs
+ bint
+ binx * self->geom.numBinT
+ biny * (self->geom.numBinX * self->geom.numBinT) ;
float *src = self->convTmp2 ;
int frameSizeX = self->geom.binSizeX * (self->geom.numBinX - 1) + 1 ;
int frameSizeY = self->geom.binSizeY * (self->geom.numBinY - 1) + 1 ;
int descrSize = vl_dsift_get_descriptor_size (self) ;
wx *= self->geom.binSizeX ;
w = wx * wy ;
for (framey = self->boundMinY ;
framey <= self->boundMaxY - frameSizeY + 1 ;
framey += self->stepY) {
for (framex = self->boundMinX ;
framex <= self->boundMaxX - frameSizeX + 1 ;
framex += self->stepX) {
*dst = w * src [(framex + binx * self->geom.binSizeX) * 1 +
(framey + biny * self->geom.binSizeY) * self->imWidth] ;
dst += descrSize ;
} /* framex */
} /* framey */
} /* binx */
} /* biny */
} /* bint */
}
VL_INLINE void
_vl_dsift_with_gaussian_window (VlDsiftFilter * self)
{
int binx, biny, bint ;
int framex, framey ;
float *xker, *yker ;
int Wx = self->geom.binSizeX - 1 ;
int Wy = self->geom.binSizeY - 1 ;
for (biny = 0 ; biny < self->geom.numBinY ; ++biny) {
yker = _vl_dsift_new_kernel (self->geom.binSizeY,
self->geom.numBinY,
biny,
self->windowSize) ;
for (binx = 0 ; binx < self->geom.numBinX ; ++binx) {
xker = _vl_dsift_new_kernel(self->geom.binSizeX,
self->geom.numBinX,
binx,
self->windowSize) ;
for (bint = 0 ; bint < self->geom.numBinT ; ++bint) {
vl_imconvcol_vf (self->convTmp1, self->imHeight,
self->grads[bint], self->imWidth, self->imHeight,
self->imWidth,
yker, -Wy, +Wy, 1,
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
vl_imconvcol_vf (self->convTmp2, self->imWidth,
self->convTmp1, self->imHeight, self->imWidth,
self->imHeight,
xker, -Wx, +Wx, 1,
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
{
float *dst = self->descrs
+ bint
+ binx * self->geom.numBinT
+ biny * (self->geom.numBinX * self->geom.numBinT) ;
float *src = self->convTmp2 ;
int frameSizeX = self->geom.binSizeX * (self->geom.numBinX - 1) + 1 ;
int frameSizeY = self->geom.binSizeY * (self->geom.numBinY - 1) + 1 ;
int descrSize = vl_dsift_get_descriptor_size (self) ;
for (framey = self->boundMinY ;
framey <= self->boundMaxY - frameSizeY + 1 ;
framey += self->stepY) {
for (framex = self->boundMinX ;
framex <= self->boundMaxX - frameSizeX + 1 ;
framex += self->stepX) {
*dst = src [(framex + binx * self->geom.binSizeX) * 1 +
(framey + biny * self->geom.binSizeY) * self->imWidth] ;
dst += descrSize ;
} /* framex */
} /* framey */
}
} /* for bint */
vl_free (xker) ;
} /* for binx */
vl_free (yker) ;
} /* for biny */
}
void DSift::Extract(const float* gray_img, const uint32_t width,
const uint32_t height, vector<VlDsiftKeypoint>* frames,
vector<float>* descrs, uint32_t* dim)
{
if (frames != NULL)
frames->clear();
if (descrs == NULL || dim == NULL)
{
cerr << "NULL pointer for descriptors and dim" << endl;
exit(-1);
}
descrs->clear();
*dim = 0;
const uint32_t max_sz = *(std::max_element(sizes_.begin(), sizes_.end()));
const uint32_t len_img = width * height;
if (!has_setup_)
SetUp(width, height);
else
{
if (width != width_ || height != height_)
{
cerr << "ERROR: Image size not matching!" << endl;
exit(-1);
}
}
for (size_t i = 0; i < sizes_.size(); ++i)
{
const uint32_t sz = sizes_[i];
const int off = std::floor(1.5 * (max_sz - sz));
vl_dsift_set_bounds(dsift_model_, bound_minx_ + std::max(0, off),
bound_miny_ + std::max(0, off),
std::min((int) width - 1, bound_maxx_),
std::min((int) height - 1, bound_maxy_));
VlDsiftDescriptorGeometry geom;
geom.numBinX = DEFAULT_NUM_BIN_X;
geom.numBinY = DEFAULT_NUM_BIN_Y;
geom.numBinT = DEFAULT_NUM_BIN_T;
geom.binSizeX = sz;
geom.binSizeY = sz;
vl_dsift_set_geometry(dsift_model_, &geom);
/*
{
int stepX;
int stepY;
int minX;
int minY;
int maxX;
int maxY;
vl_bool useFlatWindow;
int numFrames = vl_dsift_get_keypoint_num(dsift_model_);
int descrSize = vl_dsift_get_descriptor_size(dsift_model_);
VlDsiftDescriptorGeometry g = *vl_dsift_get_geometry(dsift_model_);
vl_dsift_get_steps(dsift_model_, &stepY, &stepX);
vl_dsift_get_bounds(dsift_model_, &minY, &minX, &maxY, &maxX);
useFlatWindow = vl_dsift_get_flat_window(dsift_model_);
printf(
"vl_dsift: bounds: [minX,minY,maxX,maxY] = [%d, %d, %d, %d]\n",
minX + 1, minY + 1, maxX + 1, maxY + 1);
printf("vl_dsift: subsampling steps: stepX=%d, stepY=%d\n", stepX,
stepY);
printf(
"vl_dsift: num bins: [numBinT, numBinX, numBinY] = [%d, %d, %d]\n",
g.numBinT, g.numBinX, g.numBinY);
printf("vl_dsift: descriptor size: %d\n", descrSize);
printf("vl_dsift: bin sizes: [binSizeX, binSizeY] = [%d, %d]\n",
g.binSizeX, g.binSizeY);
printf("vl_dsift: flat window: %s\n", VL_YESNO(useFlatWindow));
printf("vl_dsift: window size: %g\n",
vl_dsift_get_window_size(dsift_model_));
printf("vl_dsift: num of features: %d\n", numFrames);
}
*/
const float sigma = 1.0 * sz / magnif_;
float* smooth_img = (float*) malloc(sizeof(float) * len_img);
memset(smooth_img, 0, sizeof(float) * len_img);
vl_imsmooth_f(smooth_img, width, gray_img, width, height, width, sigma,
sigma);
vl_dsift_process(dsift_model_, smooth_img);
free(smooth_img);
const int num_key_pts = vl_dsift_get_keypoint_num(dsift_model_);
const VlDsiftKeypoint* key_points = vl_dsift_get_keypoints(dsift_model_);
*dim = vl_dsift_get_descriptor_size(dsift_model_);
const float* features = vl_dsift_get_descriptors(dsift_model_);
float* f = (float*) malloc(sizeof(float) * num_key_pts * (*dim));
cblas_scopy(num_key_pts * (*dim), features, 1, f, 1);
cblas_sscal(num_key_pts * (*dim), 512.0f, f, 1);
for (uint32_t d = 0; d < num_key_pts; ++d)
{
const float norm = (key_points + d)->norm;
if (norm < contr_thrd_)
{
// remove low contrast
memset(f + d * (*dim), 0, sizeof(float) * (*dim));
}
else
{
for (uint32_t j = d * (*dim); j < (d + 1) * (*dim); ++j)
{
float tmp = VL_MIN(f[j], 255.0);
if (!float_desc_)
tmp = (int) tmp;
f[j] = tmp;
}
}
}
if (i == 0)
{
if (frames != NULL)
frames->reserve(num_key_pts * sizes_.size());
descrs->reserve(num_key_pts * sizes_.size() * (*dim));
}
if (frames != NULL)
frames->insert(frames->end(), key_points, key_points + num_key_pts);
descrs->insert(descrs->end(), f, f + num_key_pts * (*dim));
free(f);
}
}
VL_INLINE
void _vl_dsift_with_flat_window (VlDsiftFilter* self)
{
int binx, biny, bint ;
int framex, framey ;
/* for each orientation bin */
for (bint = 0 ; bint < self->geom.numBinT ; ++bint) {
vl_imconvcoltri_vf (self->convTmp1, self->imHeight,
self->grads [bint], self->imWidth, self->imHeight,
self->imWidth,
self->geom.binSizeY, /* filt size */
1, /* subsampling step */
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
vl_imconvcoltri_vf (self->convTmp2, self->imWidth,
self->convTmp1, self->imHeight, self->imWidth,
self->imHeight,
self->geom.binSizeX,
1,
VL_PAD_BY_CONTINUITY|VL_TRANSPOSE) ;
for (biny = 0 ; biny < self->geom.numBinY ; ++biny) {
float wy = _vl_dsift_get_bin_window_mean (self->geom.binSizeY,
self->geom.numBinY,
biny,
self->windowSize) ;
/* The convolution function uses triangualr wave with unit integral (hence height
* equal to half the bin size). Instead
* for SIFT the triangular wave is used as a partition function and must
* have unit maximum. We compoensate for this by multiplying the result
* by the bin size.
*/
wy *= self->geom.binSizeY ;
for (binx = 0 ; binx < self->geom.numBinX ; ++binx) {
float w ;
float wx = _vl_dsift_get_bin_window_mean (self->geom.binSizeX,
self->geom.numBinX,
binx,
self->windowSize) ;
float *dst = self->descrs
+ bint
+ binx * self->geom.numBinT
+ biny * (self->geom.numBinX * self->geom.numBinT) ;
float *src = self->convTmp2 ;
int frameSizeX = self->geom.binSizeX * (self->geom.numBinX - 1) + 1 ;
int frameSizeY = self->geom.binSizeY * (self->geom.numBinY - 1) + 1 ;
int descrSize = vl_dsift_get_descriptor_size (self) ;
wx *= self->geom.binSizeX ;
w = wx * wy ;
for (framey = self->boundMinY ;
framey <= self->boundMaxY - frameSizeY + 1 ;
framey += self->stepY) {
for (framex = self->boundMinX ;
framex <= self->boundMaxX - frameSizeX + 1 ;
framex += self->stepX) {
*dst = w * src [(framex + binx * self->geom.binSizeX) * 1 +
(framey + biny * self->geom.binSizeY) * self->imWidth] ;
dst += descrSize ;
} /* framex */
} /* framey */
} /* binx */
} /* biny */
} /* bint */
}
void
mexFunction(int nout, mxArray *out[],
int nin, const mxArray *in[])
{
enum {IN_I=0, IN_END} ;
enum {OUT_FRAMES=0, OUT_DESCRIPTORS} ;
int verbose = 0 ;
int opt ;
int next = IN_END ;
mxArray const *optarg ;
float const *data ;
int M, N ;
int step [2] = {1,1} ;
vl_bool norm = 0 ;
vl_bool floatDescriptors = VL_FALSE ;
vl_bool useFlatWindow = VL_FALSE ;
double windowSize = -1.0 ;
double *bounds = NULL ;
double boundBuffer [4] ;
VlDsiftDescriptorGeometry geom ;
VL_USE_MATLAB_ENV ;
geom.numBinX = 4 ;
geom.numBinY = 4 ;
geom.numBinT = 8 ;
geom.binSizeX = 3 ;
geom.binSizeY = 3 ;
/* -----------------------------------------------------------------
* Check the arguments
* -------------------------------------------------------------- */
if (nin < 1) {
vlmxError(vlmxErrNotEnoughInputArguments, NULL) ;
} else if (nout > 2) {
vlmxError(vlmxErrTooManyOutputArguments, NULL) ;
}
if (mxGetNumberOfDimensions (in[IN_I]) != 2 ||
mxGetClassID (in[IN_I]) != mxSINGLE_CLASS ) {
vlmxError(vlmxErrInvalidArgument,
"I must be a matrix of class SINGLE.") ;
}
data = (float*) mxGetData (in[IN_I]) ;
M = mxGetM (in[IN_I]) ;
N = mxGetN (in[IN_I]) ;
while ((opt = vlmxNextOption (in, nin, options, &next, &optarg)) >= 0) {
switch (opt) {
case opt_verbose :
++ verbose ;
break ;
case opt_fast :
useFlatWindow = 1 ;
break ;
case opt_norm :
norm = 1 ;
break ;
case opt_bounds :
if (!vlmxIsPlainVector(optarg, 4)) {
mexErrMsgTxt("BOUNDS must be a 4-dimensional vector.") ;
}
bounds = boundBuffer ;
bounds [0] = mxGetPr(optarg)[0] - 1 ;
bounds [1] = mxGetPr(optarg)[1] - 1 ;
bounds [2] = mxGetPr(optarg)[2] - 1 ;
bounds [3] = mxGetPr(optarg)[3] - 1 ;
break ;
case opt_size :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"SIZE is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
geom.binSizeX = (int) mxGetPr(optarg)[0] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
geom.binSizeX = (int) mxGetPr(optarg)[1] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"SIZE is neither a scalar or a 2D vector.") ;
}
if (geom.binSizeX < 1 || geom.binSizeY < 1) {
vlmxError(vlmxErrInvalidArgument,"SIZE value is invalid.") ;
}
break ;
case opt_step :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"STEP is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
step[0] = (int) mxGetPr(optarg)[0] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
step[0] = (int) mxGetPr(optarg)[1] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"STEP is neither a scalar or a 2D vector.") ;
}
if (step[0] < 1 || step[1] < 1) {
vlmxError(vlmxErrInvalidArgument,"STEP value is invalid.") ;
}
break ;
case opt_window_size :
if (!vlmxIsPlainScalar(optarg) || (windowSize = *mxGetPr(optarg)) < 0) {
vlmxError(vlmxErrInvalidArgument,"WINDOWSIZE is not a scalar or it is negative.") ;
}
break ;
case opt_float_descriptors :
floatDescriptors = VL_TRUE ;
break ;
case opt_geometry :
if (!vlmxIsPlainVector(optarg,3)) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY is not a 3D vector.") ;
}
geom.numBinY = (int)mxGetPr(optarg)[0] ;
geom.numBinX = (int)mxGetPr(optarg)[1] ;
geom.numBinT = (int)mxGetPr(optarg)[2] ;
if (geom.numBinX < 1 ||
geom.numBinY < 1 ||
geom.numBinT < 1) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY value is invalid.") ;
}
break ;
default :
abort() ;
}
}
/* -----------------------------------------------------------------
* Do job
* -------------------------------------------------------------- */
{
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
int k, i ;
VlDsiftFilter *dsift ;
/* note that the image received from MATLAB is transposed */
dsift = vl_dsift_new (M, N) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, step[0], step[1]) ;
if (bounds) {
vl_dsift_set_bounds(dsift,
VL_MAX(bounds[1], 0),
VL_MAX(bounds[0], 0),
VL_MIN(bounds[3], M - 1),
VL_MIN(bounds[2], N - 1));
}
vl_dsift_set_flat_window(dsift, useFlatWindow) ;
if (windowSize >= 0) {
vl_dsift_set_window_size(dsift, windowSize) ;
}
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
if (verbose) {
int stepX ;
int stepY ;
int minX ;
int minY ;
int maxX ;
int maxY ;
vl_bool useFlatWindow ;
vl_dsift_get_steps (dsift, &stepY, &stepX) ;
vl_dsift_get_bounds (dsift, &minY, &minX, &maxY, &maxX) ;
useFlatWindow = vl_dsift_get_flat_window(dsift) ;
mexPrintf("vl_dsift: image size [W, H] = [%d, %d]\n", N, M) ;
mexPrintf("vl_dsift: bounds: [minX,minY,maxX,maxY] = [%d, %d, %d, %d]\n",
minX+1, minY+1, maxX+1, maxY+1) ;
mexPrintf("vl_dsift: subsampling steps: stepX=%d, stepY=%d\n", stepX, stepY) ;
mexPrintf("vl_dsift: num bins: [numBinT, numBinX, numBinY] = [%d, %d, %d]\n",
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
mexPrintf("vl_dsift: descriptor size: %d\n", descrSize) ;
mexPrintf("vl_dsift: bin sizes: [binSizeX, binSizeY] = [%d, %d]\n",
geom.binSizeX,
geom.binSizeY) ;
mexPrintf("vl_dsift: flat window: %s\n", VL_YESNO(useFlatWindow)) ;
mexPrintf("vl_dsift: window size: %g\n", vl_dsift_get_window_size(dsift)) ;
mexPrintf("vl_dsift: num of features: %d\n", numFrames) ;
}
vl_dsift_process (dsift, data) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
/* ---------------------------------------------------------------
* Create output arrays
* ------------------------------------------------------------ */
{
mwSize dims [2] ;
dims [0] = descrSize ;
dims [1] = numFrames ;
if (floatDescriptors) {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxSINGLE_CLASS, mxREAL) ;
} else {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxUINT8_CLASS, mxREAL) ;
}
dims [0] = norm ? 3 : 2 ;
out[OUT_FRAMES] = mxCreateNumericArray
(2, dims, mxDOUBLE_CLASS, mxREAL) ;
}
/* ---------------------------------------------------------------
* Copy back
* ------------------------------------------------------------ */
{
float *tmpDescr = mxMalloc(sizeof(float) * descrSize) ;
double *outFrameIter = mxGetPr(out[OUT_FRAMES]) ;
void *outDescrIter = mxGetData(out[OUT_DESCRIPTORS]) ;
for (k = 0 ; k < numFrames ; ++k) {
*outFrameIter++ = frames[k].y + 1 ;
*outFrameIter++ = frames[k].x + 1 ;
/* We have an implied / 2 in the norm, because of the clipping
below */
if (norm)
*outFrameIter++ = frames [k].norm ;
vl_dsift_transpose_descriptor (tmpDescr,
descrs + descrSize * k,
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
if (floatDescriptors) {
for (i = 0 ; i < descrSize ; ++i) {
float * pt = (float*) outDescrIter ;
*pt++ = VL_MIN(512.0F * tmpDescr[i], 255.0F) ;
outDescrIter = pt ;
}
} else {
for (i = 0 ; i < descrSize ; ++i) {
vl_uint8 * pt = (vl_uint8*) outDescrIter ;
*pt++ = (vl_uint8) (VL_MIN(512.0F * tmpDescr[i], 255.0F)) ;
outDescrIter = pt ;
}
}
}
mxFree(tmpDescr) ;
}
vl_dsift_delete (dsift) ;
}
}
/** ------------------------------------------------------------------
** @brief Python entry point
**/
PyObject * vl_dsift_python(
PyArrayObject & pyArray,
int opt_step,
PyArrayObject & opt_bounds,
int opt_size,
bool opt_fast,
bool opt_verbose,
bool opt_norm)
{
// check data type
assert(pyArray.descr->type_num == PyArray_FLOAT);
assert(pyArray.flags & NPY_FORTRAN);
assert(opt_bounds.descr->type_num == PyArray_FLOAT);
int verbose = 0;
int opt;
float const *data;
int M, N;
int step = 1;
int size = 3;
vl_bool norm = 0;
vl_bool useFlatWindow = VL_FALSE;
double *bounds = NULL;
double boundBuffer[4];
/* -----------------------------------------------------------------
* Check the arguments
* -------------------------------------------------------------- */
data = (float*) pyArray.data;
M = pyArray.dimensions[0];
N = pyArray.dimensions[1];
if (opt_verbose)
++verbose;
if (opt_fast)
useFlatWindow = 1;
if (opt_norm)
norm = 1;
if (opt_bounds.nd == 1 && opt_bounds.dimensions[0] == 4) {
double * tmp = (double *) opt_bounds.data;
bounds = boundBuffer;
for (int i = 0; i < 4; i++)
bounds[i] = tmp[i];
}
if (opt_size >= 0)
size = opt_size;
if (opt_step >= 0)
step = opt_step;
// create PyTuple for outputs
PyObject * tuple = PyTuple_New(2);
/* -----------------------------------------------------------------
* Do job
* -------------------------------------------------------------- */
{
int numFrames;
int descrSize;
VlDsiftKeypoint const *frames;
VlDsiftDescriptorGeometry const *geom;
float const *descrs;
int k, i;
VlDsiftFilter *dsift;
dsift = vl_dsift_new_basic(M, N, step, size);
if (bounds) {
vl_dsift_set_bounds(dsift, VL_MAX(bounds[0], 0), VL_MAX(
bounds[1], 0), VL_MIN(bounds[2], M - 1), VL_MIN(bounds[3], N
- 1));
}
vl_dsift_set_flat_window(dsift, useFlatWindow);
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = vl_dsift_get_geometry(dsift);
if (verbose) {
int stepX;
int stepY;
int minX;
int minY;
int maxX;
int maxY;
vl_bool useFlatWindow;
vl_dsift_get_steps(dsift, &stepX, &stepY);
vl_dsift_get_bounds(dsift, &minX, &minY, &maxX, &maxY);
useFlatWindow = vl_dsift_get_flat_window(dsift);
printf("dsift: image size: %d x %d\n", N, M);
printf(
" bounds: [%d, %d, %d, %d]\n", minY, minX,
maxY, maxX);
printf(" subsampling steps: %d, %d\n", stepY, stepX);
printf(
" num bins: [%d, %d, %d]\n", geom->numBinT,
geom->numBinX, geom->numBinY);
printf(" descriptor size: %d\n", descrSize);
printf(
" bin sizes: [%d, %d]\n", geom->binSizeX,
geom->binSizeY);
printf(" flat window: %s\n", VL_YESNO(useFlatWindow));
printf(" number of frames: %d\n", numFrames);
}
vl_dsift_process(dsift, data);
frames = vl_dsift_get_keypoints(dsift);
descrs = vl_dsift_get_descriptors(dsift);
/* ---------------------------------------------------------------
* Create output arrays
* ------------------------------------------------------------ */
npy_intp dims[2];
dims[0] = descrSize;
dims[1] = numFrames;
// allocate PyArray objects
PyArrayObject * _descriptors = (PyArrayObject *) PyArray_NewFromDescr(
&PyArray_Type, PyArray_DescrFromType(PyArray_UINT8),
2, dims, NULL, NULL, NPY_F_CONTIGUOUS, NULL);
if (norm)
dims[0] = 3;
else
dims[0] = 2;
PyArrayObject * _frames = (PyArrayObject*) PyArray_NewFromDescr(
&PyArray_Type, PyArray_DescrFromType(PyArray_DOUBLE),
2, dims, NULL, NULL, NPY_F_CONTIGUOUS, NULL);
// put PyArray objects in PyTuple
PyTuple_SetItem(tuple, 0, PyArray_Return(_frames));
PyTuple_SetItem(tuple, 1, PyArray_Return(_descriptors));
/* ---------------------------------------------------------------
* Copy back
* ------------------------------------------------------------ */
{
float *tmpDescr = (float*) vl_malloc(sizeof(float) * descrSize);
double *outFrameIter = (double*) _frames->data;
vl_uint8 *outDescrIter = (vl_uint8 *) _descriptors->data;
for (k = 0; k < numFrames; ++k) {
*outFrameIter++ = frames[k].y;
*outFrameIter++ = frames[k].x;
/* We have an implied / 2 in the norm, because of the clipping
below */
if (norm)
*outFrameIter++ = frames[k].norm;
vl_dsift_transpose_descriptor(
tmpDescr, descrs + descrSize * k, geom->numBinT,
geom->numBinX, geom->numBinY);
for (i = 0; i < descrSize; ++i) {
*outDescrIter++ = (vl_uint8) (VL_MIN(
512.0F * tmpDescr[i], 255.0F));
}
}
vl_free(tmpDescr);
}
vl_dsift_delete(dsift);
}
return tuple;
}
vector<vector<float> > Encoder::extractMultiDSIFT(Mat normMat, Mat landmarks, int level){
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
int dimension = patchSize / cellSize;
vector<float> dsiftCode;
for (int l = 0; l < level; l++){
int tmpcellSize = cellSize - l;
int tmppatchSize = tmpcellSize*dimension;
for (unsigned int i = 0; i < landmarks.cols; i++){
if (landmarks.at<float>(0, i) > tmppatchSize/2 && landmarks.at<float>(1, i) > tmppatchSize/2 && landmarks.at<float>(0, i) + tmppatchSize/2 < normMat.cols && landmarks.at<float>(1, i) + tmppatchSize/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tmppatchSize/2 , landmarks.at<float>(1, i) - tmppatchSize/2, tmppatchSize, tmppatchSize));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}
vector<vector<float> > Encoder::extractTunedDSIFT(Mat normMat, Mat landmarks){
//eye, eyebrow, eye-band, nose-ver, mouth, left face, right face, jaw, forehead, nose-hor
//40x20x2, 40x20x2, 100x40, 40x60, 60x40, 40x60,40x60, 50x20, 100x40, 60x40
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
vector<float> dsiftCode;
for (int l = 0; l <= 2; l++){
for (int i = 0; i < landmarks.cols; i++){
switch (i){
case 0:
case 1:
case 2:
case 3:
tunedCols = 30 + 10*l;
tunedRows = 10 + 10*l;
break;
case 4:
tunedCols = 70 + 20*l;
tunedRows = 30 + 10*l;
break;
case 5:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 6:
tunedCols = 40 + 20*l;
tunedRows = 30 + 10*l;
break;
case 7:
case 8:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 9:
tunedCols = 40 + 10*l;
tunedRows = 20;
break;
case 10:
tunedCols = 60 + 20*l;
tunedRows = 30;
break;
case 11:
tunedCols = 40 + 10*l;
tunedRows = 20 + 10*l;
break;
default:
cout<<"Wrong landmark size: "<<i<<endl;
exit(1);
}
if (landmarks.at<float>(0, i) > tunedCols/2 && landmarks.at<float>(1, i) > tunedRows/2 && landmarks.at<float>(0, i) + tunedCols/2 < normMat.cols && landmarks.at<float>(1, i) + tunedRows/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tunedCols/2 , landmarks.at<float>(1, i) - tunedRows/2, tunedCols, tunedRows));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}
