Mat PyramidsVl::extractFeatures(const Mat &im, vector<KeyPoint> &keypoints, int step)
{
Q_UNUSED(step);
Mat features(0, 128, CV_32F);
double magnif = 6;
QList<int> sizes;
sizes << 4;
sizes << 6;
sizes << 8;
sizes << 10;
/* convert to float array */
assert(im.type() == CV_8U);
float *imdata = new float[im.rows * im.cols];
for (int i = 0; i < im.rows; i++)
for (int j = 0; j < im.cols; j++)
imdata[i * im.cols + j] = im.row(i).data[j];
float *smoothed = new float[im.rows * im.cols];
for (int i = 0; i < sizes.size(); i++) {
int step = sizes[i];
/* smoothing step */
double sigma = step / magnif;
vl_imsmooth_f(smoothed, im.cols, imdata, im.cols, im.rows, im.cols, sigma, sigma);
memcpy(smoothed, imdata, im.rows * im.cols * 4);
/* denset sift */
VlDsiftFilter *dsift = vl_dsift_new_basic(im.cols, im.rows, step, 8);
vl_dsift_process(dsift, smoothed);
int cnt = vl_dsift_get_keypoint_num(dsift);
const float *descs = vl_dsift_get_descriptors(dsift);
const VlDsiftKeypoint *kpts = vl_dsift_get_keypoints(dsift);
for (int i = 0; i < cnt; i++) {
Mat ft(1, 128, CV_32F);
for (int j = 0; j < 128; j++)
ft.at<float>(0, j) = qMin(descs[i * 128 + j] * 512, float(255));
features.push_back(ft);
KeyPoint kpt;
kpt.pt.x = kpts[i].x;
kpt.pt.y = kpts[i].y;
keypoints.push_back(kpt);
}
vl_dsift_delete(dsift);
}
delete [] imdata;
delete [] smoothed;
return features;
}
void DSift::Extract(const float* gray_img, const uint32_t width,
const uint32_t height, vector<VlDsiftKeypoint>* frames,
vector<float>* descrs, uint32_t* dim)
{
if (frames != NULL)
frames->clear();
if (descrs == NULL || dim == NULL)
{
cerr << "NULL pointer for descriptors and dim" << endl;
exit(-1);
}
descrs->clear();
*dim = 0;
const uint32_t max_sz = *(std::max_element(sizes_.begin(), sizes_.end()));
const uint32_t len_img = width * height;
if (!has_setup_)
SetUp(width, height);
else
{
if (width != width_ || height != height_)
{
cerr << "ERROR: Image size not matching!" << endl;
exit(-1);
}
}
for (size_t i = 0; i < sizes_.size(); ++i)
{
const uint32_t sz = sizes_[i];
const int off = std::floor(1.5 * (max_sz - sz));
vl_dsift_set_bounds(dsift_model_, bound_minx_ + std::max(0, off),
bound_miny_ + std::max(0, off),
std::min((int) width - 1, bound_maxx_),
std::min((int) height - 1, bound_maxy_));
VlDsiftDescriptorGeometry geom;
geom.numBinX = DEFAULT_NUM_BIN_X;
geom.numBinY = DEFAULT_NUM_BIN_Y;
geom.numBinT = DEFAULT_NUM_BIN_T;
geom.binSizeX = sz;
geom.binSizeY = sz;
vl_dsift_set_geometry(dsift_model_, &geom);
/*
{
int stepX;
int stepY;
int minX;
int minY;
int maxX;
int maxY;
vl_bool useFlatWindow;
int numFrames = vl_dsift_get_keypoint_num(dsift_model_);
int descrSize = vl_dsift_get_descriptor_size(dsift_model_);
VlDsiftDescriptorGeometry g = *vl_dsift_get_geometry(dsift_model_);
vl_dsift_get_steps(dsift_model_, &stepY, &stepX);
vl_dsift_get_bounds(dsift_model_, &minY, &minX, &maxY, &maxX);
useFlatWindow = vl_dsift_get_flat_window(dsift_model_);
printf(
"vl_dsift: bounds: [minX,minY,maxX,maxY] = [%d, %d, %d, %d]\n",
minX + 1, minY + 1, maxX + 1, maxY + 1);
printf("vl_dsift: subsampling steps: stepX=%d, stepY=%d\n", stepX,
stepY);
printf(
"vl_dsift: num bins: [numBinT, numBinX, numBinY] = [%d, %d, %d]\n",
g.numBinT, g.numBinX, g.numBinY);
printf("vl_dsift: descriptor size: %d\n", descrSize);
printf("vl_dsift: bin sizes: [binSizeX, binSizeY] = [%d, %d]\n",
g.binSizeX, g.binSizeY);
printf("vl_dsift: flat window: %s\n", VL_YESNO(useFlatWindow));
printf("vl_dsift: window size: %g\n",
vl_dsift_get_window_size(dsift_model_));
printf("vl_dsift: num of features: %d\n", numFrames);
}
*/
const float sigma = 1.0 * sz / magnif_;
float* smooth_img = (float*) malloc(sizeof(float) * len_img);
memset(smooth_img, 0, sizeof(float) * len_img);
vl_imsmooth_f(smooth_img, width, gray_img, width, height, width, sigma,
sigma);
vl_dsift_process(dsift_model_, smooth_img);
free(smooth_img);
const int num_key_pts = vl_dsift_get_keypoint_num(dsift_model_);
const VlDsiftKeypoint* key_points = vl_dsift_get_keypoints(dsift_model_);
*dim = vl_dsift_get_descriptor_size(dsift_model_);
const float* features = vl_dsift_get_descriptors(dsift_model_);
float* f = (float*) malloc(sizeof(float) * num_key_pts * (*dim));
cblas_scopy(num_key_pts * (*dim), features, 1, f, 1);
cblas_sscal(num_key_pts * (*dim), 512.0f, f, 1);
for (uint32_t d = 0; d < num_key_pts; ++d)
{
const float norm = (key_points + d)->norm;
if (norm < contr_thrd_)
{
// remove low contrast
memset(f + d * (*dim), 0, sizeof(float) * (*dim));
}
else
{
for (uint32_t j = d * (*dim); j < (d + 1) * (*dim); ++j)
{
float tmp = VL_MIN(f[j], 255.0);
if (!float_desc_)
tmp = (int) tmp;
f[j] = tmp;
}
}
}
if (i == 0)
{
if (frames != NULL)
frames->reserve(num_key_pts * sizes_.size());
descrs->reserve(num_key_pts * sizes_.size() * (*dim));
}
if (frames != NULL)
frames->insert(frames->end(), key_points, key_points + num_key_pts);
descrs->insert(descrs->end(), f, f + num_key_pts * (*dim));
free(f);
}
}
bool VLFeat::CalculateCommon(int f, bool all, int l) {
string msg = "VLFeat::CalculateCommon("+ToStr(f)+","+ToStr(all)+","+
ToStr(l)+") : ";
// if (!do_fisher && !do_vlad) {
// cerr << msg
// << "either encoding=fisher or encoding=vlad should be specified"
// << endl;
// return false;
// }
if (!gmm && !kmeans) {
cerr << msg << "either gmm=xxx or kmeans=xxx option should be given"
<< endl;
return false;
}
cox::tictac::func tt(tics, "VLFeat::CalculateCommon");
// obs! only some parameters here, should be in ProcessOptionsAndRemove()
// too, also scales and geometry should be made specifiable...
bool normalizeSift = false, renormalize = true, flat_window = true;
size_t step = 3, binsize = 8;
EnsureImage();
int width = Width(true), height = Height(true);
if (FrameVerbose())
cout << msg+"wxh="
<< width << "x" << height << "=" << width*height << endl;
vector<float> rgbcoeff { 0.2989, 0.5870, 0.1140 };
imagedata idata = CurrentFrame();
idata.convert(imagedata::pixeldata_float);
idata.force_one_channel(rgbcoeff);
vector<float> dsift;
size_t descr_size_orig = 0, descr_size_final = 0;
vector<float> scales { 1.0000, 0.7071, 0.5000, 0.3536, 0.2500 };
// vector<float> scales { 1.0000 };
for (size_t i=0; i<scales.size(); i++) {
if (KeyPointVerbose())
cout << "Starting vl_dsift_process() in scale " << scales[i] << endl;
imagedata simg = idata;
if (scales[i]!=1) {
scalinginfo si(simg.width(), simg.height(),
(int)floor(scales[i]*simg.width()+0.5),
(int)floor(scales[i]*simg.height()+0.5));
simg.rescale(si, 1);
}
// VlDsiftFilter *sf = vl_dsift_new(simg.width(), simg.height());
VlDsiftFilter *sf = vl_dsift_new_basic(simg.width(), simg.height(),
step, binsize);
// opts.scales = logspace(log10(1), log10(.25), 5) ;
// void vl_dsift_set_bounds ( VlDsiftFilter * self,
// int minX,
// int minY,
// int maxX,
// int maxY
// );
// VlDsiftDescriptorGeometry geom = { 8, 4, 4, 0, 0 };
// vl_dsift_set_geometry(sf, &geom);
//vl_dsift_set_steps(sf, 3, 3);
//vl_dsift_set_window_size(sf, 8);
vl_dsift_set_flat_window(sf, flat_window); // aka fast in matlab
vector<float> imgvec = simg.get_float();
const float *img_fp = &imgvec[0];
// cout << "IMAGE = " << img_fp[0] << " " << img_fp[1] << " "
// << img_fp[2] << " ... " << img_fp[41] << endl;
vl_dsift_process(sf, img_fp);
// if opts.rootSift // false
// descrs{si} = sqrt(descrs{si}) ;
// end
// if opts.normalizeSift //true
// descrs{si} = snorm(descrs{si}) ;
// end
descr_size_orig = sf->descrSize;
size_t nf = sf->numFrames;
const VlDsiftKeypoint *k = sf->frames;
float *d = sf->descrs;
if (KeyPointVerbose())
cout << " found " << sf->numFrames << " 'frames' in "
<< simg.info() << endl
<< " descriptor dim " << descr_size_orig << endl;
if (PixelVerbose())
for (size_t i=0; i<nf; i++) {
cout << " i=" << i << " x=" << k[i].x << " y=" << k[i].y
<< " s=" << k[i].s << " norm=" << k[i].norm;
if (FullVerbose()) {
cout << " RAW";
for (size_t j=0; j<descr_size_orig; j++)
cout << " " << d[i*descr_size_orig+j];
}
cout << endl;
}
if (normalizeSift) {
for (size_t i=0; i<nf; i++) {
if (PixelVerbose())
cout << " i=" << i << " x=" << k[i].x << " y=" << k[i].y
<< " s=" << k[i].s << " norm=" << k[i].norm;
double mul = 0.0;
for (size_t j=0; j<descr_size_orig; j++)
mul += d[i*descr_size_orig+j]*d[i*descr_size_orig+j];
if (mul)
mul = 1.0/sqrt(mul);
if (FullVerbose())
cout << " NORM";
for (size_t j=0; j<descr_size_orig; j++) {
d[i*descr_size_orig+j] *= mul;
if (FullVerbose())
cout << " " << d[i*descr_size_orig+j];
}
if (PixelVerbose())
cout << endl;
}
}
if (!pca.vector_length()) {
dsift.insert(dsift.end(), d, d+nf*descr_size_orig);
descr_size_final = descr_size_orig;
} else {
for (size_t i=0; i<nf; i++) {
vector<float> vin(d+i*descr_size_orig, d+(i+1)*descr_size_orig);
vector<float> vout = pca.projection_coeff(vin);
dsift.insert(dsift.end(), vout.begin(), vout.end());
}
descr_size_final = pca.base_size();
}
vl_dsift_delete(sf);
}
size_t numdata = dsift.size()/descr_size_final;
const float *datain = &dsift[0];
vector<float> enc((do_fisher?2:1)*descriptor_dim()*nclusters());
float *dataout = &enc[0];
if (do_fisher) {
if (FrameVerbose())
cout << msg << "fisher encoding " << numdata
<< " descriptors of size " << descr_size_orig
<< " => " << descr_size_final
<< " with gmm dimensionality " << descriptor_dim()
<< endl;
if (descr_size_final!=descriptor_dim()) {
cerr << msg << "dimensionality mismatch descr_size_final="
<< descr_size_final << " descriptor_dim()=" << descriptor_dim()
<< endl;
return false;
}
vl_fisher_encode(dataout, VL_TYPE_FLOAT, means(), descriptor_dim(),
nclusters(), covariances(), priors(),
datain, numdata, VL_FISHER_FLAG_IMPROVED) ;
}
if (do_vlad) {
//obs! correct use of pca?
if (FrameVerbose())
cout << msg << "vlad encoding " << numdata
<< " descriptors of size " << descr_size_final << endl;
vector<vl_uint32> indexes(numdata);
vector<float> distances(numdata);
if (kdtree)
vl_kdforest_query_with_array(kdtree, &indexes[0], 1, numdata, &distances[0], datain);
else
vl_kmeans_quantize(kmeans, &indexes[0], &distances[0], datain, numdata);
vector<float> assignments(numdata*nclusters());
for (size_t i=0; i<numdata; i++)
assignments[i * nclusters() + indexes[i]] = 1;
int vlad_flags = VL_VLAD_FLAG_SQUARE_ROOT|VL_VLAD_FLAG_NORMALIZE_COMPONENTS;
vl_vlad_encode(dataout, VL_TYPE_FLOAT, means(), descriptor_dim(),
nclusters(), datain, numdata, &assignments[0],
vlad_flags);
}
if (renormalize) {
if (PixelVerbose())
cout << " RENORM:";
double mul = 0.0;
for (size_t j=0; j<enc.size(); j++)
mul += enc[j]*enc[j];
if (mul)
mul = 1.0/sqrt(mul);
for (size_t j=0; j<enc.size(); j++) {
if (PixelVerbose())
cout << " " << enc[j];
enc[j] *= mul;
if (PixelVerbose())
cout << "->" << enc[j];
}
if (PixelVerbose())
cout << endl;
}
((VectorData*)GetData(0))->setVector(enc);
return true;
}
int main(int argc, char** argv){
VL_PRINT("Feature Test Program \n");
char* img_name = argv[1];
//load image to gray image
cv::Mat img = cv::imread(img_name, CV_LOAD_IMAGE_GRAYSCALE);
cv::Size size(640, 480);
resize(img, img, size);
int height = img.rows;
int width = img.cols;
vl_sift_pix * img_data = new vl_sift_pix[height * width];
//copy image to vl_feat
for (int i = 0; i < height; ++i){
for (int j = 0; j < width; ++j){
img_data[i*width + j] = img.at<uchar>(i, j);
}
}
VlDsiftFilter* dsiftFilt = NULL;
dsiftFilt = vl_dsift_new(width, height);
//process dense sift
vl_dsift_process(dsiftFilt, img_data);
//get key points
int nkeys = vl_dsift_get_keypoint_num(dsiftFilt);
const VlDsiftKeypoint* pKeyPoint = vl_dsift_get_keypoints(dsiftFilt);
for (int i = 0; i < nkeys; ++i){
VlDsiftKeypoint tmpKeyPoint = *(pKeyPoint + i);
cv::circle(img, cv::Point(tmpKeyPoint.x, tmpKeyPoint.y), 0.5, CV_RGB(255, 255, 0));
//get key point descriptor
const float* descriptors = new float[128];
descriptors = vl_dsift_get_descriptors(dsiftFilt);
//int k = 0;
//while (k < 128){
// printf("%d : %f", k, descriptors[k]);
// printf(";");
// ++k;
//}
}
vl_dsift_delete(dsiftFilt);
//free space
delete[] img_data;
img_data = NULL;
//show final image
cv::namedWindow("Source Image", 1);
cv::imshow("Source Image", img);
cv::waitKey(0);
return 0;
}
void
mexFunction(int nout, mxArray *out[],
int nin, const mxArray *in[])
{
enum {IN_I=0, IN_END} ;
enum {OUT_FRAMES=0, OUT_DESCRIPTORS} ;
int verbose = 0 ;
int opt ;
int next = IN_END ;
mxArray const *optarg ;
float const *data ;
int M, N ;
int step [2] = {1,1} ;
vl_bool norm = 0 ;
vl_bool floatDescriptors = VL_FALSE ;
vl_bool useFlatWindow = VL_FALSE ;
double windowSize = -1.0 ;
double *bounds = NULL ;
double boundBuffer [4] ;
VlDsiftDescriptorGeometry geom ;
VL_USE_MATLAB_ENV ;
geom.numBinX = 4 ;
geom.numBinY = 4 ;
geom.numBinT = 8 ;
geom.binSizeX = 3 ;
geom.binSizeY = 3 ;
/* -----------------------------------------------------------------
* Check the arguments
* -------------------------------------------------------------- */
if (nin < 1) {
vlmxError(vlmxErrNotEnoughInputArguments, NULL) ;
} else if (nout > 2) {
vlmxError(vlmxErrTooManyOutputArguments, NULL) ;
}
if (mxGetNumberOfDimensions (in[IN_I]) != 2 ||
mxGetClassID (in[IN_I]) != mxSINGLE_CLASS ) {
vlmxError(vlmxErrInvalidArgument,
"I must be a matrix of class SINGLE.") ;
}
data = (float*) mxGetData (in[IN_I]) ;
M = mxGetM (in[IN_I]) ;
N = mxGetN (in[IN_I]) ;
while ((opt = vlmxNextOption (in, nin, options, &next, &optarg)) >= 0) {
switch (opt) {
case opt_verbose :
++ verbose ;
break ;
case opt_fast :
useFlatWindow = 1 ;
break ;
case opt_norm :
norm = 1 ;
break ;
case opt_bounds :
if (!vlmxIsPlainVector(optarg, 4)) {
mexErrMsgTxt("BOUNDS must be a 4-dimensional vector.") ;
}
bounds = boundBuffer ;
bounds [0] = mxGetPr(optarg)[0] - 1 ;
bounds [1] = mxGetPr(optarg)[1] - 1 ;
bounds [2] = mxGetPr(optarg)[2] - 1 ;
bounds [3] = mxGetPr(optarg)[3] - 1 ;
break ;
case opt_size :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"SIZE is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
geom.binSizeX = (int) mxGetPr(optarg)[0] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
geom.binSizeX = (int) mxGetPr(optarg)[1] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"SIZE is neither a scalar or a 2D vector.") ;
}
if (geom.binSizeX < 1 || geom.binSizeY < 1) {
vlmxError(vlmxErrInvalidArgument,"SIZE value is invalid.") ;
}
break ;
case opt_step :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"STEP is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
step[0] = (int) mxGetPr(optarg)[0] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
step[0] = (int) mxGetPr(optarg)[1] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"STEP is neither a scalar or a 2D vector.") ;
}
if (step[0] < 1 || step[1] < 1) {
vlmxError(vlmxErrInvalidArgument,"STEP value is invalid.") ;
}
break ;
case opt_window_size :
if (!vlmxIsPlainScalar(optarg) || (windowSize = *mxGetPr(optarg)) < 0) {
vlmxError(vlmxErrInvalidArgument,"WINDOWSIZE is not a scalar or it is negative.") ;
}
break ;
case opt_float_descriptors :
floatDescriptors = VL_TRUE ;
break ;
case opt_geometry :
if (!vlmxIsPlainVector(optarg,3)) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY is not a 3D vector.") ;
}
geom.numBinY = (int)mxGetPr(optarg)[0] ;
geom.numBinX = (int)mxGetPr(optarg)[1] ;
geom.numBinT = (int)mxGetPr(optarg)[2] ;
if (geom.numBinX < 1 ||
geom.numBinY < 1 ||
geom.numBinT < 1) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY value is invalid.") ;
}
break ;
default :
abort() ;
}
}
/* -----------------------------------------------------------------
* Do job
* -------------------------------------------------------------- */
{
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
int k, i ;
VlDsiftFilter *dsift ;
/* note that the image received from MATLAB is transposed */
dsift = vl_dsift_new (M, N) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, step[0], step[1]) ;
if (bounds) {
vl_dsift_set_bounds(dsift,
VL_MAX(bounds[1], 0),
VL_MAX(bounds[0], 0),
VL_MIN(bounds[3], M - 1),
VL_MIN(bounds[2], N - 1));
}
vl_dsift_set_flat_window(dsift, useFlatWindow) ;
if (windowSize >= 0) {
vl_dsift_set_window_size(dsift, windowSize) ;
}
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
if (verbose) {
int stepX ;
int stepY ;
int minX ;
int minY ;
int maxX ;
int maxY ;
vl_bool useFlatWindow ;
vl_dsift_get_steps (dsift, &stepY, &stepX) ;
vl_dsift_get_bounds (dsift, &minY, &minX, &maxY, &maxX) ;
useFlatWindow = vl_dsift_get_flat_window(dsift) ;
mexPrintf("vl_dsift: image size [W, H] = [%d, %d]\n", N, M) ;
mexPrintf("vl_dsift: bounds: [minX,minY,maxX,maxY] = [%d, %d, %d, %d]\n",
minX+1, minY+1, maxX+1, maxY+1) ;
mexPrintf("vl_dsift: subsampling steps: stepX=%d, stepY=%d\n", stepX, stepY) ;
mexPrintf("vl_dsift: num bins: [numBinT, numBinX, numBinY] = [%d, %d, %d]\n",
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
mexPrintf("vl_dsift: descriptor size: %d\n", descrSize) ;
mexPrintf("vl_dsift: bin sizes: [binSizeX, binSizeY] = [%d, %d]\n",
geom.binSizeX,
geom.binSizeY) ;
mexPrintf("vl_dsift: flat window: %s\n", VL_YESNO(useFlatWindow)) ;
mexPrintf("vl_dsift: window size: %g\n", vl_dsift_get_window_size(dsift)) ;
mexPrintf("vl_dsift: num of features: %d\n", numFrames) ;
}
vl_dsift_process (dsift, data) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
/* ---------------------------------------------------------------
* Create output arrays
* ------------------------------------------------------------ */
{
mwSize dims [2] ;
dims [0] = descrSize ;
dims [1] = numFrames ;
if (floatDescriptors) {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxSINGLE_CLASS, mxREAL) ;
} else {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxUINT8_CLASS, mxREAL) ;
}
dims [0] = norm ? 3 : 2 ;
out[OUT_FRAMES] = mxCreateNumericArray
(2, dims, mxDOUBLE_CLASS, mxREAL) ;
}
/* ---------------------------------------------------------------
* Copy back
* ------------------------------------------------------------ */
{
float *tmpDescr = mxMalloc(sizeof(float) * descrSize) ;
double *outFrameIter = mxGetPr(out[OUT_FRAMES]) ;
void *outDescrIter = mxGetData(out[OUT_DESCRIPTORS]) ;
for (k = 0 ; k < numFrames ; ++k) {
*outFrameIter++ = frames[k].y + 1 ;
*outFrameIter++ = frames[k].x + 1 ;
/* We have an implied / 2 in the norm, because of the clipping
below */
if (norm)
*outFrameIter++ = frames [k].norm ;
vl_dsift_transpose_descriptor (tmpDescr,
descrs + descrSize * k,
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
if (floatDescriptors) {
for (i = 0 ; i < descrSize ; ++i) {
float * pt = (float*) outDescrIter ;
*pt++ = VL_MIN(512.0F * tmpDescr[i], 255.0F) ;
outDescrIter = pt ;
}
} else {
for (i = 0 ; i < descrSize ; ++i) {
vl_uint8 * pt = (vl_uint8*) outDescrIter ;
*pt++ = (vl_uint8) (VL_MIN(512.0F * tmpDescr[i], 255.0F)) ;
outDescrIter = pt ;
}
}
}
mxFree(tmpDescr) ;
}
vl_dsift_delete (dsift) ;
}
}
/** ------------------------------------------------------------------
** @brief Python entry point
**/
PyObject * vl_dsift_python(
PyArrayObject & pyArray,
int opt_step,
PyArrayObject & opt_bounds,
int opt_size,
bool opt_fast,
bool opt_verbose,
bool opt_norm)
{
// check data type
assert(pyArray.descr->type_num == PyArray_FLOAT);
assert(pyArray.flags & NPY_FORTRAN);
assert(opt_bounds.descr->type_num == PyArray_FLOAT);
int verbose = 0;
int opt;
float const *data;
int M, N;
int step = 1;
int size = 3;
vl_bool norm = 0;
vl_bool useFlatWindow = VL_FALSE;
double *bounds = NULL;
double boundBuffer[4];
/* -----------------------------------------------------------------
* Check the arguments
* -------------------------------------------------------------- */
data = (float*) pyArray.data;
M = pyArray.dimensions[0];
N = pyArray.dimensions[1];
if (opt_verbose)
++verbose;
if (opt_fast)
useFlatWindow = 1;
if (opt_norm)
norm = 1;
if (opt_bounds.nd == 1 && opt_bounds.dimensions[0] == 4) {
double * tmp = (double *) opt_bounds.data;
bounds = boundBuffer;
for (int i = 0; i < 4; i++)
bounds[i] = tmp[i];
}
if (opt_size >= 0)
size = opt_size;
if (opt_step >= 0)
step = opt_step;
// create PyTuple for outputs
PyObject * tuple = PyTuple_New(2);
/* -----------------------------------------------------------------
* Do job
* -------------------------------------------------------------- */
{
int numFrames;
int descrSize;
VlDsiftKeypoint const *frames;
VlDsiftDescriptorGeometry const *geom;
float const *descrs;
int k, i;
VlDsiftFilter *dsift;
dsift = vl_dsift_new_basic(M, N, step, size);
if (bounds) {
vl_dsift_set_bounds(dsift, VL_MAX(bounds[0], 0), VL_MAX(
bounds[1], 0), VL_MIN(bounds[2], M - 1), VL_MIN(bounds[3], N
- 1));
}
vl_dsift_set_flat_window(dsift, useFlatWindow);
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = vl_dsift_get_geometry(dsift);
if (verbose) {
int stepX;
int stepY;
int minX;
int minY;
int maxX;
int maxY;
vl_bool useFlatWindow;
vl_dsift_get_steps(dsift, &stepX, &stepY);
vl_dsift_get_bounds(dsift, &minX, &minY, &maxX, &maxY);
useFlatWindow = vl_dsift_get_flat_window(dsift);
printf("dsift: image size: %d x %d\n", N, M);
printf(
" bounds: [%d, %d, %d, %d]\n", minY, minX,
maxY, maxX);
printf(" subsampling steps: %d, %d\n", stepY, stepX);
printf(
" num bins: [%d, %d, %d]\n", geom->numBinT,
geom->numBinX, geom->numBinY);
printf(" descriptor size: %d\n", descrSize);
printf(
" bin sizes: [%d, %d]\n", geom->binSizeX,
geom->binSizeY);
printf(" flat window: %s\n", VL_YESNO(useFlatWindow));
printf(" number of frames: %d\n", numFrames);
}
vl_dsift_process(dsift, data);
frames = vl_dsift_get_keypoints(dsift);
descrs = vl_dsift_get_descriptors(dsift);
/* ---------------------------------------------------------------
* Create output arrays
* ------------------------------------------------------------ */
npy_intp dims[2];
dims[0] = descrSize;
dims[1] = numFrames;
// allocate PyArray objects
PyArrayObject * _descriptors = (PyArrayObject *) PyArray_NewFromDescr(
&PyArray_Type, PyArray_DescrFromType(PyArray_UINT8),
2, dims, NULL, NULL, NPY_F_CONTIGUOUS, NULL);
if (norm)
dims[0] = 3;
else
dims[0] = 2;
PyArrayObject * _frames = (PyArrayObject*) PyArray_NewFromDescr(
&PyArray_Type, PyArray_DescrFromType(PyArray_DOUBLE),
2, dims, NULL, NULL, NPY_F_CONTIGUOUS, NULL);
// put PyArray objects in PyTuple
PyTuple_SetItem(tuple, 0, PyArray_Return(_frames));
PyTuple_SetItem(tuple, 1, PyArray_Return(_descriptors));
/* ---------------------------------------------------------------
* Copy back
* ------------------------------------------------------------ */
{
float *tmpDescr = (float*) vl_malloc(sizeof(float) * descrSize);
double *outFrameIter = (double*) _frames->data;
vl_uint8 *outDescrIter = (vl_uint8 *) _descriptors->data;
for (k = 0; k < numFrames; ++k) {
*outFrameIter++ = frames[k].y;
*outFrameIter++ = frames[k].x;
/* We have an implied / 2 in the norm, because of the clipping
below */
if (norm)
*outFrameIter++ = frames[k].norm;
vl_dsift_transpose_descriptor(
tmpDescr, descrs + descrSize * k, geom->numBinT,
geom->numBinX, geom->numBinY);
for (i = 0; i < descrSize; ++i) {
*outDescrIter++ = (vl_uint8) (VL_MIN(
512.0F * tmpDescr[i], 255.0F));
}
}
vl_free(tmpDescr);
}
vl_dsift_delete(dsift);
}
return tuple;
}
vector<vector<float> > Encoder::extractMultiDSIFT(Mat normMat, Mat landmarks, int level){
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
int dimension = patchSize / cellSize;
vector<float> dsiftCode;
for (int l = 0; l < level; l++){
int tmpcellSize = cellSize - l;
int tmppatchSize = tmpcellSize*dimension;
for (unsigned int i = 0; i < landmarks.cols; i++){
if (landmarks.at<float>(0, i) > tmppatchSize/2 && landmarks.at<float>(1, i) > tmppatchSize/2 && landmarks.at<float>(0, i) + tmppatchSize/2 < normMat.cols && landmarks.at<float>(1, i) + tmppatchSize/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tmppatchSize/2 , landmarks.at<float>(1, i) - tmppatchSize/2, tmppatchSize, tmppatchSize));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}
vector<vector<float> > Encoder::extractTunedDSIFT(Mat normMat, Mat landmarks){
//eye, eyebrow, eye-band, nose-ver, mouth, left face, right face, jaw, forehead, nose-hor
//40x20x2, 40x20x2, 100x40, 40x60, 60x40, 40x60,40x60, 50x20, 100x40, 60x40
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
vector<float> dsiftCode;
for (int l = 0; l <= 2; l++){
for (int i = 0; i < landmarks.cols; i++){
switch (i){
case 0:
case 1:
case 2:
case 3:
tunedCols = 30 + 10*l;
tunedRows = 10 + 10*l;
break;
case 4:
tunedCols = 70 + 20*l;
tunedRows = 30 + 10*l;
break;
case 5:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 6:
tunedCols = 40 + 20*l;
tunedRows = 30 + 10*l;
break;
case 7:
case 8:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 9:
tunedCols = 40 + 10*l;
tunedRows = 20;
break;
case 10:
tunedCols = 60 + 20*l;
tunedRows = 30;
break;
case 11:
tunedCols = 40 + 10*l;
tunedRows = 20 + 10*l;
break;
default:
cout<<"Wrong landmark size: "<<i<<endl;
exit(1);
}
if (landmarks.at<float>(0, i) > tunedCols/2 && landmarks.at<float>(1, i) > tunedRows/2 && landmarks.at<float>(0, i) + tunedCols/2 < normMat.cols && landmarks.at<float>(1, i) + tunedRows/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tunedCols/2 , landmarks.at<float>(1, i) - tunedRows/2, tunedCols, tunedRows));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}