VL_EXPORT VlDsiftFilter *
vl_dsift_new_basic (int imWidth, int imHeight, int step, int binSize)
{
VlDsiftFilter* self = vl_dsift_new(imWidth, imHeight) ;
VlDsiftDescriptorGeometry geom = *vl_dsift_get_geometry(self) ;
geom.binSizeX = binSize ;
geom.binSizeY = binSize ;
vl_dsift_set_geometry(self, &geom) ;
vl_dsift_set_steps(self, step, step) ;
return self ;
}
void DSift::SetUp(const uint32_t width, const uint32_t height)
{
if (dsift_model_ != NULL)
vl_dsift_delete(dsift_model_);
width_ = width;
height_ = height;
dsift_model_ = vl_dsift_new(width_, height_);
vl_dsift_set_steps(dsift_model_, step_, step_);
vl_dsift_set_window_size(dsift_model_, win_size_);
vl_dsift_set_flat_window(dsift_model_, fast_);
// set the aux data
const int num_sz = sizes_.size();
off_.resize(num_sz, 0);
start_x_.resize(num_sz, 0);
start_y_.resize(num_sz, 0);
end_x_.resize(num_sz, 0);
end_y_.resize(num_sz, 0);
num_patches_.resize(num_sz, 0);
total_num_patches_ = 0;
const uint32_t max_sz = *(std::max_element(sizes_.begin(), sizes_.end()));
for (int i = 0; i < num_sz; ++i)
{
const uint32_t sz = sizes_[i];
off_[i] = std::floor(1.5 * (max_sz - sz));
start_x_[i] = off_[i] + 1.5 * sz;
start_y_[i] = off_[i] + 1.5 * sz;
end_x_[i] = width_ - 1.5 * sz;
end_y_[i] = height_ - 1.5 * sz;
const int num_x = std::ceil((end_x_[i] - start_x_[i]) * 1.0 / step_);
const int num_y = std::ceil((end_y_[i] - start_y_[i]) * 1.0 / step_);
num_patches_[i] = num_x * num_y;
total_num_patches_ += num_patches_[i];
}
has_setup_ = true;
}
vector<vector<float> > Encoder::extractMultiDSIFT(Mat normMat, Mat landmarks, int level){
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
int dimension = patchSize / cellSize;
vector<float> dsiftCode;
for (int l = 0; l < level; l++){
int tmpcellSize = cellSize - l;
int tmppatchSize = tmpcellSize*dimension;
for (unsigned int i = 0; i < landmarks.cols; i++){
if (landmarks.at<float>(0, i) > tmppatchSize/2 && landmarks.at<float>(1, i) > tmppatchSize/2 && landmarks.at<float>(0, i) + tmppatchSize/2 < normMat.cols && landmarks.at<float>(1, i) + tmppatchSize/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tmppatchSize/2 , landmarks.at<float>(1, i) - tmppatchSize/2, tmppatchSize, tmppatchSize));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}
void
mexFunction(int nout, mxArray *out[],
int nin, const mxArray *in[])
{
enum {IN_I=0, IN_END} ;
enum {OUT_FRAMES=0, OUT_DESCRIPTORS} ;
int verbose = 0 ;
int opt ;
int next = IN_END ;
mxArray const *optarg ;
float const *data ;
int M, N ;
int step [2] = {1,1} ;
vl_bool norm = 0 ;
vl_bool floatDescriptors = VL_FALSE ;
vl_bool useFlatWindow = VL_FALSE ;
double windowSize = -1.0 ;
double *bounds = NULL ;
double boundBuffer [4] ;
VlDsiftDescriptorGeometry geom ;
VL_USE_MATLAB_ENV ;
geom.numBinX = 4 ;
geom.numBinY = 4 ;
geom.numBinT = 8 ;
geom.binSizeX = 3 ;
geom.binSizeY = 3 ;
/* -----------------------------------------------------------------
* Check the arguments
* -------------------------------------------------------------- */
if (nin < 1) {
vlmxError(vlmxErrNotEnoughInputArguments, NULL) ;
} else if (nout > 2) {
vlmxError(vlmxErrTooManyOutputArguments, NULL) ;
}
if (mxGetNumberOfDimensions (in[IN_I]) != 2 ||
mxGetClassID (in[IN_I]) != mxSINGLE_CLASS ) {
vlmxError(vlmxErrInvalidArgument,
"I must be a matrix of class SINGLE.") ;
}
data = (float*) mxGetData (in[IN_I]) ;
M = mxGetM (in[IN_I]) ;
N = mxGetN (in[IN_I]) ;
while ((opt = vlmxNextOption (in, nin, options, &next, &optarg)) >= 0) {
switch (opt) {
case opt_verbose :
++ verbose ;
break ;
case opt_fast :
useFlatWindow = 1 ;
break ;
case opt_norm :
norm = 1 ;
break ;
case opt_bounds :
if (!vlmxIsPlainVector(optarg, 4)) {
mexErrMsgTxt("BOUNDS must be a 4-dimensional vector.") ;
}
bounds = boundBuffer ;
bounds [0] = mxGetPr(optarg)[0] - 1 ;
bounds [1] = mxGetPr(optarg)[1] - 1 ;
bounds [2] = mxGetPr(optarg)[2] - 1 ;
bounds [3] = mxGetPr(optarg)[3] - 1 ;
break ;
case opt_size :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"SIZE is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
geom.binSizeX = (int) mxGetPr(optarg)[0] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
geom.binSizeX = (int) mxGetPr(optarg)[1] ;
geom.binSizeY = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"SIZE is neither a scalar or a 2D vector.") ;
}
if (geom.binSizeX < 1 || geom.binSizeY < 1) {
vlmxError(vlmxErrInvalidArgument,"SIZE value is invalid.") ;
}
break ;
case opt_step :
if (!vlmxIsPlainVector(optarg,-1)) {
vlmxError(vlmxErrInvalidArgument,"STEP is not a plain vector.") ;
}
if (mxGetNumberOfElements(optarg) == 1) {
step[0] = (int) mxGetPr(optarg)[0] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else if (mxGetNumberOfElements(optarg) == 2) {
step[0] = (int) mxGetPr(optarg)[1] ;
step[1] = (int) mxGetPr(optarg)[0] ;
} else {
vlmxError(vlmxErrInvalidArgument,"STEP is neither a scalar or a 2D vector.") ;
}
if (step[0] < 1 || step[1] < 1) {
vlmxError(vlmxErrInvalidArgument,"STEP value is invalid.") ;
}
break ;
case opt_window_size :
if (!vlmxIsPlainScalar(optarg) || (windowSize = *mxGetPr(optarg)) < 0) {
vlmxError(vlmxErrInvalidArgument,"WINDOWSIZE is not a scalar or it is negative.") ;
}
break ;
case opt_float_descriptors :
floatDescriptors = VL_TRUE ;
break ;
case opt_geometry :
if (!vlmxIsPlainVector(optarg,3)) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY is not a 3D vector.") ;
}
geom.numBinY = (int)mxGetPr(optarg)[0] ;
geom.numBinX = (int)mxGetPr(optarg)[1] ;
geom.numBinT = (int)mxGetPr(optarg)[2] ;
if (geom.numBinX < 1 ||
geom.numBinY < 1 ||
geom.numBinT < 1) {
vlmxError(vlmxErrInvalidArgument, "GEOMETRY value is invalid.") ;
}
break ;
default :
abort() ;
}
}
/* -----------------------------------------------------------------
* Do job
* -------------------------------------------------------------- */
{
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
int k, i ;
VlDsiftFilter *dsift ;
/* note that the image received from MATLAB is transposed */
dsift = vl_dsift_new (M, N) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, step[0], step[1]) ;
if (bounds) {
vl_dsift_set_bounds(dsift,
VL_MAX(bounds[1], 0),
VL_MAX(bounds[0], 0),
VL_MIN(bounds[3], M - 1),
VL_MIN(bounds[2], N - 1));
}
vl_dsift_set_flat_window(dsift, useFlatWindow) ;
if (windowSize >= 0) {
vl_dsift_set_window_size(dsift, windowSize) ;
}
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
if (verbose) {
int stepX ;
int stepY ;
int minX ;
int minY ;
int maxX ;
int maxY ;
vl_bool useFlatWindow ;
vl_dsift_get_steps (dsift, &stepY, &stepX) ;
vl_dsift_get_bounds (dsift, &minY, &minX, &maxY, &maxX) ;
useFlatWindow = vl_dsift_get_flat_window(dsift) ;
mexPrintf("vl_dsift: image size [W, H] = [%d, %d]\n", N, M) ;
mexPrintf("vl_dsift: bounds: [minX,minY,maxX,maxY] = [%d, %d, %d, %d]\n",
minX+1, minY+1, maxX+1, maxY+1) ;
mexPrintf("vl_dsift: subsampling steps: stepX=%d, stepY=%d\n", stepX, stepY) ;
mexPrintf("vl_dsift: num bins: [numBinT, numBinX, numBinY] = [%d, %d, %d]\n",
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
mexPrintf("vl_dsift: descriptor size: %d\n", descrSize) ;
mexPrintf("vl_dsift: bin sizes: [binSizeX, binSizeY] = [%d, %d]\n",
geom.binSizeX,
geom.binSizeY) ;
mexPrintf("vl_dsift: flat window: %s\n", VL_YESNO(useFlatWindow)) ;
mexPrintf("vl_dsift: window size: %g\n", vl_dsift_get_window_size(dsift)) ;
mexPrintf("vl_dsift: num of features: %d\n", numFrames) ;
}
vl_dsift_process (dsift, data) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
/* ---------------------------------------------------------------
* Create output arrays
* ------------------------------------------------------------ */
{
mwSize dims [2] ;
dims [0] = descrSize ;
dims [1] = numFrames ;
if (floatDescriptors) {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxSINGLE_CLASS, mxREAL) ;
} else {
out[OUT_DESCRIPTORS] = mxCreateNumericArray
(2, dims, mxUINT8_CLASS, mxREAL) ;
}
dims [0] = norm ? 3 : 2 ;
out[OUT_FRAMES] = mxCreateNumericArray
(2, dims, mxDOUBLE_CLASS, mxREAL) ;
}
/* ---------------------------------------------------------------
* Copy back
* ------------------------------------------------------------ */
{
float *tmpDescr = mxMalloc(sizeof(float) * descrSize) ;
double *outFrameIter = mxGetPr(out[OUT_FRAMES]) ;
void *outDescrIter = mxGetData(out[OUT_DESCRIPTORS]) ;
for (k = 0 ; k < numFrames ; ++k) {
*outFrameIter++ = frames[k].y + 1 ;
*outFrameIter++ = frames[k].x + 1 ;
/* We have an implied / 2 in the norm, because of the clipping
below */
if (norm)
*outFrameIter++ = frames [k].norm ;
vl_dsift_transpose_descriptor (tmpDescr,
descrs + descrSize * k,
geom.numBinT,
geom.numBinX,
geom.numBinY) ;
if (floatDescriptors) {
for (i = 0 ; i < descrSize ; ++i) {
float * pt = (float*) outDescrIter ;
*pt++ = VL_MIN(512.0F * tmpDescr[i], 255.0F) ;
outDescrIter = pt ;
}
} else {
for (i = 0 ; i < descrSize ; ++i) {
vl_uint8 * pt = (vl_uint8*) outDescrIter ;
*pt++ = (vl_uint8) (VL_MIN(512.0F * tmpDescr[i], 255.0F)) ;
outDescrIter = pt ;
}
}
}
mxFree(tmpDescr) ;
}
vl_dsift_delete (dsift) ;
}
}
vector<vector<float> > Encoder::extractTunedDSIFT(Mat normMat, Mat landmarks){
//eye, eyebrow, eye-band, nose-ver, mouth, left face, right face, jaw, forehead, nose-hor
//40x20x2, 40x20x2, 100x40, 40x60, 60x40, 40x60,40x60, 50x20, 100x40, 60x40
vector<vector<float> > ret;
//hard code max LBP size
int tunedCellSize = 10;
int tunedCols, tunedRows;
vector<float> dsiftCode;
for (int l = 0; l <= 2; l++){
for (int i = 0; i < landmarks.cols; i++){
switch (i){
case 0:
case 1:
case 2:
case 3:
tunedCols = 30 + 10*l;
tunedRows = 10 + 10*l;
break;
case 4:
tunedCols = 70 + 20*l;
tunedRows = 30 + 10*l;
break;
case 5:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 6:
tunedCols = 40 + 20*l;
tunedRows = 30 + 10*l;
break;
case 7:
case 8:
tunedCols = 30 + 10*l;
tunedRows = 40 + 20*l;
break;
case 9:
tunedCols = 40 + 10*l;
tunedRows = 20;
break;
case 10:
tunedCols = 60 + 20*l;
tunedRows = 30;
break;
case 11:
tunedCols = 40 + 10*l;
tunedRows = 20 + 10*l;
break;
default:
cout<<"Wrong landmark size: "<<i<<endl;
exit(1);
}
if (landmarks.at<float>(0, i) > tunedCols/2 && landmarks.at<float>(1, i) > tunedRows/2 && landmarks.at<float>(0, i) + tunedCols/2 < normMat.cols && landmarks.at<float>(1, i) + tunedRows/2 < normMat.rows){
Mat roi(normMat, Rect(landmarks.at<float>(0, i) - tunedCols/2 , landmarks.at<float>(1, i) - tunedRows/2, tunedCols, tunedRows));
vector<float> data;
for (int j = 0; j < roi.cols; j++){
for (int k = 0; k < roi.rows; k++){
data.push_back((float)roi.at<unsigned char>(k, j)/255);
}
}
//dsift
int numFrames ;
int descrSize ;
VlDsiftKeypoint const *frames ;
float const *descrs ;
VlDsiftFilter *dsift ;
VlDsiftDescriptorGeometry geom ;
geom.numBinX = 2 ;
geom.numBinY = 2 ;
geom.numBinT = 4 ;
geom.binSizeX = 4 ;
geom.binSizeY = 4 ;
dsift = vl_dsift_new (roi.rows, roi.cols) ;
vl_dsift_set_geometry(dsift, &geom) ;
vl_dsift_set_steps(dsift, 2, 2) ;
vl_dsift_set_flat_window(dsift, 1) ;
numFrames = vl_dsift_get_keypoint_num (dsift) ;
descrSize = vl_dsift_get_descriptor_size (dsift) ;
geom = *vl_dsift_get_geometry (dsift) ;
vl_dsift_process (dsift, &data[0]) ;
frames = vl_dsift_get_keypoints (dsift) ;
descrs = vl_dsift_get_descriptors (dsift) ;
//cout<<"frames: "<<numFrames<<" descrs: "<<descrSize<<" cols: "<<roi.cols<<" rows: "<<roi.rows<<endl;
float tranDescr[128];
for (int f = 0; f < numFrames; f++){
vl_dsift_transpose_descriptor (tranDescr, descrs + descrSize * f, geom.numBinT, geom.numBinX, geom.numBinY) ;
for (int d = 0 ; d < descrSize ; d++) {
tranDescr[d] = VL_MIN(512.0F * tranDescr[d], 255.0F) ;
dsiftCode.push_back(tranDescr[d]);
//cout<<tmpDescr[i]<<" ";
}
}
//if (i != 0 && i != 2){
ret.push_back(dsiftCode);
dsiftCode.clear();
//}
vl_dsift_delete (dsift) ;
}
else{
cout<<"Patch out of bound: "<<landmarks.at<float>(0, i)<<" "<<landmarks.at<float>(1, i)<<endl;
cout<<"landmark: "<<i<<" Cols: "<<tunedCols<<" Rows: "<<tunedRows<<endl;
imwrite("tmp/outOfBound.jpg", normMat);
exit(1);
}
}
}
return ret;
}