std::unique_ptr<Classifier> KNearestNeighborsConstructor::deserialize(const std::string & filename) const{
CvKNearest * res = new CvKNearest();
//TODO: Not implemented in opencv2
res->load(filename.c_str());
int k = res->get_max_k();
return std::unique_ptr<Classifier>(new KNearestNeighborClassifier(std::unique_ptr<CvKNearest>(res),k));
}
//K-nearest neighbor
void knn ( Mat & trainingData , Mat & trainingClasses , Mat & testData , Mat &
testClasses , int K ) {
CvKNearest knn ( trainingData , trainingClasses , Mat () , false , K ) ;
Mat predicted ( testClasses.rows , 1 , CV_32F ) ;
for ( int i = 0; i < testData.rows ; i ++) {
const Mat sample = testData.row ( i ) ;
predicted.at < float >( i ,0) = knn.find_nearest ( sample , K ) ;
}
cout << " Accuracy_ { KNN } = " << evaluate ( predicted , testClasses ) << endl ;
plot_binary ( testData , predicted , " Predictions KNN " ) ;
}
unsigned int KNN_Node::predict(const Mat sample,float& dis)
{
CvKNearest knn;
Mat labels (data->rows,1,CV_32FC1,label);
knn.train(*data,labels,Mat(),false,k);
Mat dists;
Mat r;
Mat rr;
// TODO fix passed params
knn.find_nearest(sample,k,r,rr,dists);
dis = *(std::min_element(dists.begin<float>(),dists.end<float>()));
// set dis to min dis
return label;
}
void operator()( const cv::BlockedRange& range ) const
{
cv::AutoBuffer<float> buf(buf_sz);
for(int i = range.begin(); i < range.end(); i += 1 )
{
float* neighbor_responses = &buf[0];
float* dist = neighbor_responses + 1*k;
Cv32suf* sort_buf = (Cv32suf*)(dist + 1*k);
pointer->find_neighbors_direct( _samples, k, i, i + 1,
neighbor_responses, _neighbors, dist );
float r = pointer->write_results( k, k1, i, i + 1, neighbor_responses, dist,
_results, _neighbor_responses, _dist, sort_buf );
if( i == 0 )
*result = r;
}
}
void Model::Predict_knn( const SampleSet& samples, SampleSet& outError )
{
int true_resp = 0;
CvKNearest *model = (CvKNearest*)m_pModel;
cv::Mat result;
model->find_nearest(samples.Samples(), 1, &result);
for (int i = 0; i < samples.N(); i++)
{
if (result.at<float>(i) != samples.GetLabelAt(i))
{
outError.Add(samples.GetSampleAt(i), samples.GetLabelAt(i));
}
else
{
true_resp++;
}
}
printf("%d %d",samples.N(), true_resp);
}
bool DigitReader :: train(char* dataPath, char* labelPath, int byteOrder) {
if(readTrainData(dataPath,byteOrder)&&readTrainClasses(labelPath, byteOrder)) {
knn.train( trainData, trainClasses);
return true;
} else {
return false;
}
}
int main(int argc, char **argv) {
bool die(false);
string filename("snapshot");
string suffix(".png");
int i_snap(0),iter(0);
Mat depthMat(Size(640,480),CV_16UC1);
Mat depthf (Size(640,480),CV_8UC1);
Mat rgbMat(Size(640,480),CV_8UC3,Scalar(0));
Mat ownMat(Size(640,480),CV_8UC3,Scalar(0));
Freenect::Freenect<MyFreenectDevice> freenect;
MyFreenectDevice& device = freenect.createDevice(0);
device.setTiltDegrees(10.0);
bool registered = false;
Mat blobMaskOutput = Mat::zeros(Size(640,480),CV_8UC1),outC;
Point midBlob;
int startX = 200, sizeX = 180, num_x_reps = 18, num_y_reps = 48;
double height_over_num_y_reps = 480/num_y_reps,
width_over_num_x_reps = sizeX/num_x_reps;
vector<double> _d(num_x_reps * num_y_reps); //the descriptor
Mat descriptorMat(_d);
// CvNormalBayesClassifier classifier; //doesnt work
CvKNearest classifier;
// CvSVM classifier; //doesnt work
// CvBoost classifier; //only good for 2 classes
// CvDTree classifier;
vector<vector<double> > training_data;
vector<int> label_data;
PCA pca;
Mat labelMat, dataMat;
vector<float> label_counts(4);
bool trained = false, loaded = false;
device.startVideo();
device.startDepth();
while (!die) {
device.getVideo(rgbMat);
device.getDepth(depthMat);
// cv::imshow("rgb", rgbMat);
depthMat.convertTo(depthf, CV_8UC1, 255.0/2048.0);
cv::imshow("depth",depthf);
//interpolation & inpainting
{
Mat _tmp,_tmp1; // = (depthMat - 400.0); //minimum observed value is ~440. so shift a bit
Mat(depthMat - 400.0).convertTo(_tmp1,CV_64FC1);
_tmp.setTo(Scalar(2048), depthMat > 750.0); //cut off at 600 to create a "box" where the user interacts
// _tmp.convertTo(depthf, CV_8UC1, 255.0/1648.0); //values are 0-2048 (11bit), account for -400 = 1648
//quadratic interpolation
// cv::pow(_tmp,2.0,_tmp1);
// _tmp1 = _tmp1 * 4.0;
// try {
// cv:log(_tmp,_tmp1);
// }
// catch (cv::Exception e) {
// cerr << e.what() << endl;
// exit(0);
// }
Point minLoc; double minval,maxval;
minMaxLoc(_tmp1, &minval, &maxval, NULL, NULL);
_tmp1.convertTo(depthf, CV_8UC1, 255.0/maxval);
Mat small_depthf; resize(depthf,small_depthf,Size(),0.2,0.2);
cv::inpaint(small_depthf,(small_depthf == 255),_tmp1,5.0,INPAINT_TELEA);
resize(_tmp1, _tmp, depthf.size());
_tmp.copyTo(depthf, (depthf == 255));
}
{
// Mat smallDepth = depthf; //cv::resize(depthf,smallDepth,Size(),0.5,0.5);
// Mat edges; //Laplacian(smallDepth, edges, -1, 7, 1.0);
// Sobel(smallDepth, edges, -1, 1, 1, 7);
//medianBlur(edges, edges, 11);
// for (int x=0; x < edges.cols; x+=20) {
// for (int y=0; y < edges.rows; y+=20) {
// //int nz = countNonZero(edges(Range(y,MIN(y+20,edges.rows-1)),Range(x,MIN(x+20,edges.cols-1))));
// Mat _i = edges(Range(y,MIN(y+20,edges.rows-1)),Range(x,MIN(x+20,edges.cols-1)));
// medianBlur(_i, _i, 7);
// //rectangle(edges, Point(x,y), Point(x+20,y+20), Scalar(nz), CV_FILLED);
// }
// }
// imshow("edges", edges);
}
cvtColor(depthf, outC, CV_GRAY2BGR);
Mat blobMaskInput = depthf < 120; //anything not white is "real" depth, TODO: inpainting invalid data
vector<Point> ctr,ctr2;
//closest point to the camera
Point minLoc; double minval,maxval;
minMaxLoc(depthf, &minval, &maxval, &minLoc, NULL, blobMaskInput);
circle(outC, minLoc, 5, Scalar(0,255,0), 3);
blobMaskInput = depthf < (minval + 18);
Scalar blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob); //find contours in the foreground, choose biggest
// if (blobMaskOutput.data != NULL) {
// imshow("first", blobMaskOutput);
// }
/////// blb :
//blb[0] = x, blb[1] = y, blb[2] = 1st blob size, blb[3] = 2nd blob size.
if(blb[0]>=0 && blb[2] > 500) { //1st blob detected, and is big enough
//cvtColor(depthf, outC, CV_GRAY2BGR);
Scalar mn,stdv;
meanStdDev(depthf,mn,stdv,blobMaskInput);
//cout << "min: " << minval << ", max: " << maxval << ", mean: " << mn[0] << endl;
//now refining blob by looking at the mean depth value it has...
blobMaskInput = depthf < (mn[0] + stdv[0]);
//(very simple) bias with hand color
{
Mat hsv; cvtColor(rgbMat, hsv, CV_RGB2HSV);
Mat _col_p(hsv.size(),CV_32FC1);
int jump = 5;
for (int x=0; x < hsv.cols; x+=jump) {
for (int y=0; y < hsv.rows; y+=jump) {
Mat _i = hsv(Range(y,MIN(y+jump,hsv.rows-1)),Range(x,MIN(x+jump,hsv.cols-1)));
Scalar hsv_mean = mean(_i);
Vec2i u; u[0] = hsv_mean[0]; u[1] = hsv_mean[1];
Vec2i v; v[0] = 120; v[1] = 110;
rectangle(_col_p, Point(x,y), Point(x+jump,y+jump), Scalar(1.0-MIN(norm(u-v)/125.0,1.0)), CV_FILLED);
}
}
// hsv = hsv - Scalar(0,0,255);
Mat _t = (Mat_<double>(2,3) << 1, 0, 15, 0, 1, -20);
Mat col_p(_col_p.size(),CV_32FC1);
warpAffine(_col_p, col_p, _t, col_p.size());
GaussianBlur(col_p, col_p, Size(11.0,11.0), 2.5);
imshow("hand color",col_p);
// imshow("rgb",rgbMat);
Mat blobMaskInput_32FC1; blobMaskInput.convertTo(blobMaskInput_32FC1, CV_32FC1, 1.0/255.0);
blobMaskInput_32FC1 = blobMaskInput_32FC1.mul(col_p, 1.0);
blobMaskInput_32FC1.convertTo(blobMaskInput, CV_8UC1, 255.0);
blobMaskInput = blobMaskInput > 128;
imshow("blob bias", blobMaskInput);
}
blb = refineSegments(Mat(),blobMaskInput,blobMaskOutput,ctr,ctr2,midBlob);
imshow("blob", blobMaskOutput);
if(blb[0] >= 0 && blb[2] > 300) {
//draw contour
Scalar color(0,0,255);
for (int idx=0; idx<ctr.size()-1; idx++)
line(outC, ctr[idx], ctr[idx+1], color, 1);
line(outC, ctr[ctr.size()-1], ctr[0], color, 1);
if(ctr2.size() > 0) { //second blob detected
Scalar color2(255,0,255);
for (int idx=0; idx<ctr2.size()-1; idx++)
line(outC, ctr2[idx], ctr2[idx+1], color2, 2);
line(outC, ctr2[ctr2.size()-1], ctr2[0], color2, 2);
}
//blob center
circle(outC, Point(blb[0],blb[1]), 50, Scalar(255,0,0), 3);
{
Mat hsv; cvtColor(rgbMat, hsv, CV_RGB2HSV);
Scalar hsv_mean,hsv_stddev; meanStdDev(hsv, hsv_mean, hsv_stddev, blobMaskOutput);
stringstream ss; ss << hsv_mean[0] << "," << hsv_mean[1] << "," << hsv_mean[2];
putText(outC, ss.str(), Point(blb[0],blb[1]), CV_FONT_HERSHEY_PLAIN, 1.0, Scalar(0,255,255));
}
Mat blobDepth,blobEdge;
depthf.copyTo(blobDepth,blobMaskOutput);
Laplacian(blobDepth, blobEdge, 8);
// equalizeHist(blobEdge, blobEdge);//just for visualization
Mat logPolar(depthf.size(),CV_8UC1);
cvLogPolar(&((IplImage)blobEdge), &((IplImage)logPolar), Point2f(blb[0],blb[1]), 80.0);
// for (int i=0; i<num_x_reps+1; i++) {
// //verical lines
// line(logPolar, Point(startX+i*width_over_num_x_reps, 0), Point(startX+i*width_over_num_x_reps,479), Scalar(255), 1);
// }
// for(int i=0; i<num_y_reps+1; i++) {
// //horizontal
// line(logPolar, Point(startX, i*height_over_num_y_reps), Point(startX+sizeX,i*height_over_num_y_reps), Scalar(255), 1);
// }
double total = 0.0;
//histogram
for (int i=0; i<num_x_reps; i++) {
for(int j=0; j<num_y_reps; j++) {
Mat part = logPolar(
Range(j*height_over_num_y_reps,(j+1)*height_over_num_y_reps),
Range(startX+i*width_over_num_x_reps,startX+(i+1)*width_over_num_x_reps)
);
// int count = countNonZero(part); //TODO: use calcHist
// _d[i*num_x_reps + j] = count;
Scalar mn = mean(part);
// part.setTo(Scalar(mn[0])); //for debug: show the value in the image
_d[i*num_x_reps + j] = mn[0];
total += mn[0];
}
}
descriptorMat = descriptorMat / total;
/*
Mat images[1] = {logPolar(Range(0,30),Range(0,30))};
int nimages = 1;
int channels[1] = {0};
int dims = 1;
float range_0[]={0,256};
float* ranges[] = { range_0 };
int histSize[1] = { 5 };
calcHist(, <#int nimages#>, <#const int *channels#>, <#const Mat mask#>, <#MatND hist#>, <#int dims#>, <#const int *histSize#>, <#const float **ranges#>, <#bool uniform#>, <#bool accumulate#>)
*/
// Mat _tmp(logPolar.size(),CV_8UC1);
// cvLogPolar(&((IplImage)logPolar), &((IplImage)_tmp),Point2f(blb[0],blb[1]), 80.0, CV_WARP_INVERSE_MAP);
// imshow("descriptor", _tmp);
// imshow("logpolar", logPolar);
}
}
if(trained) {
Mat results(1,1,CV_32FC1);
Mat samples; Mat(Mat(_d).t()).convertTo(samples,CV_32FC1);
Mat samplesAfterPCA = samples; //pca.project(samples);
classifier.find_nearest(&((CvMat)samplesAfterPCA), 1, &((CvMat)results));
// ((float*)results.data)[0] = classifier.predict(&((CvMat)samples))->value;
Mat lc(label_counts); lc *= 0.9;
// label_counts[(int)((float*)results.data)[0]] *= 0.9;
label_counts[(int)((float*)results.data)[0]] += 0.1;
Point maxLoc;
minMaxLoc(lc, NULL, NULL, NULL, &maxLoc);
int res = maxLoc.y;
stringstream ss; ss << "prediction: ";
if (res == LABEL_OPEN) {
ss << "Open hand";
}
if (res == LABEL_FIST) {
ss << "Fist";
}
if (res == LABEL_THUMB) {
ss << "Thumb";
}
if (res == LABEL_GARBAGE) {
ss << "Garbage";
}
putText(outC, ss.str(), Point(20,50), CV_FONT_HERSHEY_PLAIN, 3.0, Scalar(0,0,255), 2);
}
stringstream ss; ss << "samples: " << training_data.size();
putText(outC, ss.str(), Point(30,outC.rows - 30), CV_FONT_HERSHEY_PLAIN, 2.0, Scalar(0,0,255), 1);
imshow("blobs", outC);
char k = cvWaitKey(5);
if( k == 27 ){
break;
}
if( k == 8 ) {
std::ostringstream file;
file << filename << i_snap << suffix;
cv::imwrite(file.str(),rgbMat);
i_snap++;
}
if (k == 'g') {
//put into training as 'garbage'
training_data.push_back(_d);
label_data.push_back(LABEL_GARBAGE);
cout << "learn grabage" << endl;
}
if(k == 'o') {
//put into training as 'open'
training_data.push_back(_d);
label_data.push_back(LABEL_OPEN);
cout << "learn open" << endl;
}
if(k == 'f') {
//put into training as 'fist'
training_data.push_back(_d);
label_data.push_back(LABEL_FIST);
cout << "learn fist" << endl;
}
if(k == 'h') {
//put into training as 'thumb'
training_data.push_back(_d);
label_data.push_back(LABEL_THUMB);
cout << "learn thumb" << endl;
}
if (k=='t') {
//train model
cout << "train model" << endl;
if(loaded != true) {
dataMat = Mat(training_data.size(),_d.size(),CV_32FC1); //descriptors as matrix rows
for (uint i=0; i<training_data.size(); i++) {
Mat v = dataMat(Range(i,i+1),Range::all());
Mat(Mat(training_data[i]).t()).convertTo(v,CV_32FC1,1.0);
}
Mat(label_data).convertTo(labelMat,CV_32FC1);
}
// pca = pca(dataMat,Mat(),CV_PCA_DATA_AS_ROW,15);
Mat dataAfterPCA = dataMat;
// pca.project(dataMat,dataAfterPCA);
classifier.train(&((CvMat)dataAfterPCA), &((CvMat)labelMat));
trained = true;
}
// if(k=='p' && trained) {
// //predict
// Mat results(1,1,CV_32FC1);
// Mat samples(1,64,CV_32FC1); Mat(Mat(_d).t()).convertTo(samples,CV_32FC1);
// classifier.find_nearest(&((CvMat)samples), 1, &((CvMat)results));
// cout << "prediction: " << ((float*)results.data)[0] << endl;
// }
if(k=='s') {
cout << "save training data" << endl;
// classifier.save("knn-classifier-open-fist-thumb.yaml"); //not implemented
dataMat = Mat(training_data.size(),_d.size(),CV_32FC1); //descriptors as matrix rows
for (uint i=0; i<training_data.size(); i++) {
Mat v = dataMat(Range(i,i+1),Range::all());
Mat(Mat(training_data[i]).t()).convertTo(v,CV_32FC1,1.0);
}
Mat(label_data).convertTo(labelMat,CV_32FC1);
FileStorage fs;
fs.open("data-samples-labels.yaml", CV_STORAGE_WRITE);
if (fs.isOpened()) {
fs << "samples" << dataMat;
fs << "labels" << labelMat;
fs << "startX" << startX;
fs << "sizeX" << sizeX;
fs << "num_x_reps" << num_x_reps;
fs << "num_y_reps" << num_y_reps;
loaded = true;
fs.release();
} else {
cerr << "can't open saved data" << endl;
}
}
if(k=='l') {
cout << "try to load training data" << endl;
FileStorage fs;
fs.open("data-samples-labels.yaml", CV_STORAGE_READ);
if (fs.isOpened()) {
fs["samples"] >> dataMat;
fs["labels"] >> labelMat;
fs["startX"] >> startX;
fs["sizeX"] >> sizeX;
fs["num_x_reps"] >> num_x_reps;
fs["num_y_reps"] >> num_y_reps;
height_over_num_y_reps = 480/num_y_reps;
width_over_num_x_reps = sizeX/num_x_reps;
loaded = true;
fs.release();
} else {
//**************************************************************************************************
// main
//
int main(int argc, char** argv)
{
unsigned ii,jj;
char indat[256*1024]; // huge because I am a lazy man
char *indatloc;
unsigned readcnt;
unsigned totallen;
unsigned flagbadframe;
unsigned gazeX, gazeY;
unsigned gazelabel;
unsigned numcams;
unsigned numframes;
unsigned numframesbad;
unsigned numframes_train;
unsigned numframesbad_train;
/* unsigned numframes_test;
unsigned numframesbad_test;*/
unsigned cntcorrect, cntincorrect;
float correctnessratio;
IplImage *frame, *framescaledup;
uchar *frameloc, *framescaleduploc;
IplImage *frame2, *frame2scaledup;
uchar *frame2loc, *frame2scaleduploc;
// double wide!
IplImage *framedual, *framedualscaledup;
uchar *framedualloc1, *framedualloc2, *framedualscaleduploc1, *framedualscaleduploc2;
CvMat *trainingVectors, *trainingClasses;
CvMat *testVector;
float testClass;
// gaze pictures
IplImage *gazeoverlay;
char filenameprefix[2*PATH_MAX_LEN];
char infilenamegazecoords[2*PATH_MAX_LEN];
char infilenamebadframes[2*PATH_MAX_LEN];
FILE *infilegazecoords;
FILE *infilebadframes;
char outfilenamelabelshuman[2*PATH_MAX_LEN];
char outfilenamelabelsml[2*PATH_MAX_LEN];
FILE *outfilelabelshuman;
FILE *outfilelabelsml;
unsigned frameidx_good_train, frameidx;
// process user cli
gFlagUserCliValid=0;
gFlagUserCliHelp=0;
if(0 != parseargs(argc,argv))
{
printusage(argv[0]);
exit(1);
}
if(0 == gFlagUserCliValid)
{
printusage(argv[0]);
exit(1);
}
if(0 != gFlagUserCliHelp)
{
printhelp(argv[0]);
exit(0);
}
// FIXME russ: find a way to get the correct path name!
getdeepestdirname(gPath,filenameprefix);
snprintf(infilenamegazecoords,2*PATH_MAX_LEN,"%s/%s_gazecoords.txt",gPath,filenameprefix);
snprintf(infilenamebadframes,2*PATH_MAX_LEN,"%s/%s_badframes.txt",gPath,filenameprefix);
snprintf(outfilenamelabelshuman,2*PATH_MAX_LEN,"%s/%s_labelshuman.txt",gPath,filenameprefix);
snprintf(outfilenamelabelsml,2*PATH_MAX_LEN,"%s/%s_labelsml.txt",gPath,filenameprefix);
// open input files
infilegazecoords = fopen(infilenamegazecoords,"r");
if(0 == infilegazecoords)
{
fprintf(stderr, "Could not open %s for reading gaze coordinates\n",infilenamegazecoords);
exit(1);
}
infilebadframes = fopen(infilenamebadframes,"r");
if(0 == infilebadframes)
{
fclose(infilegazecoords);
fprintf(stderr, "Could not open %s for reading bad frame flags\n",infilenamebadframes);
exit(1);
}
cntcorrect = cntincorrect = 0;
correctnessratio = 0.0f;
// sloppily grab the number of frames
numframesbad=0;
numframesbad_train=0;
// numframesbad_test=0;
while(EOF != fscanf(infilebadframes,"[%d] bad? := %d\n",&numframes,&flagbadframe))
{
numframesbad += (flagbadframe ? 1:0);
if(numframes < FIRSTTESTFRAME)
{
numframesbad_train += (flagbadframe ? 1:0);
}
/* else
{
numframesbad_test += (flagbadframe ? 1:0);
}*/
}
// 0 indexed
numframes++;
assert(FIRSTTESTFRAME < numframes);
numframes_train = FIRSTTESTFRAME;
// numframes_test = numframes - FIRSTTESTFRAME;
/* FIXME russ: fseek wasn't working
fseek(infilebadframes,0,0);*/
fclose(infilebadframes);
infilebadframes = fopen(infilenamebadframes,"r");
if(0 == infilebadframes)
{
fclose(infilegazecoords);
fprintf(stderr, "Could not open %s for reading bad frame flags\n",infilenamebadframes);
exit(1);
}
// open output files
outfilelabelshuman = fopen(outfilenamelabelshuman,"w");
if(0 == outfilenamelabelshuman)
{
fclose(infilegazecoords);
fclose(infilebadframes);
fprintf(stderr, "Could not open %s for writing human labels\n",outfilenamelabelshuman);
exit(1);
}
outfilelabelsml = fopen(outfilenamelabelsml,"w");
if(0 == outfilelabelsml)
{
fclose(infilegazecoords);
fclose(infilebadframes);
fclose(outfilelabelshuman);
fprintf(stderr, "Could not open %s for writing ML labels\n",outfilenamelabelsml);
exit(1);
}
frameidx_good_train=0;
// init our frame
frame = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
frame = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
framescaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
frame2 = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
frame2scaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
framedual = cvCreateImage(cvSize(FRAME_X_Y*2,FRAME_X_Y), IPL_DEPTH_8U, 1);
framedualscaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL*2,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
gazeoverlay = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL*2,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 3);
// appease the compiler
frameloc = framescaleduploc = 0;
frame2loc = frame2scaleduploc = 0;
framedualloc1 = framedualloc2 = framedualscaleduploc1 = framedualscaleduploc2 = 0;
trainingVectors = cvCreateMat(numframes_train-numframesbad_train, FRAME_X_Y*FRAME_X_Y, CV_32FC1);
trainingClasses = cvCreateMat(numframes_train-numframesbad_train, 1, CV_32FC1);
testVector = cvCreateMat(1, FRAME_X_Y*FRAME_X_Y, CV_32FC1);
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert(OPCODE_RESP_NUM_CAMS == (unsigned char)indat[0]);
numcams = (unsigned)readchar(stdin);
if(2 != numcams)
{
fprintf(stderr,"ERROR: this program expects 2 cameras exactly!\n");
printusage(argv[0]);
exit(1);
}
/* russ: this is better
for(ii=0; ii<FIRSTTESTFRAME; ++ii)*/
while(frameidx < FIRSTTESTFRAME-1)
{
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert( (OPCODE_FRAME == (unsigned char)indat[0]) ||
(SYMBOL_EXIT == (unsigned char)indat[0]) );
if(SYMBOL_EXIT == (unsigned char)indat[0])
{
break;
}
totallen=0;
indatloc=indat;
while(FRAME_LEN*numcams > totallen)
{
readcnt = fread(indatloc,1,(FRAME_LEN*numcams)-totallen,stdin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
for(ii = 0; ii < FRAME_X_Y; ++ii)
{
frameloc = (uchar*)(frame->imageData + (ii*frame->widthStep));
frame2loc = (uchar*)(frame2->imageData + (ii*frame2->widthStep));
framedualloc1 = (uchar*)(framedual->imageData + (ii*framedual->widthStep));
framedualloc2 = (uchar*)( framedual->imageData
+ (ii*framedual->widthStep)
+ (framedual->widthStep/2) );
for(jj = 0; jj < FRAME_X_Y; ++jj)
{
frameloc[jj] = framedualloc1[jj] = (uchar)indat[((numcams*ii)*FRAME_X_Y)+jj];
frame2loc[jj] = framedualloc2[jj] = (uchar)indat[((numcams*ii+1)*FRAME_X_Y)+jj];
// russ: does this work?
// put image into traningVectors[frameidx_good_train]
// will just get written over if this is a bad frame
trainingVectors->data.fl[frameidx_good_train*FRAME_X_Y*FRAME_X_Y + ii*FRAME_X_Y + jj]
= (float)(indat[((numcams*ii)*FRAME_X_Y)+jj] / 255.0f);
}
}
// scale up the frames
cvResize(frame,framescaledup,CV_INTER_LINEAR);
cvResize(frame2,frame2scaledup,CV_INTER_LINEAR);
cvResize(framedual,framedualscaledup,CV_INTER_LINEAR);
if(EOF == fscanf( infilegazecoords, "[%d] (x,y) := (%d,%d)\n",
&frameidx,&gazeX,&gazeY ))
{
// TODO error message? this shouldn't happen I think
break;
}
if(EOF == fscanf(infilebadframes,"[%d] bad? := %d\n",&frameidx,&flagbadframe))
{
// TODO error message? this shouldn't happen I think
break;
}
// check if bad frame and skip if so
if(0 != flagbadframe)
{
continue;
}
gazelabel = calculateLabel(gazeX,gazeY);
fprintf(outfilelabelshuman,"[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(outfilelabelshuman);
printf("[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(stdout);
// put label into traningClasses[frameidx_good_train]
trainingClasses->data.fl[frameidx_good_train] = (float)gazelabel;
++frameidx_good_train;
}
// build the classifier
CvKNearest knn = CvKNearest(trainingVectors,trainingClasses,0,false,3);
while(1)
{
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert( (OPCODE_FRAME == (unsigned char)indat[0]) ||
(SYMBOL_EXIT == (unsigned char)indat[0]) );
if(SYMBOL_EXIT == (unsigned char)indat[0])
{
break;
}
totallen=0;
indatloc=indat;
while(FRAME_LEN*numcams > totallen)
{
readcnt = fread(indatloc,1,(FRAME_LEN*numcams)-totallen,stdin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
for(ii = 0; ii < FRAME_X_Y; ++ii)
{
frameloc = (uchar*)(frame->imageData + (ii*frame->widthStep));
frame2loc = (uchar*)(frame2->imageData + (ii*frame2->widthStep));
framedualloc1 = (uchar*)(framedual->imageData + (ii*framedual->widthStep));
framedualloc2 = (uchar*)( framedual->imageData
+ (ii*framedual->widthStep)
+ (framedual->widthStep/2) );
for(jj = 0; jj < FRAME_X_Y; ++jj)
{
frameloc[jj] = framedualloc1[jj] = (uchar)indat[((numcams*ii)*FRAME_X_Y)+jj];
frame2loc[jj] = framedualloc2[jj] = (uchar)indat[((numcams*ii+1)*FRAME_X_Y)+jj];
// russ: does this work?
testVector->data.fl[ii*FRAME_X_Y + jj]
= (float)(indat[((numcams*ii)*FRAME_X_Y)+jj] / 255.0f);
}
}
// scale up the frames
cvResize(frame,framescaledup,CV_INTER_LINEAR);
cvResize(frame2,frame2scaledup,CV_INTER_LINEAR);
cvResize(framedual,framedualscaledup,CV_INTER_LINEAR);
if(EOF == fscanf( infilegazecoords, "[%d] (x,y) := (%d,%d)\n",
&frameidx,&gazeX,&gazeY ))
{
// TODO error message? this shouldn't happen I think
break;
}
if(EOF == fscanf(infilebadframes,"[%d] bad? := %d\n",&frameidx,&flagbadframe))
{
// TODO error message? this shouldn't happen I think
break;
}
// check if bad frame and skip if so
if(0 != flagbadframe)
{
continue;
}
gazelabel = calculateLabel(gazeX,gazeY);
fprintf(outfilelabelshuman,"[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(outfilelabelshuman);
printf("[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(stdout);
// TODO: classify!
testClass = knn.find_nearest(testVector,3,0,0,0,0);
fprintf(outfilelabelsml,"[%06d] label_ml := %d\n",frameidx,(unsigned)testClass);
fflush(outfilelabelsml);
printf("[%06d] label_ml := %d\n",frameidx,(unsigned)testClass);
fflush(stdout);
if((unsigned)testClass == gazelabel)
{
++cntcorrect;
}
else
{
++cntincorrect;
}
correctnessratio = ((float)cntcorrect) / (float)(cntcorrect+cntincorrect);
printf("running correctness: % 2.04f%%\n",(float)correctnessratio*100.0f);
fflush(stdout);
}
// release/destroy OpenCV objects
cvReleaseImage(&frame);
cvReleaseImage(&framescaledup);
cvReleaseImage(&frame2);
cvReleaseImage(&frame2scaledup);
cvReleaseImage(&framedual);
cvReleaseImage(&framedualscaledup);
cvReleaseImage(&gazeoverlay);
cvDestroyWindow("Gaze Overlay");
// close files
fclose(infilegazecoords);
return 0;
}
int DigitReader::recognize(CvMat* data) {
CvMat* nearests = cvCreateMat( 1, K, CV_32FC1);
int response = knn.find_nearest(data,K,0,0,nearests,0);
return response;
}
void Model::Train_knn( const SampleSet& samples )
{
CvKNearest* model = (CvKNearest*)m_pModel;
//void* para = (void*)m_trainPara;
model->train(samples.Samples(), samples.Labels());
}
int main()
{
const int train_sample_count = 300;
bool is_regression = false;
const char* filename = "data/waveform.data";
int response_idx = 21;
CvMLData data;
CvTrainTestSplit spl( train_sample_count );
if(data.read_csv(filename) != 0)
{
printf("couldn't read %s\n", filename);
exit(0);
}
data.set_response_idx(response_idx);
data.change_var_type(response_idx, CV_VAR_CATEGORICAL);
data.set_train_test_split( &spl );
const CvMat* values = data.get_values();
const CvMat* response = data.get_responses();
const CvMat* missing = data.get_missing();
const CvMat* var_types = data.get_var_types();
const CvMat* train_sidx = data.get_train_sample_idx();
const CvMat* var_idx = data.get_var_idx();
CvMat*response_map;
CvMat*ordered_response = cv_preprocess_categories(response, var_idx, response->rows, &response_map, NULL);
int num_classes = response_map->cols;
CvDTree dtree;
printf("======DTREE=====\n");
CvDTreeParams cvd_params( 10, 1, 0, false, 16, 0, false, false, 0);
dtree.train( &data, cvd_params);
print_result( dtree.calc_error( &data, CV_TRAIN_ERROR), dtree.calc_error( &data, CV_TEST_ERROR ), dtree.get_var_importance() );
#if 0
/* boosted trees are only implemented for two classes */
printf("======BOOST=====\n");
CvBoost boost;
boost.train( &data, CvBoostParams(CvBoost::DISCRETE, 100, 0.95, 2, false, 0));
print_result( boost.calc_error( &data, CV_TRAIN_ERROR ), boost.calc_error( &data, CV_TEST_ERROR), 0 );
#endif
printf("======RTREES=====\n");
CvRTrees rtrees;
rtrees.train( &data, CvRTParams( 10, 2, 0, false, 16, 0, true, 0, 100, 0, CV_TERMCRIT_ITER ));
print_result( rtrees.calc_error( &data, CV_TRAIN_ERROR), rtrees.calc_error( &data, CV_TEST_ERROR ), rtrees.get_var_importance() );
printf("======ERTREES=====\n");
CvERTrees ertrees;
ertrees.train( &data, CvRTParams( 10, 2, 0, false, 16, 0, true, 0, 100, 0, CV_TERMCRIT_ITER ));
print_result( ertrees.calc_error( &data, CV_TRAIN_ERROR), ertrees.calc_error( &data, CV_TEST_ERROR ), ertrees.get_var_importance() );
printf("======GBTREES=====\n");
CvGBTrees gbtrees;
CvGBTreesParams gbparams;
gbparams.loss_function_type = CvGBTrees::DEVIANCE_LOSS; // classification, not regression
gbtrees.train( &data, gbparams);
//gbt_print_error(&gbtrees, values, response, response_idx, train_sidx);
print_result( gbtrees.calc_error( &data, CV_TRAIN_ERROR), gbtrees.calc_error( &data, CV_TEST_ERROR ), 0);
printf("======KNEAREST=====\n");
CvKNearest knearest;
//bool CvKNearest::train( const Mat& _train_data, const Mat& _responses,
// const Mat& _sample_idx, bool _is_regression,
// int _max_k, bool _update_base )
bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL;
assert(is_classifier);
int max_k = 10;
knearest.train(values, response, train_sidx, is_regression, max_k, false);
CvMat* new_response = cvCreateMat(response->rows, 1, values->type);
//print_types();
//const CvMat* train_sidx = data.get_train_sample_idx();
knearest.find_nearest(values, max_k, new_response, 0, 0, 0);
print_result(knearest_calc_error(values, response, new_response, train_sidx, is_regression, CV_TRAIN_ERROR),
knearest_calc_error(values, response, new_response, train_sidx, is_regression, CV_TEST_ERROR), 0);
printf("======== RBF SVM =======\n");
//printf("indexes: %d / %d, responses: %d\n", train_sidx->cols, var_idx->cols, values->rows);
CvMySVM svm1;
CvSVMParams params1 = CvSVMParams(CvSVM::C_SVC, CvSVM::RBF,
/*degree*/0, /*gamma*/1, /*coef0*/0, /*C*/1,
/*nu*/0, /*p*/0, /*class_weights*/0,
cvTermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON));
//svm1.train(values, response, train_sidx, var_idx, params1);
svm1.train_auto(values, response, var_idx, train_sidx, params1);
svm_print_error(&svm1, values, response, response_idx, train_sidx);
printf("======== Linear SVM =======\n");
CvMySVM svm2;
CvSVMParams params2 = CvSVMParams(CvSVM::C_SVC, CvSVM::LINEAR,
/*degree*/0, /*gamma*/1, /*coef0*/0, /*C*/1,
/*nu*/0, /*p*/0, /*class_weights*/0,
cvTermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON));
//svm2.train(values, response, train_sidx, var_idx, params2);
svm2.train_auto(values, response, var_idx, train_sidx, params2);
svm_print_error(&svm2, values, response, response_idx, train_sidx);
printf("======NEURONAL NETWORK=====\n");
int num_layers = 3;
CvMat layers = cvMat(1, num_layers, CV_32SC1, calloc(1, sizeof(double)*num_layers*1));
cvmSetI(&layers, 0, 0, values->cols-1);
cvmSetI(&layers, 0, 1, num_classes);
cvmSetI(&layers, 0, 2, num_classes);
CvANN_MLP ann(&layers, CvANN_MLP::SIGMOID_SYM, 0.0, 0.0);
CvANN_MLP_TrainParams ann_params;
//ann_params.train_method = CvANN_MLP_TrainParams::BACKPROP;
CvMat ann_response = cvmat_make_boolean_class_columns(response, num_classes);
CvMat values2 = cvmat_remove_column(values, response_idx);
ann.train(&values2, &ann_response, NULL, train_sidx, ann_params, 0x0000);
//ann.train(values, &ann_response, NULL, train_sidx, ann_params, 0x0000);
ann_print_error(&ann, values, num_classes, &ann_response, response, response_idx, train_sidx);
#if 0 /* slow */
printf("======== Polygonal SVM =======\n");
//printf("indexes: %d / %d, responses: %d\n", train_sidx->cols, var_idx->cols, values->rows);
CvMySVM svm3;
CvSVMParams params3 = CvSVMParams(CvSVM::C_SVC, CvSVM::POLY,
/*degree*/2, /*gamma*/1, /*coef0*/0, /*C*/1,
/*nu*/0, /*p*/0, /*class_weights*/0,
cvTermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON));
//svm3.train(values, response, train_sidx, var_idx, params3);
svm3.train_auto(values, response, var_idx, train_sidx, params3);
svm_print_error(&svm3, values, response, response_idx, train_sidx);
#endif
return 0;
}
int MultipleGraphsClassifier::predict(DisjointSetForest &segmentation, const Mat_<Vec3b> &image, const Mat_<float> &mask) {
int maxGraphSize = max(segmentation.getNumberOfComponents(), this->maxTrainingGraphSize);
int minGraphSize = min(segmentation.getNumberOfComponents(), this->minTrainingGraphSize);
if (minGraphSize <= this->k) {
cout<<"the smallest graph has size "<<minGraphSize<<", cannot compute "<<this->k<<" eigenvectors"<<endl;
exit(EXIT_FAILURE);
}
// compute each feature graph for the test sample
vector<WeightedGraph> featureGraphs;
featureGraphs.reserve(this->features.size());
for (int i = 0; i < (int)this->features.size(); i++) {
featureGraphs.push_back(this->computeFeatureGraph(i, segmentation, image, mask));
}
// order graphs by feature, to compute pattern vectors feature by
// feature.
vector<vector<WeightedGraph> > graphsByFeatures(this->features.size());
// add feature graphs for the test sample at index 0
for (int i = 0; i < (int)this->features.size(); i++) {
graphsByFeatures[i].reserve(this->trainingFeatureGraphs.size() + 1);
graphsByFeatures[i].push_back(featureGraphs[i]);
}
// add feature graphs for each training sample
for (int i = 0; i < (int)this->trainingFeatureGraphs.size(); i++) {
for (int j = 0; j < (int)this->features.size(); j++) {
graphsByFeatures[j].push_back(get<0>(this->trainingFeatureGraphs[i])[j]);
}
}
// compute the corresponding pattern vectors
vector<vector<VectorXd> > patternsByFeatures(this->features.size());
for (int i = 0; i < (int)this->features.size(); i++) {
patternsByFeatures[i] = patternVectors(graphsByFeatures[i], this->k, maxGraphSize);
}
// concatenate pattern vectors by image, converting to opencv format
// to work with cv's ML module.
Mat_<float> longPatterns = Mat_<float>::zeros(this->trainingFeatureGraphs.size() + 1, maxGraphSize * k * this->features.size());
for (int i = 0; i < (int)this->trainingFeatureGraphs.size() + 1; i++) {
VectorXd longPattern(maxGraphSize * k * this->features.size());
for (int j = 0; j < this->features.size(); j++) {
longPattern.block(j * maxGraphSize * k, 0, maxGraphSize * k, 1) = patternsByFeatures[j][i];
}
Mat_<double> cvLongPattern;
eigenToCv(longPattern, cvLongPattern);
Mat_<float> castLongPattern = Mat_<float>(cvLongPattern);
castLongPattern.copyTo(longPatterns.row(i));
}
// classification of long patterns using SVM
CvKNearest svmClassifier;
Mat_<int> classes(this->trainingFeatureGraphs.size(), 1);
for (int i = 0; i < (int)this->trainingFeatureGraphs.size(); i++) {
classes(i,0) = get<1>(this->trainingFeatureGraphs[i]);
}
svmClassifier.train(longPatterns.rowRange(1, longPatterns.rows), classes);
return (int)svmClassifier.find_nearest(longPatterns.row(0), 1);
}
void train(cv::Mat& trainingData, cv::Mat& trainingClasses, int K)
{
this-knn.train(trainingData, trainingClasses, cv::Mat(), false, K,false);
this->K = K;
}