//
// glassesReadFrame: read a FRAME frame of length len into buf
//
char glassesReadFrame(FILE *infile, char buf[], unsigned len)
{
char opcode;
char *bufloc;
unsigned readlen;
unsigned readlenTotal;
readuntilchar(infile,SYMBOL_SOF);
opcode = readchar(infile);
if(OPCODE_FRAME == (unsigned char)opcode)
{
readlenTotal=0;
bufloc=buf;
while(len > readlenTotal)
{
readlen = fread(bufloc,1,len-readlenTotal,infile);
readlenTotal+=readlen;
bufloc+=readlen;
}
*bufloc = '\0';
}
return opcode;
}
void Grab_Camera_Frames()
{
// russ
unsigned ii,jj;
char indat[256*1024]; // huge because I am a lazy man
char *indatloc;
unsigned readcnt;
unsigned totallen;
uchar *eye_image_loc, *scene_image_loc;
// Russ: Glasses support added here.
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert( (OPCODE_FRAME == (unsigned char)indat[0]) ||
(SYMBOL_EXIT == (unsigned char)indat[0]) );
if(SYMBOL_EXIT == (unsigned char)indat[0])
{
Close_Logfile();
Close_GUI();
exit(0);
}
totallen=0;
indatloc=indat;
while((RESOLUTION_WIDTH*RESOLUTION_HEIGHT)*2 > totallen)
{
readcnt = fread(indatloc,1,((RESOLUTION_WIDTH*RESOLUTION_HEIGHT)*2)-totallen,stdin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
for(ii=0; ii < RESOLUTION_WIDTH; ++ii)
{
eye_image_loc = (uchar*)(eye_image->imageData + (ii*eye_image->widthStep));
scene_image_loc = (uchar*)(scene_image_grayscale->imageData + (ii*scene_image_grayscale->widthStep));
for(jj = 0; jj < RESOLUTION_HEIGHT; ++jj)
{
eye_image_loc[jj] = (uchar)indat[((2*ii)*RESOLUTION_WIDTH)+jj];
scene_image_loc[jj] = (uchar)indat[((2*ii+1)*RESOLUTION_WIDTH)+jj];
}
}
cvNormalize(eye_image,eye_image,0,255,CV_MINMAX,CV_8UC1);
cvNormalize(scene_image_grayscale,scene_image_grayscale,0,255,CV_MINMAX,CV_8UC1);
cvCvtColor(scene_image_grayscale, scene_image, CV_GRAY2RGB);
original_eye_image = cvCloneImage(eye_image);
if (frame_number == 0) {
Calculate_Avg_Intensity_Hori(eye_image);
memcpy(intensity_factor_hori, avg_intensity_hori, eye_image->height*sizeof(double));
}
frame_number++;
}
//**************************************************************************************************
// main
//
int main(int argc, char** argv)
{
char cc;
char indat[256*1024]; // huge because I am a lazy man
char *indatloc;
int readcnt;
int totallen;
unsigned numcams;
// process user cli
gFlagUserCliValid=0;
gFlagUserCliHelp=0;
gFlagUseBluetooth=0;
if(0 != parseargs(argc,argv))
{
printusage(argv[0]);
exit(1);
}
if(0 == gFlagUserCliValid)
{
printusage(argv[0]);
exit(1);
}
if(0 != gFlagUserCliHelp)
{
printhelp(argv[0]);
exit(0);
}
if(0 == gFlagUseBluetooth)
{
// russ: this is a little inelegant
gCamin = fopen("/dev/ttyACM0","r");
if(0 == gCamin)
{
fprintf(stderr, "Could not open /dev/ttyACM0 for reading; trying /dev/ttyACM1\n");
gCamin = fopen("/dev/ttyACM1", "r");
if(0 == gCamin)
{
fprintf(stderr, "Could not open /dev/ttyACM1 for reading\n");
return -1;
}
gCamout = fopen("/dev/ttyACM1","w");
if(0 == gCamout)
{
fprintf(stderr, "Could not open /dev/ttyACM1 for writing\n");
fclose(gCamin);
return -1;
}
}
else
{
gCamout = fopen("/dev/ttyACM0","w");
if(0 == gCamout)
{
fprintf(stderr, "Could not open /dev/ttyACM0 for writing\n");
fclose(gCamin);
return -1;
}
}
}
else
{
// use bluetooth
gCamin = fopen("/dev/rfcomm0", "r");
if(0 == gCamin)
{
fprintf(stderr, "Could not open /dev/rfcomm0 for reading\n");
return -1;
}
gCamout = fopen("/dev/rfcomm0","w");
if(0 == gCamout)
{
fprintf(stderr, "Could not open /dev/rfcomm0 for writing\n");
fclose(gCamin);
return -1;
}
}
// find out if the device has 1 or 2 cameras
fputc((char)SYMBOL_SOF,gCamout);
dbgPrintOp("tx: 0x%02X\n", (unsigned char)SYMBOL_SOF);
fputc((char)OPCODE_REQ_NUM_CAMS,gCamout);
dbgPrintOp("tx: 0x%02X\n", (unsigned char)OPCODE_REQ_NUM_CAMS);
fflush(gCamout);
readuntilchar(gCamin,SYMBOL_SOF);
dbgPrintOp("tx: 0x%02X\n", (unsigned char)SYMBOL_SOF);
indat[0] = readchar(gCamin);
dbgPrintOp("tx: 0x%02X\n", (unsigned char)indat[0]);
if(OPCODE_RESP_NUM_CAMS != (unsigned char)indat[0])
{
assert(OPCODE_RESP_NUM_CAMS == (unsigned char)indat[0]);
}
numcams = readchar(gCamin);
dbgPrintOp("tx: 0x%02X\n", (unsigned char)numcams);
assert((0 < numcams) && (MAX_CAMS >= numcams));
// print numcams on stdout for pipe interface
printf("%c",SYMBOL_SOF);
printf("%c",OPCODE_RESP_NUM_CAMS);
printf("%c",(unsigned char)numcams);
fflush(stdout);
while(1)
{
// TODO: should be a function
fputc((char)SYMBOL_SOF,gCamout);
fputc((char)OPCODE_SINGLE_FRAME,gCamout);
fflush(gCamout);
readuntilchar(gCamin,SYMBOL_SOF);
dbgPrintOp("rx: 0x%02X\n", (unsigned char)SYMBOL_SOF);
printf("%c",SYMBOL_SOF);
indat[0] = readchar(gCamin);
dbgPrintOp("rx: 0x%02X\n", (unsigned char)indat[0]);
printf("%c",indat[0]);
if(OPCODE_FRAME != (unsigned char)indat[0])
{
assert(OPCODE_FRAME == (unsigned char)indat[0]);
}
totallen=0;
indatloc=indat;
while(FRAME_LEN*numcams > totallen)
{
readcnt = fread(indatloc,1,(FRAME_LEN*numcams)-totallen,gCamin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
printf("%s",indat);
fflush(stdout);
cc = getch();
// look for ESC key
if(ESC_KEY == cc)
{
break;
}
}
// tell listening program that we're done here
printf("%c",SYMBOL_SOF);
printf("%c",SYMBOL_EXIT);
fflush(stdout);
fclose(gCamin);
fclose(gCamout);
return 0;
}
//**************************************************************************************************
// main
//
int main(int argc, char** argv)
{
unsigned ii,jj;
char indat[256*1024]; // huge because I am a lazy man
char *indatloc;
unsigned readcnt;
unsigned totallen;
unsigned flagbadframe;
unsigned gazeX, gazeY;
unsigned gazelabel;
unsigned numcams;
unsigned numframes;
unsigned numframesbad;
unsigned numframes_train;
unsigned numframesbad_train;
/* unsigned numframes_test;
unsigned numframesbad_test;*/
unsigned cntcorrect, cntincorrect;
float correctnessratio;
IplImage *frame, *framescaledup;
uchar *frameloc, *framescaleduploc;
IplImage *frame2, *frame2scaledup;
uchar *frame2loc, *frame2scaleduploc;
// double wide!
IplImage *framedual, *framedualscaledup;
uchar *framedualloc1, *framedualloc2, *framedualscaleduploc1, *framedualscaleduploc2;
CvMat *trainingVectors, *trainingClasses;
CvMat *testVector;
float testClass;
// gaze pictures
IplImage *gazeoverlay;
char filenameprefix[2*PATH_MAX_LEN];
char infilenamegazecoords[2*PATH_MAX_LEN];
char infilenamebadframes[2*PATH_MAX_LEN];
FILE *infilegazecoords;
FILE *infilebadframes;
char outfilenamelabelshuman[2*PATH_MAX_LEN];
char outfilenamelabelsml[2*PATH_MAX_LEN];
FILE *outfilelabelshuman;
FILE *outfilelabelsml;
unsigned frameidx_good_train, frameidx;
// process user cli
gFlagUserCliValid=0;
gFlagUserCliHelp=0;
if(0 != parseargs(argc,argv))
{
printusage(argv[0]);
exit(1);
}
if(0 == gFlagUserCliValid)
{
printusage(argv[0]);
exit(1);
}
if(0 != gFlagUserCliHelp)
{
printhelp(argv[0]);
exit(0);
}
// FIXME russ: find a way to get the correct path name!
getdeepestdirname(gPath,filenameprefix);
snprintf(infilenamegazecoords,2*PATH_MAX_LEN,"%s/%s_gazecoords.txt",gPath,filenameprefix);
snprintf(infilenamebadframes,2*PATH_MAX_LEN,"%s/%s_badframes.txt",gPath,filenameprefix);
snprintf(outfilenamelabelshuman,2*PATH_MAX_LEN,"%s/%s_labelshuman.txt",gPath,filenameprefix);
snprintf(outfilenamelabelsml,2*PATH_MAX_LEN,"%s/%s_labelsml.txt",gPath,filenameprefix);
// open input files
infilegazecoords = fopen(infilenamegazecoords,"r");
if(0 == infilegazecoords)
{
fprintf(stderr, "Could not open %s for reading gaze coordinates\n",infilenamegazecoords);
exit(1);
}
infilebadframes = fopen(infilenamebadframes,"r");
if(0 == infilebadframes)
{
fclose(infilegazecoords);
fprintf(stderr, "Could not open %s for reading bad frame flags\n",infilenamebadframes);
exit(1);
}
cntcorrect = cntincorrect = 0;
correctnessratio = 0.0f;
// sloppily grab the number of frames
numframesbad=0;
numframesbad_train=0;
// numframesbad_test=0;
while(EOF != fscanf(infilebadframes,"[%d] bad? := %d\n",&numframes,&flagbadframe))
{
numframesbad += (flagbadframe ? 1:0);
if(numframes < FIRSTTESTFRAME)
{
numframesbad_train += (flagbadframe ? 1:0);
}
/* else
{
numframesbad_test += (flagbadframe ? 1:0);
}*/
}
// 0 indexed
numframes++;
assert(FIRSTTESTFRAME < numframes);
numframes_train = FIRSTTESTFRAME;
// numframes_test = numframes - FIRSTTESTFRAME;
/* FIXME russ: fseek wasn't working
fseek(infilebadframes,0,0);*/
fclose(infilebadframes);
infilebadframes = fopen(infilenamebadframes,"r");
if(0 == infilebadframes)
{
fclose(infilegazecoords);
fprintf(stderr, "Could not open %s for reading bad frame flags\n",infilenamebadframes);
exit(1);
}
// open output files
outfilelabelshuman = fopen(outfilenamelabelshuman,"w");
if(0 == outfilenamelabelshuman)
{
fclose(infilegazecoords);
fclose(infilebadframes);
fprintf(stderr, "Could not open %s for writing human labels\n",outfilenamelabelshuman);
exit(1);
}
outfilelabelsml = fopen(outfilenamelabelsml,"w");
if(0 == outfilelabelsml)
{
fclose(infilegazecoords);
fclose(infilebadframes);
fclose(outfilelabelshuman);
fprintf(stderr, "Could not open %s for writing ML labels\n",outfilenamelabelsml);
exit(1);
}
frameidx_good_train=0;
// init our frame
frame = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
frame = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
framescaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
frame2 = cvCreateImage(cvSize(FRAME_X_Y,FRAME_X_Y), IPL_DEPTH_8U, 1);
frame2scaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
framedual = cvCreateImage(cvSize(FRAME_X_Y*2,FRAME_X_Y), IPL_DEPTH_8U, 1);
framedualscaledup = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL*2,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 1);
gazeoverlay = cvCreateImage(cvSize( FRAME_X_Y*SCALINGVAL*2,
FRAME_X_Y*SCALINGVAL ), IPL_DEPTH_8U, 3);
// appease the compiler
frameloc = framescaleduploc = 0;
frame2loc = frame2scaleduploc = 0;
framedualloc1 = framedualloc2 = framedualscaleduploc1 = framedualscaleduploc2 = 0;
trainingVectors = cvCreateMat(numframes_train-numframesbad_train, FRAME_X_Y*FRAME_X_Y, CV_32FC1);
trainingClasses = cvCreateMat(numframes_train-numframesbad_train, 1, CV_32FC1);
testVector = cvCreateMat(1, FRAME_X_Y*FRAME_X_Y, CV_32FC1);
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert(OPCODE_RESP_NUM_CAMS == (unsigned char)indat[0]);
numcams = (unsigned)readchar(stdin);
if(2 != numcams)
{
fprintf(stderr,"ERROR: this program expects 2 cameras exactly!\n");
printusage(argv[0]);
exit(1);
}
/* russ: this is better
for(ii=0; ii<FIRSTTESTFRAME; ++ii)*/
while(frameidx < FIRSTTESTFRAME-1)
{
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert( (OPCODE_FRAME == (unsigned char)indat[0]) ||
(SYMBOL_EXIT == (unsigned char)indat[0]) );
if(SYMBOL_EXIT == (unsigned char)indat[0])
{
break;
}
totallen=0;
indatloc=indat;
while(FRAME_LEN*numcams > totallen)
{
readcnt = fread(indatloc,1,(FRAME_LEN*numcams)-totallen,stdin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
for(ii = 0; ii < FRAME_X_Y; ++ii)
{
frameloc = (uchar*)(frame->imageData + (ii*frame->widthStep));
frame2loc = (uchar*)(frame2->imageData + (ii*frame2->widthStep));
framedualloc1 = (uchar*)(framedual->imageData + (ii*framedual->widthStep));
framedualloc2 = (uchar*)( framedual->imageData
+ (ii*framedual->widthStep)
+ (framedual->widthStep/2) );
for(jj = 0; jj < FRAME_X_Y; ++jj)
{
frameloc[jj] = framedualloc1[jj] = (uchar)indat[((numcams*ii)*FRAME_X_Y)+jj];
frame2loc[jj] = framedualloc2[jj] = (uchar)indat[((numcams*ii+1)*FRAME_X_Y)+jj];
// russ: does this work?
// put image into traningVectors[frameidx_good_train]
// will just get written over if this is a bad frame
trainingVectors->data.fl[frameidx_good_train*FRAME_X_Y*FRAME_X_Y + ii*FRAME_X_Y + jj]
= (float)(indat[((numcams*ii)*FRAME_X_Y)+jj] / 255.0f);
}
}
// scale up the frames
cvResize(frame,framescaledup,CV_INTER_LINEAR);
cvResize(frame2,frame2scaledup,CV_INTER_LINEAR);
cvResize(framedual,framedualscaledup,CV_INTER_LINEAR);
if(EOF == fscanf( infilegazecoords, "[%d] (x,y) := (%d,%d)\n",
&frameidx,&gazeX,&gazeY ))
{
// TODO error message? this shouldn't happen I think
break;
}
if(EOF == fscanf(infilebadframes,"[%d] bad? := %d\n",&frameidx,&flagbadframe))
{
// TODO error message? this shouldn't happen I think
break;
}
// check if bad frame and skip if so
if(0 != flagbadframe)
{
continue;
}
gazelabel = calculateLabel(gazeX,gazeY);
fprintf(outfilelabelshuman,"[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(outfilelabelshuman);
printf("[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(stdout);
// put label into traningClasses[frameidx_good_train]
trainingClasses->data.fl[frameidx_good_train] = (float)gazelabel;
++frameidx_good_train;
}
// build the classifier
CvKNearest knn = CvKNearest(trainingVectors,trainingClasses,0,false,3);
while(1)
{
readuntilchar(stdin,SYMBOL_SOF);
indat[0] = readchar(stdin);
assert( (OPCODE_FRAME == (unsigned char)indat[0]) ||
(SYMBOL_EXIT == (unsigned char)indat[0]) );
if(SYMBOL_EXIT == (unsigned char)indat[0])
{
break;
}
totallen=0;
indatloc=indat;
while(FRAME_LEN*numcams > totallen)
{
readcnt = fread(indatloc,1,(FRAME_LEN*numcams)-totallen,stdin);
totallen+=readcnt;
indatloc+=readcnt;
}
*indatloc = '\0';
for(ii = 0; ii < FRAME_X_Y; ++ii)
{
frameloc = (uchar*)(frame->imageData + (ii*frame->widthStep));
frame2loc = (uchar*)(frame2->imageData + (ii*frame2->widthStep));
framedualloc1 = (uchar*)(framedual->imageData + (ii*framedual->widthStep));
framedualloc2 = (uchar*)( framedual->imageData
+ (ii*framedual->widthStep)
+ (framedual->widthStep/2) );
for(jj = 0; jj < FRAME_X_Y; ++jj)
{
frameloc[jj] = framedualloc1[jj] = (uchar)indat[((numcams*ii)*FRAME_X_Y)+jj];
frame2loc[jj] = framedualloc2[jj] = (uchar)indat[((numcams*ii+1)*FRAME_X_Y)+jj];
// russ: does this work?
testVector->data.fl[ii*FRAME_X_Y + jj]
= (float)(indat[((numcams*ii)*FRAME_X_Y)+jj] / 255.0f);
}
}
// scale up the frames
cvResize(frame,framescaledup,CV_INTER_LINEAR);
cvResize(frame2,frame2scaledup,CV_INTER_LINEAR);
cvResize(framedual,framedualscaledup,CV_INTER_LINEAR);
if(EOF == fscanf( infilegazecoords, "[%d] (x,y) := (%d,%d)\n",
&frameidx,&gazeX,&gazeY ))
{
// TODO error message? this shouldn't happen I think
break;
}
if(EOF == fscanf(infilebadframes,"[%d] bad? := %d\n",&frameidx,&flagbadframe))
{
// TODO error message? this shouldn't happen I think
break;
}
// check if bad frame and skip if so
if(0 != flagbadframe)
{
continue;
}
gazelabel = calculateLabel(gazeX,gazeY);
fprintf(outfilelabelshuman,"[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(outfilelabelshuman);
printf("[%06d] label_human := %d\n",frameidx,gazelabel);
fflush(stdout);
// TODO: classify!
testClass = knn.find_nearest(testVector,3,0,0,0,0);
fprintf(outfilelabelsml,"[%06d] label_ml := %d\n",frameidx,(unsigned)testClass);
fflush(outfilelabelsml);
printf("[%06d] label_ml := %d\n",frameidx,(unsigned)testClass);
fflush(stdout);
if((unsigned)testClass == gazelabel)
{
++cntcorrect;
}
else
{
++cntincorrect;
}
correctnessratio = ((float)cntcorrect) / (float)(cntcorrect+cntincorrect);
printf("running correctness: % 2.04f%%\n",(float)correctnessratio*100.0f);
fflush(stdout);
}
// release/destroy OpenCV objects
cvReleaseImage(&frame);
cvReleaseImage(&framescaledup);
cvReleaseImage(&frame2);
cvReleaseImage(&frame2scaledup);
cvReleaseImage(&framedual);
cvReleaseImage(&framedualscaledup);
cvReleaseImage(&gazeoverlay);
cvDestroyWindow("Gaze Overlay");
// close files
fclose(infilegazecoords);
return 0;
}
int main( int argc, char** argv )
{
char c;
// Russ: unsigned char is required for the glasses input to work properly (GDP v0).
uchar indat;
// Russ: Glasses support added here.
readuntilchar(stdin,SYMBOL_SOF);
indat = (unsigned)readchar(stdin);
assert(OPCODE_RESP_NUM_CAMS == indat);
// Russ: Read numcams byte.
indat = (unsigned)readchar(stdin);
// Russ: Really needs 2 cameras.
assert(2 == indat);
Open_GUI();
Open_Logfile(argc,argv);
Start_Timer();
int i, j;
double T[3][3], T1[3][3];
for (j = 0; j < 3; j++) {
for (i = 0; i < 3; i++) {
T[j][i] = j*3+i+1;
}
}
T[2][0] = T[2][1] = 0;
printf("\nT: \n");
for (j = 0; j < 3; j++) {
for (i = 0; i < 3; i++) {
printf("%6.2lf ", T[j][i]);
}
printf("\n");
}
affine_matrix_inverse(T, T1);
printf("\nT1: \n");
for (j = 0; j < 3; j++) {
for (i = 0; i < 3; i++) {
printf("%6.2lf ", T1[j][i]);
}
printf("\n");
}
// Russ: Value of 370 picked to support the v1 glasses frame rate, roughly.
while ((c=cvWaitKey(370))!='q') {
if (c == 's') {
sprintf(eye_file, "eye%05d.bmp", image_no);
sprintf(scene_file, "scene%05d.bmp", image_no);
image_no++;
cvSaveImage(eye_file, eye_image);
cvSaveImage(scene_file, scene_image);
printf("thres: %d\n", pupil_edge_thres);
} else if (c == 'c') {
save_image = 1 - save_image;
printf("save_image = %d\n", save_image);
} else if (c == 'e') {
save_ellipse = 1 - save_ellipse;
printf("save_ellipse = %d\n", save_ellipse);
if (save_ellipse == 1) {
Open_Ellipse_Log();
} else {
fclose(ellipse_log);
}
}
if (start_point.x == -1 && start_point.y == -1)
Grab_Camera_Frames();
else
process_image();
if (frame_number%1==0) Update_Gui_Windows();
}
Close_Logfile();
Close_GUI();
return 0;
}
