int do_example_for_optical_flow(void)
{
/* Create an object that decodes the input video stream. */
CvCapture *input_video = cvCaptureFromFile(
//"C:\\Documents and Settings\\David Stavens\\Desktop\\223B-Demo\\optical_flow_input.avi"
//"C:\\Users\\Ran_the_User\\Documents\\Technion_Studies\\2016_A_winter\\02_Aerial_Video_PROJECT\\video-examples\\AnimalsMovingZSL17_07_14.mp4"
//"C:\\Users\\Ran_the_User\\Documents\\Technion_Studies\\2016_A_winter\\02_Aerial_Video_PROJECT\\\\AnimalsMovingZSL17_07_14.mp4"
"C:\\Users\\Ran_the_User\\Documents\\Technion_Studies\\2016_A_winter\\02_Aerial_Video_PROJECT\\video-examples\\optical_flow_input.avi"
);
if (input_video == NULL)
{
/* Either the video didn't exist OR it uses a codec OpenCV
* doesn't support.
*/
fprintf(stderr, "Error: Can't open video.\n");
return -1;
}
/* Read the video's frame size out of the AVI. */
CvSize frame_size;
frame_size.height = (int) cvGetCaptureProperty( input_video, CV_CAP_PROP_FRAME_HEIGHT );
frame_size.width = (int) cvGetCaptureProperty( input_video, CV_CAP_PROP_FRAME_WIDTH );
/* Determine the number of frames in the AVI. */
long number_of_frames;
/* Go to the end of the AVI (ie: the fraction is "1") */
cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_AVI_RATIO, 1. );
/* Now that we're at the end, read the AVI position in frames */
number_of_frames = (int) cvGetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES );
/* Return to the beginning */
cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES, 0. );
/* Create a windows called "Optical Flow" for visualizing the output.
* Have the window automatically change its size to match the output.
*/
cvNamedWindow("Optical Flow", CV_WINDOW_NORMAL); /// ran change: CV_WINDOW_AUTOSIZE);
cv::resizeWindow("Optical Flow", frame_size.width/ windowShrinkFactor, frame_size.height/ windowShrinkFactor);
long current_frame = 0;
while(true)
{
static IplImage *frame = NULL, *frame1 = NULL, *frame1_1C = NULL, *frame2_1C = NULL, *eig_image = NULL, *temp_image = NULL, *pyramid1 = NULL, *pyramid2 = NULL;
//static cvResize smaller =
// cv::resize(src, src, img.size());
/* Go to the frame we want. Important if multiple frames are queried in
* the loop which they of course are for optical flow. Note that the very
* first call to this is actually not needed. (Because the correct position
* is set outsite the for() loop.)
*/
cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES, current_frame );
/* Get the next frame of the video.
* IMPORTANT! cvQueryFrame() always returns a pointer to the _same_
* memory location. So successive calls:
* frame1 = cvQueryFrame();
* frame2 = cvQueryFrame();
* frame3 = cvQueryFrame();
* will result in (frame1 == frame2 && frame2 == frame3) being true.
* The solution is to make a copy of the cvQueryFrame() output.
*/
frame = cvQueryFrame( input_video );
if (frame == NULL)
{
/* Why did we get a NULL frame? We shouldn't be at the end. */
fprintf(stderr, "Error: Hmm. The end came sooner than we thought.\n");
return -1;
}
/* Allocate another image if not already allocated.
* Image has ONE channel of color (ie: monochrome) with 8-bit "color" depth.
* This is the image format OpenCV algorithms actually operate on (mostly).
*/
allocateOnDemand( &frame1_1C, frame_size, IPL_DEPTH_8U, 1 );
/* Convert whatever the AVI image format is into OpenCV's preferred format.
* AND flip the image vertically. Flip is a shameless hack. OpenCV reads
* in AVIs upside-down by default. (No comment :-))
*/
cvConvertImage(frame, frame1_1C, 0);// CV_CVTIMG_FLIP);
/* We'll make a full color backup of this frame so that we can draw on it.
* (It's not the best idea to draw on the static memory space of cvQueryFrame().)
*/
allocateOnDemand( &frame1, frame_size, IPL_DEPTH_8U, 3 );
cvConvertImage(frame, frame1, 0);// CV_CVTIMG_FLIP);
/* Get the second frame of video. Same principles as the first. */
frame = cvQueryFrame( input_video );
if (frame == NULL)
{
fprintf(stderr, "Error: Hmm. The end came sooner than we thought.\n");
return -1;
}
allocateOnDemand( &frame2_1C, frame_size, IPL_DEPTH_8U, 1 );
cvConvertImage(frame, frame2_1C, 0);// CV_CVTIMG_FLIP);
/* Shi and Tomasi Feature Tracking! */
/* Preparation: Allocate the necessary storage. */
allocateOnDemand( &eig_image, frame_size, IPL_DEPTH_32F, 1 );
allocateOnDemand( &temp_image, frame_size, IPL_DEPTH_32F, 1 );
/* Preparation: This array will contain the features found in frame 1. */
CvPoint2D32f frame1_features[NUM_OF_FEATURES];
/* Preparation: BEFORE the function call this variable is the array size
* (or the maximum number of features to find). AFTER the function call
* this variable is the number of features actually found.
*/
int number_of_features;
/* I'm hardcoding this at 400. But you should make this a #define so that you can
* change the number of features you use for an accuracy/speed tradeoff analysis.
*/
number_of_features = NUM_OF_FEATURES;
/* Actually run the Shi and Tomasi algorithm!!
* "frame1_1C" is the input image.
* "eig_image" and "temp_image" are just workspace for the algorithm.
* The first ".01" specifies the minimum quality of the features (based on the eigenvalues).
* The second ".01" specifies the minimum Euclidean distance between features.
* "NULL" means use the entire input image. You could point to a part of the image.
* WHEN THE ALGORITHM RETURNS:
* "frame1_features" will contain the feature points.
* "number_of_features" will be set to a value <= 400 indicating the number of feature points found.
*/
cvGoodFeaturesToTrack(frame1_1C, eig_image, temp_image, frame1_features, &number_of_features, .01, .01, NULL);
/* Pyramidal Lucas Kanade Optical Flow! */
/* This array will contain the locations of the points from frame 1 in frame 2. */
CvPoint2D32f frame2_features[NUM_OF_FEATURES];
/* The i-th element of this array will be non-zero if and only if the i-th feature of
* frame 1 was found in frame 2.
*/
char optical_flow_found_feature[NUM_OF_FEATURES];
/* The i-th element of this array is the error in the optical flow for the i-th feature
* of frame1 as found in frame 2. If the i-th feature was not found (see the array above)
* I think the i-th entry in this array is undefined.
*/
float optical_flow_feature_error[NUM_OF_FEATURES];
/* This is the window size to use to avoid the aperture problem (see slide "Optical Flow: Overview"). */
CvSize optical_flow_window = cvSize(3,3);
/* This termination criteria tells the algorithm to stop when it has either done 20 iterations or when
* epsilon is better than .3. You can play with these parameters for speed vs. accuracy but these values
* work pretty well in many situations.
*/
CvTermCriteria optical_flow_termination_criteria
= cvTermCriteria( CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, .3 );
/* This is some workspace for the algorithm.
* (The algorithm actually carves the image into pyramids of different resolutions.)
*/
allocateOnDemand( &pyramid1, frame_size, IPL_DEPTH_8U, 1 );
allocateOnDemand( &pyramid2, frame_size, IPL_DEPTH_8U, 1 );
/* Actually run Pyramidal Lucas Kanade Optical Flow!!
* "frame1_1C" is the first frame with the known features.
* "frame2_1C" is the second frame where we want to find the first frame's features.
* "pyramid1" and "pyramid2" are workspace for the algorithm.
* "frame1_features" are the features from the first frame.
* "frame2_features" is the (outputted) locations of those features in the second frame.
* "number_of_features" is the number of features in the frame1_features array.
* "optical_flow_window" is the size of the window to use to avoid the aperture problem.
* "5" is the maximum number of pyramids to use. 0 would be just one level.
* "optical_flow_found_feature" is as described above (non-zero iff feature found by the flow).
* "optical_flow_feature_error" is as described above (error in the flow for this feature).
* "optical_flow_termination_criteria" is as described above (how long the algorithm should look).
* "0" means disable enhancements. (For example, the second array isn't pre-initialized with guesses.)
*/
cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2, frame1_features, frame2_features, number_of_features, optical_flow_window, 5, optical_flow_found_feature, optical_flow_feature_error, optical_flow_termination_criteria, 0 );
/* For fun (and debugging :)), let's draw the flow field. */
for(int i = 0; i < number_of_features; i++)
{
/* If Pyramidal Lucas Kanade didn't really find the feature, skip it. */
if ( optical_flow_found_feature[i] == 0 ) continue;
int line_thickness; line_thickness = 1;
/* CV_RGB(red, green, blue) is the red, green, and blue components
* of the color you want, each out of 255.
*/
CvScalar line_color; line_color = CV_RGB(255,0,250);
/* Let's make the flow field look nice with arrows. */
/* The arrows will be a bit too short for a nice visualization because of the high framerate
* (ie: there's not much motion between the frames). So let's lengthen them by a factor of 3.
*/
CvPoint p,q;
p.x = (int) frame1_features[i].x;
p.y = (int) frame1_features[i].y;
q.x = (int) frame2_features[i].x;
q.y = (int) frame2_features[i].y;
double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x );
double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) );
/* Here we lengthen the arrow by a factor of three. */
q.x = (int) (p.x - 3 * hypotenuse * cos(angle));
q.y = (int) (p.y - 3 * hypotenuse * sin(angle));
/* Now we draw the main line of the arrow. */
/* "frame1" is the frame to draw on.
* "p" is the point where the line begins.
* "q" is the point where the line stops.
* "CV_AA" means antialiased drawing.
* "0" means no fractional bits in the center cooridinate or radius.
*/
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
/* Now draw the tips of the arrow. I do some scaling so that the
* tips look proportional to the main line of the arrow.
*/
p.x = (int) (q.x + 9 * cos(angle + pi / 4));
p.y = (int) (q.y + 9 * sin(angle + pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
p.x = (int) (q.x + 9 * cos(angle - pi / 4));
p.y = (int) (q.y + 9 * sin(angle - pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
}
/* Now display the image we drew on. Recall that "Optical Flow" is the name of
* the window we created above.
*/
cvShowImage("Optical Flow", frame1);
/* And wait for the user to press a key (so the user has time to look at the image).
* If the argument is 0 then it waits forever otherwise it waits that number of milliseconds.
* The return value is the key the user pressed.
*/
int key_pressed;
key_pressed = cvWaitKey(1); //0
/* If the users pushes "b" or "B" go back one frame.
* Otherwise go forward one frame.
*/
if (key_pressed == 'b' || key_pressed == 'B') current_frame--;
else current_frame++;
/* Don't run past the front/end of the AVI. */
if (current_frame < 0) current_frame = 0;
if (current_frame >= number_of_frames - 1) current_frame = number_of_frames - 2;
if (key_pressed == 27) break;
}
}
void MouthContours::execute(IplImage* img, IplImage* drw, CvRect mouthSearch){
CvSeq* contours;
if(CV_IS_IMAGE(imgGrey)){
cvReleaseImage(&imgGrey);
}
if(CV_IS_IMAGE(imgTempl)){
cvReleaseImage(&imgTempl);
}
allocateOnDemand( &storageTeeth );
allocateOnDemand( &imgTempl, cvSize( img->width, img->height ), IPL_DEPTH_8U, 3 );
cvCopy( img, imgTempl, 0 );
allocateOnDemand( &imgGrey, cvSize( img->width, img->height ), IPL_DEPTH_8U, 1 );
if(CV_IS_STORAGE((storageTeeth))){
contours = cvCreateSeq( CV_SEQ_KIND_GENERIC|CV_32SC2, sizeof(CvContour), sizeof(CvPoint), storageTeeth );
cvCvtColor( imgTempl, imgGrey, CV_BGR2GRAY );
int sigma = 1;
int ksize = (sigma*5)|1;
cvSetImageROI(imgGrey, mouthSearch);
cvSetImageROI(drw, mouthSearch);
cvSmooth( imgGrey , imgGrey, CV_GAUSSIAN, ksize, ksize, sigma, sigma);
//cvEqualizeHist( small_img_grey, small_img_grey );
cvCanny( imgGrey, imgGrey, 70, 70, 3 );
cvDilate( imgGrey, imgGrey, NULL, 1 );
cvErode( imgGrey, imgGrey, NULL, 1 );
cvFindContours( imgGrey, storageTeeth, &contours, sizeof(CvContour), CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, cvPoint(0,0) );
if(CV_IS_SEQ(contours)){
contours = cvApproxPoly( contours, sizeof(CvContour), storageTeeth, CV_POLY_APPROX_DP, 5, 1 );
if( contours->total > 0 ){
for( ;contours; contours = contours->h_next ){
if( contours->total < 4 )
continue;
cvDrawContours( drw, contours, CV_RGB(255,0,0), CV_RGB(0,255,0), 5, 1, CV_AA, cvPoint(0,0) );
MouthContours::TeethArcLength = cvArcLength( contours, CV_WHOLE_SEQ, -1);
MouthContours::TeethAreaContour = cvContourArea( contours, CV_WHOLE_SEQ);
time_t ltime;
struct tm *Tm;
ltime=time(NULL);
Tm=localtime(<ime);
MouthContours::MouthHH = Tm->tm_hour;
MouthContours::MouthMM = Tm->tm_min;
MouthContours::MouthSS = Tm->tm_sec;
}
}else{
MouthContours::MouthHH = 0;
MouthContours::MouthMM = 0;
MouthContours::MouthSS = 0;
MouthContours::TeethArcLength = 0;
MouthContours::TeethAreaContour = 0;
}
}else{
MouthContours::MouthHH = 0;
MouthContours::MouthMM = 0;
MouthContours::MouthSS = 0;
MouthContours::TeethArcLength = 0;
MouthContours::TeethAreaContour = 0;
}
cvClearMemStorage( storageTeeth );
}
cvResetImageROI(imgGrey);
cvResetImageROI(drw);
}
int main(void)
{
/* Input video dari kamera */
CvCapture* input_video;
input_video = cvCaptureFromCAM(CV_CAP_ANY);
/* cek keserdiaan kamera */
if (input_video == NULL)
{
fprintf(stderr, "Error: Kamera tidak terdeteksi.\n");
return -1;
}
/* mengambil frame dari video */
cvQueryFrame( input_video );
/* mengambil properti dari video */
CvSize frame_size;
frame_size.height = (int) cvGetCaptureProperty( input_video, CV_CAP_PROP_FRAME_HEIGHT );
frame_size.width = (int) cvGetCaptureProperty( input_video, CV_CAP_PROP_FRAME_WIDTH );
/* membuat window baru bernama optical flow */
cvNamedWindow("Optical Flow", CV_WINDOW_AUTOSIZE);
long current_frame = 0;
while(1)
{
static IplImage *frame = NULL, *frame1 = NULL, *frame1_1C = NULL, *frame2_1C = NULL, *eig_image = NULL, *temp_image = NULL, *pyramid1 = NULL, *pyramid2 = NULL;
/* mendapatkan frame selanjutnya */
frame = cvQueryFrame( input_video );
/* mengalokasikan gambar */
allocateOnDemand( &frame1_1C, frame_size, IPL_DEPTH_8U, 1 );
cvConvertImage(frame, frame1_1C, 0);
allocateOnDemand( &frame1, frame_size, IPL_DEPTH_8U, 3 );
cvConvertImage(frame, frame1, 0);
/* mendapatkan frame ke dua */
frame = cvQueryFrame( input_video );
allocateOnDemand( &frame2_1C, frame_size, IPL_DEPTH_8U, 1 );
cvConvertImage(frame, frame2_1C, 0);
/* Shi and Tomasi Feature Tracking! */
/* mengalokasikan gambar */
allocateOnDemand( &eig_image, frame_size, IPL_DEPTH_32F, 1 );
allocateOnDemand( &temp_image, frame_size, IPL_DEPTH_32F, 1 );
/* Preparation: This array will contain the features found in frame 1. */
CvPoint2D32f frame1_features[400];
/* menginisialisasi jumlah feature / garis panah */
int number_of_features;
number_of_features = 400;
/* menjalankan algoritma Shi dan Tomasi */
cvGoodFeaturesToTrack(frame1_1C, eig_image, temp_image, frame1_features, & number_of_features, .01, .01, NULL);
/* Pyramidal Lucas Kanade Optical Flow! */
/* menyimpan lokasi poin dari frame 1 di frame 2 dalam array */
CvPoint2D32f frame2_features[400];
char optical_flow_found_feature[400];
float optical_flow_feature_error[400];
CvSize optical_flow_window = cvSize(3,3);
CvTermCriteria optical_flow_termination_criteria = cvTermCriteria( CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, .3 );
allocateOnDemand( &pyramid1, frame_size, IPL_DEPTH_8U, 1 );
allocateOnDemand( &pyramid2, frame_size, IPL_DEPTH_8U, 1 );
/* menjalakan Pyramidal Lucas Kanade Optical Flow */
cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2, frame1_features, frame2_features, number_of_features, optical_flow_window, 5, optical_flow_found_feature, optical_flow_feature_error, optical_flow_termination_criteria, 0 );
/* membuat panah */
for(int i = 0; i < number_of_features; i++)
{
/* skip bila tidak ada feature */
if ( optical_flow_found_feature[i] == 0 ) continue;
int line_thickness; line_thickness = 1;
/* warna garis */
CvScalar line_color; line_color = CV_RGB(255,0,0);
/* menggambarkan panah */
CvPoint p,q;
p.x = (int) frame1_features[i].x;
p.y = (int) frame1_features[i].y;
q.x = (int) frame2_features[i].x;
q.y = (int) frame2_features[i].y;
double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x );
double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) );
q.x = (int) (p.x - 3 * hypotenuse * cos(angle));
q.y = (int) (p.y - 3 * hypotenuse * sin(angle));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
p.x = (int) (q.x + 9 * cos(angle + pi / 4));
p.y = (int) (q.y + 9 * sin(angle + pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
p.x = (int) (q.x + 9 * cos(angle - pi / 4));
p.y = (int) (q.y + 9 * sin(angle - pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
}
/* menampilkan gambar */
cvShowImage("Optical Flow", frame1);
/* keluar */
int key_pressed;
key_pressed = cvWaitKey(10);
if (key_pressed == 27) break;
current_frame++;
}
void MainWindow::optical_flow()
{
cvReleaseCapture(&pCapture);
CvCapture *pCapture = cvCaptureFromFile( "/home/kevin/optical_flow_input.avi" );
if (pCapture == NULL)
{
fprintf(stderr, "Error: Can't open video.\n");
}
/* Read the video's frame size out of the AVI. */
CvSize frame_size;
frame_size.height = (int) cvGetCaptureProperty( pCapture, CV_CAP_PROP_FRAME_HEIGHT );
frame_size.width = (int) cvGetCaptureProperty( pCapture, CV_CAP_PROP_FRAME_WIDTH );
/* Determine the number of frames in the AVI. */
long number_of_frames;
/* Go to the end of the AVI (ie: the fraction is "1") */
cvSetCaptureProperty( pCapture, CV_CAP_PROP_POS_AVI_RATIO, 1. );
/* Now that we're at the end, read the AVI position in frames */
number_of_frames = (int) cvGetCaptureProperty( pCapture, CV_CAP_PROP_POS_FRAMES );
/* Return to the beginning */
cvSetCaptureProperty( pCapture, CV_CAP_PROP_POS_FRAMES, 0. );
while(true)
{
static IplImage *frame = NULL, *frame1 = NULL, *frame1_1C = NULL;
static IplImage *frame2_1C = NULL, *eig_image = NULL, *temp_image = NULL;
static IplImage *pyramid1 = NULL, *pyramid2 = NULL;
cvSetCaptureProperty( pCapture, CV_CAP_PROP_POS_FRAMES, current_frame );
frame = cvQueryFrame( pCapture );
if (frame == NULL)
{
/* Why did we get a NULL frame? We shouldn't be at the end. */
fprintf(stderr, "Error: Hmm. The end came sooner than we thought.\n");
}
allocateOnDemand( &frame1_1C, frame_size, IPL_DEPTH_8U, 1 );
cvConvertImage(frame, frame1_1C, CV_CVTIMG_FLIP);
allocateOnDemand( &frame1, frame_size, IPL_DEPTH_8U, 3 );
cvConvertImage(frame, frame1, CV_CVTIMG_FLIP);
/* Get the second frame of video. Same principles as the first. */
frame = cvQueryFrame( pCapture );
if (frame == NULL)
{
fprintf(stderr, "Error: Hmm. The end came sooner than we thought.\n");
}
allocateOnDemand( &frame2_1C, frame_size, IPL_DEPTH_8U, 1 );
cvConvertImage(frame, frame2_1C, CV_CVTIMG_FLIP);
/* Preparation: Allocate the necessary storage. */
allocateOnDemand( &eig_image, frame_size, IPL_DEPTH_32F, 1 );
allocateOnDemand( &temp_image, frame_size, IPL_DEPTH_32F, 1 );
/* Preparation: This array will contain the features found in frame 1. */
CvPoint2D32f frame1_features[400];
int number_of_features;
number_of_features = 400;
cvGoodFeaturesToTrack(frame1_1C, eig_image, temp_image, frame1_features,
&number_of_features, .01, .01, NULL);
/* Pyramidal Lucas Kanade Optical Flow! */
/* This array will contain the locations of the points from frame 1 in frame 2. */
CvPoint2D32f frame2_features[400];
char optical_flow_found_feature[400];
float optical_flow_feature_error[400];
/* This is the window size to use to avoid the aperture problem (see slide "Optical Flow: Overview"). */
CvSize optical_flow_window = cvSize(3,3);
CvTermCriteria optical_flow_termination_criteria
= cvTermCriteria( CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, .3 );
/* This is some workspace for the algorithm.
* (The algorithm actually carves the image into pyramids of different resolutions.)
*/
allocateOnDemand( &pyramid1, frame_size, IPL_DEPTH_8U, 1 );
allocateOnDemand( &pyramid2, frame_size, IPL_DEPTH_8U, 1 );
cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2,
frame1_features, frame2_features, number_of_features,
optical_flow_window, 5, optical_flow_found_feature,
optical_flow_feature_error, optical_flow_termination_criteria, 0 );
/* For fun (and debugging :)), let's draw the flow field. */
for(int i = 0; i < number_of_features; i++)
{
/* If Pyramidal Lucas Kanade didn't really find the feature, skip it. */
if ( optical_flow_found_feature[i] == 0 ) continue;
int line_thickness;
line_thickness = 1;
/* CV_RGB(red, green, blue) is the red, green, and blue components
* of the color you want, each out of 255.
*/
CvScalar line_color;
line_color = CV_RGB(255,0,0);
CvPoint p,q;
p.x = (int) frame1_features[i].x;
p.y = (int) frame1_features[i].y;
q.x = (int) frame2_features[i].x;
q.y = (int) frame2_features[i].y;
double angle;
angle = atan2( (double) p.y - q.y, (double) p.x - q.x );
double hypotenuse;
hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) );
/* Here we lengthen the arrow by a factor of three. */
q.x = (int) (p.x - 3 * hypotenuse * cos(angle));
q.y = (int) (p.y - 3 * hypotenuse * sin(angle));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
/* Now draw the tips of the arrow. I do some scaling so that the
* tips look proportional to the main line of the arrow.
*/
p.x = (int) (q.x + 9 * cos(angle + pi / 4));
p.y = (int) (q.y + 9 * sin(angle + pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
p.x = (int) (q.x + 9 * cos(angle - pi / 4));
p.y = (int) (q.y + 9 * sin(angle - pi / 4));
cvLine( frame1, p, q, line_color, line_thickness, CV_AA, 0 );
}
cv::Mat frame1_mat=MainWindow::image_rotate(frame1);
cv::flip(frame1_mat,frame1_mat,1);
IplImage dst = frame1_mat;
cv::Mat frame2_mat=MainWindow::image_rotate(frame2_1C);
cv::flip(frame2_mat,frame2_mat,1);
IplImage dst1 = frame2_mat;
MainWindow::Display(frame,&dst,&dst1);
int key_pressed;
key_pressed = cvWaitKey(0);
if (key_pressed == 'b' || key_pressed == 'B')
current_frame--;
else
current_frame++;
/* Don't run past the front/end of the AVI. */
if (current_frame < 0)
current_frame = 0;
if (current_frame >= number_of_frames - 1)
current_frame = number_of_frames - 2;
}
}
int main( int argc, char** argv ) {
int cmd;
float desired[8][8] = { {1,0,0,0,0,0,0,0}, // 0
{0,1,0,0,0,0,0,0}, // 1
{0,0,1,0,0,0,0,0}, // 2
{0,0,0,1,1,0,0,0}, // 3 forward
{0,0,0,1,1,0,0,0}, // 4 forward
{0,0,0,0,0,1,0,0}, // 5
{0,0,0,0,0,0,1,0}, // 6
{0,0,0,0,0,0,0,1}}; // 7
CvPoint p,q;
CvScalar line_color = CV_RGB(0,0,255);
CvScalar out_color;
const char* name_orig = "Original: press q to save images";
const char* name_ave = "input";
const char* name_weights = "weights";
const char* inputCmdFile_name = "./inputs/dataset1/commandlist.dat";
const char* outputFile_name = "./outputs/output_";
FILE* outputFile;
FILE* inputCmdFile;
char inputName[64];
char outputName[64];
CvCapture* capture = cvCreateCameraCapture(0) ;
CvSize frame_size;
CvScalar ave = cvScalar(1);
CvRect slice_rect;
CvSize slice_size;
static IplImage* frame = NULL;
static IplImage* frame_g = NULL;
static IplImage* frame_small = NULL;
static IplImage* frame_weights = NULL;
static IplImage* frame_w_big = NULL;
static IplImage* frame_w_final = NULL;
static IplImage* frame_final = NULL;
static IplImage* ave_image = NULL;
// static IplImage *scale = NULL;
static IplImage* frame_slices[N_SLICES];
float inputs[(SIZE/N_SLICES)*SIZE];
float outputs[N_SLICES];
int choices[N_SLICES];
// float desired[N_SLICES];
// float desired[] = {0,0,0,1,1,0,0,0}; //XXX dummy test...
//Evo (int nNets, int nInputs, int nHidden, int nOuts)
Evo* evoSlice;
int ep;
int trial;
int stepCnt;
int flag = 0;
char c;
int i,j,k,s;
float tmp;
////////////////////////////////////////////////////////////////////////////////
// init stuff
inputCmdFile = fopen(inputCmdFile_name,"r");
if (inputCmdFile == NULL) {printf("Unable to open: %s",inputCmdFile_name); return 0; }
// create windows for looking at stuff
// cvNamedWindow( name_slice, CV_WINDOW_AUTOSIZE );
cvNamedWindow( name_weights, CV_WINDOW_AUTOSIZE );
cvNamedWindow( name_ave, CV_WINDOW_AUTOSIZE );
cvNamedWindow( name_orig, CV_WINDOW_AUTOSIZE );
// frame_size = cvSize(frame->width,frame->height);
frame_size = cvSize(SIZE,SIZE);
#ifdef USECAM
// capture a frame so we can get an idea of the size of the source
frame = cvQueryFrame( capture );
if( !frame ) return 0;
#else
sprintf(inputName,"./inputs/dataset1/image0000000000.jpg");
frame = cvLoadImage(inputName, 0 );
if( !frame ){ printf("ERROR OPENING: %s!!!\n",inputName); return 0;}
#endif
allocateOnDemand( &frame_g, cvSize(frame->width,frame->height), IPL_DEPTH_8U, 1 );
allocateOnDemand( &frame_w_big, cvSize(frame->width,frame->height), IPL_DEPTH_8U, 1 );
allocateOnDemand( &frame_w_final, cvSize(frame->width,frame->height), IPL_DEPTH_8U, 3 );
allocateOnDemand( &frame_final, cvSize(frame->width,frame->height+20), IPL_DEPTH_8U, 3 );
allocateOnDemand( &ave_image, frame_size, IPL_DEPTH_8U, 1 );
allocateOnDemand( &frame_small, frame_size, IPL_DEPTH_8U, 1 );
allocateOnDemand( &frame_weights, frame_size, IPL_DEPTH_8U, 1 );
slice_size = cvSize(ave_image->width/N_SLICES, ave_image->height);
for (i=0;i<N_SLICES;i++) {
allocateOnDemand( &frame_slices[i], slice_size, IPL_DEPTH_8U, 1);
}
for(trial=0;trial<N_TRIALS;trial++) {
sprintf(outputName,"%s%d.txt", outputFile_name, trial);
outputFile = fopen(outputName,"w");
// init each leariner
evoSlice = (Evo*)malloc(sizeof(Evo)*N_SLICES);
for(i=0;i<N_SLICES;i++) {
evoSlice[i] = Evo(N_NETS, (SIZE/N_SLICES)*SIZE, N_HIDDEN, 1);
evoSlice[i].choose();
choices[i] = evoSlice[i].choose();
}
ep = 0;
stepCnt = 0;
flag = 0;
while(1) {
////////////////////////////////////////////////////////////////////////////////
// Pre processing
#if 0
// make blank image...
cvSet(ave_image, cvScalar(0));
for (i=0;i<NF;i++) {
// get image
#ifdef USECAM
frame = cvQueryFrame( capture );
if( !frame ) break;
#else
sprintf(inputName,"./inputs/dataset1/image%010d.jpg",stepCnt);
frame = cvLoadImage(inputName, 0 );
if( !frame ){ printf("ERROR OPENING: %s!!!\n",inputName); return 0;}
stepCnt++;
#endif
// convert it to grey
cvConvertImage(frame, frame_g );//, CV_CVTIMG_FLIP);
// resize
cvResize(frame_g, frame_small);
// take difference
cvSub(frame_small, ave_image, ave_image);
}
for(j=0;j<SIZE;j++) {
for(k=0;k<SIZE;k++) {
PIX(ave_image,k,j) = (char)(PIX(ave_image,k,j)*10);
}
}
#endif
#if 0
frame = cvQueryFrame( capture );
if( !frame ) break;
cvConvertImage(frame, frame_g );
cvResize(frame_g, frame_small);
cvConvertImage(frame_small, ave_image );
#endif
#if 1
sprintf(inputName,"./inputs/dataset1/image%010d.jpg",stepCnt);
frame = cvLoadImage(inputName, 0 );
if( !frame ){ printf("ERROR OPENING: %s!!!\n",inputName); break;}
cvConvertImage(frame, frame_g );
cvResize(frame_g, frame_small);
cvConvertImage(frame_small, ave_image );
// cvCanny(ave_image, ave_image, 50, 40,5);
#endif
// cvDilate(ave_image, ave_image,NULL,4);
////////////////////////////////////////////////////////////////////////////////
// Generate NN inputs
// slice it up
for (i=0;i<N_SLICES;i++) {
slice_rect = cvRect(i*ave_image->width/N_SLICES, 0, ave_image->width/N_SLICES, ave_image->height);
cvSetImageROI(ave_image, slice_rect);
cvCopy(ave_image, frame_slices[i], NULL);
}
cvResetImageROI(ave_image); // remove this when we don't care about looking at the ave
////////////////////////////////////////////////////////////////////////////////
// Evaluate NN
if (stepCnt == N_LEARN)
flag = 1;
if( (flag == 1) && (stepCnt%N_LEARN == 0)) { // every N_LEARN images switch
ep++;
fprintf(outputFile,"%d",ep);
for(i=0;i<N_SLICES;i++) {
evoSlice[i].replace();
choices[i] = evoSlice[i].choose();
fprintf(outputFile,"\t%1.3f",evoSlice[i].netPool[evoSlice[i].best()].grade);
}
fprintf(outputFile,"\n");
if(ep >= N_EPISODES) break;
// draw weights image
for(s=0;s<N_SLICES;s++) {
for(j=0;j<SIZE;j++) {
for(k=0;k<(SIZE/N_SLICES);k++) {
tmp = 0;
for(i=0;i<N_HIDDEN;i++) {
tmp += evoSlice[s].mutant->nodeHidden->w[(j*(SIZE/N_SLICES))+k+1];
}
PIX(frame_weights,k+(s*SIZE/N_SLICES),j) = (char)((tmp/N_HIDDEN)*255 + 127);
// printf("%d\t",(char)((tmp/N_HIDDEN)*255));
}
}
}
cvResize(frame_weights, frame_w_big, CV_INTER_LINEAR);
cvConvertImage(frame_w_big, frame_w_final);
}
fscanf(inputCmdFile,"%d",&cmd);
printf("\nTrial: %d Episode: %d Devin's cmd: %d\n",trial,ep,cmd);
for(i=0;i<N_SLICES;i++)
printf("%1.3f\t",desired[cmd][i]);
printf("\n");
for(i=0;i<N_SLICES;i++) {
// cvShowImage( name_slice, frame_slices[i] );
// strip pixel data into a single array
for(j=0;j<SIZE;j++) {
for(k=0;k<(SIZE/N_SLICES);k++) {
inputs[(j*(SIZE/N_SLICES))+k] = (float)PIX(frame_slices[i],k,j)/255.0;
}
}
// printf("\n%d: Eval slice %d\n",stepCnt,i);
outputs[i] = evoSlice[i].eval(inputs, &desired[cmd][i]);
// outputs[i] = desired[i];
printf("%1.3f\t",outputs[i]);
}
printf("\n");
for(i=0;i<N_SLICES;i++) {
printf("%d\t",choices[i]);
}
printf("\n");
for(i=0;i<N_SLICES;i++) {
printf("%1.3f\t",evoSlice[i].mutant->grade);
}
printf("\n");
////////////////////////////////////////////////////////////////////////////////
// GUI stuff
// copy input image into larger final image
cvSetImageROI(frame_final, cvRect(0, 0, frame_w_big->width, frame_w_big->height));
cvConvertImage(frame, frame_final);
cvResetImageROI(frame_final);
// draw slice markers
for(i=1;i<N_SLICES;i++) {
// on the final frame...
p.x = (int)(i*frame_final->width/N_SLICES);
p.y = 0;
q.x = p.x;
q.y = (int)frame_final->height;
cvLine( frame_final, p, q, line_color, 2, CV_AA, 0 );
// on the weights
p.x = (int)(i*frame_w_final->width/N_SLICES);
p.y = 0;
q.x = p.x;
q.y = (int)frame_w_final->height;
cvLine( frame_w_final, p, q, line_color, 2, CV_AA, 0 );
}
// draw output indicators
for(i=0;i<N_SLICES;i++) {
out_color = CV_RGB(outputs[i]*255,0,0);
p.x = (int)(i*frame_final->width/N_SLICES);
p.y = (int)(frame_final->height-20);
q.x = (int)(p.x+frame_final->width/N_SLICES);
q.y = (int)(p.y+20);
cvRectangle( frame_final, p, q, out_color, CV_FILLED, CV_AA, 0 );
}
cvShowImage( name_ave, ave_image );
cvShowImage( name_orig, frame_final );
cvShowImage( name_weights, frame_w_final );
c = cvWaitKey(2);
if( c == 27 ) break;
else if( c == 'q') {
cvSaveImage("weights.jpg",frame_w_final);
cvSaveImage("output.jpg",frame_final);
}
stepCnt++;
if (stepCnt>=(N_STEPS-(N_STEPS%N_LEARN))) {
stepCnt=0;
rewind(inputCmdFile);
}
} // end while
free(evoSlice);
fclose(outputFile);
} // end trial for
////////////////////////////////////////////////////////////////////////////////
// clean up
// delete &evo;
fclose(inputCmdFile);
cvReleaseCapture( &capture );
cvDestroyWindow( name_ave );
cvDestroyWindow( name_orig );
cvDestroyWindow( name_weights );
}
