void testApp::applyGPUImageFilters(){ if (bLearnBakground == true){ grabFrameToGPU(gpuBGTex); bLearnBakground = false; } GLuint processedTex; //tmp = contrastFilter->apply(gpuBGTex); processedTex = subtractFilter->apply(gpuSourceTex, gpuBGTex); //blur if(bSmooth){ gaussHFilter->parameters["kernel_size"]->value = (float)smooth; gaussVFilter->parameters["kernel_size"]->value = (float)smooth; processedTex = gaussHFilter->apply(processedTex); processedTex = gaussVFilter->apply(processedTex); } //highpass if(bHighpass){ gaussHFilter2->parameters["kernel_size"]->value = (float)highpassBlur; gaussVFilter2->parameters["kernel_size"]->value = (float)highpassBlur; processedTex = gaussHFilter2->apply(processedTex); processedTex = gaussVFilter2->apply(processedTex); processedTex = subtractFilter2->apply(gaussVFilter->output_texture, processedTex); } //amplifys if(bAmplify){ } threshFilter->parameters["Threshold"]->value = (float)threshold / 255.0; threshFilter->apply(processedTex); //until the rest of the pipeline is fixed well just download the preprocessing result from the gpu and use that for the blob detection //TODO: make this part not super slow ;) glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, threshFilter->output_buffer); //glReadBuffer(gaussVFilter->output_buffer); glReadPixels(0,0,camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, gpuReadBackBuffer); gpuReadBackImage.setFromPixels(gpuReadBackBuffer, camWidth, camHeight); gpuReadBackImageGS = gpuReadBackImage; glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0); }
/****************************************************************************** * The update function runs continuously. Use it to update states and variables *****************************************************************************/ void ofxNCoreVision::_update(ofEventArgs &e) { if(debugMode) if((stream = freopen(fileName, "a", stdout)) == NULL){} bNewFrame = false; if(bcamera) //if camera { #ifdef TARGET_WIN32 if(PS3!=NULL)//ps3 camera { bNewFrame = PS3->isFrameNew(); } else if(ffmv!=NULL) { ffmv->grabFrame(); bNewFrame = true; } else if(vidGrabber !=NULL) { vidGrabber->grabFrame(); bNewFrame = vidGrabber->isFrameNew(); } else if(dsvl !=NULL) { bNewFrame = dsvl->isFrameNew(); } #else vidGrabber->grabFrame(); bNewFrame = vidGrabber->isFrameNew(); #endif } else //if video { vidPlayer->idleMovie(); bNewFrame = vidPlayer->isFrameNew(); } //if no new frame, return if(!bNewFrame) { return; } else//else process camera frame { ofBackground(0, 0, 0); // Calculate FPS of Camera frames++; float time = ofGetElapsedTimeMillis(); if (time > (lastFPSlog + 1000)) { fps = frames; frames = 0; lastFPSlog = time; }//End calculation float beforeTime = ofGetElapsedTimeMillis(); if (bGPUMode) { grabFrameToGPU(filter->gpuSourceTex); filter->applyGPUFilters(); contourFinder.findContours(filter->gpuReadBackImageGS, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false); if(contourFinder.bTrackFiducials) { grabFrameToGPU(filter_fiducial->gpuSourceTex); filter_fiducial->applyGPUFilters(); fidfinder.findFiducials( filter_fiducial->gpuReadBackImageGS ); } } else { grabFrameToCPU(); filter->applyCPUFilters( processedImg ); contourFinder.findContours(processedImg, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false); if(contourFinder.bTrackFiducials) { filter_fiducial->applyCPUFilters( processedImg_fiducial ); fidfinder.findFiducials( processedImg_fiducial ); } } //If Object tracking or Finger tracking is enabled if(contourFinder.bTrackFingers || contourFinder.bTrackObjects) { tracker.track(&contourFinder); } //Map Fiducials from camera to screen position if(contourFinder.bTrackFiducials) { tracker.doFiducialCalculation(); } //get DSP time differenceTime = ofGetElapsedTimeMillis() - beforeTime; //Dynamic Background subtraction LearRate if (filter->bDynamicBG) { filter->fLearnRate = backgroundLearnRate * .0001; //If there are no blobs, add the background faster. if (contourFinder.nBlobs > 0) //If there ARE blobs, add the background slower. { filter->fLearnRate = backgroundLearnRate * .0001; } }//End Background Learning rate //Sending TUIO messages if (myTUIO.bOSCMode || myTUIO.bTCPMode || myTUIO.bBinaryMode) { //printf("sending data osc : %d TCP : %d binary : %d\n", myTUIO.bOSCMode, myTUIO.bTCPMode, myTUIO.bBinaryMode); myTUIO.setMode(contourFinder.bTrackFingers , contourFinder.bTrackObjects, contourFinder.bTrackFiducials); // myTUIO.sendTUIO(&getBlobs(),&getObjects(),&fidfinder.fiducialsList); myTUIO.sendTUIO(tracker.getPtrTrackedBlobs(), tracker.getPtrTrackedObjects(), &fidfinder.fiducialsList); } } }
/****************************************************************************** * The setup function is run once to perform initializations in the application *****************************************************************************/ void testApp::setup() { /******************** * Initalize Variables *********************/ fuzzy.loadImage("particle.png"); fuzzy.setUseTexture(true); //For screengrab snapCounter = 6; //Background Subtraction Learning Rate fLearnRate = 0.0001f; //Intialize FPS variables frames = 0; fps = 0; lastFPSlog = 0; //Calibration Booleans bCalibration= false; bShowTargets = true; bW = false; bA = false; bS = false; bD = false; downColor = 0xFF0000; bDrawVideo = true; bFullscreen = false; //bFlowing = false; // ofSetBackgroundAuto(false); //Load Settings from config.xml file loadXMLSettings(); //Load Calibration Settings from calibration.xml file calibrate.setCamRes(camWidth, camHeight); calibrate.loadXMLSettings(); //Setup Window Properties ofSetWindowShape(winWidth,winHeight); ofSetFrameRate(camRate); //This will be based on camera fps in the future ofSetVerticalSync(false); //Set vertical sync to false for better performance //Pick the Source - camera or video if(bcamera){ activeInput = true; vidGrabber.listDevices(); vidGrabber.setDeviceID(deviceID); vidGrabber.setVerbose(true); vidGrabber.initGrabber(camWidth,camHeight); printf("Camera Mode\n"); int grabW = vidGrabber.width; int grabH = vidGrabber.height; printf("Asked for %i by %i - actual size is %i by %i \n", camWidth, camHeight, grabW, grabH); } else{ activeInput = true; //vidPlayer.loadMovie("test_videos/FrontDI.m4v"); //vidPlayer.loadMovie("test_videos/HCI_FTIR.mov"); vidPlayer.loadMovie("test_videos/raw.mp4"); //vidPlayer.loadMovie("test_videos/5point.avi"); vidPlayer.play(); printf("Video Mode\n"); camHeight = vidPlayer.height; camWidth = vidPlayer.width; } /***************************************************************************************************** * Allocate images (needed for drawing/processing images) ----Most of These won't be needed in the end ******************************************************************************************************/ processedImg.allocate(camWidth, camHeight); //main Image that'll be processed. processedImg.setUseTexture(false); sourceImg.allocate(camWidth, camHeight); //Source Image sourceImg.setUseTexture(false); //We don't need to draw this so don't create a texture //These images are used for drawing only grayImg.allocate(camWidth, camHeight); //Gray Image grayBg.allocate(camWidth, camHeight); //Background Image subtractBg.allocate(camWidth, camHeight); //Background After subtraction grayDiff.allocate(camWidth, camHeight); //Difference Image between Background and Source highpassImg.allocate(camWidth, camHeight); //Highpass Image ampImg.allocate(camWidth, camHeight); //Amplied Image fiLearn.allocate(camWidth, camHeight); //ofxFloatImage used for simple dynamic background subtracti // fiLearn.setUseTexture(false); pressureMap.allocate(camWidth, camHeight); //Pressure Map Image //For camera warp giWarped.allocate(camWidth, camHeight); //Warped Image (used for warped calibration) giWarped.setUseTexture(false); if(bFlowing){ //FLOW opticalFlowLK.allocate(camWidth, camHeight); opticalFlowBM.allocate(camWidth, camHeight); } /********************************************************************************************************/ /**********************************************************/ //GPU stuff initialization /**********************************************************/ glGenTextures(1, &gpuSourceTex); glGenTextures(1, &gpuBGTex); //initialize texture once with glTexImage2D so we can use gltexSubImage2D afetrwards (fastser) glEnable(GL_TEXTURE_2D); glBindTexture(GL_TEXTURE_2D, gpuSourceTex); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, camWidth, camHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL); glBindTexture(GL_TEXTURE_2D, gpuBGTex); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, camWidth, camHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL); glBindTexture(GL_TEXTURE_2D, 0); glDisable(GL_TEXTURE_2D); grabFrameToGPU(gpuBGTex); //so very inefficient..but only for now..until i fix the gpu blob detection and geoemtry shader for variable length output gpuReadBackBuffer = new unsigned char[camWidth*camHeight*3]; gpuReadBackImage.allocate(camWidth, camHeight); gpuReadBackImageGS.allocate(camWidth, camHeight); subtractFilter = new ImageFilter("filters/absSubtract.xml", camWidth, camHeight); subtractFilter2 = new ImageFilter("filters/subtract.xml", camWidth, camHeight); contrastFilter = new ImageFilter("filters/contrast.xml", camWidth, camHeight); gaussVFilter = new ImageFilter("filters/gaussV.xml", camWidth, camHeight); gaussHFilter = new ImageFilter("filters/gauss.xml", camWidth, camHeight); gaussVFilter2 = new ImageFilter("filters/gaussV2.xml", camWidth, camHeight); gaussHFilter2 = new ImageFilter("filters/gauss2.xml", camWidth, camHeight); threshFilter = new ImageFilter("filters/threshold.xml", camWidth, camHeight); /**********************************************************/ //Fonts - Is there a way to dynamically change font size? verdana.loadFont("verdana.ttf", 8, true, true); //Font used for small images sidebarTXT.loadFont("verdana.ttf", 8, true, true); calibrationText.loadFont("verdana.ttf", 10, true, true); bigvideo.loadFont("verdana.ttf", 13, true, true); //Font used for big images. //Static Images logo.loadImage("images/logo.jpg"); background.loadImage("images/background.jpg"); //Main (Temp?) Background //Setup green warped box warp_box.setup( 40, 30, camWidth, camHeight, camWidth/320, camHeight/240); warp_box.fHandles[0].x = handle1x; warp_box.fHandles[0].y = handle1y; warp_box.fHandles[1].x = handle2x; warp_box.fHandles[1].y = handle2y; warp_box.fHandles[2].x = handle3x; warp_box.fHandles[2].y = handle3y; warp_box.fHandles[3].x = handle4x; warp_box.fHandles[3].y = handle4y; //Warped points dstPts[0].x = 0.0f; dstPts[0].y = camHeight; dstPts[1].x = camWidth; dstPts[1].y = camHeight; dstPts[2].x = camWidth; dstPts[2].y = 0.0f; dstPts[3].x = 0.0f; dstPts[3].y = 0.0f; //Setup testApp to listen for touch events tracker.setListener(this); gui = ofxGui::Instance(this); setupGUI(); printf("Touchlib application is setup!\n"); }
/****************************************************************************** * The update function runs continuously. Use it to update states and variables *****************************************************************************/ void testApp::update() { bNewFrame = false; if(activeInput){ if(bcamera){ vidGrabber.grabFrame(); bNewFrame = vidGrabber.isFrameNew(); } else{ vidPlayer.idleMovie(); bNewFrame = vidPlayer.isFrameNew(); } if (bNewFrame) { ofBackground(110, 110, 110); //Calculate FPS of Camera frames++; float time = ofGetElapsedTimeMillis(); if(time > (lastFPSlog + 1000)){ fps = frames; frames = 0; lastFPSlog = time; }//End calculation if(bGPUMode){ grabFrameToGPU(gpuSourceTex); applyGPUImageFilters(); contourFinder.findContours(gpuReadBackImageGS, 1, (camWidth*camHeight)/25, 50, false); } else{ grabFrame(); applyImageFilters(); contourFinder.findContours(processedImg, 1, (camWidth*camHeight)/25, 50, false); if(bFlowing){ //FLOW grayImg.threshold(100); opticalFlowLK.calc(grayImg,processedImg,11); grayImg.blurHeavily(); opticalFlowBM.calc(grayImg,processedImg,5);} } //Track found contours/blobs tracker.track(&contourFinder); /************************************************** * Background subtraction LearRate * If there are no blobs, add the background faster. * If there ARE blobs, add the background slower. ***************************************************/ if(bDynamicBG){ fLearnRate = 0.01f; if(contourFinder.nBlobs > 0){ fLearnRate = 0.0003f; } }//End Background Learning rate if(bTUIOMode){ //We're not using frameseq right now with OSC //myTUIO.update(); //Start sending OSC myTUIO.sendOSC(); } } } }