void testApp::applyGPUImageFilters(){
if (bLearnBakground == true){
grabFrameToGPU(gpuBGTex);
bLearnBakground = false;
}
GLuint processedTex;
//tmp = contrastFilter->apply(gpuBGTex);
processedTex = subtractFilter->apply(gpuSourceTex, gpuBGTex);
//blur
if(bSmooth){
gaussHFilter->parameters["kernel_size"]->value = (float)smooth;
gaussVFilter->parameters["kernel_size"]->value = (float)smooth;
processedTex = gaussHFilter->apply(processedTex);
processedTex = gaussVFilter->apply(processedTex);
}
//highpass
if(bHighpass){
gaussHFilter2->parameters["kernel_size"]->value = (float)highpassBlur;
gaussVFilter2->parameters["kernel_size"]->value = (float)highpassBlur;
processedTex = gaussHFilter2->apply(processedTex);
processedTex = gaussVFilter2->apply(processedTex);
processedTex = subtractFilter2->apply(gaussVFilter->output_texture, processedTex);
}
//amplifys
if(bAmplify){
}
threshFilter->parameters["Threshold"]->value = (float)threshold / 255.0;
threshFilter->apply(processedTex);
//until the rest of the pipeline is fixed well just download the preprocessing result from the gpu and use that for the blob detection
//TODO: make this part not super slow ;)
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, threshFilter->output_buffer);
//glReadBuffer(gaussVFilter->output_buffer);
glReadPixels(0,0,camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, gpuReadBackBuffer);
gpuReadBackImage.setFromPixels(gpuReadBackBuffer, camWidth, camHeight);
gpuReadBackImageGS = gpuReadBackImage;
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0);
}
/******************************************************************************
* The update function runs continuously. Use it to update states and variables
*****************************************************************************/
void ofxNCoreVision::_update(ofEventArgs &e)
{
if(debugMode) if((stream = freopen(fileName, "a", stdout)) == NULL){}
bNewFrame = false;
if(bcamera) //if camera
{
#ifdef TARGET_WIN32
if(PS3!=NULL)//ps3 camera
{
bNewFrame = PS3->isFrameNew();
}
else if(ffmv!=NULL)
{
ffmv->grabFrame();
bNewFrame = true;
}
else if(vidGrabber !=NULL)
{
vidGrabber->grabFrame();
bNewFrame = vidGrabber->isFrameNew();
}
else if(dsvl !=NULL)
{
bNewFrame = dsvl->isFrameNew();
}
#else
vidGrabber->grabFrame();
bNewFrame = vidGrabber->isFrameNew();
#endif
}
else //if video
{
vidPlayer->idleMovie();
bNewFrame = vidPlayer->isFrameNew();
}
//if no new frame, return
if(!bNewFrame)
{
return;
}
else//else process camera frame
{
ofBackground(0, 0, 0);
// Calculate FPS of Camera
frames++;
float time = ofGetElapsedTimeMillis();
if (time > (lastFPSlog + 1000))
{
fps = frames;
frames = 0;
lastFPSlog = time;
}//End calculation
float beforeTime = ofGetElapsedTimeMillis();
if (bGPUMode)
{
grabFrameToGPU(filter->gpuSourceTex);
filter->applyGPUFilters();
contourFinder.findContours(filter->gpuReadBackImageGS, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false);
if(contourFinder.bTrackFiducials)
{
grabFrameToGPU(filter_fiducial->gpuSourceTex);
filter_fiducial->applyGPUFilters();
fidfinder.findFiducials( filter_fiducial->gpuReadBackImageGS );
}
}
else
{
grabFrameToCPU();
filter->applyCPUFilters( processedImg );
contourFinder.findContours(processedImg, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false);
if(contourFinder.bTrackFiducials)
{
filter_fiducial->applyCPUFilters( processedImg_fiducial );
fidfinder.findFiducials( processedImg_fiducial );
}
}
//If Object tracking or Finger tracking is enabled
if(contourFinder.bTrackFingers || contourFinder.bTrackObjects)
{
tracker.track(&contourFinder);
}
//Map Fiducials from camera to screen position
if(contourFinder.bTrackFiducials)
{
tracker.doFiducialCalculation();
}
//get DSP time
differenceTime = ofGetElapsedTimeMillis() - beforeTime;
//Dynamic Background subtraction LearRate
if (filter->bDynamicBG)
{
filter->fLearnRate = backgroundLearnRate * .0001; //If there are no blobs, add the background faster.
if (contourFinder.nBlobs > 0) //If there ARE blobs, add the background slower.
{
filter->fLearnRate = backgroundLearnRate * .0001;
}
}//End Background Learning rate
//Sending TUIO messages
if (myTUIO.bOSCMode || myTUIO.bTCPMode || myTUIO.bBinaryMode)
{
//printf("sending data osc : %d TCP : %d binary : %d\n", myTUIO.bOSCMode, myTUIO.bTCPMode, myTUIO.bBinaryMode);
myTUIO.setMode(contourFinder.bTrackFingers , contourFinder.bTrackObjects, contourFinder.bTrackFiducials);
// myTUIO.sendTUIO(&getBlobs(),&getObjects(),&fidfinder.fiducialsList);
myTUIO.sendTUIO(tracker.getPtrTrackedBlobs(), tracker.getPtrTrackedObjects(), &fidfinder.fiducialsList);
}
}
}
/******************************************************************************
* The setup function is run once to perform initializations in the application
*****************************************************************************/
void testApp::setup()
{
/********************
* Initalize Variables
*********************/
fuzzy.loadImage("particle.png");
fuzzy.setUseTexture(true);
//For screengrab
snapCounter = 6;
//Background Subtraction Learning Rate
fLearnRate = 0.0001f;
//Intialize FPS variables
frames = 0;
fps = 0;
lastFPSlog = 0;
//Calibration Booleans
bCalibration= false;
bShowTargets = true;
bW = false;
bA = false;
bS = false;
bD = false;
downColor = 0xFF0000;
bDrawVideo = true;
bFullscreen = false;
//bFlowing = false;
// ofSetBackgroundAuto(false);
//Load Settings from config.xml file
loadXMLSettings();
//Load Calibration Settings from calibration.xml file
calibrate.setCamRes(camWidth, camHeight);
calibrate.loadXMLSettings();
//Setup Window Properties
ofSetWindowShape(winWidth,winHeight);
ofSetFrameRate(camRate); //This will be based on camera fps in the future
ofSetVerticalSync(false); //Set vertical sync to false for better performance
//Pick the Source - camera or video
if(bcamera){
activeInput = true;
vidGrabber.listDevices();
vidGrabber.setDeviceID(deviceID);
vidGrabber.setVerbose(true);
vidGrabber.initGrabber(camWidth,camHeight);
printf("Camera Mode\n");
int grabW = vidGrabber.width;
int grabH = vidGrabber.height;
printf("Asked for %i by %i - actual size is %i by %i \n",
camWidth, camHeight, grabW, grabH);
}
else{
activeInput = true;
//vidPlayer.loadMovie("test_videos/FrontDI.m4v");
//vidPlayer.loadMovie("test_videos/HCI_FTIR.mov");
vidPlayer.loadMovie("test_videos/raw.mp4");
//vidPlayer.loadMovie("test_videos/5point.avi");
vidPlayer.play();
printf("Video Mode\n");
camHeight = vidPlayer.height;
camWidth = vidPlayer.width;
}
/*****************************************************************************************************
* Allocate images (needed for drawing/processing images) ----Most of These won't be needed in the end
******************************************************************************************************/
processedImg.allocate(camWidth, camHeight); //main Image that'll be processed.
processedImg.setUseTexture(false);
sourceImg.allocate(camWidth, camHeight); //Source Image
sourceImg.setUseTexture(false); //We don't need to draw this so don't create a texture
//These images are used for drawing only
grayImg.allocate(camWidth, camHeight); //Gray Image
grayBg.allocate(camWidth, camHeight); //Background Image
subtractBg.allocate(camWidth, camHeight); //Background After subtraction
grayDiff.allocate(camWidth, camHeight); //Difference Image between Background and Source
highpassImg.allocate(camWidth, camHeight); //Highpass Image
ampImg.allocate(camWidth, camHeight); //Amplied Image
fiLearn.allocate(camWidth, camHeight); //ofxFloatImage used for simple dynamic background subtracti
// fiLearn.setUseTexture(false);
pressureMap.allocate(camWidth, camHeight); //Pressure Map Image
//For camera warp
giWarped.allocate(camWidth, camHeight); //Warped Image (used for warped calibration)
giWarped.setUseTexture(false);
if(bFlowing){
//FLOW
opticalFlowLK.allocate(camWidth, camHeight);
opticalFlowBM.allocate(camWidth, camHeight);
}
/********************************************************************************************************/
/**********************************************************/
//GPU stuff initialization
/**********************************************************/
glGenTextures(1, &gpuSourceTex);
glGenTextures(1, &gpuBGTex);
//initialize texture once with glTexImage2D so we can use gltexSubImage2D afetrwards (fastser)
glEnable(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, gpuSourceTex);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, camWidth, camHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
glBindTexture(GL_TEXTURE_2D, gpuBGTex);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, camWidth, camHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glDisable(GL_TEXTURE_2D);
grabFrameToGPU(gpuBGTex);
//so very inefficient..but only for now..until i fix the gpu blob detection and geoemtry shader for variable length output
gpuReadBackBuffer = new unsigned char[camWidth*camHeight*3];
gpuReadBackImage.allocate(camWidth, camHeight);
gpuReadBackImageGS.allocate(camWidth, camHeight);
subtractFilter = new ImageFilter("filters/absSubtract.xml", camWidth, camHeight);
subtractFilter2 = new ImageFilter("filters/subtract.xml", camWidth, camHeight);
contrastFilter = new ImageFilter("filters/contrast.xml", camWidth, camHeight);
gaussVFilter = new ImageFilter("filters/gaussV.xml", camWidth, camHeight);
gaussHFilter = new ImageFilter("filters/gauss.xml", camWidth, camHeight);
gaussVFilter2 = new ImageFilter("filters/gaussV2.xml", camWidth, camHeight);
gaussHFilter2 = new ImageFilter("filters/gauss2.xml", camWidth, camHeight);
threshFilter = new ImageFilter("filters/threshold.xml", camWidth, camHeight);
/**********************************************************/
//Fonts - Is there a way to dynamically change font size?
verdana.loadFont("verdana.ttf", 8, true, true); //Font used for small images
sidebarTXT.loadFont("verdana.ttf", 8, true, true);
calibrationText.loadFont("verdana.ttf", 10, true, true);
bigvideo.loadFont("verdana.ttf", 13, true, true); //Font used for big images.
//Static Images
logo.loadImage("images/logo.jpg");
background.loadImage("images/background.jpg"); //Main (Temp?) Background
//Setup green warped box
warp_box.setup( 40, 30, camWidth, camHeight, camWidth/320, camHeight/240);
warp_box.fHandles[0].x = handle1x;
warp_box.fHandles[0].y = handle1y;
warp_box.fHandles[1].x = handle2x;
warp_box.fHandles[1].y = handle2y;
warp_box.fHandles[2].x = handle3x;
warp_box.fHandles[2].y = handle3y;
warp_box.fHandles[3].x = handle4x;
warp_box.fHandles[3].y = handle4y;
//Warped points
dstPts[0].x = 0.0f;
dstPts[0].y = camHeight;
dstPts[1].x = camWidth;
dstPts[1].y = camHeight;
dstPts[2].x = camWidth;
dstPts[2].y = 0.0f;
dstPts[3].x = 0.0f;
dstPts[3].y = 0.0f;
//Setup testApp to listen for touch events
tracker.setListener(this);
gui = ofxGui::Instance(this);
setupGUI();
printf("Touchlib application is setup!\n");
}
/******************************************************************************
* The update function runs continuously. Use it to update states and variables
*****************************************************************************/
void testApp::update()
{
bNewFrame = false;
if(activeInput){
if(bcamera){
vidGrabber.grabFrame();
bNewFrame = vidGrabber.isFrameNew();
}
else{
vidPlayer.idleMovie();
bNewFrame = vidPlayer.isFrameNew();
}
if (bNewFrame)
{
ofBackground(110, 110, 110);
//Calculate FPS of Camera
frames++;
float time = ofGetElapsedTimeMillis();
if(time > (lastFPSlog + 1000)){
fps = frames;
frames = 0;
lastFPSlog = time;
}//End calculation
if(bGPUMode){
grabFrameToGPU(gpuSourceTex);
applyGPUImageFilters();
contourFinder.findContours(gpuReadBackImageGS, 1, (camWidth*camHeight)/25, 50, false);
}
else{
grabFrame();
applyImageFilters();
contourFinder.findContours(processedImg, 1, (camWidth*camHeight)/25, 50, false);
if(bFlowing){
//FLOW
grayImg.threshold(100);
opticalFlowLK.calc(grayImg,processedImg,11);
grayImg.blurHeavily();
opticalFlowBM.calc(grayImg,processedImg,5);}
}
//Track found contours/blobs
tracker.track(&contourFinder);
/**************************************************
* Background subtraction LearRate
* If there are no blobs, add the background faster.
* If there ARE blobs, add the background slower.
***************************************************/
if(bDynamicBG){
fLearnRate = 0.01f;
if(contourFinder.nBlobs > 0){
fLearnRate = 0.0003f;
}
}//End Background Learning rate
if(bTUIOMode){
//We're not using frameseq right now with OSC
//myTUIO.update();
//Start sending OSC
myTUIO.sendOSC();
}
}
}
}
