Beispiel #1
0
void Kinect::KinectHandTracker::getAllHands()
{
	// List of hands evidence
	// If hand matches old positions, previous ID is assigned again
	const nite::Array<nite::HandData>& hands= this->handTrackerFrame.getHands();

	//printf("%d hands\n", hands.getSize());
	this->isTwoHands = false;

	for (int i = 0; i < hands.getSize(); ++i)
	{
		const nite::HandData& user = hands[i];

		if (!user.isTracking())
		{
			//printf("Lost hand %d\n", user.getId());
			nite::HandId id = user.getId();
			HistoryBuffer<20>* pHistory = this->g_histories[id];
			this->g_histories.erase(this->g_histories.find(id));
			delete pHistory;
		}
		else
		{
			if (user.isNew())
			{
				//printf("New hand id %d\n", user.getId());
				this->g_histories[user.getId()] = new HistoryBuffer<20>;
			}
			// Hand evidence in Buffer
			HistoryBuffer<20>* pHistory = this->g_histories[user.getId()];
			pHistory->AddPoint(user.getPosition());

			// Data for mouse
			//first find HAND = MOUSE
			if(i==0 && isCursorMovementEnable)
			{
				mouse->setSpeedUpMoving(mSpeed);
				//mouse->moveCursorMouse(user.getPosition().x/2,-1.0*user.getPosition().y/2,isClick);
				coordinateConverter.convertWorldToDepth(*mDepth, user.getPosition().x, user.getPosition().y, user.getPosition().z, &mDepthX, &mDepthY, &mDepthZ);
				mouse->moveCursorWorldCoordinates(mDepthX,mDepthY,isClick);
			}
			// TODO - further implementation should include depth information in pixels
			this->handZ[i] = user.getPosition().z;

			this->getArrayHands[i][0] = user.getPosition().x;
			this->getArrayHands[i][1] = 0 - user.getPosition().y;
			// If two hands have been found get the position of the rectangle
			if (hands.getSize() == 2)
			{
				printf("two hands found\n");
				// get positions for both hands



				this->isTwoHands = true;
			}
		}
	}
}
Beispiel #2
0
void SampleViewer::Display()
{

	nite::Status rc = m_pHandTracker->readFrame(&handFrame);
	if (rc != nite::STATUS_OK)
	{
		printf("GetNextData failed\n");
		return;
	}

	depthFrame = handFrame.getDepthFrame();

	if (m_pTexMap == NULL)
	{
		// Texture map init
		m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE);
		m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE);
		m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY];
	}

	glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	glEnable(GL_DEPTH_TEST);
	glMatrixMode(GL_PROJECTION);
	glPushMatrix();
	glLoadIdentity();
	glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -10000.0, 10000.0);



	if (depthFrame.isValid())
	{
		calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
	}

	memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel));

	float factor[3] = {1, 1, 1};
	// check if we need to draw depth frame to texture
	
	float av_x = 0;
	float av_y = 0;
	int counter= 0;



	for(int i = 0; i<=7 ; i++)
	      note_on[i] = false;

	if (depthFrame.isValid() && g_drawDepth)
	{
		const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData();
		const openni::DepthPixel* pDepthRow1 = pDepthRow;
		openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX;
		int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel);
		glPointSize(2);
		glBegin(GL_POINTS);

		for (int y = 0; y < depthFrame.getHeight(); ++y)
		{
			const openni::DepthPixel* pDepth = pDepthRow;
			openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX();
			//chord_temp = 0;
			for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex)
			{
				if (*pDepth != 0)
				{
					factor[0] = Colors[colorCount][0];
					factor[1] = Colors[colorCount][1];
					factor[2] = Colors[colorCount][2];

					int nHistValue = m_pDepthHist[*pDepth];
					pTex->r = nHistValue*factor[0];
					pTex->g = nHistValue*factor[1];
					pTex->b = nHistValue*factor[2];

					factor[0] = factor[1] = factor[2] = 1;
					

					if(*pDepth <= 800)
					{
					    //glColor3f(1,0,0);	
						glColor3f(float(*pDepth)/2000,float(*pDepth)/2000,float(*pDepth)/2000);
						
						av_x = x + av_x;
						counter++;
						av_y = y + av_y;
		
							
					}
					else{
						glColor3f(float(*pDepth)/2000,float(*pDepth)/2000,float(*pDepth)/2000);	
		
					}

					glVertex3f(2*x,2*y,-*pDepth);
				}
			}
			pDepthRow += rowSize;
			pTexRow += m_nTexMapX;
		}
		glEnd();
		//////////////////////////////////////////////////////////////////////////////////////////////////////////////////

		av_x = av_x / counter;
	    av_y = av_y / counter;

		float R_x=0;
		float R_y=0;
		float L_x=0;
		float L_y=0;
		int counter_R=0;
		int counter_L=0;

		/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////	

		for (int y = 0; y < depthFrame.getHeight(); ++y)
		{
			const openni::DepthPixel* pDepth = pDepthRow1;
			//chord_temp = 0;
			for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth)
			{
				if (*pDepth != 0)
				{

					if(*pDepth <= 800)
					{

						if(x > av_x){
							counter_R++;
							R_x = R_x +x;
							R_y = R_y +y;
						}
						if(x < av_x){
							counter_L++;
							L_x = L_x +x;
							L_y = L_y +y;
						}

					}

				}
			}
			pDepthRow1 += rowSize;
		}
		/////////////////////////////////////////////////////////////////

		R_x = R_x/counter_R;
	    R_y = R_y/counter_R;

		L_x = L_x/counter_L;
	    L_y = L_y/counter_L;

		glPointSize(30);
	    glBegin(GL_POINTS);
		glColor3f(1,0,0);
		glVertex3f(R_x*2,R_y*2,800);
		glColor3f(1,1,0);
		glVertex3f(L_x*2,L_y*2,800);
		glEnd();


		if( R_x >=75 && R_x <=175  ){
			if( R_y  <= 150  )
				{
					note_on[0] = true;
				}

				else if( R_y >= 350)
				{
					note_on[1] = true;
				}
			}
							
		if( R_x >=175 && R_x <=300){
			if( R_y <= 150 )
			{
				note_on[2] = true;	
			}

			else if( R_y >= 350 )
			{	
				note_on[3] = true;					
			}					
		}

		if( R_x>=300 && R_x<=425){
			if( R_y <= 150 )
			{		
				note_on[4] = true;	
			}

			else if( R_y >= 350 )
			{							
				note_on[5] = true;							
			}
		}

		if( R_x>=425 && R_x<=550){
			if( R_y <= 150  )
			{						
				note_on[6] = true;
			}

			else if( R_y >= 350 )
			{			
				note_on[7] = true;	
			}
		}

		////////////////////////////////////////
		if( L_x >=75 && L_x <=175  ){
			if( L_y  <= 150  )
				{
					note_on[0] = true;
				}

				else if( L_y >= 350)
				{
					note_on[1] = true;
				}
			}
							
		if( L_x >=175 && L_x <=300){
			if( L_y <= 150 )
			{
				note_on[2] = true;	
			}

			else if( L_y >= 350 )
			{	
				note_on[3] = true;					
			}					
		}

		if( L_x>=300 && L_x<=425){
			if( L_y <= 150 )
			{		
				note_on[4] = true;	
			}

			else if( L_y >= 350 )
			{							
				note_on[5] = true;							
			}
		}

		if( L_x>=425 && L_x<=550){
			if( L_y <= 150  )
			{						
				note_on[6] = true;
			}

			else if( L_y >= 350 )
			{			
				note_on[7] = true;	
			}
		}
	}




	playdrum();
	for(int i=0;i<=7 ;i++)
		last_note[i] = note_on[i]; 

	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); 
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); 
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 

	glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, BMPwidth1,BMPheight1, 0, GL_RGB, GL_UNSIGNED_BYTE, BMPimage1);

	glEnable(GL_BLEND);
	glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
	glColor4f(1,1,1,0.5);

	glEnable(GL_TEXTURE_2D);
	glBegin(GL_QUADS);

	g_nXRes = depthFrame.getVideoMode().getResolutionX();
	g_nYRes = depthFrame.getVideoMode().getResolutionY();

	// upper left
	glTexCoord2f(0,1);
	glVertex3f(0,0,-800);
	// upper right
	glTexCoord2f(1,1);
	glVertex3f(1240,0,-800);
	// bottom right
	glTexCoord2f(1,0);
	glVertex3f(1240,960,-800);
	// bottom left
	glTexCoord2f(0,0);
	glVertex3f(0,960,-800);

	glEnd();
	glDisable(GL_TEXTURE_2D);
	glDisable(GL_BLEND);  

	/////////////////////////////////////////////////////////////////////////////////chord selection

	glBegin(GL_LINES);
	glColor3f(1,0,0);
	glVertex3f(150,300,800);
	glVertex3f(1100,300,800);
	glVertex3f(150,700,800);
	glVertex3f(1100,700,800);
	glEnd();


	glPointSize(30);
	glBegin(GL_POINTS);
	glColor3f(1,1,0);
	glVertex3f(150,300,800);
	glVertex3f(350,300,800);
	glVertex3f(600,300,800);
	glVertex3f(850,300,800);
	glVertex3f(1100,300,800);


	glVertex3f(150,700,800);
	glVertex3f(350,700,800);
	glVertex3f(600,700,800);
	glVertex3f(850,700,800);
	glVertex3f(1100,700,800);
	glEnd();



	////////////////////////////////////////////////////////////////////////////////////////////////////////////
		

	const nite::Array<nite::GestureData>& gestures = handFrame.getGestures();
	for (int i = 0; i < gestures.getSize(); ++i)
	{
		if (gestures[i].isComplete())
		{
			const nite::Point3f& position = gestures[i].getCurrentPosition();
			printf("Gesture %d at (%f,%f,%f)\n", gestures[i].getType(), position.x, position.y, position.z);

			nite::HandId newId;
			m_pHandTracker->startHandTracking(gestures[i].getCurrentPosition(), &newId);
		}
	}

	const nite::Array<nite::HandData>& hands= handFrame.getHands();
	for (int i = 0; i < hands.getSize(); ++i)
	{
		const nite::HandData& user = hands[i];

		if (!user.isTracking())
		{
			printf("Lost hand %d\n", user.getId());
			nite::HandId id = user.getId();
			HistoryBuffer<20>* pHistory = g_histories[id];
			g_histories.erase(g_histories.find(id));
			delete pHistory;
		}
		else
		{
			if (user.isNew())
			{
				printf("Found hand %d\n", user.getId());
				g_histories[user.getId()] = new HistoryBuffer<20>;
			}
			// Add to history
			HistoryBuffer<20>* pHistory = g_histories[user.getId()];
			pHistory->AddPoint(user.getPosition());
			// Draw history
		    DrawHistory(m_pHandTracker, user.getId(), pHistory);
		   
		}
	}

	if (g_drawFrameId)
	{
		DrawFrameId(handFrame.getFrameIndex());
	}
	// Swap the OpenGL display buffers
	glutSwapBuffers();
	
}
void HandGesture::Display()
{
	nite::HandTrackerFrameRef handFrame;
	openni::VideoFrameRef depthFrame;
	nite::Status rc = m_pHandTracker->readFrame(&handFrame);


	IplImage* pDepthImg = cvCreateImage(cvSize(W,H), IPL_DEPTH_16U, 1);

	if (rc != nite::STATUS_OK)
	{
		printf("GetNextData failed\n");
		return;
	}

	depthFrame = handFrame.getDepthFrame();

	if (depthFrame.isValid())
	{
//		calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
//		getHandThreshold(depthFrame);
		/*Copy depth data to image data*/
		memcpy(pDepthImg->imageData, depthFrame.getData(), W*H*2);

		cvCvtScale(pDepthImg, this->pImg, DEPTH_SCALE_FACTOR);

		cvConvertImage(this->pImg, this->pRgbImg, CV_GRAY2RGB);
	}

	const nite::Array<nite::GestureData>& gestures = handFrame.getGestures();
	for (int i = 0; i < gestures.getSize(); ++i)
	{
		if (gestures[i].isComplete())
		{
			nite::HandId newId;
			m_pHandTracker->startHandTracking(gestures[i].getCurrentPosition(), &newId);
		}
	}

	const nite::Array<nite::HandData>& hands= handFrame.getHands();
	CvPoint2D32f handPoint = cvPoint2D32f(0,0);
	CvPoint cvhandPoint = cvPoint(0, 0);
	for (int i = 0; i < hands.getSize(); ++i)
	{
		const nite::HandData& hand = hands[i];

		if (!hand.isTracking())// Tracking hand is lost
		{
			printf("Lost hand %d\n", hand.getId());
			nite::HandId id = hand.getId();
			HistoryBuffer<HISBUFFER>* pHistory = g_histories[id];
			g_histories.erase(g_histories.find(id));
			delete pHistory;
			handPointClear();
		}
		else
		{
			if (hand.isNew())
			{
				printf("Found hand %d\n", hand.getId());
				nite::HandId id = hand.getId();
				g_histories[hand.getId()] = new HistoryBuffer<HISBUFFER>;
				handPoint = cvPoint2D32f(0,0);
				cvhandPoint = cvPoint(0, 0);
			}
			this->hand3DPoint = hand.getPosition();
			// Add to history
			HistoryBuffer<HISBUFFER>* pHistory = g_histories[hand.getId()];
			pHistory->AddPoint(hand.getPosition());
			// Draw history
			DrawHistory(m_pHandTracker, hand.getId(), pHistory);

			//Get coordiation of hand point
			m_pHandTracker->convertHandCoordinatesToDepth(hand3DPoint.x, hand3DPoint.y, hand3DPoint.z, (float*)&handPoint.x, (float*)&handPoint.y);
			// Convert hand point to int
			cvhandPoint = cvPointFrom32f(handPoint);

			printf("Gesture %d at (%d,%d,%d)\n", hand.getId(), cvhandPoint.x, cvhandPoint.y, handDepthPoint.d);
			
			getHandThreshold(depthFrame);

			
			cvShowImage("ThresholdImage", this->pThImg);

			cvCircle(this->pRgbImg, cvhandPoint, 2, RED, 4, CV_AA, 0);
		}
	}
	
	cvShowImage("DepthImage", this->pRgbImg);
}
Beispiel #4
0
void Kinect::KinectHandTracker::getRotatingMove()
{
	// List of hands evidence
	// If hand matches old positions, previous ID is assigned again
	const nite::Array<nite::HandData>& hands= this->handTrackerFrame.getHands();

	this->slidingHand_x = 0;
	this->slidingHand_y = 0;
	this->slidingHand_z = 0;
	this->slidingHand_type = "";
	//strcpy(this->slidingHand_type, "");

	for (int i = 0; i < hands.getSize(); ++i)
	{
		const nite::HandData& user = hands[i];

		if (!user.isTracking())
		{
			//printf("Lost hand %d\n", user.getId());
			nite::HandId id = user.getId();
			HistoryBuffer<20>* pHistory = this->g_histories[id];
			this->g_histories.erase(this->g_histories.find(id));
			delete pHistory;
		}
		else
		{
			if (user.isNew())
			{
				this->g_histories[user.getId()] = new HistoryBuffer<20>;
			}
			// Hand evidence in Buffer
			HistoryBuffer<20>* pHistory = this->g_histories[user.getId()];
			pHistory->AddPoint(user.getPosition());


			coordinateConverter.convertWorldToDepth(*mDepth, user.getPosition().x, user.getPosition().y, user.getPosition().z, &this->slidingHand_x, &this->slidingHand_y, &this->slidingHand_z);

			// printf("%lf %lf \n", this->slidingHand_x,this->slidingHand_y );


			float koordy[60] = {0};

			bool gesto_dolava = false;
			bool gesto_doprava = false;
			bool gesto_hore = false;
			bool gesto_dole = false;


			if(pHistory->GetSize() == 20)
			{ // ak je historia naplnena
				const nite::Point3f& position1 = pHistory->operator[](0);
				this->m_pHandTracker.convertHandCoordinatesToDepth(position1.x, position1.y, position1.z, &koordy[0], &koordy[1]);
				const nite::Point3f& position2 = pHistory->operator[](19);
				this->m_pHandTracker.convertHandCoordinatesToDepth(position2.x, position2.y, position2.z, &koordy[2], &koordy[3]);


				//check for output
				if(abs(koordy[0] - koordy[2]) > 100.0)
				{
					if(koordy[0] > koordy[2])
					{
						gesto_dolava = false;
						gesto_doprava = true;
					}
					else
					{
						gesto_dolava = true;
						gesto_doprava = false;
					}
				}
				if(abs(koordy[1] - koordy[3]) > 100.0)
				{
					if(koordy[1] < koordy[3])
					{
						gesto_dole = false;
						gesto_hore = true;
					}
					else
					{
						gesto_dole = true;
						gesto_hore = false;
					}
				}
				if(gesto_dolava) this->slidingHand_type = "scroll left";
				else if(gesto_doprava) this->slidingHand_type = "scroll right";
			}
		}
	}
}