void Kinect::KinectHandTracker::getAllHands()
{
// List of hands evidence
// If hand matches old positions, previous ID is assigned again
const nite::Array<nite::HandData>& hands= this->handTrackerFrame.getHands();
//printf("%d hands\n", hands.getSize());
this->isTwoHands = false;
for (int i = 0; i < hands.getSize(); ++i)
{
const nite::HandData& user = hands[i];
if (!user.isTracking())
{
//printf("Lost hand %d\n", user.getId());
nite::HandId id = user.getId();
HistoryBuffer<20>* pHistory = this->g_histories[id];
this->g_histories.erase(this->g_histories.find(id));
delete pHistory;
}
else
{
if (user.isNew())
{
//printf("New hand id %d\n", user.getId());
this->g_histories[user.getId()] = new HistoryBuffer<20>;
}
// Hand evidence in Buffer
HistoryBuffer<20>* pHistory = this->g_histories[user.getId()];
pHistory->AddPoint(user.getPosition());
// Data for mouse
//first find HAND = MOUSE
if(i==0 && isCursorMovementEnable)
{
mouse->setSpeedUpMoving(mSpeed);
//mouse->moveCursorMouse(user.getPosition().x/2,-1.0*user.getPosition().y/2,isClick);
coordinateConverter.convertWorldToDepth(*mDepth, user.getPosition().x, user.getPosition().y, user.getPosition().z, &mDepthX, &mDepthY, &mDepthZ);
mouse->moveCursorWorldCoordinates(mDepthX,mDepthY,isClick);
}
// TODO - further implementation should include depth information in pixels
this->handZ[i] = user.getPosition().z;
this->getArrayHands[i][0] = user.getPosition().x;
this->getArrayHands[i][1] = 0 - user.getPosition().y;
// If two hands have been found get the position of the rectangle
if (hands.getSize() == 2)
{
printf("two hands found\n");
// get positions for both hands
this->isTwoHands = true;
}
}
}
}
void SampleViewer::Display()
{
nite::Status rc = m_pHandTracker->readFrame(&handFrame);
if (rc != nite::STATUS_OK)
{
printf("GetNextData failed\n");
return;
}
depthFrame = handFrame.getDepthFrame();
if (m_pTexMap == NULL)
{
// Texture map init
m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE);
m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE);
m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY];
}
glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glEnable(GL_DEPTH_TEST);
glMatrixMode(GL_PROJECTION);
glPushMatrix();
glLoadIdentity();
glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -10000.0, 10000.0);
if (depthFrame.isValid())
{
calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
}
memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel));
float factor[3] = {1, 1, 1};
// check if we need to draw depth frame to texture
float av_x = 0;
float av_y = 0;
int counter= 0;
for(int i = 0; i<=7 ; i++)
note_on[i] = false;
if (depthFrame.isValid() && g_drawDepth)
{
const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData();
const openni::DepthPixel* pDepthRow1 = pDepthRow;
openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX;
int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel);
glPointSize(2);
glBegin(GL_POINTS);
for (int y = 0; y < depthFrame.getHeight(); ++y)
{
const openni::DepthPixel* pDepth = pDepthRow;
openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX();
//chord_temp = 0;
for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex)
{
if (*pDepth != 0)
{
factor[0] = Colors[colorCount][0];
factor[1] = Colors[colorCount][1];
factor[2] = Colors[colorCount][2];
int nHistValue = m_pDepthHist[*pDepth];
pTex->r = nHistValue*factor[0];
pTex->g = nHistValue*factor[1];
pTex->b = nHistValue*factor[2];
factor[0] = factor[1] = factor[2] = 1;
if(*pDepth <= 800)
{
//glColor3f(1,0,0);
glColor3f(float(*pDepth)/2000,float(*pDepth)/2000,float(*pDepth)/2000);
av_x = x + av_x;
counter++;
av_y = y + av_y;
}
else{
glColor3f(float(*pDepth)/2000,float(*pDepth)/2000,float(*pDepth)/2000);
}
glVertex3f(2*x,2*y,-*pDepth);
}
}
pDepthRow += rowSize;
pTexRow += m_nTexMapX;
}
glEnd();
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
av_x = av_x / counter;
av_y = av_y / counter;
float R_x=0;
float R_y=0;
float L_x=0;
float L_y=0;
int counter_R=0;
int counter_L=0;
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for (int y = 0; y < depthFrame.getHeight(); ++y)
{
const openni::DepthPixel* pDepth = pDepthRow1;
//chord_temp = 0;
for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth)
{
if (*pDepth != 0)
{
if(*pDepth <= 800)
{
if(x > av_x){
counter_R++;
R_x = R_x +x;
R_y = R_y +y;
}
if(x < av_x){
counter_L++;
L_x = L_x +x;
L_y = L_y +y;
}
}
}
}
pDepthRow1 += rowSize;
}
/////////////////////////////////////////////////////////////////
R_x = R_x/counter_R;
R_y = R_y/counter_R;
L_x = L_x/counter_L;
L_y = L_y/counter_L;
glPointSize(30);
glBegin(GL_POINTS);
glColor3f(1,0,0);
glVertex3f(R_x*2,R_y*2,800);
glColor3f(1,1,0);
glVertex3f(L_x*2,L_y*2,800);
glEnd();
if( R_x >=75 && R_x <=175 ){
if( R_y <= 150 )
{
note_on[0] = true;
}
else if( R_y >= 350)
{
note_on[1] = true;
}
}
if( R_x >=175 && R_x <=300){
if( R_y <= 150 )
{
note_on[2] = true;
}
else if( R_y >= 350 )
{
note_on[3] = true;
}
}
if( R_x>=300 && R_x<=425){
if( R_y <= 150 )
{
note_on[4] = true;
}
else if( R_y >= 350 )
{
note_on[5] = true;
}
}
if( R_x>=425 && R_x<=550){
if( R_y <= 150 )
{
note_on[6] = true;
}
else if( R_y >= 350 )
{
note_on[7] = true;
}
}
////////////////////////////////////////
if( L_x >=75 && L_x <=175 ){
if( L_y <= 150 )
{
note_on[0] = true;
}
else if( L_y >= 350)
{
note_on[1] = true;
}
}
if( L_x >=175 && L_x <=300){
if( L_y <= 150 )
{
note_on[2] = true;
}
else if( L_y >= 350 )
{
note_on[3] = true;
}
}
if( L_x>=300 && L_x<=425){
if( L_y <= 150 )
{
note_on[4] = true;
}
else if( L_y >= 350 )
{
note_on[5] = true;
}
}
if( L_x>=425 && L_x<=550){
if( L_y <= 150 )
{
note_on[6] = true;
}
else if( L_y >= 350 )
{
note_on[7] = true;
}
}
}
playdrum();
for(int i=0;i<=7 ;i++)
last_note[i] = note_on[i];
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, BMPwidth1,BMPheight1, 0, GL_RGB, GL_UNSIGNED_BYTE, BMPimage1);
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glColor4f(1,1,1,0.5);
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
g_nXRes = depthFrame.getVideoMode().getResolutionX();
g_nYRes = depthFrame.getVideoMode().getResolutionY();
// upper left
glTexCoord2f(0,1);
glVertex3f(0,0,-800);
// upper right
glTexCoord2f(1,1);
glVertex3f(1240,0,-800);
// bottom right
glTexCoord2f(1,0);
glVertex3f(1240,960,-800);
// bottom left
glTexCoord2f(0,0);
glVertex3f(0,960,-800);
glEnd();
glDisable(GL_TEXTURE_2D);
glDisable(GL_BLEND);
/////////////////////////////////////////////////////////////////////////////////chord selection
glBegin(GL_LINES);
glColor3f(1,0,0);
glVertex3f(150,300,800);
glVertex3f(1100,300,800);
glVertex3f(150,700,800);
glVertex3f(1100,700,800);
glEnd();
glPointSize(30);
glBegin(GL_POINTS);
glColor3f(1,1,0);
glVertex3f(150,300,800);
glVertex3f(350,300,800);
glVertex3f(600,300,800);
glVertex3f(850,300,800);
glVertex3f(1100,300,800);
glVertex3f(150,700,800);
glVertex3f(350,700,800);
glVertex3f(600,700,800);
glVertex3f(850,700,800);
glVertex3f(1100,700,800);
glEnd();
////////////////////////////////////////////////////////////////////////////////////////////////////////////
const nite::Array<nite::GestureData>& gestures = handFrame.getGestures();
for (int i = 0; i < gestures.getSize(); ++i)
{
if (gestures[i].isComplete())
{
const nite::Point3f& position = gestures[i].getCurrentPosition();
printf("Gesture %d at (%f,%f,%f)\n", gestures[i].getType(), position.x, position.y, position.z);
nite::HandId newId;
m_pHandTracker->startHandTracking(gestures[i].getCurrentPosition(), &newId);
}
}
const nite::Array<nite::HandData>& hands= handFrame.getHands();
for (int i = 0; i < hands.getSize(); ++i)
{
const nite::HandData& user = hands[i];
if (!user.isTracking())
{
printf("Lost hand %d\n", user.getId());
nite::HandId id = user.getId();
HistoryBuffer<20>* pHistory = g_histories[id];
g_histories.erase(g_histories.find(id));
delete pHistory;
}
else
{
if (user.isNew())
{
printf("Found hand %d\n", user.getId());
g_histories[user.getId()] = new HistoryBuffer<20>;
}
// Add to history
HistoryBuffer<20>* pHistory = g_histories[user.getId()];
pHistory->AddPoint(user.getPosition());
// Draw history
DrawHistory(m_pHandTracker, user.getId(), pHistory);
}
}
if (g_drawFrameId)
{
DrawFrameId(handFrame.getFrameIndex());
}
// Swap the OpenGL display buffers
glutSwapBuffers();
}
void HandGesture::Display()
{
nite::HandTrackerFrameRef handFrame;
openni::VideoFrameRef depthFrame;
nite::Status rc = m_pHandTracker->readFrame(&handFrame);
IplImage* pDepthImg = cvCreateImage(cvSize(W,H), IPL_DEPTH_16U, 1);
if (rc != nite::STATUS_OK)
{
printf("GetNextData failed\n");
return;
}
depthFrame = handFrame.getDepthFrame();
if (depthFrame.isValid())
{
// calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
// getHandThreshold(depthFrame);
/*Copy depth data to image data*/
memcpy(pDepthImg->imageData, depthFrame.getData(), W*H*2);
cvCvtScale(pDepthImg, this->pImg, DEPTH_SCALE_FACTOR);
cvConvertImage(this->pImg, this->pRgbImg, CV_GRAY2RGB);
}
const nite::Array<nite::GestureData>& gestures = handFrame.getGestures();
for (int i = 0; i < gestures.getSize(); ++i)
{
if (gestures[i].isComplete())
{
nite::HandId newId;
m_pHandTracker->startHandTracking(gestures[i].getCurrentPosition(), &newId);
}
}
const nite::Array<nite::HandData>& hands= handFrame.getHands();
CvPoint2D32f handPoint = cvPoint2D32f(0,0);
CvPoint cvhandPoint = cvPoint(0, 0);
for (int i = 0; i < hands.getSize(); ++i)
{
const nite::HandData& hand = hands[i];
if (!hand.isTracking())// Tracking hand is lost
{
printf("Lost hand %d\n", hand.getId());
nite::HandId id = hand.getId();
HistoryBuffer<HISBUFFER>* pHistory = g_histories[id];
g_histories.erase(g_histories.find(id));
delete pHistory;
handPointClear();
}
else
{
if (hand.isNew())
{
printf("Found hand %d\n", hand.getId());
nite::HandId id = hand.getId();
g_histories[hand.getId()] = new HistoryBuffer<HISBUFFER>;
handPoint = cvPoint2D32f(0,0);
cvhandPoint = cvPoint(0, 0);
}
this->hand3DPoint = hand.getPosition();
// Add to history
HistoryBuffer<HISBUFFER>* pHistory = g_histories[hand.getId()];
pHistory->AddPoint(hand.getPosition());
// Draw history
DrawHistory(m_pHandTracker, hand.getId(), pHistory);
//Get coordiation of hand point
m_pHandTracker->convertHandCoordinatesToDepth(hand3DPoint.x, hand3DPoint.y, hand3DPoint.z, (float*)&handPoint.x, (float*)&handPoint.y);
// Convert hand point to int
cvhandPoint = cvPointFrom32f(handPoint);
printf("Gesture %d at (%d,%d,%d)\n", hand.getId(), cvhandPoint.x, cvhandPoint.y, handDepthPoint.d);
getHandThreshold(depthFrame);
cvShowImage("ThresholdImage", this->pThImg);
cvCircle(this->pRgbImg, cvhandPoint, 2, RED, 4, CV_AA, 0);
}
}
cvShowImage("DepthImage", this->pRgbImg);
}
void Kinect::KinectHandTracker::getRotatingMove()
{
// List of hands evidence
// If hand matches old positions, previous ID is assigned again
const nite::Array<nite::HandData>& hands= this->handTrackerFrame.getHands();
this->slidingHand_x = 0;
this->slidingHand_y = 0;
this->slidingHand_z = 0;
this->slidingHand_type = "";
//strcpy(this->slidingHand_type, "");
for (int i = 0; i < hands.getSize(); ++i)
{
const nite::HandData& user = hands[i];
if (!user.isTracking())
{
//printf("Lost hand %d\n", user.getId());
nite::HandId id = user.getId();
HistoryBuffer<20>* pHistory = this->g_histories[id];
this->g_histories.erase(this->g_histories.find(id));
delete pHistory;
}
else
{
if (user.isNew())
{
this->g_histories[user.getId()] = new HistoryBuffer<20>;
}
// Hand evidence in Buffer
HistoryBuffer<20>* pHistory = this->g_histories[user.getId()];
pHistory->AddPoint(user.getPosition());
coordinateConverter.convertWorldToDepth(*mDepth, user.getPosition().x, user.getPosition().y, user.getPosition().z, &this->slidingHand_x, &this->slidingHand_y, &this->slidingHand_z);
// printf("%lf %lf \n", this->slidingHand_x,this->slidingHand_y );
float koordy[60] = {0};
bool gesto_dolava = false;
bool gesto_doprava = false;
bool gesto_hore = false;
bool gesto_dole = false;
if(pHistory->GetSize() == 20)
{ // ak je historia naplnena
const nite::Point3f& position1 = pHistory->operator[](0);
this->m_pHandTracker.convertHandCoordinatesToDepth(position1.x, position1.y, position1.z, &koordy[0], &koordy[1]);
const nite::Point3f& position2 = pHistory->operator[](19);
this->m_pHandTracker.convertHandCoordinatesToDepth(position2.x, position2.y, position2.z, &koordy[2], &koordy[3]);
//check for output
if(abs(koordy[0] - koordy[2]) > 100.0)
{
if(koordy[0] > koordy[2])
{
gesto_dolava = false;
gesto_doprava = true;
}
else
{
gesto_dolava = true;
gesto_doprava = false;
}
}
if(abs(koordy[1] - koordy[3]) > 100.0)
{
if(koordy[1] < koordy[3])
{
gesto_dole = false;
gesto_hore = true;
}
else
{
gesto_dole = true;
gesto_hore = false;
}
}
if(gesto_dolava) this->slidingHand_type = "scroll left";
else if(gesto_doprava) this->slidingHand_type = "scroll right";
}
}
}
}
