/// <summary>
/// Handle new skeleton data
/// </summary>
void CSkeletonBasics::ProcessSkeleton()
{
NUI_SKELETON_FRAME skeletonFrame = {0};
HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame);
if ( FAILED(hr) )
{
return;
}
// smooth out the skeleton data
m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL);
// Endure Direct2D is ready to draw
hr = EnsureDirect2DResources( );
if ( FAILED(hr) )
{
return;
}
m_pRenderTarget->BeginDraw();
m_pRenderTarget->Clear( );
RECT rct;
GetClientRect( GetDlgItem( m_hWnd, IDC_VIDEOVIEW ), &rct);
int width = rct.right;
int height = rct.bottom;
for (int i = 0 ; i < NUI_SKELETON_COUNT; ++i)
{
NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState;
if (NUI_SKELETON_TRACKED == trackingState)
{
// We're tracking the skeleton, draw it
DrawSkeleton(skeletonFrame.SkeletonData[i], width, height);
}
else if (NUI_SKELETON_POSITION_ONLY == trackingState)
{
// we've only received the center point of the skeleton, draw that
D2D1_ELLIPSE ellipse = D2D1::Ellipse(
SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height),
g_JointThickness,
g_JointThickness
);
m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked);
}
}
hr = m_pRenderTarget->EndDraw();
// Device lost, need to recreate the render target
// We'll dispose it now and retry drawing
if (D2DERR_RECREATE_TARGET == hr)
{
hr = S_OK;
DiscardDirect2DResources();
}
}
void JModel::Render( const Mat4& tm, JBoneInstance* pSkelInst, bool bIgnoreShader, bool bUseCachedData )
{
if (m_IBufID == -1)
{
Init();
}
// check geometry caching status, update if needed
g_pRenderServer->CacheIB( m_IBufID, m_Indices.GetData(), m_Indices.GetSize(), m_IBIteration, m_IBFirstByte );
int nBones = m_SkelInst.size();
// calculate world-space transforms for bones
if (!pSkelInst && nBones > 0) pSkelInst = &m_SkelInst[0];
UpdateSkeleton( Mat4::identity, pSkelInst );
// cycle through geometries and render them
int nGeom = m_Meshes.size();
int cMtl = -1;
for (int i = 0; i < nGeom; i++)
{
JMesh* pMesh = m_Meshes[i];
if (!pMesh->IsVisible())
{
continue;
}
pMesh->CacheVB();
int mtlID = pMesh->GetMaterialID();
if (mtlID >= 0 && mtlID != cMtl)
{
JMaterial* pMtl = m_Materials[mtlID];
pMtl->Render( bIgnoreShader );
cMtl = mtlID;
}
Mat4 worldTM = tm;
int hostBoneID = pMesh->GetHostBoneID();
if (hostBoneID >= 0 && pMesh->GetNSkinBones() == 0)
{
JBoneInstance& hb = pSkelInst[hostBoneID];
if (!hb.m_bVisible)
{
continue;
}
Mat4 rootTM = hb.m_ModelTM*tm;
worldTM = rootTM;
}
pMesh->Render( m_IBFirstByte, pSkelInst, worldTM, bUseCachedData );
}
if (m_bDrawSkeleton)
{
DrawSkeleton( tm, pSkelInst );
}
} // JModel::Render
static inline void DebugRenderPass(RenderState *rs) {
auto settings = GetDebugRenderSettings();
if (settings->drawSkeletons) {
for (size_t i = 0; i < rs->drawList.animated_model_draw_commands.count; i++) {
Animated_Model_Draw_Command *cmd = &rs->drawList.animated_model_draw_commands[i];
ModelAsset *model = cmd->model;
assert(model != nullptr);
DrawSkeleton(model->joints, model->jointCount, cmd->animation_state, cmd->model_matrix);
}
}
}
void DrawContent()
{
DrawMesh();
glPointSize(5.0);
DrawSkeleton();
}
void SampleViewer::Display()
{
// namespace bg = boost::geometry;
nite::UserTrackerFrameRef userTrackerFrame;
openni::VideoFrameRef depthFrame;
nite::Status rc = m_pUserTracker->readFrame(&userTrackerFrame);
if (rc != nite::STATUS_OK)
{
printf("GetNextData failed\n");
return;
}
depthFrame = userTrackerFrame.getDepthFrame();
if (m_pTexMap == NULL)
{
// Texture map init
m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE);
m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE);
m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY];
}
const nite::UserMap& userLabels = userTrackerFrame.getUserMap();
glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_PROJECTION);
glPushMatrix();
glLoadIdentity();
glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -1.0, 1.0);
if (depthFrame.isValid() && g_drawDepth)
{
calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
}
memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel));
float factor[3] = {1, 1, 1};
// check if we need to draw depth frame to texture
if (depthFrame.isValid() && g_drawDepth)
{
const nite::UserId* pLabels = userLabels.getPixels();
const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData();
openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX;
int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel);
for (int y = 0; y < depthFrame.getHeight(); ++y)
{
const openni::DepthPixel* pDepth = pDepthRow;
openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX();
for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex, ++pLabels)
{
if (*pDepth != 0)
{
if (*pLabels == 0)
{
if (!g_drawBackground)
{
factor[0] = factor[1] = factor[2] = 0;
}
else
{
factor[0] = Colors[colorCount][0];
factor[1] = Colors[colorCount][1];
factor[2] = Colors[colorCount][2];
}
}
else
{
factor[0] = Colors[*pLabels % colorCount][0];
factor[1] = Colors[*pLabels % colorCount][1];
factor[2] = Colors[*pLabels % colorCount][2];
}
// // Add debug lines - every 10cm
// else if ((*pDepth / 10) % 10 == 0)
// {
// factor[0] = factor[2] = 0;
// }
int nHistValue = m_pDepthHist[*pDepth];
pTex->r = nHistValue*factor[0];
pTex->g = nHistValue*factor[1];
pTex->b = nHistValue*factor[2];
factor[0] = factor[1] = factor[2] = 1;
}
}
pDepthRow += rowSize;
pTexRow += m_nTexMapX;
}
}
glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap);
// Display the OpenGL texture map
glColor4f(1,1,1,1);
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
g_nXRes = depthFrame.getVideoMode().getResolutionX();
g_nYRes = depthFrame.getVideoMode().getResolutionY();
// upper left
glTexCoord2f(0, 0);
glVertex2f(0, 0);
// upper right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0);
glVertex2f(GL_WIN_SIZE_X, 0);
// bottom right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y);
// bottom left
glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(0, GL_WIN_SIZE_Y);
glEnd();
glDisable(GL_TEXTURE_2D);
const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers();
// 11.16.15 remove for loop for tracking only one user
// for (int i = 0; i < users.getSize(); ++i)
// {
//const nite::UserData& user = users[i];
if (users.getSize() > 0) {
const nite::UserData& user = users[0];
updateUserState(user, userTrackerFrame.getTimestamp());
if (user.isNew())
{
m_pUserTracker->startSkeletonTracking(user.getId());
m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS);
}
else if (!user.isLost())
{
if (g_drawStatusLabel)
{
DrawStatusLabel(m_pUserTracker, user);
}
if (g_drawCenterOfMass)
{
DrawCenterOfMass(m_pUserTracker, user);
}
if (g_drawBoundingBox)
{
DrawBoundingBox(user);
}
if (user.getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton)
{
DrawSkeleton(m_pUserTracker, user);
}
}
if (m_poseUser == 0 || m_poseUser == user.getId())
{
const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS);
if (pose.isEntered())
{
// Start timer
sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s");
printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000);
m_poseUser = user.getId();
m_poseTime = userTrackerFrame.getTimestamp();
}
else if (pose.isExited())
{
memset(g_generalMessage, 0, sizeof(g_generalMessage));
printf("Count-down interrupted\n");
m_poseTime = 0;
m_poseUser = 0;
}
else if (pose.isHeld())
{
// tick
if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000)
{
printf("Count down complete. Exit...\n");
Finalize();
exit(2);
}
}
} //user
char buffer[80] = "";
// ---------------- calculate right shoulder 2 DOF ----------------------
int rightShoulderX, rightShoulderY, rightShoulderZ;
rightShoulderX = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().x -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().x;
rightShoulderY = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().y -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().y;
rightShoulderZ = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().z -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().z;
Spherical rightShoulderSpherical;
rightShoulderSpherical = Cartesian2Spherical(rightShoulderX, rightShoulderY, rightShoulderZ);
int rightShoulderPhi, rightShoulderTheta;
rightShoulderTheta = radian2Degree(rightShoulderSpherical.radianTheta, rightShoulderThetaInit); // horizontal rise it's -60 degree
rightShoulderPhi = radian2Degree(rightShoulderSpherical.radianPhi, rightShoulderPitchInit); // when hand's down, it's 70 degree
int rightShoulderYawNow = rightShoulderPhi * sin(rightShoulderTheta * PI / 180);
int rightShoulderPitchNow = rightShoulderPhi * cos(rightShoulderTheta * PI / 180);
sprintf(buffer,"(rightShoulderTheta=%d, rightShoulderYaw=%d, rightShoulderPitch=%d)", rightShoulderTheta, rightShoulderYawNow, rightShoulderPitchNow);
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(20, 20);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
// ------------------- calculate right elbow 2 DOF -----------------------
int rightElbowX, rightElbowY, rightElbowZ;
rightElbowX = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().x -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().x;
rightElbowY = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().y -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().y;
rightElbowZ = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().z -
user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().z;
Spherical rightElbowSpherical;
rightElbowSpherical = Cartesian2Spherical(rightElbowX, rightElbowY, rightElbowZ);
int rightElbowThetaNow = - radian2Degree(rightElbowSpherical.radianTheta, rightElbowThetaInit); // reverse for system
int rightElbowYawNow = radian2Degree(rightElbowSpherical.radianPhi, rightElbowYawInit);
sprintf(buffer,"(rightElbowThetaNow=%d, rightElbowYawNow=%d)", rightElbowThetaNow, rightElbowYawNow);
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(20, 60);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
// ---------------- calculate left shoulder 2 DOF ----------------------
int leftShoulderX, leftShoulderY, leftShoulderZ;
leftShoulderX = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().x -
user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().x;
leftShoulderY = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().y -
user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().y;
leftShoulderZ = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().z -
user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().z;
Spherical leftShoulderSpherical;
leftShoulderSpherical = Cartesian2Spherical(leftShoulderX, leftShoulderY, leftShoulderZ);
int leftShoulderPhi, leftShoulderTheta;
leftShoulderTheta = radian2Degree(leftShoulderSpherical.radianTheta, leftShoulderThetaInit); // horizontal rise it's -60 degree
leftShoulderPhi = radian2Degree(leftShoulderSpherical.radianPhi, leftShoulderPitchInit); // when hand's down, it's 70 degree
// need to reverse in left side
int leftShoulderYawNow = - leftShoulderPhi * sin(leftShoulderTheta * PI / 180);
int leftShoulderPitchNow = - leftShoulderPhi * cos(leftShoulderTheta * PI / 180);
sprintf(buffer,"(leftShoulderTheta=%d, leftShoulderYaw=%d, leftShoulderPitch=%d)", leftShoulderTheta, leftShoulderYawNow, leftShoulderPitchNow);
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(20, 100);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
// ------------------- calculate left elbow 2 DOF -----------------------
int leftElbowX, leftElbowY, leftElbowZ;
leftElbowX = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().x -
user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().x;
leftElbowY = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().y -
user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().y;
leftElbowZ = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().z -
user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().z;
Spherical leftElbowSpherical;
leftElbowSpherical = Cartesian2Spherical(leftElbowX, leftElbowY, leftElbowZ);
int leftElbowThetaNow = radian2Degree(leftElbowSpherical.radianTheta, leftElbowThetaInit);
int leftElbowYawNow = radian2Degree(leftElbowSpherical.radianPhi, leftElbowYawInit);
sprintf(buffer,"(leftElbowTheta=%d, leftElbowYawNow=%d)", leftElbowThetaNow, leftElbowYawNow);
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(20, 140);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
// ---------------- constraint movement and publish message ------------
int rightShoulderYawDiff = abs(rightShoulderYaw - rightShoulderYawNow);
int rightShoulderPitchDiff = abs(rightShoulderPitch - rightShoulderPitchNow);
int rightElbowThetaDiff = abs(rightElbowTheta - rightElbowThetaNow);
int rightElbowYawDiff = abs(rightElbowYaw - rightElbowYawNow);
int leftShoulderYawDiff = abs(leftShoulderYaw - leftShoulderYawNow);
int leftShoulderPitchDiff = abs(leftShoulderPitch - leftShoulderPitchNow);
int leftElbowThetaDiff = abs(leftElbowTheta - leftElbowThetaNow);
int leftElbowYawDiff = abs(leftElbowYaw - leftElbowYawNow);
if ((rightShoulderYawDiff < moveLimitDegree) && (rightShoulderPitchDiff < moveLimitDegree) && (rightElbowThetaDiff < moveLimitDegree) && (rightElbowYawDiff < moveLimitDegree) && (rightShoulderTheta >= 0)) {
// in range then refresh robot angle
rightShoulderYaw = rightShoulderYawNow;
rightShoulderPitch = rightShoulderPitchNow;
rightElbowTheta = rightElbowThetaNow;
rightElbowYaw = rightElbowYawNow;
// change the angle to 0 to 360 and publish
rightShoulderYawPub = angleHandler(rightShoulderYaw);
rightShoulderPitchPub = angleHandler(rightShoulderPitch);
rightElbowThetaPub = angleHandler(rightElbowTheta);
rightElbowYawPub = angleHandler(rightElbowYaw);
sprintf(buffer,"tracking!");
glColor3f(0.0f, 0.0f, 1.0f);
glRasterPos2i(20, 180);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
actionPublish(rightShoulderYawPub, rightShoulderPitchPub, rightElbowThetaPub, rightElbowYawPub, leftShoulderYawPub, leftShoulderPitchPub, leftElbowThetaPub, leftElbowYawPub);
}
else {
sprintf(buffer,"soulder yaw: %d, soulder pitch: %d, elbow yaw: %d, rightShoulderTheta > 0: %d", rightShoulderYaw - rightShoulderYawNow, rightShoulderPitch - rightShoulderPitchNow, rightElbowYaw - rightElbowYawNow, rightShoulderTheta);
glColor3f(1.0f, 0.0f, 0.5f);
glRasterPos2i(20, 180);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
}
if((leftShoulderYawDiff < moveLimitDegree) && (leftShoulderPitchDiff < moveLimitDegree) && (leftElbowThetaDiff < moveLimitDegree) && (leftElbowYawDiff < moveLimitDegree) && (leftShoulderTheta >= 0)) {
leftShoulderYaw = leftShoulderYawNow;
leftShoulderPitch = leftShoulderPitchNow;
leftElbowTheta = leftElbowThetaNow;
leftElbowYaw = leftElbowYawNow;
leftShoulderYawPub = angleHandler(leftShoulderYaw);
leftShoulderPitchPub = angleHandler(leftShoulderPitch);
leftElbowThetaPub = angleHandler(leftElbowTheta);
leftElbowYawPub = angleHandler(leftElbowYaw);
sprintf(buffer,"tracking!");
glColor3f(0.0f, 0.3f, 1.0f);
glRasterPos2i(20, 220);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
actionPublish(rightShoulderYawPub, rightShoulderPitchPub, rightElbowThetaPub, rightElbowYawPub, leftShoulderYawPub, leftShoulderPitchPub, leftElbowThetaPub, leftElbowYawPub);
}
else {
sprintf(buffer,"soulder yaw: %d, soulder pitch: %d, elbow yaw: %d, leftShoulderTheta > 0: %d", leftShoulderYaw - leftShoulderYawNow, leftShoulderPitch - leftShoulderPitchNow, leftElbowYaw - leftElbowYawNow, leftShoulderTheta);
glColor3f(0.7f, 0.8f, 2.0f);
glRasterPos2i(20, 220);
glPrintString(GLUT_BITMAP_HELVETICA_18, buffer);
}
}
if (g_drawFrameId)
{
DrawFrameId(userTrackerFrame.getFrameIndex());
}
if (g_generalMessage[0] != '\0')
{
char *msg = g_generalMessage;
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(100, 20);
glPrintString(GLUT_BITMAP_HELVETICA_18, msg);
}
// Swap the OpenGL display buffers
glutSwapBuffers();
}
void TestApp::OnUpdate()
{
if (mWindow.HandleMessage())
{
mRunning = false;
}
else
{
// Update our time
mTimer.Update();
// Update animation
mAnimationController.Update(mTimer.GetElapsedTime() * 1000.0f);
// Camera movement
const float kMoveSpeed = 10.0f;
const float kTurnSpeed = 5.0f;
if (mKeyStates[VK_UP] || mKeyStates['W'])
{
mCamera.Walk(kMoveSpeed * mTimer.GetElapsedTime());
}
else if (mKeyStates[VK_DOWN] || mKeyStates['S'])
{
mCamera.Walk(-kMoveSpeed * mTimer.GetElapsedTime());
}
else if (mKeyStates[VK_RIGHT] || mKeyStates['D'])
{
mCamera.Strafe(kMoveSpeed * mTimer.GetElapsedTime());
}
else if (mKeyStates[VK_LEFT] || mKeyStates['A'])
{
mCamera.Strafe(-kMoveSpeed * mTimer.GetElapsedTime());
}
else if (mKeyStates['K'])
{
mDrawSkeleton = !mDrawSkeleton;
}
// Render scene
mGraphicsSystem.BeginRender(Color::Black());
mRenderer.SetCamera(mCamera);
if(mDrawSkeleton)
{
DrawSkeleton();
}
// for each mesh
for(u32 i = 0; i < mModel.mMeshes.size(); ++i)
{
const Mesh* mesh = mModel.mMeshes[i];
// copy vertexweights
const VertexWeights& vertexWeights = mesh->GetVertexWeights();
// if vw not empty
if(!vertexWeights.empty())
{
const std::vector<Math::Matrix>& boneTransforms = mAnimationController.BoneTransforms();
// copy vertices
const Mesh::Vertex* vertices = mesh->GetVertices();
// get vertex count
const u32 count = mesh->GetVertexCount();
// create new vertexarray of size vertex count
Mesh::Vertex* newVertices = new Mesh::Vertex[count];
// for each vertex
for(u32 j = 0; j < count; ++j)
{
// create zero transform
Math::Matrix transform;
transform = transform.Zero();
// copy boneweights from this vertexweight
const BoneWeights& boneWeights = vertexWeights[j];
if(i == 1)
{
transform = boneTransforms[22];
}
else
{
// for each boneweight
for(u32 k = 0; k < boneWeights.size(); ++k)
{
const BoneWeight& boneWeight = boneWeights[k];
transform = transform + boneTransforms[i == 1 ? 22 : boneWeight.boneIndex] * boneWeight.weight;
}
}
// insert position into newVertices
newVertices[j].position = Math::TransformCoord(vertices[j].position, transform);
// insert normal into newVertices
newVertices[j].normal = Math::Normalize(Math::TransformNormal(vertices[j].normal, transform));
// insert texcoord into newVertices
newVertices[j].texcoord = vertices[j].texcoord;
}
// update mesh buffer at index with newVertices
mModel.mMeshBuffers[i]->UpdateBuffer(mGraphicsSystem, newVertices, count);
// safedelete newVertices
SafeDeleteArray(newVertices);
}
}
mModel.Render(mRenderer);
SimpleDraw::Render(mCamera);
mGraphicsSystem.EndRender();
}
}
void SampleViewer::Display()
{
if (g_pause)
return;
nite::UserTrackerFrameRef userTrackerFrame;
nite::Status rc1 = m_pUserTracker->readFrame(&userTrackerFrame);
if (rc1 != nite::STATUS_OK) {
printf("GetNextData failed\n");
return;
}
openni::VideoFrameRef depthFrameSide = userTrackerFrame.getDepthFrame();
int height = depthFrameSide.getHeight();
int width = depthFrameSide.getWidth();
if (!label) {
label = (int *)malloc(width*height*sizeof(int));
}
openni::VideoFrameRef depthFrameTop;
openni::Status rc2 = depthStreamTop.readFrame(&depthFrameTop);
if (rc2 != openni::STATUS_OK) {
printf("GetNextData failed\n");
return;
}
if (m_pTexMap == NULL)
{
// Texture map init
m_nTexMapX = MIN_CHUNKS_SIZE(depthFrameSide.getVideoMode().getResolutionX(), TEXTURE_SIZE);
m_nTexMapY = MIN_CHUNKS_SIZE(depthFrameSide.getVideoMode().getResolutionY(), TEXTURE_SIZE);
m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY];
}
const nite::UserMap& userLabels = userTrackerFrame.getUserMap();
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_PROJECTION);
glPushMatrix();
glLoadIdentity();
glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -1.0, 1.0);
if (depthFrameSide.isValid() && g_drawDepth)
{
calculateHistogram(m_pDepthHistSide, MAX_DEPTH, depthFrameSide);
}
memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel));
float factor[3] = {1, 1, 1};
// check if we need to draw depth frame to texture
if (depthFrameSide.isValid() && g_drawDepth)
{
const nite::UserId* pLabels = userLabels.getPixels();
const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrameSide.getData();
openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrameSide.getCropOriginY() * m_nTexMapX;
int rowSize = depthFrameSide.getStrideInBytes() / sizeof(openni::DepthPixel);
for (int y = 0; y < height; ++y)
{
const openni::DepthPixel* pDepth = pDepthRow;
openni::RGB888Pixel* pTex = pTexRow + depthFrameSide.getCropOriginX();
for (int x = 0; x < width; ++x, ++pDepth, ++pTex, ++pLabels)
{
if (*pDepth != 0)
{
if (*pLabels == 0)
{
if (!g_drawBackground)
{
factor[0] = factor[1] = factor[2] = 0;
}
else
{
factor[0] = Colors[colorCount][0];
factor[1] = Colors[colorCount][1];
factor[2] = Colors[colorCount][2];
}
}
else
{
factor[0] = Colors[*pLabels % colorCount][0];
factor[1] = Colors[*pLabels % colorCount][1];
factor[2] = Colors[*pLabels % colorCount][2];
}
// // Add debug lines - every 10cm
// else if ((*pDepth / 10) % 10 == 0)
// {
// factor[0] = factor[2] = 0;
// }
int nHistValue = m_pDepthHistSide[*pDepth];
pTex->r = nHistValue*factor[0];
pTex->g = nHistValue*factor[1];
pTex->b = nHistValue*factor[2];
factor[0] = factor[1] = factor[2] = 1;
}
}
pDepthRow += rowSize;
pTexRow += m_nTexMapX;
}
}
const openni::DepthPixel *imgBufferSide = (const openni::DepthPixel *)depthFrameSide.getData();
const openni::DepthPixel *imgBufferTop = (const openni::DepthPixel *)depthFrameTop.getData();
calculateHistogram(m_pDepthHistTop, MAX_DEPTH, depthFrameTop);
imgTop = Mat(depthFrameTop.getHeight(), depthFrameTop.getWidth(), CV_8UC3);
Mat(depthFrameTop.getHeight(), depthFrameTop.getWidth(), CV_16U, (void *)imgBufferTop).convertTo(depthTop, CV_8U, 1.0/256);
for (int i = 0; i < imgTop.rows; i++) {
for (int j = 0; j < imgTop.cols; j++) {
int val = (int)m_pDepthHistTop[imgBufferTop[j + i*imgTop.cols]];
imgTop.at<Vec3b>(i, j).val[0] = val;
imgTop.at<Vec3b>(i, j).val[1] = val;
imgTop.at<Vec3b>(i, j).val[2] = val;
}
}
if (g_getBackground)
bgSubtractor->processImages(depthTop);
bgSubtractor->getMask(depthTop, mask);
imshow("Mask", mask);
glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap);
// Display the OpenGL texture map
glColor4f(1,1,1,1);
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
// 320x240
g_nXRes = depthFrameSide.getVideoMode().getResolutionX();
g_nYRes = depthFrameSide.getVideoMode().getResolutionY();
// upper left
glTexCoord2f(0, 0);
glVertex2f(0, 0);
// upper right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0);
glVertex2f(GL_WIN_SIZE_X, 0);
// bottom right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y);
// bottom left
glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(0, GL_WIN_SIZE_Y);
glEnd();
glDisable(GL_TEXTURE_2D);
const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers();
float maxSize = -1;
int maxIdx = -1;
for (int i = 0; i < users.getSize(); ++i) {
const nite::UserData &user = users[i];
if (!user.isVisible())
continue;
if (getSize(user) > maxSize) {
maxSize = getSize(user);
maxIdx = i;
}
//printf("user %d: size=%f\n, lost=%d, new=%d, visible=%d\n",
// i, getSize(user), user.isLost(), user.isNew(), user.isVisible());
}
for (int i = 0; i < users.getSize(); ++i)
{
const nite::UserData &user = users[i];
updateUserState(user, userTrackerFrame.getTimestamp());
if (user.isNew())
{
m_pUserTracker->startSkeletonTracking(user.getId());
m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS);
}
else if (!user.isLost())
{
if (g_drawStatusLabel) {
DrawStatusLabel(m_pUserTracker, user);
}
if (g_drawCenterOfMass) {
DrawCenterOfMass(m_pUserTracker, user);
}
if (g_drawBoundingBox) {
DrawBoundingBox(user);
}
if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton) {
if (maxIdx == i) {
DrawSkeleton(m_pUserTracker, user);
sideSkel.setTo(Scalar(0, 0, 0));
drawSkeleton(sideSkel, sideJoints);
topSkel.setTo(Scalar(0, 0, 0));
drawSkeleton(topSkel, topJoints);
drawSkeleton(imgTop, topJoints);
}
}
}
// exit the program after a few seconds if PoseType == POSE_CROSSED_HANDS
if (m_poseUser == 0 || m_poseUser == user.getId())
{
const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS);
if (pose.isEntered())
{
// Start timer
sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s");
printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000);
m_poseUser = user.getId();
m_poseTime = userTrackerFrame.getTimestamp();
}
else if (pose.isExited())
{
memset(g_generalMessage, 0, sizeof(g_generalMessage));
printf("Count-down interrupted\n");
m_poseTime = 0;
m_poseUser = 0;
}
else if (pose.isHeld())
{
// tick
if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000)
{
printf("Count down complete. Exit...\n");
Finalize();
exit(2);
}
}
}
}
if (g_drawFrameId)
{
DrawFrameId(userTrackerFrame.getFrameIndex());
}
if (g_generalMessage[0] != '\0')
{
char *msg = g_generalMessage;
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(100, 20);
glPrintString(GLUT_BITMAP_HELVETICA_18, msg);
}
imshow("Side", sideSkel);
imshow("Top", topSkel);
imshow("DepthTop", imgTop);
if (!g_getBackground) {
knnsearch(topJoints, imgBufferTop, mask, labelTop, label, 320, 240);
drawSkeleton(labelTop, topJoints);
cv::resize(labelTop, labelTop, Size(), 2, 2);
imshow("Label", labelTop);
if (g_capture2) {
// c style
string path = outDir + "/depth-top" + to_string(nFrame) + ".txt";
FILE *f = fopen(path.c_str(), "w");
for (int i = 0; i < width*height; i++) {
fprintf(f, "%u\n", imgBufferTop[i]);
}
fclose(f);
path = outDir + "/depth-side" + to_string(nFrame) + ".txt";
f = fopen(path.c_str(), "w");
for (int i = 0; i < width*height; i++) {
fprintf(f, "%u\n", imgBufferSide[i]);
}
fclose(f);
path = outDir + "/joints-top" + to_string(nFrame) + ".txt";
f = fopen(path.c_str(), "w");
for (int i = 0; i < N_JOINTS; i++) {
fprintf(f, "%f, %f, %f, %f, %f\n", topJoints[i][0], topJoints[i][1],
topJoints[i][2], topJoints[i][3], topJoints[i][4]);
}
fclose(f);
path = outDir + "/joints-side" + to_string(nFrame) + ".txt";
f = fopen(path.c_str(), "w");
for (int i = 0; i < N_JOINTS; i++) {
fprintf(f, "%f, %f, %f, %f, %f\n", sideJoints[i][0], sideJoints[i][1],
sideJoints[i][2], sideJoints[i][3], sideJoints[i][4]);
}
fclose(f);
path = outDir + "/label-top" + to_string(nFrame) + ".txt";
f = fopen(path.c_str(), "w");
for (int i = 0; i < width*height; i++) {
fprintf(f, "%d\n", label[i]);
}
fclose(f);
path = outDir + "/label-side" + to_string(nFrame) + ".txt";
f = fopen(path.c_str(), "w");
const nite::UserId* labelsTop = userLabels.getPixels();
for (int i = 0; i < width*height; i++) {
fprintf(f, "%d\n", (int)labelsTop[i]);
}
fclose(f);
nFrame++;
}
}
// Swap the OpenGL display buffers
glutSwapBuffers();
}
void md5_render(md5_model_t md5)
{
const struct md5_mesh *mesh = md5->mesh;
vector_t org, rot;
unsigned int i;
if ( !AllocVertexArrays(mesh) )
return;
v_copy(org, md5->ent.origin);
v_copy(rot, md5->ent.angles);
if (md5->anim) {
/* TODO: interpolate motion */
Animate(md5);
}
glTranslatef(org[X], org[Y], org[Z]);
glRotatef(rot[X] - 90, 1, 0, 0);
glRotatef(rot[Y], 0, 1, 0);
glRotatef(rot[Z], 0, 0, 1);
glCullFace(GL_FRONT);
glColor4f(1.0, 1.0, 1.0, 1.0);
glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);
glEnableClientState(GL_VERTEX_ARRAY);
glEnableClientState(GL_TEXTURE_COORD_ARRAY);
//glEnableClientState(GL_VERTEX_ATTRIB_ARRAY);
//glEnableVertexAttribArray();
glEnableClientState(GL_NORMAL_ARRAY);
glUseProgram(prog);
glUniform1iARB(u_Texture,0);
glUniform1iARB(u_normalTexture,1);
/* Draw each mesh of the model */
for (i = 0; i < mesh->num_meshes; ++i) {
PrepareMesh(&mesh->meshes[i], md5->skeleton);
glActiveTextureARB(GL_TEXTURE0);
tex_bind(mesh->meshes[i].skin);
glActiveTextureARB(GL_TEXTURE1);
tex_bind(mesh->meshes[i].normalmap);
glActiveTextureARB(GL_TEXTURE0);
glVertexPointer(3, GL_FLOAT, 0, vertexArray);
glNormalPointer(GL_FLOAT, 0, normalArray);
glTexCoordPointer(2, GL_FLOAT, 0, texArray);
// glVertexAttribPointerARB(tangent_attrib, 3, GL_FLOAT, GL_FALSE, 0, tangentArray);
glDrawElements(GL_TRIANGLES, mesh->meshes[i].num_tris * 3,
GL_UNSIGNED_INT, vertexIndices);
}
glUseProgram(0);
glDisableClientState(GL_VERTEX_ARRAY);
glDisableClientState(GL_TEXTURE_COORD_ARRAY);
#if 0
/* Draw skeleton */
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
glDisable(GL_TEXTURE_2D);
glDisable(GL_CULL_FACE);
DrawSkeleton(skeleton, md5file.num_joints);
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
glEnable(GL_TEXTURE_2D);
#endif
glCullFace(GL_BACK);
}
void SampleViewer::Display()
{
nite::UserTrackerFrameRef userTrackerFrame;
openni::VideoFrameRef depthFrame;
nite::Status rc = m_pUserTracker->readFrame(&userTrackerFrame);
if (rc != nite::STATUS_OK)
{
printf("GetNextData failed\n");
return;
}
depthFrame = userTrackerFrame.getDepthFrame();
m_colorStream.readFrame(&m_colorFrame);
if (m_pTexMap == NULL)
{
// Texture map init
m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE);
m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE);
m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY];
}
const nite::UserMap& userLabels = userTrackerFrame.getUserMap();
glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_PROJECTION);
glPushMatrix();
glLoadIdentity();
glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -2000.0, 2000.0);
if (depthFrame.isValid() && g_drawDepth)
{
calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame);
}
memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel));
// check if we need to draw image frame to texture
if (m_eViewState == DISPLAY_MODE_IMAGE && m_colorFrame.isValid())
{
const openni::RGB888Pixel* pImageRow = (const openni::RGB888Pixel*)m_colorFrame.getData();
openni::RGB888Pixel* pTexRow = m_pTexMap + m_colorFrame.getCropOriginY() * m_nTexMapX;
int rowSize = m_colorFrame.getStrideInBytes() / sizeof(openni::RGB888Pixel);
for (int y = 0; y < m_colorFrame.getHeight(); ++y)
{
const openni::RGB888Pixel* pImage = pImageRow;
openni::RGB888Pixel* pTex = pTexRow + m_colorFrame.getCropOriginX();
for (int x = 0; x < m_colorFrame.getWidth(); ++x, ++pImage, ++pTex)
{
*pTex = *pImage;
}
pImageRow += rowSize;
pTexRow += m_nTexMapX;
}
}
float factor[3] = {1, 1, 1};
// check if we need to draw depth frame to texture
if (depthFrame.isValid() && g_drawDepth)
{
const nite::UserId* pLabels = userLabels.getPixels();
const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData();
openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX;
int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel);
for (int y = 0; y < depthFrame.getHeight(); ++y)
{
const openni::DepthPixel* pDepth = pDepthRow;
openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX();
for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex, ++pLabels)
{
if (*pDepth != 0)
{
if (*pLabels == 0)
{
if (!g_drawBackground)
{
factor[0] = factor[1] = factor[2] = 0;
}
else
{
factor[0] = Colors[colorCount][0];
factor[1] = Colors[colorCount][1];
factor[2] = Colors[colorCount][2];
}
}
else
{
factor[0] = Colors[*pLabels % colorCount][0];
factor[1] = Colors[*pLabels % colorCount][1];
factor[2] = Colors[*pLabels % colorCount][2];
}
// // Add debug lines - every 10cm
// else if ((*pDepth / 10) % 10 == 0)
// {
// factor[0] = factor[2] = 0;
// }
int nHistValue = m_pDepthHist[*pDepth];
pTex->r = nHistValue*factor[0];
pTex->g = nHistValue*factor[1];
pTex->b = nHistValue*factor[2];
factor[0] = factor[1] = factor[2] = 1;
}
}
pDepthRow += rowSize;
pTexRow += m_nTexMapX;
}
}
glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap);
// Display the OpenGL texture map
glColor4f(1,1,1,1);
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
g_nXRes = depthFrame.getVideoMode().getResolutionX();
g_nYRes = depthFrame.getVideoMode().getResolutionY();
// upper left
glTexCoord2f(0, 0);
glVertex2f(0, 0);
// upper right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0);
glVertex2f(GL_WIN_SIZE_X, 0);
// bottom right
glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y);
// bottom left
glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY);
glVertex2f(0, GL_WIN_SIZE_Y);
glEnd();
glDisable(GL_TEXTURE_2D);
const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers();
for (int i = 0; i < users.getSize(); ++i)
{
const nite::UserData& user = users[i];
updateUserState(user, userTrackerFrame.getTimestamp());
if (user.isNew())
{
m_pUserTracker->startSkeletonTracking(user.getId());
m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS);
}
else if (!user.isLost())
{
if (g_drawStatusLabel)
{
DrawStatusLabel(m_pUserTracker, user);
}
if (g_drawCenterOfMass)
{
DrawCenterOfMass(m_pUserTracker, user);
}
if (g_drawBoundingBox)
{
DrawBoundingBox(user);
}
if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton)
{
DrawSkeleton(m_pUserTracker, user);
}
if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawHat)
{
DrawHat(m_pUserTracker, user);
}
if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawCube)
{
DrawCube(m_pUserTracker, user);
}
if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawCubeFront)
{
DrawCubeFront(m_pUserTracker, user);
}
}
if (m_poseUser == 0 || m_poseUser == user.getId())
{
const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS);
if (pose.isEntered())
{
// Start timer
sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s");
printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000);
m_poseUser = user.getId();
m_poseTime = userTrackerFrame.getTimestamp();
}
else if (pose.isExited())
{
memset(g_generalMessage, 0, sizeof(g_generalMessage));
printf("Count-down interrupted\n");
m_poseTime = 0;
m_poseUser = 0;
}
else if (pose.isHeld())
{
// tick
if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000)
{
printf("Count down complete. Exit...\n");
Finalize();
exit(2);
}
}
}
}
if (g_drawFrameId)
{
DrawFrameId(userTrackerFrame.getFrameIndex());
}
if (g_generalMessage[0] != '\0')
{
char *msg = g_generalMessage;
glColor3f(1.0f, 0.0f, 0.0f);
glRasterPos2i(100, 20);
glPrintString(GLUT_BITMAP_HELVETICA_18, msg);
}
// Swap the OpenGL display buffers
glutSwapBuffers();
}
/// <summary>
/// Handle new skeleton data
/// </summary>
void CSkeletonBasics::ProcessSkeleton()
{
NUI_SKELETON_FRAME skeletonFrame = {0};
HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame);
if ( FAILED(hr) )
{
return;
}
// smooth out the skeleton data
m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL);
// Endure Direct2D is ready to draw
hr = EnsureDirect2DResources( );
if ( FAILED(hr) )
{
return;
}
m_pRenderTarget->BeginDraw();
m_pRenderTarget->Clear( );
RECT rct;
GetClientRect( GetDlgItem( m_hWnd, IDC_VIDEOVIEW ), &rct);
int width = rct.right;
int height = rct.bottom;
for (int i = 0 ; i < NUI_SKELETON_COUNT; ++i)
{
NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState;
if (NUI_SKELETON_TRACKED == trackingState)
{
// We're tracking the skeleton, draw it
DrawSkeleton(skeletonFrame.SkeletonData[i], width, height);
// Write commands into keyboard buffer depending on gesture
if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y))
{
SetStatusMessage(L"rechte Hand ist oben");
// Write "D" in keyboard buffer
//keybd_event(0x44, 0x20, 0, 0); //1. Ascii Tabelle 2. Scan Code
unsigned char berndmessage[] = "d";
port1.sendArray(berndmessage, 1);
unsigned char heinrichmessage[] = "d";
port2.sendArray2(heinrichmessage, 1);
}
if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "A" in keyboard buffer
SetStatusMessage(L"linke Hand ist oben");
// keybd_event(0x41, 0x1E, 0, 0);
unsigned char berndmessage[] = "a";
port1.sendArray(berndmessage, 1);
unsigned char heinrichmessage[] = "a";
port2.sendArray2(heinrichmessage, 1);
}
if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "W" in keyboard buffer
SetStatusMessage(L"beide Hände sind oben");
//putchar('w');
//keybd_event( 0x77, 0, KEYEVENTF_EXTENDEDKEY | 0,0 ); --> gibt immerhin die keys aus
//keybd_event(0x57, 0x11, 0, 0 ); //Fährt!!
unsigned char berndmessage[] = "w";
port1.sendArray(berndmessage, 1);
unsigned char heinrichmessage[] = "w";
port2.sendArray2(heinrichmessage, 1);
}
if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y))
{
// Write "S" in keyboard buffer
SetStatusMessage(L"beide Hände unterhalb der Knie");
//keybd_event(0x53, 0x1F, 0, 0);
unsigned char berndmessage[] = "s";
port1.sendArray(berndmessage, 1);
unsigned char heinrichmessage[] = "s";
port2.sendArray2(heinrichmessage, 1);
}
if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "P" in keyboard buffer
SetStatusMessage(L"beide Hände sind unten");
//keybd_event(0x4F, 0x18, 0, 0);
unsigned char berndmessage[] = "p";
port1.sendArray(berndmessage, 1);
unsigned char heinrichmessage[] = "p";
port2.sendArray2(heinrichmessage, 1);
}
}
else if (NUI_SKELETON_POSITION_ONLY == trackingState)
{
// we've only received the center point of the skeleton, draw that
D2D1_ELLIPSE ellipse = D2D1::Ellipse(
SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height),
g_JointThickness,
g_JointThickness
);
m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked);
}
}
/*
// ---> ANFANG zweite Person
for (int j = 0; j < NUI_SKELETON_COUNT; ++j)
{
NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[j].eTrackingState;
if (NUI_SKELETON_TRACKED == trackingState)
{
// We're tracking the skeleton, draw it
DrawSkeleton(skeletonFrame.SkeletonData[j], width, height);
// Write commands into keyboard buffer depending on gesture
if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y))
{
SetStatusMessage(L"zweite Person: rechte Hand ist oben");
// Write "K" in keyboard buffer
keybd_event(0x4B, 0x25, 0, 0); //1. Ascii Tabelle 2. Scan Code
}
if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "H" in keyboard buffer
SetStatusMessage(L"zweite Person: linke Hand ist oben");
keybd_event(0x48, 0x23, 0, 0);
}
if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "U" in keyboard buffer
SetStatusMessage(L"zweite Person: beide Hände sind oben");
keybd_event(0x55, 0x16, 0, 0); //Fährt!!
}
if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y))
{
// Write "J" in keyboard buffer
SetStatusMessage(L"zweite Person: beide Hände unterhalb der Knie");
keybd_event(0x4A, 0x24, 0, 0);
}
if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) &&
(skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y))
{
// Write "P" in keyboard buffer
SetStatusMessage(L"zweite Person: beide Hände sind unten");
keybd_event(0x50, 0x19, 0, 0);
}
}
else if (NUI_SKELETON_POSITION_ONLY == trackingState)
{
// we've only received the center point of the skeleton, draw that
D2D1_ELLIPSE ellipse = D2D1::Ellipse(
SkeletonToScreen(skeletonFrame.SkeletonData[j].Position, width, height),
g_JointThickness,
g_JointThickness
);
m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked);
}
}
// --> ENDE zweite Person
*/
hr = m_pRenderTarget->EndDraw();
// Device lost, need to recreate the render target
// We'll dispose it now and retry drawing
if (D2DERR_RECREATE_TARGET == hr)
{
hr = S_OK;
DiscardDirect2DResources();
}
}
/// <summary>
/// Handle new skeleton data
/// </summary>
void CDataCollection::ProcessSkeleton()
{
NUI_SKELETON_FRAME skeletonFrame = {0};
HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame);
if(m_bSaveSkeStart)
{
int framecount = m_pSkeData->GetFrameSaved();
::SetDlgItemInt(m_hWnd, static_cast<int>(IDC_FRAME_SHOW),framecount, FALSE );
}
if ( FAILED(hr) )
{
return;
}
// smooth out the skeleton data
m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL);
//m_pSkeData->AddOneFrame(skeletonFrame);
for (int i = 0 ; i < NUI_SKELETON_COUNT; i++)
{
NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState;
if (NUI_SKELETON_TRACKED ==trackingState)
{
m_pSkeData->AddOneFrame(skeletonFrame.SkeletonData[i].SkeletonPositions);
break;
}
}
if(m_bSaveSkeStart) {
if (m_pSkeData->SavingIsStopped()) {
KillTimer (m_hWnd, TIMER_SEC) ;
StringCchPrintf( statusmsg, cStatusMessageMaxLen, L"Skeleton video saved to %s", skefname);
MessageBox(m_hWnd, statusmsg, L"File saved!", MB_OK);
m_bSaveSkeStart=false;
}
}
// Endure Direct2D is ready to draw
hr = EnsureDirect2DResources( );
if ( FAILED(hr) )
{
return;
}
m_pRenderTarget->BeginDraw();
m_pRenderTarget->Clear( );
RECT rct;
GetClientRect( GetDlgItem( m_hWnd, IDC_SKELETON ), &rct);
int width = rct.right;
int height = rct.bottom;
// WCHAR statusMessage[cStatusMessageMaxLen];
int i;
//currentframetime=GetTickCount();
for ( i = 0 ; i < NUI_SKELETON_COUNT; ++i)
{
NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState;
if (NUI_SKELETON_TRACKED == trackingState)
{
// We're tracking the skeleton, draw it
DrawSkeleton(skeletonFrame.SkeletonData[i], width, height);
break;
}
else if (NUI_SKELETON_POSITION_ONLY == trackingState)
{
// we've only received the center point of the skeleton, draw that
D2D1_ELLIPSE ellipse = D2D1::Ellipse(
SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height),
g_JointThickness,
g_JointThickness
);
m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked);
}
}
hr = m_pRenderTarget->EndDraw();
// Device lost, need to recreate the render target
// We'll dispose it now and retry drawing
if (D2DERR_RECREATE_TARGET == hr)
{
hr = S_OK;
DiscardDirect2DResources();
}
}
