/// <summary> /// Handle new skeleton data /// </summary> void CSkeletonBasics::ProcessSkeleton() { NUI_SKELETON_FRAME skeletonFrame = {0}; HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame); if ( FAILED(hr) ) { return; } // smooth out the skeleton data m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL); // Endure Direct2D is ready to draw hr = EnsureDirect2DResources( ); if ( FAILED(hr) ) { return; } m_pRenderTarget->BeginDraw(); m_pRenderTarget->Clear( ); RECT rct; GetClientRect( GetDlgItem( m_hWnd, IDC_VIDEOVIEW ), &rct); int width = rct.right; int height = rct.bottom; for (int i = 0 ; i < NUI_SKELETON_COUNT; ++i) { NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState; if (NUI_SKELETON_TRACKED == trackingState) { // We're tracking the skeleton, draw it DrawSkeleton(skeletonFrame.SkeletonData[i], width, height); } else if (NUI_SKELETON_POSITION_ONLY == trackingState) { // we've only received the center point of the skeleton, draw that D2D1_ELLIPSE ellipse = D2D1::Ellipse( SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height), g_JointThickness, g_JointThickness ); m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked); } } hr = m_pRenderTarget->EndDraw(); // Device lost, need to recreate the render target // We'll dispose it now and retry drawing if (D2DERR_RECREATE_TARGET == hr) { hr = S_OK; DiscardDirect2DResources(); } }
void JModel::Render( const Mat4& tm, JBoneInstance* pSkelInst, bool bIgnoreShader, bool bUseCachedData ) { if (m_IBufID == -1) { Init(); } // check geometry caching status, update if needed g_pRenderServer->CacheIB( m_IBufID, m_Indices.GetData(), m_Indices.GetSize(), m_IBIteration, m_IBFirstByte ); int nBones = m_SkelInst.size(); // calculate world-space transforms for bones if (!pSkelInst && nBones > 0) pSkelInst = &m_SkelInst[0]; UpdateSkeleton( Mat4::identity, pSkelInst ); // cycle through geometries and render them int nGeom = m_Meshes.size(); int cMtl = -1; for (int i = 0; i < nGeom; i++) { JMesh* pMesh = m_Meshes[i]; if (!pMesh->IsVisible()) { continue; } pMesh->CacheVB(); int mtlID = pMesh->GetMaterialID(); if (mtlID >= 0 && mtlID != cMtl) { JMaterial* pMtl = m_Materials[mtlID]; pMtl->Render( bIgnoreShader ); cMtl = mtlID; } Mat4 worldTM = tm; int hostBoneID = pMesh->GetHostBoneID(); if (hostBoneID >= 0 && pMesh->GetNSkinBones() == 0) { JBoneInstance& hb = pSkelInst[hostBoneID]; if (!hb.m_bVisible) { continue; } Mat4 rootTM = hb.m_ModelTM*tm; worldTM = rootTM; } pMesh->Render( m_IBFirstByte, pSkelInst, worldTM, bUseCachedData ); } if (m_bDrawSkeleton) { DrawSkeleton( tm, pSkelInst ); } } // JModel::Render
static inline void DebugRenderPass(RenderState *rs) { auto settings = GetDebugRenderSettings(); if (settings->drawSkeletons) { for (size_t i = 0; i < rs->drawList.animated_model_draw_commands.count; i++) { Animated_Model_Draw_Command *cmd = &rs->drawList.animated_model_draw_commands[i]; ModelAsset *model = cmd->model; assert(model != nullptr); DrawSkeleton(model->joints, model->jointCount, cmd->animation_state, cmd->model_matrix); } } }
void DrawContent() { DrawMesh(); glPointSize(5.0); DrawSkeleton(); }
void SampleViewer::Display() { // namespace bg = boost::geometry; nite::UserTrackerFrameRef userTrackerFrame; openni::VideoFrameRef depthFrame; nite::Status rc = m_pUserTracker->readFrame(&userTrackerFrame); if (rc != nite::STATUS_OK) { printf("GetNextData failed\n"); return; } depthFrame = userTrackerFrame.getDepthFrame(); if (m_pTexMap == NULL) { // Texture map init m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE); m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE); m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY]; } const nite::UserMap& userLabels = userTrackerFrame.getUserMap(); glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glMatrixMode(GL_PROJECTION); glPushMatrix(); glLoadIdentity(); glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -1.0, 1.0); if (depthFrame.isValid() && g_drawDepth) { calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame); } memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel)); float factor[3] = {1, 1, 1}; // check if we need to draw depth frame to texture if (depthFrame.isValid() && g_drawDepth) { const nite::UserId* pLabels = userLabels.getPixels(); const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData(); openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX; int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel); for (int y = 0; y < depthFrame.getHeight(); ++y) { const openni::DepthPixel* pDepth = pDepthRow; openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX(); for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex, ++pLabels) { if (*pDepth != 0) { if (*pLabels == 0) { if (!g_drawBackground) { factor[0] = factor[1] = factor[2] = 0; } else { factor[0] = Colors[colorCount][0]; factor[1] = Colors[colorCount][1]; factor[2] = Colors[colorCount][2]; } } else { factor[0] = Colors[*pLabels % colorCount][0]; factor[1] = Colors[*pLabels % colorCount][1]; factor[2] = Colors[*pLabels % colorCount][2]; } // // Add debug lines - every 10cm // else if ((*pDepth / 10) % 10 == 0) // { // factor[0] = factor[2] = 0; // } int nHistValue = m_pDepthHist[*pDepth]; pTex->r = nHistValue*factor[0]; pTex->g = nHistValue*factor[1]; pTex->b = nHistValue*factor[2]; factor[0] = factor[1] = factor[2] = 1; } } pDepthRow += rowSize; pTexRow += m_nTexMapX; } } glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap); // Display the OpenGL texture map glColor4f(1,1,1,1); glEnable(GL_TEXTURE_2D); glBegin(GL_QUADS); g_nXRes = depthFrame.getVideoMode().getResolutionX(); g_nYRes = depthFrame.getVideoMode().getResolutionY(); // upper left glTexCoord2f(0, 0); glVertex2f(0, 0); // upper right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0); glVertex2f(GL_WIN_SIZE_X, 0); // bottom right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y); // bottom left glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(0, GL_WIN_SIZE_Y); glEnd(); glDisable(GL_TEXTURE_2D); const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers(); // 11.16.15 remove for loop for tracking only one user // for (int i = 0; i < users.getSize(); ++i) // { //const nite::UserData& user = users[i]; if (users.getSize() > 0) { const nite::UserData& user = users[0]; updateUserState(user, userTrackerFrame.getTimestamp()); if (user.isNew()) { m_pUserTracker->startSkeletonTracking(user.getId()); m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS); } else if (!user.isLost()) { if (g_drawStatusLabel) { DrawStatusLabel(m_pUserTracker, user); } if (g_drawCenterOfMass) { DrawCenterOfMass(m_pUserTracker, user); } if (g_drawBoundingBox) { DrawBoundingBox(user); } if (user.getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton) { DrawSkeleton(m_pUserTracker, user); } } if (m_poseUser == 0 || m_poseUser == user.getId()) { const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS); if (pose.isEntered()) { // Start timer sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s"); printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000); m_poseUser = user.getId(); m_poseTime = userTrackerFrame.getTimestamp(); } else if (pose.isExited()) { memset(g_generalMessage, 0, sizeof(g_generalMessage)); printf("Count-down interrupted\n"); m_poseTime = 0; m_poseUser = 0; } else if (pose.isHeld()) { // tick if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000) { printf("Count down complete. Exit...\n"); Finalize(); exit(2); } } } //user char buffer[80] = ""; // ---------------- calculate right shoulder 2 DOF ---------------------- int rightShoulderX, rightShoulderY, rightShoulderZ; rightShoulderX = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().x - user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().x; rightShoulderY = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().y - user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().y; rightShoulderZ = user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().z - user.getSkeleton().getJoint(nite::JOINT_RIGHT_SHOULDER).getPosition().z; Spherical rightShoulderSpherical; rightShoulderSpherical = Cartesian2Spherical(rightShoulderX, rightShoulderY, rightShoulderZ); int rightShoulderPhi, rightShoulderTheta; rightShoulderTheta = radian2Degree(rightShoulderSpherical.radianTheta, rightShoulderThetaInit); // horizontal rise it's -60 degree rightShoulderPhi = radian2Degree(rightShoulderSpherical.radianPhi, rightShoulderPitchInit); // when hand's down, it's 70 degree int rightShoulderYawNow = rightShoulderPhi * sin(rightShoulderTheta * PI / 180); int rightShoulderPitchNow = rightShoulderPhi * cos(rightShoulderTheta * PI / 180); sprintf(buffer,"(rightShoulderTheta=%d, rightShoulderYaw=%d, rightShoulderPitch=%d)", rightShoulderTheta, rightShoulderYawNow, rightShoulderPitchNow); glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(20, 20); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); // ------------------- calculate right elbow 2 DOF ----------------------- int rightElbowX, rightElbowY, rightElbowZ; rightElbowX = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().x - user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().x; rightElbowY = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().y - user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().y; rightElbowZ = user.getSkeleton().getJoint(nite::JOINT_RIGHT_HAND).getPosition().z - user.getSkeleton().getJoint(nite::JOINT_RIGHT_ELBOW).getPosition().z; Spherical rightElbowSpherical; rightElbowSpherical = Cartesian2Spherical(rightElbowX, rightElbowY, rightElbowZ); int rightElbowThetaNow = - radian2Degree(rightElbowSpherical.radianTheta, rightElbowThetaInit); // reverse for system int rightElbowYawNow = radian2Degree(rightElbowSpherical.radianPhi, rightElbowYawInit); sprintf(buffer,"(rightElbowThetaNow=%d, rightElbowYawNow=%d)", rightElbowThetaNow, rightElbowYawNow); glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(20, 60); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); // ---------------- calculate left shoulder 2 DOF ---------------------- int leftShoulderX, leftShoulderY, leftShoulderZ; leftShoulderX = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().x - user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().x; leftShoulderY = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().y - user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().y; leftShoulderZ = user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().z - user.getSkeleton().getJoint(nite::JOINT_LEFT_SHOULDER).getPosition().z; Spherical leftShoulderSpherical; leftShoulderSpherical = Cartesian2Spherical(leftShoulderX, leftShoulderY, leftShoulderZ); int leftShoulderPhi, leftShoulderTheta; leftShoulderTheta = radian2Degree(leftShoulderSpherical.radianTheta, leftShoulderThetaInit); // horizontal rise it's -60 degree leftShoulderPhi = radian2Degree(leftShoulderSpherical.radianPhi, leftShoulderPitchInit); // when hand's down, it's 70 degree // need to reverse in left side int leftShoulderYawNow = - leftShoulderPhi * sin(leftShoulderTheta * PI / 180); int leftShoulderPitchNow = - leftShoulderPhi * cos(leftShoulderTheta * PI / 180); sprintf(buffer,"(leftShoulderTheta=%d, leftShoulderYaw=%d, leftShoulderPitch=%d)", leftShoulderTheta, leftShoulderYawNow, leftShoulderPitchNow); glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(20, 100); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); // ------------------- calculate left elbow 2 DOF ----------------------- int leftElbowX, leftElbowY, leftElbowZ; leftElbowX = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().x - user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().x; leftElbowY = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().y - user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().y; leftElbowZ = user.getSkeleton().getJoint(nite::JOINT_LEFT_HAND).getPosition().z - user.getSkeleton().getJoint(nite::JOINT_LEFT_ELBOW).getPosition().z; Spherical leftElbowSpherical; leftElbowSpherical = Cartesian2Spherical(leftElbowX, leftElbowY, leftElbowZ); int leftElbowThetaNow = radian2Degree(leftElbowSpherical.radianTheta, leftElbowThetaInit); int leftElbowYawNow = radian2Degree(leftElbowSpherical.radianPhi, leftElbowYawInit); sprintf(buffer,"(leftElbowTheta=%d, leftElbowYawNow=%d)", leftElbowThetaNow, leftElbowYawNow); glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(20, 140); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); // ---------------- constraint movement and publish message ------------ int rightShoulderYawDiff = abs(rightShoulderYaw - rightShoulderYawNow); int rightShoulderPitchDiff = abs(rightShoulderPitch - rightShoulderPitchNow); int rightElbowThetaDiff = abs(rightElbowTheta - rightElbowThetaNow); int rightElbowYawDiff = abs(rightElbowYaw - rightElbowYawNow); int leftShoulderYawDiff = abs(leftShoulderYaw - leftShoulderYawNow); int leftShoulderPitchDiff = abs(leftShoulderPitch - leftShoulderPitchNow); int leftElbowThetaDiff = abs(leftElbowTheta - leftElbowThetaNow); int leftElbowYawDiff = abs(leftElbowYaw - leftElbowYawNow); if ((rightShoulderYawDiff < moveLimitDegree) && (rightShoulderPitchDiff < moveLimitDegree) && (rightElbowThetaDiff < moveLimitDegree) && (rightElbowYawDiff < moveLimitDegree) && (rightShoulderTheta >= 0)) { // in range then refresh robot angle rightShoulderYaw = rightShoulderYawNow; rightShoulderPitch = rightShoulderPitchNow; rightElbowTheta = rightElbowThetaNow; rightElbowYaw = rightElbowYawNow; // change the angle to 0 to 360 and publish rightShoulderYawPub = angleHandler(rightShoulderYaw); rightShoulderPitchPub = angleHandler(rightShoulderPitch); rightElbowThetaPub = angleHandler(rightElbowTheta); rightElbowYawPub = angleHandler(rightElbowYaw); sprintf(buffer,"tracking!"); glColor3f(0.0f, 0.0f, 1.0f); glRasterPos2i(20, 180); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); actionPublish(rightShoulderYawPub, rightShoulderPitchPub, rightElbowThetaPub, rightElbowYawPub, leftShoulderYawPub, leftShoulderPitchPub, leftElbowThetaPub, leftElbowYawPub); } else { sprintf(buffer,"soulder yaw: %d, soulder pitch: %d, elbow yaw: %d, rightShoulderTheta > 0: %d", rightShoulderYaw - rightShoulderYawNow, rightShoulderPitch - rightShoulderPitchNow, rightElbowYaw - rightElbowYawNow, rightShoulderTheta); glColor3f(1.0f, 0.0f, 0.5f); glRasterPos2i(20, 180); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); } if((leftShoulderYawDiff < moveLimitDegree) && (leftShoulderPitchDiff < moveLimitDegree) && (leftElbowThetaDiff < moveLimitDegree) && (leftElbowYawDiff < moveLimitDegree) && (leftShoulderTheta >= 0)) { leftShoulderYaw = leftShoulderYawNow; leftShoulderPitch = leftShoulderPitchNow; leftElbowTheta = leftElbowThetaNow; leftElbowYaw = leftElbowYawNow; leftShoulderYawPub = angleHandler(leftShoulderYaw); leftShoulderPitchPub = angleHandler(leftShoulderPitch); leftElbowThetaPub = angleHandler(leftElbowTheta); leftElbowYawPub = angleHandler(leftElbowYaw); sprintf(buffer,"tracking!"); glColor3f(0.0f, 0.3f, 1.0f); glRasterPos2i(20, 220); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); actionPublish(rightShoulderYawPub, rightShoulderPitchPub, rightElbowThetaPub, rightElbowYawPub, leftShoulderYawPub, leftShoulderPitchPub, leftElbowThetaPub, leftElbowYawPub); } else { sprintf(buffer,"soulder yaw: %d, soulder pitch: %d, elbow yaw: %d, leftShoulderTheta > 0: %d", leftShoulderYaw - leftShoulderYawNow, leftShoulderPitch - leftShoulderPitchNow, leftElbowYaw - leftElbowYawNow, leftShoulderTheta); glColor3f(0.7f, 0.8f, 2.0f); glRasterPos2i(20, 220); glPrintString(GLUT_BITMAP_HELVETICA_18, buffer); } } if (g_drawFrameId) { DrawFrameId(userTrackerFrame.getFrameIndex()); } if (g_generalMessage[0] != '\0') { char *msg = g_generalMessage; glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(100, 20); glPrintString(GLUT_BITMAP_HELVETICA_18, msg); } // Swap the OpenGL display buffers glutSwapBuffers(); }
void TestApp::OnUpdate() { if (mWindow.HandleMessage()) { mRunning = false; } else { // Update our time mTimer.Update(); // Update animation mAnimationController.Update(mTimer.GetElapsedTime() * 1000.0f); // Camera movement const float kMoveSpeed = 10.0f; const float kTurnSpeed = 5.0f; if (mKeyStates[VK_UP] || mKeyStates['W']) { mCamera.Walk(kMoveSpeed * mTimer.GetElapsedTime()); } else if (mKeyStates[VK_DOWN] || mKeyStates['S']) { mCamera.Walk(-kMoveSpeed * mTimer.GetElapsedTime()); } else if (mKeyStates[VK_RIGHT] || mKeyStates['D']) { mCamera.Strafe(kMoveSpeed * mTimer.GetElapsedTime()); } else if (mKeyStates[VK_LEFT] || mKeyStates['A']) { mCamera.Strafe(-kMoveSpeed * mTimer.GetElapsedTime()); } else if (mKeyStates['K']) { mDrawSkeleton = !mDrawSkeleton; } // Render scene mGraphicsSystem.BeginRender(Color::Black()); mRenderer.SetCamera(mCamera); if(mDrawSkeleton) { DrawSkeleton(); } // for each mesh for(u32 i = 0; i < mModel.mMeshes.size(); ++i) { const Mesh* mesh = mModel.mMeshes[i]; // copy vertexweights const VertexWeights& vertexWeights = mesh->GetVertexWeights(); // if vw not empty if(!vertexWeights.empty()) { const std::vector<Math::Matrix>& boneTransforms = mAnimationController.BoneTransforms(); // copy vertices const Mesh::Vertex* vertices = mesh->GetVertices(); // get vertex count const u32 count = mesh->GetVertexCount(); // create new vertexarray of size vertex count Mesh::Vertex* newVertices = new Mesh::Vertex[count]; // for each vertex for(u32 j = 0; j < count; ++j) { // create zero transform Math::Matrix transform; transform = transform.Zero(); // copy boneweights from this vertexweight const BoneWeights& boneWeights = vertexWeights[j]; if(i == 1) { transform = boneTransforms[22]; } else { // for each boneweight for(u32 k = 0; k < boneWeights.size(); ++k) { const BoneWeight& boneWeight = boneWeights[k]; transform = transform + boneTransforms[i == 1 ? 22 : boneWeight.boneIndex] * boneWeight.weight; } } // insert position into newVertices newVertices[j].position = Math::TransformCoord(vertices[j].position, transform); // insert normal into newVertices newVertices[j].normal = Math::Normalize(Math::TransformNormal(vertices[j].normal, transform)); // insert texcoord into newVertices newVertices[j].texcoord = vertices[j].texcoord; } // update mesh buffer at index with newVertices mModel.mMeshBuffers[i]->UpdateBuffer(mGraphicsSystem, newVertices, count); // safedelete newVertices SafeDeleteArray(newVertices); } } mModel.Render(mRenderer); SimpleDraw::Render(mCamera); mGraphicsSystem.EndRender(); } }
void SampleViewer::Display() { if (g_pause) return; nite::UserTrackerFrameRef userTrackerFrame; nite::Status rc1 = m_pUserTracker->readFrame(&userTrackerFrame); if (rc1 != nite::STATUS_OK) { printf("GetNextData failed\n"); return; } openni::VideoFrameRef depthFrameSide = userTrackerFrame.getDepthFrame(); int height = depthFrameSide.getHeight(); int width = depthFrameSide.getWidth(); if (!label) { label = (int *)malloc(width*height*sizeof(int)); } openni::VideoFrameRef depthFrameTop; openni::Status rc2 = depthStreamTop.readFrame(&depthFrameTop); if (rc2 != openni::STATUS_OK) { printf("GetNextData failed\n"); return; } if (m_pTexMap == NULL) { // Texture map init m_nTexMapX = MIN_CHUNKS_SIZE(depthFrameSide.getVideoMode().getResolutionX(), TEXTURE_SIZE); m_nTexMapY = MIN_CHUNKS_SIZE(depthFrameSide.getVideoMode().getResolutionY(), TEXTURE_SIZE); m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY]; } const nite::UserMap& userLabels = userTrackerFrame.getUserMap(); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glMatrixMode(GL_PROJECTION); glPushMatrix(); glLoadIdentity(); glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -1.0, 1.0); if (depthFrameSide.isValid() && g_drawDepth) { calculateHistogram(m_pDepthHistSide, MAX_DEPTH, depthFrameSide); } memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel)); float factor[3] = {1, 1, 1}; // check if we need to draw depth frame to texture if (depthFrameSide.isValid() && g_drawDepth) { const nite::UserId* pLabels = userLabels.getPixels(); const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrameSide.getData(); openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrameSide.getCropOriginY() * m_nTexMapX; int rowSize = depthFrameSide.getStrideInBytes() / sizeof(openni::DepthPixel); for (int y = 0; y < height; ++y) { const openni::DepthPixel* pDepth = pDepthRow; openni::RGB888Pixel* pTex = pTexRow + depthFrameSide.getCropOriginX(); for (int x = 0; x < width; ++x, ++pDepth, ++pTex, ++pLabels) { if (*pDepth != 0) { if (*pLabels == 0) { if (!g_drawBackground) { factor[0] = factor[1] = factor[2] = 0; } else { factor[0] = Colors[colorCount][0]; factor[1] = Colors[colorCount][1]; factor[2] = Colors[colorCount][2]; } } else { factor[0] = Colors[*pLabels % colorCount][0]; factor[1] = Colors[*pLabels % colorCount][1]; factor[2] = Colors[*pLabels % colorCount][2]; } // // Add debug lines - every 10cm // else if ((*pDepth / 10) % 10 == 0) // { // factor[0] = factor[2] = 0; // } int nHistValue = m_pDepthHistSide[*pDepth]; pTex->r = nHistValue*factor[0]; pTex->g = nHistValue*factor[1]; pTex->b = nHistValue*factor[2]; factor[0] = factor[1] = factor[2] = 1; } } pDepthRow += rowSize; pTexRow += m_nTexMapX; } } const openni::DepthPixel *imgBufferSide = (const openni::DepthPixel *)depthFrameSide.getData(); const openni::DepthPixel *imgBufferTop = (const openni::DepthPixel *)depthFrameTop.getData(); calculateHistogram(m_pDepthHistTop, MAX_DEPTH, depthFrameTop); imgTop = Mat(depthFrameTop.getHeight(), depthFrameTop.getWidth(), CV_8UC3); Mat(depthFrameTop.getHeight(), depthFrameTop.getWidth(), CV_16U, (void *)imgBufferTop).convertTo(depthTop, CV_8U, 1.0/256); for (int i = 0; i < imgTop.rows; i++) { for (int j = 0; j < imgTop.cols; j++) { int val = (int)m_pDepthHistTop[imgBufferTop[j + i*imgTop.cols]]; imgTop.at<Vec3b>(i, j).val[0] = val; imgTop.at<Vec3b>(i, j).val[1] = val; imgTop.at<Vec3b>(i, j).val[2] = val; } } if (g_getBackground) bgSubtractor->processImages(depthTop); bgSubtractor->getMask(depthTop, mask); imshow("Mask", mask); glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap); // Display the OpenGL texture map glColor4f(1,1,1,1); glEnable(GL_TEXTURE_2D); glBegin(GL_QUADS); // 320x240 g_nXRes = depthFrameSide.getVideoMode().getResolutionX(); g_nYRes = depthFrameSide.getVideoMode().getResolutionY(); // upper left glTexCoord2f(0, 0); glVertex2f(0, 0); // upper right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0); glVertex2f(GL_WIN_SIZE_X, 0); // bottom right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y); // bottom left glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(0, GL_WIN_SIZE_Y); glEnd(); glDisable(GL_TEXTURE_2D); const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers(); float maxSize = -1; int maxIdx = -1; for (int i = 0; i < users.getSize(); ++i) { const nite::UserData &user = users[i]; if (!user.isVisible()) continue; if (getSize(user) > maxSize) { maxSize = getSize(user); maxIdx = i; } //printf("user %d: size=%f\n, lost=%d, new=%d, visible=%d\n", // i, getSize(user), user.isLost(), user.isNew(), user.isVisible()); } for (int i = 0; i < users.getSize(); ++i) { const nite::UserData &user = users[i]; updateUserState(user, userTrackerFrame.getTimestamp()); if (user.isNew()) { m_pUserTracker->startSkeletonTracking(user.getId()); m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS); } else if (!user.isLost()) { if (g_drawStatusLabel) { DrawStatusLabel(m_pUserTracker, user); } if (g_drawCenterOfMass) { DrawCenterOfMass(m_pUserTracker, user); } if (g_drawBoundingBox) { DrawBoundingBox(user); } if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton) { if (maxIdx == i) { DrawSkeleton(m_pUserTracker, user); sideSkel.setTo(Scalar(0, 0, 0)); drawSkeleton(sideSkel, sideJoints); topSkel.setTo(Scalar(0, 0, 0)); drawSkeleton(topSkel, topJoints); drawSkeleton(imgTop, topJoints); } } } // exit the program after a few seconds if PoseType == POSE_CROSSED_HANDS if (m_poseUser == 0 || m_poseUser == user.getId()) { const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS); if (pose.isEntered()) { // Start timer sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s"); printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000); m_poseUser = user.getId(); m_poseTime = userTrackerFrame.getTimestamp(); } else if (pose.isExited()) { memset(g_generalMessage, 0, sizeof(g_generalMessage)); printf("Count-down interrupted\n"); m_poseTime = 0; m_poseUser = 0; } else if (pose.isHeld()) { // tick if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000) { printf("Count down complete. Exit...\n"); Finalize(); exit(2); } } } } if (g_drawFrameId) { DrawFrameId(userTrackerFrame.getFrameIndex()); } if (g_generalMessage[0] != '\0') { char *msg = g_generalMessage; glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(100, 20); glPrintString(GLUT_BITMAP_HELVETICA_18, msg); } imshow("Side", sideSkel); imshow("Top", topSkel); imshow("DepthTop", imgTop); if (!g_getBackground) { knnsearch(topJoints, imgBufferTop, mask, labelTop, label, 320, 240); drawSkeleton(labelTop, topJoints); cv::resize(labelTop, labelTop, Size(), 2, 2); imshow("Label", labelTop); if (g_capture2) { // c style string path = outDir + "/depth-top" + to_string(nFrame) + ".txt"; FILE *f = fopen(path.c_str(), "w"); for (int i = 0; i < width*height; i++) { fprintf(f, "%u\n", imgBufferTop[i]); } fclose(f); path = outDir + "/depth-side" + to_string(nFrame) + ".txt"; f = fopen(path.c_str(), "w"); for (int i = 0; i < width*height; i++) { fprintf(f, "%u\n", imgBufferSide[i]); } fclose(f); path = outDir + "/joints-top" + to_string(nFrame) + ".txt"; f = fopen(path.c_str(), "w"); for (int i = 0; i < N_JOINTS; i++) { fprintf(f, "%f, %f, %f, %f, %f\n", topJoints[i][0], topJoints[i][1], topJoints[i][2], topJoints[i][3], topJoints[i][4]); } fclose(f); path = outDir + "/joints-side" + to_string(nFrame) + ".txt"; f = fopen(path.c_str(), "w"); for (int i = 0; i < N_JOINTS; i++) { fprintf(f, "%f, %f, %f, %f, %f\n", sideJoints[i][0], sideJoints[i][1], sideJoints[i][2], sideJoints[i][3], sideJoints[i][4]); } fclose(f); path = outDir + "/label-top" + to_string(nFrame) + ".txt"; f = fopen(path.c_str(), "w"); for (int i = 0; i < width*height; i++) { fprintf(f, "%d\n", label[i]); } fclose(f); path = outDir + "/label-side" + to_string(nFrame) + ".txt"; f = fopen(path.c_str(), "w"); const nite::UserId* labelsTop = userLabels.getPixels(); for (int i = 0; i < width*height; i++) { fprintf(f, "%d\n", (int)labelsTop[i]); } fclose(f); nFrame++; } } // Swap the OpenGL display buffers glutSwapBuffers(); }
void md5_render(md5_model_t md5) { const struct md5_mesh *mesh = md5->mesh; vector_t org, rot; unsigned int i; if ( !AllocVertexArrays(mesh) ) return; v_copy(org, md5->ent.origin); v_copy(rot, md5->ent.angles); if (md5->anim) { /* TODO: interpolate motion */ Animate(md5); } glTranslatef(org[X], org[Y], org[Z]); glRotatef(rot[X] - 90, 1, 0, 0); glRotatef(rot[Y], 0, 1, 0); glRotatef(rot[Z], 0, 0, 1); glCullFace(GL_FRONT); glColor4f(1.0, 1.0, 1.0, 1.0); glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE); glEnableClientState(GL_VERTEX_ARRAY); glEnableClientState(GL_TEXTURE_COORD_ARRAY); //glEnableClientState(GL_VERTEX_ATTRIB_ARRAY); //glEnableVertexAttribArray(); glEnableClientState(GL_NORMAL_ARRAY); glUseProgram(prog); glUniform1iARB(u_Texture,0); glUniform1iARB(u_normalTexture,1); /* Draw each mesh of the model */ for (i = 0; i < mesh->num_meshes; ++i) { PrepareMesh(&mesh->meshes[i], md5->skeleton); glActiveTextureARB(GL_TEXTURE0); tex_bind(mesh->meshes[i].skin); glActiveTextureARB(GL_TEXTURE1); tex_bind(mesh->meshes[i].normalmap); glActiveTextureARB(GL_TEXTURE0); glVertexPointer(3, GL_FLOAT, 0, vertexArray); glNormalPointer(GL_FLOAT, 0, normalArray); glTexCoordPointer(2, GL_FLOAT, 0, texArray); // glVertexAttribPointerARB(tangent_attrib, 3, GL_FLOAT, GL_FALSE, 0, tangentArray); glDrawElements(GL_TRIANGLES, mesh->meshes[i].num_tris * 3, GL_UNSIGNED_INT, vertexIndices); } glUseProgram(0); glDisableClientState(GL_VERTEX_ARRAY); glDisableClientState(GL_TEXTURE_COORD_ARRAY); #if 0 /* Draw skeleton */ glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); glDisable(GL_TEXTURE_2D); glDisable(GL_CULL_FACE); DrawSkeleton(skeleton, md5file.num_joints); glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); glEnable(GL_TEXTURE_2D); #endif glCullFace(GL_BACK); }
void SampleViewer::Display() { nite::UserTrackerFrameRef userTrackerFrame; openni::VideoFrameRef depthFrame; nite::Status rc = m_pUserTracker->readFrame(&userTrackerFrame); if (rc != nite::STATUS_OK) { printf("GetNextData failed\n"); return; } depthFrame = userTrackerFrame.getDepthFrame(); m_colorStream.readFrame(&m_colorFrame); if (m_pTexMap == NULL) { // Texture map init m_nTexMapX = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionX(), TEXTURE_SIZE); m_nTexMapY = MIN_CHUNKS_SIZE(depthFrame.getVideoMode().getResolutionY(), TEXTURE_SIZE); m_pTexMap = new openni::RGB888Pixel[m_nTexMapX * m_nTexMapY]; } const nite::UserMap& userLabels = userTrackerFrame.getUserMap(); glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glMatrixMode(GL_PROJECTION); glPushMatrix(); glLoadIdentity(); glOrtho(0, GL_WIN_SIZE_X, GL_WIN_SIZE_Y, 0, -2000.0, 2000.0); if (depthFrame.isValid() && g_drawDepth) { calculateHistogram(m_pDepthHist, MAX_DEPTH, depthFrame); } memset(m_pTexMap, 0, m_nTexMapX*m_nTexMapY*sizeof(openni::RGB888Pixel)); // check if we need to draw image frame to texture if (m_eViewState == DISPLAY_MODE_IMAGE && m_colorFrame.isValid()) { const openni::RGB888Pixel* pImageRow = (const openni::RGB888Pixel*)m_colorFrame.getData(); openni::RGB888Pixel* pTexRow = m_pTexMap + m_colorFrame.getCropOriginY() * m_nTexMapX; int rowSize = m_colorFrame.getStrideInBytes() / sizeof(openni::RGB888Pixel); for (int y = 0; y < m_colorFrame.getHeight(); ++y) { const openni::RGB888Pixel* pImage = pImageRow; openni::RGB888Pixel* pTex = pTexRow + m_colorFrame.getCropOriginX(); for (int x = 0; x < m_colorFrame.getWidth(); ++x, ++pImage, ++pTex) { *pTex = *pImage; } pImageRow += rowSize; pTexRow += m_nTexMapX; } } float factor[3] = {1, 1, 1}; // check if we need to draw depth frame to texture if (depthFrame.isValid() && g_drawDepth) { const nite::UserId* pLabels = userLabels.getPixels(); const openni::DepthPixel* pDepthRow = (const openni::DepthPixel*)depthFrame.getData(); openni::RGB888Pixel* pTexRow = m_pTexMap + depthFrame.getCropOriginY() * m_nTexMapX; int rowSize = depthFrame.getStrideInBytes() / sizeof(openni::DepthPixel); for (int y = 0; y < depthFrame.getHeight(); ++y) { const openni::DepthPixel* pDepth = pDepthRow; openni::RGB888Pixel* pTex = pTexRow + depthFrame.getCropOriginX(); for (int x = 0; x < depthFrame.getWidth(); ++x, ++pDepth, ++pTex, ++pLabels) { if (*pDepth != 0) { if (*pLabels == 0) { if (!g_drawBackground) { factor[0] = factor[1] = factor[2] = 0; } else { factor[0] = Colors[colorCount][0]; factor[1] = Colors[colorCount][1]; factor[2] = Colors[colorCount][2]; } } else { factor[0] = Colors[*pLabels % colorCount][0]; factor[1] = Colors[*pLabels % colorCount][1]; factor[2] = Colors[*pLabels % colorCount][2]; } // // Add debug lines - every 10cm // else if ((*pDepth / 10) % 10 == 0) // { // factor[0] = factor[2] = 0; // } int nHistValue = m_pDepthHist[*pDepth]; pTex->r = nHistValue*factor[0]; pTex->g = nHistValue*factor[1]; pTex->b = nHistValue*factor[2]; factor[0] = factor[1] = factor[2] = 1; } } pDepthRow += rowSize; pTexRow += m_nTexMapX; } } glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, m_nTexMapX, m_nTexMapY, 0, GL_RGB, GL_UNSIGNED_BYTE, m_pTexMap); // Display the OpenGL texture map glColor4f(1,1,1,1); glEnable(GL_TEXTURE_2D); glBegin(GL_QUADS); g_nXRes = depthFrame.getVideoMode().getResolutionX(); g_nYRes = depthFrame.getVideoMode().getResolutionY(); // upper left glTexCoord2f(0, 0); glVertex2f(0, 0); // upper right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, 0); glVertex2f(GL_WIN_SIZE_X, 0); // bottom right glTexCoord2f((float)g_nXRes/(float)m_nTexMapX, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(GL_WIN_SIZE_X, GL_WIN_SIZE_Y); // bottom left glTexCoord2f(0, (float)g_nYRes/(float)m_nTexMapY); glVertex2f(0, GL_WIN_SIZE_Y); glEnd(); glDisable(GL_TEXTURE_2D); const nite::Array<nite::UserData>& users = userTrackerFrame.getUsers(); for (int i = 0; i < users.getSize(); ++i) { const nite::UserData& user = users[i]; updateUserState(user, userTrackerFrame.getTimestamp()); if (user.isNew()) { m_pUserTracker->startSkeletonTracking(user.getId()); m_pUserTracker->startPoseDetection(user.getId(), nite::POSE_CROSSED_HANDS); } else if (!user.isLost()) { if (g_drawStatusLabel) { DrawStatusLabel(m_pUserTracker, user); } if (g_drawCenterOfMass) { DrawCenterOfMass(m_pUserTracker, user); } if (g_drawBoundingBox) { DrawBoundingBox(user); } if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawSkeleton) { DrawSkeleton(m_pUserTracker, user); } if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawHat) { DrawHat(m_pUserTracker, user); } if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawCube) { DrawCube(m_pUserTracker, user); } if (users[i].getSkeleton().getState() == nite::SKELETON_TRACKED && g_drawCubeFront) { DrawCubeFront(m_pUserTracker, user); } } if (m_poseUser == 0 || m_poseUser == user.getId()) { const nite::PoseData& pose = user.getPose(nite::POSE_CROSSED_HANDS); if (pose.isEntered()) { // Start timer sprintf(g_generalMessage, "In exit pose. Keep it for %d second%s to exit\n", g_poseTimeoutToExit/1000, g_poseTimeoutToExit/1000 == 1 ? "" : "s"); printf("Counting down %d second to exit\n", g_poseTimeoutToExit/1000); m_poseUser = user.getId(); m_poseTime = userTrackerFrame.getTimestamp(); } else if (pose.isExited()) { memset(g_generalMessage, 0, sizeof(g_generalMessage)); printf("Count-down interrupted\n"); m_poseTime = 0; m_poseUser = 0; } else if (pose.isHeld()) { // tick if (userTrackerFrame.getTimestamp() - m_poseTime > g_poseTimeoutToExit * 1000) { printf("Count down complete. Exit...\n"); Finalize(); exit(2); } } } } if (g_drawFrameId) { DrawFrameId(userTrackerFrame.getFrameIndex()); } if (g_generalMessage[0] != '\0') { char *msg = g_generalMessage; glColor3f(1.0f, 0.0f, 0.0f); glRasterPos2i(100, 20); glPrintString(GLUT_BITMAP_HELVETICA_18, msg); } // Swap the OpenGL display buffers glutSwapBuffers(); }
/// <summary> /// Handle new skeleton data /// </summary> void CSkeletonBasics::ProcessSkeleton() { NUI_SKELETON_FRAME skeletonFrame = {0}; HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame); if ( FAILED(hr) ) { return; } // smooth out the skeleton data m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL); // Endure Direct2D is ready to draw hr = EnsureDirect2DResources( ); if ( FAILED(hr) ) { return; } m_pRenderTarget->BeginDraw(); m_pRenderTarget->Clear( ); RECT rct; GetClientRect( GetDlgItem( m_hWnd, IDC_VIDEOVIEW ), &rct); int width = rct.right; int height = rct.bottom; for (int i = 0 ; i < NUI_SKELETON_COUNT; ++i) { NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState; if (NUI_SKELETON_TRACKED == trackingState) { // We're tracking the skeleton, draw it DrawSkeleton(skeletonFrame.SkeletonData[i], width, height); // Write commands into keyboard buffer depending on gesture if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y)) { SetStatusMessage(L"rechte Hand ist oben"); // Write "D" in keyboard buffer //keybd_event(0x44, 0x20, 0, 0); //1. Ascii Tabelle 2. Scan Code unsigned char berndmessage[] = "d"; port1.sendArray(berndmessage, 1); unsigned char heinrichmessage[] = "d"; port2.sendArray2(heinrichmessage, 1); } if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "A" in keyboard buffer SetStatusMessage(L"linke Hand ist oben"); // keybd_event(0x41, 0x1E, 0, 0); unsigned char berndmessage[] = "a"; port1.sendArray(berndmessage, 1); unsigned char heinrichmessage[] = "a"; port2.sendArray2(heinrichmessage, 1); } if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "W" in keyboard buffer SetStatusMessage(L"beide Hände sind oben"); //putchar('w'); //keybd_event( 0x77, 0, KEYEVENTF_EXTENDEDKEY | 0,0 ); --> gibt immerhin die keys aus //keybd_event(0x57, 0x11, 0, 0 ); //Fährt!! unsigned char berndmessage[] = "w"; port1.sendArray(berndmessage, 1); unsigned char heinrichmessage[] = "w"; port2.sendArray2(heinrichmessage, 1); } if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y)) { // Write "S" in keyboard buffer SetStatusMessage(L"beide Hände unterhalb der Knie"); //keybd_event(0x53, 0x1F, 0, 0); unsigned char berndmessage[] = "s"; port1.sendArray(berndmessage, 1); unsigned char heinrichmessage[] = "s"; port2.sendArray2(heinrichmessage, 1); } if ((skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[i].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "P" in keyboard buffer SetStatusMessage(L"beide Hände sind unten"); //keybd_event(0x4F, 0x18, 0, 0); unsigned char berndmessage[] = "p"; port1.sendArray(berndmessage, 1); unsigned char heinrichmessage[] = "p"; port2.sendArray2(heinrichmessage, 1); } } else if (NUI_SKELETON_POSITION_ONLY == trackingState) { // we've only received the center point of the skeleton, draw that D2D1_ELLIPSE ellipse = D2D1::Ellipse( SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height), g_JointThickness, g_JointThickness ); m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked); } } /* // ---> ANFANG zweite Person for (int j = 0; j < NUI_SKELETON_COUNT; ++j) { NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[j].eTrackingState; if (NUI_SKELETON_TRACKED == trackingState) { // We're tracking the skeleton, draw it DrawSkeleton(skeletonFrame.SkeletonData[j], width, height); // Write commands into keyboard buffer depending on gesture if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y)) { SetStatusMessage(L"zweite Person: rechte Hand ist oben"); // Write "K" in keyboard buffer keybd_event(0x4B, 0x25, 0, 0); //1. Ascii Tabelle 2. Scan Code } if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "H" in keyboard buffer SetStatusMessage(L"zweite Person: linke Hand ist oben"); keybd_event(0x48, 0x23, 0, 0); } if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "U" in keyboard buffer SetStatusMessage(L"zweite Person: beide Hände sind oben"); keybd_event(0x55, 0x16, 0, 0); //Fährt!! } if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y)) { // Write "J" in keyboard buffer SetStatusMessage(L"zweite Person: beide Hände unterhalb der Knie"); keybd_event(0x4A, 0x24, 0, 0); } if ((skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_LEFT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y > skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_KNEE_RIGHT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_LEFT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_LEFT].y) && (skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_HAND_RIGHT].y < skeletonFrame.SkeletonData[j].SkeletonPositions[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y)) { // Write "P" in keyboard buffer SetStatusMessage(L"zweite Person: beide Hände sind unten"); keybd_event(0x50, 0x19, 0, 0); } } else if (NUI_SKELETON_POSITION_ONLY == trackingState) { // we've only received the center point of the skeleton, draw that D2D1_ELLIPSE ellipse = D2D1::Ellipse( SkeletonToScreen(skeletonFrame.SkeletonData[j].Position, width, height), g_JointThickness, g_JointThickness ); m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked); } } // --> ENDE zweite Person */ hr = m_pRenderTarget->EndDraw(); // Device lost, need to recreate the render target // We'll dispose it now and retry drawing if (D2DERR_RECREATE_TARGET == hr) { hr = S_OK; DiscardDirect2DResources(); } }
/// <summary> /// Handle new skeleton data /// </summary> void CDataCollection::ProcessSkeleton() { NUI_SKELETON_FRAME skeletonFrame = {0}; HRESULT hr = m_pNuiSensor->NuiSkeletonGetNextFrame(0, &skeletonFrame); if(m_bSaveSkeStart) { int framecount = m_pSkeData->GetFrameSaved(); ::SetDlgItemInt(m_hWnd, static_cast<int>(IDC_FRAME_SHOW),framecount, FALSE ); } if ( FAILED(hr) ) { return; } // smooth out the skeleton data m_pNuiSensor->NuiTransformSmooth(&skeletonFrame, NULL); //m_pSkeData->AddOneFrame(skeletonFrame); for (int i = 0 ; i < NUI_SKELETON_COUNT; i++) { NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState; if (NUI_SKELETON_TRACKED ==trackingState) { m_pSkeData->AddOneFrame(skeletonFrame.SkeletonData[i].SkeletonPositions); break; } } if(m_bSaveSkeStart) { if (m_pSkeData->SavingIsStopped()) { KillTimer (m_hWnd, TIMER_SEC) ; StringCchPrintf( statusmsg, cStatusMessageMaxLen, L"Skeleton video saved to %s", skefname); MessageBox(m_hWnd, statusmsg, L"File saved!", MB_OK); m_bSaveSkeStart=false; } } // Endure Direct2D is ready to draw hr = EnsureDirect2DResources( ); if ( FAILED(hr) ) { return; } m_pRenderTarget->BeginDraw(); m_pRenderTarget->Clear( ); RECT rct; GetClientRect( GetDlgItem( m_hWnd, IDC_SKELETON ), &rct); int width = rct.right; int height = rct.bottom; // WCHAR statusMessage[cStatusMessageMaxLen]; int i; //currentframetime=GetTickCount(); for ( i = 0 ; i < NUI_SKELETON_COUNT; ++i) { NUI_SKELETON_TRACKING_STATE trackingState = skeletonFrame.SkeletonData[i].eTrackingState; if (NUI_SKELETON_TRACKED == trackingState) { // We're tracking the skeleton, draw it DrawSkeleton(skeletonFrame.SkeletonData[i], width, height); break; } else if (NUI_SKELETON_POSITION_ONLY == trackingState) { // we've only received the center point of the skeleton, draw that D2D1_ELLIPSE ellipse = D2D1::Ellipse( SkeletonToScreen(skeletonFrame.SkeletonData[i].Position, width, height), g_JointThickness, g_JointThickness ); m_pRenderTarget->DrawEllipse(ellipse, m_pBrushJointTracked); } } hr = m_pRenderTarget->EndDraw(); // Device lost, need to recreate the render target // We'll dispose it now and retry drawing if (D2DERR_RECREATE_TARGET == hr) { hr = S_OK; DiscardDirect2DResources(); } }