/// <summary> /// /// </summary> void ProcessColor(){ HRESULT hr; NUI_IMAGE_FRAME imageFrame; hr = m_pNuiSensor->NuiImageStreamGetNextFrame(m_pStreamColorHandle, 0, &imageFrame); if (FAILED(hr)) { return; } INuiFrameTexture * pTexture = imageFrame.pFrameTexture; NUI_LOCKED_RECT LockedRect; // Lock the frame data so the Kinect knows not to modify it while we're reading it pTexture->LockRect(0, &LockedRect, NULL, 0); // Make sure we've received valid data if (LockedRect.Pitch != 0) { cv::Mat colorFrame(cHeight, cWidth, CV_8UC3); for(int i = 0; i < cHeight; i++) { uchar *ptr = colorFrame.ptr<uchar>(i); uchar *pBuffer = (uchar*)(LockedRect.pBits) + i * LockedRect.Pitch; for(int j = 0; j < cWidth; j++) { ptr[3*j] = pBuffer[4*j]; ptr[3*j+1] = pBuffer[4*j+1]; ptr[3*j+2] = pBuffer[4*j+2]; } } // Draw image if (m_bShow) { cv::imshow("Color", colorFrame); cv::waitKey(1); } // If m_bRecord if (m_bRecord) { // Retrieve the path to My Photos WCHAR screenshotPath[MAX_PATH]; // Write out the bitmap to disk GetScreenshotFileName(screenshotPath, _countof(screenshotPath), COLOR); std::wstring screenshotPathWStr(screenshotPath); std::string screenshotPathStr(screenshotPathWStr.begin(), screenshotPathWStr.end()); cv::imwrite(screenshotPathStr, colorFrame); } } pTexture->UnlockRect(0); m_pNuiSensor->NuiImageStreamReleaseFrame(m_pStreamColorHandle, &imageFrame); }
BOOL HandleKeyPress(DWORD vKey) { switch(vKey) { case VK_F4: { if(bShowNameTags) { bShowNameTags = FALSE; break; } else { bShowNameTags = TRUE; break; } } case VK_F7: pChatWindow->ToggleEnabled(); break; case VK_F8: { CScreenshot ScreenShot(pD3DDevice); std::string sFileName; GetScreenshotFileName(sFileName); if(ScreenShot.TakeScreenShot((PCHAR)sFileName.c_str())) { pChatWindow->AddInfoMessage("Screenshot Taken - %s",sFileName.c_str()); } else { pChatWindow->AddInfoMessage("Unable to take a screenshot"); } } break; case VK_RETURN: pCmdWindow->ProcessInput(); break; } return FALSE; }
/////////////////// // Take a screenshot static void TakeScreenshot(const std::string& scr_path, const std::string& additional_data) { if (scr_path.empty()) // Check return; notes << "Save screenshot to " << scr_path << endl; std::string extension; // Set the extension switch (tLXOptions->iScreenshotFormat) { case FMT_BMP: extension = ".bmp"; break; case FMT_PNG: extension = ".png"; break; case FMT_JPG: extension = ".jpg"; break; case FMT_GIF: extension = ".gif"; break; default: extension = ".png"; } // Save the surface SaveSurface(VideoPostProcessor::videoBufferSurface(), GetScreenshotFileName(scr_path, extension), tLXOptions->iScreenshotFormat, additional_data); }
/// <summary> /// Handle new depth and color data /// <param name="nTime">timestamp of frame</param> /// <param name="pDepthBuffer">pointer to depth frame data</param> /// <param name="nDepthWidth">width (in pixels) of input depth image data</param> /// <param name="nDepthHeight">height (in pixels) of input depth image data</param> /// <param name="pColorBuffer">pointer to color frame data</param> /// <param name="nColorWidth">width (in pixels) of input color image data</param> /// <param name="nColorHeight">height (in pixels) of input color image data</param> /// <param name="pBodyIndexBuffer">pointer to body index frame data</param> /// <param name="nBodyIndexWidth">width (in pixels) of input body index data</param> /// <param name="nBodyIndexHeight">height (in pixels) of input body index data</param> /// </summary> void CCoordinateMappingBasics::ProcessFrame(INT64 nTime, const UINT16* pDepthBuffer, int nDepthWidth, int nDepthHeight, const RGBQUAD* pColorBuffer, int nColorWidth, int nColorHeight, const BYTE* pBodyIndexBuffer, int nBodyIndexWidth, int nBodyIndexHeight, int nBodyCount, IBody** ppBodies) { if (m_hWnd) { if (!m_nStartTime) { m_nStartTime = nTime; } double fps = 0.0; LARGE_INTEGER qpcNow = {0}; if (m_fFreq) { if (QueryPerformanceCounter(&qpcNow)) { if (m_nLastCounter) { m_nFramesSinceUpdate++; fps = m_fFreq * m_nFramesSinceUpdate / double(qpcNow.QuadPart - m_nLastCounter); } } } WCHAR szStatusMessage[64]; StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L" FPS = %0.2f Time = %I64d", fps, (nTime - m_nStartTime)); if (SetStatusMessage(szStatusMessage, 1000, false)) { m_nLastCounter = qpcNow.QuadPart; m_nFramesSinceUpdate = 0; } } // Make sure we've received valid data if (m_pCoordinateMapper && m_pColorCoordinates && m_pOutputRGBX && pDepthBuffer && (nDepthWidth == cDepthWidth) && (nDepthHeight == cDepthHeight) && pColorBuffer && (nColorWidth == cColorWidth) && (nColorHeight == cColorHeight) && pBodyIndexBuffer && (nBodyIndexWidth == cDepthWidth) && (nBodyIndexHeight == cDepthHeight)) { HRESULT hr = m_pCoordinateMapper->MapDepthFrameToColorSpace(nDepthWidth * nDepthHeight, (UINT16*)pDepthBuffer,nDepthWidth * nDepthHeight, m_pColorCoordinates); if (SUCCEEDED(hr)) { RGBQUAD c_green = {0, 255, 0}; // loop over pixel of the output for (int depthIndex = 0; depthIndex < (nDepthWidth * nDepthHeight); ++depthIndex) { // default setting source to copy from the background pixel const RGBQUAD* pSrc = (m_pBackgroundRGBX) ? (m_pBackgroundRGBX + depthIndex) : &c_green; BYTE player = pBodyIndexBuffer[depthIndex]; // if we're tracking a player for the current pixel, draw from the color camera if (player != 0xff) { // retrieve the depth to color mapping for the current depth pixel ColorSpacePoint colorPoint = m_pColorCoordinates[depthIndex]; // make sure the depth pixel maps to a valid point in color space int colorX = (int)(floor(colorPoint.X + 0.5)); int colorY = (int)(floor(colorPoint.Y + 0.5)); if ((colorX >= 0) && (colorX < nColorWidth) && (colorY >= 0) && (colorY < nColorHeight)) { // calculate index into color array int colorIndex = colorX + (colorY * nColorWidth); // set source for copy to the color pixel pSrc = m_pColorRGBX + colorIndex; } } // write output m_pOutputRGBX[depthIndex] = *pSrc; } // Draw the data with Direct2D m_pDrawCoordinateMapping->Draw(reinterpret_cast<BYTE*>(m_pOutputRGBX), cDepthWidth * cDepthHeight * sizeof(RGBQUAD)); if (m_bSaveScreenshot) { WCHAR szScreenshotPath[MAX_PATH]; // Retrieve the path to My Photos GetScreenshotFileName(szScreenshotPath, _countof(szScreenshotPath)); // Write out the bitmap to disk HRESULT hr = SaveBitmapToFile(reinterpret_cast<BYTE*>(m_pOutputRGBX), nDepthWidth, nDepthHeight, sizeof(RGBQUAD) * 8, szScreenshotPath); WCHAR szStatusMessage[64 + MAX_PATH]; if (SUCCEEDED(hr)) { // Set the status bar to show where the screenshot was saved StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Screenshot saved to %s", szScreenshotPath); } else { StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Failed to write screenshot to %s", szScreenshotPath); } SetStatusMessage(szStatusMessage, 5000, true); // toggle off so we don't save a screenshot again next frame m_bSaveScreenshot = false; } } } D2D1_POINT_2F center; center.x = 400.0; center.y = 100.0; int width = 0; int height = 0; if (m_pCoordinateMapper) { RECT rct; GetClientRect(GetDlgItem(m_hWnd, IDC_VIDEOVIEW), &rct); width = rct.right; height = rct.bottom; DWORD clipedge = 0; for (int i = 0; i < nBodyCount; ++i) { IBody* pBody = ppBodies[i]; if (pBody) { BOOLEAN bTracked = false; HRESULT hr = pBody->get_IsTracked(&bTracked); // Engaged()は使えるみたい。これは、視野に入ってきた人を認識するものだろう。 //hr = pBody->get_Engaged(&nEngaged[i]); // 以下はまだ使えないようだ //hr = pBody->GetAppearanceDetectionResults((UINT)i, &nEngaged[i]); pBody->get_ClippedEdges(&clipedge); if (SUCCEEDED(hr) && bTracked) { Joint joints[JointType_Count]; D2D1_POINT_2F jointPoints[JointType_Count]; HandState leftHandState = HandState_Unknown; HandState rightHandState = HandState_Unknown; pBody->get_HandLeftState(&leftHandState); pBody->get_HandRightState(&rightHandState); hr = pBody->GetJoints(_countof(joints), joints); if (SUCCEEDED(hr)) { for (int j = 0; j < _countof(joints); ++j) { jointPoints[j] = BodyToScreen(joints[j].Position, width, height); } m_pDrawCoordinateMapping->DrawBody(joints, jointPoints); // ここに頭部に丸を描いて、ボディ番号を表示 m_pDrawCoordinateMapping->DrawHead(jointPoints[JointType_Head], i, clipedge/*, nEngaged[i]*/); m_pDrawCoordinateMapping->DrawHand(leftHandState, jointPoints[JointType_HandLeft]); m_pDrawCoordinateMapping->DrawHand(rightHandState, jointPoints[JointType_HandRight]); // 手先がある領域にきたら実行 // ボタンのような // 現状、複数人が認識されても実行するので、本来は最初に認識された一人のみにする必要がある。 float xy[2] = { 0.0 }; xy[0] = jointPoints[JointType_HandTipRight].x - center.x; xy[1] = jointPoints[JointType_HandTipRight].y - center.y; m_nButton = 0; if (sqrt(xy[0] * xy[0] + xy[1] * xy[1]) < 50.0) { m_nButton = 1; } m_pDrawCoordinateMapping->DrawButton(center, m_nButton); } } } } m_pDrawCoordinateMapping->EndDraw(); } }
/// <summary> /// Handle new color data /// <param name="nTime">timestamp of frame</param> /// <param name="pBuffer">pointer to frame data</param> /// <param name="nWidth">width (in pixels) of input image data</param> /// <param name="nHeight">height (in pixels) of input image data</param> /// </summary> void CBodyBasics::ProcessColor(INT64 nTime, RGBQUAD* pBuffer, int nWidth, int nHeight) { if (m_hWnd) { if (!m_nStartTime) { m_nStartTime = nTime; } double fps = 0.0; LARGE_INTEGER qpcNow = { 0 }; if (m_fFreq) { if (QueryPerformanceCounter(&qpcNow)) { if (m_nLastCounter) { m_nFramesSinceUpdate++; fps = m_fFreq * m_nFramesSinceUpdate / double(qpcNow.QuadPart - m_nLastCounter); } } } WCHAR szStatusMessage[64]; StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L" FPS = %0.2f Time = %I64d", fps, (nTime - m_nStartTime)); if (SetStatusMessage(szStatusMessage, 1000, false)) { m_nLastCounter = qpcNow.QuadPart; m_nFramesSinceUpdate = 0; } } // Make sure we've received valid data if (pBuffer && (nWidth == cColorWidth) && (nHeight == cColorHeight)) { // Draw the data with Direct2D m_pDrawColor->Draw(reinterpret_cast<BYTE*>(pBuffer), cColorWidth * cColorHeight * sizeof(RGBQUAD)); if (m_bSaveScreenshot) { WCHAR szScreenshotPath[MAX_PATH]; // Retrieve the path to My Photos GetScreenshotFileName(szScreenshotPath, _countof(szScreenshotPath)); // Write out the bitmap to disk HRESULT hr = SaveBitmapToFile(reinterpret_cast<BYTE*>(pBuffer), nWidth, nHeight, sizeof(RGBQUAD)* 8, szScreenshotPath); WCHAR szStatusMessage[64 + MAX_PATH]; if (SUCCEEDED(hr)) { // Set the status bar to show where the screenshot was saved StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Screenshot saved to %s", szScreenshotPath); } else { StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Failed to write screenshot to %s", szScreenshotPath); } SetStatusMessage(szStatusMessage, 5000, true); // toggle off so we don't save a screenshot again next frame m_bSaveScreenshot = false; } } }
/// <summary> /// Handle new color data /// </summary> /// <returns>indicates success or failure</returns> void CColorBasics::ProcessColor(DASHout* dasher) { HRESULT hr; NUI_IMAGE_FRAME imageFrame; colourFrame *cFrame; // Attempt to get the color frame hr = m_pNuiSensor->NuiImageStreamGetNextFrame(m_pColorStreamHandle, 0, &imageFrame); if (FAILED(hr)) { printf("failed in processcolor \n "); return; } INuiFrameTexture * pTexture = imageFrame.pFrameTexture; NUI_LOCKED_RECT LockedRect; // Lock the frame data so the Kinect knows not to modify it while we're reading it pTexture->LockRect(0, &LockedRect, NULL, 0); // Make sure we've received valid data if (LockedRect.Pitch != 0) { //init the frame to be parsed in FFMPEG cFrame = init_cFrame(cColorWidth*cColorHeight*3); if(!cFrame){ printf("fudge!\n"); gf_free(cFrame); return; } //Frame number cFrame->number = imageFrame.dwFrameNumber; if (!dasher->sys_start) { dasher->sys_start = gf_sys_clock_high_res(); dasher->prev_pts = 0; cFrame->pts = 0; } else { //TODO: fix timing here and cFrame->pts = 30*(gf_sys_clock_high_res() - dasher->sys_start)/1000000; printf("CTS diff is %d ms\n", (u32) (cFrame->pts - dasher->prev_pts) / 1000); dasher->prev_pts = cFrame->pts; } //convert RGBA to RGB unsigned char * currFrame = (unsigned char *)LockedRect.pBits; int j = 0; for (int i = 0; i < cColorWidth * cColorHeight * 4; i += 4){ (cFrame->kinectFrame)[i - j] = currFrame[i + 2]; (cFrame->kinectFrame)[i - j + 1] = currFrame[i + 1]; (cFrame->kinectFrame)[i - j + 2] = currFrame[i]; j++; } dasher->nextColourFrame = cFrame; // Draw the data with Direct2D m_pDrawColor->Draw(static_cast<BYTE *>(LockedRect.pBits), LockedRect.size); #if 0 // If the user pressed the screenshot button, save a screenshot if (m_bSaveScreenshot) { WCHAR statusMessage[cStatusMessageMaxLen]; // Retrieve the path to My Photos WCHAR screenshotPath[MAX_PATH]; GetScreenshotFileName(screenshotPath, _countof(screenshotPath)); // Write out the bitmap to disk hr = SaveBitmapToFile(static_cast<BYTE *>(LockedRect.pBits), cColorWidth, cColorHeight, 32, screenshotPath); if (SUCCEEDED(hr)) { // Set the status bar to show where the screenshot was saved StringCchPrintf( statusMessage, cStatusMessageMaxLen, L"Screenshot saved to %s", screenshotPath); } else { StringCchPrintf( statusMessage, cStatusMessageMaxLen, L"Failed to write screenshot to %s", screenshotPath); } SetStatusMessage(statusMessage); // toggle off so we don't save a screenshot again next frame m_bSaveScreenshot = false; } #endif } // We're done with the texture so unlock it pTexture->UnlockRect(0); // Release the frame m_pNuiSensor->NuiImageStreamReleaseFrame(m_pColorStreamHandle, &imageFrame); }
/// <summary> /// Handle new depth data /// <param name="nTime">timestamp of frame</param> /// <param name="pBuffer">pointer to frame data</param> /// <param name="nWidth">width (in pixels) of input image data</param> /// <param name="nHeight">height (in pixels) of input image data</param> /// <param name="nMinDepth">minimum reliable depth</param> /// <param name="nMaxDepth">maximum reliable depth</param> /// </summary> void CDepthBasics::ProcessDepth(INT64 nTime, const UINT16* pBuffer, int nWidth, int nHeight, USHORT nMinDepth, USHORT nMaxDepth) { if (m_hWnd) { if (!m_nStartTime) { m_nStartTime = nTime; } double fps = 0.0; LARGE_INTEGER qpcNow = {0}; if (m_fFreq) { if (QueryPerformanceCounter(&qpcNow)) { if (m_nLastCounter) { m_nFramesSinceUpdate++; fps = m_fFreq * m_nFramesSinceUpdate / double(qpcNow.QuadPart - m_nLastCounter); } } } WCHAR szStatusMessage[64]; StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L" FPS = %0.2f Time = %I64d", fps, (nTime - m_nStartTime)); if (SetStatusMessage(szStatusMessage, 1000, false)) { m_nLastCounter = qpcNow.QuadPart; m_nFramesSinceUpdate = 0; } } int size = nWidth*nHeight; // Make sure we've received valid data if (m_pDepthRGBX && pBuffer && (nWidth == cDepthWidth) && (nHeight == cDepthHeight)) { RGBQUAD* pRGBX = m_pDepthRGBX; // Make a copy of received depth frame UINT16* depthFrameCopy = new UINT16[nWidth * nHeight]; const UINT16* srcDepthFrame = pBuffer; memcpy(depthFrameCopy, srcDepthFrame, nWidth*nHeight*sizeof(UINT16)); // Collect 10 frames and calculate reference depth image for background removal if (m_collectImagesCount < 10) { m_collectImagesCount++; m_depthFrames.push_back(depthFrameCopy); if (m_collectImagesCount == 10) { // Allocate memory for 32 bit reference depth frame to avoid range problem with 16 bit integers m_pDepthRef32 = new UINT32[size]; memset(m_pDepthRef32, 0, size* sizeof(UINT32)); // Iterate images for (int image = 0; image < m_collectImagesCount; image++) { UINT16* pDepthBufferCopy = m_depthFrames.at(image); int dr_index = 0; // Loop through all pixels in image for (int y = 0; y < nHeight; y++) { int zero_fill = 0; int count = 0; // Calculate pixel average from stride for (int j = 0; j < nWidth; j++) { int value = pDepthBufferCopy[j + (y * nWidth)]; if (value != 0) { zero_fill += value; count++; } } if (count != 0) { zero_fill /= count; } // Fill zero pixels with average value in reference frame for (int x = 0; x < nWidth; x++) { int value = pDepthBufferCopy[x + (y * nWidth)]; m_pDepthRef32[dr_index++] += (value != 0 ? value : zero_fill); } } } // Use memory from first received depth frame m_pDepthRef = m_depthFrames.at(0); // Calculate average of each pixel and store them in 16 bit depth frame for (int i = 0; i < size; i++) { m_pDepthRef[i] = m_pDepthRef32[i] / 10; } // Deallocate collected frames except first for (int i = 1; i < m_depthFrames.size(); i++) { delete [] m_depthFrames.at(i); } m_depthFrames.clear(); // Deallocate 32bit reference depth frame memory delete [] m_pDepthRef32; m_pDepthRef32 = NULL; return; } else return; } // Calculate difference of received depth frame and reference depth image for (int i = 0; i < size; i++) { UINT16 depth_val = depthFrameCopy[i]; if (depth_val == 0) depth_val = m_pDepthRef[i]; int diff = m_pDepthRef[i] - depth_val; // If difference of pixels is greater than 200 use reference value else 5000 (background) depthFrameCopy[i] = (diff > 200 ? depth_val : 5000); } // Clear kernel information memset(kernels_, 0, sizeof(PixelBox) * HorizontalBlocks * VerticalBlocks); // Traverse through depth frame with and calculate background pixel count for each 8x8 kernel int kernelIndex = 0; int kernelIndexStartY = 0; unsigned int min_x = cDepthWidth, max_x = 0, min_y = cDepthHeight, max_y = 0; int kernelRow = 0; for (int y = 0; y < cDepthHeight; y++) { for (int x = 0; x < cDepthWidth; x++) { unsigned short pixelValue = depthFrameCopy[(y * cDepthWidth) + x]; // If pixel value is 5000 count it in if (pixelValue == 5000) { kernels_[kernelIndex].backgroundPixels++; } if (x % KernelSize == 0 && x != 0) { kernelIndex += 1; } if (x == (cDepthWidth-1)) // Last pixel was handled on current row { if (y % KernelSize == 0 && y != 0) // Last horizontal kernel handled { // Calculate 2d coordinates for each kernel (horizontally) for (int kernel = 0; kernel < HorizontalBlocks; kernel++) { kernels_[kernelIndexStartY + kernel].tlX = kernel * KernelSize; kernels_[kernelIndexStartY + kernel].tlY = kernelRow * KernelSize; } kernelIndexStartY += HorizontalBlocks; // Let's handle next horizontal row kernelRow += 1; } // Index to kernel array to access next kernel kernelIndex = kernelIndexStartY; } } } // Traverse through kernels for (int kernel = 0; kernel < HorizontalBlocks*VerticalBlocks; kernel++) { // If pixel count in kernel is less than BackgroundPixelCountThreshold // treat kernel as foreground box and check for min and max x y if (kernels_[kernel].backgroundPixels < BackgroundPixelCountThreshold) { if (kernels_[kernel].tlX < min_x) min_x = kernels_[kernel].tlX; if (kernels_[kernel].tlX > max_x) max_x = kernels_[kernel].tlX; if (kernels_[kernel].tlY < min_y) min_y = kernels_[kernel].tlY; if (kernels_[kernel].tlY > max_y) max_y = kernels_[kernel].tlY; } } const UINT16* pBufferEnd = depthFrameCopy + (nWidth * nHeight); while (depthFrameCopy < pBufferEnd) { USHORT depth = *depthFrameCopy; // To convert to a byte, we're discarding the most-significant // rather than least-significant bits. // We're preserving detail, although the intensity will "wrap." // Values outside the reliable depth range are mapped to 0 (black). // Note: Using conditionals in this loop could degrade performance. // Consider using a lookup table instead when writing production code. BYTE intensity = static_cast<BYTE>((depth >= nMinDepth) && (depth <= nMaxDepth) ? (depth % 256) : 0); pRGBX->rgbRed = intensity; pRGBX->rgbGreen = intensity; pRGBX->rgbBlue = intensity; ++pRGBX; ++depthFrameCopy; } // Draw the data with Direct2D m_pDrawDepth->Draw(reinterpret_cast<BYTE*>(m_pDepthRGBX), cDepthWidth * cDepthHeight * sizeof(RGBQUAD), min_x, min_y, max_x, max_y); if (m_bSaveScreenshot) { WCHAR szScreenshotPath[MAX_PATH]; // Retrieve the path to My Photos GetScreenshotFileName(szScreenshotPath, _countof(szScreenshotPath)); // Write out the bitmap to disk HRESULT hr = SaveBitmapToFile(reinterpret_cast<BYTE*>(m_pDepthRGBX), nWidth, nHeight, sizeof(RGBQUAD) * 8, szScreenshotPath); WCHAR szStatusMessage[64 + MAX_PATH]; if (SUCCEEDED(hr)) { // Set the status bar to show where the screenshot was saved StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Screenshot saved to %s", szScreenshotPath); } else { StringCchPrintf(szStatusMessage, _countof(szStatusMessage), L"Failed to write screenshot to %s", szScreenshotPath); } SetStatusMessage(szStatusMessage, 5000, true); // toggle off so we don't save a screenshot again next frame m_bSaveScreenshot = false; } } }