//Newer version, without the failing void URealSenseTask::HandProducerTick() { if (!HandsEnabled) return; //Data is up to date from earlier if (collector->handData == nullptr) return; if (!collector->handTickDataRead) return; collector->handTickDataRead = false; PXCHandData* handData = collector->handData; collector->numberOfDetectedHands = handData->QueryNumberOfHands(); for (pxcI32 i = 0; i < collector->numberOfDetectedHands; i++) { //get hand information PXCHandData::IHand* hand; if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR) { //Store hand data collector->tempHandData[i] = hand; //Iterate Joints - lambda capture works... PXCHandData::JointData jointData; for (int j = 0; j < PXCHandData::NUMBER_OF_JOINTS; j++) { hand->QueryTrackedJoint((PXCHandData::JointType)j, jointData); collector->tempJointData[i][j] = jointData; collector->jointDataReady[i][j] = true; //This event should also emit if a frame is missed by caching the latest input /*RunLambdaOnGameThread([&, jointData]() { PXCHandData::JointData tempJointData = jointData; tempJoint->setFromRealSenseJoint(&tempJointData); if (collector->interfaceDelegate != nullptr) IRealSenseInterface::Execute_JointMoved(collector->interfaceDelegate, tempJoint); //} });*/ } } } }
void ITA_ForcesApp::updateRS() { if (mIsRunning) { if (mRS->AcquireFrame(false,0) >= PXC_STATUS_NO_ERROR) { mHandData->Update(); getRSCursorPos(); getRSGestureState(); auto sample = mRS->QuerySample(); if (sample != nullptr) { auto rgb = sample->color; if (rgb != nullptr) { PXCImage::ImageData rgbData; if (rgb->AcquireAccess(PXCImage::ACCESS_READ, PXCImage::PIXEL_FORMAT_RGB32, &rgbData) >= PXC_STATUS_NO_ERROR) { auto srf = Surface8u::create(rgbData.planes[0], RGB_SIZE.x, RGB_SIZE.y, rgbData.pitches[0], SurfaceChannelOrder::BGRA); auto chan = Channel8u::create(*srf); mTexRgb = gl::Texture2d::create(*chan); rgb->ReleaseAccess(&rgbData); } } } mRS->ReleaseFrame(); } } }
void ITA_ForcesApp::getRSCursorPos() { mHasHand = false; mMousePos.clear(); auto numHands = mHandData->QueryNumberOfHands(); pxcUID handId; mNumInputs = 0.0f; for (int i = 0; i < numHands; ++i) { if (mHandData->QueryHandId(PXCHandData::ACCESS_ORDER_BY_ID, i, handId) >= PXC_STATUS_NO_ERROR) { PXCHandData::IHand *hand; if (mHandData->QueryHandDataById(handId, hand) >= PXC_STATUS_NO_ERROR) { if (hand->HasCursor()) { PXCHandData::ICursor *cursor; if (hand->QueryCursor(cursor) >= PXC_STATUS_NO_ERROR) { auto cursorPos = cursor->QueryPointImage(); // Mapping block PXCPoint3DF32 uvz[1]{ cursorPos }; uvz[0].z *= 1000.0f; PXCPointF32 ijj[1]; auto stat = mMapper->MapDepthToColor(1, uvz, ijj); // if(!mMouseInput) mMousePos.push_back( vec3(remapPos(vec2(ijj[0].x, ijj[0].y)), 0.0)); mHasHand = true; mNumInputs += 1.0f; } } } } } if (numHands < 1) mIdle = true; else mIdle = false; mForceMode = mRepulsing ? FORCE_R : FORCE_A; }
void ITA_ForcesApp::getRSGestureState() { auto numGestures = mHandData->QueryFiredGesturesNumber(); for (int j = 0; j < numGestures; ++j) { PXCHandData::GestureData gestData; if (mHandData->QueryFiredGestureData(j, gestData) >= PXC_STATUS_NO_ERROR) { PXCHandData::IHand *hand; if (mHandData->QueryHandDataById(gestData.handId, hand) >= PXC_STATUS_NO_ERROR) { std::wstring str(gestData.name); if (str.find(L"cursor_click") != std::string::npos) { if (mHasHand) mRepulsing = !mRepulsing; } } } } }
void URealSenseTask::HandProducerTickOld() { if (!HandsEnabled) return; //Data is up to date from earlier if (collector->handData == nullptr) return; PXCHandData* handData = collector->handData; for (pxcI32 i = 0; i < handData->QueryNumberOfHands(); i++) { //get hand information PXCHandData::IHand* hand; if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR) { { FScopeLock lock(&JointCriticalSection); collector->tempHandData[i] = hand; } //Hand data RunLambdaOnGameThread([&, i]() { { FScopeLock lock(&JointCriticalSection); if (!collector->handDataRead[i]) { tempHand->setFromRealSenseHand(hand);//tempHandData[i]); if (collector->interfaceDelegate != nullptr) IRealSenseInterface::Execute_HandMoved(collector->interfaceDelegate, tempHand); //clear it collector->handDataRead[i] = true; } } }); //Iterate Joints PXCHandData::JointData jointData; for (int j = 0; j < PXCHandData::NUMBER_OF_JOINTS; j++) { hand->QueryTrackedJoint((PXCHandData::JointType)j, jointData); //Copy data only if it hasn't been read { FScopeLock lock(&JointCriticalSection); if (collector->jointDataRead[i][j]) { collector->tempJointData[i][j] = jointData; FVector vect = FVector(jointData.positionWorld.x, jointData.positionWorld.y, jointData.positionWorld.z); //UE_LOG(RealSensePluginLog, Log, TEXT("Copied Joint: %s"), *vect.ToString()); collector->jointDataRead[i][j] = false; } } //This event should also emit if a frame is missed by caching the latest input RunLambdaOnGameThread([&, i, j]() { //{ //FScopeLock lock(&JointCriticalSection); if (!collector->jointDataRead[i][j]) { //FVector vect = FVector(tempJointData[i][j].positionWorld.x, tempJointData[i][j].positionWorld.y, tempJointData[i][j].positionWorld.z); //UE_LOG(RealSensePluginLog, Log, TEXT("Reading Joint: %s"), *vect.ToString()); tempJoint->setFromRealSenseJoint(&collector->tempJointData[i][j]); if (collector->interfaceDelegate != nullptr) IRealSenseInterface::Execute_JointMoved(collector->interfaceDelegate, tempJoint); //clear it collector->jointDataRead[i][j] = true; } //} }); } } } }
void URealSenseTask::GestureProducerTick() { PXCHandData* handData = collector->handData; PXCHandData::GestureData gestureData; for (pxcI32 i = 0; i < handData->QueryNumberOfHands(); i++) { //get hand information PXCHandData::IHand* hand; if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR) { //spread fingers if (!GesturesEnabled) return; //FScopeLock lock(&GestureCriticalSection); //tempHand->setFromRealSenseHand(hand); //URealSenseHand* copyHand = DuplicateObject(tempHand, nullptr); if (handData->IsGestureFired(L"spreadfingers", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_SpreadFingersGestureDetected(collector->interfaceDelegate, tempHand); }); } //fist if (handData->IsGestureFired(L"fist", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_FistGestureDetected(collector->interfaceDelegate, tempHand); }); } //tap if (handData->IsGestureFired(L"tap", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_TapGestureDetected(collector->interfaceDelegate, tempHand); }); } //thumb_down if (handData->IsGestureFired(L"thumb_down", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_ThumbDownGestureDetected(collector->interfaceDelegate, tempHand); }); } //thumb_up if (handData->IsGestureFired(L"thumb_up", gestureData)) { //If the gesture was fired by the same hand runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_ThumbUpGestureDetected(collector->interfaceDelegate, tempHand); }); } //two_fingers_pinch_open if (handData->IsGestureFired(L"two_fingers_pinch_open", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_TwoFingersPinchOpenGestureDetected(collector->interfaceDelegate, tempHand); }); } //v_sign if (handData->IsGestureFired(L"v_sign", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_VSignGestureDetected(collector->interfaceDelegate, tempHand); }); } //full_pinch if (handData->IsGestureFired(L"full_pinch", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_FullPinchGestureDetected(collector->interfaceDelegate, tempHand); }); } //swipe if (handData->IsGestureFired(L"swipe", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_SwipeGestureDetected(collector->interfaceDelegate, tempHand); }); } //wave if (handData->IsGestureFired(L"wave", gestureData)) { runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]() { tempHand->setFromRealSenseHand(hand); IRealSenseInterface::Execute_WaveGestureDetected(collector->interfaceDelegate, tempHand); }); } } } }
int main(char* args,char* argsc) { pxcStatus status = PXC_STATUS_ALLOC_FAILED; PXCSenseManager *session = PXCSenseManager::CreateInstance(); if(!session) { printf("Instance create failed\n"); return 10; } status = session->EnableHand(nullptr); if(status = PXC_STATUS_NO_ERROR) { printf("Hand data unavailable\n"); return 20; } session->Init(); if(status = PXC_STATUS_NO_ERROR) { printf("init failed\n"); return 30; } PXCHandModule* handTracker = session->QueryHand(); if(status = PXC_STATUS_NO_ERROR) { printf("no hand tracking support\n"); return 40; } PXCHandConfiguration* handConfig = handTracker->CreateActiveConfiguration(); handConfig->EnableAllGestures(); handConfig->ApplyChanges(); PXCHandData* handData = handTracker->CreateOutput(); bool running = true; status = session->EnableStream(PXCCapture::StreamType::STREAM_TYPE_DEPTH,1920,1080,30.0); if (status = PXC_STATUS_NO_ERROR) { printf("Unknown error when enabling stream."); return 50; } while(running) { //printf("Acquire frame. "); status = session->AcquireFrame(true); //printf("Got frame.\n"); if(status >= PXC_STATUS_NO_ERROR) { printf("He's dead Jim.\n"); return 50; } handData->Update(); //printf("Got %i gestures for %i hands.\n", handData->QueryFiredGesturesNumber(),handData->QueryNumberOfHands()); for(int i=0; i<handData->QueryFiredGesturesNumber(); i++) { PXCHandData::GestureData gestureData; handData->QueryFiredGestureData(i,gestureData); wchar_t* name = (wchar_t*)gestureData.name; std::wstring nameStr(name); std::wcout << nameStr << std::endl; //printf("%s - len %d\n",nameStr.c_str(), nameStr.size()); if(nameStr == L"v_sign") { running = false; } } PXCCapture::Sample* capture = session->QuerySample(); PXCImage* depthImage = capture->depth; //std::this_thread::sleep_for(std::chrono::milliseconds(10)); session->ReleaseFrame(); } session->Release(); return 0; }