//Newer version, without the failing
void URealSenseTask::HandProducerTick()
{
if (!HandsEnabled)
return;
//Data is up to date from earlier
if (collector->handData == nullptr)
return;
if (!collector->handTickDataRead)
return;
collector->handTickDataRead = false;
PXCHandData* handData = collector->handData;
collector->numberOfDetectedHands = handData->QueryNumberOfHands();
for (pxcI32 i = 0; i < collector->numberOfDetectedHands; i++)
{
//get hand information
PXCHandData::IHand* hand;
if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR)
{
//Store hand data
collector->tempHandData[i] = hand;
//Iterate Joints - lambda capture works...
PXCHandData::JointData jointData;
for (int j = 0; j < PXCHandData::NUMBER_OF_JOINTS; j++)
{
hand->QueryTrackedJoint((PXCHandData::JointType)j, jointData);
collector->tempJointData[i][j] = jointData;
collector->jointDataReady[i][j] = true;
//This event should also emit if a frame is missed by caching the latest input
/*RunLambdaOnGameThread([&, jointData]()
{
PXCHandData::JointData tempJointData = jointData;
tempJoint->setFromRealSenseJoint(&tempJointData);
if (collector->interfaceDelegate != nullptr)
IRealSenseInterface::Execute_JointMoved(collector->interfaceDelegate, tempJoint);
//}
});*/
}
}
}
}
void ITA_ForcesApp::updateRS()
{
if (mIsRunning)
{
if (mRS->AcquireFrame(false,0) >= PXC_STATUS_NO_ERROR)
{
mHandData->Update();
getRSCursorPos();
getRSGestureState();
auto sample = mRS->QuerySample();
if (sample != nullptr)
{
auto rgb = sample->color;
if (rgb != nullptr)
{
PXCImage::ImageData rgbData;
if (rgb->AcquireAccess(PXCImage::ACCESS_READ, PXCImage::PIXEL_FORMAT_RGB32, &rgbData) >= PXC_STATUS_NO_ERROR)
{
auto srf = Surface8u::create(rgbData.planes[0], RGB_SIZE.x, RGB_SIZE.y, rgbData.pitches[0], SurfaceChannelOrder::BGRA);
auto chan = Channel8u::create(*srf);
mTexRgb = gl::Texture2d::create(*chan);
rgb->ReleaseAccess(&rgbData);
}
}
}
mRS->ReleaseFrame();
}
}
}
void ITA_ForcesApp::getRSCursorPos()
{
mHasHand = false;
mMousePos.clear();
auto numHands = mHandData->QueryNumberOfHands();
pxcUID handId;
mNumInputs = 0.0f;
for (int i = 0; i < numHands; ++i)
{
if (mHandData->QueryHandId(PXCHandData::ACCESS_ORDER_BY_ID, i, handId) >= PXC_STATUS_NO_ERROR)
{
PXCHandData::IHand *hand;
if (mHandData->QueryHandDataById(handId, hand) >= PXC_STATUS_NO_ERROR)
{
if (hand->HasCursor())
{
PXCHandData::ICursor *cursor;
if (hand->QueryCursor(cursor) >= PXC_STATUS_NO_ERROR)
{
auto cursorPos = cursor->QueryPointImage();
// Mapping block
PXCPoint3DF32 uvz[1]{ cursorPos };
uvz[0].z *= 1000.0f;
PXCPointF32 ijj[1];
auto stat = mMapper->MapDepthToColor(1, uvz, ijj);
//
if(!mMouseInput)
mMousePos.push_back( vec3(remapPos(vec2(ijj[0].x, ijj[0].y)), 0.0));
mHasHand = true;
mNumInputs += 1.0f;
}
}
}
}
}
if (numHands < 1)
mIdle = true;
else
mIdle = false;
mForceMode = mRepulsing ? FORCE_R : FORCE_A;
}
void ITA_ForcesApp::getRSGestureState()
{
auto numGestures = mHandData->QueryFiredGesturesNumber();
for (int j = 0; j < numGestures; ++j)
{
PXCHandData::GestureData gestData;
if (mHandData->QueryFiredGestureData(j, gestData) >= PXC_STATUS_NO_ERROR)
{
PXCHandData::IHand *hand;
if (mHandData->QueryHandDataById(gestData.handId, hand) >= PXC_STATUS_NO_ERROR)
{
std::wstring str(gestData.name);
if (str.find(L"cursor_click") != std::string::npos)
{
if (mHasHand)
mRepulsing = !mRepulsing;
}
}
}
}
}
void URealSenseTask::HandProducerTickOld()
{
if (!HandsEnabled)
return;
//Data is up to date from earlier
if (collector->handData == nullptr)
return;
PXCHandData* handData = collector->handData;
for (pxcI32 i = 0; i < handData->QueryNumberOfHands(); i++)
{
//get hand information
PXCHandData::IHand* hand;
if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR)
{
{
FScopeLock lock(&JointCriticalSection);
collector->tempHandData[i] = hand;
}
//Hand data
RunLambdaOnGameThread([&, i]()
{
{
FScopeLock lock(&JointCriticalSection);
if (!collector->handDataRead[i])
{
tempHand->setFromRealSenseHand(hand);//tempHandData[i]);
if (collector->interfaceDelegate != nullptr)
IRealSenseInterface::Execute_HandMoved(collector->interfaceDelegate, tempHand);
//clear it
collector->handDataRead[i] = true;
}
}
});
//Iterate Joints
PXCHandData::JointData jointData;
for (int j = 0; j < PXCHandData::NUMBER_OF_JOINTS; j++)
{
hand->QueryTrackedJoint((PXCHandData::JointType)j, jointData);
//Copy data only if it hasn't been read
{
FScopeLock lock(&JointCriticalSection);
if (collector->jointDataRead[i][j])
{
collector->tempJointData[i][j] = jointData;
FVector vect = FVector(jointData.positionWorld.x, jointData.positionWorld.y, jointData.positionWorld.z);
//UE_LOG(RealSensePluginLog, Log, TEXT("Copied Joint: %s"), *vect.ToString());
collector->jointDataRead[i][j] = false;
}
}
//This event should also emit if a frame is missed by caching the latest input
RunLambdaOnGameThread([&, i, j]()
{
//{
//FScopeLock lock(&JointCriticalSection);
if (!collector->jointDataRead[i][j])
{
//FVector vect = FVector(tempJointData[i][j].positionWorld.x, tempJointData[i][j].positionWorld.y, tempJointData[i][j].positionWorld.z);
//UE_LOG(RealSensePluginLog, Log, TEXT("Reading Joint: %s"), *vect.ToString());
tempJoint->setFromRealSenseJoint(&collector->tempJointData[i][j]);
if (collector->interfaceDelegate != nullptr)
IRealSenseInterface::Execute_JointMoved(collector->interfaceDelegate, tempJoint);
//clear it
collector->jointDataRead[i][j] = true;
}
//}
});
}
}
}
}
void URealSenseTask::GestureProducerTick()
{
PXCHandData* handData = collector->handData;
PXCHandData::GestureData gestureData;
for (pxcI32 i = 0; i < handData->QueryNumberOfHands(); i++)
{
//get hand information
PXCHandData::IHand* hand;
if (handData->QueryHandData(PXCHandData::AccessOrderType::ACCESS_ORDER_BY_TIME, i, hand) == PXC_STATUS_NO_ERROR)
{
//spread fingers
if (!GesturesEnabled)
return;
//FScopeLock lock(&GestureCriticalSection);
//tempHand->setFromRealSenseHand(hand);
//URealSenseHand* copyHand = DuplicateObject(tempHand, nullptr);
if (handData->IsGestureFired(L"spreadfingers", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_SpreadFingersGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//fist
if (handData->IsGestureFired(L"fist", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_FistGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//tap
if (handData->IsGestureFired(L"tap", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_TapGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//thumb_down
if (handData->IsGestureFired(L"thumb_down", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_ThumbDownGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//thumb_up
if (handData->IsGestureFired(L"thumb_up", gestureData))
{
//If the gesture was fired by the same hand
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_ThumbUpGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//two_fingers_pinch_open
if (handData->IsGestureFired(L"two_fingers_pinch_open", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_TwoFingersPinchOpenGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//v_sign
if (handData->IsGestureFired(L"v_sign", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_VSignGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//full_pinch
if (handData->IsGestureFired(L"full_pinch", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_FullPinchGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//swipe
if (handData->IsGestureFired(L"swipe", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_SwipeGestureDetected(collector->interfaceDelegate, tempHand);
});
}
//wave
if (handData->IsGestureFired(L"wave", gestureData))
{
runGestureOnGameThread(gestureData.handId, hand->QueryUniqueId(), [&]()
{
tempHand->setFromRealSenseHand(hand);
IRealSenseInterface::Execute_WaveGestureDetected(collector->interfaceDelegate, tempHand);
});
}
}
}
}
int main(char* args,char* argsc) {
pxcStatus status = PXC_STATUS_ALLOC_FAILED;
PXCSenseManager *session = PXCSenseManager::CreateInstance();
if(!session) {
printf("Instance create failed\n");
return 10;
}
status = session->EnableHand(nullptr);
if(status = PXC_STATUS_NO_ERROR) {
printf("Hand data unavailable\n");
return 20;
}
session->Init();
if(status = PXC_STATUS_NO_ERROR) {
printf("init failed\n");
return 30;
}
PXCHandModule* handTracker = session->QueryHand();
if(status = PXC_STATUS_NO_ERROR) {
printf("no hand tracking support\n");
return 40;
}
PXCHandConfiguration* handConfig = handTracker->CreateActiveConfiguration();
handConfig->EnableAllGestures();
handConfig->ApplyChanges();
PXCHandData* handData = handTracker->CreateOutput();
bool running = true;
status = session->EnableStream(PXCCapture::StreamType::STREAM_TYPE_DEPTH,1920,1080,30.0);
if (status = PXC_STATUS_NO_ERROR) {
printf("Unknown error when enabling stream.");
return 50;
}
while(running) {
//printf("Acquire frame. ");
status = session->AcquireFrame(true);
//printf("Got frame.\n");
if(status >= PXC_STATUS_NO_ERROR) {
printf("He's dead Jim.\n");
return 50;
}
handData->Update();
//printf("Got %i gestures for %i hands.\n", handData->QueryFiredGesturesNumber(),handData->QueryNumberOfHands());
for(int i=0; i<handData->QueryFiredGesturesNumber(); i++) {
PXCHandData::GestureData gestureData;
handData->QueryFiredGestureData(i,gestureData);
wchar_t* name = (wchar_t*)gestureData.name;
std::wstring nameStr(name);
std::wcout << nameStr << std::endl;
//printf("%s - len %d\n",nameStr.c_str(), nameStr.size());
if(nameStr == L"v_sign") {
running = false;
}
}
PXCCapture::Sample* capture = session->QuerySample();
PXCImage* depthImage = capture->depth;
//std::this_thread::sleep_for(std::chrono::milliseconds(10));
session->ReleaseFrame();
}
session->Release();
return 0;
}
