void MyListener::onFrame(const Controller& controller)
{
const Frame frame = controller.frame();
const GestureList gestures = frame.gestures();
HandList hands = frame.hands();
for (HandList::const_iterator hl = hands.begin(); hl != hands.end(); ++hl) {
const Hand hand = *hl;
// Smoothing and stabilization is performed in order to make this
// value more suitable for interaction with 2D content. The stabilized
// position lags behind the palm position by a variable amount,
// depending primarily on the speed of movement.
Vector position = hand.stabilizedPalmPosition();
if (m_positionChanged)
m_positionChanged(position[0], position[1], position[2],
frame.fingers().extended().count(),
hand.direction(),
hand.palmVelocity());
if (m_pinch)
m_pinch(hand.pinchStrength());
if (m_grab)
m_grab(hand.grabStrength());
}
for (int g = 0; g < gestures.count(); ++g) {
Gesture gesture = gestures[g];
switch (gesture.type()) {
case Gesture::TYPE_KEY_TAP:
case Gesture::TYPE_SCREEN_TAP:
if (m_tapped)
m_tapped();
break;
case Gesture::TYPE_SWIPE:
break;
}
}
}
void Window::paintGL()
{
// Clear
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
//LeapMotion
bool drawflag = true;
bool drawflag2 = true;
if( m_controller.isConnected()) //controller is a Controller object
{
Frame frame = m_controller.frame(); //The latest frame
if (!frame.hands().isEmpty())
{
Leap::HandList hands = frame.hands();
Hand firstHand = hands[0];
Leap::Vector handCenter = firstHand.palmPosition();
m_transform.setToIdentity();
//m_transform.rotate(180,QVector3D(0,0,-1));
m_transform.translate(QVector3D(-handCenter.x,handCenter.y,handCenter.z));
float yaw = firstHand.direction().yaw();
m_transform.rotate(-(yaw/3.14)*180,QVector3D(0,-1,0));
float roll = firstHand.palmNormal().roll();
m_transform.rotate((roll/3.14)*180,QVector3D(0,0,-1));
//Frame previous = controller.frame(1); //The previous frame
if (firstHand.grabStrength()>0.5) {
drawflag = false;
}
Hand secondHand = hands[1];
Leap::Vector handCenter2 = secondHand.palmPosition();
m_righthandtransform.setToIdentity();
//m_transform.rotate(180,QVector3D(0,0,-1));
m_righthandtransform.translate(QVector3D(-handCenter2.x,handCenter2.y,handCenter2.z));
float yaw2 = secondHand.direction().yaw();
m_righthandtransform.rotate(-(yaw2/3.14)*180,QVector3D(0,-1,0));
float roll2 = secondHand.palmNormal().roll();
m_righthandtransform.rotate((roll2/3.14)*180,QVector3D(0,0,-1));
//Frame previous = controller.frame(1); //The previous frame
if (secondHand.grabStrength()>0.5) {
drawflag2 = false;
}
}
else
{
m_transform.rotate(2,QVector3D(1,1,0));
m_transform.rotate(2,QVector3D(0,1,0));
m_transform.setTranslation(QVector3D(40,150,0));
//m_transform.setScale(QVector3D(5,5,5));
m_righthandtransform.rotate(2,QVector3D(0,1,0));
m_righthandtransform.setTranslation(QVector3D(-30,150,0));
//m_righthandtransform.setScale(QVector3D(250,250,250));
}
}
else
{
m_transform.setTranslation(QVector3D(0,150,0));
}
//create dynamic enviroment mapping
calcuSSBB(m_transform.getMatrix(),m_modelview,m_projection);
//opencv
Mat opencv_image = cvQueryFrame(capture);
Mat dest;
cvtColor(opencv_image, dest,CV_BGR2RGB);
QImage image((uchar*)dest.data, dest.cols, dest.rows,QImage::Format_RGB888);
//m_backTexture->bind(0);
//m_backTexture->setData(image,QOpenGLTexture::DontGenerateMipMaps);
//glActiveTexture(GL_TEXTURE0);
//generate FBO
// bind a framebuffer object
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, m_backfbo);
//glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
// Set Drawing buffers
GLuint attachments[1] = {GL_COLOR_ATTACHMENT0};
glDrawBuffers(1, attachments);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glViewport(0,0,1280,720);
//glViewport(0,0,1280*2,720*2);
m_backTexture->bind();
m_backTexture->load(image);
m_backprogram->bind();
m_backprogram->setUniformValue("ourTexture", 0);
m_backprogram->setUniformValue(u_worldToCameraFloor, m_camera.getMatrix());
m_backprogram->setUniformValue(u_cameraToViewFloor, m_backgroundProjection);
if(m_back!= NULL)
{
m_back->draw();
}
m_backprogram->release();
m_backTexture->unbind();
if(drawflag2){
m_noise->bind();
//glEnable(GL_TEXTURE_3D);
m_righthandprogram->bind();
//m_program->setUniformValue(u_worldToCamera, m_camera.getMatrix());
m_righthandprogram->setUniformValue(u_worldToCamera, m_modelview);
m_righthandprogram->setUniformValue(u_cameraToView, m_projection);
m_righthandprogram->setUniformValue(u_modelToWorld, m_righthandtransform.getMatrix());
m_righthandprogram->setUniformValue("noise", 0);
m_bunny->draw();
m_righthandprogram->release();
m_noise->unbind();}
//m_environment->unbind();
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, m_fbo);
//glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
GLuint attachments1[1] = {GL_COLOR_ATTACHMENT0};
glDrawBuffers(1, attachments1);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glViewport(0,0,512,512);
//+X
//m_backTexture->bind();
//m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
m_cubefaceprogram->bind();
m_cubeface->create(
m_ssbb[2],m_ssbb[3],
m_ssbb[2] + ( ( 1.0f - m_ssbb[2] ) / 2.0f ) ,m_ssbb[3] + ( ( 1.0f - m_ssbb[3] ) / 2.0f ),
m_ssbb[2] + ( ( 1.0f - m_ssbb[2] ) / 2.0f ) ,m_ssbb[1] / 2.0f,
m_ssbb[2],m_ssbb[1]
);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
//m_backTexture->unbind();
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_CUBE_MAP_NEGATIVE_X, texFBO, 0);
//-X
//m_backTexture->bind();
//m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
m_cubefaceprogram->bind();
m_cubeface->create(
m_ssbb[0] / 2.0f,m_ssbb[3] + ( ( 1.0f - m_ssbb[3] ) / 2.0f ),
m_ssbb[0],m_ssbb[3],
m_ssbb[0],m_ssbb[1],
m_ssbb[0]/2.0f,m_ssbb[1] / 2.0f
);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
//m_backTexture->unbind();
// //+Y
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_CUBE_MAP_POSITIVE_Y, texFBO, 0);
//m_backTexture->bind();
//m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
m_cubefaceprogram->bind();
m_cubeface->create(
m_ssbb[2] + ( ( 1.0f - m_ssbb[2] ) / 2.0f ),m_ssbb[3] + ( ( 1.0f - m_ssbb[3] ) / 2.0f ),
m_ssbb[0] / 2.0f,m_ssbb[3] + ( ( 1.0f - m_ssbb[3] ) / 2.0f ),
m_ssbb[0],m_ssbb[3],
m_ssbb[2],m_ssbb[3]
);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
// m_backTexture->unbind();
//
// //-Y
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_CUBE_MAP_NEGATIVE_Y, texFBO, 0);
//m_backTexture->bind();
//m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
m_cubefaceprogram->bind();
m_cubeface->create(
m_ssbb[2],m_ssbb[1],
m_ssbb[0],m_ssbb[1],
m_ssbb[0]/2.0f,m_ssbb[1] / 2.0f,
m_ssbb[2] + ( ( 1.0f - m_ssbb[2] ) / 2.0f ) ,( m_ssbb[1] / 2.0f )
);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
m_backTexture->unbind();
//+Z
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_CUBE_MAP_POSITIVE_Z, texFBO, 0);
//m_backTexture->bind();
//m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
m_cubefaceprogram->bind();
m_cubeface->create( m_ssbb[0],m_ssbb[3],
m_ssbb[2],m_ssbb[3],
m_ssbb[2],m_ssbb[1],
m_ssbb[0],m_ssbb[1]);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
//m_backTexture->unbind();
// //-Z
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_CUBE_MAP_NEGATIVE_Z, texFBO, 0);
m_backTexture->bind();
m_backTexture->load(image);
glBindTexture(GL_TEXTURE_2D, backFBO);
m_cubefaceprogram->bind();
m_cubeface->create(
m_ssbb[0] / 2.0f,m_ssbb[3] + ( ( 1.0f - m_ssbb[3] ) / 2.0f ),
m_ssbb[0],m_ssbb[3],
m_ssbb[0],m_ssbb[1],
m_ssbb[0]/2.0f,m_ssbb[1] / 2.0f
);
m_cubefaceprogram->setUniformValue("screenTexture", 0);
m_cubeface->draw();
m_cubefaceprogram->release();
m_backTexture->unbind();
//
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
glViewport(0,0,1280,720);
m_backTexture->bind();
m_backTexture->load(image);
//glBindTexture(GL_TEXTURE_2D, backFBO);
m_backprogram->bind();
m_backprogram->setUniformValue("ourTexture", 0);
m_backprogram->setUniformValue(u_worldToCameraFloor, m_camera.getMatrix());
m_backprogram->setUniformValue(u_cameraToViewFloor, m_backgroundProjection);
if(m_back!= NULL)
{
m_back->draw();
}
m_backprogram->release();
m_backTexture->unbind();
//m_environment->bind();
glBindTexture(GL_TEXTURE_CUBE_MAP, texFBO);
//glEnable(GL_TEXTURE_2D);
m_program->bind();
//m_program->setUniformValue(u_worldToCamera, m_camera.getMatrix());
m_program->setUniformValue(u_worldToCamera, m_modelview);
m_program->setUniformValue(u_cameraToView, m_projection);
m_program->setUniformValue(u_modelToWorld, m_transform.getMatrix());
m_program->setUniformValue("envTex", 0);
if(drawflag)
m_model->draw();
m_program->release();
//m_environment->unbind();
m_noise->bind();
m_righthandprogram->bind();
m_righthandprogram->setUniformValue(u_worldToCamera, m_modelview);
m_righthandprogram->setUniformValue(u_cameraToView, m_projection);
m_righthandprogram->setUniformValue(u_modelToWorld, m_righthandtransform.getMatrix());
m_righthandprogram->setUniformValue("noise", 0);
if(drawflag2)
m_bunny->draw();
m_righthandprogram->release();
m_noise->unbind();
//m_environment->unbind();
}
void QTVS_Leap::HandLogic()
{
//TODO: Fix this
if (hands.count() == 1)
{
int iHandToFingerShift = hand.isLeft() ? 5 : 0;
for (int iFingerCounter = iHandToFingerShift;
iFingerCounter <= iHandToFingerShift + 4;
iFingerCounter ++)
QCoreApplication::postEvent(fingerTraces.at(iFingerCounter), new QHideEvent());
iHandToFingerShift = hand.isLeft() ? 0 : 5;
if(ui.checkBox_ShowFingers->isChecked())
{
for (int iFingerCounter = iHandToFingerShift;
iFingerCounter <= iHandToFingerShift + 4;
iFingerCounter ++)
QCoreApplication::postEvent(fingerTraces.at(iFingerCounter), new QShowEvent());
}
}
else if (hands.isEmpty())
{
foreach (FingerTraceWindow * fTrace, fingerTraces)
QCoreApplication::postEvent(fTrace, new QHideEvent());
// QCoreApplication::postEvent(thumbTrace, new QHideEvent());
// QCoreApplication::postEvent(indexTrace, new QHideEvent());
// QCoreApplication::postEvent(middleTrace, new QHideEvent());
// QCoreApplication::postEvent(ringTrace, new QHideEvent());
// QCoreApplication::postEvent(pinkieTrace, new QHideEvent());
}
else
{
if(ui.checkBox_ShowFingers->isChecked())
{
foreach (FingerTraceWindow * fTrace, fingerTraces)
QCoreApplication::postEvent(fTrace, new QShowEvent());
}
// QCoreApplication::postEvent(thumbTrace, new QShowEvent());
// QCoreApplication::postEvent(indexTrace, new QShowEvent());
// QCoreApplication::postEvent(middleTrace, new QShowEvent());
// QCoreApplication::postEvent(ringTrace, new QShowEvent());
// QCoreApplication::postEvent(pinkieTrace, new QShowEvent());
}
for (HandList::const_iterator hl = hands.begin(); hl != hands.end(); ++hl) {
// Get the first hand
hand = *hl;
//TODO: Perhaps move this to gestures?
if (ui.checkBox_Crunch->isChecked())
{
if (hands.count() == 2)
{
// we check if one hand's dragging and the other's closed
if (hand.isLeft())
{
// if this hand is left, and the other hand (right) is dragging something..
if (debugWindowDrag_Right.left != -1)
{
debugDisplayString = QString::number(hand.grabStrength());
//pretty much closed
if (hand.grabStrength() >= 0.7)
{
SendMessage(debugWindowHWND_Right, WM_SYSCOMMAND, SC_CLOSE, 0);
// DestroyWindow();
}
}
}
else
{
// if this hand is left, and the other hand (right) is dragging something..
if (debugWindowDrag_Left.left != -1)
{
//pretty much closed
if (hand.grabStrength() >= 0.7)
{
// DestroyWindow(debugWindowHWND_Left);
SendMessage(debugWindowHWND_Left, WM_SYSCOMMAND, SC_CLOSE, 0);
}
}
}
}
}
// std::string handType = hand.isLeft() ? "Left hand" : "Right hand";
// std::cout << std::string(2, ' ') << handType << ", id: " << hand.id()
// << ", palm position: " << hand.palmPosition() << std::endl;
// Get the hand's normal vector and direction
const Vector normal = hand.palmNormal();
const Vector direction = hand.direction();
// Calculate the hand's pitch, roll, and yaw angles
// debugDisplayString = QString(", palm position: " + QString(hand.palmPosition().toString().data() ));
// debugDisplayString = QString::number(hand.palmPosition().x); //20
// debugDisplayString.append("\n");
// debugDisplayString.append(QString::number(hand.palmPosition().y)); // 5
// debugDisplayString.append("\n");
// debugDisplayString.append(QString::number(hand.palmPosition().z));
// debugDisplayString.append("\n");
// debugDisplayString.append("roll:" + QString::number(normal.roll()));
///--------------------------------------------------
if (debug_extendedFingerCounter != 0 && ui.checkBox_palmMouse->isChecked())
HandCursorPosition(hand.stabilizedPalmPosition());
if (ui.checkBox_HandRollDrag->isChecked())
{
if (bDebug_HandRollDrag)
{
if (normal.roll() > 0.7)
{
bDebug_HandRollDrag = false;
if (debug_extendedFingerCounter != 0)
MouseKeyboardEmulation::MouseLeftClickDown();
}
}
}
if (!bDebug_HandRollDrag)
{
if (normal.roll() < 0.5)
{
bDebug_HandRollDrag = true;
MouseKeyboardEmulation::MouseLeftClickUp();
}
}
///-----------------------------------------------
debug_extendedFingerCounter = 0;
foreach (Finger finger, fingers)
{
if (finger.isExtended())
debug_extendedFingerCounter++;
}
// Fist Scrolling
if (debug_extendedFingerCounter == 0 && ui.checkBox_palmScroll->isChecked())
{
if (fFistPositionY == 0)
fFistPositionY = hand.palmPosition().y;
if (hand.palmPosition().y > fFistPositionY + 10)
{
float fDifference = hand.palmPosition().y - fFistPositionY + 10;
fDifference /= 10;
MouseKeyboardEmulation::MouseWheelUp(fDifference);
}
else if (hand.palmPosition().y < fFistPositionY - 10)
{
float fDifference = abs(hand.palmPosition().y - fFistPositionY - 10);
fDifference /= 10;
MouseKeyboardEmulation::MouseWheelUp(-1 * fDifference);
}
// debugDisplayString = QString::number(ThumbMiddleDifference_X); //20
// debugDisplayString.append("\n");
// debugDisplayString.append(QString::number(abs(middleFinger.stabilizedTipPosition().y - indexFinger.stabilizedTipPosition().y))); // 5
// debugDisplayString.append("\n");
// debugDisplayString.append(QString::number(debug_extendedFingerCounter));
//works
// if(direction.pitch() * RAD_TO_DEG > 30)
// MouseKeyboardEmulation::MouseWheelUp((direction.pitch() * RAD_TO_DEG)/10);
// if(direction.pitch() * RAD_TO_DEG < 30)
// MouseKeyboardEmulation::MouseWheelDown(5);
}
else
{
if (fFistPositionY != 0)
fFistPositionY = 0;
}
// std::cout << std::string(2, ' ') << "pitch: " << direction.pitch() * RAD_TO_DEG << " degrees, "
// << "roll: " << normal.roll() * RAD_TO_DEG << " degrees, "
// << "yaw: " << direction.yaw() * RAD_TO_DEG << " degrees" << std::endl;
// Get the Arm bone
Arm arm = hand.arm();
// std::cout << std::string(2, ' ') << "Arm direction: " << arm.direction()
// << " wrist position: " << arm.wristPosition()
// << " elbow position: " << arm.elbowPosition() << std::endl;
// Get fingers
fingers = hand.fingers();
FingerLogic(hand.isLeft() ? handLeft : handRight);
}
}