void Camera::MoveBackward(float units)
{
glm::vec3 forwardVector(camTarget.x - camPos.x, 0.0f, camTarget.z - camPos.z);
glm::normalize(forwardVector);
camTarget -= units * forwardVector;
camPos = ResolveCamPosition();
}
void Camera::TranslateForward(const float aAmount)
{
Vector3f position = myTransformation.GetPosition();
Vector3f forwardVector(myTransformation.m31, myTransformation.m32, myTransformation.m33);
position += forwardVector * aAmount;
myTransformation.SetPosition(position);
}
void Camera::TranslateSideways(const float aAmount)
{
Vector3f position = myTransformation.GetPosition();
Vector3f forwardVector(myTransformation.m11, myTransformation.m12, myTransformation.m13);
position += forwardVector * aAmount;
myTransformation.SetPosition(position);
}
void Camera::MoveRight(float units)
{
glm::vec3 forwardVector(camTarget.x - camPos.x, 0.0f, camTarget.z - camPos.z);
glm::vec3 rightVector = glm::cross(forwardVector, upVector);
glm::normalize(rightVector);
camTarget += units * rightVector;
camPos = ResolveCamPosition();
}
void SkeletonModel::stretchArm(int jointIndex, const glm::vec3& position) {
// find out where the hand is pointing
glm::quat handRotation;
getJointRotation(jointIndex, handRotation, true);
const FBXGeometry& geometry = _geometry->getFBXGeometry();
glm::vec3 forwardVector(jointIndex == geometry.rightHandJointIndex ? -1.0f : 1.0f, 0.0f, 0.0f);
glm::vec3 handVector = handRotation * forwardVector;
// align elbow with hand
const FBXJoint& joint = geometry.joints.at(jointIndex);
if (joint.parentIndex == -1) {
return;
}
glm::quat elbowRotation;
getJointRotation(joint.parentIndex, elbowRotation, true);
applyRotationDelta(joint.parentIndex, rotationBetween(elbowRotation * forwardVector, handVector), false);
// set position according to normal length
float scale = extractUniformScale(_scale);
glm::vec3 handPosition = position - _translation;
glm::vec3 elbowPosition = handPosition - handVector * joint.distanceToParent * scale;
// set shoulder orientation to point to elbow
const FBXJoint& parentJoint = geometry.joints.at(joint.parentIndex);
if (parentJoint.parentIndex == -1) {
return;
}
glm::quat shoulderRotation;
getJointRotation(parentJoint.parentIndex, shoulderRotation, true);
applyRotationDelta(parentJoint.parentIndex, rotationBetween(shoulderRotation * forwardVector,
elbowPosition - extractTranslation(_jointStates.at(parentJoint.parentIndex).transform)), false);
// update the shoulder state
updateJointState(parentJoint.parentIndex);
// adjust the elbow's local translation
setJointTranslation(joint.parentIndex, elbowPosition);
}
void SkeletonModel::setHandPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation) {
// this algorithm is from sample code from sixense
const FBXGeometry& geometry = _geometry->getFBXGeometry();
int elbowJointIndex = geometry.joints.at(jointIndex).parentIndex;
if (elbowJointIndex == -1) {
return;
}
int shoulderJointIndex = geometry.joints.at(elbowJointIndex).parentIndex;
glm::vec3 shoulderPosition;
if (!getJointPosition(shoulderJointIndex, shoulderPosition)) {
return;
}
// precomputed lengths
float scale = extractUniformScale(_scale);
float upperArmLength = geometry.joints.at(elbowJointIndex).distanceToParent * scale;
float lowerArmLength = geometry.joints.at(jointIndex).distanceToParent * scale;
// first set wrist position
glm::vec3 wristPosition = position;
glm::vec3 shoulderToWrist = wristPosition - shoulderPosition;
float distanceToWrist = glm::length(shoulderToWrist);
// prevent gimbal lock
if (distanceToWrist > upperArmLength + lowerArmLength - EPSILON) {
distanceToWrist = upperArmLength + lowerArmLength - EPSILON;
shoulderToWrist = glm::normalize(shoulderToWrist) * distanceToWrist;
wristPosition = shoulderPosition + shoulderToWrist;
}
// cosine of angle from upper arm to hand vector
float cosA = (upperArmLength * upperArmLength + distanceToWrist * distanceToWrist - lowerArmLength * lowerArmLength) /
(2 * upperArmLength * distanceToWrist);
float mid = upperArmLength * cosA;
float height = sqrt(upperArmLength * upperArmLength + mid * mid - 2 * upperArmLength * mid * cosA);
// direction of the elbow
glm::vec3 handNormal = glm::cross(rotation * glm::vec3(0.0f, 1.0f, 0.0f), shoulderToWrist); // elbow rotating with wrist
glm::vec3 relaxedNormal = glm::cross(glm::vec3(0.0f, 1.0f, 0.0f), shoulderToWrist); // elbow pointing straight down
const float NORMAL_WEIGHT = 0.5f;
glm::vec3 finalNormal = glm::mix(relaxedNormal, handNormal, NORMAL_WEIGHT);
bool rightHand = (jointIndex == geometry.rightHandJointIndex);
if (rightHand ? (finalNormal.y > 0.0f) : (finalNormal.y < 0.0f)) {
finalNormal.y = 0.0f; // dont allow elbows to point inward (y is vertical axis)
}
glm::vec3 tangent = glm::normalize(glm::cross(shoulderToWrist, finalNormal));
// ik solution
glm::vec3 elbowPosition = shoulderPosition + glm::normalize(shoulderToWrist) * mid - tangent * height;
glm::vec3 forwardVector(rightHand ? -1.0f : 1.0f, 0.0f, 0.0f);
glm::quat shoulderRotation = rotationBetween(forwardVector, elbowPosition - shoulderPosition);
JointState& shoulderState = _jointStates[shoulderJointIndex];
shoulderState.setRotationInBindFrame(shoulderRotation, PALM_PRIORITY);
JointState& elbowState = _jointStates[elbowJointIndex];
elbowState.setRotationInBindFrame(rotationBetween(shoulderRotation * forwardVector, wristPosition - elbowPosition) * shoulderRotation, PALM_PRIORITY);
JointState& handState = _jointStates[jointIndex];
handState.setRotationInBindFrame(rotation, PALM_PRIORITY);
}
void MainController::run()
{
while(!pangolin::ShouldQuit() && !((!logReader->hasMore()) && quiet) && !(eFusion->getTick() == end && quiet))
{
if(!gui->pause->Get() || pangolin::Pushed(*gui->step))
{
if((logReader->hasMore() || rewind) && eFusion->getTick() < end)
{
TICK("LogRead");
if(rewind)
{
if(!logReader->hasMore())
{
logReader->getBack();
}
else
{
logReader->getNext();
}
if(logReader->rewound())
{
logReader->currentFrame = 0;
}
}
else
{
logReader->getNext();
}
TOCK("LogRead");
if(eFusion->getTick() < start)
{
eFusion->setTick(start);
logReader->fastForward(start);
}
float weightMultiplier = framesToSkip + 1;
if(framesToSkip > 0)
{
eFusion->setTick(eFusion->getTick() + framesToSkip);
logReader->fastForward(logReader->currentFrame + framesToSkip);
framesToSkip = 0;
}
Eigen::Matrix4f * currentPose = 0;
if(groundTruthOdometry)
{
currentPose = new Eigen::Matrix4f;
currentPose->setIdentity();
*currentPose = groundTruthOdometry->getIncrementalTransformation(logReader->timestamp);
}
eFusion->processFrame(logReader->rgb, logReader->depth, logReader->timestamp, currentPose, weightMultiplier);
if(currentPose)
{
delete currentPose;
}
if(frameskip && Stopwatch::getInstance().getTimings().at("Run") > 1000.f / 30.f)
{
framesToSkip = int(Stopwatch::getInstance().getTimings().at("Run") / (1000.f / 30.f));
}
}
}
else
{
eFusion->predict();
}
TICK("GUI");
if(gui->followPose->Get())
{
pangolin::OpenGlMatrix mv;
Eigen::Matrix4f currPose = eFusion->getCurrPose();
Eigen::Matrix3f currRot = currPose.topLeftCorner(3, 3);
Eigen::Quaternionf currQuat(currRot);
Eigen::Vector3f forwardVector(0, 0, 1);
Eigen::Vector3f upVector(0, iclnuim ? 1 : -1, 0);
Eigen::Vector3f forward = (currQuat * forwardVector).normalized();
Eigen::Vector3f up = (currQuat * upVector).normalized();
Eigen::Vector3f eye(currPose(0, 3), currPose(1, 3), currPose(2, 3));
eye -= forward;
Eigen::Vector3f at = eye + forward;
Eigen::Vector3f z = (eye - at).normalized(); // Forward
Eigen::Vector3f x = up.cross(z).normalized(); // Right
Eigen::Vector3f y = z.cross(x);
Eigen::Matrix4d m;
m << x(0), x(1), x(2), -(x.dot(eye)),
y(0), y(1), y(2), -(y.dot(eye)),
z(0), z(1), z(2), -(z.dot(eye)),
0, 0, 0, 1;
memcpy(&mv.m[0], m.data(), sizeof(Eigen::Matrix4d));
gui->s_cam.SetModelViewMatrix(mv);
}
gui->preCall();
std::stringstream stri;
stri << eFusion->getModelToModel().lastICPCount;
gui->trackInliers->Ref().Set(stri.str());
std::stringstream stre;
stre << (isnan(eFusion->getModelToModel().lastICPError) ? 0 : eFusion->getModelToModel().lastICPError);
gui->trackRes->Ref().Set(stre.str());
if(!gui->pause->Get())
{
gui->resLog.Log((isnan(eFusion->getModelToModel().lastICPError) ? std::numeric_limits<float>::max() : eFusion->getModelToModel().lastICPError), icpErrThresh);
gui->inLog.Log(eFusion->getModelToModel().lastICPCount, icpCountThresh);
}
Eigen::Matrix4f pose = eFusion->getCurrPose();
if(gui->drawRawCloud->Get() || gui->drawFilteredCloud->Get())
{
eFusion->computeFeedbackBuffers();
}
if(gui->drawRawCloud->Get())
{
eFusion->getFeedbackBuffers().at(FeedbackBuffer::RAW)->render(gui->s_cam.GetProjectionModelViewMatrix(), pose, gui->drawNormals->Get(), gui->drawColors->Get());
}
if(gui->drawFilteredCloud->Get())
{
eFusion->getFeedbackBuffers().at(FeedbackBuffer::FILTERED)->render(gui->s_cam.GetProjectionModelViewMatrix(), pose, gui->drawNormals->Get(), gui->drawColors->Get());
}
if(gui->drawGlobalModel->Get())
{
glFinish();
TICK("Global");
if(gui->drawFxaa->Get())
{
gui->drawFXAA(gui->s_cam.GetProjectionModelViewMatrix(),
gui->s_cam.GetModelViewMatrix(),
eFusion->getGlobalModel().model(),
eFusion->getConfidenceThreshold(),
eFusion->getTick(),
eFusion->getTimeDelta(),
iclnuim);
}
else
{
eFusion->getGlobalModel().renderPointCloud(gui->s_cam.GetProjectionModelViewMatrix(),
eFusion->getConfidenceThreshold(),
gui->drawUnstable->Get(),
gui->drawNormals->Get(),
gui->drawColors->Get(),
gui->drawPoints->Get(),
gui->drawWindow->Get(),
gui->drawTimes->Get(),
eFusion->getTick(),
eFusion->getTimeDelta());
}
glFinish();
TOCK("Global");
}
if(eFusion->getLost())
{
glColor3f(1, 1, 0);
}
else
{
glColor3f(1, 0, 1);
}
gui->drawFrustum(pose);
glColor3f(1, 1, 1);
if(gui->drawFerns->Get())
{
glColor3f(0, 0, 0);
for(size_t i = 0; i < eFusion->getFerns().frames.size(); i++)
{
if((int)i == eFusion->getFerns().lastClosest)
continue;
gui->drawFrustum(eFusion->getFerns().frames.at(i)->pose);
}
glColor3f(1, 1, 1);
}
if(gui->drawDefGraph->Get())
{
const std::vector<GraphNode*> & graph = eFusion->getLocalDeformation().getGraph();
for(size_t i = 0; i < graph.size(); i++)
{
pangolin::glDrawCross(graph.at(i)->position(0),
graph.at(i)->position(1),
graph.at(i)->position(2),
0.1);
for(size_t j = 0; j < graph.at(i)->neighbours.size(); j++)
{
pangolin::glDrawLine(graph.at(i)->position(0),
graph.at(i)->position(1),
graph.at(i)->position(2),
graph.at(graph.at(i)->neighbours.at(j))->position(0),
graph.at(graph.at(i)->neighbours.at(j))->position(1),
graph.at(graph.at(i)->neighbours.at(j))->position(2));
}
}
}
if(eFusion->getFerns().lastClosest != -1)
{
glColor3f(1, 0, 0);
gui->drawFrustum(eFusion->getFerns().frames.at(eFusion->getFerns().lastClosest)->pose);
glColor3f(1, 1, 1);
}
const std::vector<PoseMatch> & poseMatches = eFusion->getPoseMatches();
int maxDiff = 0;
for(size_t i = 0; i < poseMatches.size(); i++)
{
if(poseMatches.at(i).secondId - poseMatches.at(i).firstId > maxDiff)
{
maxDiff = poseMatches.at(i).secondId - poseMatches.at(i).firstId;
}
}
for(size_t i = 0; i < poseMatches.size(); i++)
{
if(gui->drawDeforms->Get())
{
if(poseMatches.at(i).fern)
{
glColor3f(1, 0, 0);
}
else
{
glColor3f(0, 1, 0);
}
for(size_t j = 0; j < poseMatches.at(i).constraints.size(); j++)
{
pangolin::glDrawLine(poseMatches.at(i).constraints.at(j).sourcePoint(0), poseMatches.at(i).constraints.at(j).sourcePoint(1), poseMatches.at(i).constraints.at(j).sourcePoint(2),
poseMatches.at(i).constraints.at(j).targetPoint(0), poseMatches.at(i).constraints.at(j).targetPoint(1), poseMatches.at(i).constraints.at(j).targetPoint(2));
}
}
}
glColor3f(1, 1, 1);
eFusion->normaliseDepth(0.3f, gui->depthCutoff->Get());
for(std::map<std::string, GPUTexture*>::const_iterator it = eFusion->getTextures().begin(); it != eFusion->getTextures().end(); ++it)
{
if(it->second->draw)
{
gui->displayImg(it->first, it->second);
}
}
eFusion->getIndexMap().renderDepth(gui->depthCutoff->Get());
gui->displayImg("ModelImg", eFusion->getIndexMap().imageTex());
gui->displayImg("Model", eFusion->getIndexMap().drawTex());
std::stringstream strs;
strs << eFusion->getGlobalModel().lastCount();
gui->totalPoints->operator=(strs.str());
std::stringstream strs2;
strs2 << eFusion->getLocalDeformation().getGraph().size();
gui->totalNodes->operator=(strs2.str());
std::stringstream strs3;
strs3 << eFusion->getFerns().frames.size();
gui->totalFerns->operator=(strs3.str());
std::stringstream strs4;
strs4 << eFusion->getDeforms();
gui->totalDefs->operator=(strs4.str());
std::stringstream strs5;
strs5 << eFusion->getTick() << "/" << logReader->getNumFrames();
gui->logProgress->operator=(strs5.str());
std::stringstream strs6;
strs6 << eFusion->getFernDeforms();
gui->totalFernDefs->operator=(strs6.str());
gui->postCall();
logReader->flipColors = gui->flipColors->Get();
eFusion->setRgbOnly(gui->rgbOnly->Get());
eFusion->setPyramid(gui->pyramid->Get());
eFusion->setFastOdom(gui->fastOdom->Get());
eFusion->setConfidenceThreshold(gui->confidenceThreshold->Get());
eFusion->setDepthCutoff(gui->depthCutoff->Get());
eFusion->setIcpWeight(gui->icpWeight->Get());
eFusion->setSo3(gui->so3->Get());
eFusion->setFrameToFrameRGB(gui->frameToFrameRGB->Get());
resetButton = pangolin::Pushed(*gui->reset);
if(gui->autoSettings)
{
static bool last = gui->autoSettings->Get();
if(gui->autoSettings->Get() != last)
{
last = gui->autoSettings->Get();
static_cast<LiveLogReader *>(logReader)->setAuto(last);
}
}
Stopwatch::getInstance().sendAll();
if(resetButton)
{
break;
}
if(pangolin::Pushed(*gui->save))
{
eFusion->savePly();
}
TOCK("GUI");
}
}