void IrrlichtRenderTarget::unprojectPoint(const GeometryBuffer& buff, const glm::vec& p_in, glm::vec& p_out) const { if (!d_matrixValid) updateMatrix(); const IrrlichtGeometryBuffer& gb = static_cast<const IrrlichtGeometryBuffer&>(buff); const irr::f32 midx = d_area.getWidth() * 0.5f; const irr::f32 midy = d_area.getHeight() * 0.5f; // viewport matrix const irr::f32 vpmat_[] = { midx, 0, 0, 0, 0, -midy, 0, 0, 0, 0, 1, 0, d_area.left() + midx, d_area.top() + midy, 0, 1 }; irr::core::matrix4 vpmat; vpmat.setM(vpmat_); // matrices used for projecting and unprojecting points const irr::core::matrix4 proj(gb.getMatrix() * d_matrix * vpmat); irr::core::matrix4 unproj(proj); unproj.makeInverse(); irr::core::vector3df in; // unproject the ends of the ray in.X = midx; in.Y = midy; in.Z = -d_viewDistance; irr::core::vector3df r1; unproj.transformVect(r1, in); in.X = p_in.d_x; in.Y = p_in.d_y; in.Z = 0; irr::core::vector3df r2; unproj.transformVect(r2, in); // calculate vector of picking ray const irr::core::vector3df rv(r1 - r2); // project points to orientate them with GeometryBuffer plane in.X = 0.0; in.Y = 0.0; irr::core::vector3df p1; proj.transformVect(p1, in); in.X = 1.0; in.Y = 0.0; irr::core::vector3df p2; proj.transformVect(p2, in); in.X = 0.0; in.Y = 1.0; irr::core::vector3df p3; proj.transformVect(p3, in); // calculate the plane normal const irr::core::vector3df pn((p2 - p1).crossProduct(p3 - p1)); // calculate distance from origin const irr::f32 plen = pn.getLength(); const irr::f32 dist = -(p1.X * (pn.X / plen) + p1.Y * (pn.Y / plen) + p1.Z * (pn.Z / plen)); // calculate intersection of ray and plane const irr::f32 pn_dot_rv = pn.dotProduct(rv); const irr::f32 tmp = pn_dot_rv != 0.0 ? (pn.dotProduct(r1) + dist) / pn_dot_rv : 0.0f; p_out.d_x = static_cast<float>(r1.X - rv.X * tmp) * d_viewDistance; p_out.d_y = static_cast<float>(r1.Y - rv.Y * tmp) * d_viewDistance; }