void mrpt::vision::pnp::posit::POS() { Eigen::Vector3d I0, J0 , r1, r2, r3; double I0_norm, J0_norm; int i; double scale; for(i=0;i<3;i++) { I0(i)=obj_matrix.row(i).dot(img_vecs.col(0)); J0(i)=obj_matrix.row(i).dot(img_vecs.col(1)); } I0_norm=I0.norm(); J0_norm=J0.norm(); scale=(I0_norm + J0_norm)/2; /*Computing TRANSLATION */ t(0)=img_pts(0,0)/scale; t(1)=img_pts(0,1)/scale; t(2)=f/scale; /* Computing ROTATION */ r1=I0/I0_norm; r2=J0/J0_norm; r3=r1.cross(r2); R.row(0)=r1; R.row(1)=r2; R.row(2)=r3; }
bool mrpt::vision::pnp::rpnp::compute_pose(Eigen::Ref<Eigen::Matrix3d> R_, Eigen::Ref<Eigen::Vector3d> t_) { // selecting an edge $P_{ i1 }P_{ i2 }$ by n random sampling int i1 = 0, i2 = 1; double lmin = Q(0, i1)*Q(0, i2) + Q(1, i1)*Q(1, i2) + Q(2, i1)*Q(2, i2); Eigen::MatrixXi rij (n,2); R_=Eigen::MatrixXd::Identity(3,3); t_=Eigen::Vector3d::Zero(); for (int i = 0; i < n; i++) for (int j = 0; j < 2; j++) rij(i, j) = rand() % n; for (int ii = 0; ii < n; ii++) { int i = rij(ii, 0), j = rij(ii,1); if (i == j) continue; double l = Q(0, i)*Q(0, j) + Q(1, i)*Q(1, j) + Q(2, i)*Q(2, j); if (l < lmin) { i1 = i; i2 = j; lmin = l; } } // calculating the rotation matrix of $O_aX_aY_aZ_a$. Eigen::Vector3d p1, p2, p0, x, y, z, dum_vec; p1 = P.col(i1); p2 = P.col(i2); p0 = (p1 + p2) / 2; x = p2 - p0; x /= x.norm(); if (abs(x(1)) < abs(x(2)) ) { dum_vec << 0, 1, 0; z = x.cross(dum_vec); z /= z.norm(); y = z.cross(x); y /= y.norm(); } else { dum_vec << 0, 0, 1; y = dum_vec.cross(x); y /= y.norm(); z = x.cross(y); x /= x.norm(); } Eigen::Matrix3d R0; R0.col(0) = x; R0.col(1) =y; R0.col(2) = z; for (int i = 0; i < n; i++) P.col(i) = R0.transpose() * (P.col(i) - p0); // Dividing the n - point set into(n - 2) 3 - point subsets // and setting up the P3P equations Eigen::Vector3d v1 = Q.col(i1), v2 = Q.col(i2); double cg1 = v1.dot(v2); double sg1 = sqrt(1 - cg1*cg1); double D1 = (P.col(i1) - P.col(i2)).norm(); Eigen::MatrixXd D4(n - 2, 5); int j = 0; Eigen::Vector3d vi; Eigen::VectorXd rowvec(5); for (int i = 0; i < n; i++) { if (i == i1 || i == i2) continue; vi = Q.col(i); double cg2 = v1.dot(vi); double cg3 = v2.dot(vi); double sg2 = sqrt(1 - cg2*cg2); double D2 = (P.col(i1) - P.col(i)).norm(); double D3 = (P.col(i) - P.col(i2)).norm(); // get the coefficients of the P3P equation from each subset. rowvec = getp3p(cg1, cg2, cg3, sg1, sg2, D1, D2, D3); D4.row(j) = rowvec; j += 1; if(j>n-3) break; } Eigen::VectorXd D7(8), dumvec(8), dumvec1(5); D7.setZero(); for (int i = 0; i < n-2; i++) { dumvec1 = D4.row(i); dumvec= getpoly7(dumvec1); D7 += dumvec; } Eigen::PolynomialSolver<double, 7> psolve(D7.reverse()); Eigen::VectorXcd comp_roots = psolve.roots().transpose(); Eigen::VectorXd real_comp, imag_comp; real_comp = comp_roots.real(); imag_comp = comp_roots.imag(); Eigen::VectorXd::Index max_index; double max_real= real_comp.cwiseAbs().maxCoeff(&max_index); std::vector<double> act_roots_; int cnt=0; for (int i=0; i<imag_comp.size(); i++ ) { if(abs(imag_comp(i))/max_real<0.001) { act_roots_.push_back(real_comp(i)); cnt++; } } double* ptr = &act_roots_[0]; Eigen::Map<Eigen::VectorXd> act_roots(ptr, cnt); if (cnt==0) { return false; } Eigen::VectorXd act_roots1(cnt); act_roots1 << act_roots.segment(0,cnt); std::vector<Eigen::Matrix3d> R_cum(cnt); std::vector<Eigen::Vector3d> t_cum(cnt); std::vector<double> err_cum(cnt); for(int i=0; i<cnt; i++) { double root = act_roots(i); // Compute the rotation matrix double d2 = cg1 + root; Eigen::Vector3d unitx, unity, unitz; unitx << 1,0,0; unity << 0,1,0; unitz << 0,0,1; x = v2*d2 -v1; x/=x.norm(); if (abs(unity.dot(x)) < abs(unitz.dot(x))) { z = x.cross(unity);z/=z.norm(); y=z.cross(x); y/y.norm(); } else { y=unitz.cross(x); y/=y.norm(); z = x.cross(y); z/=z.norm(); } R.col(0)=x; R.col(1)=y; R.col(2)=z; //calculating c, s, tx, ty, tz Eigen::MatrixXd D(2 * n, 6); D.setZero(); R0 = R.transpose(); Eigen::VectorXd r(Eigen::Map<Eigen::VectorXd>(R0.data(), R0.cols()*R0.rows())); for (int j = 0; j<n; j++) { double ui = img_pts(j, 0), vi = img_pts(j, 1), xi = P(0, j), yi = P(1, j), zi = P(2, j); D.row(2 * j) << -r(1)*yi + ui*(r(7)*yi + r(8)*zi) - r(2)*zi, -r(2)*yi + ui*(r(8)*yi - r(7)*zi) + r(1)*zi, -1, 0, ui, ui*r(6)*xi - r(0)*xi; D.row(2 * j + 1) << -r(4)*yi + vi*(r(7)*yi + r(8)*zi) - r(5)*zi, -r(5)*yi + vi*(r(8)*yi - r(7)*zi) + r(4)*zi, 0, -1, vi, vi*r(6)*xi - r(3)*xi; } Eigen::MatrixXd DTD = D.transpose()*D; Eigen::EigenSolver<Eigen::MatrixXd> es(DTD); Eigen::VectorXd Diag = es.pseudoEigenvalueMatrix().diagonal(); Eigen::MatrixXd V_mat = es.pseudoEigenvectors(); Eigen::MatrixXd::Index min_index; Diag.minCoeff(&min_index); Eigen::VectorXd V = V_mat.col(min_index); V /= V(5); double c = V(0), s = V(1); t << V(2), V(3), V(4); //calculating the camera pose by 3d alignment Eigen::VectorXd xi, yi, zi; xi = P.row(0); yi = P.row(1); zi = P.row(2); Eigen::MatrixXd XXcs(3, n), XXc(3,n); XXc.setZero(); XXcs.row(0) = r(0)*xi + (r(1)*c + r(2)*s)*yi + (-r(1)*s + r(2)*c)*zi + t(0)*Eigen::VectorXd::Ones(n); XXcs.row(1) = r(3)*xi + (r(4)*c + r(5)*s)*yi + (-r(4)*s + r(5)*c)*zi + t(1)*Eigen::VectorXd::Ones(n); XXcs.row(2) = r(6)*xi + (r(7)*c + r(8)*s)*yi + (-r(7)*s + r(8)*c)*zi + t(2)*Eigen::VectorXd::Ones(n); for (int ii = 0; ii < n; ii++) XXc.col(ii) = Q.col(ii)*XXcs.col(ii).norm(); Eigen::Matrix3d R2; Eigen::Vector3d t2; Eigen::MatrixXd XXw = obj_pts.transpose(); calcampose(XXc, XXw, R2, t2); R_cum[i] = R2; t_cum[i] = t2; for (int k = 0; k < n; k++) XXc.col(k) = R2 * XXw.col(k) + t2; Eigen::MatrixXd xxc(2, n); xxc.row(0) = XXc.row(0).array() / XXc.row(2).array(); xxc.row(1) = XXc.row(1).array() / XXc.row(2).array(); double res = ((xxc.row(0) - img_pts.col(0).transpose()).norm() + (xxc.row(1) - img_pts.col(1).transpose()).norm()) / 2; err_cum[i] = res; } int pos_cum = std::min_element(err_cum.begin(), err_cum.end()) - err_cum.begin(); R_ = R_cum[pos_cum]; t_ = t_cum[pos_cum]; return true; }
bool Estimator::initialStructure() { TicToc t_sfm; //check imu observibility { map<double, ImageFrame>::iterator frame_it; Vector3d sum_g; for (frame_it = all_image_frame.begin(), frame_it++; frame_it != all_image_frame.end(); frame_it++) { double dt = frame_it->second.pre_integration->sum_dt; Vector3d tmp_g = frame_it->second.pre_integration->delta_v / dt; sum_g += tmp_g; } Vector3d aver_g; aver_g = sum_g * 1.0 / ((int)all_image_frame.size() - 1); double var = 0; for (frame_it = all_image_frame.begin(), frame_it++; frame_it != all_image_frame.end(); frame_it++) { double dt = frame_it->second.pre_integration->sum_dt; Vector3d tmp_g = frame_it->second.pre_integration->delta_v / dt; var += (tmp_g - aver_g).transpose() * (tmp_g - aver_g); //cout << "frame g " << tmp_g.transpose() << endl; } var = sqrt(var / ((int)all_image_frame.size() - 1)); //ROS_WARN("IMU variation %f!", var); if(var < 0.25) { ROS_INFO("IMU excitation not enouth!"); //return false; } } // global sfm Quaterniond Q[frame_count + 1]; Vector3d T[frame_count + 1]; map<int, Vector3d> sfm_tracked_points; vector<SFMFeature> sfm_f; for (auto &it_per_id : f_manager.feature) { int imu_j = it_per_id.start_frame - 1; SFMFeature tmp_feature; tmp_feature.state = false; tmp_feature.id = it_per_id.feature_id; for (auto &it_per_frame : it_per_id.feature_per_frame) { imu_j++; Vector3d pts_j = it_per_frame.point; tmp_feature.observation.push_back(make_pair(imu_j, Eigen::Vector2d{pts_j.x(), pts_j.y()})); } sfm_f.push_back(tmp_feature); } Matrix3d relative_R; Vector3d relative_T; int l; if (!relativePose(relative_R, relative_T, l)) { ROS_INFO("Not enough features or parallax; Move device around"); return false; } GlobalSFM sfm; if(!sfm.construct(frame_count + 1, Q, T, l, relative_R, relative_T, sfm_f, sfm_tracked_points)) { ROS_DEBUG("global SFM failed!"); marginalization_flag = MARGIN_OLD; return false; } //solve pnp for all frame map<double, ImageFrame>::iterator frame_it; map<int, Vector3d>::iterator it; frame_it = all_image_frame.begin( ); for (int i = 0; frame_it != all_image_frame.end( ); frame_it++) { // provide initial guess cv::Mat r, rvec, t, D, tmp_r; if((frame_it->first) == Headers[i].stamp.toSec()) { frame_it->second.is_key_frame = true; frame_it->second.R = Q[i].toRotationMatrix() * RIC[0].transpose(); frame_it->second.T = T[i]; i++; continue; } if((frame_it->first) > Headers[i].stamp.toSec()) { i++; } Matrix3d R_inital = (Q[i].inverse()).toRotationMatrix(); Vector3d P_inital = - R_inital * T[i]; cv::eigen2cv(R_inital, tmp_r); cv::Rodrigues(tmp_r, rvec); cv::eigen2cv(P_inital, t); frame_it->second.is_key_frame = false; vector<cv::Point3f> pts_3_vector; vector<cv::Point2f> pts_2_vector; for (auto &id_pts : frame_it->second.points) { int feature_id = id_pts.first; //cout << "feature id " << feature_id; for (auto &i_p : id_pts.second) { //cout << " pts image_frame " << (i_p.second.head<2>() * 460 ).transpose() << endl; it = sfm_tracked_points.find(feature_id); if(it != sfm_tracked_points.end()) { Vector3d world_pts = it->second; cv::Point3f pts_3(world_pts(0), world_pts(1), world_pts(2)); pts_3_vector.push_back(pts_3); Vector2d img_pts = i_p.second.head<2>(); cv::Point2f pts_2(img_pts(0), img_pts(1)); pts_2_vector.push_back(pts_2); } } } cv::Mat K = (cv::Mat_<double>(3, 3) << 1, 0, 0, 0, 1, 0, 0, 0, 1); if(pts_3_vector.size() < 6) { //cout << "pts_3_vector size " << pts_3_vector.size() << endl; ROS_DEBUG("Not enough points for solve pnp !"); return false; } if (! cv::solvePnP(pts_3_vector, pts_2_vector, K, D, rvec, t, 1)) { ROS_DEBUG("solve pnp fail!"); return false; } cv::Rodrigues(rvec, r); MatrixXd R_pnp,tmp_R_pnp; cv::cv2eigen(r, tmp_R_pnp); R_pnp = tmp_R_pnp.transpose(); MatrixXd T_pnp; cv::cv2eigen(t, T_pnp); T_pnp = R_pnp * (-T_pnp); frame_it->second.R = R_pnp * RIC[0].transpose(); frame_it->second.T = T_pnp; } if (visualInitialAlign()) return true; else { ROS_INFO("misalign visual structure with IMU"); return false; } }