/// Compute the Root Mean Square Error of the residuals
static double RMSE(const SfM_Data & sfm_data)
{
// Compute residuals for each observation
std::vector<double> vec;
for(Landmarks::const_iterator iterTracks = sfm_data.GetLandmarks().begin();
iterTracks != sfm_data.GetLandmarks().end();
++iterTracks)
{
const Observations & obs = iterTracks->second.obs;
for(Observations::const_iterator itObs = obs.begin();
itObs != obs.end(); ++itObs)
{
const View * view = sfm_data.GetViews().find(itObs->first)->second.get();
const geometry::Pose3 pose = sfm_data.GetPoseOrDie(view);
const std::shared_ptr<cameras::IntrinsicBase> intrinsic = sfm_data.GetIntrinsics().at(view->id_intrinsic);
const Vec2 residual = intrinsic->residual(pose, iterTracks->second.X, itObs->second.x);
//std::cout << residual << " ";
vec.push_back( residual(0) );
vec.push_back( residual(1) );
}
}
const Eigen::Map<Eigen::RowVectorXd> residuals(&vec[0], vec.size());
const double RMSE = std::sqrt(residuals.squaredNorm() / vec.size());
return RMSE;
}
void Frustum_Filter::init_z_near_z_far_depth
(
const SfM_Data & sfm_data,
const double zNear,
const double zFar
)
{
// If z_near & z_far are -1 and structure if not empty,
// compute the values for each camera and the structure
const bool bComputed_Z = (zNear == -1. && zFar == -1.) && !sfm_data.structure.empty();
if (bComputed_Z) // Compute the near & far planes from the structure and view observations
{
for (Landmarks::const_iterator itL = sfm_data.GetLandmarks().begin();
itL != sfm_data.GetLandmarks().end(); ++itL)
{
const Landmark & landmark = itL->second;
const Vec3 & X = landmark.X;
for (Observations::const_iterator iterO = landmark.obs.begin();
iterO != landmark.obs.end(); ++iterO)
{
const IndexT id_view = iterO->first;
const View * view = sfm_data.GetViews().at(id_view).get();
if (!sfm_data.IsPoseAndIntrinsicDefined(view))
continue;
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const double z = Depth(pose.rotation(), pose.translation(), X);
NearFarPlanesT::iterator itZ = z_near_z_far_perView.find(id_view);
if (itZ != z_near_z_far_perView.end())
{
if ( z < itZ->second.first)
itZ->second.first = z;
else
if ( z > itZ->second.second)
itZ->second.second = z;
}
else
z_near_z_far_perView[id_view] = {z,z};
}
}
}
else
{
// Init the same near & far limit for all the valid views
for (Views::const_iterator it = sfm_data.GetViews().begin();
it != sfm_data.GetViews().end(); ++it)
{
const View * view = it->second.get();
if (!sfm_data.IsPoseAndIntrinsicDefined(view))
continue;
if (z_near_z_far_perView.find(view->id_view) == z_near_z_far_perView.end())
z_near_z_far_perView[view->id_view] = {zNear, zFar};
}
}
}
/// Export camera poses positions as a Vec3 vector
void GetCameraPositions(const SfM_Data & sfm_data, std::vector<Vec3> & vec_camPosition)
{
for (const auto & view : sfm_data.GetViews())
{
if (sfm_data.IsPoseAndIntrinsicDefined(view.second.get()))
{
const geometry::Pose3 pose = sfm_data.GetPoseOrDie(view.second.get());
vec_camPosition.push_back(pose.center());
}
}
}
/// Triangulate a given track from a selection of observations
Vec3 SfM_Data_Structure_Computation_Robust::track_sample_triangulation(
const SfM_Data & sfm_data,
const Observations & obs,
const std::set<IndexT> & samples) const
{
Triangulation trianObj;
for (auto& it : samples)
{
const IndexT & idx = it;
Observations::const_iterator itObs = obs.begin();
std::advance(itObs, idx);
const View * view = sfm_data.views.at(itObs->first).get();
const IntrinsicBase * cam = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
trianObj.add(
cam->get_projective_equivalent(pose),
cam->get_ud_pixel(itObs->second.x));
}
return trianObj.compute();
}
// Init a frustum for each valid views of the SfM scene
void Frustum_Filter::initFrustum
(
const SfM_Data & sfm_data
)
{
for (NearFarPlanesT::const_iterator it = z_near_z_far_perView.begin();
it != z_near_z_far_perView.end(); ++it)
{
const View * view = sfm_data.GetViews().at(it->first).get();
if (!sfm_data.IsPoseAndIntrinsicDefined(view))
continue;
Intrinsics::const_iterator iterIntrinsic = sfm_data.GetIntrinsics().find(view->id_intrinsic);
if (!isPinhole(iterIntrinsic->second->getType()))
continue;
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const Pinhole_Intrinsic * cam = dynamic_cast<const Pinhole_Intrinsic*>(iterIntrinsic->second.get());
if (!cam)
continue;
if (!_bTruncated) // use infinite frustum
{
const Frustum f(
cam->w(), cam->h(), cam->K(),
pose.rotation(), pose.center());
frustum_perView[view->id_view] = f;
}
else // use truncated frustum with defined Near and Far planes
{
const Frustum f(cam->w(), cam->h(), cam->K(),
pose.rotation(), pose.center(), it->second.first, it->second.second);
frustum_perView[view->id_view] = f;
}
}
}
inline bool Generate_SfM_Report
(
const SfM_Data & sfm_data,
const std::string & htmlFilename
)
{
// Compute mean,max,median residual values per View
IndexT residualCount = 0;
Hash_Map< IndexT, std::vector<double> > residuals_per_view;
for ( const auto & iterTracks : sfm_data.GetLandmarks() )
{
const Observations & obs = iterTracks.second.obs;
for ( const auto & itObs : obs )
{
const View * view = sfm_data.GetViews().at(itObs.first).get();
const geometry::Pose3 pose = sfm_data.GetPoseOrDie(view);
const cameras::IntrinsicBase * intrinsic = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
// Use absolute values
const Vec2 residual = intrinsic->residual(pose, iterTracks.second.X, itObs.second.x).array().abs();
residuals_per_view[itObs.first].push_back(residual(0));
residuals_per_view[itObs.first].push_back(residual(1));
++residualCount;
}
}
using namespace htmlDocument;
// extract directory from htmlFilename
const std::string sTableBegin = "<table border=\"1\">",
sTableEnd = "</table>",
sRowBegin= "<tr>", sRowEnd = "</tr>",
sColBegin = "<td>", sColEnd = "</td>",
sNewLine = "<br>", sFullLine = "<hr>";
htmlDocument::htmlDocumentStream htmlDocStream("SFM report.");
htmlDocStream.pushInfo(
htmlDocument::htmlMarkup("h1", std::string("SFM report.")));
htmlDocStream.pushInfo(sFullLine);
htmlDocStream.pushInfo( "Dataset info:" + sNewLine );
std::ostringstream os;
os << " #views: " << sfm_data.GetViews().size() << sNewLine
<< " #poses: " << sfm_data.GetPoses().size() << sNewLine
<< " #intrinsics: " << sfm_data.GetIntrinsics().size() << sNewLine
<< " #tracks: " << sfm_data.GetLandmarks().size() << sNewLine
<< " #residuals: " << residualCount << sNewLine;
htmlDocStream.pushInfo( os.str() );
htmlDocStream.pushInfo( sFullLine );
htmlDocStream.pushInfo( sTableBegin);
os.str("");
os << sRowBegin
<< sColBegin + "IdView" + sColEnd
<< sColBegin + "Basename" + sColEnd
<< sColBegin + "#Observations" + sColEnd
<< sColBegin + "Residuals min" + sColEnd
<< sColBegin + "Residuals median" + sColEnd
<< sColBegin + "Residuals mean" + sColEnd
<< sColBegin + "Residuals max" + sColEnd
<< sRowEnd;
htmlDocStream.pushInfo( os.str() );
for (const auto & iterV : sfm_data.GetViews() )
{
const View * v = iterV.second.get();
const IndexT id_view = v->id_view;
os.str("");
os << sRowBegin
<< sColBegin << id_view << sColEnd
<< sColBegin + stlplus::basename_part(v->s_Img_path) + sColEnd;
// IdView | basename | #Observations | residuals min | residual median | residual max
if (sfm_data.IsPoseAndIntrinsicDefined(v))
{
if( residuals_per_view.find(id_view) != residuals_per_view.end() )
{
const std::vector<double> & residuals = residuals_per_view.at(id_view);
if (!residuals.empty())
{
double min, max, mean, median;
minMaxMeanMedian(residuals.begin(), residuals.end(), min, max, mean, median);
os << sColBegin << residuals.size()/2 << sColEnd // #observations
<< sColBegin << min << sColEnd
<< sColBegin << median << sColEnd
<< sColBegin << mean << sColEnd
<< sColBegin << max <<sColEnd;
}
}
}
os << sRowEnd;
htmlDocStream.pushInfo( os.str() );
}
htmlDocStream.pushInfo( sTableEnd );
htmlDocStream.pushInfo( sFullLine );
// combine all residual values into one vector
// export the SVG histogram
{
IndexT residualCount = 0;
for (Hash_Map< IndexT, std::vector<double> >::const_iterator
it = residuals_per_view.begin();
it != residuals_per_view.end();
++it)
{
residualCount += it->second.size();
}
// Concat per view residual values into one vector
std::vector<double> residuals(residualCount);
residualCount = 0;
for (Hash_Map< IndexT, std::vector<double> >::const_iterator
it = residuals_per_view.begin();
it != residuals_per_view.end();
++it)
{
std::copy(it->second.begin(),
it->second.begin()+it->second.size(),
residuals.begin()+residualCount);
residualCount += it->second.size();
}
if (!residuals.empty())
{
// RMSE computation
const Eigen::Map<Eigen::RowVectorXd> residuals_mapping(&residuals[0], residuals.size());
const double RMSE = std::sqrt(residuals_mapping.squaredNorm() / (double)residuals.size());
os.str("");
os << sFullLine << "SfM Scene RMSE: " << RMSE << sFullLine;
htmlDocStream.pushInfo(os.str());
const double maxRange = *max_element(residuals.begin(), residuals.end());
Histogram<double> histo(0.0, maxRange, 100);
histo.Add(residuals.begin(), residuals.end());
svg::svgHisto svg_Histo;
svg_Histo.draw(histo.GetHist(), std::pair<float,float>(0.f, maxRange),
stlplus::create_filespec(stlplus::folder_part(htmlFilename), "residuals_histogram", "svg"),
600, 200);
os.str("");
os << sNewLine<< "Residuals histogram" << sNewLine;
os << "<img src=\""
<< "residuals_histogram.svg"
<< "\" height=\"300\" width =\"800\">\n";
htmlDocStream.pushInfo(os.str());
}
}
std::ofstream htmlFileStream(htmlFilename.c_str());
htmlFileStream << htmlDocStream.getDoc();
const bool bOk = !htmlFileStream.bad();
return bOk;
}
/// Use guided matching to find corresponding 2-view correspondences
void SfM_Data_Structure_Estimation_From_Known_Poses::match(
const SfM_Data & sfm_data,
const Pair_Set & pairs,
const std::shared_ptr<Regions_Provider> & regions_provider)
{
C_Progress_display my_progress_bar( pairs.size(), std::cout,
"Compute pairwise fundamental guided matching:\n" );
#ifdef OPENMVG_USE_OPENMP
#pragma omp parallel
#endif // OPENMVG_USE_OPENMP
for (Pair_Set::const_iterator it = pairs.begin(); it != pairs.end(); ++it)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp single nowait
#endif // OPENMVG_USE_OPENMP
{
// --
// Perform GUIDED MATCHING
// --
// Use the computed model to check valid correspondences
// - by considering geometric error and descriptor distance ratio.
std::vector<IndMatch> vec_corresponding_indexes;
const View * viewL = sfm_data.GetViews().at(it->first).get();
const Pose3 poseL = sfm_data.GetPoseOrDie(viewL);
const Intrinsics::const_iterator iterIntrinsicL = sfm_data.GetIntrinsics().find(viewL->id_intrinsic);
const View * viewR = sfm_data.GetViews().at(it->second).get();
const Pose3 poseR = sfm_data.GetPoseOrDie(viewR);
const Intrinsics::const_iterator iterIntrinsicR = sfm_data.GetIntrinsics().find(viewR->id_intrinsic);
Mat xL, xR;
PointsToMat(iterIntrinsicL->second.get(), regions_provider->regions_per_view.at(it->first)->GetRegionsPositions(), xL);
PointsToMat(iterIntrinsicR->second.get(), regions_provider->regions_per_view.at(it->second)->GetRegionsPositions(), xR);
const Mat34 P_L = iterIntrinsicL->second.get()->get_projective_equivalent(poseL);
const Mat34 P_R = iterIntrinsicR->second.get()->get_projective_equivalent(poseR);
const Mat3 F_lr = F_from_P(P_L, P_R);
const double thresholdF = 4.0;
#if defined(EXHAUSTIVE_MATCHING)
// Guided matching considering geometric error and descriptor distance ratio
geometry_aware::GuidedMatching
<Mat3, openMVG::fundamental::kernel::EpipolarDistanceError,
DescriptorT, L2_Vectorized<DescriptorT::bin_type> >(
F_lr, xL, desc_provider.at(it->first), xR, desc_provider.at(it->second),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes);
#else
const Vec3 epipole2 = epipole_from_P(P_R, poseL);
const features::Regions * regions = regions_provider->regions_per_view.at(it->first).get();
if (regions->IsScalar())
{
// L2 Metric (Handle descriptor internal type)
if(regions->Type_id() == typeid(unsigned char).name())
{
geometry_aware::GuidedMatching_Fundamental_Fast<
openMVG::fundamental::kernel::EpipolarDistanceError,
L2_Vectorized<unsigned char> >
( F_lr,
epipole2,
regions_provider->regions_per_view.at(it->first).get(),
iterIntrinsicR->second.get()->w(), iterIntrinsicR->second.get()->h(),
regions_provider->regions_per_view.at(it->second).get(),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes);
}
else
if(regions->Type_id() == typeid(float).name())
{
geometry_aware::GuidedMatching_Fundamental_Fast<
openMVG::fundamental::kernel::EpipolarDistanceError,
L2_Vectorized<float> >
( F_lr,
epipole2,
regions_provider->regions_per_view.at(it->first).get(),
iterIntrinsicR->second.get()->w(), iterIntrinsicR->second.get()->h(),
regions_provider->regions_per_view.at(it->second).get(),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes);
}
else
if(regions->Type_id() == typeid(double).name())
{
geometry_aware::GuidedMatching_Fundamental_Fast<
openMVG::fundamental::kernel::EpipolarDistanceError,
L2_Vectorized<double> >
( F_lr,
epipole2,
regions_provider->regions_per_view.at(it->first).get(),
iterIntrinsicR->second.get()->w(), iterIntrinsicR->second.get()->h(),
regions_provider->regions_per_view.at(it->second).get(),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes);
}
}
else
if (regions->IsBinary() && regions->Type_id() == typeid(unsigned char).name())
{
// Hamming metric
geometry_aware::GuidedMatching_Fundamental_Fast<
openMVG::fundamental::kernel::EpipolarDistanceError,
Hamming<unsigned char> >
( F_lr,
epipole2,
regions_provider->regions_per_view.at(it->first).get(),
iterIntrinsicR->second.get()->w(), iterIntrinsicR->second.get()->h(),
regions_provider->regions_per_view.at(it->second).get(),
Square(thresholdF), 0.8,
vec_corresponding_indexes);
}
#endif
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif // OPENMVG_USE_OPENMP
{
++my_progress_bar;
for (size_t i = 0; i < vec_corresponding_indexes.size(); ++i)
putatives_matches[*it].push_back(vec_corresponding_indexes[i]);
}
}
}
}
/// Filter inconsistent correspondences by using 3-view correspondences on view triplets
void SfM_Data_Structure_Estimation_From_Known_Poses::filter(
const SfM_Data & sfm_data,
const Pair_Set & pairs,
const std::shared_ptr<Regions_Provider> & regions_provider)
{
// Compute triplets
// Triangulate triplet tracks
// - keep valid one
typedef std::vector< graph::Triplet > Triplets;
const Triplets triplets = graph::tripletListing(pairs);
C_Progress_display my_progress_bar( triplets.size(), std::cout,
"Per triplet tracks validation (discard spurious correspondences):\n" );
#ifdef OPENMVG_USE_OPENMP
#pragma omp parallel
#endif // OPENMVG_USE_OPENMP
for( Triplets::const_iterator it = triplets.begin(); it != triplets.end(); ++it)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp single nowait
#endif // OPENMVG_USE_OPENMP
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif // OPENMVG_USE_OPENMP
{++my_progress_bar;}
const graph::Triplet & triplet = *it;
const IndexT I = triplet.i, J = triplet.j , K = triplet.k;
openMVG::tracks::STLMAPTracks map_tracksCommon;
openMVG::tracks::TracksBuilder tracksBuilder;
{
PairWiseMatches map_matchesIJK;
if(putatives_matches.find(std::make_pair(I,J)) != putatives_matches.end())
map_matchesIJK.insert(*putatives_matches.find(std::make_pair(I,J)));
if(putatives_matches.find(std::make_pair(I,K)) != putatives_matches.end())
map_matchesIJK.insert(*putatives_matches.find(std::make_pair(I,K)));
if(putatives_matches.find(std::make_pair(J,K)) != putatives_matches.end())
map_matchesIJK.insert(*putatives_matches.find(std::make_pair(J,K)));
if (map_matchesIJK.size() >= 2) {
tracksBuilder.Build(map_matchesIJK);
tracksBuilder.Filter(3);
tracksBuilder.ExportToSTL(map_tracksCommon);
}
// Triangulate the tracks
for (tracks::STLMAPTracks::const_iterator iterTracks = map_tracksCommon.begin();
iterTracks != map_tracksCommon.end(); ++iterTracks) {
{
const tracks::submapTrack & subTrack = iterTracks->second;
Triangulation trianObj;
for (tracks::submapTrack::const_iterator iter = subTrack.begin(); iter != subTrack.end(); ++iter) {
const size_t imaIndex = iter->first;
const size_t featIndex = iter->second;
const View * view = sfm_data.GetViews().at(imaIndex).get();
const IntrinsicBase * cam = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const Vec2 pt = regions_provider->regions_per_view.at(imaIndex)->GetRegionPosition(featIndex);
trianObj.add(cam->get_projective_equivalent(pose), cam->get_ud_pixel(pt));
}
const Vec3 Xs = trianObj.compute();
if (trianObj.minDepth() > 0 && trianObj.error() < 4.0)
// TODO: Add an angular check ?
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif // OPENMVG_USE_OPENMP
{
openMVG::tracks::submapTrack::const_iterator iterI, iterJ, iterK;
iterI = iterJ = iterK = subTrack.begin();
std::advance(iterJ,1);
std::advance(iterK,2);
triplets_matches[std::make_pair(I,J)].push_back(IndMatch(iterI->second, iterJ->second));
triplets_matches[std::make_pair(J,K)].push_back(IndMatch(iterJ->second, iterK->second));
triplets_matches[std::make_pair(I,K)].push_back(IndMatch(iterI->second, iterK->second));
}
}
}
}
}
}
}
// Clear putatives matches since they are no longer required
matching::PairWiseMatches().swap(putatives_matches);
}
/// Use guided matching to find corresponding 2-view correspondences
void SfM_Data_Structure_Estimation_From_Known_Poses::match(
const SfM_Data & sfm_data,
const Pair_Set & pairs,
const std::shared_ptr<Regions_Provider> & regions_provider)
{
C_Progress_display my_progress_bar( pairs.size(), std::cout,
"Compute pairwise fundamental guided matching:\n" );
#ifdef OPENMVG_USE_OPENMP
#pragma omp parallel
#endif // OPENMVG_USE_OPENMP
for (Pair_Set::const_iterator it = pairs.begin(); it != pairs.end(); ++it)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp single nowait
#endif // OPENMVG_USE_OPENMP
{
// --
// Perform GUIDED MATCHING
// --
// Use the computed model to check valid correspondences
// - by considering geometric error and descriptor distance ratio.
std::vector<IndMatch> vec_corresponding_indexes;
const View * viewL = sfm_data.GetViews().at(it->first).get();
const Pose3 poseL = sfm_data.GetPoseOrDie(viewL);
const Intrinsics::const_iterator iterIntrinsicL = sfm_data.GetIntrinsics().find(viewL->id_intrinsic);
const View * viewR = sfm_data.GetViews().at(it->second).get();
const Pose3 poseR = sfm_data.GetPoseOrDie(viewR);
const Intrinsics::const_iterator iterIntrinsicR = sfm_data.GetIntrinsics().find(viewR->id_intrinsic);
if (sfm_data.GetIntrinsics().count(viewL->id_intrinsic) != 0 ||
sfm_data.GetIntrinsics().count(viewR->id_intrinsic) != 0)
{
const Mat34 P_L = iterIntrinsicL->second.get()->get_projective_equivalent(poseL);
const Mat34 P_R = iterIntrinsicR->second.get()->get_projective_equivalent(poseR);
const Mat3 F_lr = F_from_P(P_L, P_R);
const double thresholdF = 4.0;
#if defined(EXHAUSTIVE_MATCHING)
geometry_aware::GuidedMatching
<Mat3, openMVG::fundamental::kernel::EpipolarDistanceError>
(
F_lr,
iterIntrinsicL->second.get(),
*regions_provider->regions_per_view.at(it->first),
iterIntrinsicR->second.get(),
*regions_provider->regions_per_view.at(it->second),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes
);
#else
const Vec3 epipole2 = epipole_from_P(P_R, poseL);
const features::Regions * regions = regions_provider->regions_per_view.at(it->first).get();
geometry_aware::GuidedMatching_Fundamental_Fast
<openMVG::fundamental::kernel::EpipolarDistanceError>
(
F_lr,
epipole2,
iterIntrinsicL->second.get(),
*regions_provider->regions_per_view.at(it->first),
iterIntrinsicR->second.get(),
*regions_provider->regions_per_view.at(it->second),
iterIntrinsicR->second->w(), iterIntrinsicR->second->h(),
Square(thresholdF), Square(0.8),
vec_corresponding_indexes
);
#endif
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif // OPENMVG_USE_OPENMP
{
++my_progress_bar;
for (size_t i = 0; i < vec_corresponding_indexes.size(); ++i)
putatives_matches[*it].push_back(vec_corresponding_indexes[i]);
}
}
}
}
}
/* OpenGL draw function & timing */
static void draw(void)
{
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
{
// convert opengl coordinates into the document information coordinates
glPushMatrix();
glMultMatrixf((GLfloat*)m_convert);
// apply view offset
openMVG::Mat4 offset_w = l2w_Camera(Mat3::Identity(), Vec3(x_offset,y_offset,z_offset));
glMultMatrixd((GLdouble*)offset_w.data());
// then apply current camera transformation
const View * view = sfm_data.GetViews().at(vec_cameras[current_cam]).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const openMVG::Mat4 l2w = l2w_Camera(pose.rotation(), pose.translation());
glPushMatrix();
glMultMatrixd((GLdouble*)l2w.data());
glPointSize(3);
glDisable(GL_TEXTURE_2D);
glDisable(GL_LIGHTING);
//Draw Structure in GREEN (as seen from the current camera)
size_t nbPoint = sfm_data.GetLandmarks().size();
size_t cpt = 0;
glBegin(GL_POINTS);
glColor3f(0.f,1.f,0.f);
for (Landmarks::const_iterator iter = sfm_data.GetLandmarks().begin();
iter != sfm_data.GetLandmarks().end(); ++iter)
{
const Landmark & landmark = iter->second;
glVertex3d(landmark.X(0), landmark.X(1), landmark.X(2));
}
glEnd();
glDisable(GL_CULL_FACE);
for (int i_cam=0; i_cam < vec_cameras.size(); ++i_cam)
{
const View * view = sfm_data.GetViews().at(vec_cameras[i_cam]).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const IntrinsicBase * cam = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
if (isPinhole(cam->getType()))
{
const Pinhole_Intrinsic * camPinhole = dynamic_cast<const Pinhole_Intrinsic*>(cam);
// Move frame to draw the camera i_cam by applying its inverse transformation
// Warning: translation has to be "fixed" to remove the current camera rotation
// Fix camera_i translation with current camera rotation inverse
const Vec3 trans = pose.rotation().transpose() * pose.translation();
// compute inverse transformation matrix from local to world
const openMVG::Mat4 l2w_i = l2w_Camera(pose.rotation().transpose(), -trans);
// stack it and use it
glPushMatrix();
glMultMatrixd((GLdouble*)l2w_i.data());
// 1. Draw optical center (RED) and image center (BLUE)
glPointSize(3);
glDisable(GL_TEXTURE_2D);
glDisable(GL_LIGHTING);
glBegin(GL_POINTS);
glColor3f(1.f,0.f,0.f);
glVertex3f(0, 0, 0); // optical center
glColor3f(0.f,0.f,1.f);
glVertex3f(0, 0, normalized_focal); // image center
glEnd();
// compute image corners coordinated with normalized focal (f=normalized_focal)
const int w = camPinhole->w();
const int h = camPinhole->h();
const double focal = camPinhole->focal();
// use principal point to adjust image center
const Vec2 pp = camPinhole->principal_point();
Vec3 c1( -pp[0]/focal * normalized_focal, (-pp[1]+h)/focal * normalized_focal, normalized_focal);
Vec3 c2((-pp[0]+w)/focal * normalized_focal, (-pp[1]+h)/focal * normalized_focal, normalized_focal);
Vec3 c3((-pp[0]+w)/focal * normalized_focal, -pp[1]/focal * normalized_focal, normalized_focal);
Vec3 c4( -pp[0]/focal * normalized_focal, -pp[1]/focal * normalized_focal, normalized_focal);
// 2. Draw thumbnail
if (i_cam == current_cam)
{
glEnable(GL_TEXTURE_2D);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glBindTexture(GL_TEXTURE_2D, m_image_vector[i_cam].texture);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glEnable(GL_BLEND);
glDisable(GL_DEPTH_TEST);
if (i_cam == current_cam) {
glColor4f(0.5f,0.5f,0.5f, 0.7f);
} else {
glColor4f(0.5f,0.5f,0.5f, 0.5f);
}
glBegin(GL_QUADS);
glTexCoord2d(0.0,1.0); glVertex3d(c1[0], c1[1], c1[2]);
glTexCoord2d(1.0,1.0); glVertex3d(c2[0], c2[1], c2[2]);
glTexCoord2d(1.0,0.0); glVertex3d(c3[0], c3[1], c3[2]);
glTexCoord2d(0.0,0.0); glVertex3d(c4[0], c4[1], c4[2]);
glEnd();
glDisable(GL_TEXTURE_2D);
glDisable(GL_BLEND); glEnable(GL_DEPTH_TEST);
}
// 3. Draw camera cone
if (i_cam == current_cam) {
glColor3f(1.f,1.f,0.f);
} else {
glColor3f(1.f,0.f,0.f);
}
glBegin(GL_LINES);
glVertex3d(0.0,0.0,0.0); glVertex3d(c1[0], c1[1], c1[2]);
glVertex3d(0.0,0.0,0.0); glVertex3d(c2[0], c2[1], c2[2]);
glVertex3d(0.0,0.0,0.0); glVertex3d(c3[0], c3[1], c3[2]);
glVertex3d(0.0,0.0,0.0); glVertex3d(c4[0], c4[1], c4[2]);
glVertex3d(c1[0], c1[1], c1[2]); glVertex3d(c2[0], c2[1], c2[2]);
glVertex3d(c2[0], c2[1], c2[2]); glVertex3d(c3[0], c3[1], c3[2]);
glVertex3d(c3[0], c3[1], c3[2]); glVertex3d(c4[0], c4[1], c4[2]);
glVertex3d(c4[0], c4[1], c4[2]); glVertex3d(c1[0], c1[1], c1[2]);
glEnd();
glPopMatrix(); // go back to current camera frame
}
}
glPopMatrix(); // go back to (document +offset) frame
glPopMatrix(); // go back to identity
}
}
bool CreateImageFile( const SfM_Data & sfm_data,
const std::string & sImagesFilename)
{
/* images.txt
# Image list with two lines of data per image:
# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME
# POINTS2D[] as (X, Y, POINT3D_ID)
# Number of images: X, mean observations per image: Y
*/
// Header
std::ofstream images_file( sImagesFilename );
if ( ! images_file )
{
std::cerr << "Cannot write file" << sImagesFilename << std::endl;
return false;
}
images_file << "# Image list with two lines of data per image:\n";
images_file << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n";
images_file << "# POINTS2D[] as (X, Y, POINT3D_ID)\n";
images_file << "# Number of images: X, mean observations per image: Y\n";
std::map< IndexT, std::vector< std::tuple<double, double, IndexT> > > viewIdToPoints2D;
const Landmarks & landmarks = sfm_data.GetLandmarks();
{
for ( Landmarks::const_iterator iterLandmarks = landmarks.begin();
iterLandmarks != landmarks.end(); ++iterLandmarks)
{
const IndexT point3d_id = iterLandmarks->first;
// Tally set of feature observations
const Observations & obs = iterLandmarks->second.obs;
for ( Observations::const_iterator itObs = obs.begin(); itObs != obs.end(); ++itObs )
{
const IndexT currentViewId = itObs->first;
const Observation & ob = itObs->second;
viewIdToPoints2D[currentViewId].push_back(std::make_tuple(ob.x( 0 ), ob.x( 1 ), point3d_id));
}
}
}
{
C_Progress_display my_progress_bar( sfm_data.GetViews().size(), std::cout, "\n- CREATE IMAGE FILE -\n" );
for (Views::const_iterator iter = sfm_data.GetViews().begin();
iter != sfm_data.GetViews().end(); ++iter, ++my_progress_bar)
{
const View * view = iter->second.get();
if ( !sfm_data.IsPoseAndIntrinsicDefined( view ) )
{
continue;
}
const Pose3 pose = sfm_data.GetPoseOrDie( view );
const Mat3 rotation = pose.rotation();
const Vec3 translation = pose.translation();
const double Tx = translation[0];
const double Ty = translation[1];
const double Tz = translation[2];
Eigen::Quaterniond q( rotation );
const double Qx = q.x();
const double Qy = q.y();
const double Qz = q.z();
const double Qw = q.w();
const IndexT image_id = view->id_view;
// Colmap's camera_ids correspond to openMVG's intrinsic ids
const IndexT camera_id = view->id_intrinsic;
const std::string image_name = view->s_Img_path;
// first line per image
//IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME
images_file << image_id << " "
<< Qw << " "
<< Qx << " "
<< Qy << " "
<< Qz << " "
<< Tx << " "
<< Ty << " "
<< Tz << " "
<< camera_id << " "
<< image_name << " "
<< "\n";
// second line per image
//POINTS2D[] as (X, Y, POINT3D_ID)
for (auto point2D: viewIdToPoints2D[image_id])
{
images_file << std::get<0>(point2D) << " " <<
std::get<1>(point2D) << " " <<
std::get<2>(point2D) << " ";
}
images_file << "\n";
}
}
return true;
}
void SfM_Data_Structure_Computation_Blind::triangulate(SfM_Data & sfm_data) const
{
std::deque<IndexT> rejectedId;
std::unique_ptr<C_Progress_display> my_progress_bar;
if (_bConsoleVerbose)
my_progress_bar.reset( new C_Progress_display(
sfm_data.structure.size(),
std::cout,
"Blind triangulation progress:\n" ));
#ifdef OPENMVG_USE_OPENMP
#pragma omp parallel
#endif
for(Landmarks::iterator iterTracks = sfm_data.structure.begin();
iterTracks != sfm_data.structure.end();
++iterTracks)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp single nowait
#endif
{
if (_bConsoleVerbose)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif
++(*my_progress_bar);
}
// Triangulate each landmark
Triangulation trianObj;
const Observations & obs = iterTracks->second.obs;
for(Observations::const_iterator itObs = obs.begin();
itObs != obs.end(); ++itObs)
{
const View * view = sfm_data.views.at(itObs->first).get();
if (sfm_data.IsPoseAndIntrinsicDefined(view))
{
const IntrinsicBase * cam = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
trianObj.add(
cam->get_projective_equivalent(pose),
cam->get_ud_pixel(itObs->second.x));
}
}
if (trianObj.size() < 2)
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif
{
rejectedId.push_front(iterTracks->first);
}
}
else
{
// Compute the 3D point
const Vec3 X = trianObj.compute();
if (trianObj.minDepth() > 0) // Keep the point only if it have a positive depth
{
iterTracks->second.X = X;
}
else
{
#ifdef OPENMVG_USE_OPENMP
#pragma omp critical
#endif
{
rejectedId.push_front(iterTracks->first);
}
}
}
}
}
// Erase the unsuccessful triangulated tracks
for (auto& it : rejectedId)
{
sfm_data.structure.erase(it);
}
}
/// Robustly try to estimate the best 3D point using a ransac Scheme
/// Return true for a successful triangulation
bool SfM_Data_Structure_Computation_Robust::robust_triangulation(
const SfM_Data & sfm_data,
const Observations & obs,
Vec3 & X,
const IndexT min_required_inliers,
const IndexT min_sample_index) const
{
const double dThresholdPixel = 4.0; // TODO: make this parameter customizable
const IndexT nbIter = obs.size(); // TODO: automatic computation of the number of iterations?
// - Ransac variables
Vec3 best_model;
std::set<IndexT> best_inlier_set;
double best_error = std::numeric_limits<double>::max();
// - Ransac loop
for (IndexT i = 0; i < nbIter; ++i)
{
std::vector<size_t> vec_samples;
robust::UniformSample(min_sample_index, obs.size(), &vec_samples);
const std::set<IndexT> samples(vec_samples.begin(), vec_samples.end());
// Hypothesis generation.
const Vec3 current_model = track_sample_triangulation(sfm_data, obs, samples);
// Test validity of the hypothesis
// - chierality (for the samples)
// - residual error
// Chierality (Check the point is in front of the sampled cameras)
bool bChierality = true;
for (auto& it : samples){
Observations::const_iterator itObs = obs.begin();
std::advance(itObs, it);
const View * view = sfm_data.views.at(itObs->first).get();
const IntrinsicBase * cam = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const double z = pose.depth(current_model); // TODO: cam->depth(pose(X));
bChierality &= z > 0;
}
if (!bChierality)
continue;
std::set<IndexT> inlier_set;
double current_error = 0.0;
// Classification as inlier/outlier according pixel residual errors.
for (Observations::const_iterator itObs = obs.begin();
itObs != obs.end(); ++itObs)
{
const View * view = sfm_data.views.at(itObs->first).get();
const IntrinsicBase * intrinsic = sfm_data.GetIntrinsics().at(view->id_intrinsic).get();
const Pose3 pose = sfm_data.GetPoseOrDie(view);
const Vec2 residual = intrinsic->residual(pose, current_model, itObs->second.x);
const double residual_d = residual.norm();
if (residual_d < dThresholdPixel)
{
inlier_set.insert(itObs->first);
current_error += residual_d;
}
else
{
current_error += dThresholdPixel;
}
}
// Does the hypothesis is the best one we have seen and have sufficient inliers.
if (current_error < best_error && inlier_set.size() >= min_required_inliers)
{
X = best_model = current_model;
best_inlier_set = inlier_set;
best_error = current_error;
}
}
return !best_inlier_set.empty();
}
bool exportToCMPMVSFormat(
const SfM_Data & sfm_data,
const std::string & sOutDirectory // Output CMPMVS files directory
)
{
bool bOk = true;
// Create basis directory structure
if (!stlplus::is_folder(sOutDirectory))
{
stlplus::folder_create(sOutDirectory);
bOk = stlplus::is_folder(sOutDirectory);
}
if (!bOk)
{
std::cerr << "Cannot access to one of the desired output directory" << std::endl;
return false;
}
else
{
// Export data :
C_Progress_display my_progress_bar( sfm_data.GetViews().size()*2 );
// Since CMPMVS requires contiguous camera index, and that some views can have some missing poses,
// we reindex the poses to ensure a contiguous pose list.
Hash_Map<IndexT, IndexT> map_viewIdToContiguous;
// Export valid views as Projective Cameras:
for(Views::const_iterator iter = sfm_data.GetViews().begin();
iter != sfm_data.GetViews().end(); ++iter, ++my_progress_bar)
{
const View * view = iter->second.get();
if (!sfm_data.IsPoseAndIntrinsicDefined(view))
continue;
const Pose3 pose = sfm_data.GetPoseOrDie(view);
Intrinsics::const_iterator iterIntrinsic = sfm_data.GetIntrinsics().find(view->id_intrinsic);
// View Id re-indexing
map_viewIdToContiguous.insert(std::make_pair(view->id_view, map_viewIdToContiguous.size()));
// We have a valid view with a corresponding camera & pose
const Mat34 P = iterIntrinsic->second.get()->get_projective_equivalent(pose);
std::ostringstream os;
os << std::setw(5) << std::setfill('0') << map_viewIdToContiguous[view->id_view] << "_P";
std::ofstream file(
stlplus::create_filespec(stlplus::folder_append_separator(sOutDirectory),
os.str() ,"txt").c_str());
file << "CONTOUR" << os.widen('\n')
<< P.row(0) <<"\n"<< P.row(1) <<"\n"<< P.row(2) << os.widen('\n');
file.close();
}
// Export (calibrated) views as undistorted images
std::pair<unsigned int, unsigned int> w_h_image_size;
Image<RGBColor> image, image_ud;
for(Views::const_iterator iter = sfm_data.GetViews().begin();
iter != sfm_data.GetViews().end(); ++iter, ++my_progress_bar)
{
const View * view = iter->second.get();
if (!sfm_data.IsPoseAndIntrinsicDefined(view))
continue;
Intrinsics::const_iterator iterIntrinsic = sfm_data.GetIntrinsics().find(view->id_intrinsic);
// We have a valid view with a corresponding camera & pose
const std::string srcImage = stlplus::create_filespec(sfm_data.s_root_path, view->s_Img_path);
std::ostringstream os;
os << std::setw(5) << std::setfill('0') << map_viewIdToContiguous[view->id_view];
std::string dstImage = stlplus::create_filespec(
stlplus::folder_append_separator(sOutDirectory), os.str(),"jpg");
const IntrinsicBase * cam = iterIntrinsic->second.get();
if (map_viewIdToContiguous[view->id_view] == 0)
w_h_image_size = std::make_pair(cam->w(), cam->h());
else
{
// check that there is no image sizing change (CMPMVS support only images of the same size)
if (cam->w() != w_h_image_size.first ||
cam->h() != w_h_image_size.second)
{
std::cerr << "CMPMVS support only image having the same image size";
return false;
}
}
if (cam->have_disto())
{
// undistort the image and save it
ReadImage( srcImage.c_str(), &image);
UndistortImage(image, cam, image_ud, BLACK);
WriteImage(dstImage.c_str(), image_ud);
}
else // (no distortion)
{
// copy the image if extension match
if (stlplus::extension_part(srcImage) == "JPG" ||
stlplus::extension_part(srcImage) == "jpg")
{
stlplus::file_copy(srcImage, dstImage);
}
else
{
ReadImage( srcImage.c_str(), &image);
WriteImage( dstImage.c_str(), image);
}
}
}
// Write the mvs_firstRun script
std::ostringstream os;
os << "[global]" << os.widen('\n')
<< "dirName=\"" << stlplus::folder_append_separator(sOutDirectory) <<"\"" << os.widen('\n')
<< "prefix=\"\"" << os.widen('\n')
<< "imgExt=\"jpg\"" << os.widen('\n')
<< "ncams=" << map_viewIdToContiguous.size() << os.widen('\n')
<< "width=" << w_h_image_size.first << os.widen('\n')
<< "height=" << w_h_image_size.second << os.widen('\n')
<< "scale=2" << os.widen('\n')
<< "workDirName=\"_tmp_fast\"" << os.widen('\n')
<< "doPrepareData=TRUE" << os.widen('\n')
<< "doPrematchSifts=TRUE" << os.widen('\n')
<< "doPlaneSweepingSGM=TRUE" << os.widen('\n')
<< "doFuse=TRUE" << os.widen('\n')
<< "nTimesSimplify=10" << os.widen('\n')
<< os.widen('\n')
<< "[prematching]" << os.widen('\n')
<< "minAngle=3.0" << os.widen('\n')
<< os.widen('\n')
<< "[grow]" << os.widen('\n')
<< "minNumOfConsistentCams=6" << os.widen('\n')
<< os.widen('\n')
<< "[filter]" << os.widen('\n')
<< "minNumOfConsistentCams=2" << os.widen('\n')
<< os.widen('\n')
<< "#do not erase empy lines after this comment otherwise it will crash ... bug" << os.widen('\n')
<< os.widen('\n')
<< os.widen('\n');
std::ofstream file(
stlplus::create_filespec(stlplus::folder_append_separator(sOutDirectory),
"01_mvs_firstRun" ,"ini").c_str());
file << os.str();
file.close();
// limitedScale
os.str("");
os << "[global]" << os.widen('\n')
<< "dirName=\"" << stlplus::folder_append_separator(sOutDirectory) <<"\"" << os.widen('\n')
<< "prefix=\"\"" << os.widen('\n')
<< "imgExt=\"jpg\"" << os.widen('\n')
<< "ncams=" << map_viewIdToContiguous.size() << os.widen('\n')
<< "width=" << w_h_image_size.first << os.widen('\n')
<< "height=" << w_h_image_size.second << os.widen('\n')
<< "scale=2" << os.widen('\n')
<< "workDirName=\"_tmp_fast\"" << os.widen('\n')
<< "doPrepareData=FALSE" << os.widen('\n')
<< "doPrematchSifts=FALSE" << os.widen('\n')
<< "doPlaneSweepingSGM=FALSE" << os.widen('\n')
<< "doFuse=FALSE" << os.widen('\n')
<< os.widen('\n')
<< "[uvatlas]" << os.widen('\n')
<< "texSide=1024" << os.widen('\n')
<< "scale=1" << os.widen('\n')
<< os.widen('\n')
<< "[delanuaycut]" << os.widen('\n')
<< "saveMeshTextured=FALSE" << os.widen('\n')
<< os.widen('\n')
<< "[hallucinationsFiltering]" << os.widen('\n')
<< "useSkyPrior=FALSE" << os.widen('\n')
<< "doLeaveLargestFullSegmentOnly=FALSE" << os.widen('\n')
<< "doRemoveHugeTriangles=TRUE" << os.widen('\n')
<< os.widen('\n')
<< "[largeScale]" << os.widen('\n')
<< "doGenerateAndReconstructSpaceMaxPts=TRUE" << os.widen('\n')
<< "doGenerateSpace=TRUE" << os.widen('\n')
<< "planMaxPts=3000000" << os.widen('\n')
<< "doComputeDEMandOrtoPhoto=FALSE" << os.widen('\n')
<< "doGenerateVideoFrames=FALSE" << os.widen('\n')
<< os.widen('\n')
<< "[meshEnergyOpt]" << os.widen('\n')
<< "doOptimizeOrSmoothMesh=FALSE" << os.widen('\n')
<< os.widen('\n')
<< os.widen('\n')
<< "#EOF" << os.widen('\n')
<< os.widen('\n')
<< os.widen('\n');
std::ofstream file2(
stlplus::create_filespec(stlplus::folder_append_separator(sOutDirectory),
"02_mvs_limitedScale" ,"ini").c_str());
file2 << os.str();
file2.close();
}
return bOk;
}