/** Create a TexturePatchCandidate by calculating the faces' bounding box projected into the view,
* relative texture coordinates and extacting the texture views relevant part
*/
TexturePatchCandidate
generate_candidate(int label, TextureView const & texture_view,
std::vector<std::size_t> const & faces, mve::TriangleMesh::ConstPtr mesh) {
mve::ByteImage::Ptr view_image = texture_view.get_image();
int min_x = view_image->width(), min_y = view_image->height();
int max_x = 0, max_y = 0;
mve::TriangleMesh::FaceList const & mesh_faces = mesh->get_faces();
mve::TriangleMesh::VertexList const & vertices = mesh->get_vertices();
std::vector<math::Vec2f> texcoords;
for (std::size_t i = 0; i < faces.size(); ++i) {
for (std::size_t j = 0; j < 3; ++j) {
math::Vec3f vertex = vertices[mesh_faces[faces[i] * 3 + j]];
math::Vec2f pixel = texture_view.get_pixel_coords(vertex);
texcoords.push_back(pixel);
min_x = std::min(static_cast<int>(std::floor(pixel[0])) - texture_patch_border, min_x);
min_y = std::min(static_cast<int>(std::floor(pixel[1])) - texture_patch_border, min_y);
max_x = std::max(static_cast<int>(std::ceil(pixel[0])) + texture_patch_border, max_x);
max_y = std::max(static_cast<int>(std::ceil(pixel[1])) + texture_patch_border, max_y);
}
}
/* Calculate the relative texcoords. */
math::Vec2f min(min_x, min_y);
for (std::size_t i = 0; i < texcoords.size(); ++i) {
texcoords[i] = texcoords[i] - min;
}
int const width = max_x - min_x;
int const height = max_y - min_y;
/* Check for valid projections/erroneous labeling files. */
assert(min_x >= 0 - texture_patch_border);
assert(min_y >= 0 - texture_patch_border);
assert(max_x < view_image->width() + texture_patch_border);
assert(max_y < view_image->height() + texture_patch_border);
mve::ByteImage::Ptr image;
image = mve::image::crop(view_image, width, height, min_x, min_y, *math::Vec3uc(255, 0, 255));
mve::image::gamma_correct(image, 2.2f);
TexturePatchCandidate texture_patch_candidate =
{Rect<int>(min_x, min_y, max_x, max_y),
TexturePatch::create(label, faces, texcoords, image)};
return texture_patch_candidate;
}
void
calculate_data_costs(mve::TriangleMesh::ConstPtr mesh, std::vector<TextureView> * texture_views,
Settings const & settings, DataCosts * data_costs) {
std::size_t const num_faces = mesh->get_faces().size() / 3;
std::size_t const num_views = texture_views->size();
assert(num_faces < std::numeric_limits<std::uint32_t>::max());
assert(num_views < std::numeric_limits<std::uint16_t>::max());
assert(MRF_MAX_ENERGYTERM < std::numeric_limits<float>::max());
FaceProjectionInfos face_projection_infos(num_faces);
calculate_face_projection_infos(mesh, texture_views, settings, &face_projection_infos);
postprocess_face_infos(settings, &face_projection_infos, data_costs);
}
TEX_NAMESPACE_BEGIN
void
find_seam_edges(UniGraph const & graph, mve::TriangleMesh::ConstPtr mesh,
std::vector<MeshEdge> * seam_edges) {
mve::TriangleMesh::FaceList const & faces = mesh->get_faces();
seam_edges->clear();
// Is it possible that a single edge is part of more than three faces whichs' label is non zero???
for (std::size_t node = 0; node < graph.num_nodes(); ++node) {
std::vector<std::size_t> const & adj_nodes = graph.get_adj_nodes(node);
for (std::size_t adj_node : adj_nodes) {
/* Add each edge only once. */
if (node > adj_node) continue;
int label1 = graph.get_label(node);
int label2 = graph.get_label(adj_node);
/* Add only seam edges. */
//if (label1 == 0 || label2 == 0 || label1 == label2) continue;
if (label1 == label2) continue;
/* Find shared edge of the faces. */
std::vector<std::size_t> shared_edge;
for (int i = 0; i < 3; ++i){
std::size_t v1 = faces[3 * node + i];
for (int j = 0; j < 3; j++){
std::size_t v2 = faces[3 * adj_node + j];
if (v1 == v2) shared_edge.push_back(v1);
}
}
assert(shared_edge.size() == 2);
std::size_t v1 = shared_edge[0];
std::size_t v2 = shared_edge[1];
assert(v1 != v2);
if (v1 > v2) std::swap(v1, v2);
MeshEdge seam_edge = {v1, v2};
seam_edges->push_back(seam_edge);
}
}
}
void
calculate_data_costs(mve::TriangleMesh::ConstPtr mesh, std::vector<TextureView> * texture_views,
Settings const & settings, ST * data_costs) {
mve::TriangleMesh::FaceList const & faces = mesh->get_faces();
mve::TriangleMesh::VertexList const & vertices = mesh->get_vertices();
mve::TriangleMesh::NormalList const & face_normals = mesh->get_face_normals();
std::size_t const num_faces = faces.size() / 3;
std::size_t const num_views = texture_views->size();
CollisionModel3D* model = newCollisionModel3D(true);
if (settings.geometric_visibility_test) {
/* Build up acceleration structure for the visibility test. */
ProgressCounter face_counter("\tBuilding collision model", num_faces);
model->setTriangleNumber(num_faces);
for (std::size_t i = 0; i < faces.size(); i += 3) {
face_counter.progress<SIMPLE>();
math::Vec3f v1 = vertices[faces[i]];
math::Vec3f v2 = vertices[faces[i + 1]];
math::Vec3f v3 = vertices[faces[i + 2]];
model->addTriangle(*v1, *v2, *v3);
face_counter.inc();
}
model->finalize();
}
std::vector<std::vector<ProjectedFaceInfo> > projected_face_infos(num_faces);
ProgressCounter view_counter("\tCalculating face qualities", num_views);
#pragma omp parallel
{
std::vector<std::pair<std::size_t, ProjectedFaceInfo> > projected_face_view_infos;
#pragma omp for schedule(dynamic)
for (std::uint16_t j = 0; j < texture_views->size(); ++j) {
view_counter.progress<SIMPLE>();
TextureView * texture_view = &texture_views->at(j);
texture_view->load_image();
texture_view->generate_validity_mask();
if (settings.data_term == GMI) {
texture_view->generate_gradient_magnitude();
texture_view->erode_validity_mask();
}
math::Vec3f const & view_pos = texture_view->get_pos();
math::Vec3f const & viewing_direction = texture_view->get_viewing_direction();
for (std::size_t i = 0; i < faces.size(); i += 3) {
std::size_t face_id = i / 3;
math::Vec3f const & v1 = vertices[faces[i]];
math::Vec3f const & v2 = vertices[faces[i + 1]];
math::Vec3f const & v3 = vertices[faces[i + 2]];
math::Vec3f const & face_normal = face_normals[face_id];
math::Vec3f const face_center = (v1 + v2 + v3) / 3.0f;
/* Check visibility and compute quality */
math::Vec3f view_to_face_vec = (face_center - view_pos).normalized();
math::Vec3f face_to_view_vec = (view_pos - face_center).normalized();
/* Backface culling */
float viewing_angle = face_to_view_vec.dot(face_normal);
if (viewing_angle < 0.0f || viewing_direction.dot(view_to_face_vec) < 0.0f)
continue;
if (std::acos(viewing_angle) > MATH_DEG2RAD(75.0f))
continue;
/* Projects into the valid part of the TextureView? */
if (!texture_view->inside(v1, v2, v3))
continue;
if (settings.geometric_visibility_test) {
/* Viewing rays do not collide? */
bool visible = true;
math::Vec3f const * samples[] = {&v1, &v2, &v3};
// TODO: random monte carlo samples...
for (std::size_t k = 0; k < sizeof(samples) / sizeof(samples[0]); ++k) {
math::Vec3f vertex = *samples[k];
math::Vec3f dir = view_pos - vertex;
float const dir_length = dir.norm();
dir.normalize();
if (model->rayCollision(*vertex, *dir, false, dir_length * 0.0001f, dir_length)) {
visible = false;
break;
}
}
if (!visible) continue;
}
ProjectedFaceInfo info = {j, 0.0f, math::Vec3f(0.0f, 0.0f, 0.0f)};
/* Calculate quality. */
texture_view->get_face_info(v1, v2, v3, &info, settings);
if (info.quality == 0.0) continue;
/* Change color space. */
mve::image::color_rgb_to_ycbcr(*(info.mean_color));
std::pair<std::size_t, ProjectedFaceInfo> pair(face_id, info);
projected_face_view_infos.push_back(pair);
}
texture_view->release_image();
texture_view->release_validity_mask();
if (settings.data_term == GMI) {
texture_view->release_gradient_magnitude();
}
view_counter.inc();
}
//std::sort(projected_face_view_infos.begin(), projected_face_view_infos.end());
#pragma omp critical
{
for (std::size_t i = projected_face_view_infos.size(); 0 < i; --i) {
std::size_t face_id = projected_face_view_infos[i - 1].first;
ProjectedFaceInfo const & info = projected_face_view_infos[i - 1].second;
projected_face_infos[face_id].push_back(info);
}
projected_face_view_infos.clear();
}
}
delete model;
model = NULL;
ProgressCounter face_counter("\tPostprocessing face infos", num_faces);
#pragma omp parallel for schedule(dynamic)
for (std::size_t i = 0; i < projected_face_infos.size(); ++i) {
face_counter.progress<SIMPLE>();
std::vector<ProjectedFaceInfo> & infos = projected_face_infos[i];
if (settings.outlier_removal != NONE) {
photometric_outlier_detection(&infos, settings);
infos.erase(std::remove_if(infos.begin(), infos.end(),
[](ProjectedFaceInfo const & info) -> bool {return info.quality == 0.0f;}),
infos.end());
}
std::sort(infos.begin(), infos.end());
face_counter.inc();
}
/* Determine the function for the normlization. */
float max_quality = 0.0f;
for (std::size_t i = 0; i < projected_face_infos.size(); ++i)
for (std::size_t j = 0; j < projected_face_infos[i].size(); ++j)
max_quality = std::max(max_quality, projected_face_infos[i][j].quality);
Histogram hist_qualities(0.0f, max_quality, 10000);
for (std::size_t i = 0; i < projected_face_infos.size(); ++i)
for (std::size_t j = 0; j < projected_face_infos[i].size(); ++j)
hist_qualities.add_value(projected_face_infos[i][j].quality);
float percentile = hist_qualities.get_approx_percentile(0.995f);
/* Calculate the costs. */
assert(num_faces < std::numeric_limits<std::uint32_t>::max());
assert(num_views < std::numeric_limits<std::uint16_t>::max());
assert(MRF_MAX_ENERGYTERM < std::numeric_limits<float>::max());
for (std::uint32_t i = 0; i < static_cast<std::uint32_t>(projected_face_infos.size()); ++i) {
for (std::size_t j = 0; j < projected_face_infos[i].size(); ++j) {
ProjectedFaceInfo const & info = projected_face_infos[i][j];
/* Clamp to percentile and normalize. */
float normalized_quality = std::min(1.0f, info.quality / percentile);
float data_cost = (1.0f - normalized_quality) * MRF_MAX_ENERGYTERM;
data_costs->set_value(i, info.view_id, data_cost);
}
/* Ensure that all memory is freeed. */
projected_face_infos[i].clear();
projected_face_infos[i].shrink_to_fit();
}
std::cout << "\tMaximum quality of a face within an image: " << max_quality << std::endl;
std::cout << "\tClamping qualities to " << percentile << " within normalization." << std::endl;
}
void
calculate_face_projection_infos(mve::TriangleMesh::ConstPtr mesh,
std::vector<TextureView> * texture_views, Settings const & settings,
FaceProjectionInfos * face_projection_infos) {
std::vector<unsigned int> const & faces = mesh->get_faces();
std::vector<math::Vec3f> const & vertices = mesh->get_vertices();
mve::TriangleMesh::NormalList const & face_normals = mesh->get_face_normals();
std::size_t const num_views = texture_views->size();
util::WallTimer timer;
std::cout << "\tBuilding BVH from " << faces.size() / 3 << " faces... " << std::flush;
BVHTree bvh_tree(faces, vertices);
std::cout << "done. (Took: " << timer.get_elapsed() << " ms)" << std::endl;
ProgressCounter view_counter("\tCalculating face qualities", num_views);
#pragma omp parallel
{
std::vector<std::pair<std::size_t, FaceProjectionInfo> > projected_face_view_infos;
#pragma omp for schedule(dynamic)
for (std::uint16_t j = 0; j < texture_views->size(); ++j) {
view_counter.progress<SIMPLE>();
TextureView * texture_view = &texture_views->at(j);
texture_view->load_image();
texture_view->generate_validity_mask();
if (settings.data_term == DATA_TERM_GMI) {
texture_view->generate_gradient_magnitude();
texture_view->erode_validity_mask();
}
math::Vec3f const & view_pos = texture_view->get_pos();
math::Vec3f const & viewing_direction = texture_view->get_viewing_direction();
for (std::size_t i = 0; i < faces.size(); i += 3) {
std::size_t face_id = i / 3;
math::Vec3f const & v1 = vertices[faces[i]];
math::Vec3f const & v2 = vertices[faces[i + 1]];
math::Vec3f const & v3 = vertices[faces[i + 2]];
math::Vec3f const & face_normal = face_normals[face_id];
math::Vec3f const face_center = (v1 + v2 + v3) / 3.0f;
/* Check visibility and compute quality */
math::Vec3f view_to_face_vec = (face_center - view_pos).normalized();
math::Vec3f face_to_view_vec = (view_pos - face_center).normalized();
/* Backface and basic frustum culling */
float viewing_angle = face_to_view_vec.dot(face_normal);
if (viewing_angle < 0.0f || viewing_direction.dot(view_to_face_vec) < 0.0f)
continue;
if (std::acos(viewing_angle) > MATH_DEG2RAD(75.0f))
continue;
/* Projects into the valid part of the TextureView? */
if (!texture_view->inside(v1, v2, v3))
continue;
if (settings.geometric_visibility_test) {
/* Viewing rays do not collide? */
bool visible = true;
math::Vec3f const * samples[] = {&v1, &v2, &v3};
// TODO: random monte carlo samples...
for (std::size_t k = 0; k < sizeof(samples) / sizeof(samples[0]); ++k) {
BVHTree::Ray ray;
ray.origin = *samples[k];
ray.dir = view_pos - ray.origin;
ray.tmax = ray.dir.norm();
ray.tmin = ray.tmax * 0.0001f;
ray.dir.normalize();
BVHTree::Hit hit;
if (bvh_tree.intersect(ray, &hit)) {
visible = false;
break;
}
}
if (!visible) continue;
}
FaceProjectionInfo info = {j, 0.0f, math::Vec3f(0.0f, 0.0f, 0.0f)};
/* Calculate quality. */
texture_view->get_face_info(v1, v2, v3, &info, settings);
if (info.quality == 0.0) continue;
/* Change color space. */
mve::image::color_rgb_to_ycbcr(*(info.mean_color));
std::pair<std::size_t, FaceProjectionInfo> pair(face_id, info);
projected_face_view_infos.push_back(pair);
}
texture_view->release_image();
texture_view->release_validity_mask();
if (settings.data_term == DATA_TERM_GMI) {
texture_view->release_gradient_magnitude();
}
view_counter.inc();
}
//std::sort(projected_face_view_infos.begin(), projected_face_view_infos.end());
#pragma omp critical
{
for (std::size_t i = projected_face_view_infos.size(); 0 < i; --i) {
std::size_t face_id = projected_face_view_infos[i - 1].first;
FaceProjectionInfo const & info = projected_face_view_infos[i - 1].second;
face_projection_infos->at(face_id).push_back(info);
}
projected_face_view_infos.clear();
}
}
}
void
generate_texture_patches(UniGraph const & graph, mve::TriangleMesh::ConstPtr mesh,
mve::MeshInfo const & mesh_info,
std::vector<TextureView> * texture_views, Settings const & settings,
std::vector<std::vector<VertexProjectionInfo> > * vertex_projection_infos,
std::vector<TexturePatch::Ptr> * texture_patches) {
util::WallTimer timer;
mve::TriangleMesh::FaceList const & mesh_faces = mesh->get_faces();
mve::TriangleMesh::VertexList const & vertices = mesh->get_vertices();
vertex_projection_infos->resize(vertices.size());
std::size_t num_patches = 0;
std::cout << "\tRunning... " << std::flush;
#pragma omp parallel for schedule(dynamic)
for (std::size_t i = 0; i < texture_views->size(); ++i) {
std::vector<std::vector<std::size_t> > subgraphs;
int const label = i + 1;
graph.get_subgraphs(label, &subgraphs);
TextureView * texture_view = &texture_views->at(i);
texture_view->load_image();
std::list<TexturePatchCandidate> candidates;
for (std::size_t j = 0; j < subgraphs.size(); ++j) {
candidates.push_back(generate_candidate(label, *texture_view, subgraphs[j], mesh));
}
texture_view->release_image();
/* Merge candidates which contain the same image content. */
std::list<TexturePatchCandidate>::iterator it, sit;
for (it = candidates.begin(); it != candidates.end(); ++it) {
for (sit = candidates.begin(); sit != candidates.end();) {
Rect<int> bounding_box = sit->bounding_box;
if (it != sit && bounding_box.is_inside(&it->bounding_box)) {
TexturePatch::Faces & faces = it->texture_patch->get_faces();
TexturePatch::Faces & ofaces = sit->texture_patch->get_faces();
faces.insert(faces.end(), ofaces.begin(), ofaces.end());
TexturePatch::Texcoords & texcoords = it->texture_patch->get_texcoords();
TexturePatch::Texcoords & otexcoords = sit->texture_patch->get_texcoords();
math::Vec2f offset;
offset[0] = sit->bounding_box.min_x - it->bounding_box.min_x;
offset[1] = sit->bounding_box.min_y - it->bounding_box.min_y;
for (std::size_t i = 0; i < otexcoords.size(); ++i) {
texcoords.push_back(otexcoords[i] + offset);
}
sit = candidates.erase(sit);
} else {
++sit;
}
}
}
it = candidates.begin();
for (; it != candidates.end(); ++it) {
std::size_t texture_patch_id;
#pragma omp critical
{
texture_patches->push_back(it->texture_patch);
texture_patch_id = num_patches++;
}
std::vector<std::size_t> const & faces = it->texture_patch->get_faces();
std::vector<math::Vec2f> const & texcoords = it->texture_patch->get_texcoords();
for (std::size_t i = 0; i < faces.size(); ++i) {
std::size_t const face_id = faces[i];
std::size_t const face_pos = face_id * 3;
for (std::size_t j = 0; j < 3; ++j) {
std::size_t const vertex_id = mesh_faces[face_pos + j];
math::Vec2f const projection = texcoords[i * 3 + j];
VertexProjectionInfo info = {texture_patch_id, projection, {face_id}};
#pragma omp critical
vertex_projection_infos->at(vertex_id).push_back(info);
}
}
}
}
merge_vertex_projection_infos(vertex_projection_infos);
{
std::vector<std::size_t> unseen_faces;
std::vector<std::vector<std::size_t> > subgraphs;
graph.get_subgraphs(0, &subgraphs);
#pragma omp parallel for schedule(dynamic)
for (std::size_t i = 0; i < subgraphs.size(); ++i) {
std::vector<std::size_t> const & subgraph = subgraphs[i];
bool success = false;
if (settings.hole_filling) {
success = fill_hole(subgraph, graph, mesh, mesh_info,
vertex_projection_infos, texture_patches);
}
if (success) {
num_patches += 1;
} else {
if (settings.keep_unseen_faces) {
#pragma omp critical
unseen_faces.insert(unseen_faces.end(),
subgraph.begin(), subgraph.end());
}
}
}
if (!unseen_faces.empty()) {
mve::ByteImage::Ptr image = mve::ByteImage::create(3, 3, 3);
std::vector<math::Vec2f> texcoords;
for (std::size_t i = 0; i < unseen_faces.size(); ++i) {
math::Vec2f projections[] = {{2.0f, 1.0f}, {1.0f, 1.0f}, {1.0f, 2.0f}};
texcoords.insert(texcoords.end(), &projections[0], &projections[3]);
}
TexturePatch::Ptr texture_patch
= TexturePatch::create(0, unseen_faces, texcoords, image);
texture_patches->push_back(texture_patch);
std::size_t texture_patch_id = texture_patches->size() - 1;
for (std::size_t i = 0; i < unseen_faces.size(); ++i) {
std::size_t const face_id = unseen_faces[i];
std::size_t const face_pos = face_id * 3;
for (std::size_t j = 0; j < 3; ++j) {
std::size_t const vertex_id = mesh_faces[face_pos + j];
math::Vec2f const projection = texcoords[i * 3 + j];
VertexProjectionInfo info = {texture_patch_id, projection, {face_id}};
#pragma omp critical
vertex_projection_infos->at(vertex_id).push_back(info);
}
}
}
}
merge_vertex_projection_infos(vertex_projection_infos);
std::cout << "done. (Took " << timer.get_elapsed_sec() << "s)" << std::endl;
std::cout << "\t" << num_patches << " texture patches." << std::endl;
}
bool fill_hole(std::vector<std::size_t> const & hole, UniGraph const & graph,
mve::TriangleMesh::ConstPtr mesh, mve::MeshInfo const & mesh_info,
std::vector<std::vector<VertexProjectionInfo> > * vertex_projection_infos,
std::vector<TexturePatch::Ptr> * texture_patches) {
mve::TriangleMesh::FaceList const & mesh_faces = mesh->get_faces();
mve::TriangleMesh::VertexList const & vertices = mesh->get_vertices();
std::map<std::size_t, std::set<std::size_t> > tmp;
for (std::size_t const face_id : hole) {
std::size_t const v0 = mesh_faces[face_id * 3];
std::size_t const v1 = mesh_faces[face_id * 3 + 1];
std::size_t const v2 = mesh_faces[face_id * 3 + 2];
tmp[v0].insert(face_id);
tmp[v1].insert(face_id);
tmp[v2].insert(face_id);
}
std::size_t const num_vertices = tmp.size();
/* Only fill small holes. */
if (num_vertices > MAX_HOLE_NUM_FACES) return false;
/* Calculate 2D parameterization using the technique from libremesh/patch2d,
* which was published as sourcecode accompanying the following paper:
*
* Isotropic Surface Remeshing
* Simon Fuhrmann, Jens Ackermann, Thomas Kalbe, Michael Goesele
*/
std::size_t seed = -1;
std::vector<bool> is_border(num_vertices, false);
std::vector<std::vector<std::size_t> > adj_verts_via_border(num_vertices);
/* Index structures to map from local <-> global vertex id. */
std::map<std::size_t, std::size_t> g2l;
std::vector<std::size_t> l2g(num_vertices);
/* Index structure to determine column in matrix/vector. */
std::vector<std::size_t> idx(num_vertices);
std::size_t num_border_vertices = 0;
bool disk_topology = true;
std::map<std::size_t, std::set<std::size_t> >::iterator it = tmp.begin();
for (std::size_t j = 0; j < num_vertices; ++j, ++it) {
std::size_t vertex_id = it->first;
g2l[vertex_id] = j;
l2g[j] = vertex_id;
/* Check topology in original mesh. */
if (mesh_info[vertex_id].vclass != mve::MeshInfo::VERTEX_CLASS_SIMPLE) {
/* Complex/Border vertex in original mesh */
disk_topology = false;
break;
}
/* Check new topology and determine if vertex is now at the border. */
std::vector<std::size_t> const & adj_faces = mesh_info[vertex_id].faces;
std::set<std::size_t> const & adj_hole_faces = it->second;
std::vector<std::pair<std::size_t, std::size_t> > fan;
for (std::size_t k = 0; k < adj_faces.size(); ++k) {
std::size_t adj_face = adj_faces[k];
if (graph.get_label(adj_faces[k]) == 0 &&
adj_hole_faces.find(adj_face) != adj_hole_faces.end()) {
std::size_t curr = adj_faces[k];
std::size_t next = adj_faces[(k + 1) % adj_faces.size()];
std::pair<std::size_t, std::size_t> pair(curr, next);
fan.push_back(pair);
}
}
std::size_t gaps = 0;
for (std::size_t k = 0; k < fan.size(); k++) {
std::size_t curr = fan[k].first;
std::size_t next = fan[(k + 1) % fan.size()].first;
if (fan[k].second != next) {
++gaps;
for (std::size_t l = 0; l < 3; ++l) {
if(mesh_faces[curr * 3 + l] == vertex_id) {
std::size_t second = mesh_faces[curr * 3 + (l + 2) % 3];
adj_verts_via_border[j].push_back(second);
}
if(mesh_faces[next * 3 + l] == vertex_id) {
std::size_t first = mesh_faces[next * 3 + (l + 1) % 3];
adj_verts_via_border[j].push_back(first);
}
}
}
}
is_border[j] = gaps == 1;
/* Check if vertex is now complex. */
if (gaps > 1) {
/* Complex vertex in hole */
disk_topology = false;
break;
}
if (is_border[j]) {
idx[j] = num_border_vertices++;
seed = vertex_id;
} else {
idx[j] = j - num_border_vertices;
}
}
tmp.clear();
/* No disk or genus zero topology */
if (!disk_topology || num_border_vertices == 0) return false;
std::vector<std::size_t> border; border.reserve(num_border_vertices);
std::size_t prev = seed;
std::size_t curr = seed;
while (prev == seed || curr != seed) {
std::size_t next = std::numeric_limits<std::size_t>::max();
std::vector<std::size_t> const & adj_verts = adj_verts_via_border[g2l[curr]];
for (std::size_t adj_vert : adj_verts) {
assert(is_border[g2l[adj_vert]]);
if (adj_vert != prev && adj_vert != curr) {
next = adj_vert;
break;
}
}
if (next != std::numeric_limits<std::size_t>::max()) {
prev = curr;
curr = next;
border.push_back(next);
} else {
/* No new border vertex */
border.clear();
break;
}
/* Loop within border */
if (border.size() > num_border_vertices) break;
}
if (border.size() != num_border_vertices) return false;
float total_length = 0.0f;
float total_projection_length = 0.0f;
for (std::size_t j = 0; j < border.size(); ++j) {
std::size_t vi0 = border[j];
std::size_t vi1 = border[(j + 1) % border.size()];
std::vector<VertexProjectionInfo> const & vpi0 = vertex_projection_infos->at(vi0);
std::vector<VertexProjectionInfo> const & vpi1 = vertex_projection_infos->at(vi0);
/* According to the previous checks (vertex class within the origial
* mesh and boundary) there already has to be at least one projection
* of each border vertex. */
assert(!vpi0.empty() && !vpi1.empty());
math::Vec2f vp0(0.0f), vp1(0.0f);
for (VertexProjectionInfo const & info0 : vpi0) {
for (VertexProjectionInfo const & info1 : vpi1) {
if (info0.texture_patch_id == info1.texture_patch_id) {
vp0 = info0.projection;
vp1 = info1.projection;
break;
}
}
}
total_projection_length += (vp0 - vp1).norm();
math::Vec3f const & v0 = vertices[vi0];
math::Vec3f const & v1 = vertices[vi1];
total_length += (v0 - v1).norm();
}
float radius = total_projection_length / (2.0f * MATH_PI);
if (total_length < std::numeric_limits<float>::epsilon()) return false;
float length = 0.0f;
std::vector<math::Vec2f> projections(num_vertices);
for (std::size_t j = 0; j < border.size(); ++j) {
float angle = 2.0f * MATH_PI * (length / total_length);
projections[g2l[border[j]]] = math::Vec2f(std::cos(angle), std::sin(angle));
math::Vec3f const & v0 = vertices[border[j]];
math::Vec3f const & v1 = vertices[border[(j + 1) % border.size()]];
length += (v0 - v1).norm();
}
typedef Eigen::Triplet<float, int> Triplet;
std::vector<Triplet> coeff;
std::size_t matrix_size = num_vertices - border.size();
Eigen::VectorXf xx(matrix_size), xy(matrix_size);
if (matrix_size != 0) {
Eigen::VectorXf bx(matrix_size);
Eigen::VectorXf by(matrix_size);
for (std::size_t j = 0; j < num_vertices; ++j) {
if (is_border[j]) continue;
std::size_t const vertex_id = l2g[j];
/* Calculate "Mean Value Coordinates" as proposed by Michael S. Floater */
std::map<std::size_t, float> weights;
std::vector<std::size_t> const & adj_faces = mesh_info[vertex_id].faces;
for (std::size_t adj_face : adj_faces) {
std::size_t v0 = mesh_faces[adj_face * 3];
std::size_t v1 = mesh_faces[adj_face * 3 + 1];
std::size_t v2 = mesh_faces[adj_face * 3 + 2];
if (v1 == vertex_id) std::swap(v1, v0);
if (v2 == vertex_id) std::swap(v2, v0);
math::Vec3f v01 = vertices[v1] - vertices[v0];
float v01n = v01.norm();
math::Vec3f v02 = vertices[v2] - vertices[v0];
float v02n = v02.norm();
/* Ensure numerical stability */
if (v01n * v02n < std::numeric_limits<float>::epsilon()) return false;
float alpha = std::acos(v01.dot(v02) / (v01n * v02n));
weights[g2l[v1]] += std::tan(alpha / 2.0f) / v01n;
weights[g2l[v2]] += std::tan(alpha / 2.0f) / v02n;
}
std::map<std::size_t, float>::iterator it;
float sum = 0.0f;
for (it = weights.begin(); it != weights.end(); ++it)
sum += it->second;
assert(sum > 0.0f);
for (it = weights.begin(); it != weights.end(); ++it)
it->second /= sum;
bx[idx[j]] = 0.0f;
by[idx[j]] = 0.0f;
for (it = weights.begin(); it != weights.end(); ++it) {
if (is_border[it->first]) {
std::size_t border_vertex_id = border[idx[it->first]];
bx[idx[j]] += projections[g2l[border_vertex_id]][0] * it->second;
by[idx[j]] += projections[g2l[border_vertex_id]][1] * it->second;
} else {
coeff.push_back(Triplet(idx[j], idx[it->first], -it->second));
}
}
}
for (std::size_t j = 0; j < matrix_size; ++j) {
coeff.push_back(Triplet(j, j, 1.0f));
}
typedef Eigen::SparseMatrix<float> SpMat;
SpMat A(matrix_size, matrix_size);
A.setFromTriplets(coeff.begin(), coeff.end());
Eigen::SparseLU<SpMat> solver;
solver.analyzePattern(A);
solver.factorize(A);
xx = solver.solve(bx);
xy = solver.solve(by);
}
float const max_hole_patch_size = MAX_HOLE_PATCH_SIZE;
int image_size = std::min(std::floor(radius * 1.1f) * 2.0f, max_hole_patch_size);
/* Ensure a minimum scale of one */
image_size += 2 * (1 + texture_patch_border);
int scale = image_size / 2 - texture_patch_border;
for (std::size_t j = 0, k = 0; j < num_vertices; ++j) {
if (is_border[j]) {
projections[j] = projections[j] * scale + image_size / 2;
} else {
projections[j] = math::Vec2f(xx[k], xy[k]) * scale + image_size / 2;
++k;
}
}
mve::ByteImage::Ptr image = mve::ByteImage::create(image_size, image_size, 3);
//DEBUG image->fill_color(*math::Vec4uc(0, 255, 0, 255));
std::vector<math::Vec2f> texcoords; texcoords.reserve(hole.size());
for (std::size_t const face_id : hole) {
for (std::size_t j = 0; j < 3; ++j) {
std::size_t const vertex_id = mesh_faces[face_id * 3 + j];
math::Vec2f const & projection = projections[g2l[vertex_id]];
texcoords.push_back(projection);
}
}
TexturePatch::Ptr texture_patch = TexturePatch::create(0, hole, texcoords, image);
std::size_t texture_patch_id;
#pragma omp critical
{
texture_patches->push_back(texture_patch);
texture_patch_id = texture_patches->size() - 1;
}
for (std::size_t j = 0; j < num_vertices; ++j) {
std::size_t const vertex_id = l2g[j];
std::vector<std::size_t> const & adj_faces = mesh_info[vertex_id].faces;
std::vector<std::size_t> faces; faces.reserve(adj_faces.size());
for (std::size_t adj_face : adj_faces) {
if (graph.get_label(adj_face) == 0) {
faces.push_back(adj_face);
}
}
VertexProjectionInfo info = {texture_patch_id, projections[j], faces};
#pragma omp critical
vertex_projection_infos->at(vertex_id).push_back(info);
}
return true;
}
void
generate_texture_patches(UniGraph const & graph, std::vector<TextureView> const & texture_views,
mve::TriangleMesh::ConstPtr mesh, mve::VertexInfoList::ConstPtr vertex_infos,
std::vector<std::vector<VertexProjectionInfo> > * vertex_projection_infos,
std::vector<TexturePatch> * texture_patches) {
util::WallTimer timer;
mve::TriangleMesh::FaceList const & mesh_faces = mesh->get_faces();
mve::TriangleMesh::VertexList const & vertices = mesh->get_vertices();
vertex_projection_infos->resize(vertices.size());
std::size_t num_patches = 0;
std::cout << "\tRunning... " << std::flush;
#pragma omp parallel for schedule(dynamic)
for (std::size_t i = 0; i < texture_views.size(); ++i) {
std::vector<std::vector<std::size_t> > subgraphs;
int const label = i + 1;
graph.get_subgraphs(label, &subgraphs);
std::list<TexturePatchCandidate> candidates;
for (std::size_t j = 0; j < subgraphs.size(); ++j) {
candidates.push_back(generate_candidate(label, texture_views[i], subgraphs[j], mesh));
}
/* Merge candidates which contain the same image content. */
std::list<TexturePatchCandidate>::iterator it, sit;
for (it = candidates.begin(); it != candidates.end(); ++it) {
for (sit = candidates.begin(); sit != candidates.end();) {
Rect<int> bounding_box = sit->bounding_box;
if (it != sit && bounding_box.is_inside(&it->bounding_box)) {
TexturePatch::Faces & faces = it->texture_patch.get_faces();
TexturePatch::Faces & ofaces = sit->texture_patch.get_faces();
faces.insert(faces.end(), ofaces.begin(), ofaces.end());
TexturePatch::Texcoords & texcoords = it->texture_patch.get_texcoords();
TexturePatch::Texcoords & otexcoords = sit->texture_patch.get_texcoords();
math::Vec2f offset;
offset[0] = sit->bounding_box.min_x - it->bounding_box.min_x;
offset[1] = sit->bounding_box.min_y - it->bounding_box.min_y;
for (std::size_t i = 0; i < otexcoords.size(); ++i) {
texcoords.push_back(otexcoords[i] + offset);
}
sit = candidates.erase(sit);
} else {
++sit;
}
}
}
it = candidates.begin();
for (; it != candidates.end(); ++it) {
std::size_t texture_patch_id;
#pragma omp critical
{
texture_patches->push_back(it->texture_patch);
texture_patch_id = num_patches++;
}
std::vector<std::size_t> const & faces = it->texture_patch.get_faces();
std::vector<math::Vec2f> const & texcoords = it->texture_patch.get_texcoords();
for (std::size_t i = 0; i < faces.size(); ++i) {
std::size_t const face_id = faces[i];
std::size_t const face_pos = face_id * 3;
for (std::size_t j = 0; j < 3; ++j) {
std::size_t const vertex_id = mesh_faces[face_pos + j];
math::Vec2f const projection = texcoords[i * 3 + j];
VertexProjectionInfo info = {texture_patch_id, projection, {face_id}};
#pragma omp critical
vertex_projection_infos->at(vertex_id).push_back(info);
}
}
}
}
merge_vertex_projection_infos(vertex_projection_infos);
std::size_t num_holes = 0;
std::size_t num_hole_faces = 0;
//if (!settings.skip_hole_filling) {
{
std::vector<std::vector<std::size_t> > subgraphs;
graph.get_subgraphs(0, &subgraphs);
#pragma omp parallel for schedule(dynamic)
for (std::size_t i = 0; i < subgraphs.size(); ++i) {
std::vector<std::size_t> const & subgraph = subgraphs[i];
std::map<std::size_t, std::set<std::size_t> > tmp;
for (std::size_t const face_id : subgraph) {
std::size_t const v0 = mesh_faces[face_id * 3];
std::size_t const v1 = mesh_faces[face_id * 3 + 1];
std::size_t const v2 = mesh_faces[face_id * 3 + 2];
tmp[v0].insert(face_id);
tmp[v1].insert(face_id);
tmp[v2].insert(face_id);
}
std::size_t const num_vertices = tmp.size();
/* Only fill small holes. */
if (num_vertices > 100) {
//std::cerr << "Hole to large" << std::endl;
continue;
}
/* Calculate 2D parameterization using the technique from libremesh/patch2d,
* which was published as sourcecode accompanying the following paper:
*
* Isotropic Surface Remeshing
* Simon Fuhrmann, Jens Ackermann, Thomas Kalbe, Michael Goesele
*/
std::size_t seed = -1;
std::vector<bool> is_border(num_vertices, false);
std::vector<std::vector<std::size_t> > adj_verts_via_border(num_vertices);
/* Index structures to map from local <-> global vertex id. */
std::map<std::size_t, std::size_t> g2l;
std::vector<std::size_t> l2g(num_vertices);
/* Index structure to determine column in matrix/vector. */
std::vector<std::size_t> idx(num_vertices);
std::size_t num_border_vertices = 0;
bool disk_topology = true;
std::map<std::size_t, std::set<std::size_t> >::iterator it = tmp.begin();
for (std::size_t j = 0; j < num_vertices; ++j, ++it) {
std::size_t vertex_id = it->first;
g2l[vertex_id] = j;
l2g[j] = vertex_id;
/* Check topology in original mesh. */
if (vertex_infos->at(vertex_id).vclass != mve::VERTEX_CLASS_SIMPLE) {
//std::cerr << "Complex/Border vertex in original mesh" << std::endl;
disk_topology = false;
break;
}
/* Check new topology and determine if vertex is now at the border. */
std::vector<std::size_t> const & adj_faces = vertex_infos->at(vertex_id).faces;
std::set<std::size_t> const & adj_hole_faces = it->second;
std::vector<std::pair<std::size_t, std::size_t> > fan;
for (std::size_t k = 0; k < adj_faces.size(); ++k) {
std::size_t adj_face = adj_faces[k];
if (graph.get_label(adj_faces[k]) == 0 &&
adj_hole_faces.find(adj_face) != adj_hole_faces.end()) {
std::size_t curr = adj_faces[k];
std::size_t next = adj_faces[(k + 1) % adj_faces.size()];
std::pair<std::size_t, std::size_t> pair(curr, next);
fan.push_back(pair);
}
}
std::size_t gaps = 0;
for (std::size_t k = 0; k < fan.size(); k++) {
std::size_t curr = fan[k].first;
std::size_t next = fan[(k + 1) % fan.size()].first;
if (fan[k].second != next) {
++gaps;
for (std::size_t l = 0; l < 3; ++l) {
if(mesh_faces[curr * 3 + l] == vertex_id) {
std::size_t second = mesh_faces[curr * 3 + (l + 2) % 3];
adj_verts_via_border[j].push_back(second);
}
if(mesh_faces[next * 3 + l] == vertex_id) {
std::size_t first = mesh_faces[next * 3 + (l + 1) % 3];
adj_verts_via_border[j].push_back(first);
}
}
}
}
is_border[j] = gaps == 1;
/* Check if vertex is now complex. */
if (gaps > 1) {
//std::cerr << "Complex vertex in hole" << std::endl;
disk_topology = false;
break;
}
if (is_border[j]) {
idx[j] = num_border_vertices++;
seed = vertex_id;
} else {
idx[j] = j - num_border_vertices;
}
}
tmp.clear();
if (!disk_topology) continue;
if (num_border_vertices == 0) {
//std::cerr << "Genus zero topology" << std::endl;
continue;
}
std::vector<std::size_t> border; border.reserve(num_border_vertices);
std::size_t prev = seed;
std::size_t curr = seed;
while (prev == seed || curr != seed) {
std::size_t next = std::numeric_limits<std::size_t>::max();
std::vector<std::size_t> const & adj_verts = adj_verts_via_border[g2l[curr]];
for (std::size_t adj_vert : adj_verts) {
assert(is_border[g2l[adj_vert]]);
if (adj_vert != prev && adj_vert != curr) {
next = adj_vert;
break;
}
}
if (next != std::numeric_limits<std::size_t>::max()) {
prev = curr;
curr = next;
border.push_back(next);
} else {
//std::cerr << "No new border vertex" << std::endl;
border.clear();
break;
}
if (border.size() > num_border_vertices) {
//std::cerr << "Loop within border" << std::endl;
break;
}
}
if (border.size() != num_border_vertices) {
continue;
}
float total_length = 0.0f;
float total_projection_length = 0.0f;
for (std::size_t j = 0; j < border.size(); ++j) {
std::size_t vi0 = border[j];
std::size_t vi1 = border[(j + 1) % border.size()];
std::vector<VertexProjectionInfo> const & vpi0 = vertex_projection_infos->at(vi0);
std::vector<VertexProjectionInfo> const & vpi1 = vertex_projection_infos->at(vi0);
/* According to the previous checks (vertex class within the origial
* mesh and boundary) there already has to be at least one projection
* of each border vertex. */
assert(!vpi0.empty() && !vpi1.empty());
math::Vec2f vp0(0.0f), vp1(0.0f);
for (VertexProjectionInfo const & info0 : vpi0) {
for (VertexProjectionInfo const & info1 : vpi1) {
if (info0.texture_patch_id == info1.texture_patch_id) {
vp0 = info0.projection;
vp1 = info1.projection;
break;
}
}
}
total_projection_length += (vp0 - vp1).norm();
math::Vec3f const & v0 = vertices[vi0];
math::Vec3f const & v1 = vertices[vi1];
total_length += (v0 - v1).norm();
}
float radius = total_projection_length / (2.0f * MATH_PI);
float length = 0.0f;
std::vector<math::Vec2f> projections(num_vertices);
for (std::size_t j = 0; j < border.size(); ++j) {
float angle = 2.0f * MATH_PI * (length / total_length);
projections[g2l[border[j]]] = math::Vec2f(std::cos(angle), std::sin(angle));
math::Vec3f const & v0 = vertices[border[j]];
math::Vec3f const & v1 = vertices[border[(j + 1) % border.size()]];
length += (v0 - v1).norm();
}
typedef Eigen::Triplet<float, int> Triplet;
std::vector<Triplet> coeff;
std::size_t matrix_size = num_vertices - border.size();
Eigen::VectorXf xx(matrix_size), xy(matrix_size);
if (matrix_size != 0) {
Eigen::VectorXf bx(matrix_size);
Eigen::VectorXf by(matrix_size);
for (std::size_t j = 0; j < num_vertices; ++j) {
if (is_border[j]) continue;
std::size_t const vertex_id = l2g[j];
/* Calculate "Mean Value Coordinates" as proposed by Michael S. Floater */
std::map<std::size_t, float> weights;
std::vector<std::size_t> const & adj_faces = vertex_infos->at(vertex_id).faces;
for (std::size_t adj_face : adj_faces) {
std::size_t v0 = mesh_faces[adj_face * 3];
std::size_t v1 = mesh_faces[adj_face * 3 + 1];
std::size_t v2 = mesh_faces[adj_face * 3 + 2];
if (v1 == vertex_id) std::swap(v1, v0);
if (v2 == vertex_id) std::swap(v2, v0);
math::Vec3f v01 = vertices[v1] - vertices[v0];
float v01n = v01.norm();
math::Vec3f v02 = vertices[v2] - vertices[v0];
float v02n = v02.norm();
float alpha = std::acos(v01.dot(v02) / (v01n * v02n));
weights[g2l[v1]] += std::tan(alpha / 2.0f) / v01n;
weights[g2l[v2]] += std::tan(alpha / 2.0f) / v02n;
}
std::map<std::size_t, float>::iterator it;
float sum = 0.0f;
for (it = weights.begin(); it != weights.end(); ++it)
sum += it->second;
for (it = weights.begin(); it != weights.end(); ++it)
it->second /= sum;
bx[idx[j]] = 0.0f;
by[idx[j]] = 0.0f;
for (it = weights.begin(); it != weights.end(); ++it) {
if (is_border[it->first]) {
std::size_t border_vertex_id = border[idx[it->first]];
bx[idx[j]] += projections[g2l[border_vertex_id]][0] * it->second;
by[idx[j]] += projections[g2l[border_vertex_id]][1] * it->second;
} else {
coeff.push_back(Triplet(idx[j], idx[it->first], -it->second));
}
}
}
for (std::size_t j = 0; j < matrix_size; ++j) {
coeff.push_back(Triplet(j, j, 1.0f));
}
typedef Eigen::SparseMatrix<float> SpMat;
SpMat A(matrix_size, matrix_size);
A.setFromTriplets(coeff.begin(), coeff.end());
Eigen::SparseLU<SpMat> solver;
solver.analyzePattern(A);
solver.factorize(A);
xx = solver.solve(bx);
xy = solver.solve(by);
}
int image_size = std::floor(radius * 1.1f) * 2 + 4;
int scale = image_size / 2 - texture_patch_border;
for (std::size_t j = 0, k = 0; j < num_vertices; ++j) {
if (!is_border[j]) {
projections[j] = math::Vec2f(xx[k], xy[k]) * scale + image_size / 2;
++k;
} else {
projections[j] = projections[j] * scale + image_size / 2;
}
}
mve::ByteImage::Ptr image = mve::ByteImage::create(image_size, image_size, 3);
//DEBUG image->fill_color(*math::Vec4uc(0, 255, 0, 255));
std::vector<math::Vec2f> texcoords; texcoords.reserve(subgraph.size());
for (std::size_t const face_id : subgraph) {
for (std::size_t j = 0; j < 3; ++j) {
std::size_t const vertex_id = mesh_faces[face_id * 3 + j];
math::Vec2f const & projection = projections[g2l[vertex_id]];
texcoords.push_back(projection);
}
}
TexturePatch texture_patch(0, subgraph, texcoords, image);
std::size_t texture_patch_id;
#pragma omp critical
{
texture_patches->push_back(texture_patch);
texture_patch_id = num_patches++;
num_hole_faces += subgraph.size();
++num_holes;
}
for (std::size_t j = 0; j < num_vertices; ++j) {
std::size_t const vertex_id = l2g[j];
std::vector<std::size_t> const & adj_faces = vertex_infos->at(vertex_id).faces;
std::vector<std::size_t> faces; faces.reserve(adj_faces.size());
for (std::size_t adj_face : adj_faces) {
if (graph.get_label(adj_face) == 0) {
faces.push_back(adj_face);
}
}
VertexProjectionInfo info = {texture_patch_id, projections[j], faces};
#pragma omp critical
vertex_projection_infos->at(vertex_id).push_back(info);
}
}
}
merge_vertex_projection_infos(vertex_projection_infos);
std::cout << "done. (Took " << timer.get_elapsed_sec() << "s)" << std::endl;
std::cout << "\t" << num_patches << " texture patches." << std::endl;
std::cout << "\t" << num_holes << " holes (" << num_hole_faces << " faces)." << std::endl;
}