uint64_t used_memory() const {
const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
const uint64_t members = nmembers * sizeof(MemberMeta);
const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
const uint64_t members_buffer_capacity = m_members_buffer.capacity();
std::cout << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
std::cout << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
std::cout << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
std::cout << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
std::cout << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
std::cout << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
std::cout << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
std::cout << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
std::cout << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
std::cout << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
std::cout << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
std::cout << " total .................................. = " << std::setw(12) << total << "\n";
std::cout << " =======================================================\n";
return relations_buffer_capacity + members_buffer_capacity + relations + members;
}
/**
* Tell the Collector that you are interested in this relation
* and want it kept until all members have been assembled and
* it is handed back to you.
*
* The relation is copied and stored in a buffer inside the
* collector.
*/
void add_relation(const osmium::Relation& relation) {
const size_t offset = m_relations_buffer.committed();
m_relations_buffer.add_item(relation);
RelationMeta relation_meta(offset);
int n = 0;
for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
relation_meta.increment_need_members();
} else {
member.ref(0); // set member id to zero to indicate we are not interested
}
++n;
}
assert(offset == m_relations_buffer.committed());
if (relation_meta.has_all_members()) {
m_relations_buffer.rollback();
} else {
m_relations_buffer.commit();
m_relations.push_back(std::move(relation_meta));
// std::cerr << "added relation id=" << relation.id() << "\n";
}
}
/**
* Parses one line in OPL format. The line must not have a newline
* character at the end. Buffer.commit() is called automatically if the
* write succeeded.
*
* @param data Line must be in this zero-delimited string.
* @param buffer Result will be written to this buffer.
*
* @returns true if an entity was parsed, false otherwise (for instance
* when the line is empty).
* @throws osmium::opl_error If the parsing fails.
*/
inline bool opl_parse(const char* data, osmium::memory::Buffer& buffer) {
try {
const bool wrote_something = osmium::io::detail::opl_parse_line(0, data, buffer);
buffer.commit();
return wrote_something;
} catch (const osmium::opl_error&) {
buffer.rollback();
throw;
}
}
explicit Builder(osmium::memory::Buffer& buffer, Builder* parent, osmium::memory::item_size_type size) :
m_buffer(buffer),
m_parent(parent),
m_item_offset(buffer.written()) {
m_buffer.reserve_space(size);
assert(buffer.is_aligned());
if (m_parent) {
m_parent->add_size(size);
}
}
std::string operator()() {
osmium::apply(m_input_buffer.cbegin(), m_input_buffer.cend(), *this);
if (m_write_change_ops) {
open_close_op_tag();
}
std::string out;
std::swap(out, m_out);
return out;
}
/**
* Decide whether to purge removed members and then do it.
*
* Currently the purging is done every thousand calls.
* This could probably be improved upon.
*/
void possibly_purge_removed_members() {
++m_count_complete;
if (m_count_complete > 10000) { // XXX
const size_t size_before = m_members_buffer.committed();
m_members_buffer.purge_removed(this);
const size_t size_after = m_members_buffer.committed();
double percent = static_cast<double>(size_before - size_after);
percent /= size_before;
percent *= 100;
std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
m_count_complete = 0;
}
}
/**
* Assemble an area from the given way.
* The resulting area is put into the out_buffer.
*
* @returns false if there was some kind of error building the
* area, true otherwise.
*/
bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) {
if (!config().create_way_polygons) {
return true;
}
if (config().problem_reporter) {
config().problem_reporter->set_object(osmium::item_type::way, way.id());
config().problem_reporter->set_nodes(way.nodes().size());
}
// Ignore (but count) ways without segments.
if (way.nodes().size() < 2) {
++stats().short_ways;
return false;
}
if (!way.ends_have_same_id()) {
++stats().duplicate_nodes;
if (config().problem_reporter) {
config().problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location());
}
}
++stats().from_ways;
stats().invalid_locations = segment_list().extract_segments_from_way(config().problem_reporter,
stats().duplicate_nodes,
way);
if (!config().ignore_invalid_locations && stats().invalid_locations > 0) {
return false;
}
if (config().debug_level > 0) {
std::cerr << "\nAssembling way " << way.id() << " containing " << segment_list().size() << " nodes\n";
}
// Now create the Area object and add the attributes and tags
// from the way.
const bool okay = create_area(out_buffer, way);
if (okay) {
out_buffer.commit();
} else {
out_buffer.rollback();
}
if (debug()) {
std::cerr << "Done: " << stats() << "\n";
}
return okay;
}
bool middle_pgsql_t::relations_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
char tmp[16];
char const *paramValues[1];
PGconn *sql_conn = rel_table->sql_conn;
taglist_t member_temp;
// Make sure we're out of copy mode */
pgsql_endCopy(rel_table);
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
paramValues[0] = tmp;
PGresult *res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK);
// Fields are: members, tags, member_count */
if (PQntuples(res) != 1) {
PQclear(res);
return false;
}
{
osmium::builder::RelationBuilder builder(buffer);
builder.set_id(id);
pgsql_parse_members(PQgetvalue(res, 0, 0), buffer, builder);
pgsql_parse_tags(PQgetvalue(res, 0, 1), buffer, builder);
}
PQclear(res);
buffer.commit();
return true;
}
bool middle_pgsql_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
char const *paramValues[1];
PGconn *sql_conn = way_table->sql_conn;
// Make sure we're out of copy mode */
pgsql_endCopy(way_table);
char tmp[16];
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
paramValues[0] = tmp;
PGresult *res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK);
if (PQntuples(res) != 1) {
PQclear(res);
return false;
}
{
osmium::builder::WayBuilder builder(buffer);
builder.set_id(id);
pgsql_parse_nodes(PQgetvalue(res, 0, 0), buffer, builder);
pgsql_parse_tags(PQgetvalue(res, 0, 1), buffer, builder);
}
PQclear(res);
buffer.commit();
return true;
}
size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const
{
if (ids.empty())
return 0;
char tmp[16];
std::unique_ptr<char[]> tmp2(new (std::nothrow) char[ids.size() * 16]);
char const *paramValues[1];
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2.get(), "{");
for (auto id : ids) {
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", id);
strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2));
}
tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/
pgsql_endCopy(way_table);
PGconn *sql_conn = way_table->sql_conn;
paramValues[0] = tmp2.get();
PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
idlist_t wayidspg;
for (int i = 0; i < countPG; i++) {
wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10));
}
// Match the list of ways coming from postgres in a different order
// back to the list of ways given by the caller */
int outres = 0;
for (auto id : ids) {
for (int j = 0; j < countPG; j++) {
if (id == wayidspg[j]) {
{
osmium::builder::WayBuilder builder(buffer);
builder.set_id(id);
pgsql_parse_nodes(PQgetvalue(res, j, 1), buffer, builder);
pgsql_parse_tags(PQgetvalue(res, j, 2), buffer, builder);
}
buffer.commit();
outres++;
break;
}
}
}
PQclear(res);
return outres;
}
/**
* Assemble an area from the given relation and its members.
* The resulting area is put into the out_buffer.
*
* @returns false if there was some kind of error building the
* area(s), true otherwise.
*/
bool operator()(const osmium::Relation& relation, const std::vector<const osmium::Way*>& members, osmium::memory::Buffer& out_buffer) {
if (!config().create_new_style_polygons) {
return true;
}
assert(relation.cmembers().size() >= members.size());
if (config().problem_reporter) {
config().problem_reporter->set_object(osmium::item_type::relation, relation.id());
}
if (relation.members().empty()) {
++stats().no_way_in_mp_relation;
return false;
}
++stats().from_relations;
stats().invalid_locations = segment_list().extract_segments_from_ways(config().problem_reporter,
stats().duplicate_nodes,
stats().duplicate_ways,
relation,
members);
if (!config().ignore_invalid_locations && stats().invalid_locations > 0) {
return false;
}
stats().member_ways = members.size();
if (stats().member_ways == 1) {
++stats().single_way_in_mp_relation;
}
if (config().debug_level > 0) {
std::cerr << "\nAssembling relation " << relation.id() << " containing " << members.size() << " way members with " << segment_list().size() << " nodes\n";
}
// Now create the Area object and add the attributes and tags
// from the relation.
bool okay = create_area(out_buffer, relation, members);
if (okay) {
out_buffer.commit();
} else {
out_buffer.rollback();
}
return okay;
}
std::size_t parse_polygon_array(const rapidjson::Value& value, osmium::memory::Buffer& buffer) {
{
osmium::builder::AreaBuilder builder{buffer};
parse_rings(value, builder);
}
return buffer.commit();
}
void run() final {
osmium::thread::set_thread_name("_osmium_opl_in");
line_by_line(*this);
if (m_buffer.committed() > 0) {
send_to_output_queue(std::move(m_buffer));
}
}
void maybe_flush() {
if (m_buffer.committed() > 800*1024) {
osmium::memory::Buffer buffer{1024*1024};
using std::swap;
swap(m_buffer, buffer);
send_to_output_queue(std::move(buffer));
}
}
/**
* Assemble an area from the given way.
*
* The resulting area is put into the out_buffer.
*
* @returns false if there was some kind of error building the
* area, true otherwise.
*/
bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) {
segment_list().extract_segments_from_way(config().problem_reporter, stats().duplicate_nodes, way);
if (!create_rings()) {
return false;
}
{
osmium::builder::AreaBuilder builder{out_buffer};
builder.initialize_from_object(way);
add_rings_to_area(builder);
}
out_buffer.commit();
return true;
}
std::vector<Edge> parok()
{
std::vector<Edge> par_lista;
for(unsigned int i=0;i<utak.capacity();i++)
{
osmium::WayNodeList * utnodeok;
utnodeok=&utak.get<osmium::Way>(i).nodes();
for(unsigned int j=0;j<(*utnodeok).size()-1;j++)
{
Edge par(((*utnodeok)[j]),((*utnodeok)[j+1]));
/*par.first=((*utnodeok)[j]).ref();
par.second=((*utnodeok)[j+1]).ref();*/
par_lista.push_back(par);
}
}
return par_lista;
}
std::size_t parse_multipolygon_array(const rapidjson::Value& value, osmium::memory::Buffer& buffer) {
assert(value.IsArray());
const auto array = value.GetArray();
if (array.Empty()) {
throw config_error{"Multipolygon must contain at least one polygon array."};
}
{
osmium::builder::AreaBuilder builder{buffer};
for (const auto& polygon : array) {
if (!polygon.IsArray()) {
throw config_error{"Polygon must be an array."};
}
parse_rings(polygon, builder);
}
}
return buffer.commit();
}
buffer_iterator_pair get_next_iter() {
m_buffer = read();
return std::make_pair(m_buffer.begin(), m_buffer.end());
}
osmium::item_type operator()(const osmium::memory::Buffer& buffer) {
return operator()(buffer.cbegin(), buffer.cend());
}
// XXX
void operator()(const osmium::memory::Buffer& buffer) {
osmium::io::detail::reliable_write(m_data_fd, buffer.data(), buffer.committed());
osmium::apply(buffer.begin(), buffer.end(), *this);
}
#include "catch.hpp"
#include <osmium/memory/buffer.hpp>
#include <array>
#include <stdexcept>
TEST_CASE("Buffer basics") {
osmium::memory::Buffer invalid_buffer1;
osmium::memory::Buffer invalid_buffer2;
osmium::memory::Buffer empty_buffer1{1024};
osmium::memory::Buffer empty_buffer2{2048};
REQUIRE_FALSE(invalid_buffer1);
REQUIRE_FALSE(invalid_buffer2);
REQUIRE(empty_buffer1);
REQUIRE(empty_buffer2);
REQUIRE(invalid_buffer1 == invalid_buffer2);
REQUIRE(invalid_buffer1 != empty_buffer1);
REQUIRE(empty_buffer1 != empty_buffer2);
REQUIRE(invalid_buffer1.capacity() == 0);
REQUIRE(invalid_buffer1.written() == 0);
REQUIRE(invalid_buffer1.committed() == 0);
REQUIRE(invalid_buffer1.clear() == 0);
REQUIRE(empty_buffer1.capacity() == 1024);
REQUIRE(empty_buffer1.written() == 0);
REQUIRE(empty_buffer1.committed() == 0);
REQUIRE(empty_buffer1.is_aligned());
#include <osmium/io/any_compression.hpp>
#include <osmium/io/xml_input.hpp>
TEST_CASE("Single relation") {
SECTION("Should extract the relation because it uses a node in the bounding box") {
using id_set = std_id_set;
osmium::io::File file(test_data_file("test_single_relation.osm"));
osmium::Box box(osmium::Location(double(-1), double(-1)),
osmium::Location(double( 1), double( 1)));
hsplitter::HistoryExtractReader<id_set> reader =
hsplitter::read_extract<id_set>(file, box);
REQUIRE(!reader.eof());
osmium::memory::Buffer buffer = reader.read();
REQUIRE(std::distance(buffer.begin(), buffer.end()) == 2);
osmium::memory::Buffer::iterator itr = buffer.begin();
{
REQUIRE(itr->type() == osmium::item_type::node);
auto &node = static_cast<const osmium::Node &>(*itr++);
REQUIRE(node.id() == 1);
}
{
REQUIRE(itr->type() == osmium::item_type::relation);
auto &rel = static_cast<const osmium::Relation &>(*itr++);
REQUIRE(rel.id() == 1);
}
}
SECTION("Should extract the relation because it uses a way which uses a node in the bbox") {
osmium::item_type operator()(osmium::memory::Buffer& buffer) {
return operator()(buffer.begin(), buffer.end());
}
inline void apply_diff(const osmium::memory::Buffer& buffer, THandlers&... handlers) {
apply_diff(buffer.cbegin(), buffer.cend(), handlers...);
}
TEST_CASE("Write with mock compressor") {
std::string fail_in;
osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
[&](int, osmium::io::fsync) { return new MockCompressor(fail_in); },
[](int) { return nullptr; },
[](const char*, size_t) { return nullptr; }
);
osmium::io::Header header;
header.set("generator", "test_writer_with_mock_compression.cpp");
osmium::io::Reader reader{with_data_dir("t/io/data.osm")};
osmium::memory::Buffer buffer = reader.read();
REQUIRE(buffer);
REQUIRE(buffer.committed() > 0);
REQUIRE(buffer.select<osmium::OSMObject>().size() > 0);
SECTION("fail on construction") {
fail_in = "constructor";
REQUIRE_THROWS_AS([&](){
osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm.gz", header, osmium::io::overwrite::allow);
writer(std::move(buffer));
writer.close();
}(), const std::logic_error&);
}
void possibly_flush_output_buffer() {
if (m_output_buffer.committed() > max_buffer_size_for_flush) {
flush_output_buffer();
}
}
osmium::item_type operator()(osmium::memory::Buffer& buffer, const osmium::item_type type) {
return operator()(buffer.begin(), buffer.end(), type);
}
TEST_CASE("Split relations") {
SECTION("should account for nodes.") {
using map_t = tileset;
constexpr size_t buffer_size = 10000;
unsigned char data[buffer_size];
map_t node_tiles, way_tiles, extra_node_tiles;
node_tiles.m_map[1] = std::set<hsplitter::tile_t>({1});
node_tiles.m_map[2] = std::set<hsplitter::tile_t>({1});
node_tiles.m_map[3] = std::set<hsplitter::tile_t>({1});
node_tiles.freeze();
way_tiles.freeze();
extra_node_tiles.freeze();
osmium::memory::Buffer rels(data, buffer_size, 0);
push_back_rel(rels, 1, 1, true, 1, 1, {1, 2, 3}, {}, {});
REQUIRE(std::distance(rels.begin(), rels.end()) == 1);
auto it = rels.begin();
auto pair = hsplitter::tiles_for_relations<map_t>(it, rels.end(), node_tiles, way_tiles, extra_node_tiles);
map_t rel_tiles = std::move(pair.first);
map_t extra_rel_tiles = std::move(pair.second);
REQUIRE(rel_tiles.m_map.size() == 1);
REQUIRE(rel_tiles.m_map.count(1) == 1);
REQUIRE(rel_tiles.m_map[1].size() == 1);
REQUIRE(rel_tiles.m_map[1].count(1) == 1);
REQUIRE(extra_rel_tiles.m_map.empty());