void
QcOsmPbfReader::read_dense_nodes(OSMPBF::PrimitiveGroup primitive_group)
{
enter_node_transactions();
OSMPBF::DenseNodes dense_node = primitive_group.dense();
DeltaCodedInt64 node_id;
DeltaCodedInt64 longitude;
DeltaCodedInt64 latitude;
int number_of_nodes = dense_node.id_size();
for (int i = 0; i < number_of_nodes; i++) {
node_id.update(dense_node.id(i));
longitude.update(dense_node.lon(i));
latitude.update(dense_node.lat(i));
// qDebug() << " dense node" << node_id() << to_wgs(longitude(), latitude());
yield_node(node_id(), longitude(), latitude());
}
// The storage pattern is: ((<keyid> <valid>)* '0' )*
bool is_key = true;
int key_id = 0;
for (int i = 0, l = dense_node.keys_vals_size(), node_index = 0;
i < l; i++) {
int32_t key_val_id = dense_node.keys_vals(i);
if (key_val_id) {
if (is_key)
key_id = key_val_id;
else {
// qDebug() << " attr" << node_index << m_string_table[key_id] << "=" << m_string_table[key_val_id];
yield_node_attribute(node_index, key_id, key_val_id);
}
is_key = not(is_key);
}
else
node_index++;
}
if (m_read_metadatas and dense_node.has_denseinfo()) {
// qDebug().nospace() << " with meta-info";
OSMPBF::DenseInfo dense_info = dense_node.denseinfo();
DeltaCodedInt64 timestamp;
DeltaCodedInt64 changeset;
DeltaCodedInt64 uid;
DeltaCodedInt64 user_sid;
for (int i = 0; i < number_of_nodes; i++) {
int32_t version = dense_info.version(i);
timestamp.update(to_timestamp(dense_info.timestamp(i)));
changeset.update(dense_info.changeset(i));
uid.update(dense_info.uid(i));
user_sid.update(dense_info.user_sid(i));
// bool visible = dense_info.visible(i);
// qDebug() << "Meta information:" << version << timestamp() << changeset() << uid() << user_sid();
// yield_node_metadata(i, version, timestamp(), changeset(), uid(), user_sid());
}
}
leave_node_transactions();
}
bool OsmPbfReader::parse(std::istream &input) {
std::vector<std::pair<uint64_t, std::string> > roadsWithoutRef;
swedishTextTree = nullptr;
node2Coord = nullptr;
nodeNames = nullptr;
wayNames = nullptr;
relationNames = nullptr;
wayNodes = nullptr;
relMembers = nullptr;
sweden = nullptr;
size_t count_named_nodes = 0, count_named_ways = 0, count_named_relations = 0;
if (!input || !input.good())
return false;
swedishTextTree = new SwedishTextTree();
if (swedishTextTree == nullptr)
Error::err("Could not allocate memory for swedishTextTree");
node2Coord = new IdTree<Coord>();
if (node2Coord == nullptr)
Error::err("Could not allocate memory for node2Coord");
nodeNames = new IdTree<WriteableString>();
if (nodeNames == nullptr)
Error::err("Could not allocate memory for nodeNames");
wayNames = new IdTree<WriteableString>();
if (wayNames == nullptr)
Error::err("Could not allocate memory for wayNames");
relationNames = new IdTree<WriteableString>();
if (relationNames == nullptr)
Error::err("Could not allocate memory for relationNames");
wayNodes = new IdTree<WayNodes>();
if (wayNodes == nullptr)
Error::err("Could not allocate memory for wayNodes");
relMembers = new IdTree<RelationMem>();
if (relMembers == nullptr)
Error::err("Could not allocate memory for relmem");
sweden = new Sweden();
if (sweden == nullptr)
Error::err("Could not allocate memory for Sweden");
doneWaySimplification = false;
boost::thread waySimplificationThread(consumerWaySimplification);
size_t max_queue_size = 0;
#ifdef CPUTIMER
Timer primitiveGroupTimer;
int64_t accumulatedPrimitiveGroupTime = 0;
#endif // CPUTIMER
/// Read while the file has not reached its end
while (input.good()) {
/// Storage of size, used multiple times
int32_t sz;
/// Read the first 4 bytes of the file, this is the size of the blob-header
input.read((char *)(&sz), sizeof(sz));
if (!input.good()) {
break; /// End of file reached
}
/// Convert the size from network byte-order to host byte-order
sz = ntohl(sz);
/// Ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
if (sz > OSMPBF::max_blob_header_size) {
Error::err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
}
// read the blob-header from the file
input.read(buffer, sz);
if (input.gcount() != sz || input.fail()) {
Error::err("unable to read blob-header from file");
}
// parse the blob-header from the read-buffer
if (!blobheader.ParseFromArray(buffer, sz)) {
Error::err("unable to parse blob header");
}
// size of the following blob
sz = blobheader.datasize();
//debug(" datasize = %u", sz);
// ensure the blob is smaller then MAX_BLOB_SIZE
if (sz > OSMPBF::max_uncompressed_blob_size) {
Error::err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
}
// read the blob from the file
input.read(buffer, sz);
if (input.gcount() != sz || input.fail()) {
Error::err("unable to read blob from file");
} else if (isatty(1))
std::cout << (sz > (1 << 18) ? "*" : (sz > (1 << 16) ? ":" : ".")) << std::flush;
// parse the blob from the read-buffer
if (!blob.ParseFromArray(buffer, sz)) {
Error::err("unable to parse blob");
}
// set when we find at least one data stream
bool found_data = false;
// if the blob has uncompressed data
if (blob.has_raw()) {
// we have at least one datastream
found_data = true;
// size of the blob-data
sz = blob.raw().size();
// check that raw_size is set correctly
if (sz != blob.raw_size()) {
Error::warn(" reports wrong raw_size: %u bytes", blob.raw_size());
}
// tell about the blob-data
//debug(" contains uncompressed data: %u bytes", sz);
// copy the uncompressed data over to the unpack_buffer
memcpy(unpack_buffer, buffer, sz);
}
// if the blob has zlib-compressed data
if (blob.has_zlib_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
Error::warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// the size of the compressesd data
sz = blob.zlib_data().size();
// zlib information
z_stream z;
// next byte to decompress
z.next_in = (unsigned char *) blob.zlib_data().c_str();
// number of bytes to decompress
z.avail_in = sz;
// place of next decompressed byte
z.next_out = (unsigned char *) unpack_buffer;
// space for decompressed data
z.avail_out = blob.raw_size();
// misc
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
if (inflateInit(&z) != Z_OK) {
Error::err(" failed to init zlib stream");
}
if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
Error::err(" failed to inflate zlib stream");
}
if (inflateEnd(&z) != Z_OK) {
Error::err(" failed to deinit zlib stream");
}
// unpacked size
sz = z.total_out;
}
// if the blob has lzma-compressed data
if (blob.has_lzma_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
Error::warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// issue a warning, lzma compression is not yet supported
Error::err(" lzma-decompression is not supported");
}
// check we have at least one data-stream
if (!found_data) {
Error::err(" does not contain any known data stream");
}
// switch between different blob-types
if (blobheader.type() == "OSMHeader") {
// parse the HeaderBlock from the blob
if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
Error::err("unable to parse header block");
}
} else if (blobheader.type() == "OSMData") {
// parse the PrimitiveBlock from the blob
if (!primblock.ParseFromArray(unpack_buffer, sz)) {
Error::err("unable to parse primitive block");
}
// iterate over all PrimitiveGroups
for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
// one PrimitiveGroup from the the Block
OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);
bool found_items = false;
const double coord_scale = 0.000000001;
#ifdef CPUTIMER
primitiveGroupTimer.start();
#endif // CPUTIMER
if (pg.nodes_size() > 0) {
found_items = true;
const int maxnodes = pg.nodes_size();
for (int j = 0; j < maxnodes; ++j) {
const uint64_t id = pg.nodes(j).id();
OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
/// Track various names like 'name', 'name:en', or 'name:bridge:dk'
std::map<std::string, std::string> name_set;
const double lat = coord_scale * (primblock.lat_offset() + (primblock.granularity() * pg.nodes(j).lat()));
const double lon = coord_scale * (primblock.lon_offset() + (primblock.granularity() * pg.nodes(j).lon()));
node2Coord->insert(id, Coord::fromLonLat(lon, lat));
bool node_is_county = false, node_is_municipality = false, node_is_traffic_sign = false;
for (int k = 0; k < pg.nodes(j).keys_size(); ++k) {
const char *ckey = primblock.stringtable().s(pg.nodes(j).keys(k)).c_str();
if (strcmp("name", ckey) == 0) {
/// Store 'name' string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.nodes(j).keys(k)), primblock.stringtable().s(pg.nodes(j).vals(k))));
++count_named_nodes;
} else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
/// Store name string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.nodes(j).keys(k)), primblock.stringtable().s(pg.nodes(j).vals(k))));
} else if (strcmp("place", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.nodes(j).vals(k)).c_str();
if (strcmp("county", cvalue) == 0) {
/// FIX OSM DATA
/// Counties should not be represented by nodes, but by relations
/// representing an area
// realworld_type = OSMElement::PlaceLargeArea;
node_is_county = true;
} else if (strcmp("municipality", cvalue) == 0) {
/// FIX OSM DATA
/// Counties should not be represented by nodes, but by relations
/// representing an area
// realworld_type = OSMElement::PlaceLargeArea;
node_is_municipality = true;
} else if (strcmp("traffic_sign", cvalue) == 0) {
/// Traffic signs may simply point to a location, but not be *at* this location.
/// Thus, their names (if set) may be misleading and so they should be ignored.
node_is_traffic_sign = true;
}
if (strcmp("city", cvalue) == 0 || strcmp("municipality", cvalue) == 0)
realworld_type = OSMElement::PlaceLarge;
else if (strcmp("borough", cvalue) == 0 || strcmp("suburb", cvalue) == 0 || strcmp("town", cvalue) == 0 || strcmp("village", cvalue) == 0)
realworld_type = OSMElement::PlaceMedium;
else if (strcmp("quarter", cvalue) == 0 || strcmp("neighbourhood", cvalue) == 0 || strcmp("hamlet", cvalue) == 0 || strcmp("isolated_dwelling", cvalue) == 0)
/// Disabling 'city_block' as those may have misleading names like node 3188612201 ('Skaraborg') in Södermalm, Stockholm
realworld_type = OSMElement::PlaceSmall;
else if (strcmp("island", cvalue) == 0)
realworld_type = OSMElement::Island;
else {
/// Skipping other types of places:
/// * Administrative boundaries should be checked elsewhere like SCBareas or NUTS3areas
/// * Very small places like farms or plots neither
}
} else if (strcmp("natural", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.nodes(j).vals(k)).c_str();
if (strcmp("water", cvalue) == 0)
realworld_type = OSMElement::Water;
}
}
if (node_is_municipality)
Error::info("Municipality '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), id);
else if (node_is_county)
Error::info("County '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), id);
else if (node_is_traffic_sign)
Error::info("Node %llu with name '%s' is a traffic sign, not recoding node's name", id, name_set["name"].c_str());
else if (!name_set.empty() /** implicitly: not node_is_municipality and not node_is_county and not node_is_traffic_sign */)
insertNames(id, OSMElement::Node, realworld_type, name_set);
}
}
if (pg.has_dense()) {
found_items = true;
uint64_t last_id = 0;
int last_keyvals_pos = 0;
double last_lat = 0.0, last_lon = 0.0;
const int idmax = pg.dense().id_size();
for (int j = 0; j < idmax; ++j) {
OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
/// Track various names like 'name', 'name:en', or 'name:bridge:dk'
std::map<std::string, std::string> name_set;
last_id += pg.dense().id(j);
last_lat += coord_scale * (primblock.lat_offset() + (primblock.granularity() * pg.dense().lat(j)));
last_lon += coord_scale * (primblock.lon_offset() + (primblock.granularity() * pg.dense().lon(j)));
node2Coord->insert(last_id, Coord::fromLonLat(last_lon, last_lat));
bool isKey = true;
int key = 0, value = 0;
bool node_is_county = false, node_is_municipality = false, node_is_traffic_sign = false;
while (last_keyvals_pos < pg.dense().keys_vals_size()) {
const int key_val = pg.dense().keys_vals(last_keyvals_pos);
++last_keyvals_pos;
if (key_val == 0) break;
if (isKey) {
key = key_val;
isKey = false;
} else { /// must be value
value = key_val;
isKey = true;
const char *ckey = primblock.stringtable().s(key).c_str();
if (strcmp("name", ckey) == 0) {
/// Store 'name' string for later use
name_set.insert(make_pair(primblock.stringtable().s(key), primblock.stringtable().s(value)));
++count_named_nodes;
} else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
/// Store name string for later use
name_set.insert(make_pair(primblock.stringtable().s(key), primblock.stringtable().s(value)));
} else if (strcmp("place", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(value).c_str();
if (strcmp("county", cvalue) == 0) {
/// FIX OSM DATA
/// Counties should not be represented by nodes, but by relations
/// representing an area
// realworld_type = OSMElement::PlaceLargeArea;
node_is_county = true;
} else if (strcmp("municipality", cvalue) == 0) {
/// FIX OSM DATA
/// Counties should not be represented by nodes, but by relations
/// representing an area
// realworld_type = OSMElement::PlaceLargeArea;
node_is_municipality = true;
} else if (strcmp("traffic_sign", cvalue) == 0) {
/// Traffic signs may simply point to a location, but not be *at* this location.
/// Thus, their names (if set) may be misleading and so they should be ignored.
node_is_traffic_sign = true;
}
if (strcmp("city", cvalue) == 0 || strcmp("municipality", cvalue) == 0)
realworld_type = OSMElement::PlaceLarge;
else if (strcmp("borough", cvalue) == 0 || strcmp("suburb", cvalue) == 0 || strcmp("town", cvalue) == 0 || strcmp("village", cvalue) == 0)
realworld_type = OSMElement::PlaceMedium;
else if (strcmp("quarter", cvalue) == 0 || strcmp("neighbourhood", cvalue) == 0 || strcmp("hamlet", cvalue) == 0 || strcmp("isolated_dwelling", cvalue) == 0)
/// Disabling 'city_block' as those may have misleading names like node 3188612201 ('Skaraborg') in Södermalm, Stockholm
realworld_type = OSMElement::PlaceSmall;
else if (strcmp("island", cvalue) == 0)
realworld_type = OSMElement::Island;
else {
/// Skipping other types of places:
/// * Administrative boundaries should be checked elsewhere like SCBareas or NUTS3areas
/// * Very small places like farms or plots neither
}
} else if (strcmp("natural", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(value).c_str();
if (strcmp("water", cvalue) == 0)
realworld_type = OSMElement::Water;
}
}
}
if (node_is_municipality)
Error::info("Municipality '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), last_id);
else if (node_is_county)
Error::info("County '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), last_id);
else if (node_is_traffic_sign)
Error::info("Node %llu with name '%s' is a traffic sign, not recoding node's name", last_id, name_set["name"].c_str());
else if (!name_set.empty() /** implicitly: not node_is_municipality and not node_is_county and not node_is_traffic_sign */)
insertNames(last_id, OSMElement::Node, realworld_type, name_set);
}
}
if (pg.ways_size() > 0) {
found_items = true;
char buffer_ref[SHORT_STRING_BUFFER_SIZE], buffer_highway[SHORT_STRING_BUFFER_SIZE];
const int maxways = pg.ways_size();
for (int w = 0; w < maxways; ++w) {
const uint64_t wayId = pg.ways(w).id();
const int way_size = pg.ways(w).refs_size();
if (way_size < 2) {
/// Rare but exists in map: a node with only one node (or no node?)
/// -> ignore those artefacts
Error::warn("Way %llu has only %d node(s)", wayId, way_size);
continue;
}
OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
/// Track various names like 'name', 'name:en', or 'name:bridge:dk'
std::map<std::string, std::string> name_set;
buffer_ref[0] = buffer_highway[0] = '\0'; /// clear buffers
for (int k = 0; k < pg.ways(w).keys_size(); ++k) {
const char *ckey = primblock.stringtable().s(pg.ways(w).keys(k)).c_str();
if (strcmp("name", ckey) == 0) {
/// Store 'name' string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.ways(w).keys(k)), primblock.stringtable().s(pg.ways(w).vals(k))));
++count_named_nodes;
} else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
/// Store name string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.ways(w).keys(k)), primblock.stringtable().s(pg.ways(w).vals(k))));
} else if (strcmp("highway", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();
/// Store 'highway' string for later use
strncpy(buffer_highway, cvalue, SHORT_STRING_BUFFER_SIZE);
if (strcmp("motorway", cvalue) == 0 || strcmp("trunk", cvalue) == 0 || strcmp("primary", cvalue) == 0)
realworld_type = OSMElement::RoadMajor;
else if (strcmp("secondary", cvalue) == 0 || strcmp("tertiary", cvalue) == 0)
realworld_type = OSMElement::RoadMedium;
else if (strcmp("unclassified", cvalue) == 0 || strcmp("residential", cvalue) == 0 || strcmp("service", cvalue) == 0)
realworld_type = OSMElement::RoadMinor;
else {
/// Skipping other types of roads:
/// * Cycle or pedestrian ways
/// * Hiking and 'offroad'
/// * Special cases like turning circles
}
} else if (strcmp("ref", ckey) == 0)
/// Store 'ref' string for later use
strncpy(buffer_ref, primblock.stringtable().s(pg.ways(w).vals(k)).c_str(), SHORT_STRING_BUFFER_SIZE);
else if (strcmp("building", ckey) == 0)
/// Remember if way is a building
realworld_type = OSMElement::Building;
else if (strcmp("place", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();
if (strcmp("island", cvalue) == 0)
realworld_type = OSMElement::Island;
} else if (strcmp("natural", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();
if (strcmp("water", cvalue) == 0)
realworld_type = OSMElement::Water;
}
}
/// If 'ref' string is not empty and 'highway' string is 'primary', 'secondary', or 'tertiary' ...
if (buffer_ref[0] != '\0' && buffer_highway[0] != '\0' && (strcmp(buffer_highway, "primary") == 0 || strcmp(buffer_highway, "secondary") == 0 || strcmp(buffer_highway, "tertiary") == 0 || strcmp(buffer_highway, "trunk") == 0 || strcmp(buffer_highway, "motorway") == 0))
/// ... assume that this way is part of a national or primary regional road
sweden->insertWayAsRoad(wayId, buffer_ref);
/// This main thread is the 'producer' of ways,
/// pushing ways into a queue. Another thread,
/// the consumer, will pop ways, simplify them
/// (removing superfluous nodes), and store them
/// for searches later.
queueWaySimplification.push(new OSMWay(pg.ways(w)));
/// Keep track of queue size for statistical purposes
++queueWaySimplificationSize;
if (queueWaySimplificationSize > max_queue_size) max_queue_size = queueWaySimplificationSize;
if (queueWaySimplificationSize > queueWaySimplification_recommendedSize - 16) {
/// Give consumer thread time to catch up
boost::this_thread::sleep(boost::posix_time::milliseconds(100));
}
if (way_size > 3 && buffer_ref[0] == '\0' && buffer_highway[0] != '\0' && (strcmp(buffer_highway, "primary") == 0 || strcmp(buffer_highway, "secondary") == 0 || strcmp(buffer_highway, "tertiary") == 0 || strcmp(buffer_highway, "trunk") == 0 || strcmp(buffer_highway, "motorway") == 0))
roadsWithoutRef.push_back(std::make_pair(wayId, std::string(buffer_highway)));
if (!name_set.empty())
insertNames(wayId, OSMElement::Way, realworld_type, name_set);
}
}
if (pg.relations_size() > 0) {
found_items = true;
const int maxrelations = pg.relations_size();
for (int i = 0; i < maxrelations; ++i) {
const uint64_t relId = pg.relations(i).id();
/// Some relations should be ignored, e.g. for roads outside of Sweden
/// which just happend to be included in the map data
/// To sort: echo '3, 1, 2' | sed -e 's/ //g' | tr ',' '\n' | sort -u -n | tr '\n' ',' | sed -e 's/,/, /g'
static const uint64_t blacklistedRelIds[] = {2545969, 3189514, 5518156, 5756777, 5794315, 5794316, 0};
/// To count: echo '3, 1, 2' | sed -e 's/ //g' | tr ',' '\n' | wc -l
static const size_t blacklistedRelIds_count = 6;
if (inSortedArray(blacklistedRelIds, blacklistedRelIds_count, relId)) continue;
OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
/// Track various names like 'name', 'name:en', or 'name:bridge:dk'
std::map<std::string, std::string> name_set;
std::string type, route, boundary;
int admin_level = 0;
const int maxkv = pg.relations(i).keys_size();
for (int k = 0; k < maxkv; ++k) {
const char *ckey = primblock.stringtable().s(pg.relations(i).keys(k)).c_str();
if (strcmp("name", ckey) == 0) {
/// Store 'name' string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.relations(i).keys(k)), primblock.stringtable().s(pg.relations(i).vals(k))));
++count_named_nodes;
} else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
/// Store name string for later use
name_set.insert(make_pair(primblock.stringtable().s(pg.relations(i).keys(k)), primblock.stringtable().s(pg.relations(i).vals(k))));
} else if (strcmp("type", ckey) == 0) {
/// Store 'type' string for later use
type = primblock.stringtable().s(pg.relations(i).vals(k));
} else if (strcmp("route", ckey) == 0) {
/// Store 'route' string for later use
route = primblock.stringtable().s(pg.relations(i).vals(k));
} else if (strcmp("ref:scb", ckey) == 0 || strcmp("ref:se:scb", ckey) == 0) {
/// Found SCB reference (two digits for lands, four digits for municipalities
const char *s = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
errno = 0;
const long int v = strtol(s, NULL, 10);
if (errno == 0)
sweden->insertSCBarea(v, relId);
else
Error::warn("Cannot convert '%s' to a number", s);
} else if (strcmp("ref:nuts:3", ckey) == 0) {
/// Found three-digit NUTS reference (SEnnn)
const char *s = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
if (s[0] == 'S' && s[1] == 'E' && s[2] >= '0' && s[2] <= '9') {
errno = 0;
const long int v = strtol(s + 2 /** adding 2 to skip 'SE' prefix */, NULL, 10);
if (errno == 0 && v > 0)
sweden->insertNUTS3area(v, relId);
else
Error::warn("Cannot convert '%s' to a number", s + 2);
}
} else if (strcmp("boundary", ckey) == 0) {
/// Store 'boundary' string for later use
boundary = primblock.stringtable().s(pg.relations(i).vals(k));
} else if (strcmp("admin_level", ckey) == 0) {
/// Store 'admin_level' string for later use
std::stringstream ss(primblock.stringtable().s(pg.relations(i).vals(k)));
ss >> admin_level;
} else if (strcmp("building", ckey) == 0)
/// Remember if way is a building
realworld_type = OSMElement::Building;
else if (strcmp("place", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
if (strcmp("island", cvalue) == 0)
realworld_type = OSMElement::Island;
} else if (strcmp("natural", ckey) == 0) {
const char *cvalue = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
if (strcmp("water", cvalue) == 0)
realworld_type = OSMElement::Water;
}
// TODO cover different types of relations to set 'realworld_type' properly
}
if (realworld_type == OSMElement::UnknownRealWorldType && type.compare("route") == 0 && route.compare("road") == 0)
realworld_type = OSMElement::RoadMajor;
else if (realworld_type == OSMElement::UnknownRealWorldType && boundary.compare("administrative") == 0)
realworld_type = OSMElement::PlaceLargeArea;
const auto name(name_set["name"]);
if (admin_level > 0 && name.length() > 1 && (boundary.compare("administrative") == 0 || boundary.compare("historic") == 0))
sweden->insertAdministrativeRegion(name, admin_level, relId);
RelationMem rm(pg.relations(i).memids_size());
uint64_t memId = 0;
for (int k = 0; k < pg.relations(i).memids_size(); ++k) {
memId += pg.relations(i).memids(k);
uint16_t flags = 0;
if (strcmp("outer", primblock.stringtable().s(pg.relations(i).roles_sid(k)).c_str()) == 0)
flags |= RelationFlags::RoleOuter;
else if (strcmp("inner", primblock.stringtable().s(pg.relations(i).roles_sid(k)).c_str()) == 0)
flags |= RelationFlags::RoleInner;
OSMElement::ElementType type = OSMElement::UnknownElementType;
if (pg.relations(i).types(k) == 0)
type = OSMElement::Node;
else if (pg.relations(i).types(k) == 1)
type = OSMElement::Way;
else if (pg.relations(i).types(k) == 2)
type = OSMElement::Relation;
else
Error::warn("Unknown relation type for member %llu in relation %llu : type=%d", memId, relId, pg.relations(i).types(k));
rm.members[k] = OSMElement(memId, type, OSMElement::UnknownRealWorldType);
rm.member_flags[k] = flags;
}
relMembers->insert(relId, rm);
if (!name_set.empty())
insertNames(relId, OSMElement::Relation, realworld_type, name_set);
}
}
if (!found_items) {
Error::warn(" contains no items");
}
}
// application main method
int main(int argc, char *argv[]) {
// check if the output is a tty so we can use colors
#ifdef WIN32
usecolor = 0;
#else
usecolor = isatty(1);
#endif
static struct option long_options[] = {
{"color", no_argument, 0, 'c'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "c", long_options, 0);
if (c == -1) {
break;
}
switch (c) {
case 'c':
usecolor = true;
break;
default:
exit(1);
}
}
// check for proper command line args
if (optind != argc-1) {
err("usage: %s [--color] file.osm.pbf", argv[0]);
}
// open specified file
FILE *fp = fopen(argv[optind], "rb");
if (!fp) {
err("can't open file '%s'", argv[optind]);
}
// read while the file has not reached its end
while (!feof(fp)) {
// storage of size, used multiple times
int32_t sz;
// read the first 4 bytes of the file, this is the size of the blob-header
if (fread(&sz, sizeof(sz), 1, fp) != 1) {
break; // end of file reached
}
// convert the size from network byte-order to host byte-order
sz = ntohl(sz);
// ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
if (sz > OSMPBF::max_blob_header_size) {
err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
}
// read the blob-header from the file
if (fread(buffer, sz, 1, fp) != 1) {
err("unable to read blob-header from file");
}
// parse the blob-header from the read-buffer
if (!blobheader.ParseFromArray(buffer, sz)) {
err("unable to parse blob header");
}
// tell about the blob-header
info("\nBlobHeader (%d bytes)", sz);
debug(" type = %s", blobheader.type().c_str());
// size of the following blob
sz = blobheader.datasize();
debug(" datasize = %u", sz);
// optional indexdata
if (blobheader.has_indexdata()) {
debug(" indexdata = %u bytes", blobheader.indexdata().size());
}
// ensure the blob is smaller then MAX_BLOB_SIZE
if (sz > OSMPBF::max_uncompressed_blob_size) {
err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
}
// read the blob from the file
if (fread(buffer, sz, 1, fp) != 1) {
err("unable to read blob from file");
}
// parse the blob from the read-buffer
if (!blob.ParseFromArray(buffer, sz)) {
err("unable to parse blob");
}
// tell about the blob-header
info("Blob (%d bytes)", sz);
// set when we find at least one data stream
bool found_data = false;
// if the blob has uncompressed data
if (blob.has_raw()) {
// we have at least one datastream
found_data = true;
// size of the blob-data
sz = blob.raw().size();
// check that raw_size is set correctly
if (sz != blob.raw_size()) {
warn(" reports wrong raw_size: %u bytes", blob.raw_size());
}
// tell about the blob-data
debug(" contains uncompressed data: %u bytes", sz);
// copy the uncompressed data over to the unpack_buffer
memcpy(unpack_buffer, buffer, sz);
}
// if the blob has zlib-compressed data
if (blob.has_zlib_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// the size of the compressesd data
sz = blob.zlib_data().size();
// tell about the compressed data
debug(" contains zlib-compressed data: %u bytes", sz);
debug(" uncompressed size: %u bytes", blob.raw_size());
// zlib information
z_stream z;
// next byte to decompress
z.next_in = (unsigned char*) blob.zlib_data().c_str();
// number of bytes to decompress
z.avail_in = sz;
// place of next decompressed byte
z.next_out = (unsigned char*) unpack_buffer;
// space for decompressed data
z.avail_out = blob.raw_size();
// misc
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
if (inflateInit(&z) != Z_OK) {
err(" failed to init zlib stream");
}
if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
err(" failed to inflate zlib stream");
}
if (inflateEnd(&z) != Z_OK) {
err(" failed to deinit zlib stream");
}
// unpacked size
sz = z.total_out;
}
// if the blob has lzma-compressed data
if (blob.has_lzma_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// tell about the compressed data
debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size());
debug(" uncompressed size: %u bytes", blob.raw_size());
// issue a warning, lzma compression is not yet supported
err(" lzma-decompression is not supported");
}
// check we have at least one data-stream
if (!found_data) {
err(" does not contain any known data stream");
}
// switch between different blob-types
if (blobheader.type() == "OSMHeader") {
// tell about the OSMHeader blob
info(" OSMHeader");
// parse the HeaderBlock from the blob
if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
err("unable to parse header block");
}
// tell about the bbox
if (headerblock.has_bbox()) {
OSMPBF::HeaderBBox bbox = headerblock.bbox();
debug(" bbox: %.7f,%.7f,%.7f,%.7f",
(double)bbox.left() / OSMPBF::lonlat_resolution,
(double)bbox.bottom() / OSMPBF::lonlat_resolution,
(double)bbox.right() / OSMPBF::lonlat_resolution,
(double)bbox.top() / OSMPBF::lonlat_resolution);
}
// tell about the required features
for (int i = 0, l = headerblock.required_features_size(); i < l; i++) {
debug(" required_feature: %s", headerblock.required_features(i).c_str());
}
// tell about the optional features
for (int i = 0, l = headerblock.optional_features_size(); i < l; i++) {
debug(" optional_feature: %s", headerblock.optional_features(i).c_str());
}
// tell about the writing program
if (headerblock.has_writingprogram()) {
debug(" writingprogram: %s", headerblock.writingprogram().c_str());
}
// tell about the source
if (headerblock.has_source()) {
debug(" source: %s", headerblock.source().c_str());
}
} else if (blobheader.type() == "OSMData") {
// tell about the OSMData blob
info(" OSMData");
// parse the PrimitiveBlock from the blob
if (!primblock.ParseFromArray(unpack_buffer, sz)) {
err("unable to parse primitive block");
}
// tell about the block's meta info
debug(" granularity: %u", primblock.granularity());
debug(" lat_offset: %u", primblock.lat_offset());
debug(" lon_offset: %u", primblock.lon_offset());
debug(" date_granularity: %u", primblock.date_granularity());
// tell about the stringtable
debug(" stringtable: %u items", primblock.stringtable().s_size());
// number of PrimitiveGroups
debug(" primitivegroups: %u groups", primblock.primitivegroup_size());
// iterate over all PrimitiveGroups
for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
// one PrimitiveGroup from the the Block
OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);
bool found_items=false;
// tell about nodes
if (pg.nodes_size() > 0) {
found_items = true;
debug(" nodes: %d", pg.nodes_size());
if (pg.nodes(0).has_info()) {
debug(" with meta-info");
}
}
// tell about dense nodes
if (pg.has_dense()) {
found_items = true;
debug(" dense nodes: %d", pg.dense().id_size());
if (pg.dense().has_denseinfo()) {
debug(" with meta-info");
}
}
// tell about ways
if (pg.ways_size() > 0) {
found_items = true;
debug(" ways: %d", pg.ways_size());
if (pg.ways(0).has_info()) {
debug(" with meta-info");
}
}
// tell about relations
if (pg.relations_size() > 0) {
found_items = true;
debug(" relations: %d", pg.relations_size());
if (pg.relations(0).has_info()) {
debug(" with meta-info");
}
}
if (!found_items) {
warn(" contains no items");
}
}
}
else {
// unknown blob type
warn(" unknown blob type: %s", blobheader.type().c_str());
}
}
// close the file pointer
fclose(fp);
// clean up the protobuf lib
google::protobuf::ShutdownProtobufLibrary();
}