Esempio n. 1
0
void
QcOsmPbfReader::read_dense_nodes(OSMPBF::PrimitiveGroup primitive_group)
{
  enter_node_transactions();

  OSMPBF::DenseNodes dense_node = primitive_group.dense();

  DeltaCodedInt64 node_id;
  DeltaCodedInt64 longitude;
  DeltaCodedInt64 latitude;
  int number_of_nodes = dense_node.id_size();
  for (int i = 0; i < number_of_nodes; i++) {
    node_id.update(dense_node.id(i));
    longitude.update(dense_node.lon(i));
    latitude.update(dense_node.lat(i));
    // qDebug() << "        dense node" << node_id() << to_wgs(longitude(), latitude());
    yield_node(node_id(), longitude(), latitude());
  }

  // The storage pattern is: ((<keyid> <valid>)* '0' )*
  bool is_key = true;
  int key_id = 0;
  for (int i = 0, l = dense_node.keys_vals_size(), node_index = 0;
       i < l; i++) {
    int32_t key_val_id = dense_node.keys_vals(i);
    if (key_val_id) {
      if (is_key)
        key_id = key_val_id;
      else {
        // qDebug() << "        attr" << node_index << m_string_table[key_id] << "=" << m_string_table[key_val_id];
        yield_node_attribute(node_index, key_id, key_val_id);
      }
      is_key = not(is_key);
    }
    else
      node_index++;
  }

  if (m_read_metadatas and dense_node.has_denseinfo()) {
    // qDebug().nospace() << "        with meta-info";
    OSMPBF::DenseInfo dense_info = dense_node.denseinfo();
    DeltaCodedInt64 timestamp;
    DeltaCodedInt64 changeset;
    DeltaCodedInt64 uid;
    DeltaCodedInt64 user_sid;
    for (int i = 0; i < number_of_nodes; i++) {
      int32_t version = dense_info.version(i);
      timestamp.update(to_timestamp(dense_info.timestamp(i)));
      changeset.update(dense_info.changeset(i));
      uid.update(dense_info.uid(i));
      user_sid.update(dense_info.user_sid(i));
      // bool visible = dense_info.visible(i);
      // qDebug() << "Meta information:" << version << timestamp() << changeset() << uid() << user_sid();
      // yield_node_metadata(i, version, timestamp(), changeset(), uid(), user_sid());
    }
  }

  leave_node_transactions();
}
bool OsmPbfReader::parse(std::istream &input) {
    std::vector<std::pair<uint64_t, std::string> > roadsWithoutRef;
    swedishTextTree = nullptr;
    node2Coord = nullptr;
    nodeNames = nullptr;
    wayNames = nullptr;
    relationNames = nullptr;
    wayNodes = nullptr;
    relMembers = nullptr;
    sweden = nullptr;
    size_t count_named_nodes = 0, count_named_ways = 0, count_named_relations = 0;

    if (!input || !input.good())
        return false;

    swedishTextTree = new SwedishTextTree();
    if (swedishTextTree == nullptr)
        Error::err("Could not allocate memory for swedishTextTree");
    node2Coord = new IdTree<Coord>();
    if (node2Coord == nullptr)
        Error::err("Could not allocate memory for node2Coord");
    nodeNames = new IdTree<WriteableString>();
    if (nodeNames == nullptr)
        Error::err("Could not allocate memory for nodeNames");
    wayNames = new IdTree<WriteableString>();
    if (wayNames == nullptr)
        Error::err("Could not allocate memory for wayNames");
    relationNames = new IdTree<WriteableString>();
    if (relationNames == nullptr)
        Error::err("Could not allocate memory for relationNames");
    wayNodes = new IdTree<WayNodes>();
    if (wayNodes == nullptr)
        Error::err("Could not allocate memory for wayNodes");
    relMembers = new IdTree<RelationMem>();
    if (relMembers == nullptr)
        Error::err("Could not allocate memory for relmem");
    sweden = new Sweden();
    if (sweden == nullptr)
        Error::err("Could not allocate memory for Sweden");

    doneWaySimplification = false;
    boost::thread waySimplificationThread(consumerWaySimplification);
    size_t max_queue_size = 0;

#ifdef CPUTIMER
    Timer primitiveGroupTimer;
    int64_t accumulatedPrimitiveGroupTime = 0;
#endif // CPUTIMER

    /// Read while the file has not reached its end
    while (input.good()) {
        /// Storage of size, used multiple times
        int32_t sz;

        /// Read the first 4 bytes of the file, this is the size of the blob-header
        input.read((char *)(&sz), sizeof(sz));
        if (!input.good()) {
            break; /// End of file reached
        }

        /// Convert the size from network byte-order to host byte-order
        sz = ntohl(sz);

        /// Ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
        if (sz > OSMPBF::max_blob_header_size) {
            Error::err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
        }

        // read the blob-header from the file
        input.read(buffer, sz);
        if (input.gcount() != sz || input.fail()) {
            Error::err("unable to read blob-header from file");
        }

        // parse the blob-header from the read-buffer
        if (!blobheader.ParseFromArray(buffer, sz)) {
            Error::err("unable to parse blob header");
        }

        // size of the following blob
        sz = blobheader.datasize();
        //debug("  datasize = %u", sz);

        // ensure the blob is smaller then MAX_BLOB_SIZE
        if (sz > OSMPBF::max_uncompressed_blob_size) {
            Error::err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
        }

        // read the blob from the file
        input.read(buffer, sz);
        if (input.gcount() != sz || input.fail()) {
            Error::err("unable to read blob from file");
        } else if (isatty(1))
            std::cout << (sz > (1 << 18) ? "*" : (sz > (1 << 16) ? ":" : ".")) << std::flush;

        // parse the blob from the read-buffer
        if (!blob.ParseFromArray(buffer, sz)) {
            Error::err("unable to parse blob");
        }

        // set when we find at least one data stream
        bool found_data = false;

        // if the blob has uncompressed data
        if (blob.has_raw()) {
            // we have at least one datastream
            found_data = true;

            // size of the blob-data
            sz = blob.raw().size();

            // check that raw_size is set correctly
            if (sz != blob.raw_size()) {
                Error::warn("  reports wrong raw_size: %u bytes", blob.raw_size());
            }

            // tell about the blob-data
            //debug("  contains uncompressed data: %u bytes", sz);

            // copy the uncompressed data over to the unpack_buffer
            memcpy(unpack_buffer, buffer, sz);
        }

        // if the blob has zlib-compressed data
        if (blob.has_zlib_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                Error::warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // the size of the compressesd data
            sz = blob.zlib_data().size();

            // zlib information
            z_stream z;

            // next byte to decompress
            z.next_in   = (unsigned char *) blob.zlib_data().c_str();

            // number of bytes to decompress
            z.avail_in  = sz;

            // place of next decompressed byte
            z.next_out  = (unsigned char *) unpack_buffer;

            // space for decompressed data
            z.avail_out = blob.raw_size();

            // misc
            z.zalloc    = Z_NULL;
            z.zfree     = Z_NULL;
            z.opaque    = Z_NULL;

            if (inflateInit(&z) != Z_OK) {
                Error::err("  failed to init zlib stream");
            }
            if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
                Error::err("  failed to inflate zlib stream");
            }
            if (inflateEnd(&z) != Z_OK) {
                Error::err("  failed to deinit zlib stream");
            }

            // unpacked size
            sz = z.total_out;
        }

        // if the blob has lzma-compressed data
        if (blob.has_lzma_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                Error::warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // issue a warning, lzma compression is not yet supported
            Error::err("  lzma-decompression is not supported");
        }

        // check we have at least one data-stream
        if (!found_data) {
            Error::err("  does not contain any known data stream");
        }

        // switch between different blob-types
        if (blobheader.type() == "OSMHeader") {
            // parse the HeaderBlock from the blob
            if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
                Error::err("unable to parse header block");
            }
        } else if (blobheader.type() == "OSMData") {
            // parse the PrimitiveBlock from the blob
            if (!primblock.ParseFromArray(unpack_buffer, sz)) {
                Error::err("unable to parse primitive block");
            }

            // iterate over all PrimitiveGroups
            for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
                // one PrimitiveGroup from the the Block
                OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);

                bool found_items = false;
                const double coord_scale = 0.000000001;

#ifdef CPUTIMER
                primitiveGroupTimer.start();
#endif // CPUTIMER
                if (pg.nodes_size() > 0) {
                    found_items = true;

                    const int maxnodes = pg.nodes_size();
                    for (int j = 0; j < maxnodes; ++j) {
                        const uint64_t id = pg.nodes(j).id();
                        OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
                        /// Track various names like 'name', 'name:en', or 'name:bridge:dk'
                        std::map<std::string, std::string> name_set;

                        const double lat = coord_scale * (primblock.lat_offset() + (primblock.granularity() * pg.nodes(j).lat()));
                        const double lon = coord_scale * (primblock.lon_offset() + (primblock.granularity() * pg.nodes(j).lon()));
                        node2Coord->insert(id, Coord::fromLonLat(lon, lat));

                        bool node_is_county = false, node_is_municipality = false, node_is_traffic_sign = false;
                        for (int k = 0; k < pg.nodes(j).keys_size(); ++k) {
                            const char *ckey = primblock.stringtable().s(pg.nodes(j).keys(k)).c_str();
                            if (strcmp("name", ckey) == 0) {
                                /// Store 'name' string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.nodes(j).keys(k)), primblock.stringtable().s(pg.nodes(j).vals(k))));
                                ++count_named_nodes;
                            } else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
                                /// Store name string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.nodes(j).keys(k)), primblock.stringtable().s(pg.nodes(j).vals(k))));
                            } else if (strcmp("place", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.nodes(j).vals(k)).c_str();

                                if (strcmp("county", cvalue) == 0) {
                                    /// FIX OSM DATA
                                    /// Counties should not be represented by nodes, but by relations
                                    /// representing an area
                                    // realworld_type = OSMElement::PlaceLargeArea;
                                    node_is_county = true;
                                } else if (strcmp("municipality", cvalue) == 0) {
                                    /// FIX OSM DATA
                                    /// Counties should not be represented by nodes, but by relations
                                    /// representing an area
                                    // realworld_type = OSMElement::PlaceLargeArea;
                                    node_is_municipality = true;
                                } else if (strcmp("traffic_sign", cvalue) == 0) {
                                    /// Traffic signs may simply point to a location, but not be *at* this location.
                                    /// Thus, their names (if set) may be misleading and so they should be ignored.
                                    node_is_traffic_sign = true;
                                }

                                if (strcmp("city", cvalue) == 0 || strcmp("municipality", cvalue) == 0)
                                    realworld_type = OSMElement::PlaceLarge;
                                else if (strcmp("borough", cvalue) == 0 || strcmp("suburb", cvalue) == 0 || strcmp("town", cvalue) == 0 || strcmp("village", cvalue) == 0)
                                    realworld_type = OSMElement::PlaceMedium;
                                else if (strcmp("quarter", cvalue) == 0 || strcmp("neighbourhood", cvalue) == 0 || strcmp("hamlet", cvalue) == 0 || strcmp("isolated_dwelling", cvalue) == 0)
                                    /// Disabling 'city_block' as those may have misleading names like node 3188612201 ('Skaraborg') in Södermalm, Stockholm
                                    realworld_type = OSMElement::PlaceSmall;
                                else if (strcmp("island", cvalue) == 0)
                                    realworld_type = OSMElement::Island;
                                else {
                                    /// Skipping other types of places:
                                    /// * Administrative boundaries should be checked elsewhere like SCBareas or NUTS3areas
                                    /// * Very small places like farms or plots neither
                                }
                            } else if (strcmp("natural", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.nodes(j).vals(k)).c_str();
                                if (strcmp("water", cvalue) == 0)
                                    realworld_type = OSMElement::Water;
                            }
                        }

                        if (node_is_municipality)
                            Error::info("Municipality '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), id);
                        else if (node_is_county)
                            Error::info("County '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), id);
                        else if (node_is_traffic_sign)
                            Error::info("Node %llu with name '%s' is a traffic sign, not recoding node's name", id, name_set["name"].c_str());
                        else if (!name_set.empty() /** implicitly: not node_is_municipality and not node_is_county and not node_is_traffic_sign */)
                            insertNames(id, OSMElement::Node, realworld_type, name_set);
                    }
                }

                if (pg.has_dense()) {
                    found_items = true;

                    uint64_t last_id = 0;
                    int last_keyvals_pos = 0;
                    double last_lat = 0.0, last_lon = 0.0;
                    const int idmax = pg.dense().id_size();
                    for (int j = 0; j < idmax; ++j) {
                        OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
                        /// Track various names like 'name', 'name:en', or 'name:bridge:dk'
                        std::map<std::string, std::string> name_set;

                        last_id += pg.dense().id(j);
                        last_lat += coord_scale * (primblock.lat_offset() + (primblock.granularity() * pg.dense().lat(j)));
                        last_lon += coord_scale * (primblock.lon_offset() + (primblock.granularity() * pg.dense().lon(j)));
                        node2Coord->insert(last_id, Coord::fromLonLat(last_lon, last_lat));

                        bool isKey = true;
                        int key = 0, value = 0;
                        bool node_is_county = false, node_is_municipality = false, node_is_traffic_sign = false;
                        while (last_keyvals_pos < pg.dense().keys_vals_size()) {
                            const int key_val = pg.dense().keys_vals(last_keyvals_pos);
                            ++last_keyvals_pos;
                            if (key_val == 0) break;
                            if (isKey) {
                                key = key_val;
                                isKey = false;
                            } else { /// must be value
                                value = key_val;
                                isKey = true;

                                const char *ckey = primblock.stringtable().s(key).c_str();
                                if (strcmp("name", ckey) == 0) {
                                    /// Store 'name' string for later use
                                    name_set.insert(make_pair(primblock.stringtable().s(key), primblock.stringtable().s(value)));
                                    ++count_named_nodes;
                                } else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
                                    /// Store name string for later use
                                    name_set.insert(make_pair(primblock.stringtable().s(key), primblock.stringtable().s(value)));
                                } else if (strcmp("place", ckey) == 0) {
                                    const char *cvalue = primblock.stringtable().s(value).c_str();

                                    if (strcmp("county", cvalue) == 0) {
                                        /// FIX OSM DATA
                                        /// Counties should not be represented by nodes, but by relations
                                        /// representing an area
                                        // realworld_type = OSMElement::PlaceLargeArea;
                                        node_is_county = true;
                                    } else if (strcmp("municipality", cvalue) == 0) {
                                        /// FIX OSM DATA
                                        /// Counties should not be represented by nodes, but by relations
                                        /// representing an area
                                        // realworld_type = OSMElement::PlaceLargeArea;
                                        node_is_municipality = true;
                                    } else if (strcmp("traffic_sign", cvalue) == 0) {
                                        /// Traffic signs may simply point to a location, but not be *at* this location.
                                        /// Thus, their names (if set) may be misleading and so they should be ignored.
                                        node_is_traffic_sign = true;
                                    }

                                    if (strcmp("city", cvalue) == 0 || strcmp("municipality", cvalue) == 0)
                                        realworld_type = OSMElement::PlaceLarge;
                                    else if (strcmp("borough", cvalue) == 0 || strcmp("suburb", cvalue) == 0 || strcmp("town", cvalue) == 0 || strcmp("village", cvalue) == 0)
                                        realworld_type = OSMElement::PlaceMedium;
                                    else if (strcmp("quarter", cvalue) == 0 || strcmp("neighbourhood", cvalue) == 0 || strcmp("hamlet", cvalue) == 0 || strcmp("isolated_dwelling", cvalue) == 0)
                                        /// Disabling 'city_block' as those may have misleading names like node 3188612201 ('Skaraborg') in Södermalm, Stockholm
                                        realworld_type = OSMElement::PlaceSmall;
                                    else if (strcmp("island", cvalue) == 0)
                                        realworld_type = OSMElement::Island;
                                    else {
                                        /// Skipping other types of places:
                                        /// * Administrative boundaries should be checked elsewhere like SCBareas or NUTS3areas
                                        /// * Very small places like farms or plots neither
                                    }
                                } else if (strcmp("natural", ckey) == 0) {
                                    const char *cvalue = primblock.stringtable().s(value).c_str();
                                    if (strcmp("water", cvalue) == 0)
                                        realworld_type = OSMElement::Water;
                                }
                            }
                        }

                        if (node_is_municipality)
                            Error::info("Municipality '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), last_id);
                        else if (node_is_county)
                            Error::info("County '%s' is represented by node %llu, not recoding node's name", name_set["name"].c_str(), last_id);
                        else if (node_is_traffic_sign)
                            Error::info("Node %llu with name '%s' is a traffic sign, not recoding node's name", last_id, name_set["name"].c_str());
                        else if (!name_set.empty() /** implicitly: not node_is_municipality and not node_is_county and not node_is_traffic_sign */)
                            insertNames(last_id, OSMElement::Node, realworld_type, name_set);
                    }
                }

                if (pg.ways_size() > 0) {
                    found_items = true;

                    char buffer_ref[SHORT_STRING_BUFFER_SIZE], buffer_highway[SHORT_STRING_BUFFER_SIZE];
                    const int maxways = pg.ways_size();
                    for (int w = 0; w < maxways; ++w) {
                        const uint64_t wayId = pg.ways(w).id();
                        const int way_size = pg.ways(w).refs_size();

                        if (way_size < 2) {
                            /// Rare but exists in map: a node with only one node (or no node?)
                            /// -> ignore those artefacts
                            Error::warn("Way %llu has only %d node(s)", wayId, way_size);
                            continue;
                        }

                        OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
                        /// Track various names like 'name', 'name:en', or 'name:bridge:dk'
                        std::map<std::string, std::string> name_set;

                        buffer_ref[0] = buffer_highway[0] = '\0'; /// clear buffers
                        for (int k = 0; k < pg.ways(w).keys_size(); ++k) {
                            const char *ckey = primblock.stringtable().s(pg.ways(w).keys(k)).c_str();
                            if (strcmp("name", ckey) == 0) {
                                /// Store 'name' string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.ways(w).keys(k)), primblock.stringtable().s(pg.ways(w).vals(k))));
                                ++count_named_nodes;
                            } else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
                                /// Store name string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.ways(w).keys(k)), primblock.stringtable().s(pg.ways(w).vals(k))));
                            } else if (strcmp("highway", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();

                                /// Store 'highway' string for later use
                                strncpy(buffer_highway, cvalue, SHORT_STRING_BUFFER_SIZE);

                                if (strcmp("motorway", cvalue) == 0 || strcmp("trunk", cvalue) == 0 || strcmp("primary", cvalue) == 0)
                                    realworld_type = OSMElement::RoadMajor;
                                else if (strcmp("secondary", cvalue) == 0 || strcmp("tertiary", cvalue) == 0)
                                    realworld_type = OSMElement::RoadMedium;
                                else if (strcmp("unclassified", cvalue) == 0 || strcmp("residential", cvalue) == 0 || strcmp("service", cvalue) == 0)
                                    realworld_type = OSMElement::RoadMinor;
                                else {
                                    /// Skipping other types of roads:
                                    /// * Cycle or pedestrian ways
                                    /// * Hiking and 'offroad'
                                    /// * Special cases like turning circles
                                }
                            } else if (strcmp("ref", ckey) == 0)
                                /// Store 'ref' string for later use
                                strncpy(buffer_ref, primblock.stringtable().s(pg.ways(w).vals(k)).c_str(), SHORT_STRING_BUFFER_SIZE);
                            else if (strcmp("building", ckey) == 0)
                                /// Remember if way is a building
                                realworld_type = OSMElement::Building;
                            else if (strcmp("place", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();
                                if (strcmp("island", cvalue) == 0)
                                    realworld_type = OSMElement::Island;
                            } else if (strcmp("natural", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.ways(w).vals(k)).c_str();
                                if (strcmp("water", cvalue) == 0)
                                    realworld_type = OSMElement::Water;
                            }
                        }

                        /// If 'ref' string is not empty and 'highway' string is 'primary', 'secondary', or 'tertiary' ...
                        if (buffer_ref[0] != '\0' && buffer_highway[0] != '\0' && (strcmp(buffer_highway, "primary") == 0 || strcmp(buffer_highway, "secondary") == 0 || strcmp(buffer_highway, "tertiary") == 0 || strcmp(buffer_highway, "trunk") == 0 || strcmp(buffer_highway, "motorway") == 0))
                            /// ... assume that this way is part of a national or primary regional road
                            sweden->insertWayAsRoad(wayId, buffer_ref);

                        /// This main thread is the 'producer' of ways,
                        /// pushing ways into a queue. Another thread,
                        /// the consumer, will pop ways, simplify them
                        /// (removing superfluous nodes), and store them
                        /// for searches later.
                        queueWaySimplification.push(new OSMWay(pg.ways(w)));
                        /// Keep track of queue size for statistical purposes
                        ++queueWaySimplificationSize;
                        if (queueWaySimplificationSize > max_queue_size) max_queue_size = queueWaySimplificationSize;
                        if (queueWaySimplificationSize > queueWaySimplification_recommendedSize - 16) {
                            /// Give consumer thread time to catch up
                            boost::this_thread::sleep(boost::posix_time::milliseconds(100));
                        }

                        if (way_size > 3 && buffer_ref[0] == '\0' && buffer_highway[0] != '\0' && (strcmp(buffer_highway, "primary") == 0 || strcmp(buffer_highway, "secondary") == 0 || strcmp(buffer_highway, "tertiary") == 0 || strcmp(buffer_highway, "trunk") == 0 || strcmp(buffer_highway, "motorway") == 0))
                            roadsWithoutRef.push_back(std::make_pair(wayId, std::string(buffer_highway)));

                        if (!name_set.empty())
                            insertNames(wayId, OSMElement::Way, realworld_type, name_set);
                    }
                }

                if (pg.relations_size() > 0) {
                    found_items = true;

                    const int maxrelations = pg.relations_size();
                    for (int i = 0; i < maxrelations; ++i) {
                        const uint64_t relId = pg.relations(i).id();

                        /// Some relations should be ignored, e.g. for roads outside of Sweden
                        /// which just happend to be included in the map data
                        /// To sort:  echo '3, 1, 2' | sed -e 's/ //g' | tr ',' '\n' | sort -u -n | tr '\n' ',' | sed -e 's/,/, /g'
                        static const uint64_t blacklistedRelIds[] = {2545969, 3189514, 5518156, 5756777, 5794315, 5794316, 0};
                        /// To count: echo '3, 1, 2' | sed -e 's/ //g' | tr ',' '\n' | wc -l
                        static const size_t blacklistedRelIds_count = 6;
                        if (inSortedArray(blacklistedRelIds, blacklistedRelIds_count, relId)) continue;

                        OSMElement::RealWorldType realworld_type = OSMElement::UnknownRealWorldType;
                        /// Track various names like 'name', 'name:en', or 'name:bridge:dk'
                        std::map<std::string, std::string> name_set;
                        std::string type, route, boundary;
                        int admin_level = 0;
                        const int maxkv = pg.relations(i).keys_size();
                        for (int k = 0; k < maxkv; ++k) {
                            const char *ckey = primblock.stringtable().s(pg.relations(i).keys(k)).c_str();
                            if (strcmp("name", ckey) == 0) {
                                /// Store 'name' string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.relations(i).keys(k)), primblock.stringtable().s(pg.relations(i).vals(k))));
                                ++count_named_nodes;
                            } else if (strncmp("name:", ckey, 5) == 0 || strcmp("alt_name", ckey) == 0 || strncmp("alt_name:", ckey, 9) == 0 || strcmp("old_name", ckey) == 0 || strncmp("old_name:", ckey, 9) == 0 || strcmp("loc_name", ckey) == 0 || strncmp("loc_name:", ckey, 9) == 0 || strcmp("short_name", ckey) == 0 || strncmp("short_name:", ckey, 11) == 0 || strcmp("official_name", ckey) == 0 || strncmp("official_name:", ckey, 14) == 0) {
                                /// Store name string for later use
                                name_set.insert(make_pair(primblock.stringtable().s(pg.relations(i).keys(k)), primblock.stringtable().s(pg.relations(i).vals(k))));
                            } else if (strcmp("type", ckey) == 0) {
                                /// Store 'type' string for later use
                                type = primblock.stringtable().s(pg.relations(i).vals(k));
                            } else if (strcmp("route", ckey) == 0) {
                                /// Store 'route' string for later use
                                route = primblock.stringtable().s(pg.relations(i).vals(k));
                            } else if (strcmp("ref:scb", ckey) == 0 || strcmp("ref:se:scb", ckey) == 0) {
                                /// Found SCB reference (two digits for lands, four digits for municipalities
                                const char *s = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
                                errno = 0;
                                const long int v = strtol(s, NULL, 10);
                                if (errno == 0)
                                    sweden->insertSCBarea(v, relId);
                                else
                                    Error::warn("Cannot convert '%s' to a number", s);
                            } else if (strcmp("ref:nuts:3", ckey) == 0) {
                                /// Found three-digit NUTS reference (SEnnn)
                                const char *s = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
                                if (s[0] == 'S' && s[1] == 'E' && s[2] >= '0' && s[2] <= '9') {
                                    errno = 0;
                                    const long int v = strtol(s + 2 /** adding 2 to skip 'SE' prefix */, NULL, 10);
                                    if (errno == 0 && v > 0)
                                        sweden->insertNUTS3area(v, relId);
                                    else
                                        Error::warn("Cannot convert '%s' to a number", s + 2);
                                }
                            } else if (strcmp("boundary", ckey) == 0) {
                                /// Store 'boundary' string for later use
                                boundary = primblock.stringtable().s(pg.relations(i).vals(k));
                            } else if (strcmp("admin_level", ckey) == 0) {
                                /// Store 'admin_level' string for later use
                                std::stringstream ss(primblock.stringtable().s(pg.relations(i).vals(k)));
                                ss >> admin_level;
                            } else if (strcmp("building", ckey) == 0)
                                /// Remember if way is a building
                                realworld_type = OSMElement::Building;
                            else if (strcmp("place", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
                                if (strcmp("island", cvalue) == 0)
                                    realworld_type = OSMElement::Island;
                            } else if (strcmp("natural", ckey) == 0) {
                                const char *cvalue = primblock.stringtable().s(pg.relations(i).vals(k)).c_str();
                                if (strcmp("water", cvalue) == 0)
                                    realworld_type = OSMElement::Water;
                            }
                            // TODO cover different types of relations to set 'realworld_type' properly
                        }

                        if (realworld_type == OSMElement::UnknownRealWorldType && type.compare("route") == 0 && route.compare("road") == 0)
                            realworld_type = OSMElement::RoadMajor;
                        else if (realworld_type == OSMElement::UnknownRealWorldType && boundary.compare("administrative") == 0)
                            realworld_type = OSMElement::PlaceLargeArea;

                        const auto name(name_set["name"]);
                        if (admin_level > 0 && name.length() > 1 && (boundary.compare("administrative") == 0 || boundary.compare("historic") == 0))
                            sweden->insertAdministrativeRegion(name, admin_level, relId);

                        RelationMem rm(pg.relations(i).memids_size());
                        uint64_t memId = 0;
                        for (int k = 0; k < pg.relations(i).memids_size(); ++k) {
                            memId += pg.relations(i).memids(k);
                            uint16_t flags = 0;
                            if (strcmp("outer", primblock.stringtable().s(pg.relations(i).roles_sid(k)).c_str()) == 0)
                                flags |= RelationFlags::RoleOuter;
                            else if (strcmp("inner", primblock.stringtable().s(pg.relations(i).roles_sid(k)).c_str()) == 0)
                                flags |= RelationFlags::RoleInner;
                            OSMElement::ElementType type = OSMElement::UnknownElementType;
                            if (pg.relations(i).types(k) == 0)
                                type = OSMElement::Node;
                            else if (pg.relations(i).types(k) == 1)
                                type = OSMElement::Way;
                            else if (pg.relations(i).types(k) == 2)
                                type = OSMElement::Relation;
                            else
                                Error::warn("Unknown relation type for member %llu in relation %llu : type=%d", memId, relId, pg.relations(i).types(k));
                            rm.members[k] = OSMElement(memId, type, OSMElement::UnknownRealWorldType);
                            rm.member_flags[k] = flags;
                        }
                        relMembers->insert(relId, rm);

                        if (!name_set.empty())
                            insertNames(relId, OSMElement::Relation, realworld_type, name_set);
                    }
                }

                if (!found_items) {
                    Error::warn("      contains no items");
                }
            }
Esempio n. 3
0
// application main method
int main(int argc, char *argv[]) {
    // check if the output is a tty so we can use colors

#ifdef WIN32
    usecolor = 0;
#else
    usecolor = isatty(1);
#endif

    static struct option long_options[] = {
        {"color", no_argument, 0, 'c'},
        {0, 0, 0, 0}
    };

    while (1) {
        int c = getopt_long(argc, argv, "c", long_options, 0);

        if (c == -1) {
            break;
        }

        switch (c) {
            case 'c':
                usecolor = true;
                break;
            default:
                exit(1);
        }
    }

    // check for proper command line args
    if (optind != argc-1) {
        err("usage: %s [--color] file.osm.pbf", argv[0]);
    }

    // open specified file
    FILE *fp = fopen(argv[optind], "rb");

    if (!fp) {
        err("can't open file '%s'", argv[optind]);
    }

    // read while the file has not reached its end
    while (!feof(fp)) {
        // storage of size, used multiple times
        int32_t sz;

        // read the first 4 bytes of the file, this is the size of the blob-header
        if (fread(&sz, sizeof(sz), 1, fp) != 1) {
            break; // end of file reached
        }

        // convert the size from network byte-order to host byte-order
        sz = ntohl(sz);

        // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
        if (sz > OSMPBF::max_blob_header_size) {
            err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
        }

        // read the blob-header from the file
        if (fread(buffer, sz, 1, fp) != 1) {
            err("unable to read blob-header from file");
        }

        // parse the blob-header from the read-buffer
        if (!blobheader.ParseFromArray(buffer, sz)) {
            err("unable to parse blob header");
        }

        // tell about the blob-header
        info("\nBlobHeader (%d bytes)", sz);
        debug("  type = %s", blobheader.type().c_str());

        // size of the following blob
        sz = blobheader.datasize();
        debug("  datasize = %u", sz);

        // optional indexdata
        if (blobheader.has_indexdata()) {
            debug("  indexdata = %u bytes", blobheader.indexdata().size());
        }

        // ensure the blob is smaller then MAX_BLOB_SIZE
        if (sz > OSMPBF::max_uncompressed_blob_size) {
            err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
        }

        // read the blob from the file
        if (fread(buffer, sz, 1, fp) != 1) {
            err("unable to read blob from file");
        }

        // parse the blob from the read-buffer
        if (!blob.ParseFromArray(buffer, sz)) {
            err("unable to parse blob");
        }

        // tell about the blob-header
        info("Blob (%d bytes)", sz);

        // set when we find at least one data stream
        bool found_data = false;

        // if the blob has uncompressed data
        if (blob.has_raw()) {
            // we have at least one datastream
            found_data = true;

            // size of the blob-data
            sz = blob.raw().size();

            // check that raw_size is set correctly
            if (sz != blob.raw_size()) {
                warn("  reports wrong raw_size: %u bytes", blob.raw_size());
            }

            // tell about the blob-data
            debug("  contains uncompressed data: %u bytes", sz);

            // copy the uncompressed data over to the unpack_buffer
            memcpy(unpack_buffer, buffer, sz);
        }

        // if the blob has zlib-compressed data
        if (blob.has_zlib_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // the size of the compressesd data
            sz = blob.zlib_data().size();

            // tell about the compressed data
            debug("  contains zlib-compressed data: %u bytes", sz);
            debug("  uncompressed size: %u bytes", blob.raw_size());

            // zlib information
            z_stream z;

            // next byte to decompress
            z.next_in   = (unsigned char*) blob.zlib_data().c_str();

            // number of bytes to decompress
            z.avail_in  = sz;

            // place of next decompressed byte
            z.next_out  = (unsigned char*) unpack_buffer;

            // space for decompressed data
            z.avail_out = blob.raw_size();

            // misc
            z.zalloc    = Z_NULL;
            z.zfree     = Z_NULL;
            z.opaque    = Z_NULL;

            if (inflateInit(&z) != Z_OK) {
                err("  failed to init zlib stream");
            }
            if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
                err("  failed to inflate zlib stream");
            }
            if (inflateEnd(&z) != Z_OK) {
                err("  failed to deinit zlib stream");
            }

            // unpacked size
            sz = z.total_out;
        }

        // if the blob has lzma-compressed data
        if (blob.has_lzma_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // tell about the compressed data
            debug("  contains lzma-compressed data: %u bytes", blob.lzma_data().size());
            debug("  uncompressed size: %u bytes", blob.raw_size());

            // issue a warning, lzma compression is not yet supported
            err("  lzma-decompression is not supported");
        }

        // check we have at least one data-stream
        if (!found_data) {
            err("  does not contain any known data stream");
        }

        // switch between different blob-types
        if (blobheader.type() == "OSMHeader") {
            // tell about the OSMHeader blob
            info("  OSMHeader");

            // parse the HeaderBlock from the blob
            if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
                err("unable to parse header block");
            }

            // tell about the bbox
            if (headerblock.has_bbox()) {
                OSMPBF::HeaderBBox bbox = headerblock.bbox();
                debug("    bbox: %.7f,%.7f,%.7f,%.7f",
                    (double)bbox.left() / OSMPBF::lonlat_resolution,
                    (double)bbox.bottom() / OSMPBF::lonlat_resolution,
                    (double)bbox.right() / OSMPBF::lonlat_resolution,
                    (double)bbox.top() / OSMPBF::lonlat_resolution);
            }

            // tell about the required features
            for (int i = 0, l = headerblock.required_features_size(); i < l; i++) {
                debug("    required_feature: %s", headerblock.required_features(i).c_str());
            }

            // tell about the optional features
            for (int i = 0, l = headerblock.optional_features_size(); i < l; i++) {
                debug("    optional_feature: %s", headerblock.optional_features(i).c_str());
            }

            // tell about the writing program
            if (headerblock.has_writingprogram()) {
                debug("    writingprogram: %s", headerblock.writingprogram().c_str());
            }

            // tell about the source
            if (headerblock.has_source()) {
                debug("    source: %s", headerblock.source().c_str());
            }
        } else if (blobheader.type() == "OSMData") {
            // tell about the OSMData blob
            info("  OSMData");

            // parse the PrimitiveBlock from the blob
            if (!primblock.ParseFromArray(unpack_buffer, sz)) {
                err("unable to parse primitive block");
            }

            // tell about the block's meta info
            debug("    granularity: %u", primblock.granularity());
            debug("    lat_offset: %u", primblock.lat_offset());
            debug("    lon_offset: %u", primblock.lon_offset());
            debug("    date_granularity: %u", primblock.date_granularity());

            // tell about the stringtable
            debug("    stringtable: %u items", primblock.stringtable().s_size());

            // number of PrimitiveGroups
            debug("    primitivegroups: %u groups", primblock.primitivegroup_size());

            // iterate over all PrimitiveGroups
            for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
                // one PrimitiveGroup from the the Block
                OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);

                bool found_items=false;

                // tell about nodes
                if (pg.nodes_size() > 0) {
                    found_items = true;

                    debug("      nodes: %d", pg.nodes_size());
                    if (pg.nodes(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                // tell about dense nodes
                if (pg.has_dense()) {
                    found_items = true;

                    debug("      dense nodes: %d", pg.dense().id_size());
                    if (pg.dense().has_denseinfo()) {
                        debug("        with meta-info");
                    }
                }

                // tell about ways
                if (pg.ways_size() > 0) {
                    found_items = true;

                    debug("      ways: %d", pg.ways_size());
                    if (pg.ways(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                // tell about relations
                if (pg.relations_size() > 0) {
                    found_items = true;

                    debug("      relations: %d", pg.relations_size());
                    if (pg.relations(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                if (!found_items) {
                    warn("      contains no items");
                }
            }
        }

        else {
            // unknown blob type
            warn("  unknown blob type: %s", blobheader.type().c_str());
        }
    }

    // close the file pointer
    fclose(fp);

    // clean up the protobuf lib
    google::protobuf::ShutdownProtobufLibrary();
}