Exemple #1
0
            /**
             * Calculate length of way.
             */
            inline double distance(const osmium::WayNodeList& wnl) {
                double sum_length = 0;

                for (auto it = wnl.begin(); it != wnl.end(); ++it) {
                    if (std::next(it) != wnl.end()) {
                        sum_length += distance(it->location(), std::next(it)->location());
                    }
                }

                return sum_length;
            }
Exemple #2
0
            linestring_type create_linestring(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) {
                linestring_start();
                size_t num_points = 0;

                if (un == use_nodes::unique) {
                    osmium::Location last_location;
                    switch (dir) {
                        case direction::forward:
                            num_points = fill_linestring_unique(wnl.cbegin(), wnl.cend());
                            break;
                        case direction::backward:
                            num_points = fill_linestring_unique(wnl.crbegin(), wnl.crend());
                            break;
                    }
                } else {
                    switch (dir) {
                        case direction::forward:
                            num_points = fill_linestring(wnl.cbegin(), wnl.cend());
                            break;
                        case direction::backward:
                            num_points = fill_linestring(wnl.crbegin(), wnl.crend());
                            break;
                    }
                }

                if (num_points < 2) {
                    throw osmium::geometry_error("not enough points for linestring");
                }

                return linestring_finish(num_points);
            }
Exemple #3
0
            polygon_type create_polygon(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) {
                polygon_start();
                size_t num_points = 0;

                if (un == use_nodes::unique) {
                    osmium::Location last_location;
                    switch (dir) {
                        case direction::forward:
                            num_points = fill_polygon_unique(wnl.cbegin(), wnl.cend());
                            break;
                        case direction::backward:
                            num_points = fill_polygon_unique(wnl.crbegin(), wnl.crend());
                            break;
                    }
                } else {
                    switch (dir) {
                        case direction::forward:
                            num_points = fill_polygon(wnl.cbegin(), wnl.cend());
                            break;
                        case direction::backward:
                            num_points = fill_polygon(wnl.crbegin(), wnl.crend());
                            break;
                    }
                }

                if (num_points < 4) {
                    throw osmium::geometry_error{"need at least four points for polygon"};
                }

                return polygon_finish(num_points);
            }
Exemple #4
0
/* Implements the mid-layer processing for osm2pgsql
 * using several PostgreSQL tables
 *
 * This layer stores data read in from the planet.osm file
 * and is then read by the backend processing code to
 * emit the final geometry-enabled output formats
*/

#include "config.h"

#ifdef _WIN32
using namespace std;
#endif

#ifdef _MSC_VER
#define alloca _alloca
#endif

#include <stdexcept>
#include <unordered_map>

#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <future>

#include <boost/format.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/builder/osm_object_builder.hpp>

#include <libpq-fe.h>

#include "middle-pgsql.hpp"
#include "node-persistent-cache.hpp"
#include "node-ram-cache.hpp"
#include "options.hpp"
#include "osmtypes.hpp"
#include "output-pgsql.hpp"
#include "pgsql.hpp"
#include "util.hpp"

enum table_id {
    t_node, t_way, t_rel
} ;

middle_pgsql_t::table_desc::table_desc(const char *name_,
                                       const char *start_,
                                       const char *create_,
                                       const char *create_index_,
                                       const char *prepare_,
                                       const char *prepare_intarray_,
                                       const char *copy_,
                                       const char *analyze_,
                                       const char *stop_,
                                       const char *array_indexes_)
    : name(name_),
      start(start_),
      create(create_),
      create_index(create_index_),
      prepare(prepare_),
      prepare_intarray(prepare_intarray_),
      copy(copy_),
      analyze(analyze_),
      stop(stop_),
      array_indexes(array_indexes_),
      copyMode(0),
      transactionMode(0),
      sql_conn(nullptr)
{}

namespace {
// Decodes a portion of an array literal from postgres */
// Argument should point to beginning of literal, on return points to delimiter */
inline const char *decode_upto( const char *src, char *dst )
{
  int quoted = (*src == '"');
  if( quoted ) src++;

  while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
  {
    if( *src == '\\' )
    {
      switch( src[1] )
      {
        case 'n': *dst++ = '\n'; break;
        case 't': *dst++ = '\t'; break;
        default: *dst++ = src[1]; break;
      }
      src+=2;
    }
    else
      *dst++ = *src++;
  }
  if( quoted ) src++;
  *dst = 0;
  return src;
}

template <typename T>
void pgsql_parse_tags(const char *string, osmium::memory::Buffer &buffer, T &obuilder)
{
    if( *string++ != '{' )
        return;

    char key[1024];
    char val[1024];
    osmium::builder::TagListBuilder builder(buffer, &obuilder);

    while( *string != '}' ) {
        string = decode_upto(string, key);
        // String points to the comma */
        string++;
        string = decode_upto(string, val);
        builder.add_tag(key, val);
        // String points to the comma or closing '}' */
        if( *string == ',' ) {
            string++;
        }
    }
}

void pgsql_parse_members(const char *string, osmium::memory::Buffer &buffer,
                         osmium::builder::RelationBuilder &obuilder)
{
    if( *string++ != '{' )
        return;

    char role[1024];
    osmium::builder::RelationMemberListBuilder builder(buffer, &obuilder);

    while( *string != '}' ) {
        char type = string[0];
        char *endp;
        osmid_t id = strtoosmid(string + 1, &endp, 10);
        // String points to the comma */
        string = decode_upto(endp + 1, role);
        builder.add_member(osmium::char_to_item_type(type), id, role);
        // String points to the comma or closing '}' */
        if( *string == ',' ) {
            string++;
        }
    }
}

void pgsql_parse_nodes(const char *string, osmium::memory::Buffer &buffer,
                         osmium::builder::WayBuilder &builder)
{
    if (*string++ == '{') {
        osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder);
        while (*string != '}') {
            char *ptr;
            wnl_builder.add_node_ref(strtoosmid(string, &ptr, 10));
            string = ptr;
            if (*string == ',') {
                string++;
            }
        }
    }
}

int pgsql_endCopy(middle_pgsql_t::table_desc *table)
{
    // Terminate any pending COPY */
    if (table->copyMode) {
        PGconn *sql_conn = table->sql_conn;
        int stop = PQputCopyEnd(sql_conn, nullptr);
        if (stop != 1) {
            fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
            util::exit_nicely();
        }

        PGresult *res = PQgetResult(sql_conn);
        if (PQresultStatus(res) != PGRES_COMMAND_OK) {
            fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
            PQclear(res);
            util::exit_nicely();
        }
        PQclear(res);
        table->copyMode = 0;
    }
    return 0;
}
} // anonymous namespace


void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
{
    for (char const c: in) {
        switch (c) {
            case '"':
                if (escape) copy_buffer += "\\";
                copy_buffer += "\\\"";
                break;
            case '\\':
                if (escape) copy_buffer += "\\\\";
                copy_buffer += "\\\\";
                break;
            case '\n':
                if (escape) copy_buffer += "\\";
                copy_buffer += "\\n";
                break;
            case '\r':
                if (escape) copy_buffer += "\\";
                copy_buffer += "\\r";
                break;
            case '\t':
                if (escape) copy_buffer += "\\";
                copy_buffer += "\\t";
                break;
            default:
                copy_buffer += c;
                break;
        }
    }
}

// escape means we return '\N' for copy mode, otherwise we return just nullptr
void middle_pgsql_t::buffer_store_tags(osmium::OSMObject const &obj, bool attrs,
                                       bool escape)
{
    copy_buffer += "{";

    for (auto const &it : obj.tags()) {
        copy_buffer += "\"";
        buffer_store_string(it.key(), escape);
        copy_buffer += "\",\"";
        buffer_store_string(it.value(), escape);
        copy_buffer += "\",";
    }
    if (attrs) {
        taglist_t extra;
        extra.add_attributes(obj);
        for (auto const &it : extra) {
            copy_buffer += "\"";
            copy_buffer += it.key;
            copy_buffer += "\",\"";
            buffer_store_string(it.value.c_str(), escape);
            copy_buffer += "\",";
        }
    }

    copy_buffer[copy_buffer.size() - 1] = '}';
}

void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
{
    if (copy_buffer.c_str() != param[0]) {
        auto diff = copy_buffer.c_str() - param[0];
        for (size_t i = 0; i < size; ++i) {
            if (param[i]) {
                param[i] += diff;
            }
        }
    }
}

void middle_pgsql_t::local_nodes_set(osmium::Node const &node,
                                     double lat, double lon)
{
    copy_buffer.reserve(node.tags().byte_size() + 100);

    bool copy = node_table->copyMode;
    char delim = copy ? '\t' : '\0';
    const char *paramValues[4] = { copy_buffer.c_str(), };

    copy_buffer = std::to_string(node.id());
    copy_buffer += delim;

#ifdef FIXED_POINT
    ramNode n(lon, lat);
    paramValues[1] = paramValues[0] + copy_buffer.size();
    copy_buffer += std::to_string(n.int_lat());
    copy_buffer += delim;

    paramValues[2] = paramValues[0] + copy_buffer.size();
    copy_buffer += std::to_string(n.int_lon());
    copy_buffer += delim;
#else
    paramValues[1] = paramValues[0] + copy_buffer.size();
    copy_buffer += std::to_string(lat);
    copy_buffer += delim;

    paramValues[2] = paramValues[0] + copy_buffer.size();
    copy_buffer += std::to_string(lon);
    copy_buffer += delim;
#endif

    if (node.tags().empty() && !out_options->extra_attributes) {
        paramValues[3] = nullptr;
        copy_buffer += "\\N";
    } else {
        paramValues[3] = paramValues[0] + copy_buffer.size();
        buffer_store_tags(node, out_options->extra_attributes, copy);
    }

    if (copy) {
        copy_buffer += '\n';
        pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
    } else {
        buffer_correct_params(paramValues, 4);
        pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
                           (const char * const *)paramValues, PGRES_COMMAND_OK);
    }
}

// This should be made more efficient by using an IN(ARRAY[]) construct */
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const
{
    assert(out.empty());

    char tmp[16];

    char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
    if (tmp2 == nullptr) return 0; //failed to allocate memory, return */


    // create a list of ids in tmp2 to query the database  */
    sprintf(tmp2, "{");
    int countDB = 0;
    out.reserve(nds.size());
    for (auto const &n : nds) {
        // Check cache first */
        osmNode loc;
        if (cache->get(&loc, n.ref()) == 0) {
            out.push_back(loc);
            continue;
        }

        countDB++;
        // Mark nodes as needing to be fetched from the DB */
        out.push_back(osmNode());

        snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", n.ref());
        strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
    }
    tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/

    if (countDB == 0) {
        free(tmp2);
        return nds.size(); // All ids where in cache, so nothing more to do */
    }

    pgsql_endCopy(node_table);

    PGconn *sql_conn = node_table->sql_conn;

    char const *paramValues[1];
    paramValues[0] = tmp2;
    PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
    int countPG = PQntuples(res);

    //store the pg results in a hashmap and telling it how many we expect
    std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);

    for (int i = 0; i < countPG; i++) {
        osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
        osmNode node;
#ifdef FIXED_POINT
        ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
                  (int) strtol(PQgetvalue(res, i, 1), nullptr, 10));

        node.lat = n.lat();
        node.lon = n.lon();
#else
        node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
        node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
#endif
        pg_nodes.emplace(id, node);
    }

    PQclear(res);
    free(tmp2);

    // If some of the nodes in the way don't exist, the returning list has holes.
    // Merge the two lists removing any holes.
    size_t wrtidx = 0;
    for (size_t i = 0; i < nds.size(); ++i) {
        if (std::isnan(out[i].lat)) {
            std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i].ref());
            if(found != pg_nodes.end()) {
                out[wrtidx] = found->second;
                ++wrtidx;
            }
        } else {
            if (wrtidx < i)
                out[wrtidx] = out[i];
            ++wrtidx;
        }
    }
    out.resize(wrtidx);

    return wrtidx;
}