/**
* Calculate length of way.
*/
inline double distance(const osmium::WayNodeList& wnl) {
double sum_length = 0;
for (auto it = wnl.begin(); it != wnl.end(); ++it) {
if (std::next(it) != wnl.end()) {
sum_length += distance(it->location(), std::next(it)->location());
}
}
return sum_length;
}
linestring_type create_linestring(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) {
linestring_start();
size_t num_points = 0;
if (un == use_nodes::unique) {
osmium::Location last_location;
switch (dir) {
case direction::forward:
num_points = fill_linestring_unique(wnl.cbegin(), wnl.cend());
break;
case direction::backward:
num_points = fill_linestring_unique(wnl.crbegin(), wnl.crend());
break;
}
} else {
switch (dir) {
case direction::forward:
num_points = fill_linestring(wnl.cbegin(), wnl.cend());
break;
case direction::backward:
num_points = fill_linestring(wnl.crbegin(), wnl.crend());
break;
}
}
if (num_points < 2) {
throw osmium::geometry_error("not enough points for linestring");
}
return linestring_finish(num_points);
}
polygon_type create_polygon(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) {
polygon_start();
size_t num_points = 0;
if (un == use_nodes::unique) {
osmium::Location last_location;
switch (dir) {
case direction::forward:
num_points = fill_polygon_unique(wnl.cbegin(), wnl.cend());
break;
case direction::backward:
num_points = fill_polygon_unique(wnl.crbegin(), wnl.crend());
break;
}
} else {
switch (dir) {
case direction::forward:
num_points = fill_polygon(wnl.cbegin(), wnl.cend());
break;
case direction::backward:
num_points = fill_polygon(wnl.crbegin(), wnl.crend());
break;
}
}
if (num_points < 4) {
throw osmium::geometry_error{"need at least four points for polygon"};
}
return polygon_finish(num_points);
}
/* Implements the mid-layer processing for osm2pgsql
* using several PostgreSQL tables
*
* This layer stores data read in from the planet.osm file
* and is then read by the backend processing code to
* emit the final geometry-enabled output formats
*/
#include "config.h"
#ifdef _WIN32
using namespace std;
#endif
#ifdef _MSC_VER
#define alloca _alloca
#endif
#include <stdexcept>
#include <unordered_map>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <future>
#include <boost/format.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <libpq-fe.h>
#include "middle-pgsql.hpp"
#include "node-persistent-cache.hpp"
#include "node-ram-cache.hpp"
#include "options.hpp"
#include "osmtypes.hpp"
#include "output-pgsql.hpp"
#include "pgsql.hpp"
#include "util.hpp"
enum table_id {
t_node, t_way, t_rel
} ;
middle_pgsql_t::table_desc::table_desc(const char *name_,
const char *start_,
const char *create_,
const char *create_index_,
const char *prepare_,
const char *prepare_intarray_,
const char *copy_,
const char *analyze_,
const char *stop_,
const char *array_indexes_)
: name(name_),
start(start_),
create(create_),
create_index(create_index_),
prepare(prepare_),
prepare_intarray(prepare_intarray_),
copy(copy_),
analyze(analyze_),
stop(stop_),
array_indexes(array_indexes_),
copyMode(0),
transactionMode(0),
sql_conn(nullptr)
{}
namespace {
// Decodes a portion of an array literal from postgres */
// Argument should point to beginning of literal, on return points to delimiter */
inline const char *decode_upto( const char *src, char *dst )
{
int quoted = (*src == '"');
if( quoted ) src++;
while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
{
if( *src == '\\' )
{
switch( src[1] )
{
case 'n': *dst++ = '\n'; break;
case 't': *dst++ = '\t'; break;
default: *dst++ = src[1]; break;
}
src+=2;
}
else
*dst++ = *src++;
}
if( quoted ) src++;
*dst = 0;
return src;
}
template <typename T>
void pgsql_parse_tags(const char *string, osmium::memory::Buffer &buffer, T &obuilder)
{
if( *string++ != '{' )
return;
char key[1024];
char val[1024];
osmium::builder::TagListBuilder builder(buffer, &obuilder);
while( *string != '}' ) {
string = decode_upto(string, key);
// String points to the comma */
string++;
string = decode_upto(string, val);
builder.add_tag(key, val);
// String points to the comma or closing '}' */
if( *string == ',' ) {
string++;
}
}
}
void pgsql_parse_members(const char *string, osmium::memory::Buffer &buffer,
osmium::builder::RelationBuilder &obuilder)
{
if( *string++ != '{' )
return;
char role[1024];
osmium::builder::RelationMemberListBuilder builder(buffer, &obuilder);
while( *string != '}' ) {
char type = string[0];
char *endp;
osmid_t id = strtoosmid(string + 1, &endp, 10);
// String points to the comma */
string = decode_upto(endp + 1, role);
builder.add_member(osmium::char_to_item_type(type), id, role);
// String points to the comma or closing '}' */
if( *string == ',' ) {
string++;
}
}
}
void pgsql_parse_nodes(const char *string, osmium::memory::Buffer &buffer,
osmium::builder::WayBuilder &builder)
{
if (*string++ == '{') {
osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder);
while (*string != '}') {
char *ptr;
wnl_builder.add_node_ref(strtoosmid(string, &ptr, 10));
string = ptr;
if (*string == ',') {
string++;
}
}
}
}
int pgsql_endCopy(middle_pgsql_t::table_desc *table)
{
// Terminate any pending COPY */
if (table->copyMode) {
PGconn *sql_conn = table->sql_conn;
int stop = PQputCopyEnd(sql_conn, nullptr);
if (stop != 1) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
util::exit_nicely();
}
PGresult *res = PQgetResult(sql_conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
PQclear(res);
util::exit_nicely();
}
PQclear(res);
table->copyMode = 0;
}
return 0;
}
} // anonymous namespace
void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
{
for (char const c: in) {
switch (c) {
case '"':
if (escape) copy_buffer += "\\";
copy_buffer += "\\\"";
break;
case '\\':
if (escape) copy_buffer += "\\\\";
copy_buffer += "\\\\";
break;
case '\n':
if (escape) copy_buffer += "\\";
copy_buffer += "\\n";
break;
case '\r':
if (escape) copy_buffer += "\\";
copy_buffer += "\\r";
break;
case '\t':
if (escape) copy_buffer += "\\";
copy_buffer += "\\t";
break;
default:
copy_buffer += c;
break;
}
}
}
// escape means we return '\N' for copy mode, otherwise we return just nullptr
void middle_pgsql_t::buffer_store_tags(osmium::OSMObject const &obj, bool attrs,
bool escape)
{
copy_buffer += "{";
for (auto const &it : obj.tags()) {
copy_buffer += "\"";
buffer_store_string(it.key(), escape);
copy_buffer += "\",\"";
buffer_store_string(it.value(), escape);
copy_buffer += "\",";
}
if (attrs) {
taglist_t extra;
extra.add_attributes(obj);
for (auto const &it : extra) {
copy_buffer += "\"";
copy_buffer += it.key;
copy_buffer += "\",\"";
buffer_store_string(it.value.c_str(), escape);
copy_buffer += "\",";
}
}
copy_buffer[copy_buffer.size() - 1] = '}';
}
void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
{
if (copy_buffer.c_str() != param[0]) {
auto diff = copy_buffer.c_str() - param[0];
for (size_t i = 0; i < size; ++i) {
if (param[i]) {
param[i] += diff;
}
}
}
}
void middle_pgsql_t::local_nodes_set(osmium::Node const &node,
double lat, double lon)
{
copy_buffer.reserve(node.tags().byte_size() + 100);
bool copy = node_table->copyMode;
char delim = copy ? '\t' : '\0';
const char *paramValues[4] = { copy_buffer.c_str(), };
copy_buffer = std::to_string(node.id());
copy_buffer += delim;
#ifdef FIXED_POINT
ramNode n(lon, lat);
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lat());
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lon());
copy_buffer += delim;
#else
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lat);
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lon);
copy_buffer += delim;
#endif
if (node.tags().empty() && !out_options->extra_attributes) {
paramValues[3] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[3] = paramValues[0] + copy_buffer.size();
buffer_store_tags(node, out_options->extra_attributes, copy);
}
if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 4);
pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
(const char * const *)paramValues, PGRES_COMMAND_OK);
}
}
// This should be made more efficient by using an IN(ARRAY[]) construct */
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const
{
assert(out.empty());
char tmp[16];
char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2, "{");
int countDB = 0;
out.reserve(nds.size());
for (auto const &n : nds) {
// Check cache first */
osmNode loc;
if (cache->get(&loc, n.ref()) == 0) {
out.push_back(loc);
continue;
}
countDB++;
// Mark nodes as needing to be fetched from the DB */
out.push_back(osmNode());
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", n.ref());
strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
}
tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
if (countDB == 0) {
free(tmp2);
return nds.size(); // All ids where in cache, so nothing more to do */
}
pgsql_endCopy(node_table);
PGconn *sql_conn = node_table->sql_conn;
char const *paramValues[1];
paramValues[0] = tmp2;
PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
//store the pg results in a hashmap and telling it how many we expect
std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);
for (int i = 0; i < countPG; i++) {
osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
osmNode node;
#ifdef FIXED_POINT
ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
(int) strtol(PQgetvalue(res, i, 1), nullptr, 10));
node.lat = n.lat();
node.lon = n.lon();
#else
node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
#endif
pg_nodes.emplace(id, node);
}
PQclear(res);
free(tmp2);
// If some of the nodes in the way don't exist, the returning list has holes.
// Merge the two lists removing any holes.
size_t wrtidx = 0;
for (size_t i = 0; i < nds.size(); ++i) {
if (std::isnan(out[i].lat)) {
std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i].ref());
if(found != pg_nodes.end()) {
out[wrtidx] = found->second;
++wrtidx;
}
} else {
if (wrtidx < i)
out[wrtidx] = out[i];
++wrtidx;
}
}
out.resize(wrtidx);
return wrtidx;
}