size_t middle_ram_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
multitaglist_t &tags, multinodelist_t &nodes) const
{
if (ids.empty())
{
return 0;
}
assert(way_ids.empty());
tags.assign(ids.size(), taglist_t());
nodes.assign(ids.size(), nodelist_t());
size_t count = 0;
for (idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
if (ways_get(*it, tags[count], nodes[count])) {
way_ids.push_back(*it);
count++;
} else {
tags[count].clear();
nodes[count].clear();
}
}
if (count < ids.size()) {
tags.resize(count);
nodes.resize(count);
}
return int(count);
}
size_t middle_ram_t::nodes_get_list(nodelist_t &out, const idlist_t nds) const
{
for (idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) {
osmNode n;
if (!cache->get(&n, *it))
out.push_back(n);
}
return int(out.size());
}
/*
COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu
ilding, bridge, layer, way) FROM stdin;
198497 Bedford Road \N \N \N \N \N \N residential \N \N \N \N \N \N \N 0102000020E610000004000000452BF702B342D5BF1C60E63BF8DF49406B9C4D470037D5BF5471E316F3DF4940DFA815A6EF35D5BF9AE95E27F5DF4940B41EB
E4C1421D5BF24D06053E7DF4940
212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D
5BF5BB39597FCDF4940
*/
int output_pgsql_t::pgsql_out_way(osmid_t id, const taglist_t &tags, const nodelist_t &nodes, int exists)
{
int polygon = 0, roads = 0;
double split_at;
/* If the flag says this object may exist already, delete it first */
if (exists) {
pgsql_delete_way_from_output(id);
// TODO: this now only has an effect when called from the iterate_ways
// call-back, so we need some alternative way to trigger this within
// osmdata_t.
const idlist_t rel_ids = m_mid->relations_using_way(id);
for (idlist_t::const_iterator itr = rel_ids.begin();
itr != rel_ids.end(); ++itr) {
rels_pending_tracker->mark(*itr);
}
}
taglist_t outtags;
if (m_tagtransform->filter_way_tags(tags, &polygon, &roads, *m_export_list.get(),
outtags))
return 0;
/* Split long ways after around 1 degree or 100km */
if (m_options.projection->get_proj_id() == PROJ_LATLONG)
split_at = 1;
else
split_at = 100 * 1000;
tag *areatag = 0;
geometry_builder::maybe_wkts_t wkts = builder.get_wkt_split(nodes, polygon, split_at);
for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) {
/* FIXME: there should be a better way to detect polygons */
if (boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) {
expire->from_nodes_poly(nodes, id);
if ((wkt->area > 0.0) && m_enable_way_area) {
char tmp[32];
snprintf(tmp, sizeof(tmp), "%g", wkt->area);
if (!areatag) {
outtags.push_dedupe(tag("way_area", tmp));
areatag = outtags.find("way_area");
} else
areatag->value = tmp;
}
m_tables[t_poly]->write_wkt(id, outtags, wkt->geom.c_str());
} else {
expire->from_nodes_line(nodes);
m_tables[t_line]->write_wkt(id, outtags, wkt->geom.c_str());
if (roads)
m_tables[t_roads]->write_wkt(id, outtags, wkt->geom.c_str());
}
}
return 0;
}
size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
multitaglist_t &tags, multinodelist_t &nodes) const {
if (ids.empty())
return 0;
char tmp[16];
std::unique_ptr<char[]> tmp2(new (std::nothrow) char[ids.size() * 16]);
char const *paramValues[1];
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2.get(), "{");
for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2));
}
tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/
pgsql_endCopy(way_table);
PGconn *sql_conn = way_table->sql_conn;
paramValues[0] = tmp2.get();
PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
idlist_t wayidspg;
for (int i = 0; i < countPG; i++) {
wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10));
}
// Match the list of ways coming from postgres in a different order
// back to the list of ways given by the caller */
for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
for (int j = 0; j < countPG; j++) {
if (*it == wayidspg[j]) {
way_ids.push_back(*it);
tags.push_back(taglist_t());
pgsql_parse_tags(PQgetvalue(res, j, 2), tags.back());
size_t num_nodes = strtoul(PQgetvalue(res, j, 3), nullptr, 10);
idlist_t list;
pgsql_parse_nodes( PQgetvalue(res, j, 1), list);
if (num_nodes != list.size()) {
fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n",
tmp, num_nodes, list.size());
util::exit_nicely();
}
nodes.push_back(nodelist_t());
nodes_get_list(nodes.back(), list);
break;
}
}
}
assert(way_ids.size() <= ids.size());
PQclear(res);
return way_ids.size();
}
/* Implements the mid-layer processing for osm2pgsql
* using several PostgreSQL tables
*
* This layer stores data read in from the planet.osm file
* and is then read by the backend processing code to
* emit the final geometry-enabled output formats
*/
#include "config.h"
#ifdef _WIN32
using namespace std;
#endif
#ifdef _MSC_VER
#define alloca _alloca
#endif
#include <stdexcept>
#include <unordered_map>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <future>
#include <boost/format.hpp>
#include <libpq-fe.h>
#include "middle-pgsql.hpp"
#include "node-persistent-cache.hpp"
#include "node-ram-cache.hpp"
#include "options.hpp"
#include "osmtypes.hpp"
#include "output-pgsql.hpp"
#include "pgsql.hpp"
#include "util.hpp"
enum table_id {
t_node, t_way, t_rel
} ;
middle_pgsql_t::table_desc::table_desc(const char *name_,
const char *start_,
const char *create_,
const char *create_index_,
const char *prepare_,
const char *prepare_intarray_,
const char *copy_,
const char *analyze_,
const char *stop_,
const char *array_indexes_)
: name(name_),
start(start_),
create(create_),
create_index(create_index_),
prepare(prepare_),
prepare_intarray(prepare_intarray_),
copy(copy_),
analyze(analyze_),
stop(stop_),
array_indexes(array_indexes_),
copyMode(0),
transactionMode(0),
sql_conn(nullptr)
{}
namespace {
// Decodes a portion of an array literal from postgres */
// Argument should point to beginning of literal, on return points to delimiter */
inline const char *decode_upto( const char *src, char *dst )
{
int quoted = (*src == '"');
if( quoted ) src++;
while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
{
if( *src == '\\' )
{
switch( src[1] )
{
case 'n': *dst++ = '\n'; break;
case 't': *dst++ = '\t'; break;
default: *dst++ = src[1]; break;
}
src+=2;
}
else
*dst++ = *src++;
}
if( quoted ) src++;
*dst = 0;
return src;
}
void pgsql_parse_tags(const char *string, taglist_t &tags)
{
char key[1024];
char val[1024];
if( *string == '\0' )
return;
if( *string++ != '{' )
return;
while( *string != '}' )
{
string = decode_upto( string, key );
// String points to the comma */
string++;
string = decode_upto( string, val );
// String points to the comma or closing '}' */
tags.push_back(tag_t(key, val));
if( *string == ',' )
string++;
}
}
// Parses an array of integers */
void pgsql_parse_nodes(const char *string, idlist_t &nds)
{
if( *string++ != '{' )
return;
while( *string != '}' )
{
char *ptr;
nds.push_back(strtoosmid( string, &ptr, 10 ));
string = ptr;
if( *string == ',' )
string++;
}
}
int pgsql_endCopy(middle_pgsql_t::table_desc *table)
{
// Terminate any pending COPY */
if (table->copyMode) {
PGconn *sql_conn = table->sql_conn;
int stop = PQputCopyEnd(sql_conn, nullptr);
if (stop != 1) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
util::exit_nicely();
}
PGresult *res = PQgetResult(sql_conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
PQclear(res);
util::exit_nicely();
}
PQclear(res);
table->copyMode = 0;
}
return 0;
}
} // anonymous namespace
void middle_pgsql_t::buffer_store_nodes(idlist_t const &nds)
{
if (nds.size() == 0) {
copy_buffer += "{}";
return;
}
copy_buffer += "{";
for (auto const &it : nds) {
copy_buffer += std::to_string(it);
copy_buffer += ',';
}
copy_buffer[copy_buffer.size() - 1] = '}';
}
void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
{
for (char const c: in) {
switch (c) {
case '"':
if (escape) copy_buffer += "\\";
copy_buffer += "\\\"";
break;
case '\\':
if (escape) copy_buffer += "\\\\";
copy_buffer += "\\\\";
break;
case '\n':
if (escape) copy_buffer += "\\";
copy_buffer += "\\n";
break;
case '\r':
if (escape) copy_buffer += "\\";
copy_buffer += "\\r";
break;
case '\t':
if (escape) copy_buffer += "\\";
copy_buffer += "\\t";
break;
default:
copy_buffer += c;
break;
}
}
}
// escape means we return '\N' for copy mode, otherwise we return just nullptr
void middle_pgsql_t::buffer_store_tags(taglist_t const &tags, bool escape)
{
copy_buffer += "{";
bool first = true;
for (auto const &it : tags) {
if (!first) {
copy_buffer += ',';
}
copy_buffer += "\"";
buffer_store_string(it.key, escape);
copy_buffer += "\",\"";
buffer_store_string(it.value, escape);
copy_buffer += '"';
first = false;
}
copy_buffer += "}";
}
void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
{
if (copy_buffer.c_str() != param[0]) {
auto diff = copy_buffer.c_str() - param[0];
for (size_t i = 0; i < size; ++i) {
if (param[i]) {
param[i] += diff;
}
}
}
}
void middle_pgsql_t::local_nodes_set(osmid_t id, double lat, double lon,
const taglist_t &tags)
{
copy_buffer.reserve(tags.size() * 24 + 64);
bool copy = node_table->copyMode;
char delim = copy ? '\t' : '\0';
const char *paramValues[4] = { copy_buffer.c_str(), };
copy_buffer = std::to_string(id);
copy_buffer += delim;
#ifdef FIXED_POINT
ramNode n(lon, lat);
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lat());
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lon());
copy_buffer += delim;
#else
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lat);
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lon);
copy_buffer += delim;
#endif
if (tags.size() == 0) {
paramValues[3] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[3] = paramValues[0] + copy_buffer.size();
buffer_store_tags(tags, copy);
}
if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 4);
pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
(const char * const *)paramValues, PGRES_COMMAND_OK);
}
}
// This should be made more efficient by using an IN(ARRAY[]) construct */
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, const idlist_t nds) const
{
assert(out.empty());
char tmp[16];
char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2, "{");
int countDB = 0;
for(idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) {
// Check cache first */
osmNode loc;
if (cache->get(&loc, *it) == 0) {
out.push_back(loc);
continue;
}
countDB++;
// Mark nodes as needing to be fetched from the DB */
out.push_back(osmNode());
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
}
tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
if (countDB == 0) {
free(tmp2);
return nds.size(); // All ids where in cache, so nothing more to do */
}
pgsql_endCopy(node_table);
PGconn *sql_conn = node_table->sql_conn;
char const *paramValues[1];
paramValues[0] = tmp2;
PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
//store the pg results in a hashmap and telling it how many we expect
std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);
for (int i = 0; i < countPG; i++) {
osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
osmNode node;
#ifdef FIXED_POINT
ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
(int) strtol(PQgetvalue(res, i, 1), nullptr, 10));
node.lat = n.lat();
node.lon = n.lon();
#else
node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
#endif
pg_nodes.emplace(id, node);
}
PQclear(res);
free(tmp2);
// If some of the nodes in the way don't exist, the returning list has holes.
// Merge the two lists removing any holes.
size_t wrtidx = 0;
for (size_t i = 0; i < nds.size(); ++i) {
if (std::isnan(out[i].lat)) {
std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i]);
if(found != pg_nodes.end()) {
out[wrtidx] = found->second;
++wrtidx;
}
} else {
if (wrtidx < i)
out[wrtidx] = out[i];
++wrtidx;
}
}
out.resize(wrtidx);
return wrtidx;
}