size_t middle_ram_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
multitaglist_t &tags, multinodelist_t &nodes) const
{
if (ids.empty())
{
return 0;
}
assert(way_ids.empty());
tags.assign(ids.size(), taglist_t());
nodes.assign(ids.size(), nodelist_t());
size_t count = 0;
for (idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
if (ways_get(*it, tags[count], nodes[count])) {
way_ids.push_back(*it);
count++;
} else {
tags[count].clear();
nodes[count].clear();
}
}
if (count < ids.size()) {
tags.resize(count);
nodes.resize(count);
}
return int(count);
}
size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const
{
if (ids.empty())
return 0;
char tmp[16];
std::unique_ptr<char[]> tmp2(new (std::nothrow) char[ids.size() * 16]);
char const *paramValues[1];
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2.get(), "{");
for (auto id : ids) {
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", id);
strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2));
}
tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/
pgsql_endCopy(way_table);
PGconn *sql_conn = way_table->sql_conn;
paramValues[0] = tmp2.get();
PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
idlist_t wayidspg;
for (int i = 0; i < countPG; i++) {
wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10));
}
// Match the list of ways coming from postgres in a different order
// back to the list of ways given by the caller */
int outres = 0;
for (auto id : ids) {
for (int j = 0; j < countPG; j++) {
if (id == wayidspg[j]) {
{
osmium::builder::WayBuilder builder(buffer);
builder.set_id(id);
pgsql_parse_nodes(PQgetvalue(res, j, 1), buffer, builder);
pgsql_parse_tags(PQgetvalue(res, j, 2), buffer, builder);
}
buffer.commit();
outres++;
break;
}
}
}
PQclear(res);
return outres;
}
int way_add(osmid_t id, const idlist_t &nds, const taglist_t &tags) {
assert(id > 0);
sum_ids += id;
num_ways += 1;
assert(nds.size() >= 0);
num_nds += uint64_t(nds.size());
return 0;
}
void middle_pgsql_t::ways_set(osmid_t way_id, const idlist_t &nds, const taglist_t &tags)
{
copy_buffer.reserve(nds.size() * 10 + tags.size() * 24 + 64);
bool copy = way_table->copyMode;
char delim = copy ? '\t' : '\0';
// Three params: id, nodes, tags */
const char *paramValues[4] = { copy_buffer.c_str(), };
copy_buffer = std::to_string(way_id);
copy_buffer += delim;
paramValues[1] = paramValues[0] + copy_buffer.size();
buffer_store_nodes(nds);
copy_buffer += delim;
if (tags.size() == 0) {
paramValues[2] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[2] = paramValues[0] + copy_buffer.size();
buffer_store_tags(tags, copy);
}
if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, way_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 3);
pgsql_execPrepared(way_table->sql_conn, "insert_way", 3,
(const char * const *)paramValues, PGRES_COMMAND_OK);
}
}
void middle_pgsql_t::buffer_store_nodes(idlist_t const &nds)
{
if (nds.size() == 0) {
copy_buffer += "{}";
return;
}
copy_buffer += "{";
for (auto const &it : nds) {
copy_buffer += std::to_string(it);
copy_buffer += ',';
}
copy_buffer[copy_buffer.size() - 1] = '}';
}
int output_multi_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
if (m_processor->interests(geometry_processor::interest_way) && nodes.size() > 1) {
return process_way(id, nodes, tags);
}
return 0;
}
size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
multitaglist_t &tags, multinodelist_t &nodes) const {
if (ids.empty())
return 0;
char tmp[16];
std::unique_ptr<char[]> tmp2(new (std::nothrow) char[ids.size() * 16]);
char const *paramValues[1];
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2.get(), "{");
for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2));
}
tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/
pgsql_endCopy(way_table);
PGconn *sql_conn = way_table->sql_conn;
paramValues[0] = tmp2.get();
PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
idlist_t wayidspg;
for (int i = 0; i < countPG; i++) {
wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10));
}
// Match the list of ways coming from postgres in a different order
// back to the list of ways given by the caller */
for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
for (int j = 0; j < countPG; j++) {
if (*it == wayidspg[j]) {
way_ids.push_back(*it);
tags.push_back(taglist_t());
pgsql_parse_tags(PQgetvalue(res, j, 2), tags.back());
size_t num_nodes = strtoul(PQgetvalue(res, j, 3), nullptr, 10);
idlist_t list;
pgsql_parse_nodes( PQgetvalue(res, j, 1), list);
if (num_nodes != list.size()) {
fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n",
tmp, num_nodes, list.size());
util::exit_nicely();
}
nodes.push_back(nodelist_t());
nodes_get_list(nodes.back(), list);
break;
}
}
}
assert(way_ids.size() <= ids.size());
PQclear(res);
return way_ids.size();
}
/* Implements the mid-layer processing for osm2pgsql
* using several PostgreSQL tables
*
* This layer stores data read in from the planet.osm file
* and is then read by the backend processing code to
* emit the final geometry-enabled output formats
*/
#include "config.h"
#ifdef _WIN32
using namespace std;
#endif
#ifdef _MSC_VER
#define alloca _alloca
#endif
#include <stdexcept>
#include <unordered_map>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <future>
#include <boost/format.hpp>
#include <libpq-fe.h>
#include "middle-pgsql.hpp"
#include "node-persistent-cache.hpp"
#include "node-ram-cache.hpp"
#include "options.hpp"
#include "osmtypes.hpp"
#include "output-pgsql.hpp"
#include "pgsql.hpp"
#include "util.hpp"
enum table_id {
t_node, t_way, t_rel
} ;
middle_pgsql_t::table_desc::table_desc(const char *name_,
const char *start_,
const char *create_,
const char *create_index_,
const char *prepare_,
const char *prepare_intarray_,
const char *copy_,
const char *analyze_,
const char *stop_,
const char *array_indexes_)
: name(name_),
start(start_),
create(create_),
create_index(create_index_),
prepare(prepare_),
prepare_intarray(prepare_intarray_),
copy(copy_),
analyze(analyze_),
stop(stop_),
array_indexes(array_indexes_),
copyMode(0),
transactionMode(0),
sql_conn(nullptr)
{}
namespace {
// Decodes a portion of an array literal from postgres */
// Argument should point to beginning of literal, on return points to delimiter */
inline const char *decode_upto( const char *src, char *dst )
{
int quoted = (*src == '"');
if( quoted ) src++;
while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
{
if( *src == '\\' )
{
switch( src[1] )
{
case 'n': *dst++ = '\n'; break;
case 't': *dst++ = '\t'; break;
default: *dst++ = src[1]; break;
}
src+=2;
}
else
*dst++ = *src++;
}
if( quoted ) src++;
*dst = 0;
return src;
}
void pgsql_parse_tags(const char *string, taglist_t &tags)
{
char key[1024];
char val[1024];
if( *string == '\0' )
return;
if( *string++ != '{' )
return;
while( *string != '}' )
{
string = decode_upto( string, key );
// String points to the comma */
string++;
string = decode_upto( string, val );
// String points to the comma or closing '}' */
tags.push_back(tag_t(key, val));
if( *string == ',' )
string++;
}
}
// Parses an array of integers */
void pgsql_parse_nodes(const char *string, idlist_t &nds)
{
if( *string++ != '{' )
return;
while( *string != '}' )
{
char *ptr;
nds.push_back(strtoosmid( string, &ptr, 10 ));
string = ptr;
if( *string == ',' )
string++;
}
}
int pgsql_endCopy(middle_pgsql_t::table_desc *table)
{
// Terminate any pending COPY */
if (table->copyMode) {
PGconn *sql_conn = table->sql_conn;
int stop = PQputCopyEnd(sql_conn, nullptr);
if (stop != 1) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
util::exit_nicely();
}
PGresult *res = PQgetResult(sql_conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
PQclear(res);
util::exit_nicely();
}
PQclear(res);
table->copyMode = 0;
}
return 0;
}
} // anonymous namespace
void middle_pgsql_t::buffer_store_nodes(idlist_t const &nds)
{
if (nds.size() == 0) {
copy_buffer += "{}";
return;
}
copy_buffer += "{";
for (auto const &it : nds) {
copy_buffer += std::to_string(it);
copy_buffer += ',';
}
copy_buffer[copy_buffer.size() - 1] = '}';
}
void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
{
for (char const c: in) {
switch (c) {
case '"':
if (escape) copy_buffer += "\\";
copy_buffer += "\\\"";
break;
case '\\':
if (escape) copy_buffer += "\\\\";
copy_buffer += "\\\\";
break;
case '\n':
if (escape) copy_buffer += "\\";
copy_buffer += "\\n";
break;
case '\r':
if (escape) copy_buffer += "\\";
copy_buffer += "\\r";
break;
case '\t':
if (escape) copy_buffer += "\\";
copy_buffer += "\\t";
break;
default:
copy_buffer += c;
break;
}
}
}
// escape means we return '\N' for copy mode, otherwise we return just nullptr
void middle_pgsql_t::buffer_store_tags(taglist_t const &tags, bool escape)
{
copy_buffer += "{";
bool first = true;
for (auto const &it : tags) {
if (!first) {
copy_buffer += ',';
}
copy_buffer += "\"";
buffer_store_string(it.key, escape);
copy_buffer += "\",\"";
buffer_store_string(it.value, escape);
copy_buffer += '"';
first = false;
}
copy_buffer += "}";
}
void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
{
if (copy_buffer.c_str() != param[0]) {
auto diff = copy_buffer.c_str() - param[0];
for (size_t i = 0; i < size; ++i) {
if (param[i]) {
param[i] += diff;
}
}
}
}
void middle_pgsql_t::local_nodes_set(osmid_t id, double lat, double lon,
const taglist_t &tags)
{
copy_buffer.reserve(tags.size() * 24 + 64);
bool copy = node_table->copyMode;
char delim = copy ? '\t' : '\0';
const char *paramValues[4] = { copy_buffer.c_str(), };
copy_buffer = std::to_string(id);
copy_buffer += delim;
#ifdef FIXED_POINT
ramNode n(lon, lat);
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lat());
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lon());
copy_buffer += delim;
#else
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lat);
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lon);
copy_buffer += delim;
#endif
if (tags.size() == 0) {
paramValues[3] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[3] = paramValues[0] + copy_buffer.size();
buffer_store_tags(tags, copy);
}
if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 4);
pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
(const char * const *)paramValues, PGRES_COMMAND_OK);
}
}
// This should be made more efficient by using an IN(ARRAY[]) construct */
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, const idlist_t nds) const
{
assert(out.empty());
char tmp[16];
char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
// create a list of ids in tmp2 to query the database */
sprintf(tmp2, "{");
int countDB = 0;
for(idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) {
// Check cache first */
osmNode loc;
if (cache->get(&loc, *it) == 0) {
out.push_back(loc);
continue;
}
countDB++;
// Mark nodes as needing to be fetched from the DB */
out.push_back(osmNode());
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
}
tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
if (countDB == 0) {
free(tmp2);
return nds.size(); // All ids where in cache, so nothing more to do */
}
pgsql_endCopy(node_table);
PGconn *sql_conn = node_table->sql_conn;
char const *paramValues[1];
paramValues[0] = tmp2;
PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);
//store the pg results in a hashmap and telling it how many we expect
std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);
for (int i = 0; i < countPG; i++) {
osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
osmNode node;
#ifdef FIXED_POINT
ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
(int) strtol(PQgetvalue(res, i, 1), nullptr, 10));
node.lat = n.lat();
node.lon = n.lon();
#else
node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
#endif
pg_nodes.emplace(id, node);
}
PQclear(res);
free(tmp2);
// If some of the nodes in the way don't exist, the returning list has holes.
// Merge the two lists removing any holes.
size_t wrtidx = 0;
for (size_t i = 0; i < nds.size(); ++i) {
if (std::isnan(out[i].lat)) {
std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i]);
if(found != pg_nodes.end()) {
out[wrtidx] = found->second;
++wrtidx;
}
} else {
if (wrtidx < i)
out[wrtidx] = out[i];
++wrtidx;
}
}
out.resize(wrtidx);
return wrtidx;
}
int output_pgsql_t::pgsql_out_relation(osmid_t id, const taglist_t &rel_tags,
const multinodelist_t &xnodes, const multitaglist_t & xtags,
const idlist_t &xid, const rolelist_t &xrole,
bool pending)
{
if (xnodes.empty())
return 0;
int roads = 0;
int make_polygon = 0;
int make_boundary = 0;
double split_at;
std::vector<int> members_superseeded(xnodes.size(), 0);
taglist_t outtags;
//if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true
if (m_tagtransform->filter_rel_member_tags(rel_tags, xtags, xrole,
&(members_superseeded[0]), &make_boundary, &make_polygon, &roads,
*m_export_list.get(), outtags)) {
return 0;
}
/* Split long linear ways after around 1 degree or 100km (polygons not effected) */
if (m_options.projection->get_proj_id() == PROJ_LATLONG)
split_at = 1;
else
split_at = 100 * 1000;
//this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above
//TODO: pick one or the other based on which we expect to care about
auto wkbs = builder.build_both(xnodes, make_polygon, m_options.enable_multi, split_at, id);
if (wkbs.empty()) {
return 0;
}
tag_t *areatag = 0;
char tmp[32];
for (const auto& wkb: wkbs) {
expire->from_wkb(wkb.geom.c_str(), -id);
/* FIXME: there should be a better way to detect polygons */
if (wkb.is_polygon()) {
if ((wkb.area > 0.0) && m_enable_way_area) {
snprintf(tmp, sizeof(tmp), "%g", wkb.area);
if (!areatag) {
outtags.push_dedupe(tag_t("way_area", tmp));
areatag = outtags.find("way_area");
}
}
m_tables[t_poly]->write_row(-id, outtags, wkb.geom);
} else {
m_tables[t_line]->write_row(-id, outtags, wkb.geom);
if (roads)
m_tables[t_roads]->write_row(-id, outtags, wkb.geom);
}
}
/* Tagtransform will have marked those member ways of the relation that
* have fully been dealt with as part of the multi-polygon entry.
* Set them in the database as done and delete their entry to not
* have duplicates */
//dont do this when working with pending relations as its not needed
if (make_polygon) {
for (size_t i=0; i < xid.size(); i++) {
if (members_superseeded[i]) {
pgsql_delete_way_from_output(xid[i]);
if(!pending)
ways_done_tracker->mark(xid[i]);
}
}
}
// If the tag transform said the polygon looked like a boundary we want to make that as well
// If we are making a boundary then also try adding any relations which form complete rings
// The linear variants will have already been processed above
if (make_boundary) {
wkbs = builder.build_polygons(xnodes, m_options.enable_multi, id);
for (const auto& wkb: wkbs) {
expire->from_wkb(wkb.geom.c_str(), -id);
if ((wkb.area > 0.0) && m_enable_way_area) {
snprintf(tmp, sizeof(tmp), "%g", wkb.area);
if (!areatag) {
outtags.push_dedupe(tag_t("way_area", tmp));
areatag = outtags.find("way_area");
}
}
m_tables[t_poly]->write_row(-id, outtags, wkb.geom);
}
}
return 0;
}