geos_datasource::geos_datasource(parameters const& params, bool bind)
: datasource(params),
extent_(),
extent_initialized_(false),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")),
geometry_data_(""),
geometry_data_name_("name"),
geometry_id_(1)
{
boost::optional<std::string> geometry = params.get<std::string>("wkt");
if (!geometry) throw datasource_exception("missing <wkt> parameter");
geometry_string_ = *geometry;
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false);
boost::optional<std::string> ext = params_.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<int> id = params_.get<int>("gid");
if (id) geometry_id_ = *id;
boost::optional<std::string> gdata = params_.get<std::string>("field_data");
if (gdata) geometry_data_ = *gdata;
boost::optional<std::string> gdata_name = params_.get<std::string>("field_name");
if (gdata_name) geometry_data_name_ = *gdata_name;
desc_.add_descriptor(attribute_descriptor(geometry_data_name_, mapnik::String));
if (bind)
{
this->bind();
}
}
osm_datasource::osm_datasource(const parameters& params)
: datasource (params),
extent_(),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8"))
{
osm_data_ = NULL;
std::string osm_filename = *params.get<std::string>("file", "");
std::string parser = *params.get<std::string>("parser", "libxml2");
std::string url = *params.get<std::string>("url", "");
std::string bbox = *params.get<std::string>("bbox", "");
// load the data
if (url != "" && bbox != "")
{
// if we supplied a url and a bounding box, load from the url
MAPNIK_LOG_DEBUG(osm) << "osm_datasource: loading_from_url url=" << url << ",bbox=" << bbox;
if ((osm_data_ = dataset_deliverer::load_from_url(url, bbox, parser)) == NULL)
{
throw datasource_exception("Error loading from URL");
}
}
else if (osm_filename != "")
{
// if we supplied a filename, load from file
if ((osm_data_ = dataset_deliverer::load_from_file(osm_filename, parser)) == NULL)
{
std::string s("OSM Plugin: Error loading from file '");
s += osm_filename + "'";
throw datasource_exception(s);
}
}
else
{
throw datasource_exception("OSM Plugin: Neither 'file' nor 'url' and 'bbox' specified");
}
osm_tag_types tagtypes;
tagtypes.add_type("maxspeed", mapnik::Integer);
tagtypes.add_type("z_order", mapnik::Integer);
osm_data_->rewind();
// Need code to get the attributes of all the data
std::set<std::string> keys = osm_data_->get_keys();
// Add the attributes to the datasource descriptor - assume they are
// all of type String
for (auto const& key : keys)
{
desc_.add_descriptor(attribute_descriptor(key, tagtypes.get_type(key)));
}
// Get the bounds of the data and set extent_ accordingly
bounds b = osm_data_->get_bounds();
extent_ = box2d<double>(b.w,b.s,b.e,b.n);
}
osm_datasource::osm_datasource(const parameters ¶ms)
: datasource (params),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8"))
{
osm_data_ = NULL;
std::string osm_filename= *params.get<std::string>("file","");
std::string parser = *params.get<std::string>("parser","libxml2");
std::string url = *params.get<std::string>("url","");
std::string bbox = *params.get<std::string>("bbox","");
bool do_process=false;
// load the data
// if we supplied a filename, load from file
if (url!="" && bbox!="")
{
// otherwise if we supplied a url and a bounding box, load from the url
#ifdef MAPNIK_DEBUG
cerr<<"loading_from_rul: url="<<url << " bbox="<<bbox<<endl;
#endif
if((osm_data_=dataset_deliverer::load_from_url
(url,bbox,parser))==NULL)
{
throw datasource_exception("Error loading from URL");
}
do_process=true;
}
else if(osm_filename!="")
{
if ((osm_data_=
dataset_deliverer::load_from_file(osm_filename,parser))==NULL)
{
throw datasource_exception("Error loading from file");
}
do_process=true;
}
if(do_process==true)
{
osm_tag_types tagtypes;
tagtypes.add_type("maxspeed",mapnik::Integer);
tagtypes.add_type("z_order",mapnik::Integer);
osm_data_->rewind();
// Need code to get the attributes of all the data
std::set<std::string> keys= osm_data_->get_keys();
// Add the attributes to the datasource descriptor - assume they are
// all of type String
for(std::set<std::string>::iterator i=keys.begin(); i!=keys.end(); i++)
desc_.add_descriptor(attribute_descriptor(*i,tagtypes.get_type(*i)));
// Get the bounds of the data and set extent_ accordingly
bounds b = osm_data_->get_bounds();
extent_ = box2d<double>(b.w,b.s,b.e,b.n);
}
}
void occi_datasource::bind() const
{
if (is_bound_) return;
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::bind");
#endif
// connect to environment
if (use_connection_pool_)
{
try
{
Environment* env = occi_environment::get_environment();
pool_ = env->createStatelessConnectionPool(
*params_.get<std::string>("user"),
*params_.get<std::string>("password"),
*params_.get<std::string>("host"),
*params_.get<int>("max_size", 5),
*params_.get<int>("initial_size", 1),
1,
StatelessConnectionPool::HOMOGENEOUS);
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
else
{
try
{
Environment* env = occi_environment::get_environment();
conn_ = env->createConnection(
*params_.get<std::string>("user"),
*params_.get<std::string>("password"),
*params_.get<std::string>("host"));
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
// extract real table name
table_name_ = mapnik::sql_utils::table_from_sql(table_);
// get SRID and/or GEOMETRY_FIELD from metadata table only if we need to
if (! srid_initialized_ || geometry_field_ == "")
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_srid_and_geometry_field");
#endif
std::ostringstream s;
s << "SELECT srid, column_name FROM " << METADATA_TABLE << " WHERE";
s << " LOWER(table_name) = LOWER('" << table_name_ << "')";
if (geometry_field_ != "")
{
s << " AND LOWER(column_name) = LOWER('" << geometry_field_ << "')";
}
MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str();
try
{
occi_connection_ptr conn;
if (use_connection_pool_) conn.set_pool(pool_);
else conn.set_connection(conn_, false);
ResultSet* rs = conn.execute_query(s.str());
if (rs && rs->next ())
{
if (! srid_initialized_)
{
srid_ = rs->getInt(1);
srid_initialized_ = true;
}
if (geometry_field_ == "")
{
geometry_field_ = rs->getString(2);
}
}
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
// get columns description
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_column_description");
#endif
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_name_ << ") WHERE rownum < 1";
MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str();
try
{
occi_connection_ptr conn;
if (use_connection_pool_) conn.set_pool(pool_);
else conn.set_connection(conn_, false);
ResultSet* rs = conn.execute_query(s.str());
if (rs)
{
std::vector<MetaData> listOfColumns = rs->getColumnListMetaData();
for (unsigned int i = 0; i < listOfColumns.size(); ++i)
{
MetaData columnObj = listOfColumns[i];
std::string fld_name = columnObj.getString(MetaData::ATTR_NAME);
int type_oid = columnObj.getInt(MetaData::ATTR_DATA_TYPE);
/*
int type_code = columnObj.getInt(MetaData::ATTR_TYPECODE);
if (type_code == OCCI_TYPECODE_OBJECT)
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object));
continue;
}
*/
switch (type_oid)
{
case oracle::occi::OCCIBOOL:
case oracle::occi::OCCIINT:
case oracle::occi::OCCIUNSIGNED_INT:
case oracle::occi::OCCIROWID:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case oracle::occi::OCCIFLOAT:
case oracle::occi::OCCIBFLOAT:
case oracle::occi::OCCIDOUBLE:
case oracle::occi::OCCIBDOUBLE:
case oracle::occi::OCCINUMBER:
case oracle::occi::OCCI_SQLT_NUM:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case oracle::occi::OCCICHAR:
case oracle::occi::OCCISTRING:
case oracle::occi::OCCI_SQLT_AFC:
case oracle::occi::OCCI_SQLT_AVC:
case oracle::occi::OCCI_SQLT_CHR:
case oracle::occi::OCCI_SQLT_LVC:
case oracle::occi::OCCI_SQLT_RDD:
case oracle::occi::OCCI_SQLT_STR:
case oracle::occi::OCCI_SQLT_VCS:
case oracle::occi::OCCI_SQLT_VNU:
case oracle::occi::OCCI_SQLT_VBI:
case oracle::occi::OCCI_SQLT_VST:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case oracle::occi::OCCIDATE:
case oracle::occi::OCCITIMESTAMP:
case oracle::occi::OCCIINTERVALDS:
case oracle::occi::OCCIINTERVALYM:
case oracle::occi::OCCI_SQLT_DAT:
case oracle::occi::OCCI_SQLT_DATE:
case oracle::occi::OCCI_SQLT_TIME:
case oracle::occi::OCCI_SQLT_TIME_TZ:
case oracle::occi::OCCI_SQLT_TIMESTAMP:
case oracle::occi::OCCI_SQLT_TIMESTAMP_LTZ:
case oracle::occi::OCCI_SQLT_TIMESTAMP_TZ:
case oracle::occi::OCCI_SQLT_INTERVAL_YM:
case oracle::occi::OCCI_SQLT_INTERVAL_DS:
case oracle::occi::OCCIANYDATA:
case oracle::occi::OCCIBLOB:
case oracle::occi::OCCIBFILE:
case oracle::occi::OCCIBYTES:
case oracle::occi::OCCICLOB:
case oracle::occi::OCCIVECTOR:
case oracle::occi::OCCIMETADATA:
case oracle::occi::OCCIPOBJECT:
case oracle::occi::OCCIREF:
case oracle::occi::OCCIREFANY:
case oracle::occi::OCCISTREAM:
case oracle::occi::OCCICURSOR:
case oracle::occi::OCCI_SQLT_FILE:
case oracle::occi::OCCI_SQLT_CFILE:
case oracle::occi::OCCI_SQLT_REF:
case oracle::occi::OCCI_SQLT_CLOB:
case oracle::occi::OCCI_SQLT_BLOB:
case oracle::occi::OCCI_SQLT_RSET:
MAPNIK_LOG_WARN(occi) << "occi_datasource: Unsupported datatype "
<< occi_enums::resolve_datatype(type_oid)
<< " (type_oid=" << type_oid << ")";
break;
default:
MAPNIK_LOG_WARN(occi) << "occi_datasource: Unknown datatype "
<< "(type_oid=" << type_oid << ")";
break;
}
}
}
}
catch (SQLException& ex)
{
throw datasource_exception(ex.getMessage());
}
}
is_bound_ = true;
}
pgraster_datasource::pgraster_datasource(parameters const& params)
: datasource(params),
table_(*params.get<std::string>("table", "")),
schema_(""),
raster_table_(*params.get<std::string>("raster_table", "")),
raster_field_(*params.get<std::string>("raster_field", "")),
key_field_(*params.get<std::string>("key_field", "")),
cursor_fetch_size_(*params.get<mapnik::value_integer>("cursor_size", 0)),
row_limit_(*params.get<value_integer>("row_limit", 0)),
type_(datasource::Raster),
srid_(*params.get<value_integer>("srid", 0)),
band_(*params.get<value_integer>("band", 0)),
extent_initialized_(false),
prescale_rasters_(*params.get<mapnik::boolean_type>("prescale_rasters", false)),
use_overviews_(*params.get<mapnik::boolean_type>("use_overviews", false)),
clip_rasters_(*params.get<mapnik::boolean_type>("clip_rasters", false)),
desc_(*params.get<std::string>("type"), "utf-8"),
creator_(params.get<std::string>("host"),
params.get<std::string>("port"),
params.get<std::string>("dbname"),
params.get<std::string>("user"),
params.get<std::string>("password"),
params.get<std::string>("connect_timeout", "4")),
bbox_token_("!bbox!"),
scale_denom_token_("!scale_denominator!"),
pixel_width_token_("!pixel_width!"),
pixel_height_token_("!pixel_height!"),
pool_max_size_(*params_.get<value_integer>("max_size", 10)),
persist_connection_(*params.get<mapnik::boolean_type>("persist_connection", true)),
extent_from_subquery_(*params.get<mapnik::boolean_type>("extent_from_subquery", false)),
estimate_extent_(*params.get<mapnik::boolean_type>("estimate_extent", false)),
max_async_connections_(*params_.get<value_integer>("max_async_connection", 1)),
asynchronous_request_(false),
// params below are for testing purposes only and may be removed at any time
intersect_min_scale_(*params.get<value_integer>("intersect_min_scale", 0)),
intersect_max_scale_(*params.get<value_integer>("intersect_max_scale", 0))
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::init");
#endif
if (table_.empty())
{
throw mapnik::datasource_exception("Pgraster Plugin: missing <table> parameter");
}
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext && !ext->empty())
{
extent_initialized_ = extent_.from_string(*ext);
}
// NOTE: In multithread environment, pool_max_size_ should be
// max_async_connections_ * num_threads
if(max_async_connections_ > 1)
{
if(max_async_connections_ > pool_max_size_)
{
std::ostringstream err;
err << "PostGIS Plugin: Error: 'max_async_connections ("
<< max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")";
throw mapnik::datasource_exception(err.str());
}
asynchronous_request_ = true;
}
boost::optional<value_integer> initial_size = params.get<value_integer>("initial_size", 1);
boost::optional<mapnik::boolean_type> autodetect_key_field = params.get<mapnik::boolean_type>("autodetect_key_field", false);
ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_);
CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id());
if (pool)
{
shared_ptr<Connection> conn = pool->borrowObject();
if (!conn) return;
if (conn->isOK())
{
desc_.set_encoding(conn->client_encoding());
if (raster_table_.empty())
{
raster_table_ = mapnik::sql_utils::table_from_sql(table_);
// non-trivial subqueries (having no FROM) make it
// impossible to use overviews
// TODO: improve "table_from_sql" ?
if ( raster_table_[raster_table_.find_first_not_of(" \t\r\n")] == '(' )
{
raster_table_.clear();
if ( use_overviews_ )
{
std::ostringstream err;
err << "Pgraster Plugin: overviews cannot be used "
"with non-trivial subqueries";
MAPNIK_LOG_WARN(pgraster) << err.str();
use_overviews_ = false;
}
if ( ! extent_from_subquery_ ) {
std::ostringstream err;
err << "Pgraster Plugin: extent can only be computed "
"from subquery as we could not found table source";
MAPNIK_LOG_WARN(pgraster) << err.str();
extent_from_subquery_ = true;
}
}
}
std::string::size_type idx = raster_table_.find_last_of('.');
if (idx != std::string::npos)
{
schema_ = raster_table_.substr(0, idx);
raster_table_ = raster_table_.substr(idx + 1);
}
// If we do not know either the geometry_field or the srid or we
// want to use overviews but do not know about schema, or
// no extent was specified, then attempt to fetch the missing
// information from a raster_columns entry.
//
// This will return no records if we are querying a bogus table returned
// from the simplistic table parsing in table_from_sql() or if
// the table parameter references a table, view, or subselect not
// registered in the geometry columns.
//
geometryColumn_ = mapnik::sql_utils::unquote_double(raster_field_);
if (!raster_table_.empty() && (
geometryColumn_.empty() || srid_ == 0 ||
(schema_.empty() && use_overviews_) ||
! extent_initialized_
))
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::init(get_srid_and_geometry_column)");
#endif
std::ostringstream s;
try
{
s << "SELECT r_raster_column col, srid, r_table_schema";
if ( ! extent_initialized_ ) {
s << ", st_xmin(extent) xmin, st_ymin(extent) ymin"
<< ", st_xmax(extent) xmax, st_ymax(extent) ymax";
}
s << " FROM "
<< RASTER_COLUMNS << " WHERE r_table_name='"
<< mapnik::sql_utils::unquote_double(raster_table_)
<< "'";
if (! schema_.empty())
{
s << " AND r_table_schema='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "'";
}
if (! raster_field_.empty())
{
s << " AND r_raster_column='"
<< mapnik::sql_utils::unquote_double(raster_field_)
<< "'";
}
MAPNIK_LOG_DEBUG(pgraster) <<
"pgraster_datasource: running query " << s.str();
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
geometryColumn_ = rs->getValue("col");
if ( ! extent_initialized_ )
{
double lox, loy, hix, hiy;
if (mapnik::util::string2double(rs->getValue("xmin"), lox) &&
mapnik::util::string2double(rs->getValue("ymin"), loy) &&
mapnik::util::string2double(rs->getValue("xmax"), hix) &&
mapnik::util::string2double(rs->getValue("ymax"), hiy))
{
extent_.init(lox, loy, hix, hiy);
extent_initialized_ = true;
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Layer extent=" << extent_;
}
else
{
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str();
}
}
if (srid_ == 0)
{
const char* srid_c = rs->getValue("srid");
if (srid_c != nullptr)
{
int result = 0;
const char * end = srid_c + std::strlen(srid_c);
if (mapnik::util::string2int(srid_c, end, result))
{
srid_ = result;
}
}
}
if ( schema_.empty() )
{
schema_ = rs->getValue("r_table_schema");
}
}
else
{
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no response from metadata query " << s.str();
}
rs->close();
}
catch (mapnik::datasource_exception const& ex) {
// let this pass on query error and use the fallback below
MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: metadata query failed: " << ex.what();
}
// If we still do not know the srid then we can try to fetch
// it from the 'table_' parameter, which should work even if it is
// a subselect as long as we know the geometry_field to query
if (! geometryColumn_.empty() && srid_ <= 0)
{
s.str("");
s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM "
<< populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
const char* srid_c = rs->getValue("srid");
if (srid_c != nullptr)
{
int result = 0;
const char * end = srid_c + std::strlen(srid_c);
if (mapnik::util::string2int(srid_c, end, result))
{
srid_ = result;
}
}
}
rs->close();
}
}
// If overviews were requested, take note of the max scale
// of each available overview, sorted by scale descending
if ( use_overviews_ )
{
std::ostringstream err;
if ( schema_.empty() )
{
err << "Pgraster Plugin: unable to lookup available table"
<< " overviews due to unknown schema";
throw mapnik::datasource_exception(err.str());
}
if ( geometryColumn_.empty() )
{
err << "Pgraster Plugin: unable to lookup available table"
<< " overviews due to unknown column name";
throw mapnik::datasource_exception(err.str());
}
std::ostringstream s;
s << "select "
"r.r_table_schema sch, "
"r.r_table_name tab, "
"r.r_raster_column col, "
"greatest(abs(r.scale_x), abs(r.scale_y)) scl "
"from"
" raster_overviews o,"
" raster_columns r "
"where"
" o.r_table_schema = '"
<< mapnik::sql_utils::unquote_double(schema_)
<< "' and o.r_table_name = '"
<< mapnik::sql_utils::unquote_double(raster_table_)
<< "' and o.r_raster_column = '"
<< mapnik::sql_utils::unquote_double(geometryColumn_)
<< "' and r.r_table_schema = o.o_table_schema"
" and r.r_table_name = o.o_table_name"
" and r.r_raster_column = o.o_raster_column"
" ORDER BY scl ASC";
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: running query " << s.str();
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
while (rs->next())
{
pgraster_overview ov = pgraster_overview();
ov.schema = rs->getValue("sch");
ov.table = rs->getValue("tab");
ov.column = rs->getValue("col");
ov.scale = atof(rs->getValue("scl"));
if(ov.scale == 0.0f)
{
MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: found invalid overview "
<< ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale;
continue;
}
overviews_.push_back(ov);
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: found overview " << ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale;
}
rs->close();
if ( overviews_.empty() ) {
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no overview found for " << schema_ << "." << raster_table_ << "." << geometryColumn_;
}
}
// detect primary key
if (*autodetect_key_field && key_field_.empty())
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_primary_key)");
#endif
std::ostringstream s;
s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') "
"AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i "
"WHERE a.attnum > 0 AND a.attrelid = c.oid "
"AND a.atttypid = t.oid AND c.relnamespace = n.oid "
"AND c.oid = i.indrelid AND i.indisprimary = 't' "
"AND t.typname !~ '^geom' AND c.relname ="
<< " '" << mapnik::sql_utils::unquote_double(raster_table_) << "' "
//"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1
<< "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum "
"OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum "
"OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum "
"OR i.indkey[9]=a.attnum) ";
if (! schema_.empty())
{
s << "AND n.nspname='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "' ";
}
s << "ORDER BY a.attnum";
shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str());
if (rs_key->next())
{
unsigned int result_rows = rs_key->size();
if (result_rows == 1)
{
bool is_int = (std::string(rs_key->getValue(3)) == "t");
if (is_int)
{
const char* key_field_string = rs_key->getValue(0);
if (key_field_string)
{
key_field_ = std::string(key_field_string);
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: auto-detected key field of '"
<< key_field_ << "' on table '" << raster_table_ << "'";
}
}
else
{
// throw for cases like a numeric primary key, which is invalid
// as it should be floating point (int numerics are useless)
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< rs_key->getValue(0)
<< "' on table '"
<< raster_table_
<< "' is not a valid integer primary key field\n";
throw mapnik::datasource_exception(err.str());
}
}
else if (result_rows > 1)
{
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< "multi column primary key detected but is not supported";
throw mapnik::datasource_exception(err.str());
}
}
rs_key->close();
}
// if a globally unique key field/primary key is required
// but still not known at this point, then throw
if (*autodetect_key_field && key_field_.empty())
{
throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required")
+ " but could not be detected for table '" +
raster_table_ + "', please supply 'key_field' option to specify field to use for primary key");
}
if (srid_ == 0)
{
srid_ = -1;
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Table " << table_ << " is using SRID=" << srid_;
}
// At this point the geometry_field may still not be known
// but we'll catch that where more useful...
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using SRID=" << srid_;
MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using geometry_column=" << geometryColumn_;
// collect attribute desc
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_column_description)");
#endif
std::ostringstream s;
s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0";
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
int count = rs->getNumFields();
bool found_key_field = false;
for (int i = 0; i < count; ++i)
{
std::string fld_name = rs->getFieldName(i);
int type_oid = rs->getTypeOID(i);
// validate type of key_field
if (! found_key_field && ! key_field_.empty() && fld_name == key_field_)
{
if (type_oid == 20 || type_oid == 21 || type_oid == 23)
{
found_key_field = true;
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
}
else
{
std::ostringstream error_s;
error_s << "invalid type '";
std::ostringstream type_s;
type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str());
if (rs_oid->next())
{
error_s << rs_oid->getValue("typname")
<< "' (oid:" << rs_oid->getValue("oid") << ")";
}
else
{
error_s << "oid:" << type_oid << "'";
}
rs_oid->close();
error_s << " for key_field '" << fld_name << "' - "
<< "must be an integer primary key";
rs->close();
throw mapnik::datasource_exception(error_s.str());
}
}
else
{
switch (type_oid)
{
case 16: // bool
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean));
break;
case 20: // int8
case 21: // int2
case 23: // int4
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break;
case 700: // float4
case 701: // float8
case 1700: // numeric
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
break;
case 1042: // bpchar
case 1043: // varchar
case 25: // text
case 705: // literal
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break;
default: // should not get here
#ifdef MAPNIK_LOG
s.str("");
s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str());
if (rs_oid->next())
{
std::string typname(rs_oid->getValue("typname"));
if (typname != "geometry" && typname != "raster")
{
MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type=" << typname
<< " (oid:" << rs_oid->getValue("oid") << ")";
}
}
else
{
MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type_oid=" << type_oid;
}
rs_oid->close();
#endif
break;
}
}
}
rs->close();
}
// Close explicitly the connection so we can 'fork()' without sharing open connections
conn->close();
}
}
sqlite_datasource::sqlite_datasource(parameters const& params)
: datasource(params),
extent_(),
extent_initialized_(false),
type_(datasource::Vector),
table_(*params.get<std::string>("table", "")),
fields_(*params.get<std::string>("fields", "*")),
metadata_(*params.get<std::string>("metadata", "")),
geometry_table_(*params.get<std::string>("geometry_table", "")),
geometry_field_(*params.get<std::string>("geometry_field", "")),
index_table_(*params.get<std::string>("index_table", "")),
key_field_(*params.get<std::string>("key_field", "")),
row_offset_(*params.get<mapnik::value_integer>("row_offset", 0)),
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
intersects_token_("!intersects!"),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8")),
format_(mapnik::wkbAuto)
{
/* TODO
- throw if no primary key but spatial index is present?
- remove auto-indexing
- if spatialite - leverage more of the metadata for geometry type detection
*/
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "sqlite_datasource::init");
#endif
boost::optional<std::string> file = params.get<std::string>("file");
if (! file) throw datasource_exception("Sqlite Plugin: missing <file> parameter");
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
dataset_name_ = *base + "/" + *file;
else
dataset_name_ = *file;
if ((dataset_name_.compare(":memory:") != 0) && (!mapnik::util::exists(dataset_name_)))
{
throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
}
use_spatial_index_ = *params.get<mapnik::boolean>("use_spatial_index", true);
// TODO - remove this option once all datasources have an indexing api
bool auto_index = *params.get<mapnik::boolean>("auto_index", true);
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<std::string> wkb = params.get<std::string>("wkb_format");
if (wkb)
{
if (*wkb == "spatialite")
{
format_ = mapnik::wkbSpatiaLite;
}
else if (*wkb == "generic")
{
format_ = mapnik::wkbGeneric;
}
else
{
format_ = mapnik::wkbAuto;
}
}
// Populate init_statements_
// 1. Build attach database statements from the "attachdb" parameter
// 2. Add explicit init statements from "initdb" parameter
// Note that we do some extra work to make sure that any attached
// databases are relative to directory containing dataset_name_. Sqlite
// will default to attaching from cwd. Typicaly usage means that the
// map loader will produce full paths here.
boost::optional<std::string> attachdb = params.get<std::string>("attachdb");
if (attachdb)
{
parse_attachdb(*attachdb);
}
boost::optional<std::string> initdb = params.get<std::string>("initdb");
if (initdb)
{
init_statements_.push_back(*initdb);
}
// now actually create the connection and start executing setup sql
dataset_ = std::make_shared<sqlite_connection>(dataset_name_);
boost::optional<mapnik::value_integer> table_by_index = params.get<mapnik::value_integer>("table_by_index");
int passed_parameters = 0;
passed_parameters += params.get<std::string>("table") ? 1 : 0;
passed_parameters += table_by_index ? 1 : 0;
if (passed_parameters > 1)
{
throw datasource_exception("SQLite Plugin: you can only select an by name "
"('table' parameter), by number ('table_by_index' parameter), "
"do not supply 2 or more of them at the same time" );
}
if (table_by_index)
{
std::vector<std::string> tables;
sqlite_utils::get_tables(dataset_,tables);
if (*table_by_index < 0 || *table_by_index >= static_cast<int>(tables.size()))
{
std::ostringstream s;
s << "SQLite Plugin: only "
<< tables.size()
<< " table(s) exist, cannot find table by index '" << *table_by_index << "'";
throw datasource_exception(s.str());
}
table_ = tables[*table_by_index];
}
if (table_.empty())
{
throw mapnik::datasource_exception("Sqlite Plugin: missing <table> parameter");
}
if (geometry_table_.empty())
{
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
// if 'table_' is a subquery then we try to deduce names
// and types from the first row returned from that query
using_subquery_ = false;
if (table_ != geometry_table_)
{
using_subquery_ = true;
}
else
{
// attempt to auto-quote table if needed
if (sqlite_utils::needs_quoting(table_))
{
table_ = std::string("[") + table_ + "]";
geometry_table_ = table_;
}
}
// Execute init_statements_
for (std::vector<std::string>::const_iterator iter = init_statements_.begin();
iter != init_statements_.end(); ++iter)
{
MAPNIK_LOG_DEBUG(sqlite) << "sqlite_datasource: Execute init sql=" << *iter;
dataset_->execute(*iter);
}
bool found_types_via_subquery = false;
if (using_subquery_)
{
std::ostringstream s;
std::string query = populate_tokens(table_);
s << "SELECT " << fields_ << " FROM (" << query << ") LIMIT 1";
found_types_via_subquery = sqlite_utils::detect_types_from_subquery(
s.str(),
geometry_field_,
desc_,
dataset_);
}
// TODO - consider removing this
if (key_field_ == "rowid")
{
desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer));
}
bool found_table = sqlite_utils::table_info(key_field_,
found_types_via_subquery,
geometry_field_,
geometry_table_,
desc_,
dataset_);
if (! found_table)
{
std::ostringstream s;
s << "Sqlite Plugin: could not query table '" << geometry_table_ << "'";
if (using_subquery_)
{
s << " from subquery '" << table_ << "'";
}
// report get available tables
std::vector<std::string> tables;
sqlite_utils::get_tables(dataset_,tables);
if (tables.size() > 0)
{
s << " (available tables for " << dataset_name_ << " are: '" << boost::algorithm::join(tables, ", ") << "')";
}
throw datasource_exception(s.str());
}
if (geometry_field_.empty())
{
std::ostringstream s;
s << "Sqlite Plugin: unable to detect the column "
<< "containing a valid geometry on table '" << geometry_table_ << "'. "
<< "Please provide a column name by passing the 'geometry_field' option "
<< "or indicate a different spatial table to use by passing the 'geometry_table' option";
throw datasource_exception(s.str());
}
if (index_table_.empty())
{
// Generate implicit index_table name - need to do this after
// we have discovered meta-data or else we don't know the column name
index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_);
}
std::string index_db = sqlite_utils::index_for_db(dataset_name_);
has_spatial_index_ = false;
if (use_spatial_index_)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(use_spatial_index)");
#endif
if (mapnik::util::exists(index_db))
{
dataset_->execute("attach database '" + index_db + "' as " + index_table_);
}
has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
if (!has_spatial_index_ && auto_index)
{
if (! key_field_.empty())
{
std::ostringstream query;
query << "SELECT "
<< geometry_field_
<< "," << key_field_
<< " FROM ("
<< geometry_table_ << ")";
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(create_spatial_index)");
#endif
std::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str());
if (sqlite_utils::create_spatial_index(index_db,index_table_,rs))
{
//extent_initialized_ = true;
has_spatial_index_ = true;
if (mapnik::util::exists(index_db))
{
dataset_->execute("attach database '" + index_db + "' as " + index_table_);
}
}
}
else
{
std::ostringstream s;
s << "Sqlite Plugin: could not generate spatial index"
<< " for table '" << geometry_table_ << "'"
<< " as no primary key can be detected."
<< " You should either declare an INTEGER PRIMARY KEY"
<< " or set the 'key_field' option to force a"
<< " given field to be used as the primary key";
throw datasource_exception(s.str());
}
}
}
if (! extent_initialized_)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(detect_extent)");
#endif
// TODO - clean this up - reducing arguments
std::string query = populate_tokens(table_);
if (!sqlite_utils::detect_extent(dataset_,
has_spatial_index_,
extent_,
index_table_,
metadata_,
geometry_field_,
geometry_table_,
key_field_,
query))
{
std::ostringstream s;
s << "Sqlite Plugin: extent could not be determined for table '"
<< geometry_table_ << "' and geometry field '" << geometry_field_ << "'"
<< " because an rtree spatial index is missing or empty."
<< " - either set the table 'extent' or create an rtree spatial index";
throw datasource_exception(s.str());
}
}
}
geos_datasource::geos_datasource(parameters const& params)
: datasource(params),
extent_(),
extent_initialized_(false),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8")),
geometry_data_(""),
geometry_data_name_("name"),
geometry_id_(1)
{
boost::optional<std::string> geometry = params.get<std::string>("wkt");
if (! geometry) throw datasource_exception("missing <wkt> parameter");
geometry_string_ = *geometry;
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<int> id = params.get<int>("gid");
if (id) geometry_id_ = *id;
boost::optional<std::string> gdata = params.get<std::string>("field_data");
if (gdata) geometry_data_ = *gdata;
boost::optional<std::string> gdata_name = params.get<std::string>("field_name");
if (gdata_name) geometry_data_name_ = *gdata_name;
desc_.add_descriptor(attribute_descriptor(geometry_data_name_, mapnik::String));
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "geos_datasource::init");
#endif
// open geos driver
initGEOS(geos_notice, geos_error);
// parse the string into geometry
geometry_.set_feature(GEOSGeomFromWKT(geometry_string_.c_str()));
if (*geometry_ == NULL || ! GEOSisValid(*geometry_))
{
throw datasource_exception("GEOS Plugin: invalid <wkt> geometry specified");
}
// try to obtain the extent from the geometry itself
if (! extent_initialized_)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "geos_datasource::init(initialize_extent)");
#endif
MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Initializing extent from geometry";
if (GEOSGeomTypeId(*geometry_) == GEOS_POINT)
{
double x, y;
unsigned int size;
const GEOSCoordSequence* cs = GEOSGeom_getCoordSeq(*geometry_);
GEOSCoordSeq_getSize(cs, &size);
GEOSCoordSeq_getX(cs, 0, &x);
GEOSCoordSeq_getY(cs, 0, &y);
extent_.init(x, y, x, y);
extent_initialized_ = true;
}
else
{
geos_feature_ptr envelope (GEOSEnvelope(*geometry_));
if (*envelope != NULL && GEOSisValid(*envelope))
{
#ifdef MAPNIK_LOG
char* wkt = GEOSGeomToWKT(*envelope);
MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Getting coord sequence from=" << wkt;
GEOSFree(wkt);
#endif
const GEOSGeometry* exterior = GEOSGetExteriorRing(*envelope);
if (exterior != NULL && GEOSisValid(exterior))
{
const GEOSCoordSequence* cs = GEOSGeom_getCoordSeq(exterior);
if (cs != NULL)
{
MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Iterating boundary points";
double x, y;
double minx = std::numeric_limits<float>::max(),
miny = std::numeric_limits<float>::max(),
maxx = -std::numeric_limits<float>::max(),
maxy = -std::numeric_limits<float>::max();
unsigned int num_points;
GEOSCoordSeq_getSize(cs, &num_points);
for (unsigned int i = 0; i < num_points; ++i)
{
GEOSCoordSeq_getX(cs, i, &x);
GEOSCoordSeq_getY(cs, i, &y);
if (x < minx) minx = x;
if (x > maxx) maxx = x;
if (y < miny) miny = y;
if (y > maxy) maxy = y;
}
extent_.init(minx, miny, maxx, maxy);
extent_initialized_ = true;
}
}
}
}
}
if (! extent_initialized_)
{
throw datasource_exception("GEOS Plugin: cannot determine extent for <wkt> geometry");
}
}
void shape_datasource::bind() const
{
if (is_bound_) return;
if (!boost::filesystem::exists(shape_name_ + ".shp"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' does not exist");
}
if (boost::filesystem::is_directory(shape_name_ + ".shp"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' appears to be a directory not a file");
}
if (!boost::filesystem::exists(shape_name_ + ".dbf"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".dbf' does not exist");
}
try
{
boost::shared_ptr<shape_io> shape_ref = boost::make_shared<shape_io>(shape_name_);
init(*shape_ref);
for (int i=0;i<shape_ref->dbf().num_fields();++i)
{
field_descriptor const& fd=shape_ref->dbf().descriptor(i);
std::string fld_name=fd.name_;
switch (fd.type_)
{
case 'C': // character
case 'D': // Date
case 'M': // Memo, a string
case 'L': // logical
case '@': // timestamp
desc_.add_descriptor(attribute_descriptor(fld_name, String));
break;
case 'N':
case 'O': // double
case 'F': // float
{
if (fd.dec_>0)
{
desc_.add_descriptor(attribute_descriptor(fld_name,Double,false,8));
}
else
{
desc_.add_descriptor(attribute_descriptor(fld_name,Integer,false,4));
}
break;
}
default:
#ifdef MAPNIK_DEBUG
// I - long
// G - ole
// + - autoincrement
std::clog << "Shape Plugin: unknown type " << fd.type_ << std::endl;
#endif
break;
}
}
// for indexed shapefiles we keep open the file descriptor for fast reads
if (indexed_) {
shape_ = shape_ref;
}
}
catch (const datasource_exception& ex)
{
std::clog << "Shape Plugin: error processing field attributes, " << ex.what() << std::endl;
throw;
}
catch (const std::exception& ex)
{
std::clog << "Shape Plugin: error processing field attributes, " << ex.what() << std::endl;
throw;
}
catch (...) // exception: pipe_select_interrupter: Too many open files
{
std::clog << "Shape Plugin: error processing field attributes" << std::endl;
throw;
}
is_bound_ = true;
}
void sqlite_datasource::bind() const
{
if (is_bound_) return;
if (!boost::filesystem::exists(dataset_name_))
throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
dataset_ = new sqlite_connection (dataset_name_);
std::string table_name = mapnik::table_from_sql(table_);
if (metadata_ != "" && ! extent_initialized_)
{
std::ostringstream s;
s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_;
s << " WHERE LOWER(f_table_name) = LOWER('" << table_name << "')";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent_.init (xmin,ymin,xmax,ymax);
extent_initialized_ = true;
}
}
if (use_spatial_index_)
{
std::ostringstream s;
s << "SELECT COUNT (*) FROM sqlite_master";
s << " WHERE LOWER(name) = LOWER('idx_" << table_name << "_" << geometry_field_ << "')";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
use_spatial_index_ = rs->column_integer (0) == 1;
}
#ifdef MAPNIK_DEBUG
if (! use_spatial_index_)
std::clog << "Sqlite Plugin: cannot use the spatial index " << std::endl;
#endif
}
{
/*
XXX - This is problematic, if we do not have at least a row,
we cannot determine the right columns types and names
as all column_type are SQLITE_NULL
*/
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_name << ") LIMIT 1";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
for (int i = 0; i < rs->column_count (); ++i)
{
const int type_oid = rs->column_type (i);
const char* fld_name = rs->column_name (i);
switch (type_oid)
{
case SQLITE_INTEGER:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case SQLITE_FLOAT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case SQLITE_TEXT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case SQLITE_NULL:
case SQLITE_BLOB:
break;
default:
#ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl;
#endif
break;
}
}
}
}
is_bound_ = true;
}
shape_datasource::shape_datasource(parameters const& params)
: datasource (params),
type_(datasource::Vector),
file_length_(0),
indexed_(false),
row_limit_(*params.get<mapnik::value_integer>("row_limit",0)),
desc_(shape_datasource::name(), *params.get<std::string>("encoding","utf-8"))
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "shape_datasource::init");
#endif
boost::optional<std::string> file = params.get<std::string>("file");
if (!file) throw datasource_exception("Shape Plugin: missing <file> parameter");
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
shape_name_ = *base + "/" + *file;
else
shape_name_ = *file;
boost::algorithm::ireplace_last(shape_name_,".shp","");
if (!mapnik::util::exists(shape_name_ + ".shp"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' does not exist");
}
if (mapnik::util::is_directory(shape_name_ + ".shp"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' appears to be a directory not a file");
}
if (!mapnik::util::exists(shape_name_ + ".dbf"))
{
throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".dbf' does not exist");
}
try
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "shape_datasource::init(get_column_description)");
#endif
std::unique_ptr<shape_io> shape_ref = std::make_unique<shape_io>(shape_name_);
init(*shape_ref);
for (int i=0;i<shape_ref->dbf().num_fields();++i)
{
field_descriptor const& fd=shape_ref->dbf().descriptor(i);
std::string fld_name=fd.name_;
switch (fd.type_)
{
case 'C': // character
case 'D': // date
desc_.add_descriptor(attribute_descriptor(fld_name, String));
break;
case 'L': // logical
desc_.add_descriptor(attribute_descriptor(fld_name, Boolean));
break;
case 'N': // numeric
case 'O': // double
case 'F': // float
{
if (fd.dec_>0)
{
desc_.add_descriptor(attribute_descriptor(fld_name,Double,false,8));
}
else
{
desc_.add_descriptor(attribute_descriptor(fld_name,Integer,false,4));
}
break;
}
default:
// I - long
// G - ole
// + - autoincrement
// @ - timestamp
// B - binary
// l - long
// M - memo
MAPNIK_LOG_ERROR(shape) << "shape_datasource: Unknown type=" << fd.type_;
break;
}
}
}
catch (datasource_exception const& ex)
{
MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes, " << ex.what();
throw;
}
catch (const std::exception& ex)
{
MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes, " << ex.what();
throw;
}
catch (...) // exception: pipe_select_interrupter: Too many open files
{
MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes";
throw;
}
}
void ogr_datasource::init(mapnik::parameters const& params)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "ogr_datasource::init");
#endif
boost::optional<std::string> file = params.get<std::string>("file");
boost::optional<std::string> string = params.get<std::string>("string");
if (!string) string = params.get<std::string>("inline");
if (! file && ! string)
{
throw datasource_exception("missing <file> or <string> parameter");
}
if (string)
{
dataset_name_ = *string;
}
else
{
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
{
dataset_name_ = *base + "/" + *file;
}
else
{
dataset_name_ = *file;
}
}
std::string driver = *params.get<std::string>("driver","");
if (! driver.empty())
{
#if GDAL_VERSION_MAJOR >= 2
unsigned int nOpenFlags = GDAL_OF_READONLY | GDAL_OF_VECTOR;
const char* papszAllowedDrivers[] = { driver.c_str(), nullptr };
dataset_ = reinterpret_cast<gdal_dataset_type>(GDALOpenEx(dataset_name_.c_str(),nOpenFlags,papszAllowedDrivers, nullptr, nullptr));
#else
OGRSFDriver * ogr_driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver.c_str());
if (ogr_driver && ogr_driver != nullptr)
{
dataset_ = ogr_driver->Open((dataset_name_).c_str(), false);
}
#endif
}
else
{
// open ogr driver
#if GDAL_VERSION_MAJOR >= 2
dataset_ = reinterpret_cast<gdal_dataset_type>(OGROpen(dataset_name_.c_str(), false, nullptr));
#else
dataset_ = OGRSFDriverRegistrar::Open(dataset_name_.c_str(), false);
#endif
}
if (! dataset_)
{
const std::string err = CPLGetLastErrorMsg();
if (err.size() == 0)
{
throw datasource_exception("OGR Plugin: connection failed: " + dataset_name_ + " was not found or is not a supported format");
}
else
{
throw datasource_exception("OGR Plugin: " + err);
}
}
// initialize layer
boost::optional<std::string> layer_by_name = params.get<std::string>("layer");
boost::optional<mapnik::value_integer> layer_by_index = params.get<mapnik::value_integer>("layer_by_index");
boost::optional<std::string> layer_by_sql = params.get<std::string>("layer_by_sql");
int passed_parameters = 0;
passed_parameters += layer_by_name ? 1 : 0;
passed_parameters += layer_by_index ? 1 : 0;
passed_parameters += layer_by_sql ? 1 : 0;
if (passed_parameters > 1)
{
throw datasource_exception("OGR Plugin: you can only select an ogr layer by name "
"('layer' parameter), by number ('layer_by_index' parameter), "
"or by sql ('layer_by_sql' parameter), "
"do not supply 2 or more of them at the same time" );
}
if (layer_by_name)
{
layer_name_ = *layer_by_name;
layer_.layer_by_name(dataset_, layer_name_);
}
else if (layer_by_index)
{
int num_layers = dataset_->GetLayerCount();
if (*layer_by_index >= num_layers)
{
std::ostringstream s;
s << "OGR Plugin: only " << num_layers << " layer(s) exist, cannot find layer by index '" << *layer_by_index << "'";
throw datasource_exception(s.str());
}
layer_.layer_by_index(dataset_, *layer_by_index);
layer_name_ = layer_.layer_name();
}
else if (layer_by_sql)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats_sql__(std::clog, "ogr_datasource::init(layer_by_sql)");
#endif
layer_.layer_by_sql(dataset_, *layer_by_sql);
layer_name_ = layer_.layer_name();
}
else
{
std::string s("OGR Plugin: missing <layer> or <layer_by_index> or <layer_by_sql> parameter, available layers are: ");
unsigned num_layers = dataset_->GetLayerCount();
bool layer_found = false;
std::vector<std::string> layer_names;
for (unsigned i = 0; i < num_layers; ++i )
{
OGRLayer* ogr_layer = dataset_->GetLayer(i);
OGRFeatureDefn* ogr_layer_def = ogr_layer->GetLayerDefn();
if (ogr_layer_def != 0)
{
layer_found = true;
layer_names.push_back(std::string("'") + ogr_layer_def->GetName() + std::string("'"));
}
}
if (! layer_found)
{
s += "None (no layers were found in dataset)";
}
else
{
s += boost::algorithm::join(layer_names,", ");
}
throw datasource_exception(s);
}
if (! layer_.is_valid())
{
std::ostringstream s;
s << "OGR Plugin: ";
if (layer_by_name)
{
s << "cannot find layer by name '" << *layer_by_name;
}
else if (layer_by_index)
{
s << "cannot find layer by index number '" << *layer_by_index;
}
else if (layer_by_sql)
{
s << "cannot find layer by sql query '" << *layer_by_sql;
}
s << "' in dataset '" << dataset_name_ << "'";
throw datasource_exception(s.str());
}
// work with real OGR layer
OGRLayer* layer = layer_.layer();
// initialize envelope
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext && !ext->empty())
{
extent_.from_string(*ext);
}
else
{
OGREnvelope envelope;
OGRErr e = layer->GetExtent(&envelope);
if (e == OGRERR_FAILURE)
{
if (layer->GetFeatureCount() == 0)
{
MAPNIK_LOG_ERROR(ogr) << "could not determine extent, layer '" << layer->GetLayerDefn()->GetName() << "' appears to have no features";
}
else
{
std::ostringstream s;
s << "OGR Plugin: Cannot determine extent for layer '" << layer->GetLayerDefn()->GetName() << "'. Please provide a manual extent string (minx,miny,maxx,maxy).";
throw datasource_exception(s.str());
}
}
extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY);
}
// scan for index file
// TODO - layer names don't match dataset name, so this will break for
// any layer types of ogr than shapefiles, etc
// fix here and in ogrindex
size_t breakpoint = dataset_name_.find_last_of(".");
if (breakpoint == std::string::npos)
{
breakpoint = dataset_name_.length();
}
index_name_ = dataset_name_.substr(0, breakpoint) + ".ogrindex";
#if defined (_WINDOWS)
std::ifstream index_file(mapnik::utf8_to_utf16(index_name_), std::ios::in | std::ios::binary);
#else
std::ifstream index_file(index_name_.c_str(), std::ios::in | std::ios::binary);
#endif
if (index_file)
{
indexed_ = true;
index_file.close();
}
#if 0
// TODO - enable this warning once the ogrindex tool is a bit more stable/mature
else
{
MAPNIK_LOG_DEBUG(ogr) << "ogr_datasource: no ogrindex file found for " << dataset_name_
<< ", use the 'ogrindex' program to build an index for faster rendering";
}
#endif
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "ogr_datasource::init(get_column_description)");
#endif
// deal with attributes descriptions
OGRFeatureDefn* def = layer->GetLayerDefn();
if (def != 0)
{
const int fld_count = def->GetFieldCount();
for (int i = 0; i < fld_count; i++)
{
OGRFieldDefn* fld = def->GetFieldDefn(i);
const std::string fld_name = fld->GetNameRef();
const OGRFieldType type_oid = fld->GetType();
switch (type_oid)
{
case OFTInteger:
#if GDAL_VERSION_MAJOR >= 2
case OFTInteger64:
#endif
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break;
case OFTReal:
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
break;
case OFTString:
case OFTWideString: // deprecated
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break;
case OFTBinary:
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object));
break;
case OFTIntegerList:
#if GDAL_VERSION_MAJOR >= 2
case OFTInteger64List:
#endif
case OFTRealList:
case OFTStringList:
case OFTWideStringList: // deprecated !
MAPNIK_LOG_WARN(ogr) << "ogr_datasource: Unhandled type_oid=" << type_oid;
break;
case OFTDate:
case OFTTime:
case OFTDateTime: // unhandled !
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object));
MAPNIK_LOG_WARN(ogr) << "ogr_datasource: Unhandled type_oid=" << type_oid;
break;
}
}
}
mapnik::parameters & extra_params = desc_.get_extra_parameters();
OGRSpatialReference * srs_ref = layer->GetSpatialRef();
char * srs_output = nullptr;
if (srs_ref && srs_ref->exportToProj4( &srs_output ) == OGRERR_NONE ) {
extra_params["proj4"] = mapnik::util::trim_copy(srs_output);
}
CPLFree(srs_output);
}
void postgis_datasource::bind() const
{
if (is_bound_) return;
boost::optional<int> initial_size = params_.get<int>("initial_size",1);
boost::optional<int> max_size = params_.get<int>("max_size",10);
ConnectionManager *mgr=ConnectionManager::instance();
mgr->registerPool(creator_, *initial_size, *max_size);
shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
if (pool)
{
shared_ptr<Connection> conn = pool->borrowObject();
if (conn && conn->isOK())
{
is_bound_ = true;
PoolGuard<shared_ptr<Connection>,
shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
desc_.set_encoding(conn->client_encoding());
if(geometry_table_.empty())
{
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
std::string::size_type idx = geometry_table_.find_last_of('.');
if (idx!=std::string::npos)
{
schema_ = geometry_table_.substr(0,idx);
geometry_table_ = geometry_table_.substr(idx+1);
}
else
{
geometry_table_ = geometry_table_.substr(0);
}
// If we do not know both the geometry_field and the srid
// then first attempt to fetch the geometry name from a geometry_columns entry.
// This will return no records if we are querying a bogus table returned
// from the simplistic table parsing in table_from_sql() or if
// the table parameter references a table, view, or subselect not
// registered in the geometry columns.
geometryColumn_ = geometry_field_;
if (!geometryColumn_.length() > 0 || srid_ == 0)
{
std::ostringstream s;
s << "SELECT f_geometry_column, srid FROM ";
s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) <<"'";
if (schema_.length() > 0)
s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'";
if (geometry_field_.length() > 0)
s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'";
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
if (rs->next())
{
geometryColumn_ = rs->getValue("f_geometry_column");
if (srid_ == 0)
{
try
{
srid_ = lexical_cast<int>(rs->getValue("srid"));
}
catch (bad_lexical_cast &ex)
{
std::clog << "Postgis Plugin: SRID=" << rs->getValue("srid") << " " << ex.what() << std::endl;
}
}
}
rs->close();
// If we still do not know the srid then we can try to fetch
// it from the 'table_' parameter, which should work even if it is
// a subselect as long as we know the geometry_field to query
if (geometryColumn_.length() && srid_ <= 0)
{
s.str("");
s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM ";
s << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
if (rs->next())
{
try
{
srid_ = lexical_cast<int>(rs->getValue("srid"));
}
catch (bad_lexical_cast &ex)
{
std::clog << "Postgis Plugin: SRID=" << rs->getValue("srid") << " " << ex.what() << std::endl;
}
}
rs->close();
}
}
if (srid_ == 0)
{
srid_ = -1;
std::clog << "Postgis Plugin: SRID warning, using srid=-1 for '" << table_ << "'" << std::endl;
}
// At this point the geometry_field may still not be known
// but we'll catch that where more useful...
#ifdef MAPNIK_DEBUG
std::clog << "Postgis Plugin: using SRID=" << srid_ << std::endl;
std::clog << "Postgis Plugin: using geometry_column=" << geometryColumn_ << std::endl;
#endif
// collect attribute desc
std::ostringstream s;
s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0";
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
int count = rs->getNumFields();
bool found_key_field = false;
for (int i=0;i<count;++i)
{
std::string fld_name=rs->getFieldName(i);
int type_oid = rs->getTypeOID(i);
// validate type of key_field
if (!found_key_field && !key_field_.empty() && fld_name == key_field_)
{
found_key_field = true;
if (type_oid == 20 || type_oid == 21 || type_oid == 23)
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
}
else
{
std::ostringstream error_s;
error_s << "invalid type '";
std::ostringstream type_s;
type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str());
if (rs_oid->next())
{
error_s << rs_oid->getValue("typname")
<< "' (oid:" << rs_oid->getValue("oid") << ")";
}
else
{
error_s << "oid:" << type_oid << "'";
}
rs_oid->close();
error_s << " for key_field '" << fld_name << "' - "
<< "must be an integer primary key";
rs->close();
throw mapnik::datasource_exception( error_s.str() );
}
}
else
{
switch (type_oid)
{
case 16: // bool
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Boolean));
break;
case 20: // int8
case 21: // int2
case 23: // int4
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case 700: // float4
case 701: // float8
case 1700: // numeric ??
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
case 1042: // bpchar
case 1043: // varchar
case 25: // text
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
default: // should not get here
#ifdef MAPNIK_DEBUG
s.str("");
s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str());
if (rs_oid->next())
{
std::clog << "Postgis Plugin: unknown type = " << rs_oid->getValue("typname")
<< " (oid:" << rs_oid->getValue("oid") << ")" << std::endl;
}
else
{
std::clog << "Postgis Plugin: unknown oid type =" << type_oid << std::endl;
}
rs_oid->close();
#endif
break;
}
}
}
rs->close();
}
}
}
void postgis_datasource::bind() const
{
if (is_bound_)
{
return;
}
boost::optional<int> initial_size = params_.get<int>("initial_size", 1);
boost::optional<int> max_size = params_.get<int>("max_size", 10);
boost::optional<mapnik::boolean> autodetect_key_field = params_.get<mapnik::boolean>("autodetect_key_field", false);
ConnectionManager* mgr = ConnectionManager::instance();
mgr->registerPool(creator_, *initial_size, *max_size);
shared_ptr< Pool<Connection,ConnectionCreator> > pool = mgr->getPool(creator_.id());
if (pool)
{
shared_ptr<Connection> conn = pool->borrowObject();
if (conn && conn->isOK())
{
PoolGuard<shared_ptr<Connection>,
shared_ptr< Pool<Connection,ConnectionCreator> > > guard(conn, pool);
desc_.set_encoding(conn->client_encoding());
if (geometry_table_.empty())
{
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
std::string::size_type idx = geometry_table_.find_last_of('.');
if (idx != std::string::npos)
{
schema_ = geometry_table_.substr(0, idx);
geometry_table_ = geometry_table_.substr(idx + 1);
}
else
{
geometry_table_ = geometry_table_.substr(0);
}
// If we do not know both the geometry_field and the srid
// then first attempt to fetch the geometry name from a geometry_columns entry.
// This will return no records if we are querying a bogus table returned
// from the simplistic table parsing in table_from_sql() or if
// the table parameter references a table, view, or subselect not
// registered in the geometry columns.
geometryColumn_ = geometry_field_;
if (geometryColumn_.empty() || srid_ == 0)
{
std::ostringstream s;
s << "SELECT f_geometry_column, srid FROM "
<< GEOMETRY_COLUMNS <<" WHERE f_table_name='"
<< mapnik::sql_utils::unquote_double(geometry_table_)
<< "'";
if (! schema_.empty())
{
s << " AND f_table_schema='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "'";
}
if (! geometry_field_.empty())
{
s << " AND f_geometry_column='"
<< mapnik::sql_utils::unquote_double(geometry_field_)
<< "'";
}
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
geometryColumn_ = rs->getValue("f_geometry_column");
if (srid_ == 0)
{
const char* srid_c = rs->getValue("srid");
if (srid_c != NULL)
{
int result = 0;
if (mapnik::util::string2int(srid_c, result))
{
srid_ = result;
}
}
}
}
rs->close();
// If we still do not know the srid then we can try to fetch
// it from the 'table_' parameter, which should work even if it is
// a subselect as long as we know the geometry_field to query
if (! geometryColumn_.empty() && srid_ <= 0)
{
s.str("");
s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM "
<< populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
const char* srid_c = rs->getValue("srid");
if (srid_c != NULL)
{
int result = 0;
if (mapnik::util::string2int(srid_c, result))
{
srid_ = result;
}
}
}
rs->close();
}
}
// detect primary key
if (*autodetect_key_field && key_field_.empty())
{
std::ostringstream s;
s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') "
"AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i "
"WHERE a.attnum > 0 AND a.attrelid = c.oid "
"AND a.atttypid = t.oid AND c.relnamespace = n.oid "
"AND c.oid = i.indrelid AND i.indisprimary = 't' "
"AND t.typname !~ '^geom' AND c.relname ="
<< " '" << mapnik::sql_utils::unquote_double(geometry_table_) << "' "
//"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1
<< "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum "
"OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum "
"OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum "
"OR i.indkey[9]=a.attnum) ";
if (! schema_.empty())
{
s << "AND n.nspname='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "' ";
}
s << "ORDER BY a.attnum";
shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str());
if (rs_key->next())
{
unsigned int result_rows = rs_key->size();
if (result_rows == 1)
{
bool is_int = (std::string(rs_key->getValue(3)) == "t");
if (is_int)
{
const char* key_field_string = rs_key->getValue(0);
if (key_field_string)
{
key_field_ = std::string(key_field_string);
#ifdef MAPNIK_DEBUG
std::clog << "PostGIS Plugin: auto-detected key field of '"
<< key_field_ << "' on table '"
<< geometry_table_ << "'\n";
#endif
}
}
else
{
// throw for cases like a numeric primary key, which is invalid
// as it should be floating point (int numerics are useless)
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< rs_key->getValue(0)
<< "' on table '"
<< geometry_table_
<< "' is not a valid integer primary key field\n";
throw mapnik::datasource_exception(err.str());
}
}
else if (result_rows > 1)
{
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< "multi column primary key detected but is not supported";
throw mapnik::datasource_exception(err.str());
}
}
rs_key->close();
}
// if a globally unique key field/primary key is required
// but still not known at this point, then throw
if (*autodetect_key_field && key_field_.empty())
{
throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required")
+ " but could not be detected for table '" +
geometry_table_ + "', please supply 'key_field' option to specify field to use for primary key");
}
if (srid_ == 0)
{
srid_ = -1;
#ifdef MAPNIK_DEBUG
std::clog << "Postgis Plugin: SRID warning, using srid=-1 for '" << table_ << "'" << std::endl;
#endif
}
// At this point the geometry_field may still not be known
// but we'll catch that where more useful...
#ifdef MAPNIK_DEBUG
std::clog << "Postgis Plugin: using SRID=" << srid_ << std::endl;
std::clog << "Postgis Plugin: using geometry_column=" << geometryColumn_ << std::endl;
#endif
// collect attribute desc
std::ostringstream s;
s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0";
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
int count = rs->getNumFields();
bool found_key_field = false;
for (int i = 0; i < count; ++i)
{
std::string fld_name = rs->getFieldName(i);
int type_oid = rs->getTypeOID(i);
// validate type of key_field
if (! found_key_field && ! key_field_.empty() && fld_name == key_field_)
{
if (type_oid == 20 || type_oid == 21 || type_oid == 23)
{
found_key_field = true;
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
}
else
{
std::ostringstream error_s;
error_s << "invalid type '";
std::ostringstream type_s;
type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str());
if (rs_oid->next())
{
error_s << rs_oid->getValue("typname")
<< "' (oid:" << rs_oid->getValue("oid") << ")";
}
else
{
error_s << "oid:" << type_oid << "'";
}
rs_oid->close();
error_s << " for key_field '" << fld_name << "' - "
<< "must be an integer primary key";
rs->close();
throw mapnik::datasource_exception(error_s.str());
}
}
else
{
switch (type_oid)
{
case 16: // bool
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean));
break;
case 20: // int8
case 21: // int2
case 23: // int4
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break;
case 700: // float4
case 701: // float8
case 1700: // numeric
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
case 1042: // bpchar
case 1043: // varchar
case 25: // text
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break;
default: // should not get here
#ifdef MAPNIK_DEBUG
s.str("");
s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
/*
if (show_queries_)
{
std::clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}
*/
shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str());
if (rs_oid->next())
{
std::clog << "Postgis Plugin: unknown type = " << rs_oid->getValue("typname")
<< " (oid:" << rs_oid->getValue("oid") << ")" << std::endl;
}
else
{
std::clog << "Postgis Plugin: unknown oid type =" << type_oid << std::endl;
}
rs_oid->close();
#endif
break;
}
}
}
rs->close();
is_bound_ = true;
}
}
}
ogr_datasource::ogr_datasource(parameters const& params)
: datasource(params),
extent_(),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")),
indexed_(false)
{
OGRRegisterAll();
boost::optional<std::string> file = params.get<std::string>("file");
if (!file) throw datasource_exception("missing <file> parameter");
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false);
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
dataset_name_ = *base + "/" + *file;
else
dataset_name_ = *file;
// open ogr driver
dataset_ = OGRSFDriverRegistrar::Open ((dataset_name_).c_str(), FALSE);
if (!dataset_)
{
std::string err = CPLGetLastErrorMsg();
if( err.size() == 0 )
{
throw datasource_exception("Connection failed: " + dataset_name_ + " was not found or is not a supported format");
} else {
throw datasource_exception(err);
}
}
// initialize layer
boost::optional<std::string> layer = params.get<std::string>("layer");
if (!layer)
{
std::string s ("missing <layer> parameter, available layers are: ");
unsigned num_layers = dataset_->GetLayerCount();
for (unsigned i = 0; i < num_layers; ++i )
{
OGRLayer *ogr_layer = dataset_->GetLayer(i);
OGRFeatureDefn* def = ogr_layer->GetLayerDefn();
if (def != 0) {
s += " '";
s += def->GetName();
s += "' ";
} else {
s += "No layers found!";
}
}
throw datasource_exception(s);
}
layerName_ = *layer;
layer_ = dataset_->GetLayerByName (layerName_.c_str());
if (! layer_) throw datasource_exception("cannot find <layer> in dataset");
// initialize envelope
OGREnvelope envelope;
layer_->GetExtent (&envelope);
extent_.init (envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY);
// scan for index file
size_t breakpoint = dataset_name_.find_last_of (".");
if (breakpoint == std::string::npos) breakpoint = dataset_name_.length();
index_name_ = dataset_name_.substr(0, breakpoint) + ".index";
std::ifstream index_file (index_name_.c_str(), std::ios::in | std::ios::binary);
if (index_file)
{
indexed_=true;
index_file.close();
}
// deal with attributes descriptions
OGRFeatureDefn* def = layer_->GetLayerDefn ();
if (def != 0)
{
int fld_count = def->GetFieldCount ();
for (int i = 0; i < fld_count; i++)
{
OGRFieldDefn* fld = def->GetFieldDefn (i);
std::string fld_name = fld->GetNameRef ();
OGRFieldType type_oid = fld->GetType ();
switch (type_oid)
{
case OFTInteger:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case OFTReal:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case OFTString:
case OFTWideString: // deprecated
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case OFTBinary:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object));
break;
case OFTIntegerList:
case OFTRealList:
case OFTStringList:
case OFTWideStringList: // deprecated !
#ifdef MAPNIK_DEBUG
clog << "unhandled type_oid=" << type_oid << endl;
#endif
break;
case OFTDate:
case OFTTime:
case OFTDateTime: // unhandled !
#ifdef MAPNIK_DEBUG
clog << "unhandled type_oid=" << type_oid << endl;
#endif
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object));
break;
default: // unknown
#ifdef MAPNIK_DEBUG
clog << "unknown type_oid=" << type_oid << endl;
#endif
break;
}
}
}
}
void ogr_datasource::bind() const
{
if (is_bound_) return;
// initialize ogr formats
OGRRegisterAll();
std::string driver = *params_.get<std::string>("driver","");
if (! driver.empty())
{
OGRSFDriver * ogr_driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver.c_str());
if (ogr_driver && ogr_driver != NULL)
{
dataset_ = ogr_driver->Open((dataset_name_).c_str(), FALSE);
}
}
else
{
// open ogr driver
dataset_ = OGRSFDriverRegistrar::Open((dataset_name_).c_str(), FALSE);
}
if (! dataset_)
{
const std::string err = CPLGetLastErrorMsg();
if (err.size() == 0)
{
throw datasource_exception("OGR Plugin: connection failed: " + dataset_name_ + " was not found or is not a supported format");
}
else
{
throw datasource_exception("OGR Plugin: " + err);
}
}
// initialize layer
boost::optional<std::string> layer_by_name = params_.get<std::string>("layer");
boost::optional<unsigned> layer_by_index = params_.get<unsigned>("layer_by_index");
boost::optional<std::string> layer_by_sql = params_.get<std::string>("layer_by_sql");
int passed_parameters = 0;
passed_parameters += layer_by_name ? 1 : 0;
passed_parameters += layer_by_index ? 1 : 0;
passed_parameters += layer_by_sql ? 1 : 0;
if (passed_parameters > 1)
{
throw datasource_exception("OGR Plugin: you can only select an ogr layer by name "
"('layer' parameter), by number ('layer_by_index' parameter), "
"or by sql ('layer_by_sql' parameter), "
"do not supply 2 or more of them at the same time" );
}
if (layer_by_name)
{
layer_name_ = *layer_by_name;
layer_.layer_by_name(dataset_, layer_name_);
}
else if (layer_by_index)
{
const unsigned num_layers = dataset_->GetLayerCount();
if (*layer_by_index >= num_layers)
{
std::ostringstream s;
s << "OGR Plugin: only ";
s << num_layers;
s << " layer(s) exist, cannot find layer by index '" << *layer_by_index << "'";
throw datasource_exception(s.str());
}
layer_.layer_by_index(dataset_, *layer_by_index);
layer_name_ = layer_.layer_name();
}
else if (layer_by_sql)
{
layer_.layer_by_sql(dataset_, *layer_by_sql);
layer_name_ = layer_.layer_name();
}
else
{
std::ostringstream s;
s << "OGR Plugin: missing <layer> or <layer_by_index> or <layer_by_sql> "
<< "parameter, available layers are: ";
unsigned num_layers = dataset_->GetLayerCount();
bool layer_found = false;
for (unsigned i = 0; i < num_layers; ++i )
{
OGRLayer* ogr_layer = dataset_->GetLayer(i);
OGRFeatureDefn* ogr_layer_def = ogr_layer->GetLayerDefn();
if (ogr_layer_def != 0)
{
layer_found = true;
s << " '" << ogr_layer_def->GetName() << "' ";
}
}
if (! layer_found)
{
s << "None (no layers were found in dataset)";
}
throw datasource_exception(s.str());
}
if (! layer_.is_valid())
{
std::string s("OGR Plugin: ");
if (layer_by_name)
{
s += "cannot find layer by name '" + *layer_by_name;
}
else if (layer_by_index)
{
s += "cannot find layer by index number '" + *layer_by_index;
}
else if (layer_by_sql)
{
s += "cannot find layer by sql query '" + *layer_by_sql;
}
s += "' in dataset '" + dataset_name_ + "'";
throw datasource_exception(s);
}
// work with real OGR layer
OGRLayer* layer = layer_.layer();
// initialize envelope
OGREnvelope envelope;
layer->GetExtent(&envelope);
extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY);
// scan for index file
// TODO - layer names don't match dataset name, so this will break for
// any layer types of ogr than shapefiles, etc
// fix here and in ogrindex
size_t breakpoint = dataset_name_.find_last_of(".");
if (breakpoint == std::string::npos)
{
breakpoint = dataset_name_.length();
}
index_name_ = dataset_name_.substr(0, breakpoint) + ".ogrindex";
std::ifstream index_file(index_name_.c_str(), std::ios::in | std::ios::binary);
if (index_file)
{
indexed_ = true;
index_file.close();
}
#if 0
// TODO - enable this warning once the ogrindex tool is a bit more stable/mature
else
{
std::clog << "### Notice: no ogrindex file found for " << dataset_name_
<< ", use the 'ogrindex' program to build an index for faster rendering"
<< std::endl;
}
#endif
// deal with attributes descriptions
OGRFeatureDefn* def = layer->GetLayerDefn();
if (def != 0)
{
const int fld_count = def->GetFieldCount();
for (int i = 0; i < fld_count; i++)
{
OGRFieldDefn* fld = def->GetFieldDefn(i);
const std::string fld_name = fld->GetNameRef();
const OGRFieldType type_oid = fld->GetType();
switch (type_oid)
{
case OFTInteger:
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break;
case OFTReal:
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
break;
case OFTString:
case OFTWideString: // deprecated
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break;
case OFTBinary:
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object));
break;
case OFTIntegerList:
case OFTRealList:
case OFTStringList:
case OFTWideStringList: // deprecated !
#ifdef MAPNIK_DEBUG
std::clog << "OGR Plugin: unhandled type_oid=" << type_oid << std::endl;
#endif
break;
case OFTDate:
case OFTTime:
case OFTDateTime: // unhandled !
#ifdef MAPNIK_DEBUG
std::clog << "OGR Plugin: unhandled type_oid=" << type_oid << std::endl;
#endif
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object));
break;
default: // unknown
#ifdef MAPNIK_DEBUG
std::clog << "OGR Plugin: unknown type_oid=" << type_oid << std::endl;
#endif
break;
}
}
}
is_bound_ = true;
}
postgis_datasource::postgis_datasource(parameters const& params)
: datasource(params),
table_(*params.get<std::string>("table", "")),
schema_(""),
geometry_table_(*params.get<std::string>("geometry_table", "")),
geometry_field_(*params.get<std::string>("geometry_field", "")),
key_field_(*params.get<std::string>("key_field", "")),
cursor_fetch_size_(*params.get<mapnik::value_integer>("cursor_size", 0)),
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
type_(datasource::Vector),
srid_(*params.get<mapnik::value_integer>("srid", 0)),
extent_initialized_(false),
simplify_geometries_(false),
desc_(postgis_datasource::name(), "utf-8"),
creator_(params.get<std::string>("host"),
params.get<std::string>("port"),
params.get<std::string>("dbname"),
params.get<std::string>("user"),
params.get<std::string>("password"),
params.get<std::string>("connect_timeout", "4")),
bbox_token_("!bbox!"),
scale_denom_token_("!scale_denominator!"),
pixel_width_token_("!pixel_width!"),
pixel_height_token_("!pixel_height!"),
pool_max_size_(*params_.get<mapnik::value_integer>("max_size", 10)),
persist_connection_(*params.get<mapnik::boolean_type>("persist_connection", true)),
extent_from_subquery_(*params.get<mapnik::boolean_type>("extent_from_subquery", false)),
max_async_connections_(*params_.get<mapnik::value_integer>("max_async_connection", 1)),
asynchronous_request_(false),
// TODO - use for known tokens too: "(@\\w+|!\\w+!)"
pattern_(boost::regex("(@\\w+)",boost::regex::normal | boost::regbase::icase)),
// params below are for testing purposes only and may be removed at any time
intersect_min_scale_(*params.get<mapnik::value_integer>("intersect_min_scale", 0)),
intersect_max_scale_(*params.get<mapnik::value_integer>("intersect_max_scale", 0))
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "postgis_datasource::init");
#endif
if (table_.empty())
{
throw mapnik::datasource_exception("Postgis Plugin: missing <table> parameter");
}
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext && !ext->empty())
{
extent_initialized_ = extent_.from_string(*ext);
}
// NOTE: In multithread environment, pool_max_size_ should be
// max_async_connections_ * num_threads
if(max_async_connections_ > 1)
{
if(max_async_connections_ > pool_max_size_)
{
std::ostringstream err;
err << "PostGIS Plugin: Error: 'max_async_connections ("
<< max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")";
throw mapnik::datasource_exception(err.str());
}
asynchronous_request_ = true;
}
boost::optional<mapnik::value_integer> initial_size = params.get<mapnik::value_integer>("initial_size", 1);
boost::optional<mapnik::boolean_type> autodetect_key_field = params.get<mapnik::boolean_type>("autodetect_key_field", false);
boost::optional<mapnik::boolean_type> estimate_extent = params.get<mapnik::boolean_type>("estimate_extent", false);
estimate_extent_ = estimate_extent && *estimate_extent;
boost::optional<mapnik::boolean_type> simplify_opt = params.get<mapnik::boolean_type>("simplify_geometries", false);
simplify_geometries_ = simplify_opt && *simplify_opt;
ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_);
CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id());
if (pool)
{
shared_ptr<Connection> conn = pool->borrowObject();
if (!conn) return;
if (conn->isOK())
{
desc_.set_encoding(conn->client_encoding());
if (geometry_table_.empty())
{
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
std::string::size_type idx = geometry_table_.find_last_of('.');
if (idx != std::string::npos)
{
schema_ = geometry_table_.substr(0, idx);
geometry_table_ = geometry_table_.substr(idx + 1);
}
else
{
geometry_table_ = geometry_table_.substr(0);
}
// NOTE: geometry_table_ how should ideally be a table name, but
// there are known edge cases where this will break down and
// geometry_table_ may even be empty: https://github.com/mapnik/mapnik/issues/2718
// If we do not know both the geometry_field and the srid
// then first attempt to fetch the geometry name from a geometry_columns entry.
// This will return no records if we are querying a bogus table returned
// from the simplistic table parsing in table_from_sql() or if
// the table parameter references a table, view, or subselect not
// registered in the geometry columns.
geometryColumn_ = geometry_field_;
if (!geometry_table_.empty() && (geometryColumn_.empty() || srid_ == 0))
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::init(get_srid_and_geometry_column)");
#endif
std::ostringstream s;
try
{
s << "SELECT f_geometry_column, srid FROM "
<< GEOMETRY_COLUMNS <<" WHERE f_table_name='"
<< mapnik::sql_utils::unquote_double(geometry_table_)
<< "'";
if (! schema_.empty())
{
s << " AND f_table_schema='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "'";
}
if (! geometry_field_.empty())
{
s << " AND f_geometry_column='"
<< mapnik::sql_utils::unquote_double(geometry_field_)
<< "'";
}
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
geometryColumn_ = rs->getValue("f_geometry_column");
// only accept srid from geometry_tables if
// user has not provided as option
if (srid_ == 0)
{
const char* srid_c = rs->getValue("srid");
if (srid_c != nullptr)
{
int result = 0;
const char * end = srid_c + std::strlen(srid_c);
if (mapnik::util::string2int(srid_c, end, result))
{
srid_ = result;
}
}
}
}
rs->close();
}
catch (mapnik::datasource_exception const& ex)
{
// let this pass on query error and use the fallback below
MAPNIK_LOG_WARN(postgis) << "postgis_datasource: metadata query failed: " << ex.what();
}
}
// If we still do not know the srid then we can try to fetch
// it from the 'geometry_table_' parameter, which should work even if it is
// a subselect as long as we know the geometry_field to query
if (!geometryColumn_.empty() && srid_ <= 0)
{
std::ostringstream s;
s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM ";
if (!geometry_table_.empty())
{
s << geometry_table_;
}
else
{
s << populate_tokens(table_);
}
s << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
if (rs->next())
{
const char* srid_c = rs->getValue("srid");
if (srid_c != nullptr)
{
int result = 0;
const char * end = srid_c + std::strlen(srid_c);
if (mapnik::util::string2int(srid_c, end, result))
{
srid_ = result;
}
}
}
rs->close();
}
// detect primary key
if (*autodetect_key_field && key_field_.empty())
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_primary_key)");
#endif
std::ostringstream s;
s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') "
"AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i "
"WHERE a.attnum > 0 AND a.attrelid = c.oid "
"AND a.atttypid = t.oid AND c.relnamespace = n.oid "
"AND c.oid = i.indrelid AND i.indisprimary = 't' "
"AND t.typname !~ '^geom' AND c.relname ="
<< " '" << mapnik::sql_utils::unquote_double(geometry_table_) << "' "
//"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1
<< "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum "
"OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum "
"OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum "
"OR i.indkey[9]=a.attnum) ";
if (! schema_.empty())
{
s << "AND n.nspname='"
<< mapnik::sql_utils::unquote_double(schema_)
<< "' ";
}
s << "ORDER BY a.attnum";
shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str());
if (rs_key->next())
{
unsigned int result_rows = rs_key->size();
if (result_rows == 1)
{
bool is_int = (std::string(rs_key->getValue(3)) == "t");
if (is_int)
{
const char* key_field_string = rs_key->getValue(0);
if (key_field_string)
{
key_field_ = std::string(key_field_string);
MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: auto-detected key field of '"
<< key_field_ << "' on table '" << geometry_table_ << "'";
}
}
else
{
// throw for cases like a numeric primary key, which is invalid
// as it should be floating point (int numerics are useless)
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< rs_key->getValue(0)
<< "' on table '"
<< geometry_table_
<< "' is not a valid integer primary key field\n";
throw mapnik::datasource_exception(err.str());
}
}
else if (result_rows > 1)
{
std::ostringstream err;
err << "PostGIS Plugin: Error: '"
<< "multi column primary key detected but is not supported";
throw mapnik::datasource_exception(err.str());
}
}
rs_key->close();
}
// if a globally unique key field/primary key is required
// but still not known at this point, then throw
if (*autodetect_key_field && key_field_.empty())
{
throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required")
+ " but could not be detected for table '" +
geometry_table_ + "', please supply 'key_field' option to specify field to use for primary key");
}
if (srid_ == 0)
{
srid_ = -1;
MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Table " << table_ << " is using SRID=" << srid_;
}
// At this point the geometry_field may still not be known
// but we'll catch that where more useful...
MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Using SRID=" << srid_;
MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Using geometry_column=" << geometryColumn_;
// collect attribute desc
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_column_description)");
#endif
std::ostringstream s;
s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0";
shared_ptr<ResultSet> rs = conn->executeQuery(s.str());
int count = rs->getNumFields();
bool found_key_field = false;
for (int i = 0; i < count; ++i)
{
std::string fld_name = rs->getFieldName(i);
int type_oid = rs->getTypeOID(i);
// validate type of key_field
if (! found_key_field && ! key_field_.empty() && fld_name == key_field_)
{
if (type_oid == 20 || type_oid == 21 || type_oid == 23)
{
found_key_field = true;
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
}
else
{
std::ostringstream error_s;
error_s << "invalid type '";
std::ostringstream type_s;
type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str());
if (rs_oid->next())
{
error_s << rs_oid->getValue("typname")
<< "' (oid:" << rs_oid->getValue("oid") << ")";
}
else
{
error_s << "oid:" << type_oid << "'";
}
rs_oid->close();
error_s << " for key_field '" << fld_name << "' - "
<< "must be an integer primary key";
rs->close();
throw mapnik::datasource_exception(error_s.str());
}
}
else
{
switch (type_oid)
{
case 16: // bool
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean));
break;
case 20: // int8
case 21: // int2
case 23: // int4
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break;
case 700: // float4
case 701: // float8
case 1700: // numeric
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
break;
case 1042: // bpchar
case 1043: // varchar
case 25: // text
case 705: // literal
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break;
default: // should not get here
#ifdef MAPNIK_LOG
s.str("");
s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid;
shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str());
if (rs_oid->next())
{
std::string typname(rs_oid->getValue("typname"));
if (typname != "geometry")
{
MAPNIK_LOG_WARN(postgis) << "postgis_datasource: Unknown type=" << typname
<< " (oid:" << rs_oid->getValue("oid") << ")";
}
}
else
{
MAPNIK_LOG_WARN(postgis) << "postgis_datasource: Unknown type_oid=" << type_oid;
}
rs_oid->close();
#endif
break;
}
}
}
rs->close();
}
// Close explicitly the connection so we can 'fork()' without sharing open connections
conn->close();
// Finally, add unique metadata to layer descriptor
mapnik::parameters & extra_params = desc_.get_extra_parameters();
// explicitly make copies of values due to https://github.com/mapnik/mapnik/issues/2651
extra_params["srid"] = srid_;
if (!key_field_.empty())
{
extra_params["key_field"] = key_field_;
}
}
}
occi_datasource::occi_datasource(parameters const& params)
: datasource (params),
type_(datasource::Vector),
fields_(*params.get<std::string>("fields", "*")),
geometry_field_(*params.get<std::string>("geometry_field", "")),
srid_initialized_(false),
extent_initialized_(false),
bbox_token_("!bbox!"),
scale_denom_token_("!scale_denominator!"),
pixel_width_token_("!pixel_width!"),
pixel_height_token_("!pixel_height!"),
desc_(occi_datasource::name(), *params.get<std::string>("encoding", "utf-8")),
use_wkb_(*params.get<mapnik::boolean_type>("use_wkb", false)),
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
row_prefetch_(*params.get<mapnik::value_integer>("row_prefetch", 100)),
pool_(0),
conn_(0)
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::init");
#endif
if (! params.get<std::string>("user")) throw datasource_exception("OCCI Plugin: no <user> specified");
if (! params.get<std::string>("password")) throw datasource_exception("OCCI Plugin: no <password> specified");
if (! params.get<std::string>("host")) throw datasource_exception("OCCI Plugin: no <host> string specified");
boost::optional<std::string> table = params.get<std::string>("table");
if (! table)
{
throw datasource_exception("OCCI Plugin: no <table> parameter specified");
}
else
{
table_ = *table;
}
estimate_extent_ = *params.get<mapnik::boolean_type>("estimate_extent",false);
use_spatial_index_ = *params.get<mapnik::boolean_type>("use_spatial_index",true);
use_connection_pool_ = *params.get<mapnik::boolean_type>("use_connection_pool",true);
boost::optional<std::string> ext = params.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<mapnik::value_integer> srid = params.get<mapnik::value_integer>("srid");
if (srid)
{
srid_ = *srid;
srid_initialized_ = true;
}
// connect to environment
if (use_connection_pool_)
{
try
{
pool_ = occi_environment::instance().create_pool(
*params.get<std::string>("user"),
*params.get<std::string>("password"),
*params.get<std::string>("host"),
*params.get<mapnik::value_integer>("max_size", 5),
*params.get<mapnik::value_integer>("initial_size", 1),
1);
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
else
{
try
{
conn_ = occi_environment::instance().create_connection(
*params.get<std::string>("user"),
*params.get<std::string>("password"),
*params.get<std::string>("host"));
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
// extract real table name
table_name_ = mapnik::sql_utils::table_from_sql(table_);
// get SRID and/or GEOMETRY_FIELD from metadata table only if we need to
if (! srid_initialized_ || geometry_field_ == "")
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_srid_and_geometry_field");
#endif
std::ostringstream s;
s << "SELECT srid, column_name FROM " << METADATA_TABLE << " WHERE";
s << " LOWER(table_name) = LOWER('" << table_name_ << "')";
if (geometry_field_ != "")
{
s << " AND LOWER(column_name) = LOWER('" << geometry_field_ << "')";
}
MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str();
try
{
occi_connection_ptr conn;
if (use_connection_pool_) conn.set_pool(pool_);
else conn.set_connection(conn_, false);
ResultSet* rs = conn.execute_query(s.str());
if (rs && rs->next ())
{
if (! srid_initialized_)
{
srid_ = rs->getInt(1);
srid_initialized_ = true;
}
if (geometry_field_ == "")
{
geometry_field_ = rs->getString(2);
}
}
}
catch (SQLException& ex)
{
throw datasource_exception("OCCI Plugin: " + ex.getMessage());
}
}
// get columns description
{
#ifdef MAPNIK_STATS
mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_column_description");
#endif
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_name_ << ") WHERE ROWNUM < 1";
MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str();
try
{
occi_connection_ptr conn;
if (use_connection_pool_) conn.set_pool(pool_);
else conn.set_connection(conn_, false);
ResultSet* rs = conn.execute_query(s.str());
if (rs)
{
std::vector<MetaData> listOfColumns = rs->getColumnListMetaData();
for (unsigned int i = 0; i < listOfColumns.size(); ++i)
{
MetaData columnObj = listOfColumns[i];
std::string fld_name = columnObj.getString(MetaData::ATTR_NAME);
int type_oid = columnObj.getInt(MetaData::ATTR_DATA_TYPE);
/*
int type_code = columnObj.getInt(MetaData::ATTR_TYPECODE);
if (type_code == OCCI_TYPECODE_OBJECT)
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object));
continue;
}
*/
switch (type_oid)
{
case oracle::occi::OCCIBOOL:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Boolean));
break;
case oracle::occi::OCCIINT:
case oracle::occi::OCCIUNSIGNED_INT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case oracle::occi::OCCIFLOAT:
case oracle::occi::OCCIBFLOAT:
case oracle::occi::OCCIDOUBLE:
case oracle::occi::OCCIBDOUBLE:
case oracle::occi::OCCINUMBER:
case oracle::occi::OCCI_SQLT_NUM:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case oracle::occi::OCCICHAR:
case oracle::occi::OCCISTRING:
case oracle::occi::OCCI_SQLT_AFC:
case oracle::occi::OCCI_SQLT_AVC:
case oracle::occi::OCCI_SQLT_CHR:
case oracle::occi::OCCI_SQLT_LNG:
case oracle::occi::OCCI_SQLT_LVC:
case oracle::occi::OCCI_SQLT_STR:
case oracle::occi::OCCI_SQLT_VCS:
case oracle::occi::OCCI_SQLT_VNU:
case oracle::occi::OCCI_SQLT_VBI:
case oracle::occi::OCCI_SQLT_VST:
case oracle::occi::OCCIROWID:
case oracle::occi::OCCI_SQLT_RDD:
case oracle::occi::OCCI_SQLT_RID:
case oracle::occi::OCCIDATE:
case oracle::occi::OCCI_SQLT_DAT:
case oracle::occi::OCCI_SQLT_DATE:
case oracle::occi::OCCI_SQLT_TIME:
case oracle::occi::OCCI_SQLT_TIME_TZ:
case oracle::occi::OCCITIMESTAMP:
case oracle::occi::OCCI_SQLT_TIMESTAMP:
case oracle::occi::OCCI_SQLT_TIMESTAMP_LTZ:
case oracle::occi::OCCI_SQLT_TIMESTAMP_TZ:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case oracle::occi::OCCIINTERVALDS:
case oracle::occi::OCCIINTERVALYM:
case oracle::occi::OCCI_SQLT_INTERVAL_YM:
case oracle::occi::OCCI_SQLT_INTERVAL_DS:
case oracle::occi::OCCIANYDATA:
case oracle::occi::OCCIBLOB:
case oracle::occi::OCCIBFILE:
case oracle::occi::OCCIBYTES:
case oracle::occi::OCCICLOB:
case oracle::occi::OCCIVECTOR:
case oracle::occi::OCCIMETADATA:
case oracle::occi::OCCIPOBJECT:
case oracle::occi::OCCIREF:
case oracle::occi::OCCIREFANY:
case oracle::occi::OCCISTREAM:
case oracle::occi::OCCICURSOR:
case oracle::occi::OCCI_SQLT_FILE:
case oracle::occi::OCCI_SQLT_CFILE:
case oracle::occi::OCCI_SQLT_REF:
case oracle::occi::OCCI_SQLT_CLOB:
case oracle::occi::OCCI_SQLT_BLOB:
case oracle::occi::OCCI_SQLT_RSET:
MAPNIK_LOG_WARN(occi) << "occi_datasource: Unsupported datatype "
<< occi_enums::resolve_datatype(type_oid)
<< " (type_oid=" << type_oid << ")";
break;
default:
MAPNIK_LOG_WARN(occi) << "occi_datasource: Unknown datatype "
<< "(type_oid=" << type_oid << ")";
break;
}
}
}
}
catch (SQLException& ex)
{
throw datasource_exception(ex.getMessage());
}
}
}
void sqlite_datasource::bind() const
{
if (is_bound_) return;
if (!boost::filesystem::exists(dataset_name_))
throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
dataset_ = new sqlite_connection (dataset_name_);
if(geometry_table_.empty())
{
geometry_table_ = mapnik::table_from_sql(table_);
}
// should we deduce column names and types using PRAGMA?
bool use_pragma_table_info = true;
if (table_ != geometry_table_)
{
// if 'table_' is a subquery then we try to deduce names
// and types from the first row returned from that query
use_pragma_table_info = false;
}
if (!use_pragma_table_info)
{
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
for (int i = 0; i < rs->column_count (); ++i)
{
const int type_oid = rs->column_type (i);
const char* fld_name = rs->column_name (i);
switch (type_oid)
{
case SQLITE_INTEGER:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case SQLITE_FLOAT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case SQLITE_TEXT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case SQLITE_NULL:
// sqlite reports based on value, not actual column type unless
// PRAGMA table_info is used so here we assume the column is a string
// which is a lesser evil than altogether dropping the column
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
case SQLITE_BLOB:
if (geometry_field_.empty() &&
(boost::algorithm::icontains(fld_name,"geom") ||
boost::algorithm::icontains(fld_name,"point") ||
boost::algorithm::icontains(fld_name,"linestring") ||
boost::algorithm::icontains(fld_name,"polygon"))
)
geometry_field_ = std::string(fld_name);
break;
default:
#ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl;
#endif
break;
}
}
}
else
{
// if we do not have at least a row and
// we cannot determine the right columns types and names
// as all column_type are SQLITE_NULL
// so we fallback to using PRAGMA table_info
use_pragma_table_info = true;
}
}
// TODO - ensure that the supplied key_field is a valid "integer primary key"
desc_.add_descriptor(attribute_descriptor("rowid",mapnik::Integer));
if (use_pragma_table_info)
{
std::ostringstream s;
s << "PRAGMA table_info(" << geometry_table_ << ")";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
bool found_table = false;
while (rs->is_valid () && rs->step_next())
{
found_table = true;
const char* fld_name = rs->column_text(1);
std::string fld_type(rs->column_text(2));
boost::algorithm::to_lower(fld_type);
// see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html
if (geometry_field_.empty() &&
(
(boost::algorithm::icontains(fld_name,"geom") ||
boost::algorithm::icontains(fld_name,"point") ||
boost::algorithm::icontains(fld_name,"linestring") ||
boost::algorithm::icontains(fld_name,"polygon"))
||
(boost::algorithm::contains(fld_type,"geom") ||
boost::algorithm::contains(fld_type,"point") ||
boost::algorithm::contains(fld_type,"linestring") ||
boost::algorithm::contains(fld_type,"polygon"))
)
)
geometry_field_ = std::string(fld_name);
else if (boost::algorithm::contains(fld_type,"int"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
}
else if (boost::algorithm::contains(fld_type,"text") ||
boost::algorithm::contains(fld_type,"char") ||
boost::algorithm::contains(fld_type,"clob"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
}
else if (boost::algorithm::contains(fld_type,"real") ||
boost::algorithm::contains(fld_type,"float") ||
boost::algorithm::contains(fld_type,"double"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
}
else if (boost::algorithm::contains(fld_type,"blob") && !geometry_field_.empty())
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
#ifdef MAPNIK_DEBUG
else
{
// "Column Affinity" says default to "Numeric" but for now we pass..
//desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
std::clog << "Sqlite Plugin: column '" << std::string(fld_name) << "' unhandled due to unknown type: " << fld_type << std::endl;
}
#endif
}
if (!found_table)
{
throw datasource_exception("Sqlite Plugin: could not query table '" + geometry_table_ + "' using pragma table_info(" + geometry_table_ + ");");
}
}
if (geometry_field_.empty()) {
throw datasource_exception("Sqlite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use.");
}
if (use_spatial_index_)
{
std::ostringstream s;
s << "SELECT COUNT (*) FROM sqlite_master";
s << " WHERE LOWER(name) = LOWER('idx_" << geometry_table_ << "_" << geometry_field_ << "')";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
use_spatial_index_ = rs->column_integer (0) == 1;
}
if (!use_spatial_index_)
std::clog << "Sqlite Plugin: spatial index not found for table '"
<< geometry_table_ << "'" << std::endl;
}
if (metadata_ != "" && !extent_initialized_)
{
std::ostringstream s;
s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_;
s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent_.init (xmin,ymin,xmax,ymax);
extent_initialized_ = true;
}
}
if (!extent_initialized_ && use_spatial_index_)
{
std::ostringstream s;
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
<< "idx_" << geometry_table_ << "_" << geometry_field_;
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent_.init (xmin,ymin,xmax,ymax);
extent_initialized_ = true;
}
}
if (!extent_initialized_) {
std::ostringstream s;
s << "Sqlite Plugin: extent could not be determined for table|geometry '"
<< geometry_table_ << "|" << geometry_field_ << "'"
<< " because an rtree spatial index is missing."
<< " - either set the table 'extent' or create an rtree spatial index";
throw datasource_exception(s.str());
}
is_bound_ = true;
}
void spatialite_datasource::bind() const
{
if (is_bound_) return;
boost::optional<std::string> file = params_.get<std::string>("file");
if (!file) throw datasource_exception("Spatialite Plugin: missing <file> parameter");
boost::optional<std::string> key_field_name = params_.get<std::string>("key_field");
if (key_field_name) {
std::string const& key_field_string = *key_field_name;
if (key_field_string.empty()) {
key_field_ = "rowid";
} else {
key_field_ = key_field_string;
}
}
else
{
key_field_ = "rowid";
}
boost::optional<std::string> wkb = params_.get<std::string>("wkb_format");
if (wkb)
{
if (*wkb == "spatialite")
format_ = mapnik::wkbSpatiaLite;
}
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false);
use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index",true);
has_spatial_index_ = false;
using_subquery_ = false;
boost::optional<std::string> ext = params_.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<std::string> base = params_.get<std::string>("base");
if (base)
dataset_name_ = *base + "/" + *file;
else
dataset_name_ = *file;
// Populate init_statements_
// 1. Build attach database statements from the "attachdb" parameter
// 2. Add explicit init statements from "initdb" parameter
// Note that we do some extra work to make sure that any attached
// databases are relative to directory containing dataset_name_. Sqlite
// will default to attaching from cwd. Typicaly usage means that the
// map loader will produce full paths here.
boost::optional<std::string> attachdb = params_.get<std::string>("attachdb");
if (attachdb) {
parse_attachdb(*attachdb);
}
boost::optional<std::string> initdb = params_.get<std::string>("initdb");
if (initdb) {
init_statements_.push_back(*initdb);
}
if (!boost::filesystem::exists(dataset_name_))
throw datasource_exception("Spatialite Plugin: " + dataset_name_ + " does not exist");
dataset_ = new sqlite_connection (dataset_name_);
// Execute init_statements_
for (std::vector<std::string>::const_iterator iter=init_statements_.begin(); iter!=init_statements_.end(); ++iter)
{
#ifdef MAPNIK_DEBUG
std::clog << "Spatialite Plugin: Execute init sql: " << *iter << std::endl;
#endif
dataset_->execute(*iter);
}
if(geometry_table_.empty())
{
geometry_table_ = mapnik::table_from_sql(table_);
}
// should we deduce column names and types using PRAGMA?
bool use_pragma_table_info = true;
if (table_ != geometry_table_)
{
// if 'table_' is a subquery then we try to deduce names
// and types from the first row returned from that query
using_subquery_ = true;
use_pragma_table_info = false;
}
if (!use_pragma_table_info)
{
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1";
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
for (int i = 0; i < rs->column_count (); ++i)
{
const int type_oid = rs->column_type (i);
const char* fld_name = rs->column_name (i);
switch (type_oid)
{
case SQLITE_INTEGER:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
break;
case SQLITE_FLOAT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
break;
case SQLITE_TEXT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
break;
case SQLITE_NULL:
// sqlite reports based on value, not actual column type unless
// PRAGMA table_info is used so here we assume the column is a string
// which is a lesser evil than altogether dropping the column
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
case SQLITE_BLOB:
if (geometry_field_.empty() &&
(boost::algorithm::icontains(fld_name,"geom") ||
boost::algorithm::icontains(fld_name,"point") ||
boost::algorithm::icontains(fld_name,"linestring") ||
boost::algorithm::icontains(fld_name,"polygon"))
)
geometry_field_ = std::string(fld_name);
break;
default:
#ifdef MAPNIK_DEBUG
std::clog << "Spatialite Plugin: unknown type_oid=" << type_oid << std::endl;
#endif
break;
}
}
}
else
{
// if we do not have at least a row and
// we cannot determine the right columns types and names
// as all column_type are SQLITE_NULL
// so we fallback to using PRAGMA table_info
use_pragma_table_info = true;
}
}
if (key_field_ == "rowid")
desc_.add_descriptor(attribute_descriptor("rowid",mapnik::Integer));
if (use_pragma_table_info)
{
std::ostringstream s;
s << "PRAGMA table_info(" << geometry_table_ << ")";
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
bool found_table = false;
while (rs->is_valid() && rs->step_next())
{
found_table = true;
// TODO - support unicode strings?
const char* fld_name = rs->column_text(1);
std::string fld_type(rs->column_text(2));
boost::algorithm::to_lower(fld_type);
// see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html
if (geometry_field_.empty() &&
(
(boost::algorithm::icontains(fld_name,"geom") ||
boost::algorithm::icontains(fld_name,"point") ||
boost::algorithm::icontains(fld_name,"linestring") ||
boost::algorithm::icontains(fld_name,"polygon"))
||
(boost::algorithm::contains(fld_type,"geom") ||
boost::algorithm::contains(fld_type,"point") ||
boost::algorithm::contains(fld_type,"linestring") ||
boost::algorithm::contains(fld_type,"polygon"))
)
)
geometry_field_ = std::string(fld_name);
else if (boost::algorithm::contains(fld_type,"int"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer));
}
else if (boost::algorithm::contains(fld_type,"text") ||
boost::algorithm::contains(fld_type,"char") ||
boost::algorithm::contains(fld_type,"clob"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
}
else if (boost::algorithm::contains(fld_type,"real") ||
boost::algorithm::contains(fld_type,"float") ||
boost::algorithm::contains(fld_type,"double"))
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
}
else if (boost::algorithm::contains(fld_type,"blob") && !geometry_field_.empty())
{
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String));
}
#ifdef MAPNIK_DEBUG
else
{
// "Column Affinity" says default to "Numeric" but for now we pass..
//desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
std::clog << "Spatialite Plugin: column '" << std::string(fld_name) << "' unhandled due to unknown type: " << fld_type << std::endl;
}
#endif
}
if (!found_table)
{
std::ostringstream s;
s << "Spatialite Plugin: could not query table '" << geometry_table_ << "' ";
if (using_subquery_)
s << " from subquery '" << table_ << "' ";
s << "using 'PRAGMA table_info(" << geometry_table_ << ")' ";
std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_)));
if (sq_err != "unknown error")
s << ": " << sq_err;
throw datasource_exception(s.str());
}
}
if (geometry_field_.empty()) {
throw datasource_exception("Spatialite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use.");
}
if (index_table_.size() == 0) {
// Generate implicit index_table name - need to do this after
// we have discovered meta-data or else we don't know the column name
index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\"";
}
if (use_spatial_index_)
{
std::ostringstream s;
s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_;
s << " LIMIT 0";
if (dataset_->execute_with_code(s.str()) == SQLITE_OK)
{
has_spatial_index_ = true;
}
#ifdef MAPNIK_DEBUG
else
{
std::clog << "Spatialite Plugin: rtree index lookup did not succeed: '" << sqlite3_errmsg(*(*dataset_)) << "'\n";
}
#endif
}
if (metadata_ != "" && !extent_initialized_)
{
std::ostringstream s;
s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_;
s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')";
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent_.init (xmin,ymin,xmax,ymax);
extent_initialized_ = true;
}
}
if (!extent_initialized_ && has_spatial_index_)
{
std::ostringstream s;
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
<< index_table_;
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
if (!rs->column_isnull(0)) {
try
{
double xmin = lexical_cast<double>(rs->column_double(0));
double ymin = lexical_cast<double>(rs->column_double(1));
double xmax = lexical_cast<double>(rs->column_double(2));
double ymax = lexical_cast<double>(rs->column_double(3));
extent_.init (xmin,ymin,xmax,ymax);
extent_initialized_ = true;
}
catch (bad_lexical_cast &ex)
{
std::clog << boost::format("Spatialite Plugin: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what() << std::endl;
}
}
}
}
#ifdef MAPNIK_DEBUG
if (!has_spatial_index_
&& use_spatial_index_
&& using_subquery_
&& key_field_ == "rowid"
&& !boost::algorithm::icontains(table_,"rowid")) {
// this is an impossible situation because rowid will be null via a subquery
std::clog << "Spatialite Plugin: WARNING: spatial index usage will fail because rowid is not present in your subquery. You have 4 options: 1) Add rowid into your select statement, 2) alias your primary key as rowid, 3) supply a 'key_field' value that references the primary key of your spatial table, or 4) avoid using a spatial index by setting 'use_spatial_index'=false";
}
#endif
// final fallback to gather extent
if (!extent_initialized_ || !has_spatial_index_) {
std::ostringstream s;
s << "SELECT " << geometry_field_ << "," << key_field_
<< " FROM " << geometry_table_;
if (row_limit_ > 0) {
s << " LIMIT " << row_limit_;
}
if (row_offset_ > 0) {
s << " OFFSET " << row_offset_;
}
#ifdef MAPNIK_DEBUG
std::clog << "Spatialite Plugin: " << s.str() << std::endl;
#endif
boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
// spatial index sql
std::string spatial_index_sql = "create virtual table " + index_table_
+ " using rtree(pkid, xmin, xmax, ymin, ymax)";
std::string spatial_index_insert_sql = "insert into " + index_table_
+ " values (?,?,?,?,?)" ;
sqlite3_stmt* stmt = 0;
if (use_spatial_index_) {
dataset_->execute(spatial_index_sql);
int rc = sqlite3_prepare_v2 (*(*dataset_), spatial_index_insert_sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK)
{
std::ostringstream index_error;
index_error << "Spatialite Plugin: auto-index table creation failed: '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql;
throw datasource_exception(index_error.str());
}
}
bool first = true;
while (rs->is_valid() && rs->step_next())
{
int size;
const char* data = (const char *) rs->column_blob (0, size);
if (data) {
// create a tmp feature to be able to parse geometry
// ideally we would not have to do this.
// see: http://trac.mapnik.org/ticket/745
mapnik::feature_ptr feature(mapnik::feature_factory::create(0));
mapnik::geometry_utils::from_wkb(feature->paths(),data,size,multiple_geometries_,format_);
mapnik::box2d<double> const& bbox = feature->envelope();
if (bbox.valid()) {
extent_initialized_ = true;
if (first) {
first = false;
extent_ = bbox;
} else {
extent_.expand_to_include(bbox);
}
// index creation
if (use_spatial_index_) {
const int type_oid = rs->column_type(1);
if (type_oid != SQLITE_INTEGER) {
std::ostringstream type_error;
type_error << "Spatialite Plugin: invalid type for key field '"
<< key_field_ << "' when creating index '" << index_table_
<< "' type was: " << type_oid << "";
throw datasource_exception(type_error.str());
}
sqlite_int64 pkid = rs->column_integer64(1);
if (sqlite3_bind_int64(stmt, 1 , pkid ) != SQLITE_OK)
{
throw datasource_exception("invalid value for for key field while generating index");
}
if ((sqlite3_bind_double(stmt, 2 , bbox.minx() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 3 , bbox.maxx() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 4 , bbox.miny() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 5 , bbox.maxy() ) != SQLITE_OK)) {
throw datasource_exception("invalid value for for extent while generating index");
}
int res = sqlite3_step(stmt);
if (res != SQLITE_DONE) {
std::ostringstream s;
s << "Spatialite Plugin: inserting bbox into rtree index failed: "
<< "error code " << sqlite3_errcode(*(*dataset_)) << ": '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql;
throw datasource_exception(s.str());
}
sqlite3_reset(stmt);
}
}
else {
std::ostringstream index_error;
index_error << "Spatialite Plugin: encountered invalid bbox at '"
<< key_field_ << "' == " << rs->column_integer64(1);
throw datasource_exception(index_error.str());
}
}
}
int res = sqlite3_finalize(stmt);
if (res != SQLITE_OK)
{
throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error");
}
}
if (!extent_initialized_) {
std::ostringstream s;
s << "Spatialite Plugin: extent could not be determined for table '"
<< geometry_table_ << "' and geometry field '" << geometry_field_ << "'"
<< " because an rtree spatial index is missing or empty."
<< " - either set the table 'extent' or create an rtree spatial index";
throw datasource_exception(s.str());
}
is_bound_ = true;
}
