geos_datasource::geos_datasource(parameters const& params, bool bind) : datasource(params), extent_(), extent_initialized_(false), type_(datasource::Vector), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")), geometry_data_(""), geometry_data_name_("name"), geometry_id_(1) { boost::optional<std::string> geometry = params.get<std::string>("wkt"); if (!geometry) throw datasource_exception("missing <wkt> parameter"); geometry_string_ = *geometry; multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false); boost::optional<std::string> ext = params_.get<std::string>("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); boost::optional<int> id = params_.get<int>("gid"); if (id) geometry_id_ = *id; boost::optional<std::string> gdata = params_.get<std::string>("field_data"); if (gdata) geometry_data_ = *gdata; boost::optional<std::string> gdata_name = params_.get<std::string>("field_name"); if (gdata_name) geometry_data_name_ = *gdata_name; desc_.add_descriptor(attribute_descriptor(geometry_data_name_, mapnik::String)); if (bind) { this->bind(); } }
osm_datasource::osm_datasource(const parameters& params) : datasource (params), extent_(), type_(datasource::Vector), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8")) { osm_data_ = NULL; std::string osm_filename = *params.get<std::string>("file", ""); std::string parser = *params.get<std::string>("parser", "libxml2"); std::string url = *params.get<std::string>("url", ""); std::string bbox = *params.get<std::string>("bbox", ""); // load the data if (url != "" && bbox != "") { // if we supplied a url and a bounding box, load from the url MAPNIK_LOG_DEBUG(osm) << "osm_datasource: loading_from_url url=" << url << ",bbox=" << bbox; if ((osm_data_ = dataset_deliverer::load_from_url(url, bbox, parser)) == NULL) { throw datasource_exception("Error loading from URL"); } } else if (osm_filename != "") { // if we supplied a filename, load from file if ((osm_data_ = dataset_deliverer::load_from_file(osm_filename, parser)) == NULL) { std::string s("OSM Plugin: Error loading from file '"); s += osm_filename + "'"; throw datasource_exception(s); } } else { throw datasource_exception("OSM Plugin: Neither 'file' nor 'url' and 'bbox' specified"); } osm_tag_types tagtypes; tagtypes.add_type("maxspeed", mapnik::Integer); tagtypes.add_type("z_order", mapnik::Integer); osm_data_->rewind(); // Need code to get the attributes of all the data std::set<std::string> keys = osm_data_->get_keys(); // Add the attributes to the datasource descriptor - assume they are // all of type String for (auto const& key : keys) { desc_.add_descriptor(attribute_descriptor(key, tagtypes.get_type(key))); } // Get the bounds of the data and set extent_ accordingly bounds b = osm_data_->get_bounds(); extent_ = box2d<double>(b.w,b.s,b.e,b.n); }
osm_datasource::osm_datasource(const parameters ¶ms) : datasource (params), type_(datasource::Vector), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")) { osm_data_ = NULL; std::string osm_filename= *params.get<std::string>("file",""); std::string parser = *params.get<std::string>("parser","libxml2"); std::string url = *params.get<std::string>("url",""); std::string bbox = *params.get<std::string>("bbox",""); bool do_process=false; // load the data // if we supplied a filename, load from file if (url!="" && bbox!="") { // otherwise if we supplied a url and a bounding box, load from the url #ifdef MAPNIK_DEBUG cerr<<"loading_from_rul: url="<<url << " bbox="<<bbox<<endl; #endif if((osm_data_=dataset_deliverer::load_from_url (url,bbox,parser))==NULL) { throw datasource_exception("Error loading from URL"); } do_process=true; } else if(osm_filename!="") { if ((osm_data_= dataset_deliverer::load_from_file(osm_filename,parser))==NULL) { throw datasource_exception("Error loading from file"); } do_process=true; } if(do_process==true) { osm_tag_types tagtypes; tagtypes.add_type("maxspeed",mapnik::Integer); tagtypes.add_type("z_order",mapnik::Integer); osm_data_->rewind(); // Need code to get the attributes of all the data std::set<std::string> keys= osm_data_->get_keys(); // Add the attributes to the datasource descriptor - assume they are // all of type String for(std::set<std::string>::iterator i=keys.begin(); i!=keys.end(); i++) desc_.add_descriptor(attribute_descriptor(*i,tagtypes.get_type(*i))); // Get the bounds of the data and set extent_ accordingly bounds b = osm_data_->get_bounds(); extent_ = box2d<double>(b.w,b.s,b.e,b.n); } }
void occi_datasource::bind() const { if (is_bound_) return; #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::bind"); #endif // connect to environment if (use_connection_pool_) { try { Environment* env = occi_environment::get_environment(); pool_ = env->createStatelessConnectionPool( *params_.get<std::string>("user"), *params_.get<std::string>("password"), *params_.get<std::string>("host"), *params_.get<int>("max_size", 5), *params_.get<int>("initial_size", 1), 1, StatelessConnectionPool::HOMOGENEOUS); } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } else { try { Environment* env = occi_environment::get_environment(); conn_ = env->createConnection( *params_.get<std::string>("user"), *params_.get<std::string>("password"), *params_.get<std::string>("host")); } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } // extract real table name table_name_ = mapnik::sql_utils::table_from_sql(table_); // get SRID and/or GEOMETRY_FIELD from metadata table only if we need to if (! srid_initialized_ || geometry_field_ == "") { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_srid_and_geometry_field"); #endif std::ostringstream s; s << "SELECT srid, column_name FROM " << METADATA_TABLE << " WHERE"; s << " LOWER(table_name) = LOWER('" << table_name_ << "')"; if (geometry_field_ != "") { s << " AND LOWER(column_name) = LOWER('" << geometry_field_ << "')"; } MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str(); try { occi_connection_ptr conn; if (use_connection_pool_) conn.set_pool(pool_); else conn.set_connection(conn_, false); ResultSet* rs = conn.execute_query(s.str()); if (rs && rs->next ()) { if (! srid_initialized_) { srid_ = rs->getInt(1); srid_initialized_ = true; } if (geometry_field_ == "") { geometry_field_ = rs->getString(2); } } } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } // get columns description { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_column_description"); #endif std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_name_ << ") WHERE rownum < 1"; MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str(); try { occi_connection_ptr conn; if (use_connection_pool_) conn.set_pool(pool_); else conn.set_connection(conn_, false); ResultSet* rs = conn.execute_query(s.str()); if (rs) { std::vector<MetaData> listOfColumns = rs->getColumnListMetaData(); for (unsigned int i = 0; i < listOfColumns.size(); ++i) { MetaData columnObj = listOfColumns[i]; std::string fld_name = columnObj.getString(MetaData::ATTR_NAME); int type_oid = columnObj.getInt(MetaData::ATTR_DATA_TYPE); /* int type_code = columnObj.getInt(MetaData::ATTR_TYPECODE); if (type_code == OCCI_TYPECODE_OBJECT) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object)); continue; } */ switch (type_oid) { case oracle::occi::OCCIBOOL: case oracle::occi::OCCIINT: case oracle::occi::OCCIUNSIGNED_INT: case oracle::occi::OCCIROWID: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case oracle::occi::OCCIFLOAT: case oracle::occi::OCCIBFLOAT: case oracle::occi::OCCIDOUBLE: case oracle::occi::OCCIBDOUBLE: case oracle::occi::OCCINUMBER: case oracle::occi::OCCI_SQLT_NUM: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case oracle::occi::OCCICHAR: case oracle::occi::OCCISTRING: case oracle::occi::OCCI_SQLT_AFC: case oracle::occi::OCCI_SQLT_AVC: case oracle::occi::OCCI_SQLT_CHR: case oracle::occi::OCCI_SQLT_LVC: case oracle::occi::OCCI_SQLT_RDD: case oracle::occi::OCCI_SQLT_STR: case oracle::occi::OCCI_SQLT_VCS: case oracle::occi::OCCI_SQLT_VNU: case oracle::occi::OCCI_SQLT_VBI: case oracle::occi::OCCI_SQLT_VST: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case oracle::occi::OCCIDATE: case oracle::occi::OCCITIMESTAMP: case oracle::occi::OCCIINTERVALDS: case oracle::occi::OCCIINTERVALYM: case oracle::occi::OCCI_SQLT_DAT: case oracle::occi::OCCI_SQLT_DATE: case oracle::occi::OCCI_SQLT_TIME: case oracle::occi::OCCI_SQLT_TIME_TZ: case oracle::occi::OCCI_SQLT_TIMESTAMP: case oracle::occi::OCCI_SQLT_TIMESTAMP_LTZ: case oracle::occi::OCCI_SQLT_TIMESTAMP_TZ: case oracle::occi::OCCI_SQLT_INTERVAL_YM: case oracle::occi::OCCI_SQLT_INTERVAL_DS: case oracle::occi::OCCIANYDATA: case oracle::occi::OCCIBLOB: case oracle::occi::OCCIBFILE: case oracle::occi::OCCIBYTES: case oracle::occi::OCCICLOB: case oracle::occi::OCCIVECTOR: case oracle::occi::OCCIMETADATA: case oracle::occi::OCCIPOBJECT: case oracle::occi::OCCIREF: case oracle::occi::OCCIREFANY: case oracle::occi::OCCISTREAM: case oracle::occi::OCCICURSOR: case oracle::occi::OCCI_SQLT_FILE: case oracle::occi::OCCI_SQLT_CFILE: case oracle::occi::OCCI_SQLT_REF: case oracle::occi::OCCI_SQLT_CLOB: case oracle::occi::OCCI_SQLT_BLOB: case oracle::occi::OCCI_SQLT_RSET: MAPNIK_LOG_WARN(occi) << "occi_datasource: Unsupported datatype " << occi_enums::resolve_datatype(type_oid) << " (type_oid=" << type_oid << ")"; break; default: MAPNIK_LOG_WARN(occi) << "occi_datasource: Unknown datatype " << "(type_oid=" << type_oid << ")"; break; } } } } catch (SQLException& ex) { throw datasource_exception(ex.getMessage()); } } is_bound_ = true; }
pgraster_datasource::pgraster_datasource(parameters const& params) : datasource(params), table_(*params.get<std::string>("table", "")), schema_(""), raster_table_(*params.get<std::string>("raster_table", "")), raster_field_(*params.get<std::string>("raster_field", "")), key_field_(*params.get<std::string>("key_field", "")), cursor_fetch_size_(*params.get<mapnik::value_integer>("cursor_size", 0)), row_limit_(*params.get<value_integer>("row_limit", 0)), type_(datasource::Raster), srid_(*params.get<value_integer>("srid", 0)), band_(*params.get<value_integer>("band", 0)), extent_initialized_(false), prescale_rasters_(*params.get<mapnik::boolean_type>("prescale_rasters", false)), use_overviews_(*params.get<mapnik::boolean_type>("use_overviews", false)), clip_rasters_(*params.get<mapnik::boolean_type>("clip_rasters", false)), desc_(*params.get<std::string>("type"), "utf-8"), creator_(params.get<std::string>("host"), params.get<std::string>("port"), params.get<std::string>("dbname"), params.get<std::string>("user"), params.get<std::string>("password"), params.get<std::string>("connect_timeout", "4")), bbox_token_("!bbox!"), scale_denom_token_("!scale_denominator!"), pixel_width_token_("!pixel_width!"), pixel_height_token_("!pixel_height!"), pool_max_size_(*params_.get<value_integer>("max_size", 10)), persist_connection_(*params.get<mapnik::boolean_type>("persist_connection", true)), extent_from_subquery_(*params.get<mapnik::boolean_type>("extent_from_subquery", false)), estimate_extent_(*params.get<mapnik::boolean_type>("estimate_extent", false)), max_async_connections_(*params_.get<value_integer>("max_async_connection", 1)), asynchronous_request_(false), // params below are for testing purposes only and may be removed at any time intersect_min_scale_(*params.get<value_integer>("intersect_min_scale", 0)), intersect_max_scale_(*params.get<value_integer>("intersect_max_scale", 0)) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::init"); #endif if (table_.empty()) { throw mapnik::datasource_exception("Pgraster Plugin: missing <table> parameter"); } boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext && !ext->empty()) { extent_initialized_ = extent_.from_string(*ext); } // NOTE: In multithread environment, pool_max_size_ should be // max_async_connections_ * num_threads if(max_async_connections_ > 1) { if(max_async_connections_ > pool_max_size_) { std::ostringstream err; err << "PostGIS Plugin: Error: 'max_async_connections (" << max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")"; throw mapnik::datasource_exception(err.str()); } asynchronous_request_ = true; } boost::optional<value_integer> initial_size = params.get<value_integer>("initial_size", 1); boost::optional<mapnik::boolean_type> autodetect_key_field = params.get<mapnik::boolean_type>("autodetect_key_field", false); ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_); CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); if (pool) { shared_ptr<Connection> conn = pool->borrowObject(); if (!conn) return; if (conn->isOK()) { desc_.set_encoding(conn->client_encoding()); if (raster_table_.empty()) { raster_table_ = mapnik::sql_utils::table_from_sql(table_); // non-trivial subqueries (having no FROM) make it // impossible to use overviews // TODO: improve "table_from_sql" ? if ( raster_table_[raster_table_.find_first_not_of(" \t\r\n")] == '(' ) { raster_table_.clear(); if ( use_overviews_ ) { std::ostringstream err; err << "Pgraster Plugin: overviews cannot be used " "with non-trivial subqueries"; MAPNIK_LOG_WARN(pgraster) << err.str(); use_overviews_ = false; } if ( ! extent_from_subquery_ ) { std::ostringstream err; err << "Pgraster Plugin: extent can only be computed " "from subquery as we could not found table source"; MAPNIK_LOG_WARN(pgraster) << err.str(); extent_from_subquery_ = true; } } } std::string::size_type idx = raster_table_.find_last_of('.'); if (idx != std::string::npos) { schema_ = raster_table_.substr(0, idx); raster_table_ = raster_table_.substr(idx + 1); } // If we do not know either the geometry_field or the srid or we // want to use overviews but do not know about schema, or // no extent was specified, then attempt to fetch the missing // information from a raster_columns entry. // // This will return no records if we are querying a bogus table returned // from the simplistic table parsing in table_from_sql() or if // the table parameter references a table, view, or subselect not // registered in the geometry columns. // geometryColumn_ = mapnik::sql_utils::unquote_double(raster_field_); if (!raster_table_.empty() && ( geometryColumn_.empty() || srid_ == 0 || (schema_.empty() && use_overviews_) || ! extent_initialized_ )) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::init(get_srid_and_geometry_column)"); #endif std::ostringstream s; try { s << "SELECT r_raster_column col, srid, r_table_schema"; if ( ! extent_initialized_ ) { s << ", st_xmin(extent) xmin, st_ymin(extent) ymin" << ", st_xmax(extent) xmax, st_ymax(extent) ymax"; } s << " FROM " << RASTER_COLUMNS << " WHERE r_table_name='" << mapnik::sql_utils::unquote_double(raster_table_) << "'"; if (! schema_.empty()) { s << " AND r_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'"; } if (! raster_field_.empty()) { s << " AND r_raster_column='" << mapnik::sql_utils::unquote_double(raster_field_) << "'"; } MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: running query " << s.str(); shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { geometryColumn_ = rs->getValue("col"); if ( ! extent_initialized_ ) { double lox, loy, hix, hiy; if (mapnik::util::string2double(rs->getValue("xmin"), lox) && mapnik::util::string2double(rs->getValue("ymin"), loy) && mapnik::util::string2double(rs->getValue("xmax"), hix) && mapnik::util::string2double(rs->getValue("ymax"), hiy)) { extent_.init(lox, loy, hix, hiy); extent_initialized_ = true; MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Layer extent=" << extent_; } else { MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str(); } } if (srid_ == 0) { const char* srid_c = rs->getValue("srid"); if (srid_c != nullptr) { int result = 0; const char * end = srid_c + std::strlen(srid_c); if (mapnik::util::string2int(srid_c, end, result)) { srid_ = result; } } } if ( schema_.empty() ) { schema_ = rs->getValue("r_table_schema"); } } else { MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no response from metadata query " << s.str(); } rs->close(); } catch (mapnik::datasource_exception const& ex) { // let this pass on query error and use the fallback below MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: metadata query failed: " << ex.what(); } // If we still do not know the srid then we can try to fetch // it from the 'table_' parameter, which should work even if it is // a subselect as long as we know the geometry_field to query if (! geometryColumn_.empty() && srid_ <= 0) { s.str(""); s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM " << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { const char* srid_c = rs->getValue("srid"); if (srid_c != nullptr) { int result = 0; const char * end = srid_c + std::strlen(srid_c); if (mapnik::util::string2int(srid_c, end, result)) { srid_ = result; } } } rs->close(); } } // If overviews were requested, take note of the max scale // of each available overview, sorted by scale descending if ( use_overviews_ ) { std::ostringstream err; if ( schema_.empty() ) { err << "Pgraster Plugin: unable to lookup available table" << " overviews due to unknown schema"; throw mapnik::datasource_exception(err.str()); } if ( geometryColumn_.empty() ) { err << "Pgraster Plugin: unable to lookup available table" << " overviews due to unknown column name"; throw mapnik::datasource_exception(err.str()); } std::ostringstream s; s << "select " "r.r_table_schema sch, " "r.r_table_name tab, " "r.r_raster_column col, " "greatest(abs(r.scale_x), abs(r.scale_y)) scl " "from" " raster_overviews o," " raster_columns r " "where" " o.r_table_schema = '" << mapnik::sql_utils::unquote_double(schema_) << "' and o.r_table_name = '" << mapnik::sql_utils::unquote_double(raster_table_) << "' and o.r_raster_column = '" << mapnik::sql_utils::unquote_double(geometryColumn_) << "' and r.r_table_schema = o.o_table_schema" " and r.r_table_name = o.o_table_name" " and r.r_raster_column = o.o_raster_column" " ORDER BY scl ASC"; MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: running query " << s.str(); shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); while (rs->next()) { pgraster_overview ov = pgraster_overview(); ov.schema = rs->getValue("sch"); ov.table = rs->getValue("tab"); ov.column = rs->getValue("col"); ov.scale = atof(rs->getValue("scl")); if(ov.scale == 0.0f) { MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: found invalid overview " << ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale; continue; } overviews_.push_back(ov); MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: found overview " << ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale; } rs->close(); if ( overviews_.empty() ) { MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no overview found for " << schema_ << "." << raster_table_ << "." << geometryColumn_; } } // detect primary key if (*autodetect_key_field && key_field_.empty()) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_primary_key)"); #endif std::ostringstream s; s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') " "AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i " "WHERE a.attnum > 0 AND a.attrelid = c.oid " "AND a.atttypid = t.oid AND c.relnamespace = n.oid " "AND c.oid = i.indrelid AND i.indisprimary = 't' " "AND t.typname !~ '^geom' AND c.relname =" << " '" << mapnik::sql_utils::unquote_double(raster_table_) << "' " //"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1 << "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum " "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum " "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum " "OR i.indkey[9]=a.attnum) "; if (! schema_.empty()) { s << "AND n.nspname='" << mapnik::sql_utils::unquote_double(schema_) << "' "; } s << "ORDER BY a.attnum"; shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str()); if (rs_key->next()) { unsigned int result_rows = rs_key->size(); if (result_rows == 1) { bool is_int = (std::string(rs_key->getValue(3)) == "t"); if (is_int) { const char* key_field_string = rs_key->getValue(0); if (key_field_string) { key_field_ = std::string(key_field_string); MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: auto-detected key field of '" << key_field_ << "' on table '" << raster_table_ << "'"; } } else { // throw for cases like a numeric primary key, which is invalid // as it should be floating point (int numerics are useless) std::ostringstream err; err << "PostGIS Plugin: Error: '" << rs_key->getValue(0) << "' on table '" << raster_table_ << "' is not a valid integer primary key field\n"; throw mapnik::datasource_exception(err.str()); } } else if (result_rows > 1) { std::ostringstream err; err << "PostGIS Plugin: Error: '" << "multi column primary key detected but is not supported"; throw mapnik::datasource_exception(err.str()); } } rs_key->close(); } // if a globally unique key field/primary key is required // but still not known at this point, then throw if (*autodetect_key_field && key_field_.empty()) { throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required") + " but could not be detected for table '" + raster_table_ + "', please supply 'key_field' option to specify field to use for primary key"); } if (srid_ == 0) { srid_ = -1; MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Table " << table_ << " is using SRID=" << srid_; } // At this point the geometry_field may still not be known // but we'll catch that where more useful... MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using SRID=" << srid_; MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using geometry_column=" << geometryColumn_; // collect attribute desc #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_column_description)"); #endif std::ostringstream s; s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); int count = rs->getNumFields(); bool found_key_field = false; for (int i = 0; i < count; ++i) { std::string fld_name = rs->getFieldName(i); int type_oid = rs->getTypeOID(i); // validate type of key_field if (! found_key_field && ! key_field_.empty() && fld_name == key_field_) { if (type_oid == 20 || type_oid == 21 || type_oid == 23) { found_key_field = true; desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); } else { std::ostringstream error_s; error_s << "invalid type '"; std::ostringstream type_s; type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str()); if (rs_oid->next()) { error_s << rs_oid->getValue("typname") << "' (oid:" << rs_oid->getValue("oid") << ")"; } else { error_s << "oid:" << type_oid << "'"; } rs_oid->close(); error_s << " for key_field '" << fld_name << "' - " << "must be an integer primary key"; rs->close(); throw mapnik::datasource_exception(error_s.str()); } } else { switch (type_oid) { case 16: // bool desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean)); break; case 20: // int8 case 21: // int2 case 23: // int4 desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); break; case 700: // float4 case 701: // float8 case 1700: // numeric desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); break; case 1042: // bpchar case 1043: // varchar case 25: // text case 705: // literal desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); break; default: // should not get here #ifdef MAPNIK_LOG s.str(""); s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str()); if (rs_oid->next()) { std::string typname(rs_oid->getValue("typname")); if (typname != "geometry" && typname != "raster") { MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type=" << typname << " (oid:" << rs_oid->getValue("oid") << ")"; } } else { MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type_oid=" << type_oid; } rs_oid->close(); #endif break; } } } rs->close(); } // Close explicitly the connection so we can 'fork()' without sharing open connections conn->close(); } }
sqlite_datasource::sqlite_datasource(parameters const& params) : datasource(params), extent_(), extent_initialized_(false), type_(datasource::Vector), table_(*params.get<std::string>("table", "")), fields_(*params.get<std::string>("fields", "*")), metadata_(*params.get<std::string>("metadata", "")), geometry_table_(*params.get<std::string>("geometry_table", "")), geometry_field_(*params.get<std::string>("geometry_field", "")), index_table_(*params.get<std::string>("index_table", "")), key_field_(*params.get<std::string>("key_field", "")), row_offset_(*params.get<mapnik::value_integer>("row_offset", 0)), row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)), intersects_token_("!intersects!"), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8")), format_(mapnik::wkbAuto) { /* TODO - throw if no primary key but spatial index is present? - remove auto-indexing - if spatialite - leverage more of the metadata for geometry type detection */ #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "sqlite_datasource::init"); #endif boost::optional<std::string> file = params.get<std::string>("file"); if (! file) throw datasource_exception("Sqlite Plugin: missing <file> parameter"); boost::optional<std::string> base = params.get<std::string>("base"); if (base) dataset_name_ = *base + "/" + *file; else dataset_name_ = *file; if ((dataset_name_.compare(":memory:") != 0) && (!mapnik::util::exists(dataset_name_))) { throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); } use_spatial_index_ = *params.get<mapnik::boolean>("use_spatial_index", true); // TODO - remove this option once all datasources have an indexing api bool auto_index = *params.get<mapnik::boolean>("auto_index", true); boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); boost::optional<std::string> wkb = params.get<std::string>("wkb_format"); if (wkb) { if (*wkb == "spatialite") { format_ = mapnik::wkbSpatiaLite; } else if (*wkb == "generic") { format_ = mapnik::wkbGeneric; } else { format_ = mapnik::wkbAuto; } } // Populate init_statements_ // 1. Build attach database statements from the "attachdb" parameter // 2. Add explicit init statements from "initdb" parameter // Note that we do some extra work to make sure that any attached // databases are relative to directory containing dataset_name_. Sqlite // will default to attaching from cwd. Typicaly usage means that the // map loader will produce full paths here. boost::optional<std::string> attachdb = params.get<std::string>("attachdb"); if (attachdb) { parse_attachdb(*attachdb); } boost::optional<std::string> initdb = params.get<std::string>("initdb"); if (initdb) { init_statements_.push_back(*initdb); } // now actually create the connection and start executing setup sql dataset_ = std::make_shared<sqlite_connection>(dataset_name_); boost::optional<mapnik::value_integer> table_by_index = params.get<mapnik::value_integer>("table_by_index"); int passed_parameters = 0; passed_parameters += params.get<std::string>("table") ? 1 : 0; passed_parameters += table_by_index ? 1 : 0; if (passed_parameters > 1) { throw datasource_exception("SQLite Plugin: you can only select an by name " "('table' parameter), by number ('table_by_index' parameter), " "do not supply 2 or more of them at the same time" ); } if (table_by_index) { std::vector<std::string> tables; sqlite_utils::get_tables(dataset_,tables); if (*table_by_index < 0 || *table_by_index >= static_cast<int>(tables.size())) { std::ostringstream s; s << "SQLite Plugin: only " << tables.size() << " table(s) exist, cannot find table by index '" << *table_by_index << "'"; throw datasource_exception(s.str()); } table_ = tables[*table_by_index]; } if (table_.empty()) { throw mapnik::datasource_exception("Sqlite Plugin: missing <table> parameter"); } if (geometry_table_.empty()) { geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } // if 'table_' is a subquery then we try to deduce names // and types from the first row returned from that query using_subquery_ = false; if (table_ != geometry_table_) { using_subquery_ = true; } else { // attempt to auto-quote table if needed if (sqlite_utils::needs_quoting(table_)) { table_ = std::string("[") + table_ + "]"; geometry_table_ = table_; } } // Execute init_statements_ for (std::vector<std::string>::const_iterator iter = init_statements_.begin(); iter != init_statements_.end(); ++iter) { MAPNIK_LOG_DEBUG(sqlite) << "sqlite_datasource: Execute init sql=" << *iter; dataset_->execute(*iter); } bool found_types_via_subquery = false; if (using_subquery_) { std::ostringstream s; std::string query = populate_tokens(table_); s << "SELECT " << fields_ << " FROM (" << query << ") LIMIT 1"; found_types_via_subquery = sqlite_utils::detect_types_from_subquery( s.str(), geometry_field_, desc_, dataset_); } // TODO - consider removing this if (key_field_ == "rowid") { desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer)); } bool found_table = sqlite_utils::table_info(key_field_, found_types_via_subquery, geometry_field_, geometry_table_, desc_, dataset_); if (! found_table) { std::ostringstream s; s << "Sqlite Plugin: could not query table '" << geometry_table_ << "'"; if (using_subquery_) { s << " from subquery '" << table_ << "'"; } // report get available tables std::vector<std::string> tables; sqlite_utils::get_tables(dataset_,tables); if (tables.size() > 0) { s << " (available tables for " << dataset_name_ << " are: '" << boost::algorithm::join(tables, ", ") << "')"; } throw datasource_exception(s.str()); } if (geometry_field_.empty()) { std::ostringstream s; s << "Sqlite Plugin: unable to detect the column " << "containing a valid geometry on table '" << geometry_table_ << "'. " << "Please provide a column name by passing the 'geometry_field' option " << "or indicate a different spatial table to use by passing the 'geometry_table' option"; throw datasource_exception(s.str()); } if (index_table_.empty()) { // Generate implicit index_table name - need to do this after // we have discovered meta-data or else we don't know the column name index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_); } std::string index_db = sqlite_utils::index_for_db(dataset_name_); has_spatial_index_ = false; if (use_spatial_index_) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(use_spatial_index)"); #endif if (mapnik::util::exists(index_db)) { dataset_->execute("attach database '" + index_db + "' as " + index_table_); } has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_); if (!has_spatial_index_ && auto_index) { if (! key_field_.empty()) { std::ostringstream query; query << "SELECT " << geometry_field_ << "," << key_field_ << " FROM (" << geometry_table_ << ")"; #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(create_spatial_index)"); #endif std::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str()); if (sqlite_utils::create_spatial_index(index_db,index_table_,rs)) { //extent_initialized_ = true; has_spatial_index_ = true; if (mapnik::util::exists(index_db)) { dataset_->execute("attach database '" + index_db + "' as " + index_table_); } } } else { std::ostringstream s; s << "Sqlite Plugin: could not generate spatial index" << " for table '" << geometry_table_ << "'" << " as no primary key can be detected." << " You should either declare an INTEGER PRIMARY KEY" << " or set the 'key_field' option to force a" << " given field to be used as the primary key"; throw datasource_exception(s.str()); } } } if (! extent_initialized_) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "sqlite_datasource::init(detect_extent)"); #endif // TODO - clean this up - reducing arguments std::string query = populate_tokens(table_); if (!sqlite_utils::detect_extent(dataset_, has_spatial_index_, extent_, index_table_, metadata_, geometry_field_, geometry_table_, key_field_, query)) { std::ostringstream s; s << "Sqlite Plugin: extent could not be determined for table '" << geometry_table_ << "' and geometry field '" << geometry_field_ << "'" << " because an rtree spatial index is missing or empty." << " - either set the table 'extent' or create an rtree spatial index"; throw datasource_exception(s.str()); } } }
geos_datasource::geos_datasource(parameters const& params) : datasource(params), extent_(), extent_initialized_(false), type_(datasource::Vector), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding", "utf-8")), geometry_data_(""), geometry_data_name_("name"), geometry_id_(1) { boost::optional<std::string> geometry = params.get<std::string>("wkt"); if (! geometry) throw datasource_exception("missing <wkt> parameter"); geometry_string_ = *geometry; boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); boost::optional<int> id = params.get<int>("gid"); if (id) geometry_id_ = *id; boost::optional<std::string> gdata = params.get<std::string>("field_data"); if (gdata) geometry_data_ = *gdata; boost::optional<std::string> gdata_name = params.get<std::string>("field_name"); if (gdata_name) geometry_data_name_ = *gdata_name; desc_.add_descriptor(attribute_descriptor(geometry_data_name_, mapnik::String)); #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "geos_datasource::init"); #endif // open geos driver initGEOS(geos_notice, geos_error); // parse the string into geometry geometry_.set_feature(GEOSGeomFromWKT(geometry_string_.c_str())); if (*geometry_ == NULL || ! GEOSisValid(*geometry_)) { throw datasource_exception("GEOS Plugin: invalid <wkt> geometry specified"); } // try to obtain the extent from the geometry itself if (! extent_initialized_) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "geos_datasource::init(initialize_extent)"); #endif MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Initializing extent from geometry"; if (GEOSGeomTypeId(*geometry_) == GEOS_POINT) { double x, y; unsigned int size; const GEOSCoordSequence* cs = GEOSGeom_getCoordSeq(*geometry_); GEOSCoordSeq_getSize(cs, &size); GEOSCoordSeq_getX(cs, 0, &x); GEOSCoordSeq_getY(cs, 0, &y); extent_.init(x, y, x, y); extent_initialized_ = true; } else { geos_feature_ptr envelope (GEOSEnvelope(*geometry_)); if (*envelope != NULL && GEOSisValid(*envelope)) { #ifdef MAPNIK_LOG char* wkt = GEOSGeomToWKT(*envelope); MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Getting coord sequence from=" << wkt; GEOSFree(wkt); #endif const GEOSGeometry* exterior = GEOSGetExteriorRing(*envelope); if (exterior != NULL && GEOSisValid(exterior)) { const GEOSCoordSequence* cs = GEOSGeom_getCoordSeq(exterior); if (cs != NULL) { MAPNIK_LOG_DEBUG(geos) << "geos_datasource: Iterating boundary points"; double x, y; double minx = std::numeric_limits<float>::max(), miny = std::numeric_limits<float>::max(), maxx = -std::numeric_limits<float>::max(), maxy = -std::numeric_limits<float>::max(); unsigned int num_points; GEOSCoordSeq_getSize(cs, &num_points); for (unsigned int i = 0; i < num_points; ++i) { GEOSCoordSeq_getX(cs, i, &x); GEOSCoordSeq_getY(cs, i, &y); if (x < minx) minx = x; if (x > maxx) maxx = x; if (y < miny) miny = y; if (y > maxy) maxy = y; } extent_.init(minx, miny, maxx, maxy); extent_initialized_ = true; } } } } } if (! extent_initialized_) { throw datasource_exception("GEOS Plugin: cannot determine extent for <wkt> geometry"); } }
void shape_datasource::bind() const { if (is_bound_) return; if (!boost::filesystem::exists(shape_name_ + ".shp")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' does not exist"); } if (boost::filesystem::is_directory(shape_name_ + ".shp")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' appears to be a directory not a file"); } if (!boost::filesystem::exists(shape_name_ + ".dbf")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".dbf' does not exist"); } try { boost::shared_ptr<shape_io> shape_ref = boost::make_shared<shape_io>(shape_name_); init(*shape_ref); for (int i=0;i<shape_ref->dbf().num_fields();++i) { field_descriptor const& fd=shape_ref->dbf().descriptor(i); std::string fld_name=fd.name_; switch (fd.type_) { case 'C': // character case 'D': // Date case 'M': // Memo, a string case 'L': // logical case '@': // timestamp desc_.add_descriptor(attribute_descriptor(fld_name, String)); break; case 'N': case 'O': // double case 'F': // float { if (fd.dec_>0) { desc_.add_descriptor(attribute_descriptor(fld_name,Double,false,8)); } else { desc_.add_descriptor(attribute_descriptor(fld_name,Integer,false,4)); } break; } default: #ifdef MAPNIK_DEBUG // I - long // G - ole // + - autoincrement std::clog << "Shape Plugin: unknown type " << fd.type_ << std::endl; #endif break; } } // for indexed shapefiles we keep open the file descriptor for fast reads if (indexed_) { shape_ = shape_ref; } } catch (const datasource_exception& ex) { std::clog << "Shape Plugin: error processing field attributes, " << ex.what() << std::endl; throw; } catch (const std::exception& ex) { std::clog << "Shape Plugin: error processing field attributes, " << ex.what() << std::endl; throw; } catch (...) // exception: pipe_select_interrupter: Too many open files { std::clog << "Shape Plugin: error processing field attributes" << std::endl; throw; } is_bound_ = true; }
void sqlite_datasource::bind() const { if (is_bound_) return; if (!boost::filesystem::exists(dataset_name_)) throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); dataset_ = new sqlite_connection (dataset_name_); std::string table_name = mapnik::table_from_sql(table_); if (metadata_ != "" && ! extent_initialized_) { std::ostringstream s; s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_; s << " WHERE LOWER(f_table_name) = LOWER('" << table_name << "')"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { double xmin = rs->column_double (0); double ymin = rs->column_double (1); double xmax = rs->column_double (2); double ymax = rs->column_double (3); extent_.init (xmin,ymin,xmax,ymax); extent_initialized_ = true; } } if (use_spatial_index_) { std::ostringstream s; s << "SELECT COUNT (*) FROM sqlite_master"; s << " WHERE LOWER(name) = LOWER('idx_" << table_name << "_" << geometry_field_ << "')"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { use_spatial_index_ = rs->column_integer (0) == 1; } #ifdef MAPNIK_DEBUG if (! use_spatial_index_) std::clog << "Sqlite Plugin: cannot use the spatial index " << std::endl; #endif } { /* XXX - This is problematic, if we do not have at least a row, we cannot determine the right columns types and names as all column_type are SQLITE_NULL */ std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_name << ") LIMIT 1"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { for (int i = 0; i < rs->column_count (); ++i) { const int type_oid = rs->column_type (i); const char* fld_name = rs->column_name (i); switch (type_oid) { case SQLITE_INTEGER: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case SQLITE_FLOAT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case SQLITE_TEXT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case SQLITE_NULL: case SQLITE_BLOB: break; default: #ifdef MAPNIK_DEBUG std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl; #endif break; } } } } is_bound_ = true; }
shape_datasource::shape_datasource(parameters const& params) : datasource (params), type_(datasource::Vector), file_length_(0), indexed_(false), row_limit_(*params.get<mapnik::value_integer>("row_limit",0)), desc_(shape_datasource::name(), *params.get<std::string>("encoding","utf-8")) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "shape_datasource::init"); #endif boost::optional<std::string> file = params.get<std::string>("file"); if (!file) throw datasource_exception("Shape Plugin: missing <file> parameter"); boost::optional<std::string> base = params.get<std::string>("base"); if (base) shape_name_ = *base + "/" + *file; else shape_name_ = *file; boost::algorithm::ireplace_last(shape_name_,".shp",""); if (!mapnik::util::exists(shape_name_ + ".shp")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' does not exist"); } if (mapnik::util::is_directory(shape_name_ + ".shp")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".shp' appears to be a directory not a file"); } if (!mapnik::util::exists(shape_name_ + ".dbf")) { throw datasource_exception("Shape Plugin: shapefile '" + shape_name_ + ".dbf' does not exist"); } try { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "shape_datasource::init(get_column_description)"); #endif std::unique_ptr<shape_io> shape_ref = std::make_unique<shape_io>(shape_name_); init(*shape_ref); for (int i=0;i<shape_ref->dbf().num_fields();++i) { field_descriptor const& fd=shape_ref->dbf().descriptor(i); std::string fld_name=fd.name_; switch (fd.type_) { case 'C': // character case 'D': // date desc_.add_descriptor(attribute_descriptor(fld_name, String)); break; case 'L': // logical desc_.add_descriptor(attribute_descriptor(fld_name, Boolean)); break; case 'N': // numeric case 'O': // double case 'F': // float { if (fd.dec_>0) { desc_.add_descriptor(attribute_descriptor(fld_name,Double,false,8)); } else { desc_.add_descriptor(attribute_descriptor(fld_name,Integer,false,4)); } break; } default: // I - long // G - ole // + - autoincrement // @ - timestamp // B - binary // l - long // M - memo MAPNIK_LOG_ERROR(shape) << "shape_datasource: Unknown type=" << fd.type_; break; } } } catch (datasource_exception const& ex) { MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes, " << ex.what(); throw; } catch (const std::exception& ex) { MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes, " << ex.what(); throw; } catch (...) // exception: pipe_select_interrupter: Too many open files { MAPNIK_LOG_ERROR(shape) << "Shape Plugin: error processing field attributes"; throw; } }
void ogr_datasource::init(mapnik::parameters const& params) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "ogr_datasource::init"); #endif boost::optional<std::string> file = params.get<std::string>("file"); boost::optional<std::string> string = params.get<std::string>("string"); if (!string) string = params.get<std::string>("inline"); if (! file && ! string) { throw datasource_exception("missing <file> or <string> parameter"); } if (string) { dataset_name_ = *string; } else { boost::optional<std::string> base = params.get<std::string>("base"); if (base) { dataset_name_ = *base + "/" + *file; } else { dataset_name_ = *file; } } std::string driver = *params.get<std::string>("driver",""); if (! driver.empty()) { #if GDAL_VERSION_MAJOR >= 2 unsigned int nOpenFlags = GDAL_OF_READONLY | GDAL_OF_VECTOR; const char* papszAllowedDrivers[] = { driver.c_str(), nullptr }; dataset_ = reinterpret_cast<gdal_dataset_type>(GDALOpenEx(dataset_name_.c_str(),nOpenFlags,papszAllowedDrivers, nullptr, nullptr)); #else OGRSFDriver * ogr_driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver.c_str()); if (ogr_driver && ogr_driver != nullptr) { dataset_ = ogr_driver->Open((dataset_name_).c_str(), false); } #endif } else { // open ogr driver #if GDAL_VERSION_MAJOR >= 2 dataset_ = reinterpret_cast<gdal_dataset_type>(OGROpen(dataset_name_.c_str(), false, nullptr)); #else dataset_ = OGRSFDriverRegistrar::Open(dataset_name_.c_str(), false); #endif } if (! dataset_) { const std::string err = CPLGetLastErrorMsg(); if (err.size() == 0) { throw datasource_exception("OGR Plugin: connection failed: " + dataset_name_ + " was not found or is not a supported format"); } else { throw datasource_exception("OGR Plugin: " + err); } } // initialize layer boost::optional<std::string> layer_by_name = params.get<std::string>("layer"); boost::optional<mapnik::value_integer> layer_by_index = params.get<mapnik::value_integer>("layer_by_index"); boost::optional<std::string> layer_by_sql = params.get<std::string>("layer_by_sql"); int passed_parameters = 0; passed_parameters += layer_by_name ? 1 : 0; passed_parameters += layer_by_index ? 1 : 0; passed_parameters += layer_by_sql ? 1 : 0; if (passed_parameters > 1) { throw datasource_exception("OGR Plugin: you can only select an ogr layer by name " "('layer' parameter), by number ('layer_by_index' parameter), " "or by sql ('layer_by_sql' parameter), " "do not supply 2 or more of them at the same time" ); } if (layer_by_name) { layer_name_ = *layer_by_name; layer_.layer_by_name(dataset_, layer_name_); } else if (layer_by_index) { int num_layers = dataset_->GetLayerCount(); if (*layer_by_index >= num_layers) { std::ostringstream s; s << "OGR Plugin: only " << num_layers << " layer(s) exist, cannot find layer by index '" << *layer_by_index << "'"; throw datasource_exception(s.str()); } layer_.layer_by_index(dataset_, *layer_by_index); layer_name_ = layer_.layer_name(); } else if (layer_by_sql) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats_sql__(std::clog, "ogr_datasource::init(layer_by_sql)"); #endif layer_.layer_by_sql(dataset_, *layer_by_sql); layer_name_ = layer_.layer_name(); } else { std::string s("OGR Plugin: missing <layer> or <layer_by_index> or <layer_by_sql> parameter, available layers are: "); unsigned num_layers = dataset_->GetLayerCount(); bool layer_found = false; std::vector<std::string> layer_names; for (unsigned i = 0; i < num_layers; ++i ) { OGRLayer* ogr_layer = dataset_->GetLayer(i); OGRFeatureDefn* ogr_layer_def = ogr_layer->GetLayerDefn(); if (ogr_layer_def != 0) { layer_found = true; layer_names.push_back(std::string("'") + ogr_layer_def->GetName() + std::string("'")); } } if (! layer_found) { s += "None (no layers were found in dataset)"; } else { s += boost::algorithm::join(layer_names,", "); } throw datasource_exception(s); } if (! layer_.is_valid()) { std::ostringstream s; s << "OGR Plugin: "; if (layer_by_name) { s << "cannot find layer by name '" << *layer_by_name; } else if (layer_by_index) { s << "cannot find layer by index number '" << *layer_by_index; } else if (layer_by_sql) { s << "cannot find layer by sql query '" << *layer_by_sql; } s << "' in dataset '" << dataset_name_ << "'"; throw datasource_exception(s.str()); } // work with real OGR layer OGRLayer* layer = layer_.layer(); // initialize envelope boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext && !ext->empty()) { extent_.from_string(*ext); } else { OGREnvelope envelope; OGRErr e = layer->GetExtent(&envelope); if (e == OGRERR_FAILURE) { if (layer->GetFeatureCount() == 0) { MAPNIK_LOG_ERROR(ogr) << "could not determine extent, layer '" << layer->GetLayerDefn()->GetName() << "' appears to have no features"; } else { std::ostringstream s; s << "OGR Plugin: Cannot determine extent for layer '" << layer->GetLayerDefn()->GetName() << "'. Please provide a manual extent string (minx,miny,maxx,maxy)."; throw datasource_exception(s.str()); } } extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY); } // scan for index file // TODO - layer names don't match dataset name, so this will break for // any layer types of ogr than shapefiles, etc // fix here and in ogrindex size_t breakpoint = dataset_name_.find_last_of("."); if (breakpoint == std::string::npos) { breakpoint = dataset_name_.length(); } index_name_ = dataset_name_.substr(0, breakpoint) + ".ogrindex"; #if defined (_WINDOWS) std::ifstream index_file(mapnik::utf8_to_utf16(index_name_), std::ios::in | std::ios::binary); #else std::ifstream index_file(index_name_.c_str(), std::ios::in | std::ios::binary); #endif if (index_file) { indexed_ = true; index_file.close(); } #if 0 // TODO - enable this warning once the ogrindex tool is a bit more stable/mature else { MAPNIK_LOG_DEBUG(ogr) << "ogr_datasource: no ogrindex file found for " << dataset_name_ << ", use the 'ogrindex' program to build an index for faster rendering"; } #endif #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "ogr_datasource::init(get_column_description)"); #endif // deal with attributes descriptions OGRFeatureDefn* def = layer->GetLayerDefn(); if (def != 0) { const int fld_count = def->GetFieldCount(); for (int i = 0; i < fld_count; i++) { OGRFieldDefn* fld = def->GetFieldDefn(i); const std::string fld_name = fld->GetNameRef(); const OGRFieldType type_oid = fld->GetType(); switch (type_oid) { case OFTInteger: #if GDAL_VERSION_MAJOR >= 2 case OFTInteger64: #endif desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); break; case OFTReal: desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); break; case OFTString: case OFTWideString: // deprecated desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); break; case OFTBinary: desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object)); break; case OFTIntegerList: #if GDAL_VERSION_MAJOR >= 2 case OFTInteger64List: #endif case OFTRealList: case OFTStringList: case OFTWideStringList: // deprecated ! MAPNIK_LOG_WARN(ogr) << "ogr_datasource: Unhandled type_oid=" << type_oid; break; case OFTDate: case OFTTime: case OFTDateTime: // unhandled ! desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object)); MAPNIK_LOG_WARN(ogr) << "ogr_datasource: Unhandled type_oid=" << type_oid; break; } } } mapnik::parameters & extra_params = desc_.get_extra_parameters(); OGRSpatialReference * srs_ref = layer->GetSpatialRef(); char * srs_output = nullptr; if (srs_ref && srs_ref->exportToProj4( &srs_output ) == OGRERR_NONE ) { extra_params["proj4"] = mapnik::util::trim_copy(srs_output); } CPLFree(srs_output); }
void postgis_datasource::bind() const { if (is_bound_) return; boost::optional<int> initial_size = params_.get<int>("initial_size",1); boost::optional<int> max_size = params_.get<int>("max_size",10); ConnectionManager *mgr=ConnectionManager::instance(); mgr->registerPool(creator_, *initial_size, *max_size); shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id()); if (pool) { shared_ptr<Connection> conn = pool->borrowObject(); if (conn && conn->isOK()) { is_bound_ = true; PoolGuard<shared_ptr<Connection>, shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool); desc_.set_encoding(conn->client_encoding()); if(geometry_table_.empty()) { geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } std::string::size_type idx = geometry_table_.find_last_of('.'); if (idx!=std::string::npos) { schema_ = geometry_table_.substr(0,idx); geometry_table_ = geometry_table_.substr(idx+1); } else { geometry_table_ = geometry_table_.substr(0); } // If we do not know both the geometry_field and the srid // then first attempt to fetch the geometry name from a geometry_columns entry. // This will return no records if we are querying a bogus table returned // from the simplistic table parsing in table_from_sql() or if // the table parameter references a table, view, or subselect not // registered in the geometry columns. geometryColumn_ = geometry_field_; if (!geometryColumn_.length() > 0 || srid_ == 0) { std::ostringstream s; s << "SELECT f_geometry_column, srid FROM "; s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) <<"'"; if (schema_.length() > 0) s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'"; if (geometry_field_.length() > 0) s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'"; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs=conn->executeQuery(s.str()); if (rs->next()) { geometryColumn_ = rs->getValue("f_geometry_column"); if (srid_ == 0) { try { srid_ = lexical_cast<int>(rs->getValue("srid")); } catch (bad_lexical_cast &ex) { std::clog << "Postgis Plugin: SRID=" << rs->getValue("srid") << " " << ex.what() << std::endl; } } } rs->close(); // If we still do not know the srid then we can try to fetch // it from the 'table_' parameter, which should work even if it is // a subselect as long as we know the geometry_field to query if (geometryColumn_.length() && srid_ <= 0) { s.str(""); s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM "; s << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs=conn->executeQuery(s.str()); if (rs->next()) { try { srid_ = lexical_cast<int>(rs->getValue("srid")); } catch (bad_lexical_cast &ex) { std::clog << "Postgis Plugin: SRID=" << rs->getValue("srid") << " " << ex.what() << std::endl; } } rs->close(); } } if (srid_ == 0) { srid_ = -1; std::clog << "Postgis Plugin: SRID warning, using srid=-1 for '" << table_ << "'" << std::endl; } // At this point the geometry_field may still not be known // but we'll catch that where more useful... #ifdef MAPNIK_DEBUG std::clog << "Postgis Plugin: using SRID=" << srid_ << std::endl; std::clog << "Postgis Plugin: using geometry_column=" << geometryColumn_ << std::endl; #endif // collect attribute desc std::ostringstream s; s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs=conn->executeQuery(s.str()); int count = rs->getNumFields(); bool found_key_field = false; for (int i=0;i<count;++i) { std::string fld_name=rs->getFieldName(i); int type_oid = rs->getTypeOID(i); // validate type of key_field if (!found_key_field && !key_field_.empty() && fld_name == key_field_) { found_key_field = true; if (type_oid == 20 || type_oid == 21 || type_oid == 23) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); } else { std::ostringstream error_s; error_s << "invalid type '"; std::ostringstream type_s; type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str()); if (rs_oid->next()) { error_s << rs_oid->getValue("typname") << "' (oid:" << rs_oid->getValue("oid") << ")"; } else { error_s << "oid:" << type_oid << "'"; } rs_oid->close(); error_s << " for key_field '" << fld_name << "' - " << "must be an integer primary key"; rs->close(); throw mapnik::datasource_exception( error_s.str() ); } } else { switch (type_oid) { case 16: // bool desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Boolean)); break; case 20: // int8 case 21: // int2 case 23: // int4 desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case 700: // float4 case 701: // float8 case 1700: // numeric ?? desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); case 1042: // bpchar case 1043: // varchar case 25: // text desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; default: // should not get here #ifdef MAPNIK_DEBUG s.str(""); s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str()); if (rs_oid->next()) { std::clog << "Postgis Plugin: unknown type = " << rs_oid->getValue("typname") << " (oid:" << rs_oid->getValue("oid") << ")" << std::endl; } else { std::clog << "Postgis Plugin: unknown oid type =" << type_oid << std::endl; } rs_oid->close(); #endif break; } } } rs->close(); } } }
void postgis_datasource::bind() const { if (is_bound_) { return; } boost::optional<int> initial_size = params_.get<int>("initial_size", 1); boost::optional<int> max_size = params_.get<int>("max_size", 10); boost::optional<mapnik::boolean> autodetect_key_field = params_.get<mapnik::boolean>("autodetect_key_field", false); ConnectionManager* mgr = ConnectionManager::instance(); mgr->registerPool(creator_, *initial_size, *max_size); shared_ptr< Pool<Connection,ConnectionCreator> > pool = mgr->getPool(creator_.id()); if (pool) { shared_ptr<Connection> conn = pool->borrowObject(); if (conn && conn->isOK()) { PoolGuard<shared_ptr<Connection>, shared_ptr< Pool<Connection,ConnectionCreator> > > guard(conn, pool); desc_.set_encoding(conn->client_encoding()); if (geometry_table_.empty()) { geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } std::string::size_type idx = geometry_table_.find_last_of('.'); if (idx != std::string::npos) { schema_ = geometry_table_.substr(0, idx); geometry_table_ = geometry_table_.substr(idx + 1); } else { geometry_table_ = geometry_table_.substr(0); } // If we do not know both the geometry_field and the srid // then first attempt to fetch the geometry name from a geometry_columns entry. // This will return no records if we are querying a bogus table returned // from the simplistic table parsing in table_from_sql() or if // the table parameter references a table, view, or subselect not // registered in the geometry columns. geometryColumn_ = geometry_field_; if (geometryColumn_.empty() || srid_ == 0) { std::ostringstream s; s << "SELECT f_geometry_column, srid FROM " << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) << "'"; if (! schema_.empty()) { s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'"; } if (! geometry_field_.empty()) { s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'"; } /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { geometryColumn_ = rs->getValue("f_geometry_column"); if (srid_ == 0) { const char* srid_c = rs->getValue("srid"); if (srid_c != NULL) { int result = 0; if (mapnik::util::string2int(srid_c, result)) { srid_ = result; } } } } rs->close(); // If we still do not know the srid then we can try to fetch // it from the 'table_' parameter, which should work even if it is // a subselect as long as we know the geometry_field to query if (! geometryColumn_.empty() && srid_ <= 0) { s.str(""); s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM " << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { const char* srid_c = rs->getValue("srid"); if (srid_c != NULL) { int result = 0; if (mapnik::util::string2int(srid_c, result)) { srid_ = result; } } } rs->close(); } } // detect primary key if (*autodetect_key_field && key_field_.empty()) { std::ostringstream s; s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') " "AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i " "WHERE a.attnum > 0 AND a.attrelid = c.oid " "AND a.atttypid = t.oid AND c.relnamespace = n.oid " "AND c.oid = i.indrelid AND i.indisprimary = 't' " "AND t.typname !~ '^geom' AND c.relname =" << " '" << mapnik::sql_utils::unquote_double(geometry_table_) << "' " //"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1 << "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum " "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum " "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum " "OR i.indkey[9]=a.attnum) "; if (! schema_.empty()) { s << "AND n.nspname='" << mapnik::sql_utils::unquote_double(schema_) << "' "; } s << "ORDER BY a.attnum"; shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str()); if (rs_key->next()) { unsigned int result_rows = rs_key->size(); if (result_rows == 1) { bool is_int = (std::string(rs_key->getValue(3)) == "t"); if (is_int) { const char* key_field_string = rs_key->getValue(0); if (key_field_string) { key_field_ = std::string(key_field_string); #ifdef MAPNIK_DEBUG std::clog << "PostGIS Plugin: auto-detected key field of '" << key_field_ << "' on table '" << geometry_table_ << "'\n"; #endif } } else { // throw for cases like a numeric primary key, which is invalid // as it should be floating point (int numerics are useless) std::ostringstream err; err << "PostGIS Plugin: Error: '" << rs_key->getValue(0) << "' on table '" << geometry_table_ << "' is not a valid integer primary key field\n"; throw mapnik::datasource_exception(err.str()); } } else if (result_rows > 1) { std::ostringstream err; err << "PostGIS Plugin: Error: '" << "multi column primary key detected but is not supported"; throw mapnik::datasource_exception(err.str()); } } rs_key->close(); } // if a globally unique key field/primary key is required // but still not known at this point, then throw if (*autodetect_key_field && key_field_.empty()) { throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required") + " but could not be detected for table '" + geometry_table_ + "', please supply 'key_field' option to specify field to use for primary key"); } if (srid_ == 0) { srid_ = -1; #ifdef MAPNIK_DEBUG std::clog << "Postgis Plugin: SRID warning, using srid=-1 for '" << table_ << "'" << std::endl; #endif } // At this point the geometry_field may still not be known // but we'll catch that where more useful... #ifdef MAPNIK_DEBUG std::clog << "Postgis Plugin: using SRID=" << srid_ << std::endl; std::clog << "Postgis Plugin: using geometry_column=" << geometryColumn_ << std::endl; #endif // collect attribute desc std::ostringstream s; s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); int count = rs->getNumFields(); bool found_key_field = false; for (int i = 0; i < count; ++i) { std::string fld_name = rs->getFieldName(i); int type_oid = rs->getTypeOID(i); // validate type of key_field if (! found_key_field && ! key_field_.empty() && fld_name == key_field_) { if (type_oid == 20 || type_oid == 21 || type_oid == 23) { found_key_field = true; desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); } else { std::ostringstream error_s; error_s << "invalid type '"; std::ostringstream type_s; type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str()); if (rs_oid->next()) { error_s << rs_oid->getValue("typname") << "' (oid:" << rs_oid->getValue("oid") << ")"; } else { error_s << "oid:" << type_oid << "'"; } rs_oid->close(); error_s << " for key_field '" << fld_name << "' - " << "must be an integer primary key"; rs->close(); throw mapnik::datasource_exception(error_s.str()); } } else { switch (type_oid) { case 16: // bool desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean)); break; case 20: // int8 case 21: // int2 case 23: // int4 desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); break; case 700: // float4 case 701: // float8 case 1700: // numeric desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); case 1042: // bpchar case 1043: // varchar case 25: // text desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); break; default: // should not get here #ifdef MAPNIK_DEBUG s.str(""); s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; /* if (show_queries_) { std::clog << boost::format("PostGIS: sending query: %s\n") % s.str(); } */ shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str()); if (rs_oid->next()) { std::clog << "Postgis Plugin: unknown type = " << rs_oid->getValue("typname") << " (oid:" << rs_oid->getValue("oid") << ")" << std::endl; } else { std::clog << "Postgis Plugin: unknown oid type =" << type_oid << std::endl; } rs_oid->close(); #endif break; } } } rs->close(); is_bound_ = true; } } }
ogr_datasource::ogr_datasource(parameters const& params) : datasource(params), extent_(), type_(datasource::Vector), desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")), indexed_(false) { OGRRegisterAll(); boost::optional<std::string> file = params.get<std::string>("file"); if (!file) throw datasource_exception("missing <file> parameter"); multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false); boost::optional<std::string> base = params.get<std::string>("base"); if (base) dataset_name_ = *base + "/" + *file; else dataset_name_ = *file; // open ogr driver dataset_ = OGRSFDriverRegistrar::Open ((dataset_name_).c_str(), FALSE); if (!dataset_) { std::string err = CPLGetLastErrorMsg(); if( err.size() == 0 ) { throw datasource_exception("Connection failed: " + dataset_name_ + " was not found or is not a supported format"); } else { throw datasource_exception(err); } } // initialize layer boost::optional<std::string> layer = params.get<std::string>("layer"); if (!layer) { std::string s ("missing <layer> parameter, available layers are: "); unsigned num_layers = dataset_->GetLayerCount(); for (unsigned i = 0; i < num_layers; ++i ) { OGRLayer *ogr_layer = dataset_->GetLayer(i); OGRFeatureDefn* def = ogr_layer->GetLayerDefn(); if (def != 0) { s += " '"; s += def->GetName(); s += "' "; } else { s += "No layers found!"; } } throw datasource_exception(s); } layerName_ = *layer; layer_ = dataset_->GetLayerByName (layerName_.c_str()); if (! layer_) throw datasource_exception("cannot find <layer> in dataset"); // initialize envelope OGREnvelope envelope; layer_->GetExtent (&envelope); extent_.init (envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY); // scan for index file size_t breakpoint = dataset_name_.find_last_of ("."); if (breakpoint == std::string::npos) breakpoint = dataset_name_.length(); index_name_ = dataset_name_.substr(0, breakpoint) + ".index"; std::ifstream index_file (index_name_.c_str(), std::ios::in | std::ios::binary); if (index_file) { indexed_=true; index_file.close(); } // deal with attributes descriptions OGRFeatureDefn* def = layer_->GetLayerDefn (); if (def != 0) { int fld_count = def->GetFieldCount (); for (int i = 0; i < fld_count; i++) { OGRFieldDefn* fld = def->GetFieldDefn (i); std::string fld_name = fld->GetNameRef (); OGRFieldType type_oid = fld->GetType (); switch (type_oid) { case OFTInteger: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case OFTReal: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case OFTString: case OFTWideString: // deprecated desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case OFTBinary: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object)); break; case OFTIntegerList: case OFTRealList: case OFTStringList: case OFTWideStringList: // deprecated ! #ifdef MAPNIK_DEBUG clog << "unhandled type_oid=" << type_oid << endl; #endif break; case OFTDate: case OFTTime: case OFTDateTime: // unhandled ! #ifdef MAPNIK_DEBUG clog << "unhandled type_oid=" << type_oid << endl; #endif desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object)); break; default: // unknown #ifdef MAPNIK_DEBUG clog << "unknown type_oid=" << type_oid << endl; #endif break; } } } }
void ogr_datasource::bind() const { if (is_bound_) return; // initialize ogr formats OGRRegisterAll(); std::string driver = *params_.get<std::string>("driver",""); if (! driver.empty()) { OGRSFDriver * ogr_driver = OGRSFDriverRegistrar::GetRegistrar()->GetDriverByName(driver.c_str()); if (ogr_driver && ogr_driver != NULL) { dataset_ = ogr_driver->Open((dataset_name_).c_str(), FALSE); } } else { // open ogr driver dataset_ = OGRSFDriverRegistrar::Open((dataset_name_).c_str(), FALSE); } if (! dataset_) { const std::string err = CPLGetLastErrorMsg(); if (err.size() == 0) { throw datasource_exception("OGR Plugin: connection failed: " + dataset_name_ + " was not found or is not a supported format"); } else { throw datasource_exception("OGR Plugin: " + err); } } // initialize layer boost::optional<std::string> layer_by_name = params_.get<std::string>("layer"); boost::optional<unsigned> layer_by_index = params_.get<unsigned>("layer_by_index"); boost::optional<std::string> layer_by_sql = params_.get<std::string>("layer_by_sql"); int passed_parameters = 0; passed_parameters += layer_by_name ? 1 : 0; passed_parameters += layer_by_index ? 1 : 0; passed_parameters += layer_by_sql ? 1 : 0; if (passed_parameters > 1) { throw datasource_exception("OGR Plugin: you can only select an ogr layer by name " "('layer' parameter), by number ('layer_by_index' parameter), " "or by sql ('layer_by_sql' parameter), " "do not supply 2 or more of them at the same time" ); } if (layer_by_name) { layer_name_ = *layer_by_name; layer_.layer_by_name(dataset_, layer_name_); } else if (layer_by_index) { const unsigned num_layers = dataset_->GetLayerCount(); if (*layer_by_index >= num_layers) { std::ostringstream s; s << "OGR Plugin: only "; s << num_layers; s << " layer(s) exist, cannot find layer by index '" << *layer_by_index << "'"; throw datasource_exception(s.str()); } layer_.layer_by_index(dataset_, *layer_by_index); layer_name_ = layer_.layer_name(); } else if (layer_by_sql) { layer_.layer_by_sql(dataset_, *layer_by_sql); layer_name_ = layer_.layer_name(); } else { std::ostringstream s; s << "OGR Plugin: missing <layer> or <layer_by_index> or <layer_by_sql> " << "parameter, available layers are: "; unsigned num_layers = dataset_->GetLayerCount(); bool layer_found = false; for (unsigned i = 0; i < num_layers; ++i ) { OGRLayer* ogr_layer = dataset_->GetLayer(i); OGRFeatureDefn* ogr_layer_def = ogr_layer->GetLayerDefn(); if (ogr_layer_def != 0) { layer_found = true; s << " '" << ogr_layer_def->GetName() << "' "; } } if (! layer_found) { s << "None (no layers were found in dataset)"; } throw datasource_exception(s.str()); } if (! layer_.is_valid()) { std::string s("OGR Plugin: "); if (layer_by_name) { s += "cannot find layer by name '" + *layer_by_name; } else if (layer_by_index) { s += "cannot find layer by index number '" + *layer_by_index; } else if (layer_by_sql) { s += "cannot find layer by sql query '" + *layer_by_sql; } s += "' in dataset '" + dataset_name_ + "'"; throw datasource_exception(s); } // work with real OGR layer OGRLayer* layer = layer_.layer(); // initialize envelope OGREnvelope envelope; layer->GetExtent(&envelope); extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY); // scan for index file // TODO - layer names don't match dataset name, so this will break for // any layer types of ogr than shapefiles, etc // fix here and in ogrindex size_t breakpoint = dataset_name_.find_last_of("."); if (breakpoint == std::string::npos) { breakpoint = dataset_name_.length(); } index_name_ = dataset_name_.substr(0, breakpoint) + ".ogrindex"; std::ifstream index_file(index_name_.c_str(), std::ios::in | std::ios::binary); if (index_file) { indexed_ = true; index_file.close(); } #if 0 // TODO - enable this warning once the ogrindex tool is a bit more stable/mature else { std::clog << "### Notice: no ogrindex file found for " << dataset_name_ << ", use the 'ogrindex' program to build an index for faster rendering" << std::endl; } #endif // deal with attributes descriptions OGRFeatureDefn* def = layer->GetLayerDefn(); if (def != 0) { const int fld_count = def->GetFieldCount(); for (int i = 0; i < fld_count; i++) { OGRFieldDefn* fld = def->GetFieldDefn(i); const std::string fld_name = fld->GetNameRef(); const OGRFieldType type_oid = fld->GetType(); switch (type_oid) { case OFTInteger: desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); break; case OFTReal: desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); break; case OFTString: case OFTWideString: // deprecated desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); break; case OFTBinary: desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object)); break; case OFTIntegerList: case OFTRealList: case OFTStringList: case OFTWideStringList: // deprecated ! #ifdef MAPNIK_DEBUG std::clog << "OGR Plugin: unhandled type_oid=" << type_oid << std::endl; #endif break; case OFTDate: case OFTTime: case OFTDateTime: // unhandled ! #ifdef MAPNIK_DEBUG std::clog << "OGR Plugin: unhandled type_oid=" << type_oid << std::endl; #endif desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Object)); break; default: // unknown #ifdef MAPNIK_DEBUG std::clog << "OGR Plugin: unknown type_oid=" << type_oid << std::endl; #endif break; } } } is_bound_ = true; }
postgis_datasource::postgis_datasource(parameters const& params) : datasource(params), table_(*params.get<std::string>("table", "")), schema_(""), geometry_table_(*params.get<std::string>("geometry_table", "")), geometry_field_(*params.get<std::string>("geometry_field", "")), key_field_(*params.get<std::string>("key_field", "")), cursor_fetch_size_(*params.get<mapnik::value_integer>("cursor_size", 0)), row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)), type_(datasource::Vector), srid_(*params.get<mapnik::value_integer>("srid", 0)), extent_initialized_(false), simplify_geometries_(false), desc_(postgis_datasource::name(), "utf-8"), creator_(params.get<std::string>("host"), params.get<std::string>("port"), params.get<std::string>("dbname"), params.get<std::string>("user"), params.get<std::string>("password"), params.get<std::string>("connect_timeout", "4")), bbox_token_("!bbox!"), scale_denom_token_("!scale_denominator!"), pixel_width_token_("!pixel_width!"), pixel_height_token_("!pixel_height!"), pool_max_size_(*params_.get<mapnik::value_integer>("max_size", 10)), persist_connection_(*params.get<mapnik::boolean_type>("persist_connection", true)), extent_from_subquery_(*params.get<mapnik::boolean_type>("extent_from_subquery", false)), max_async_connections_(*params_.get<mapnik::value_integer>("max_async_connection", 1)), asynchronous_request_(false), // TODO - use for known tokens too: "(@\\w+|!\\w+!)" pattern_(boost::regex("(@\\w+)",boost::regex::normal | boost::regbase::icase)), // params below are for testing purposes only and may be removed at any time intersect_min_scale_(*params.get<mapnik::value_integer>("intersect_min_scale", 0)), intersect_max_scale_(*params.get<mapnik::value_integer>("intersect_max_scale", 0)) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "postgis_datasource::init"); #endif if (table_.empty()) { throw mapnik::datasource_exception("Postgis Plugin: missing <table> parameter"); } boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext && !ext->empty()) { extent_initialized_ = extent_.from_string(*ext); } // NOTE: In multithread environment, pool_max_size_ should be // max_async_connections_ * num_threads if(max_async_connections_ > 1) { if(max_async_connections_ > pool_max_size_) { std::ostringstream err; err << "PostGIS Plugin: Error: 'max_async_connections (" << max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")"; throw mapnik::datasource_exception(err.str()); } asynchronous_request_ = true; } boost::optional<mapnik::value_integer> initial_size = params.get<mapnik::value_integer>("initial_size", 1); boost::optional<mapnik::boolean_type> autodetect_key_field = params.get<mapnik::boolean_type>("autodetect_key_field", false); boost::optional<mapnik::boolean_type> estimate_extent = params.get<mapnik::boolean_type>("estimate_extent", false); estimate_extent_ = estimate_extent && *estimate_extent; boost::optional<mapnik::boolean_type> simplify_opt = params.get<mapnik::boolean_type>("simplify_geometries", false); simplify_geometries_ = simplify_opt && *simplify_opt; ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_); CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); if (pool) { shared_ptr<Connection> conn = pool->borrowObject(); if (!conn) return; if (conn->isOK()) { desc_.set_encoding(conn->client_encoding()); if (geometry_table_.empty()) { geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } std::string::size_type idx = geometry_table_.find_last_of('.'); if (idx != std::string::npos) { schema_ = geometry_table_.substr(0, idx); geometry_table_ = geometry_table_.substr(idx + 1); } else { geometry_table_ = geometry_table_.substr(0); } // NOTE: geometry_table_ how should ideally be a table name, but // there are known edge cases where this will break down and // geometry_table_ may even be empty: https://github.com/mapnik/mapnik/issues/2718 // If we do not know both the geometry_field and the srid // then first attempt to fetch the geometry name from a geometry_columns entry. // This will return no records if we are querying a bogus table returned // from the simplistic table parsing in table_from_sql() or if // the table parameter references a table, view, or subselect not // registered in the geometry columns. geometryColumn_ = geometry_field_; if (!geometry_table_.empty() && (geometryColumn_.empty() || srid_ == 0)) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::init(get_srid_and_geometry_column)"); #endif std::ostringstream s; try { s << "SELECT f_geometry_column, srid FROM " << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) << "'"; if (! schema_.empty()) { s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'"; } if (! geometry_field_.empty()) { s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'"; } shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { geometryColumn_ = rs->getValue("f_geometry_column"); // only accept srid from geometry_tables if // user has not provided as option if (srid_ == 0) { const char* srid_c = rs->getValue("srid"); if (srid_c != nullptr) { int result = 0; const char * end = srid_c + std::strlen(srid_c); if (mapnik::util::string2int(srid_c, end, result)) { srid_ = result; } } } } rs->close(); } catch (mapnik::datasource_exception const& ex) { // let this pass on query error and use the fallback below MAPNIK_LOG_WARN(postgis) << "postgis_datasource: metadata query failed: " << ex.what(); } } // If we still do not know the srid then we can try to fetch // it from the 'geometry_table_' parameter, which should work even if it is // a subselect as long as we know the geometry_field to query if (!geometryColumn_.empty() && srid_ <= 0) { std::ostringstream s; s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM "; if (!geometry_table_.empty()) { s << geometry_table_; } else { s << populate_tokens(table_); } s << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); if (rs->next()) { const char* srid_c = rs->getValue("srid"); if (srid_c != nullptr) { int result = 0; const char * end = srid_c + std::strlen(srid_c); if (mapnik::util::string2int(srid_c, end, result)) { srid_ = result; } } } rs->close(); } // detect primary key if (*autodetect_key_field && key_field_.empty()) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_primary_key)"); #endif std::ostringstream s; s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') " "AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i " "WHERE a.attnum > 0 AND a.attrelid = c.oid " "AND a.atttypid = t.oid AND c.relnamespace = n.oid " "AND c.oid = i.indrelid AND i.indisprimary = 't' " "AND t.typname !~ '^geom' AND c.relname =" << " '" << mapnik::sql_utils::unquote_double(geometry_table_) << "' " //"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1 << "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum " "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum " "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum " "OR i.indkey[9]=a.attnum) "; if (! schema_.empty()) { s << "AND n.nspname='" << mapnik::sql_utils::unquote_double(schema_) << "' "; } s << "ORDER BY a.attnum"; shared_ptr<ResultSet> rs_key = conn->executeQuery(s.str()); if (rs_key->next()) { unsigned int result_rows = rs_key->size(); if (result_rows == 1) { bool is_int = (std::string(rs_key->getValue(3)) == "t"); if (is_int) { const char* key_field_string = rs_key->getValue(0); if (key_field_string) { key_field_ = std::string(key_field_string); MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: auto-detected key field of '" << key_field_ << "' on table '" << geometry_table_ << "'"; } } else { // throw for cases like a numeric primary key, which is invalid // as it should be floating point (int numerics are useless) std::ostringstream err; err << "PostGIS Plugin: Error: '" << rs_key->getValue(0) << "' on table '" << geometry_table_ << "' is not a valid integer primary key field\n"; throw mapnik::datasource_exception(err.str()); } } else if (result_rows > 1) { std::ostringstream err; err << "PostGIS Plugin: Error: '" << "multi column primary key detected but is not supported"; throw mapnik::datasource_exception(err.str()); } } rs_key->close(); } // if a globally unique key field/primary key is required // but still not known at this point, then throw if (*autodetect_key_field && key_field_.empty()) { throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required") + " but could not be detected for table '" + geometry_table_ + "', please supply 'key_field' option to specify field to use for primary key"); } if (srid_ == 0) { srid_ = -1; MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Table " << table_ << " is using SRID=" << srid_; } // At this point the geometry_field may still not be known // but we'll catch that where more useful... MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Using SRID=" << srid_; MAPNIK_LOG_DEBUG(postgis) << "postgis_datasource: Using geometry_column=" << geometryColumn_; // collect attribute desc #ifdef MAPNIK_STATS mapnik::progress_timer __stats2__(std::clog, "postgis_datasource::bind(get_column_description)"); #endif std::ostringstream s; s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; shared_ptr<ResultSet> rs = conn->executeQuery(s.str()); int count = rs->getNumFields(); bool found_key_field = false; for (int i = 0; i < count; ++i) { std::string fld_name = rs->getFieldName(i); int type_oid = rs->getTypeOID(i); // validate type of key_field if (! found_key_field && ! key_field_.empty() && fld_name == key_field_) { if (type_oid == 20 || type_oid == 21 || type_oid == 23) { found_key_field = true; desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); } else { std::ostringstream error_s; error_s << "invalid type '"; std::ostringstream type_s; type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(type_s.str()); if (rs_oid->next()) { error_s << rs_oid->getValue("typname") << "' (oid:" << rs_oid->getValue("oid") << ")"; } else { error_s << "oid:" << type_oid << "'"; } rs_oid->close(); error_s << " for key_field '" << fld_name << "' - " << "must be an integer primary key"; rs->close(); throw mapnik::datasource_exception(error_s.str()); } } else { switch (type_oid) { case 16: // bool desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean)); break; case 20: // int8 case 21: // int2 case 23: // int4 desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); break; case 700: // float4 case 701: // float8 case 1700: // numeric desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); break; case 1042: // bpchar case 1043: // varchar case 25: // text case 705: // literal desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); break; default: // should not get here #ifdef MAPNIK_LOG s.str(""); s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str()); if (rs_oid->next()) { std::string typname(rs_oid->getValue("typname")); if (typname != "geometry") { MAPNIK_LOG_WARN(postgis) << "postgis_datasource: Unknown type=" << typname << " (oid:" << rs_oid->getValue("oid") << ")"; } } else { MAPNIK_LOG_WARN(postgis) << "postgis_datasource: Unknown type_oid=" << type_oid; } rs_oid->close(); #endif break; } } } rs->close(); } // Close explicitly the connection so we can 'fork()' without sharing open connections conn->close(); // Finally, add unique metadata to layer descriptor mapnik::parameters & extra_params = desc_.get_extra_parameters(); // explicitly make copies of values due to https://github.com/mapnik/mapnik/issues/2651 extra_params["srid"] = srid_; if (!key_field_.empty()) { extra_params["key_field"] = key_field_; } } }
occi_datasource::occi_datasource(parameters const& params) : datasource (params), type_(datasource::Vector), fields_(*params.get<std::string>("fields", "*")), geometry_field_(*params.get<std::string>("geometry_field", "")), srid_initialized_(false), extent_initialized_(false), bbox_token_("!bbox!"), scale_denom_token_("!scale_denominator!"), pixel_width_token_("!pixel_width!"), pixel_height_token_("!pixel_height!"), desc_(occi_datasource::name(), *params.get<std::string>("encoding", "utf-8")), use_wkb_(*params.get<mapnik::boolean_type>("use_wkb", false)), row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)), row_prefetch_(*params.get<mapnik::value_integer>("row_prefetch", 100)), pool_(0), conn_(0) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::init"); #endif if (! params.get<std::string>("user")) throw datasource_exception("OCCI Plugin: no <user> specified"); if (! params.get<std::string>("password")) throw datasource_exception("OCCI Plugin: no <password> specified"); if (! params.get<std::string>("host")) throw datasource_exception("OCCI Plugin: no <host> string specified"); boost::optional<std::string> table = params.get<std::string>("table"); if (! table) { throw datasource_exception("OCCI Plugin: no <table> parameter specified"); } else { table_ = *table; } estimate_extent_ = *params.get<mapnik::boolean_type>("estimate_extent",false); use_spatial_index_ = *params.get<mapnik::boolean_type>("use_spatial_index",true); use_connection_pool_ = *params.get<mapnik::boolean_type>("use_connection_pool",true); boost::optional<std::string> ext = params.get<std::string>("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); boost::optional<mapnik::value_integer> srid = params.get<mapnik::value_integer>("srid"); if (srid) { srid_ = *srid; srid_initialized_ = true; } // connect to environment if (use_connection_pool_) { try { pool_ = occi_environment::instance().create_pool( *params.get<std::string>("user"), *params.get<std::string>("password"), *params.get<std::string>("host"), *params.get<mapnik::value_integer>("max_size", 5), *params.get<mapnik::value_integer>("initial_size", 1), 1); } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } else { try { conn_ = occi_environment::instance().create_connection( *params.get<std::string>("user"), *params.get<std::string>("password"), *params.get<std::string>("host")); } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } // extract real table name table_name_ = mapnik::sql_utils::table_from_sql(table_); // get SRID and/or GEOMETRY_FIELD from metadata table only if we need to if (! srid_initialized_ || geometry_field_ == "") { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_srid_and_geometry_field"); #endif std::ostringstream s; s << "SELECT srid, column_name FROM " << METADATA_TABLE << " WHERE"; s << " LOWER(table_name) = LOWER('" << table_name_ << "')"; if (geometry_field_ != "") { s << " AND LOWER(column_name) = LOWER('" << geometry_field_ << "')"; } MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str(); try { occi_connection_ptr conn; if (use_connection_pool_) conn.set_pool(pool_); else conn.set_connection(conn_, false); ResultSet* rs = conn.execute_query(s.str()); if (rs && rs->next ()) { if (! srid_initialized_) { srid_ = rs->getInt(1); srid_initialized_ = true; } if (geometry_field_ == "") { geometry_field_ = rs->getString(2); } } } catch (SQLException& ex) { throw datasource_exception("OCCI Plugin: " + ex.getMessage()); } } // get columns description { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "occi_datasource::get_column_description"); #endif std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_name_ << ") WHERE ROWNUM < 1"; MAPNIK_LOG_DEBUG(occi) << "occi_datasource: " << s.str(); try { occi_connection_ptr conn; if (use_connection_pool_) conn.set_pool(pool_); else conn.set_connection(conn_, false); ResultSet* rs = conn.execute_query(s.str()); if (rs) { std::vector<MetaData> listOfColumns = rs->getColumnListMetaData(); for (unsigned int i = 0; i < listOfColumns.size(); ++i) { MetaData columnObj = listOfColumns[i]; std::string fld_name = columnObj.getString(MetaData::ATTR_NAME); int type_oid = columnObj.getInt(MetaData::ATTR_DATA_TYPE); /* int type_code = columnObj.getInt(MetaData::ATTR_TYPECODE); if (type_code == OCCI_TYPECODE_OBJECT) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Object)); continue; } */ switch (type_oid) { case oracle::occi::OCCIBOOL: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Boolean)); break; case oracle::occi::OCCIINT: case oracle::occi::OCCIUNSIGNED_INT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case oracle::occi::OCCIFLOAT: case oracle::occi::OCCIBFLOAT: case oracle::occi::OCCIDOUBLE: case oracle::occi::OCCIBDOUBLE: case oracle::occi::OCCINUMBER: case oracle::occi::OCCI_SQLT_NUM: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case oracle::occi::OCCICHAR: case oracle::occi::OCCISTRING: case oracle::occi::OCCI_SQLT_AFC: case oracle::occi::OCCI_SQLT_AVC: case oracle::occi::OCCI_SQLT_CHR: case oracle::occi::OCCI_SQLT_LNG: case oracle::occi::OCCI_SQLT_LVC: case oracle::occi::OCCI_SQLT_STR: case oracle::occi::OCCI_SQLT_VCS: case oracle::occi::OCCI_SQLT_VNU: case oracle::occi::OCCI_SQLT_VBI: case oracle::occi::OCCI_SQLT_VST: case oracle::occi::OCCIROWID: case oracle::occi::OCCI_SQLT_RDD: case oracle::occi::OCCI_SQLT_RID: case oracle::occi::OCCIDATE: case oracle::occi::OCCI_SQLT_DAT: case oracle::occi::OCCI_SQLT_DATE: case oracle::occi::OCCI_SQLT_TIME: case oracle::occi::OCCI_SQLT_TIME_TZ: case oracle::occi::OCCITIMESTAMP: case oracle::occi::OCCI_SQLT_TIMESTAMP: case oracle::occi::OCCI_SQLT_TIMESTAMP_LTZ: case oracle::occi::OCCI_SQLT_TIMESTAMP_TZ: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case oracle::occi::OCCIINTERVALDS: case oracle::occi::OCCIINTERVALYM: case oracle::occi::OCCI_SQLT_INTERVAL_YM: case oracle::occi::OCCI_SQLT_INTERVAL_DS: case oracle::occi::OCCIANYDATA: case oracle::occi::OCCIBLOB: case oracle::occi::OCCIBFILE: case oracle::occi::OCCIBYTES: case oracle::occi::OCCICLOB: case oracle::occi::OCCIVECTOR: case oracle::occi::OCCIMETADATA: case oracle::occi::OCCIPOBJECT: case oracle::occi::OCCIREF: case oracle::occi::OCCIREFANY: case oracle::occi::OCCISTREAM: case oracle::occi::OCCICURSOR: case oracle::occi::OCCI_SQLT_FILE: case oracle::occi::OCCI_SQLT_CFILE: case oracle::occi::OCCI_SQLT_REF: case oracle::occi::OCCI_SQLT_CLOB: case oracle::occi::OCCI_SQLT_BLOB: case oracle::occi::OCCI_SQLT_RSET: MAPNIK_LOG_WARN(occi) << "occi_datasource: Unsupported datatype " << occi_enums::resolve_datatype(type_oid) << " (type_oid=" << type_oid << ")"; break; default: MAPNIK_LOG_WARN(occi) << "occi_datasource: Unknown datatype " << "(type_oid=" << type_oid << ")"; break; } } } } catch (SQLException& ex) { throw datasource_exception(ex.getMessage()); } } }
void sqlite_datasource::bind() const { if (is_bound_) return; if (!boost::filesystem::exists(dataset_name_)) throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); dataset_ = new sqlite_connection (dataset_name_); if(geometry_table_.empty()) { geometry_table_ = mapnik::table_from_sql(table_); } // should we deduce column names and types using PRAGMA? bool use_pragma_table_info = true; if (table_ != geometry_table_) { // if 'table_' is a subquery then we try to deduce names // and types from the first row returned from that query use_pragma_table_info = false; } if (!use_pragma_table_info) { std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { for (int i = 0; i < rs->column_count (); ++i) { const int type_oid = rs->column_type (i); const char* fld_name = rs->column_name (i); switch (type_oid) { case SQLITE_INTEGER: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case SQLITE_FLOAT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case SQLITE_TEXT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case SQLITE_NULL: // sqlite reports based on value, not actual column type unless // PRAGMA table_info is used so here we assume the column is a string // which is a lesser evil than altogether dropping the column desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); case SQLITE_BLOB: if (geometry_field_.empty() && (boost::algorithm::icontains(fld_name,"geom") || boost::algorithm::icontains(fld_name,"point") || boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::icontains(fld_name,"polygon")) ) geometry_field_ = std::string(fld_name); break; default: #ifdef MAPNIK_DEBUG std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl; #endif break; } } } else { // if we do not have at least a row and // we cannot determine the right columns types and names // as all column_type are SQLITE_NULL // so we fallback to using PRAGMA table_info use_pragma_table_info = true; } } // TODO - ensure that the supplied key_field is a valid "integer primary key" desc_.add_descriptor(attribute_descriptor("rowid",mapnik::Integer)); if (use_pragma_table_info) { std::ostringstream s; s << "PRAGMA table_info(" << geometry_table_ << ")"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); bool found_table = false; while (rs->is_valid () && rs->step_next()) { found_table = true; const char* fld_name = rs->column_text(1); std::string fld_type(rs->column_text(2)); boost::algorithm::to_lower(fld_type); // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html if (geometry_field_.empty() && ( (boost::algorithm::icontains(fld_name,"geom") || boost::algorithm::icontains(fld_name,"point") || boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::icontains(fld_name,"polygon")) || (boost::algorithm::contains(fld_type,"geom") || boost::algorithm::contains(fld_type,"point") || boost::algorithm::contains(fld_type,"linestring") || boost::algorithm::contains(fld_type,"polygon")) ) ) geometry_field_ = std::string(fld_name); else if (boost::algorithm::contains(fld_type,"int")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); } else if (boost::algorithm::contains(fld_type,"text") || boost::algorithm::contains(fld_type,"char") || boost::algorithm::contains(fld_type,"clob")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); } else if (boost::algorithm::contains(fld_type,"real") || boost::algorithm::contains(fld_type,"float") || boost::algorithm::contains(fld_type,"double")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); } else if (boost::algorithm::contains(fld_type,"blob") && !geometry_field_.empty()) desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); #ifdef MAPNIK_DEBUG else { // "Column Affinity" says default to "Numeric" but for now we pass.. //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); std::clog << "Sqlite Plugin: column '" << std::string(fld_name) << "' unhandled due to unknown type: " << fld_type << std::endl; } #endif } if (!found_table) { throw datasource_exception("Sqlite Plugin: could not query table '" + geometry_table_ + "' using pragma table_info(" + geometry_table_ + ");"); } } if (geometry_field_.empty()) { throw datasource_exception("Sqlite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use."); } if (use_spatial_index_) { std::ostringstream s; s << "SELECT COUNT (*) FROM sqlite_master"; s << " WHERE LOWER(name) = LOWER('idx_" << geometry_table_ << "_" << geometry_field_ << "')"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { use_spatial_index_ = rs->column_integer (0) == 1; } if (!use_spatial_index_) std::clog << "Sqlite Plugin: spatial index not found for table '" << geometry_table_ << "'" << std::endl; } if (metadata_ != "" && !extent_initialized_) { std::ostringstream s; s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_; s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')"; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { double xmin = rs->column_double (0); double ymin = rs->column_double (1); double xmax = rs->column_double (2); double ymax = rs->column_double (3); extent_.init (xmin,ymin,xmax,ymax); extent_initialized_ = true; } } if (!extent_initialized_ && use_spatial_index_) { std::ostringstream s; s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " << "idx_" << geometry_table_ << "_" << geometry_field_; boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { double xmin = rs->column_double (0); double ymin = rs->column_double (1); double xmax = rs->column_double (2); double ymax = rs->column_double (3); extent_.init (xmin,ymin,xmax,ymax); extent_initialized_ = true; } } if (!extent_initialized_) { std::ostringstream s; s << "Sqlite Plugin: extent could not be determined for table|geometry '" << geometry_table_ << "|" << geometry_field_ << "'" << " because an rtree spatial index is missing." << " - either set the table 'extent' or create an rtree spatial index"; throw datasource_exception(s.str()); } is_bound_ = true; }
void spatialite_datasource::bind() const { if (is_bound_) return; boost::optional<std::string> file = params_.get<std::string>("file"); if (!file) throw datasource_exception("Spatialite Plugin: missing <file> parameter"); boost::optional<std::string> key_field_name = params_.get<std::string>("key_field"); if (key_field_name) { std::string const& key_field_string = *key_field_name; if (key_field_string.empty()) { key_field_ = "rowid"; } else { key_field_ = key_field_string; } } else { key_field_ = "rowid"; } boost::optional<std::string> wkb = params_.get<std::string>("wkb_format"); if (wkb) { if (*wkb == "spatialite") format_ = mapnik::wkbSpatiaLite; } multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false); use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index",true); has_spatial_index_ = false; using_subquery_ = false; boost::optional<std::string> ext = params_.get<std::string>("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); boost::optional<std::string> base = params_.get<std::string>("base"); if (base) dataset_name_ = *base + "/" + *file; else dataset_name_ = *file; // Populate init_statements_ // 1. Build attach database statements from the "attachdb" parameter // 2. Add explicit init statements from "initdb" parameter // Note that we do some extra work to make sure that any attached // databases are relative to directory containing dataset_name_. Sqlite // will default to attaching from cwd. Typicaly usage means that the // map loader will produce full paths here. boost::optional<std::string> attachdb = params_.get<std::string>("attachdb"); if (attachdb) { parse_attachdb(*attachdb); } boost::optional<std::string> initdb = params_.get<std::string>("initdb"); if (initdb) { init_statements_.push_back(*initdb); } if (!boost::filesystem::exists(dataset_name_)) throw datasource_exception("Spatialite Plugin: " + dataset_name_ + " does not exist"); dataset_ = new sqlite_connection (dataset_name_); // Execute init_statements_ for (std::vector<std::string>::const_iterator iter=init_statements_.begin(); iter!=init_statements_.end(); ++iter) { #ifdef MAPNIK_DEBUG std::clog << "Spatialite Plugin: Execute init sql: " << *iter << std::endl; #endif dataset_->execute(*iter); } if(geometry_table_.empty()) { geometry_table_ = mapnik::table_from_sql(table_); } // should we deduce column names and types using PRAGMA? bool use_pragma_table_info = true; if (table_ != geometry_table_) { // if 'table_' is a subquery then we try to deduce names // and types from the first row returned from that query using_subquery_ = true; use_pragma_table_info = false; } if (!use_pragma_table_info) { std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1"; boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); if (rs->is_valid() && rs->step_next()) { for (int i = 0; i < rs->column_count (); ++i) { const int type_oid = rs->column_type (i); const char* fld_name = rs->column_name (i); switch (type_oid) { case SQLITE_INTEGER: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); break; case SQLITE_FLOAT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); break; case SQLITE_TEXT: desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); break; case SQLITE_NULL: // sqlite reports based on value, not actual column type unless // PRAGMA table_info is used so here we assume the column is a string // which is a lesser evil than altogether dropping the column desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); case SQLITE_BLOB: if (geometry_field_.empty() && (boost::algorithm::icontains(fld_name,"geom") || boost::algorithm::icontains(fld_name,"point") || boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::icontains(fld_name,"polygon")) ) geometry_field_ = std::string(fld_name); break; default: #ifdef MAPNIK_DEBUG std::clog << "Spatialite Plugin: unknown type_oid=" << type_oid << std::endl; #endif break; } } } else { // if we do not have at least a row and // we cannot determine the right columns types and names // as all column_type are SQLITE_NULL // so we fallback to using PRAGMA table_info use_pragma_table_info = true; } } if (key_field_ == "rowid") desc_.add_descriptor(attribute_descriptor("rowid",mapnik::Integer)); if (use_pragma_table_info) { std::ostringstream s; s << "PRAGMA table_info(" << geometry_table_ << ")"; boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); bool found_table = false; while (rs->is_valid() && rs->step_next()) { found_table = true; // TODO - support unicode strings? const char* fld_name = rs->column_text(1); std::string fld_type(rs->column_text(2)); boost::algorithm::to_lower(fld_type); // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html if (geometry_field_.empty() && ( (boost::algorithm::icontains(fld_name,"geom") || boost::algorithm::icontains(fld_name,"point") || boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::icontains(fld_name,"polygon")) || (boost::algorithm::contains(fld_type,"geom") || boost::algorithm::contains(fld_type,"point") || boost::algorithm::contains(fld_type,"linestring") || boost::algorithm::contains(fld_type,"polygon")) ) ) geometry_field_ = std::string(fld_name); else if (boost::algorithm::contains(fld_type,"int")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); } else if (boost::algorithm::contains(fld_type,"text") || boost::algorithm::contains(fld_type,"char") || boost::algorithm::contains(fld_type,"clob")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); } else if (boost::algorithm::contains(fld_type,"real") || boost::algorithm::contains(fld_type,"float") || boost::algorithm::contains(fld_type,"double")) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); } else if (boost::algorithm::contains(fld_type,"blob") && !geometry_field_.empty()) { desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); } #ifdef MAPNIK_DEBUG else { // "Column Affinity" says default to "Numeric" but for now we pass.. //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); std::clog << "Spatialite Plugin: column '" << std::string(fld_name) << "' unhandled due to unknown type: " << fld_type << std::endl; } #endif } if (!found_table) { std::ostringstream s; s << "Spatialite Plugin: could not query table '" << geometry_table_ << "' "; if (using_subquery_) s << " from subquery '" << table_ << "' "; s << "using 'PRAGMA table_info(" << geometry_table_ << ")' "; std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_))); if (sq_err != "unknown error") s << ": " << sq_err; throw datasource_exception(s.str()); } } if (geometry_field_.empty()) { throw datasource_exception("Spatialite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use."); } if (index_table_.size() == 0) { // Generate implicit index_table name - need to do this after // we have discovered meta-data or else we don't know the column name index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\""; } if (use_spatial_index_) { std::ostringstream s; s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_; s << " LIMIT 0"; if (dataset_->execute_with_code(s.str()) == SQLITE_OK) { has_spatial_index_ = true; } #ifdef MAPNIK_DEBUG else { std::clog << "Spatialite Plugin: rtree index lookup did not succeed: '" << sqlite3_errmsg(*(*dataset_)) << "'\n"; } #endif } if (metadata_ != "" && !extent_initialized_) { std::ostringstream s; s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_; s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')"; boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); if (rs->is_valid() && rs->step_next()) { double xmin = rs->column_double (0); double ymin = rs->column_double (1); double xmax = rs->column_double (2); double ymax = rs->column_double (3); extent_.init (xmin,ymin,xmax,ymax); extent_initialized_ = true; } } if (!extent_initialized_ && has_spatial_index_) { std::ostringstream s; s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " << index_table_; boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); if (rs->is_valid() && rs->step_next()) { if (!rs->column_isnull(0)) { try { double xmin = lexical_cast<double>(rs->column_double(0)); double ymin = lexical_cast<double>(rs->column_double(1)); double xmax = lexical_cast<double>(rs->column_double(2)); double ymax = lexical_cast<double>(rs->column_double(3)); extent_.init (xmin,ymin,xmax,ymax); extent_initialized_ = true; } catch (bad_lexical_cast &ex) { std::clog << boost::format("Spatialite Plugin: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what() << std::endl; } } } } #ifdef MAPNIK_DEBUG if (!has_spatial_index_ && use_spatial_index_ && using_subquery_ && key_field_ == "rowid" && !boost::algorithm::icontains(table_,"rowid")) { // this is an impossible situation because rowid will be null via a subquery std::clog << "Spatialite Plugin: WARNING: spatial index usage will fail because rowid is not present in your subquery. You have 4 options: 1) Add rowid into your select statement, 2) alias your primary key as rowid, 3) supply a 'key_field' value that references the primary key of your spatial table, or 4) avoid using a spatial index by setting 'use_spatial_index'=false"; } #endif // final fallback to gather extent if (!extent_initialized_ || !has_spatial_index_) { std::ostringstream s; s << "SELECT " << geometry_field_ << "," << key_field_ << " FROM " << geometry_table_; if (row_limit_ > 0) { s << " LIMIT " << row_limit_; } if (row_offset_ > 0) { s << " OFFSET " << row_offset_; } #ifdef MAPNIK_DEBUG std::clog << "Spatialite Plugin: " << s.str() << std::endl; #endif boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); // spatial index sql std::string spatial_index_sql = "create virtual table " + index_table_ + " using rtree(pkid, xmin, xmax, ymin, ymax)"; std::string spatial_index_insert_sql = "insert into " + index_table_ + " values (?,?,?,?,?)" ; sqlite3_stmt* stmt = 0; if (use_spatial_index_) { dataset_->execute(spatial_index_sql); int rc = sqlite3_prepare_v2 (*(*dataset_), spatial_index_insert_sql.c_str(), -1, &stmt, 0); if (rc != SQLITE_OK) { std::ostringstream index_error; index_error << "Spatialite Plugin: auto-index table creation failed: '" << sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql; throw datasource_exception(index_error.str()); } } bool first = true; while (rs->is_valid() && rs->step_next()) { int size; const char* data = (const char *) rs->column_blob (0, size); if (data) { // create a tmp feature to be able to parse geometry // ideally we would not have to do this. // see: http://trac.mapnik.org/ticket/745 mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); mapnik::geometry_utils::from_wkb(feature->paths(),data,size,multiple_geometries_,format_); mapnik::box2d<double> const& bbox = feature->envelope(); if (bbox.valid()) { extent_initialized_ = true; if (first) { first = false; extent_ = bbox; } else { extent_.expand_to_include(bbox); } // index creation if (use_spatial_index_) { const int type_oid = rs->column_type(1); if (type_oid != SQLITE_INTEGER) { std::ostringstream type_error; type_error << "Spatialite Plugin: invalid type for key field '" << key_field_ << "' when creating index '" << index_table_ << "' type was: " << type_oid << ""; throw datasource_exception(type_error.str()); } sqlite_int64 pkid = rs->column_integer64(1); if (sqlite3_bind_int64(stmt, 1 , pkid ) != SQLITE_OK) { throw datasource_exception("invalid value for for key field while generating index"); } if ((sqlite3_bind_double(stmt, 2 , bbox.minx() ) != SQLITE_OK) || (sqlite3_bind_double(stmt, 3 , bbox.maxx() ) != SQLITE_OK) || (sqlite3_bind_double(stmt, 4 , bbox.miny() ) != SQLITE_OK) || (sqlite3_bind_double(stmt, 5 , bbox.maxy() ) != SQLITE_OK)) { throw datasource_exception("invalid value for for extent while generating index"); } int res = sqlite3_step(stmt); if (res != SQLITE_DONE) { std::ostringstream s; s << "Spatialite Plugin: inserting bbox into rtree index failed: " << "error code " << sqlite3_errcode(*(*dataset_)) << ": '" << sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql; throw datasource_exception(s.str()); } sqlite3_reset(stmt); } } else { std::ostringstream index_error; index_error << "Spatialite Plugin: encountered invalid bbox at '" << key_field_ << "' == " << rs->column_integer64(1); throw datasource_exception(index_error.str()); } } } int res = sqlite3_finalize(stmt); if (res != SQLITE_OK) { throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error"); } } if (!extent_initialized_) { std::ostringstream s; s << "Spatialite Plugin: extent could not be determined for table '" << geometry_table_ << "' and geometry field '" << geometry_field_ << "'" << " because an rtree spatial index is missing or empty." << " - either set the table 'extent' or create an rtree spatial index"; throw datasource_exception(s.str()); } is_bound_ = true; }