bool HTTPClient::doDownload(const std::string& url, const std::string& filename) { initialize(); // download the data HTTPResponse response = this->doGet( HTTPRequest(url) ); if ( response.isOK() ) { unsigned int part_num = response.getNumParts() > 1? 1 : 0; std::istream& input_stream = response.getPartStream( part_num ); std::ofstream fout; fout.open(filename.c_str(), std::ios::out | std::ios::binary); input_stream.seekg (0, std::ios::end); int length = input_stream.tellg(); input_stream.seekg (0, std::ios::beg); char *buffer = new char[length]; input_stream.read(buffer, length); fout.write(buffer, length); delete[] buffer; fout.close(); return true; } else { OE_WARN << LC << "Error downloading file " << filename << " (" << response.getCode() << ")" << std::endl; return false; } }
bool EarthFile::readXML( const std::string& location ) { bool success = false; if ( osgDB::containsServerAddress( location ) ) { HTTPResponse response = HTTPClient::get( location ); if ( response.isOK() && response.getNumParts() > 0 ) { success = readXML( response.getPartStream( 0 ), location ); } } else { if (osgDB::fileExists(location) && (osgDB::fileType(location) == osgDB::REGULAR_FILE)) { std::ifstream in( location.c_str() ); success = readXML( in, location ); } } if ( success ) { std::string filename = location; if (!osgDB::containsServerAddress(filename)) { filename = osgDB::getRealPath( location ); } _map->setReferenceURI( filename ); } return success; }
ReadResult HTTPClient::doReadNode(const std::string& location, const osgDB::Options* options, ProgressCallback* callback) { initialize(); ReadResult result; HTTPResponse response = this->doGet(location, options, callback); if (response.isOK()) { osgDB::ReaderWriter* reader = getReader(location, response); if (!reader) { OE_WARN << LC << "Can't find an OSG plugin to read "<<location<<std::endl; result = ReadResult(ReadResult::RESULT_NO_READER); } else { osgDB::ReaderWriter::ReadResult rr = reader->readNode(response.getPartStream(0), options); if ( rr.validNode() ) { result = ReadResult(rr.takeNode(), response.getHeadersAsConfig()); } else { if ( !rr.message().empty() ) { OE_WARN << LC << "HTTP error: " << rr.message() << std::endl; } OE_WARN << LC << reader->className() << " failed to read node from " << location << std::endl; result = ReadResult(ReadResult::RESULT_READER_ERROR); } } } else { result = ReadResult( response.isCancelled() ? ReadResult::RESULT_CANCELED : response.getCode() == HTTPResponse::NOT_FOUND ? ReadResult::RESULT_NOT_FOUND : response.getCode() == HTTPResponse::SERVER_ERROR ? ReadResult::RESULT_SERVER_ERROR : ReadResult::RESULT_UNKNOWN_ERROR ); //If we have an error but it's recoverable, like a server error or timeout then set the callback to retry. if (HTTPClient::isRecoverable( result.code() ) ) { if (callback) { OE_DEBUG << "Error in HTTPClient for " << location << " but it's recoverable" << std::endl; callback->setNeedsRetry( true ); } } } return result; }
void saveResponse(HTTPResponse& response, const std::string& filename) { std::ofstream fout; fout.open(filename.c_str(), std::ios::out | std::ios::binary); std::istream& input_stream = response.getPartStream(0); input_stream.seekg (0, std::ios::end); int length = input_stream.tellg(); input_stream.seekg (0, std::ios::beg); char *buffer = new char[length]; input_stream.read(buffer, length); fout.write(buffer, length); delete[] buffer; fout.close(); }
osg::Image* WCS11Source::createImage(const TileKey& key, ProgressCallback* progress) { HTTPRequest request = createRequest( key ); OE_INFO << "[osgEarth::WCS1.1] Key=" << key.str() << " URL = " << request.getURL() << std::endl; double lon0,lat0,lon1,lat1; key.getExtent().getBounds( lon0, lat0, lon1, lat1 ); // download the data. It's a multipart-mime stream, so we have to use HTTP directly. HTTPResponse response = HTTPClient::get( request, _dbOptions.get(), progress ); if ( !response.isOK() ) { OE_WARN << "[osgEarth::WCS1.1] WARNING: HTTP request failed" << std::endl; return NULL; } //TODO: Make WCS driver use progress callback unsigned int part_num = response.getNumParts() > 1? 1 : 0; std::istream& input_stream = response.getPartStream( part_num ); //TODO: un-hard-code TIFFs osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension( "tiff" ); if ( !reader ) { OE_NOTICE << "[osgEarth::WCS1.1] WARNING: no reader for \"tiff\"" << std::endl; return NULL; } osgDB::ReaderWriter::ReadResult result = reader->readImage( input_stream ); //, getOptions() ); if ( !result.success() ) { OE_NOTICE << "[osgEarth::WCS1.1] WARNING: readImage() failed for Reader " << reader->getName() << std::endl; return NULL; } osg::Image* image = result.getImage(); //OE_NOTICE << "Returned grid is " << image->s() << "x" << image->t() << std::endl; if ( image ) image->ref(); return image; }
TileService* TileServiceReader::read( const std::string &location, const osgDB::ReaderWriter::Options* options ) { TileService *tileService = NULL; if ( osgDB::containsServerAddress( location ) ) { HTTPResponse response = HTTPClient::get( location, options); if (response.isOK() && response.getNumParts() > 0 ) { tileService = read( response.getPartStream( 0 ) ); } } else { if ((osgDB::fileExists(location)) && (osgDB::fileType(location) == osgDB::REGULAR_FILE)) { std::ifstream in( location.c_str() ); tileService = read( in ); } } return tileService; }
ReadResult HTTPClient::doReadObject(const HTTPRequest& request, const osgDB::Options* options, ProgressCallback* callback) { initialize(); ReadResult result; HTTPResponse response = this->doGet(request, options, callback); if (response.isOK()) { osgDB::ReaderWriter* reader = getReader(request.getURL(), response); if (!reader) { result = ReadResult(ReadResult::RESULT_NO_READER); } else { osgDB::ReaderWriter::ReadResult rr = reader->readObject(response.getPartStream(0), options); if ( rr.validObject() ) { result = ReadResult(rr.takeObject()); } else { if ( s_HTTP_DEBUG ) { OE_WARN << LC << reader->className() << " failed to read object from " << request.getURL() << "; message = " << rr.message() << std::endl; } result = ReadResult(ReadResult::RESULT_READER_ERROR); result.setErrorDetail( rr.message() ); } } // last-modified (file time) result.setLastModifiedTime( getCurlFileTime(_curl_handle) ); } else { result = ReadResult( response.isCancelled() ? ReadResult::RESULT_CANCELED : response.getCode() == HTTPResponse::NOT_FOUND ? ReadResult::RESULT_NOT_FOUND : response.getCode() == HTTPResponse::SERVER_ERROR ? ReadResult::RESULT_SERVER_ERROR : response.getCode() == HTTPResponse::NOT_MODIFIED ? ReadResult::RESULT_NOT_MODIFIED : ReadResult::RESULT_UNKNOWN_ERROR ); //If we have an error but it's recoverable, like a server error or timeout then set the callback to retry. if (HTTPClient::isRecoverable( result.code() ) ) { if (callback) { if ( s_HTTP_DEBUG ) { OE_NOTICE << LC << "Error in HTTPClient for " << request.getURL() << " but it's recoverable" << std::endl; } callback->setNeedsRetry( true ); } } } result.setMetadata( response.getHeadersAsConfig() ); return result; }
HTTPClient::ResultCode HTTPClient::doReadNodeFile(const std::string& filename, osg::ref_ptr<osg::Node>& output, const osgDB::ReaderWriter::Options *options, ProgressBase *callback) { ResultCode result = RESULT_OK; if ( osgDB::containsServerAddress( filename ) ) { HTTPResponse response = this->doGet(filename, options, callback); if (response.isOK()) { // Try to find a reader by file extension. If this fails, we will fetch the file // anyway and try to get a reader via mime-type. std::string ext = osgDB::getFileExtension( filename ); osgDB::ReaderWriter *reader = osgDB::Registry::instance()->getReaderWriterForExtension( ext ); //If we didn't get a reader by extension, try to get it via mime type if (!reader) { std::string mimeType = response.getMimeType(); //OE_DEBUG << LC << "Looking up extension for mime-type " << mimeType << std::endl; if ( mimeType.length() > 0 ) { reader = owUtil::Registry::instance().getReaderWriterForMimeType(mimeType); } } // if we still didn't get it, bad news if (!reader) { //OE_NOTICE<<LC<<"Error: No ReaderWriter for file "<<filename<<std::endl; result = RESULT_NO_READER; } else { osgDB::ReaderWriter::ReadResult rr = reader->readNode(response.getPartStream(0), options); if ( rr.validNode() ) { output = rr.takeNode(); } else { if ( rr.error() ) { //OE_WARN << LC << "HTTP Reader Error: " << rr.message() << std::endl; } result = RESULT_READER_ERROR; } } } else { result = response.isCancelled() ? RESULT_CANCELED : response.getCode() == HTTPResponse::NOT_FOUND ? RESULT_NOT_FOUND : response.getCode() == HTTPResponse::SERVER_ERROR ? RESULT_SERVER_ERROR : RESULT_UNKNOWN_ERROR; //If we have an error but it's recoverable, like a server error or timeout then set the callback to retry. if (HTTPClient::isRecoverable( result ) ) { if (callback) { //OE_DEBUG << "Error in HTTPClient for " << filename << " but it's recoverable" << std::endl; callback->setNeedsRetry( true ); } } /*if (response.isCancelled()) OE_NOTICE << "Request for " << filename << " was cancelled " << std::endl;*/ } } else { output = osgDB::readNodeFile( filename, options ); if ( !output.valid() ) result = RESULT_NOT_FOUND; } return result; }
bool MapService::init( const std::string& _url, const osgDB::ReaderWriter::Options* options ) { url = _url; std::string sep = url.find( "?" ) == std::string::npos ? "?" : "&"; std::string json_url = url + sep + std::string("f=pjson"); // request the data in JSON format HTTPResponse response = HTTPClient::get( json_url, options ); if ( !response.isOK() ) return setError( "Unable to read metadata from ArcGIS service" ); Json::Value doc; Json::Reader reader; if ( !reader.parse( response.getPartStream(0), doc ) ) return setError( "Unable to parse metadata; invalid JSON" ); // Read the profile. We are using "fullExtent"; perhaps an option to use "initialExtent" instead? double xmin = doc["fullExtent"].get("xmin", 0).asDouble(); double ymin = doc["fullExtent"].get("ymin", 0).asDouble(); double xmax = doc["fullExtent"].get("xmax", 0).asDouble(); double ymax = doc["fullExtent"].get("ymax", 0).asDouble(); int srs = doc["fullExtent"].get("spatialReference", "").get("wkid", 0).asInt(); //Assumes the SRS is going to be an EPSG code std::stringstream ss; ss << "epsg:" << srs; if ( ! (xmax > xmin && ymax > ymin && srs != 0 ) ) { return setError( "Map service does not define a full extent" ); } // Read the layers list Json::Value j_layers = doc["layers"]; if ( j_layers.empty() ) return setError( "Map service contains no layers" ); for( unsigned int i=0; i<j_layers.size(); i++ ) { Json::Value layer = j_layers[i]; int id = i; // layer.get("id", -1).asInt(); std::string name = layer["name"].asString(); if ( id >= 0 && !name.empty() ) { layers.push_back( MapServiceLayer( id, name ) ); } } tiled = false; std::string format = "png"; int tile_rows = 256; int tile_cols = 256; int min_level = 25; int max_level = 0; int num_tiles_wide = 1; int num_tiles_high = 1; // Read the tiling schema Json::Value j_tileinfo = doc["tileInfo"]; if ( !j_tileinfo.empty() ) { tiled = true; // return setError( "Map service does not define a tiling schema" ); // TODO: what do we do if the width <> height? tile_rows = j_tileinfo.get( "rows", 0 ).asInt(); tile_cols = j_tileinfo.get( "cols", 0 ).asInt(); if ( tile_rows <= 0 && tile_cols <= 0 ) return setError( "Map service tile size not specified" ); format = j_tileinfo.get( "format", "" ).asString(); if ( format.empty() ) return setError( "Map service tile schema does not specify an image format" ); Json::Value j_levels = j_tileinfo["lods"]; if ( j_levels.empty() ) return setError( "Map service tile schema contains no LODs" ); min_level = INT_MAX; max_level = 0; for( unsigned int i=0; i<j_levels.size(); i++ ) { int level = j_levels[i].get( "level", -1 ).asInt(); if ( level >= 0 && level < min_level ) min_level = level; if ( level >= 0 && level > max_level ) max_level = level; } if (j_levels.size() > 0) { int l = j_levels[0u].get("level", -1).asInt(); double res = j_levels[0u].get("resolution", 0.0).asDouble(); num_tiles_wide = (int)osg::round((xmax - xmin) / (res * tile_cols)); num_tiles_high = (int)osg::round((ymax - ymin) / (res * tile_cols)); //In case the first level specified isn't level 0, compute the number of tiles at level 0 for (int i = 0; i < l; i++) { num_tiles_wide /= 2; num_tiles_high /= 2; } //profile.setNumTilesWideAtLod0(num_tiles_wide); //profile.setNumTilesHighAtLod0(num_tiles_high); } } std::string ssStr; ssStr = ss.str(); osg::ref_ptr< SpatialReference > spatialReference = SpatialReference::create( ssStr ); if (spatialReference->isGeographic()) { //If we have a geographic SRS, just use the geodetic profile profile = Registry::instance()->getGlobalGeodeticProfile(); } else if (spatialReference->isMercator()) { //If we have a mercator SRS, just use the mercator profile profile = Registry::instance()->getGlobalMercatorProfile(); } else { //It's not geodetic or mercator, so try to use the full extent profile = Profile::create( spatialReference.get(), xmin, ymin, xmax, ymax, NULL, num_tiles_wide, num_tiles_high); } // now we're good. tile_info = TileInfo( tile_rows, format, min_level, max_level, num_tiles_wide, num_tiles_high); is_valid = true; return is_valid; }
ReadResult HTTPClient::doReadImage(const std::string& location, const osgDB::Options* options, ProgressCallback* callback) { initialize(); ReadResult result; HTTPResponse response = this->doGet(location, options, callback); if (response.isOK()) { osgDB::ReaderWriter* reader = getReader(location, response); if (!reader) { result = ReadResult(ReadResult::RESULT_NO_READER); } else { osgDB::ReaderWriter::ReadResult rr = reader->readImage(response.getPartStream(0), options); if ( rr.validImage() ) { result = ReadResult(rr.takeImage(), response.getHeadersAsConfig() ); } else { if ( !rr.message().empty() ) { OE_WARN << LC << "HTTP error: " << rr.message() << std::endl; } OE_WARN << LC << reader->className() << " failed to read image from " << location << std::endl; result = ReadResult(ReadResult::RESULT_READER_ERROR); } } // last-modified (file time) TimeStamp filetime = 0; if ( CURLE_OK == curl_easy_getinfo(_curl_handle, CURLINFO_FILETIME, &filetime) ) { result.setLastModifiedTime( filetime ); } } else { result = ReadResult( response.isCancelled() ? ReadResult::RESULT_CANCELED : response.getCode() == HTTPResponse::NOT_FOUND ? ReadResult::RESULT_NOT_FOUND : response.getCode() == HTTPResponse::SERVER_ERROR ? ReadResult::RESULT_SERVER_ERROR : ReadResult::RESULT_UNKNOWN_ERROR ); //If we have an error but it's recoverable, like a server error or timeout then set the callback to retry. if (HTTPClient::isRecoverable( result.code() ) ) { if (callback) { OE_DEBUG << "Error in HTTPClient for " << location << " but it's recoverable" << std::endl; callback->setNeedsRetry( true ); } } } // set the source name if ( result.getImage() ) result.getImage()->setName( location ); return result; }
HTTPClient::ResultCode HTTPClient::doReadImageFile(const std::string& filename, osg::ref_ptr<osg::Image>& output, const osgDB::ReaderWriter::Options *options, osgEarth::ProgressCallback *callback) { ResultCode result = RESULT_OK; if ( osgDB::containsServerAddress( filename ) ) { HTTPResponse response = this->doGet(filename, options, callback); if (response.isOK()) { osgDB::ReaderWriter* reader = 0L; // try to look up a reader by mime-type first: std::string mimeType = response.getMimeType(); OE_DEBUG << LC << "Looking up extension for mime-type " << mimeType << std::endl; if ( !mimeType.empty() ) { reader = osgEarth::Registry::instance()->getReaderWriterForMimeType(mimeType); } if ( !reader ) { // Try to find a reader by file extension. If this fails, we will fetch the file // anyway and try to get a reader via mime-type. std::string ext = osgDB::getFileExtension( filename ); reader = osgDB::Registry::instance()->getReaderWriterForExtension( ext ); //OE_NOTICE << "Reading " << filename << " with mime " << response.getMimeType() << std::endl; } if (!reader) { OE_WARN << LC << "Can't find an OSG plugin to read "<<filename<<std::endl; result = RESULT_NO_READER; } else { osgDB::ReaderWriter::ReadResult rr = reader->readImage(response.getPartStream(0), options); if ( rr.validImage() ) { output = rr.takeImage(); } else { if ( !rr.message().empty() ) { OE_WARN << LC << "HTTP error: " << rr.message() << std::endl; } OE_WARN << LC << reader->className() << " failed to read image from " << filename << std::endl; result = RESULT_READER_ERROR; } } } else { result = response.isCancelled() ? RESULT_CANCELED : response.getCode() == HTTPResponse::NOT_FOUND ? RESULT_NOT_FOUND : response.getCode() == HTTPResponse::SERVER_ERROR ? RESULT_SERVER_ERROR : RESULT_UNKNOWN_ERROR; //If we have an error but it's recoverable, like a server error or timeout then set the callback to retry. if (HTTPClient::isRecoverable( result ) ) { if (callback) { OE_DEBUG << "Error in HTTPClient for " << filename << " but it's recoverable" << std::endl; callback->setNeedsRetry( true ); } } //if ( response.isCancelled() ) // OE_NOTICE << "HTTP cancel: " << filename << std::endl; //else // OE_NOTICE << "HTTP ERROR " << response.getCode() << ": " << filename << std::endl; /*if (response.isCancelled()) OE_NOTICE << "Request for " << filename << " was cancelled " << std::endl;*/ } } else { output = osgDB::readImageFile( filename, options ); if ( !output.valid() ) result = RESULT_NOT_FOUND; } return result; }