void ExtractorBase::extract_url (string const &url, string const &type, Object *object) const { namespace fs = boost::filesystem; fs::path path = fs::unique_path(); int r = wget(url, path.native()); if (r != 0) { throw ExternalError(url); } extract_path(path.native(), type, object); fs::remove(path); }
void Server::loadObject (ObjectRequest const &request, Object *object) const { namespace fs = boost::filesystem; //fs::path path; bool is_url = false; if (request.url.size()) { if (request.content.size()) { throw RequestError("both url and content set"); } if (test_url(request.url)) { is_url = true; } } if (request.raw) { if (request.content.size()) { xtor.extract(request.content, request.type, object); } else if (is_url) { xtor.extract_url(request.url, request.type, object); } else { xtor.extract_path(request.url, request.type, object); } } else { if (request.content.size()) { std::istringstream is(request.content); object->read(is); } else { fs::path path; if (is_url) { // url path = fs::unique_path(); int r = wget(request.url, path.native()); if (r != 0) { throw ExternalError(request.url); } } else { path = fs::path(request.url); } ifstream is(path.native(), ios::binary); object->read(is); if (is_url) { fs::remove(path); } } } }
void Server::loadObject (ObjectRequest const &request, Object *object) const { namespace fs = boost::filesystem; fs::path path; bool is_url = false; if (request.url.size()) { if (request.content.size()) { throw RequestError("both url and content set"); } if (test_url(request.url)) { is_url = true; path = fs::unique_path(); string cmd = (boost::format("wget -O '%s' '%s'") % path.native() % request.url).str(); int r = ::system(cmd.c_str()); if (r != 0) { throw ExternalError(cmd); } } else { path = fs::path(request.url); } } if (request.raw) { if (request.content.size()) { xtor.extract(request.content, request.type, object); } else { // url std::cout<<"Inside Server::loadObject, request.raw = true"<<std::endl; xtor.extract_path(path.native(), request.type, object); } } else { if (request.content.size()) { std::istringstream is(request.content); object->read(is); } else { // url ifstream is(path.native(), ios::binary); object->read(is); } } if (is_url) { fs::remove(path); } }
void ReadURL (const std::string &url, std::string *binary) { do { if (url.compare(0, 7, "http://") == 0) break; if (url.compare(0, 8, "https://") == 0) break; if (url.compare(0, 6, "ftp://") == 0) break; // do not support other protocols ReadFile(url, binary); return; } while (false); namespace fs = boost::filesystem; fs::path tmp = fs::unique_path(); int r = wget(url, tmp.native()); if (r) { LOG(error) << "Fail to download: " << url; fs::remove(tmp); throw ExternalError(url); } else { ReadFile(tmp.native(), binary); fs::remove(tmp); } }