ofHttpResponse ofURLFileLoader::handleRequest(ofHttpRequest request) {
try {
URI uri(request.url);
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
HTTPResponse res;
ofPtr<HTTPSession> session;
istream * rs;
if(uri.getScheme()=="https"){
//const Poco::Net::Context::Ptr context( new Poco::Net::Context( Poco::Net::Context::CLIENT_USE, "", "", "rootcert.pem" ) );
HTTPSClientSession * httpsSession = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
httpsSession->setTimeout(Poco::Timespan(20,0));
httpsSession->sendRequest(req);
rs = &httpsSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpsSession);
}else{
HTTPClientSession * httpSession = new HTTPClientSession(uri.getHost(), uri.getPort());
httpSession->setTimeout(Poco::Timespan(20,0));
httpSession->sendRequest(req);
rs = &httpSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpSession);
}
if(!request.saveTo){
return ofHttpResponse(request,*rs,res.getStatus(),res.getReason());
}else{
ofFile saveTo(request.name,ofFile::WriteOnly,true);
char aux_buffer[1024];
rs->read(aux_buffer, 1024);
std::streamsize n = rs->gcount();
while (n > 0){
// we resize to size+1 initialized to 0 to have a 0 at the end for strings
saveTo.write(aux_buffer,n);
if (rs->good()){
rs->read(aux_buffer, 1024);
n = rs->gcount();
}
else n = 0;
}
return ofHttpResponse(request,res.getStatus(),res.getReason());
}
} catch (const Exception& exc) {
ofLogError("ofURLFileLoader") << "handleRequest(): "+ exc.displayText();
return ofHttpResponse(request,-1,exc.displayText());
} catch (...) {
return ofHttpResponse(request,-1,"ofURLFileLoader: fatal error, couldn't catch Exception");
}
return ofHttpResponse(request,-1,"ofURLFileLoader: fatal error, couldn't catch Exception");
}
bool ofxSimpleHttp::downloadURL(ofxSimpleHttpResponse* resp, bool sendResultThroughEvents, bool beingCalledFromMainThread, bool saveToDisk){
bool ok;
ofstream myfile;
bool fileIsAlreadyHere = false;
//create a file to save the stream to
if(saveToDisk){
if (resp->expectedChecksum.length()){ //if user provided a checksum
ofFile f;
f.open(resp->absolutePath);
if (f.exists()){
fileIsAlreadyHere = ofxChecksum::sha1(resp->absolutePath, resp->expectedChecksum);
if(fileIsAlreadyHere){
resp->checksumOK = true;
resp->status = 0;
resp->ok = true;
resp->fileWasHere = true;
ofLogVerbose() << "ofxSimpleHttp: about to download "<< resp->url << " but a file with same name and correct checksum is already here!";
ofLogVerbose() << "ofxSimpleHttp: skipping download (" << resp->expectedChecksum << ")";
}
}
f.close();
}else{
if (!onlySkipDownloadIfChecksumMatches){
ofFile f;
f.open(resp->absolutePath);
if (f.exists() && f.getSize() > 0){
resp->checksumOK = false;
resp->status = 0;
resp->ok = true;
resp->fileWasHere = true;
fileIsAlreadyHere = true;
ofLogVerbose() << "ofxSimpleHttp: about to download "<< resp->url << " but a file with same name and (size > 0) is already here!";
ofLogVerbose() << "ofxSimpleHttp: skipping download (missing checksum)";
}
f.close();
}
}
}
if (!fileIsAlreadyHere){ //if file is not here, download it!
myfile.open( resp->absolutePath.c_str(), ios_base::binary );
//myfile.open(ofToDataPath(filename).c_str()); //for not binary?
try {
ofHttpRequest request(resp->url, resp->url);
URI uri(request.url);
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
HTTPClientSession * session;
if(uri.getScheme()=="https"){
session = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
}else{
session = new HTTPClientSession(uri.getHost(), uri.getPort());
}
//resp->session = &session;
HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
req.set( "User-Agent", userAgent.c_str() );
session->setTimeout( Poco::Timespan(timeOut,0) );
session->sendRequest(req);
HTTPResponse res;
istream& rs = session->receiveResponse(res);
resp->status = res.getStatus();
try {
resp->timestamp = res.getDate();
} catch (Exception& exc) {
resp->timestamp = 0;
}
resp->reasonForStatus = res.getReasonForStatus( res.getStatus() );
resp->contentType = res.getContentType();
resp->serverReportedSize = res.getContentLength();
resp->timeTakenToDownload = ofGetElapsedTimef();
if (resp->serverReportedSize == -1) ofLogWarning("ofxSimpleHttp", "downloadURL(%s) >> Server doesn't report download size...", resp->fileName.c_str() );
ofLogVerbose("ofxSimpleHttp", "downloadURL() >> about to start download (%s, %d bytes)", resp->fileName.c_str(), res.getContentLength() );
ofLogVerbose("ofxSimpleHttp", "downloadURL() >> server reports request status: (%d-%s)", resp->status, resp->reasonForStatus.c_str() );
//StreamCopier::copyStream(rs, myfile); //write to file here!
if(saveToDisk){
streamCopyWithProgress(rs, myfile, resp->serverReportedSize, resp->downloadProgress, resp->downloadSpeed, resp->downloadCanceled);
}else{
copyToStringWithProgress(rs, resp->responseBody, resp->serverReportedSize, resp->downloadProgress, resp->downloadSpeed, resp->downloadCanceled);
}
resp->timeTakenToDownload = ofGetElapsedTimef() - resp->timeTakenToDownload;
if (resp->expectedChecksum.length() > 0){
resp->checksumOK = ofxChecksum::sha1(resp->absolutePath, resp->expectedChecksum);
if(!resp->checksumOK){
ofLogVerbose() << "ofxSimpleHttp: downloaded OK but Checksum FAILED";
ofLogVerbose() << "ofxSimpleHttp: SHA1 was meant to be: " << resp->expectedChecksum;
}
}
resp->downloadSpeed = 0;
resp->avgDownloadSpeed = 0;
resp->downloadedBytes = 0;
//resp->session = NULL;
delete session;
session = NULL;
if(saveToDisk){
myfile.close();
}
if (resp->downloadCanceled){
//delete half-baked download file
if (resp->downloadToDisk){
ofFile f;
if(f.open(resp->absolutePath)){
if (f.isFile()){
f.remove();
}
}
}
}else{
if(saveToDisk){
ofLogNotice("ofxSimpleHttp", "downloadURL() downloaded to %s", resp->fileName.c_str() );
}
if( saveToDisk ){
//ask the filesystem what is the real size of the file
ofFile file;
try{
file.open(resp->absolutePath.c_str());
resp->downloadedBytes = file.getSize();
file.close();
}catch(Exception& exc){
ofLogError("ofxSimpleHttp", "downloadURL(%s) >> Exception at file.open: %s", resp->fileName.c_str(), exc.displayText().c_str() );
}
}else{
resp->downloadedBytes = resp->responseBody.size();
}
resp->avgDownloadSpeed = (resp->downloadedBytes / 1024.) / resp->timeTakenToDownload; //kb/sec
//check download file size missmatch
if ( resp->serverReportedSize > 0 && resp->serverReportedSize != resp->downloadedBytes) {
ofLogWarning("ofxSimpleHttp", "downloadURLtoDiskBlocking() >> Download size mismatch (%s) >> Server: %d Downloaded: %d",
resp->fileName.c_str(), resp->serverReportedSize, resp->downloadedBytes );
resp->reasonForStatus = "Download size mismatch!";
resp->status = -1;
resp->ok = false;
}else{
if (resp->status == 200){
resp->ok = true;
}else{
resp->ok = false;
}
}
ofLogVerbose() << "ofxSimpleHttp: download finished! " << resp->url << " !";
ok = TRUE;
}
}catch(Exception& exc){
myfile.close();
ofLogError("ofxSimpleHttp", "downloadURL(%s) >> Exception: %s", resp->fileName.c_str(), exc.displayText().c_str() );
resp->reasonForStatus = exc.displayText();
resp->ok = false;
resp->status = -1;
ok = false;
ofLogError() << "ofxSimpleHttp: failed to download " << resp->url << " !";
}
}
//enqueue the operation result!
if (sendResultThroughEvents ){
if ( resp->notifyOnSuccess ){
if(idleTimeAfterEachDownload > 0.0){
ofSleepMillis(idleTimeAfterEachDownload * 1000);
}
if (beingCalledFromMainThread){ //we running on main thread, we can just snd the notif from here
ofNotifyEvent( httpResponse, *resp, this );
}else{ //we are running from a bg thread
if (notifyFromMainThread){ //user wants to get notified form main thread, we need to enqueue the notification
if (timeToStop == false){ //see if we have been destructed! dont forward events if so
lock();
ofxSimpleHttpResponse tempCopy = *resp;
responsesPendingNotification.push(tempCopy);
unlock();
}
}else{ //user doesnt care about main thread, the notificaiton can come from bg thread so we do it from here
ofNotifyEvent( httpResponse, *resp, this );
}
}
}
}
return ok;
}
// ----------------------------------------------------------------------
ofxHttpResponse ofxHttpUtils::getUrl(string url){
ofxHttpResponse response;
try{
URI uri(url.c_str());
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
if(auth.getUsername()!="") auth.authenticate(req);
if(sendCookies){
for(unsigned i=0; i<cookies.size(); i++){
NameValueCollection reqCookies;
reqCookies.add(cookies[i].getName(),cookies[i].getValue());
req.setCookies(reqCookies);
}
}
HTTPResponse res;
ofPtr<HTTPSession> session;
istream * rs;
if(uri.getScheme()=="https"){
HTTPSClientSession * httpsSession = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
httpsSession->setTimeout(Poco::Timespan(timeoutSeconds,0));
httpsSession->sendRequest(req);
rs = &httpsSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpsSession);
}else{
HTTPClientSession * httpSession = new HTTPClientSession(uri.getHost(), uri.getPort());
httpSession->setTimeout(Poco::Timespan(timeoutSeconds,0));
httpSession->sendRequest(req);
rs = &httpSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpSession);
}
response=ofxHttpResponse(res, *rs, url);
if(sendCookies){
cookies.insert(cookies.begin(),response.cookies.begin(),response.cookies.end());
}
if(response.status>=300 && response.status<400){
Poco::URI uri(req.getURI());
uri.resolve(res.get("Location"));
response.location = uri.toString();
}
ofNotifyEvent( newResponseEvent, response, this );
//std::cout << res.getStatus() << " " << res.getReason() << std::endl;
//StreamCopier::copyStream(rs, std::cout);
}catch (Exception& exc){
ofLogError("ofxHttpUtils") << exc.displayText();
response.status = -1;
response.reasonForStatus = exc.displayText();
ofNotifyEvent(newResponseEvent, response, this);
}
return response;
}
// ----------------------------------------------------------------------
ofxHttpResponse ofxHttpUtils::doPostForm(ofxHttpForm & form){
ofxHttpResponse response;
try{
URI uri( form.action.c_str() );
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
//HTTPClientSession session(uri.getHost(), uri.getPort());
HTTPRequest req(HTTPRequest::HTTP_POST, path, HTTPMessage::HTTP_1_1);
if(auth.getUsername()!="") auth.authenticate(req);
if(sendCookies){
for(unsigned i=0; i<cookies.size(); i++){
NameValueCollection reqCookies;
reqCookies.add(cookies[i].getName(),cookies[i].getValue());
req.setCookies(reqCookies);
}
}
for (unsigned int i = 0; i < form.headerIds.size(); ++i) {
const std::string name = form.headerIds[i].c_str();
const std::string val = form.headerValues[i].c_str();
req.set(name, val);
}
HTTPResponse res;
HTMLForm pocoForm;
// create the form data to send
if(form.formFiles.size()>0) {
pocoForm.setEncoding(HTMLForm::ENCODING_MULTIPART);
}
else {
pocoForm.setEncoding(HTMLForm::ENCODING_URL);
}
// form values
for(unsigned i=0; i<form.formIds.size(); i++){
const std::string name = form.formIds[i].c_str();
const std::string val = form.formValues[i].c_str();
pocoForm.set(name, val);
}
map<string,string>::iterator it;
for(it = form.formFiles.begin(); it!=form.formFiles.end(); it++){
string fileName = it->second.substr(it->second.find_last_of('/')+1);
ofLogVerbose("ofxHttpUtils") << "adding file: " << fileName << " path: " << it->second;
pocoForm.addPart(it->first,new FilePartSource(it->second));
}
pocoForm.prepareSubmit(req);
ofPtr<HTTPSession> session;
istream * rs;
if(uri.getScheme()=="https"){
HTTPSClientSession * httpsSession = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
httpsSession->setTimeout(Poco::Timespan(20,0));
pocoForm.write(httpsSession->sendRequest(req));
rs = &httpsSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpsSession);
}else{
HTTPClientSession * httpSession = new HTTPClientSession(uri.getHost(), uri.getPort());
httpSession->setTimeout(Poco::Timespan(20,0));
pocoForm.write(httpSession->sendRequest(req));
rs = &httpSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpSession);
}
response = ofxHttpResponse(res, *rs, form.action);
if(sendCookies){
cookies.insert(cookies.begin(),response.cookies.begin(),response.cookies.end());
}
if(response.status>=300 && response.status<400){
Poco::URI uri(req.getURI());
uri.resolve(res.get("Location"));
response.location = uri.toString();
}
ofNotifyEvent(newResponseEvent, response, this);
}catch (Exception& exc){
ofLogError("ofxHttpUtils") << "ofxHttpUtils error doPostForm -- " << form.action.c_str();
//ofNotifyEvent(notifyNewError, "time out", this);
// for now print error, need to broadcast a response
ofLogError("ofxHttpUtils") << exc.displayText();
response.status = -1;
response.reasonForStatus = exc.displayText();
ofNotifyEvent(newResponseEvent, response, this);
}
return response;
}
// ----------------------------------------------------------------------
ofxHttpResponse ofxHttpUtils::postData(string url, const ofBuffer & data, string contentType){
ofxHttpResponse response;
try{
URI uri( url.c_str() );
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
//HTTPClientSession session(uri.getHost(), uri.getPort());
HTTPRequest req(HTTPRequest::HTTP_POST, path, HTTPMessage::HTTP_1_1);
if(auth.getUsername()!="") auth.authenticate(req);
if(sendCookies){
for(unsigned i=0; i<cookies.size(); i++){
NameValueCollection reqCookies;
reqCookies.add(cookies[i].getName(),cookies[i].getValue());
req.setCookies(reqCookies);
}
}
if(contentType!=""){
req.setContentType(contentType);
}
req.setContentLength(data.size());
HTTPResponse res;
ofPtr<HTTPSession> session;
istream * rs;
if(uri.getScheme()=="https"){
HTTPSClientSession * httpsSession = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
httpsSession->setTimeout(Poco::Timespan(20,0));
httpsSession->sendRequest(req) << data;
rs = &httpsSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpsSession);
}else{
HTTPClientSession * httpSession = new HTTPClientSession(uri.getHost(), uri.getPort());
httpSession->setTimeout(Poco::Timespan(20,0));
httpSession->sendRequest(req) << data;
rs = &httpSession->receiveResponse(res);
session = ofPtr<HTTPSession>(httpSession);
}
response = ofxHttpResponse(res, *rs, url);
if(sendCookies){
cookies.insert(cookies.begin(),response.cookies.begin(),response.cookies.end());
}
if(response.status>=300 && response.status<400){
Poco::URI uri(req.getURI());
uri.resolve(res.get("Location"));
response.location = uri.toString();
}
ofNotifyEvent(newResponseEvent, response, this);
}catch (Exception& exc){
ofLogError("ofxHttpUtils") << "ofxHttpUtils error postData --";
//ofNotifyEvent(notifyNewError, "time out", this);
// for now print error, need to broadcast a response
ofLogError("ofxHttpUtils") << exc.displayText();
response.status = -1;
response.reasonForStatus = exc.displayText();
ofNotifyEvent(newResponseEvent, response, this);
}
return response;
}
void ofURLFileLoaderImpl::threadedFunction() {
thread.setName("ofURLFileLoader " + thread.name());
while( isThreadRunning() ){
int cancelled;
while(cancelRequestQueue.tryReceive(cancelled)){
cancelledRequests.insert(cancelled);
}
ofHttpRequest request;
if(requests.receive(request)){
if(cancelledRequests.find(request.getID())==cancelledRequests.end()){
ofHttpResponse response(handleRequest(request));
int status = response.status;
#if __cplusplus>=201103
if(!responses.send(move(response))){
#else
if(!responses.send(response)){
#endif
break;
}
if(status==-1){
// retry
requests.send(request);
}
}else{
cancelledRequests.erase(cancelled);
}
}else{
break;
}
}
}
ofHttpResponse ofURLFileLoaderImpl::handleRequest(ofHttpRequest request) {
try {
URI uri(request.url);
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
HTTPResponse res;
shared_ptr<HTTPSession> session;
istream * rs;
if(uri.getScheme()=="https"){
//const Poco::Net::Context::Ptr context( new Poco::Net::Context( Poco::Net::Context::CLIENT_USE, "", "", "rootcert.pem" ) );
HTTPSClientSession * httpsSession = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
httpsSession->setTimeout(Poco::Timespan(20,0));
httpsSession->sendRequest(req);
rs = &httpsSession->receiveResponse(res);
session = shared_ptr<HTTPSession>(httpsSession);
}else{
HTTPClientSession * httpSession = new HTTPClientSession(uri.getHost(), uri.getPort());
httpSession->setTimeout(Poco::Timespan(20,0));
httpSession->sendRequest(req);
rs = &httpSession->receiveResponse(res);
session = shared_ptr<HTTPSession>(httpSession);
}
if(!request.saveTo){
return ofHttpResponse(request,*rs,res.getStatus(),res.getReason());
}else{
ofFile saveTo(request.name,ofFile::WriteOnly,true);
char aux_buffer[1024];
rs->read(aux_buffer, 1024);
std::streamsize n = rs->gcount();
while (n > 0){
// we resize to size+1 initialized to 0 to have a 0 at the end for strings
saveTo.write(aux_buffer,n);
if (rs->good()){
rs->read(aux_buffer, 1024);
n = rs->gcount();
}
else n = 0;
}
return ofHttpResponse(request,res.getStatus(),res.getReason());
}
} catch (const Exception& exc) {
ofLogError("ofURLFileLoader") << "handleRequest(): "+ exc.displayText();
return ofHttpResponse(request,-1,exc.displayText());
} catch (...) {
return ofHttpResponse(request,-1,"ofURLFileLoader: fatal error, couldn't catch Exception");
}
return ofHttpResponse(request,-1,"ofURLFileLoader: fatal error, couldn't catch Exception");
}
void ofURLFileLoaderImpl::update(ofEventArgs & args){
ofHttpResponse response;
while(responses.tryReceive(response)){
ofNotifyEvent(ofURLResponseEvent(),response);
}
}
