/// checks the validity of the HTTP response
void checkResponse(const HTTPResponse& http_response)
{
BOOST_REQUIRE(http_response.getStatusCode() == 200);
BOOST_CHECK(! http_response.hasHeader(HTTPTypes::HEADER_CONTENT_LENGTH));
BOOST_REQUIRE(http_response.getContentLength() == BIG_BUF_SIZE);
BOOST_CHECK_EQUAL(memcmp(http_response.getContent(), m_big_buf, BIG_BUF_SIZE), 0);
}
void HTTPSClientSessionTest::testPostLargeChunkedKeepAlive()
{
SecureServerSocket svs(32322);
HTTPServer srv(new TestRequestHandlerFactory(), svs, new HTTPServerParams());
srv.start();
try
{
HTTPSClientSession s("localhost", srv.port());
s.setKeepAlive(true);
for (int i = 0; i < 10; ++i)
{
HTTPRequest request(HTTPRequest::HTTP_POST, "/keepAlive", HTTPMessage::HTTP_1_1);
std::string body(16000, 'x');
request.setChunkedTransferEncoding(true);
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getChunkedTransferEncoding());
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == body);
}
srv.stop();
}
catch (...)
{
srv.stop();
throw;
}
}
void HTTPSClientSessionTest::testKeepAlive()
{
HTTPSTestServer srv;
HTTPSClientSession s("localhost", srv.port());
s.setKeepAlive(true);
HTTPRequest request(HTTPRequest::HTTP_HEAD, "/keepAlive", HTTPMessage::HTTP_1_1);
s.sendRequest(request);
HTTPResponse response;
std::istream& rs1 = s.receiveResponse(response);
assert (response.getContentLength() == HTTPSTestServer::SMALL_BODY.length());
assert (response.getContentType() == "text/plain");
assert (response.getKeepAlive());
std::ostringstream ostr1;
assert (StreamCopier::copyStream(rs1, ostr1) == 0);
request.setMethod(HTTPRequest::HTTP_GET);
request.setURI("/small");
s.sendRequest(request);
std::istream& rs2 = s.receiveResponse(response);
assert (response.getContentLength() == HTTPSTestServer::SMALL_BODY.length());
assert (response.getKeepAlive());
std::ostringstream ostr2;
StreamCopier::copyStream(rs2, ostr2);
assert (ostr2.str() == HTTPSTestServer::SMALL_BODY);
request.setMethod(HTTPRequest::HTTP_GET);
request.setURI("/large");
s.sendRequest(request);
std::istream& rs3 = s.receiveResponse(response);
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
assert (response.getChunkedTransferEncoding());
assert (response.getKeepAlive());
std::ostringstream ostr3;
int n = StreamCopier::copyStream(rs3, ostr3);
assert (ostr3.str() == HTTPSTestServer::LARGE_BODY);
request.setMethod(HTTPRequest::HTTP_HEAD);
request.setURI("/large");
s.sendRequest(request);
std::istream& rs4= s.receiveResponse(response);
assert (response.getContentLength() == HTTPSTestServer::LARGE_BODY.length());
assert (response.getContentType() == "text/plain");
assert (!response.getKeepAlive());
std::ostringstream ostr4;
assert (StreamCopier::copyStream(rs4, ostr4) == 0);
}
TEST(HTTPClientSession, testHead)
{
HTTPClientSession s("localhost", 80);
HTTPRequest request(HTTPRequest::HTTP_HEAD, "/large");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
EXPECT_TRUE (response.getContentLength() > 0);
EXPECT_TRUE (response.getContentType() == "text/plain");
std::ostringstream ostr;
EXPECT_TRUE (StreamCopier::copyStream(rs, ostr) == 0);
}
void HTTPSClientSessionTest::testHead()
{
HTTPSTestServer srv;
HTTPSClientSession s("localhost", srv.port());
HTTPRequest request(HTTPRequest::HTTP_HEAD, "/large");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == HTTPSTestServer::LARGE_BODY.length());
assert (response.getContentType() == "text/plain");
std::ostringstream ostr;
assert (StreamCopier::copyStream(rs, ostr) == 0);
}
void HTTPResponseTest::testRead3()
{
std::string s("HTTP/1.1 200 \r\nContent-Length: 0\r\n\r\n");
std::istringstream istr(s);
HTTPResponse response;
response.read(istr);
assert (response.getVersion() == HTTPMessage::HTTP_1_1);
assert (response.getStatus() == HTTPResponse::HTTP_OK);
assert (response.getReason() == "");
assert (response.size() == 1);
assert (response.getContentLength() == 0);
assert (istr.get() == -1);
}
TEST(HTTPClientSession, testGetLarge)
{
HTTPClientSession s("localhost", 80);
HTTPRequest request(HTTPRequest::HTTP_GET, "/index.html");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
EXPECT_TRUE (response.getContentLength() > 0);
EXPECT_TRUE (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
//EXPECT_TRUE (ostr.str() == HTTPTestServer::LARGE_BODY);
}
void HTTPSClientSessionTest::testGetSmall()
{
HTTPSTestServer srv;
HTTPSClientSession s("localhost", srv.port());
HTTPRequest request(HTTPRequest::HTTP_GET, "/small");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == HTTPSTestServer::SMALL_BODY.length());
assert (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == HTTPSTestServer::SMALL_BODY);
}
TEST(HTTPClientSession, testPostSmallIdentity)
{
HTTPClientSession s("localhost", 80);
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body("this is a random request body\r\n0\r\n");
request.setContentLength((int) body.length());
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
EXPECT_TRUE (response.getContentLength() == body.length());
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
EXPECT_TRUE (ostr.str() == body);
}
TEST(HTTPClientSession, testPostLargeClose)
{
HTTPClientSession s("localhost", 80);
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body(8000, 'x');
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
EXPECT_TRUE (!response.getChunkedTransferEncoding());
EXPECT_TRUE (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
EXPECT_TRUE (ostr.str() == body);
}
void HTTPSClientSessionTest::testUnknownContentLength()
{
HTTPSTestServer srv;
HTTPSClientSession s("127.0.0.1", srv.port());
HTTPRequest request(HTTPRequest::HTTP_GET, "/nolength");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
assert (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == HTTPSTestServer::SMALL_BODY);
}
void HTTPSClientSessionTest::testPostSmallIdentity()
{
HTTPSTestServer srv;
HTTPSClientSession s("localhost", srv.port());
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body("this is a random request body\r\n0\r\n");
request.setContentLength((int) body.length());
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == body.length());
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == body);
}
void HTTPSClientSessionTest::testPostLargeClose()
{
HTTPSTestServer srv;
HTTPSClientSession s("localhost", srv.port());
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body(8000, 'x');
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (!response.getChunkedTransferEncoding());
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == body);
}
void HTTPClientSessionTest::testProxy()
{
HTTPTestServer srv;
HTTPClientSession s("www.somehost.com");
s.setProxy("localhost", srv.port());
HTTPRequest request(HTTPRequest::HTTP_GET, "/large");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == HTTPTestServer::LARGE_BODY.length());
assert (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == HTTPTestServer::LARGE_BODY);
}
void HTTPSClientSessionTest::testPostSmallChunked()
{
HTTPSTestServer srv;
HTTPSClientSession s("127.0.0.1", srv.port());
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body("this is a random request body");
request.setChunkedTransferEncoding(true);
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getChunkedTransferEncoding());
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == body);
}
void HTTPSClientSessionTest::testServerAbort()
{
HTTPSTestServer srv;
HTTPSClientSession s("127.0.0.1", srv.port());
HTTPRequest request(HTTPRequest::HTTP_GET, "/nolength/connection/abort");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == HTTPMessage::UNKNOWN_CONTENT_LENGTH);
assert (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == HTTPSTestServer::SMALL_BODY);
assert ( dynamic_cast<const Poco::Net::SSLConnectionUnexpectedlyClosedException*>(
s.networkException()) != NULL );
}
void HTTPSClientSessionTest::testPostLargeIdentity()
{
HTTPSTestServer srv;
HTTPSClientSession s("127.0.0.1", srv.port());
HTTPRequest request(HTTPRequest::HTTP_POST, "/echo");
std::string body(8000, 'x');
body.append("\r\n0\r\n");
request.setContentLength((int) body.length());
s.sendRequest(request) << body;
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
assert (response.getContentLength() == body.length());
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
assert (ostr.str() == body);
}
shared_ptr<FDSObject> GalaxyFDSClient::getObject(const string& bucketName,
const string& objectName, long pos, long len) {
string uri = formatUri(_pConfig->getDownloadBaseUri(), bucketName + "/" +
objectName, vector<string>());
URI pocoUri(uri);
shared_ptr<HTTPClientSession> pSession(_pSessionFacotry
->createClientSession(pocoUri));
pSession->setHost(pocoUri.getHost());
pSession->setPort(pocoUri.getPort());
HTTPRequest request(HTTPRequest::HTTP_GET, uri, HTTPMessage::HTTP_1_1);
FDSObjectMetadata metadata;
if (pos >= 0 || len > 0) {
std::ostringstream os;
os << "bytes=" << pos << "-";
if (len > 0) {
os << (pos + len - 1); // inclusive
}
metadata.add(Constants::RANGE, os.str());
}
prepareRequestHeaders(uri, HTTPRequest::HTTP_GET, "", _emptyStream,
metadata, request);
HTTPResponse response;
pSession->sendRequest(request);
istream& rs = pSession->receiveResponse(response);
if (response.getStatus() != HTTPResponse::HTTP_OK &&
response.getStatus() != HTTPResponse::HTTP_PARTIAL_CONTENT) {
stringstream msg;
msg << "Get object failed, status=" << response.getStatus() << ", reason=";
StreamCopier::copyStream(rs, msg);
throw GalaxyFDSClientException(response.getStatus(), msg.str());
}
shared_ptr<FDSObjectSummary> pObjectSummary(new FDSObjectSummary());
pObjectSummary->setBucketName(bucketName);
pObjectSummary->setObjectName(objectName);
pObjectSummary->setSize(response.getContentLength());
shared_ptr<FDSObject> pObject(new FDSObject());
pObject->setObjectSummary(pObjectSummary);
pObject->setObjectMetadata(this->parseMetadata(response));
pObject->setSession(pSession);
pObject->setObjectContent(rs);
return pObject;
}
TEST(HTTPClientSession, testProxyAuth)
{
HTTPClientSession s("www.somehost.com");
s.setProxy("localhost", 80);
s.setProxyCredentials("user", "pass");
HTTPRequest request(HTTPRequest::HTTP_GET, "/large");
s.sendRequest(request);
HTTPResponse response;
std::istream& rs = s.receiveResponse(response);
EXPECT_TRUE (response.getContentLength() > 0);
EXPECT_TRUE (response.getContentType() == "text/plain");
std::ostringstream ostr;
StreamCopier::copyStream(rs, ostr);
//EXPECT_TRUE (ostr.str() == HTTPTestServer::LARGE_BODY);
//std::string r = srv.lastRequest();
//EXPECT_TRUE (r.find("Proxy-Authorization: Basic dXNlcjpwYXNz\r\n") != std::string::npos);
}
bool ofxSimpleHttp::downloadURL(ofxSimpleHttpResponse* resp, bool sendResultThroughEvents, bool beingCalledFromMainThread, bool saveToDisk){
bool ok;
ofstream myfile;
bool fileIsAlreadyHere = false;
//create a file to save the stream to
if(saveToDisk){
if (resp->expectedChecksum.length()){ //if user provided a checksum
ofFile f;
f.open(resp->absolutePath);
if (f.exists()){
fileIsAlreadyHere = ofxChecksum::sha1(resp->absolutePath, resp->expectedChecksum);
if(fileIsAlreadyHere){
resp->checksumOK = true;
resp->status = 0;
resp->ok = true;
resp->fileWasHere = true;
ofLogVerbose() << "ofxSimpleHttp: about to download "<< resp->url << " but a file with same name and correct checksum is already here!";
ofLogVerbose() << "ofxSimpleHttp: skipping download (" << resp->expectedChecksum << ")";
}
}
f.close();
}else{
if (!onlySkipDownloadIfChecksumMatches){
ofFile f;
f.open(resp->absolutePath);
if (f.exists() && f.getSize() > 0){
resp->checksumOK = false;
resp->status = 0;
resp->ok = true;
resp->fileWasHere = true;
fileIsAlreadyHere = true;
ofLogVerbose() << "ofxSimpleHttp: about to download "<< resp->url << " but a file with same name and (size > 0) is already here!";
ofLogVerbose() << "ofxSimpleHttp: skipping download (missing checksum)";
}
f.close();
}
}
}
if (!fileIsAlreadyHere){ //if file is not here, download it!
myfile.open( resp->absolutePath.c_str(), ios_base::binary );
//myfile.open(ofToDataPath(filename).c_str()); //for not binary?
try {
ofHttpRequest request(resp->url, resp->url);
URI uri(request.url);
std::string path(uri.getPathAndQuery());
if (path.empty()) path = "/";
HTTPClientSession * session;
if(uri.getScheme()=="https"){
session = new HTTPSClientSession(uri.getHost(), uri.getPort());//,context);
}else{
session = new HTTPClientSession(uri.getHost(), uri.getPort());
}
//resp->session = &session;
HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
req.set( "User-Agent", userAgent.c_str() );
session->setTimeout( Poco::Timespan(timeOut,0) );
session->sendRequest(req);
HTTPResponse res;
istream& rs = session->receiveResponse(res);
resp->status = res.getStatus();
try {
resp->timestamp = res.getDate();
} catch (Exception& exc) {
resp->timestamp = 0;
}
resp->reasonForStatus = res.getReasonForStatus( res.getStatus() );
resp->contentType = res.getContentType();
resp->serverReportedSize = res.getContentLength();
resp->timeTakenToDownload = ofGetElapsedTimef();
if (resp->serverReportedSize == -1) ofLogWarning("ofxSimpleHttp", "downloadURL(%s) >> Server doesn't report download size...", resp->fileName.c_str() );
ofLogVerbose("ofxSimpleHttp", "downloadURL() >> about to start download (%s, %d bytes)", resp->fileName.c_str(), res.getContentLength() );
ofLogVerbose("ofxSimpleHttp", "downloadURL() >> server reports request status: (%d-%s)", resp->status, resp->reasonForStatus.c_str() );
//StreamCopier::copyStream(rs, myfile); //write to file here!
if(saveToDisk){
streamCopyWithProgress(rs, myfile, resp->serverReportedSize, resp->downloadProgress, resp->downloadSpeed, resp->downloadCanceled);
}else{
copyToStringWithProgress(rs, resp->responseBody, resp->serverReportedSize, resp->downloadProgress, resp->downloadSpeed, resp->downloadCanceled);
}
resp->timeTakenToDownload = ofGetElapsedTimef() - resp->timeTakenToDownload;
if (resp->expectedChecksum.length() > 0){
resp->checksumOK = ofxChecksum::sha1(resp->absolutePath, resp->expectedChecksum);
if(!resp->checksumOK){
ofLogVerbose() << "ofxSimpleHttp: downloaded OK but Checksum FAILED";
ofLogVerbose() << "ofxSimpleHttp: SHA1 was meant to be: " << resp->expectedChecksum;
}
}
resp->downloadSpeed = 0;
resp->avgDownloadSpeed = 0;
resp->downloadedBytes = 0;
//resp->session = NULL;
delete session;
session = NULL;
if(saveToDisk){
myfile.close();
}
if (resp->downloadCanceled){
//delete half-baked download file
if (resp->downloadToDisk){
ofFile f;
if(f.open(resp->absolutePath)){
if (f.isFile()){
f.remove();
}
}
}
}else{
if(saveToDisk){
ofLogNotice("ofxSimpleHttp", "downloadURL() downloaded to %s", resp->fileName.c_str() );
}
if( saveToDisk ){
//ask the filesystem what is the real size of the file
ofFile file;
try{
file.open(resp->absolutePath.c_str());
resp->downloadedBytes = file.getSize();
file.close();
}catch(Exception& exc){
ofLogError("ofxSimpleHttp", "downloadURL(%s) >> Exception at file.open: %s", resp->fileName.c_str(), exc.displayText().c_str() );
}
}else{
resp->downloadedBytes = resp->responseBody.size();
}
resp->avgDownloadSpeed = (resp->downloadedBytes / 1024.) / resp->timeTakenToDownload; //kb/sec
//check download file size missmatch
if ( resp->serverReportedSize > 0 && resp->serverReportedSize != resp->downloadedBytes) {
ofLogWarning("ofxSimpleHttp", "downloadURLtoDiskBlocking() >> Download size mismatch (%s) >> Server: %d Downloaded: %d",
resp->fileName.c_str(), resp->serverReportedSize, resp->downloadedBytes );
resp->reasonForStatus = "Download size mismatch!";
resp->status = -1;
resp->ok = false;
}else{
if (resp->status == 200){
resp->ok = true;
}else{
resp->ok = false;
}
}
ofLogVerbose() << "ofxSimpleHttp: download finished! " << resp->url << " !";
ok = TRUE;
}
}catch(Exception& exc){
myfile.close();
ofLogError("ofxSimpleHttp", "downloadURL(%s) >> Exception: %s", resp->fileName.c_str(), exc.displayText().c_str() );
resp->reasonForStatus = exc.displayText();
resp->ok = false;
resp->status = -1;
ok = false;
ofLogError() << "ofxSimpleHttp: failed to download " << resp->url << " !";
}
}
//enqueue the operation result!
if (sendResultThroughEvents ){
if ( resp->notifyOnSuccess ){
if(idleTimeAfterEachDownload > 0.0){
ofSleepMillis(idleTimeAfterEachDownload * 1000);
}
if (beingCalledFromMainThread){ //we running on main thread, we can just snd the notif from here
ofNotifyEvent( httpResponse, *resp, this );
}else{ //we are running from a bg thread
if (notifyFromMainThread){ //user wants to get notified form main thread, we need to enqueue the notification
if (timeToStop == false){ //see if we have been destructed! dont forward events if so
lock();
ofxSimpleHttpResponse tempCopy = *resp;
responsesPendingNotification.push(tempCopy);
unlock();
}
}else{ //user doesnt care about main thread, the notificaiton can come from bg thread so we do it from here
ofNotifyEvent( httpResponse, *resp, this );
}
}
}
}
return ok;
}
