bool UrlReadToMemory( const string & urlString, vector< uchar > & data, map<string, string> & header ) { data.clear(); UniformResourceLocator u( urlString ); if ( u.protocol == UrlProtocol_INVALID ) { return false; } uint ip = GetIpAddress( u.hostname ); Socket sock; if ( sock.Connect( ip, u.port ) == false ) { return false; } char buf[512]; int sz; if( header.count("etag") == 0 ) { r3Sprintf( buf, "GET %s HTTP/1.1\r\nHost: %s:%d\r\n\r\n", u.path.c_str(), u.hostname.c_str(), u.port ); } else { r3Sprintf( buf, "GET %s HTTP/1.1\r\nIf-None-Match: \"%s\"\r\nHost: %s:%d\r\n\r\n", u.path.c_str(), header["etag"].c_str(), u.hostname.c_str(), u.port ); } sz = (int)strlen( buf ); Output( "Sending http request (%d chars): %s", sz, buf ); sock.Write( buf, sz ); InputStream is( sock ); // add If-None-Match with etag in the header.... int read_try_count = 5; while( read_try_count > 0 && sock.CanRead() == false ) { SleepMilliseconds( 500 ); read_try_count--; } if( read_try_count == 0 ) { Output( "Socket read for %s timed out.", urlString.c_str() ); sock.Disconnect(); return false; } HttpResponse resp ( is ); if( resp.code == 404 || resp.code == 304 ) { Output( "Http exiting read with code %d", resp.code ); sock.Disconnect(); return false; } header = resp.header; /* Output( "%s %d %s", resp.protocol.c_str(), resp.code, resp.reason.c_str() ); for( map<string, string>::iterator it = resp.header.begin(); it != resp.header.end(); ++it ) { Output( " %s: %s", it->first.c_str(), it->second.c_str() ); } */ int bytes = resp.GetInt("Content-Length"); if ( bytes > 0 ) { // not chunked - single payload int r = is.Read( bytes, data ); Output( "content length = %d, and read = %d", bytes, r ); } else if ( resp.GetString( "Transfer-Encoding" ).find( "chunked" ) != string::npos ) { // chunks while( 1 ) { string chunkHeader = is.GetLine(); if( chunkHeader.size() < 2 ) { break; } r3Sscanf( chunkHeader.c_str(), "%x", &bytes ); if( bytes > 0 ) { is.Read( bytes, data ); is.GetChar(); is.GetChar(); // skip the CRLF } else { break; } } //Output( "chunked output, abandon hope! (or implement me)"); } sock.Disconnect(); return true; }