void IStreamUrlImplCurl::seekRelative( off_t relativeOffset )
{
// if this move stays inside the current buffer, we're good
if( ( mBufferOffset + relativeOffset >= 0 ) && ( mBufferOffset + relativeOffset < mBufferedBytes ) ) {
mBufferOffset += relativeOffset;
return;
}
else if( relativeOffset < 0 ) { // if we're moving backwards out of the buffer, we have to reset
throw StreamExc(); // need to implement this
}
else { // moving forward off the end of the buffer - keep buffering til we're in range
throw StreamExc(); // need to implement this
}
}
void IStreamUrlImplWinInet::fillBuffer( int wantBytes ) const
{
// we've already got all the data we need
if( bufferDataRemaining() > wantBytes )
return;
// if we want more bytes than will fit in the rest of the buffer, let's make some room
if( mBufferSize - mBufferedBytes < wantBytes ) {
int bytesCulled = mBufferOffset;
memmove( mBuffer, &mBuffer[mBufferOffset], mBufferedBytes - bytesCulled );
mBufferedBytes -= bytesCulled;
mBufferOffset = 0;
mBufferFileOffset += bytesCulled;
}
// now if we've made all the room there is to make, and we still aren't big enough, reallocate
if( wantBytes > mBufferSize - mBufferedBytes ) {
// not enough space in buffer
int oldBufferSize = mBufferSize;
while( mBufferSize - mBufferedBytes < wantBytes )
mBufferSize *= 2;
uint8_t *newBuff = reinterpret_cast<uint8_t*>( realloc( mBuffer, mBufferSize ) );
if( ! newBuff ) {
throw StreamExc();
}
else {
// realloc suceeded increase buffer size
mBuffer = newBuff;
}
}
do {
::DWORD bytesAvailable, bytesToRead, bytesRead;
if( ! ::InternetQueryDataAvailable( mRequest.get(), &bytesAvailable, 0, 0 ) )
throw StreamExc();
if( bytesAvailable == 0 ) {
mIsFinished = true;
break;
}
bytesToRead = std::min<int>( bytesAvailable, wantBytes );
if( ! ::InternetReadFile( mRequest.get(), mBuffer + mBufferedBytes, bytesToRead, &bytesRead ) )
throw StreamExc();
mBufferedBytes += bytesRead;
wantBytes -= bytesRead;
if( wantBytes < 0 )
wantBytes = 0;
} while( wantBytes );
}
IStreamUrlImplCurl::IStreamUrlImplCurl( const std::string &url, const std::string &user, const std::string &password )
: IStreamUrlImpl( user, password ), still_running( 1 ), mSizeCached( false ), mBufferFileOffset( 0 ), mStartedRead( false ),
mEffectiveUrl( 0 ), mResponseCode( 0 )
{
if( ! CURLLib::instance() )
throw StreamExc(); // for some reason the curl lib isn't initialized, and we're screwed
mMulti = curl_multi_init();
mCurl = curl_easy_init();
curl_easy_setopt( mCurl, CURLOPT_URL, url.c_str() );
curl_easy_setopt( mCurl, CURLOPT_WRITEDATA, this );
curl_easy_setopt( mCurl, CURLOPT_VERBOSE, 0L );
curl_easy_setopt( mCurl, CURLOPT_FOLLOWLOCATION, 1L );
curl_easy_setopt( mCurl, CURLOPT_WRITEFUNCTION, IStreamUrlImplCurl::writeCallback );
if( ( ! mUser.empty() ) || ( ! mPassword.empty() ) ) {
mUserColonPassword = mUser + ":" + mPassword;
curl_easy_setopt( mCurl, CURLOPT_USERPWD, mUserColonPassword.c_str() );
curl_easy_setopt( mCurl, CURLOPT_HTTPAUTH, CURLAUTH_ANY );
}
curl_multi_add_handle( mMulti, mCurl );
// we fill the buffer just to get things rolling
mBufferSize = DEFAULT_BUFFER_SIZE;
mBuffer = (uint8_t*)malloc( mBufferSize );
mBufferOffset = 0;
mBufferedBytes = 0;
mBufferFileOffset = 0;
// fillBuffer( Stream::MINIMUM_BUFFER_SIZE );
}
void DataSourcePath::createBuffer()
{
// no-op - we already supplied the buffer in the constructor
IStreamFileRef stream = loadFileStream( mFilePath );
if( ! stream )
throw StreamExc();
mBuffer = loadStreamBuffer( stream );
}
void IStreamUrlImplCurl::IORead( void *dest, size_t size )
{
fillBuffer( size );
// check if theres data in the buffer - if not fillBuffer() either errored or EOF
if( bufferRemaining() < (off_t)size )
throw StreamExc();
memcpy( dest, mBuffer + mBufferOffset, size );
mBufferOffset += size;
}
void IStreamUrlImplCurl::fillBuffer( int wantBytes ) const
{
// first make sure we've started reading, and do so if not
if( ! mStartedRead ) {
while( curl_multi_perform( mMulti, &still_running ) == CURLM_CALL_MULTI_PERFORM );
if( ( bufferRemaining() == 0 ) && ( ! still_running ) ) {
throw StreamExc();
}
mStartedRead = true;
}
// only attempt to fill buffer if transactions still running and buffer
// doesnt exceed required size already
if( ( ! still_running ) || ( bufferRemaining() >= wantBytes ) )
return;
// if we want more bytes than will fit in the rest of the buffer, let's make some room
if( mBufferSize - mBufferedBytes < wantBytes ) {
int bytesCulled = mBufferOffset;
memmove( mBuffer, &mBuffer[mBufferOffset], mBufferedBytes - bytesCulled );
mBufferedBytes -= bytesCulled;
mBufferOffset = 0;
mBufferFileOffset += bytesCulled;
}
// attempt to fill buffer
do {
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
struct timeval timeout;
FD_ZERO( &fdread );
FD_ZERO( &fdwrite );
FD_ZERO( &fdexcep );
// set a suitable timeout to fail on
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
// get file descriptors from the transfers
curl_multi_fdset( mMulti, &fdread, &fdwrite, &fdexcep, &maxfd );
int rc = select( maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout );
switch( rc ) {
case -1:
throw StreamExc();
break;
case 0:
break;
default:
// timeout or readable/writable sockets
// note we *could* be more efficient and not wait for
// CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry
// but that gets messy
while( curl_multi_perform( mMulti, &still_running ) == CURLM_CALL_MULTI_PERFORM );
break;
}
} while( still_running && ( bufferRemaining() < wantBytes ) );
}
IStreamUrlImplWinInet::IStreamUrlImplWinInet( const std::string &url, const std::string &user, const std::string &password )
: IStreamUrlImpl( user, password ), mIsFinished( false ), mBuffer( 0 ), mBufferFileOffset( 0 )
{
std::wstring wideUrl = toUtf16( url );
// we need to break the URL up into its constituent parts so we can choose a scheme
URL_COMPONENTS urlComponents;
::memset( &urlComponents, 0, sizeof(urlComponents) );
urlComponents.dwStructSize = sizeof(urlComponents);
urlComponents.dwSchemeLength = 1;
urlComponents.dwHostNameLength = 1;
urlComponents.dwHostNameLength = 1;
urlComponents.dwUrlPathLength = 1;
BOOL success = ::InternetCrackUrl( wideUrl.c_str(), 0, 0, &urlComponents );
if( ! success )
throw StreamExc();
// TODO this should be made safe against buffer overflows
WCHAR host[1024], path[2048];
memcpy( host, urlComponents.lpszHostName, urlComponents.dwHostNameLength * sizeof(WCHAR) );
host[urlComponents.dwHostNameLength] = 0;
memcpy( path, urlComponents.lpszUrlPath, urlComponents.dwUrlPathLength * sizeof(WCHAR) );
path[urlComponents.dwUrlPathLength] = 0;
// make sure this a scheme we know about - HTTP(S) or FTP
switch( urlComponents.nScheme ) {
case INTERNET_SCHEME_HTTP: case INTERNET_SCHEME_HTTPS: case INTERNET_SCHEME_FTP: break;
default: throw StreamExc();
}
mSession = std::shared_ptr<void>( ::InternetOpen( AGENT_NAME, INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0 ), safeInternetCloseHandle );
if( ! mSession )
throw StreamExc();
std::wstring wideUser = toUtf16( user );
std::wstring widePassword = toUtf16( password );
//check for HTTP and HTTPS here because they both require the same flag in InternetConnect()
if( ( urlComponents.nScheme == INTERNET_SCHEME_HTTP ) ||
( urlComponents.nScheme == INTERNET_SCHEME_HTTPS ) ) {
mConnection = std::shared_ptr<void>( ::InternetConnect( mSession.get(), host, urlComponents.nPort, (wideUser.empty()) ? NULL : wideUser.c_str(), (widePassword.empty()) ? NULL : widePassword.c_str(), INTERNET_SERVICE_HTTP, 0, NULL ),
safeInternetCloseHandle );
}else{
//otherwise we just want to take our best shot at the Scheme type.
mConnection = std::shared_ptr<void>( ::InternetConnect( mSession.get(), host, urlComponents.nPort, (wideUser.empty()) ? NULL : wideUser.c_str(), (widePassword.empty()) ? NULL : widePassword.c_str(), urlComponents.nScheme, 0, NULL ),
safeInternetCloseHandle );
}
if( ! mConnection )
throw StreamExc();
//http and https cases broken out incase someone wishes to modify connection based off of type.
//it is wrong to group http with https.
//http
if(urlComponents.nScheme == INTERNET_SCHEME_HTTP ) {
static LPCTSTR lpszAcceptTypes[] = { L"*/*", NULL };
mRequest = std::shared_ptr<void>( ::HttpOpenRequest( mConnection.get(), L"GET", path, NULL, NULL, lpszAcceptTypes,
INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_NO_COOKIES | INTERNET_FLAG_RELOAD, NULL ),
safeInternetCloseHandle );
if( ! mRequest )
throw StreamExc();
BOOL success = ::HttpSendRequest( mRequest.get(), NULL, 0, NULL, 0);
if( ! success )
throw StreamExc();
}
//https
else if(urlComponents.nScheme == INTERNET_SCHEME_HTTPS ) {
static LPCTSTR lpszAcceptTypes[] = { L"*/*", NULL };
mRequest = std::shared_ptr<void>( ::HttpOpenRequest( mConnection.get(), L"GET", path, NULL, NULL, lpszAcceptTypes,
INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_NO_COOKIES | INTERNET_FLAG_RELOAD | INTERNET_FLAG_SECURE, NULL ),
safeInternetCloseHandle );
if( ! mRequest )
throw StreamExc();
BOOL success = ::HttpSendRequest( mRequest.get(), NULL, 0, NULL, 0);
if( ! success )
throw StreamExc();
}
//ftp
else if( urlComponents.nScheme == INTERNET_SCHEME_FTP ) {
mRequest = std::shared_ptr<void>( ::FtpOpenFile( mConnection.get(), path, GENERIC_READ, FTP_TRANSFER_TYPE_BINARY, NULL ),
safeInternetCloseHandle );
if( ! mRequest )
throw StreamExc();
}
mBufferSize = DEFAULT_BUFFER_SIZE;
mBuffer = (uint8_t*)malloc( mBufferSize );
mBufferOffset = 0;
mBufferedBytes = 0;
mBufferFileOffset = 0;
}
