int VSIMemFilesystemHandler::Rmdir( const char * pszPathname )
{
CPLMutexHolder oHolder( &hMutex );
return Unlink( pszPathname );
}
int VSIMemFilesystemHandler::Rename( const char *pszOldPath,
const char *pszNewPath )
{
CPLMutexHolder oHolder( &hMutex );
CPLString osOldPath = pszOldPath;
CPLString osNewPath = pszNewPath;
NormalizePath( osOldPath );
NormalizePath( osNewPath );
if ( osOldPath.compare(osNewPath) == 0 )
return 0;
if( oFileList.find(osOldPath) == oFileList.end() )
{
errno = ENOENT;
return -1;
}
else
{
VSIMemFile* poFile = oFileList[osOldPath];
oFileList.erase( oFileList.find(osOldPath) );
Unlink(osNewPath);
oFileList[osNewPath] = poFile;
poFile->osFilename = osNewPath;
return 0;
}
}
int VSIMemFilesystemHandler::Unlink( const char * pszFilename )
{
CPLMutexHolder oHolder( &hMutex );
CPLString osFilename = pszFilename;
NormalizePath( osFilename );
VSIMemFile *poFile;
if( oFileList.find(osFilename) == oFileList.end() )
{
errno = ENOENT;
return -1;
}
else
{
poFile = oFileList[osFilename];
if( --(poFile->nRefCount) == 0 )
delete poFile;
oFileList.erase( oFileList.find(osFilename) );
return 0;
}
}
int VSIMemFilesystemHandler::Mkdir( const char * pszPathname,
long nMode )
{
(void) nMode;
CPLMutexHolder oHolder( &hMutex );
CPLString osPathname = pszPathname;
NormalizePath( osPathname );
if( oFileList.find(osPathname) != oFileList.end() )
{
errno = EEXIST;
return -1;
}
VSIMemFile *poFile = new VSIMemFile;
poFile->osFilename = osPathname;
poFile->bIsDirectory = TRUE;
oFileList[osPathname] = poFile;
poFile->nRefCount++; /* referenced by file list */
return 0;
}
VSIFileManager *VSIFileManager::Get()
{
static volatile GPtrDiff_t nConstructerPID = 0;
if( poManager != NULL )
{
if( nConstructerPID != 0 )
{
GPtrDiff_t nCurrentPID = static_cast<GPtrDiff_t>(CPLGetPID());
if( nConstructerPID != nCurrentPID )
{
{
CPLMutexHolder oHolder( &hVSIFileManagerMutex );
}
if ( nConstructerPID != 0 )
{
VSIDebug1( "nConstructerPID != 0: %d", nConstructerPID);
assert(false);
}
}
}
return poManager;
}
CPLMutexHolder oHolder2( &hVSIFileManagerMutex );
if( poManager == NULL )
{
nConstructerPID = static_cast<GPtrDiff_t>(CPLGetPID());
#ifdef DEBUG_VERBOSE
printf("Thread %d: VSIFileManager in construction\n", nConstructerPID);
#endif
poManager = new VSIFileManager;
VSIInstallLargeFileHandler();
VSIInstallSubFileHandler();
VSIInstallMemFileHandler();
#ifdef HAVE_LIBZ
VSIInstallGZipFileHandler();
VSIInstallZipFileHandler();
#endif
#ifdef HAVE_CURL
VSIInstallCurlFileHandler();
VSIInstallCurlStreamingFileHandler();
VSIInstallS3FileHandler();
VSIInstallS3StreamingFileHandler();
#endif
VSIInstallStdinHandler();
VSIInstallStdoutHandler();
VSIInstallSparseFileHandler();
VSIInstallTarFileHandler();
VSIInstallCryptFileHandler();
//printf("Thread %d: VSIFileManager construction finished\n", nConstructerPID);
nConstructerPID = 0;
}
return poManager;
}
VSIFileManager *VSIFileManager::Get()
{
static volatile int nConstructerPID = 0;
if( poManager != NULL )
{
if( nConstructerPID != 0 )
{
int nCurrentPID = (int)CPLGetPID();
if( nConstructerPID != nCurrentPID )
{
//printf("Thread %d: Waiting for VSIFileManager to be finished by other thread.\n", nCurrentPID);
{
CPLMutexHolder oHolder( &hVSIFileManagerMutex );
}
//printf("Thread %d: End of wait for VSIFileManager construction to be finished\n", nCurrentPID);
CPLAssert(nConstructerPID == 0);
}
}
return poManager;
}
CPLMutexHolder oHolder2( &hVSIFileManagerMutex );
if( poManager == NULL )
{
nConstructerPID = (int)CPLGetPID();
//printf("Thread %d: VSIFileManager in construction\n", nConstructerPID);
poManager = new VSIFileManager;
VSIInstallLargeFileHandler();
VSIInstallSubFileHandler();
VSIInstallMemFileHandler();
#ifdef HAVE_LIBZ
VSIInstallGZipFileHandler();
VSIInstallZipFileHandler();
#endif
#ifdef HAVE_CURL
VSIInstallCurlFileHandler();
VSIInstallCurlStreamingFileHandler();
#endif
VSIInstallStdinHandler();
VSIInstallStdoutHandler();
VSIInstallSparseFileHandler();
VSIInstallTarFileHandler();
//printf("Thread %d: VSIFileManager construction finished\n", nConstructerPID);
nConstructerPID = 0;
}
return poManager;
}
int VSIMemFilesystemHandler::Stat( const char * pszFilename,
VSIStatBufL * pStatBuf,
int nFlags )
{
(void) nFlags;
CPLMutexHolder oHolder( &hMutex );
CPLString osFilename = pszFilename;
NormalizePath( osFilename );
memset( pStatBuf, 0, sizeof(VSIStatBufL) );
if ( osFilename == "/vsimem/" )
{
pStatBuf->st_size = 0;
pStatBuf->st_mode = S_IFDIR;
return 0;
}
if( oFileList.find(osFilename) == oFileList.end() )
{
errno = ENOENT;
return -1;
}
VSIMemFile *poFile = oFileList[osFilename];
memset( pStatBuf, 0, sizeof(VSIStatBufL) );
if( poFile->bIsDirectory )
{
pStatBuf->st_size = 0;
pStatBuf->st_mode = S_IFDIR;
}
else
{
pStatBuf->st_size = poFile->nLength;
pStatBuf->st_mode = S_IFREG;
pStatBuf->st_mtime = poFile->mTime;
}
return 0;
}
GByte *VSIGetMemFileBuffer( const char *pszFilename,
vsi_l_offset *pnDataLength,
int bUnlinkAndSeize )
{
VSIMemFilesystemHandler *poHandler = (VSIMemFilesystemHandler *)
VSIFileManager::GetHandler("/vsimem/");
if (pszFilename == NULL)
return NULL;
CPLString osFilename = pszFilename;
VSIMemFilesystemHandler::NormalizePath( osFilename );
CPLMutexHolder oHolder( &poHandler->hMutex );
if( poHandler->oFileList.find(osFilename) == poHandler->oFileList.end() )
return NULL;
VSIMemFile *poFile = poHandler->oFileList[osFilename];
GByte *pabyData;
pabyData = poFile->pabyData;
if( pnDataLength != NULL )
*pnDataLength = poFile->nLength;
if( bUnlinkAndSeize )
{
if( !poFile->bOwnData )
CPLDebug( "VSIMemFile",
"File doesn't own data in VSIGetMemFileBuffer!" );
else
poFile->bOwnData = FALSE;
poHandler->oFileList.erase( poHandler->oFileList.find(osFilename) );
--(poFile->nRefCount);
delete poFile;
}
return pabyData;
}
int VSIMemFilesystemHandler::Rename( const char *pszOldPath,
const char *pszNewPath )
{
CPLMutexHolder oHolder( &hMutex );
CPLString osOldPath = pszOldPath;
CPLString osNewPath = pszNewPath;
NormalizePath( osOldPath );
NormalizePath( osNewPath );
if ( osOldPath.compare(osNewPath) == 0 )
return 0;
if( oFileList.find(osOldPath) == oFileList.end() )
{
errno = ENOENT;
return -1;
}
else
{
std::map<CPLString,VSIMemFile*>::iterator it = oFileList.find(osOldPath);
while (it != oFileList.end() && it->first.ifind(osOldPath) == 0)
{
const CPLString osRemainder = it->first.substr(osOldPath.size());
if (osRemainder.empty() || osRemainder[0] == '/')
{
const CPLString osNewFullPath = osNewPath + osRemainder;
Unlink_unlocked(osNewFullPath);
oFileList[osNewFullPath] = it->second;
it->second->osFilename = osNewFullPath;
oFileList.erase(it++);
}
else ++it;
}
return 0;
}
}
VSILFILE *VSIFileFromMemBuffer( const char *pszFilename,
GByte *pabyData,
vsi_l_offset nDataLength,
int bTakeOwnership )
{
if( VSIFileManager::GetHandler("")
== VSIFileManager::GetHandler("/vsimem/") )
VSIInstallMemFileHandler();
VSIMemFilesystemHandler *poHandler = (VSIMemFilesystemHandler *)
VSIFileManager::GetHandler("/vsimem/");
if (pszFilename == NULL)
return NULL;
CPLString osFilename = pszFilename;
VSIMemFilesystemHandler::NormalizePath( osFilename );
VSIMemFile *poFile = new VSIMemFile;
poFile->osFilename = osFilename;
poFile->bOwnData = bTakeOwnership;
poFile->pabyData = pabyData;
poFile->nLength = nDataLength;
poFile->nAllocLength = nDataLength;
{
CPLMutexHolder oHolder( &poHandler->hMutex );
poHandler->Unlink(osFilename);
poHandler->oFileList[poFile->osFilename] = poFile;
poFile->nRefCount++;
}
return (VSILFILE *) poHandler->Open( osFilename, "r+" );
}
VSIVirtualHandle *
VSIMemFilesystemHandler::Open( const char *pszFilename,
const char *pszAccess )
{
CPLMutexHolder oHolder( &hMutex );
VSIMemFile *poFile;
CPLString osFilename = pszFilename;
NormalizePath( osFilename );
/* -------------------------------------------------------------------- */
/* Get the filename we are opening, create if needed. */
/* -------------------------------------------------------------------- */
if( oFileList.find(osFilename) == oFileList.end() )
poFile = NULL;
else
poFile = oFileList[osFilename];
if( strstr(pszAccess,"w") == NULL && poFile == NULL )
{
errno = ENOENT;
return NULL;
}
if( strstr(pszAccess,"w") )
{
if( poFile )
poFile->SetLength( 0 );
else
{
poFile = new VSIMemFile;
poFile->osFilename = osFilename;
oFileList[poFile->osFilename] = poFile;
poFile->nRefCount++; // for file list
}
}
if( poFile->bIsDirectory )
{
errno = EISDIR;
return NULL;
}
/* -------------------------------------------------------------------- */
/* Setup the file handle on this file. */
/* -------------------------------------------------------------------- */
VSIMemHandle *poHandle = new VSIMemHandle;
poHandle->poFile = poFile;
poHandle->nOffset = 0;
if( strstr(pszAccess,"w") || strstr(pszAccess,"+")
|| strstr(pszAccess,"a") )
poHandle->bUpdate = TRUE;
else
poHandle->bUpdate = FALSE;
poFile->nRefCount++;
if( strstr(pszAccess,"a") )
poHandle->nOffset = poFile->nLength;
return poHandle;
}
/**
* \brief Fetch a document from an url and return in a string.
*
* @param pszURL valid URL recognized by underlying download library (libcurl)
* @param papszOptions option list as a NULL-terminated array of strings. May be NULL.
* The following options are handled :
* <ul>
* <li>TIMEOUT=val, where val is in seconds</li>
* <li>HEADERS=val, where val is an extra header to use when getting a web page.
* For example "Accept: application/x-ogcwkt"
* <li>HTTPAUTH=[BASIC/NTLM/GSSNEGOTIATE/ANY] to specify an authentication scheme to use.
* <li>USERPWD=userid:password to specify a user and password for authentication
* <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the server
* with a POST request.
* <li>PROXY=val, to make requests go through a proxy server, where val is of the
* form proxy.server.com:port_number
* <li>PROXYUSERPWD=val, where val is of the form username:password
* <li>PROXYAUTH=[BASIC/NTLM/DIGEST/ANY] to specify an proxy authentication scheme to use.
* <li>NETRC=[YES/NO] to enable or disable use of $HOME/.netrc, default YES.
* <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL >= 1.9.0)
* <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2; ...
* <li>MAX_RETRY=val, where val is the maximum number of retry attempts if a 503 or
* 504 HTTP error occurs. Default is 0. (GDAL >= 2.0)
* <li>RETRY_DELAY=val, where val is the number of seconds between retry attempts.
* Default is 30. (GDAL >= 2.0)
* </ul>
*
* Alternatively, if not defined in the papszOptions arguments, the PROXY,
* PROXYUSERPWD, PROXYAUTH, NETRC, MAX_RETRY and RETRY_DELAY values are searched in the configuration
* options named GDAL_HTTP_PROXY, GDAL_HTTP_PROXYUSERPWD, GDAL_PROXY_AUTH,
* GDAL_HTTP_NETRC, GDAL_HTTP_MAX_RETRY and GDAL_HTTP_RETRY_DELAY.
*
* @return a CPLHTTPResult* structure that must be freed by
* CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
*/
CPLHTTPResult *CPLHTTPFetch( const char *pszURL, char **papszOptions )
{
if( strncmp(pszURL, "/vsimem/", strlen("/vsimem/")) == 0 &&
/* Disabled by default for potential security issues */
CSLTestBoolean(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")) )
{
CPLString osURL(pszURL);
const char* pszPost = CSLFetchNameValue( papszOptions, "POSTFIELDS" );
if( pszPost != NULL ) /* Hack: we append post content to filename */
{
osURL += "&POSTFIELDS=";
osURL += pszPost;
}
vsi_l_offset nLength = 0;
CPLHTTPResult* psResult = (CPLHTTPResult* )CPLCalloc(1, sizeof(CPLHTTPResult));
GByte* pabyData = VSIGetMemFileBuffer( osURL, &nLength, FALSE );
if( pabyData == NULL )
{
CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
psResult->nStatus = 1;
psResult->pszErrBuf = CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
CPLError( CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf );
}
else if( nLength != 0 )
{
psResult->nDataLen = (size_t)nLength;
psResult->pabyData = (GByte*) CPLMalloc((size_t)nLength + 1);
memcpy(psResult->pabyData, pabyData, (size_t)nLength);
psResult->pabyData[(size_t)nLength] = 0;
}
if( psResult->pabyData != NULL &&
strncmp((const char*)psResult->pabyData, "Content-Type: ",
strlen("Content-Type: ")) == 0 )
{
const char* pszContentType = (const char*)psResult->pabyData + strlen("Content-type: ");
const char* pszEOL = strchr(pszContentType, '\r');
if( pszEOL )
pszEOL = strchr(pszContentType, '\n');
if( pszEOL )
{
int nLength = pszEOL - pszContentType;
psResult->pszContentType = (char*)CPLMalloc(nLength + 1);
memcpy(psResult->pszContentType, pszContentType, nLength);
psResult->pszContentType[nLength] = 0;
}
}
return psResult;
}
#ifndef HAVE_CURL
(void) papszOptions;
(void) pszURL;
CPLError( CE_Failure, CPLE_NotSupported,
"GDAL/OGR not compiled with libcurl support, remote requests not supported." );
return NULL;
#else
/* -------------------------------------------------------------------- */
/* Are we using a persistent named session? If so, search for */
/* or create it. */
/* */
/* Currently this code does not attempt to protect against */
/* multiple threads asking for the same named session. If that */
/* occurs it will be in use in multiple threads at once which */
/* might have bad consequences depending on what guarantees */
/* libcurl gives - which I have not investigated. */
/* -------------------------------------------------------------------- */
CURL *http_handle = NULL;
const char *pszPersistent = CSLFetchNameValue( papszOptions, "PERSISTENT" );
const char *pszClosePersistent = CSLFetchNameValue( papszOptions, "CLOSE_PERSISTENT" );
if (pszPersistent)
{
CPLString osSessionName = pszPersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( poSessionMap == NULL )
poSessionMap = new std::map<CPLString,CURL*>;
if( poSessionMap->count( osSessionName ) == 0 )
{
(*poSessionMap)[osSessionName] = curl_easy_init();
CPLDebug( "HTTP", "Establish persistent session named '%s'.",
osSessionName.c_str() );
}
http_handle = (*poSessionMap)[osSessionName];
}
/* -------------------------------------------------------------------- */
/* Are we requested to close a persistent named session? */
/* -------------------------------------------------------------------- */
else if (pszClosePersistent)
{
CPLString osSessionName = pszClosePersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( poSessionMap )
{
std::map<CPLString,CURL*>::iterator oIter = poSessionMap->find( osSessionName );
if( oIter != poSessionMap->end() )
{
curl_easy_cleanup(oIter->second);
poSessionMap->erase(oIter);
if( poSessionMap->size() == 0 )
{
delete poSessionMap;
poSessionMap = NULL;
}
CPLDebug( "HTTP", "Ended persistent session named '%s'.",
osSessionName.c_str() );
}
else
{
CPLDebug( "HTTP", "Could not find persistent session named '%s'.",
osSessionName.c_str() );
}
}
return NULL;
}
else
http_handle = curl_easy_init();
/* -------------------------------------------------------------------- */
/* Setup the request. */
/* -------------------------------------------------------------------- */
char szCurlErrBuf[CURL_ERROR_SIZE+1];
CPLHTTPResult *psResult;
struct curl_slist *headers=NULL;
const char* pszArobase = strchr(pszURL, '@');
const char* pszSlash = strchr(pszURL, '/');
const char* pszColon = (pszSlash) ? strchr(pszSlash, ':') : NULL;
if (pszArobase != NULL && pszColon != NULL && pszArobase - pszColon > 0)
{
/* http://user:[email protected] */
char* pszSanitizedURL = CPLStrdup(pszURL);
pszSanitizedURL[pszColon-pszURL] = 0;
CPLDebug( "HTTP", "Fetch(%s:#password#%s)", pszSanitizedURL, pszArobase );
CPLFree(pszSanitizedURL);
}
else
{
CPLDebug( "HTTP", "Fetch(%s)", pszURL );
}
psResult = (CPLHTTPResult *) CPLCalloc(1,sizeof(CPLHTTPResult));
curl_easy_setopt(http_handle, CURLOPT_URL, pszURL );
CPLHTTPSetOptions(http_handle, papszOptions);
// turn off SSL verification, accept all servers with ssl
curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, FALSE);
/* Set Headers.*/
const char *pszHeaders = CSLFetchNameValue( papszOptions, "HEADERS" );
if( pszHeaders != NULL ) {
CPLDebug ("HTTP", "These HTTP headers were set: %s", pszHeaders);
headers = curl_slist_append(headers, pszHeaders);
curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
}
// are we making a head request
const char* pszNoBody = NULL;
if ((pszNoBody = CSLFetchNameValue( papszOptions, "NO_BODY" )) != NULL)
{
if (CSLTestBoolean(pszNoBody))
{
CPLDebug ("HTTP", "HEAD Request: %s", pszURL);
curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
}
}
// capture response headers
curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION, CPLHdrWriteFct);
curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, psResult );
curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, CPLWriteFct );
szCurlErrBuf[0] = '\0';
curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
static int bHasCheckVersion = FALSE;
static int bSupportGZip = FALSE;
if (!bHasCheckVersion)
{
bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
bHasCheckVersion = TRUE;
}
int bGZipRequested = FALSE;
if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
{
bGZipRequested = TRUE;
curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
}
/* -------------------------------------------------------------------- */
/* If 502, 503 or 504 status code retry this HTTP call until max */
/* retry has been rearched */
/* -------------------------------------------------------------------- */
const char *pszRetryDelay = CSLFetchNameValue( papszOptions, "RETRY_DELAY" );
if( pszRetryDelay == NULL )
pszRetryDelay = CPLGetConfigOption( "GDAL_HTTP_RETRY_DELAY", "30" );
const char *pszMaxRetries = CSLFetchNameValue( papszOptions, "MAX_RETRY" );
if( pszMaxRetries == NULL )
pszMaxRetries = CPLGetConfigOption( "GDAL_HTTP_MAX_RETRY", "0" );
int nRetryDelaySecs = atoi(pszRetryDelay);
int nMaxRetries = atoi(pszMaxRetries);
int nRetryCount = 0;
bool bRequestRetry;
do
{
bRequestRetry = FALSE;
/* -------------------------------------------------------------------- */
/* Execute the request, waiting for results. */
/* -------------------------------------------------------------------- */
psResult->nStatus = (int) curl_easy_perform( http_handle );
/* -------------------------------------------------------------------- */
/* Fetch content-type if possible. */
/* -------------------------------------------------------------------- */
psResult->pszContentType = NULL;
curl_easy_getinfo( http_handle, CURLINFO_CONTENT_TYPE,
&(psResult->pszContentType) );
if( psResult->pszContentType != NULL )
psResult->pszContentType = CPLStrdup(psResult->pszContentType);
/* -------------------------------------------------------------------- */
/* Have we encountered some sort of error? */
/* -------------------------------------------------------------------- */
if( strlen(szCurlErrBuf) > 0 )
{
int bSkipError = FALSE;
/* Some servers such as http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities */
/* invalidly return Content-Length as the uncompressed size, with makes curl to wait for more data */
/* and time-out finally. If we got the expected data size, then we don't emit an error */
/* but turn off GZip requests */
if (bGZipRequested &&
strstr(szCurlErrBuf, "transfer closed with") &&
strstr(szCurlErrBuf, "bytes remaining to read"))
{
const char* pszContentLength =
CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
if (pszContentLength && psResult->nDataLen != 0 &&
atoi(pszContentLength) == psResult->nDataLen)
{
const char* pszCurlGZIPOption = CPLGetConfigOption("CPL_CURL_GZIP", NULL);
if (pszCurlGZIPOption == NULL)
{
CPLSetConfigOption("CPL_CURL_GZIP", "NO");
CPLDebug("HTTP", "Disabling CPL_CURL_GZIP, because %s doesn't support it properly",
pszURL);
}
psResult->nStatus = 0;
bSkipError = TRUE;
}
}
if (!bSkipError)
{
psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
CPLError( CE_Failure, CPLE_AppDefined,
"%s", szCurlErrBuf );
}
}
else
{
/* HTTP errors do not trigger curl errors. But we need to */
/* propagate them to the caller though */
long response_code = 0;
curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
if (response_code >= 400 && response_code < 600)
{
/* If HTTP 502, 503 or 504 gateway timeout error retry after a pause */
if ((response_code >= 502 && response_code <= 504) && nRetryCount < nMaxRetries)
{
CPLError(CE_Warning, CPLE_AppDefined,
"HTTP error code: %d - %s. Retrying again in %d secs",
(int)response_code, pszURL, nRetryDelaySecs);
CPLSleep(nRetryDelaySecs);
nRetryCount++;
CPLFree(psResult->pszContentType);
psResult->pszContentType = NULL;
CSLDestroy(psResult->papszHeaders);
psResult->papszHeaders = NULL;
CPLFree(psResult->pabyData);
psResult->pabyData = NULL;
psResult->nDataLen = 0;
psResult->nDataAlloc = 0;
bRequestRetry = TRUE;
}
else
{
psResult->pszErrBuf = CPLStrdup(CPLSPrintf("HTTP error code : %d", (int)response_code));
CPLError( CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf );
}
}
}
}
while (bRequestRetry);
if (!pszPersistent)
curl_easy_cleanup( http_handle );
curl_slist_free_all(headers);
return psResult;
#endif /* def HAVE_CURL */
}
int VSIMemFilesystemHandler::Unlink( const char * pszFilename )
{
CPLMutexHolder oHolder( &hMutex );
return Unlink_unlocked(pszFilename);
}
const VSIArchiveContent* VSIArchiveFilesystemHandler::GetContentOfArchive
(const char* archiveFilename, VSIArchiveReader* poReader)
{
CPLMutexHolder oHolder( &hMutex );
if (oFileList.find(archiveFilename) != oFileList.end() )
{
return oFileList[archiveFilename];
}
int bMustClose = (poReader == NULL);
if (poReader == NULL)
{
poReader = CreateReader(archiveFilename);
if (!poReader)
return NULL;
}
if (poReader->GotoFirstFile() == FALSE)
{
if (bMustClose)
delete(poReader);
return NULL;
}
VSIArchiveContent* content = new VSIArchiveContent;
content->nEntries = 0;
content->entries = NULL;
oFileList[archiveFilename] = content;
std::set<CPLString> oSet;
do
{
CPLString osFileName = poReader->GetFileName();
const char* fileName = osFileName.c_str();
/* Remove ./ pattern at the beginning of a filename */
if (fileName[0] == '.' && fileName[1] == '/')
{
fileName += 2;
if (fileName[0] == '\0')
continue;
}
char* pszStrippedFileName = CPLStrdup(fileName);
char* pszIter;
for(pszIter = pszStrippedFileName;*pszIter;pszIter++)
{
if (*pszIter == '\\')
*pszIter = '/';
}
int bIsDir = strlen(fileName) > 0 &&
fileName[strlen(fileName)-1] == '/';
if (bIsDir)
{
/* Remove trailing slash */
pszStrippedFileName[strlen(fileName)-1] = 0;
}
if (oSet.find(pszStrippedFileName) == oSet.end())
{
oSet.insert(pszStrippedFileName);
/* Add intermediate directory structure */
for(pszIter = pszStrippedFileName;*pszIter;pszIter++)
{
if (*pszIter == '/')
{
char* pszStrippedFileName2 = CPLStrdup(pszStrippedFileName);
pszStrippedFileName2[pszIter - pszStrippedFileName] = 0;
if (oSet.find(pszStrippedFileName2) == oSet.end())
{
oSet.insert(pszStrippedFileName2);
content->entries = (VSIArchiveEntry*)CPLRealloc(content->entries,
sizeof(VSIArchiveEntry) * (content->nEntries + 1));
content->entries[content->nEntries].fileName = pszStrippedFileName2;
content->entries[content->nEntries].nModifiedTime = poReader->GetModifiedTime();
content->entries[content->nEntries].uncompressed_size = 0;
content->entries[content->nEntries].bIsDir = TRUE;
content->entries[content->nEntries].file_pos = NULL;
if (ENABLE_DEBUG)
CPLDebug("VSIArchive", "[%d] %s : " CPL_FRMT_GUIB " bytes", content->nEntries+1,
content->entries[content->nEntries].fileName,
content->entries[content->nEntries].uncompressed_size);
content->nEntries++;
}
else
{
CPLFree(pszStrippedFileName2);
}
}
}
content->entries = (VSIArchiveEntry*)CPLRealloc(content->entries,
sizeof(VSIArchiveEntry) * (content->nEntries + 1));
content->entries[content->nEntries].fileName = pszStrippedFileName;
content->entries[content->nEntries].nModifiedTime = poReader->GetModifiedTime();
content->entries[content->nEntries].uncompressed_size = poReader->GetFileSize();
content->entries[content->nEntries].bIsDir = bIsDir;
content->entries[content->nEntries].file_pos = poReader->GetFileOffset();
if (ENABLE_DEBUG)
CPLDebug("VSIArchive", "[%d] %s : " CPL_FRMT_GUIB " bytes", content->nEntries+1,
content->entries[content->nEntries].fileName,
content->entries[content->nEntries].uncompressed_size);
content->nEntries++;
}
else
{
CPLFree(pszStrippedFileName);
}
} while(poReader->GotoNextFile());
if (bMustClose)
delete(poReader);
return content;
}
char* VSIArchiveFilesystemHandler::SplitFilename(const char *pszFilename,
CPLString &osFileInArchive,
int bCheckMainFileExists)
{
int i = 0;
if (strcmp(pszFilename, GetPrefix()) == 0)
return NULL;
/* Allow natural chaining of VSI drivers without requiring double slash */
CPLString osDoubleVsi(GetPrefix());
osDoubleVsi += "/vsi";
if (strncmp(pszFilename, osDoubleVsi.c_str(), osDoubleVsi.size()) == 0)
pszFilename += strlen(GetPrefix());
else
pszFilename += strlen(GetPrefix()) + 1;
while(pszFilename[i])
{
std::vector<CPLString> oExtensions = GetExtensions();
std::vector<CPLString>::const_iterator iter;
int nToSkip = 0;
for( iter = oExtensions.begin(); iter != oExtensions.end(); ++iter )
{
const CPLString& osExtension = *iter;
if (EQUALN(pszFilename + i, osExtension.c_str(), strlen(osExtension.c_str())))
{
nToSkip = strlen(osExtension.c_str());
break;
}
}
if (nToSkip != 0)
{
VSIStatBufL statBuf;
char* archiveFilename = CPLStrdup(pszFilename);
int bArchiveFileExists = FALSE;
if (archiveFilename[i + nToSkip] == '/' ||
archiveFilename[i + nToSkip] == '\\')
{
archiveFilename[i + nToSkip] = 0;
}
if (!bCheckMainFileExists)
{
bArchiveFileExists = TRUE;
}
else
{
CPLMutexHolder oHolder( &hMutex );
if (oFileList.find(archiveFilename) != oFileList.end() )
{
bArchiveFileExists = TRUE;
}
}
if (!bArchiveFileExists)
{
VSIFilesystemHandler *poFSHandler =
VSIFileManager::GetHandler( archiveFilename );
if (poFSHandler->Stat(archiveFilename, &statBuf,
VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0 &&
!VSI_ISDIR(statBuf.st_mode))
{
bArchiveFileExists = TRUE;
}
}
if (bArchiveFileExists)
{
if (pszFilename[i + nToSkip] == '/' ||
pszFilename[i + nToSkip] == '\\')
{
char* pszArchiveInFileName = CPLStrdup(pszFilename + i + nToSkip + 1);
/* Replace a/../b by b and foo/a/../b by foo/b */
while(TRUE)
{
char* pszPrevDir = strstr(pszArchiveInFileName, "/../");
if (pszPrevDir == NULL || pszPrevDir == pszArchiveInFileName)
break;
char* pszPrevSlash = pszPrevDir - 1;
while(pszPrevSlash != pszArchiveInFileName &&
*pszPrevSlash != '/')
pszPrevSlash --;
if (pszPrevSlash == pszArchiveInFileName)
memmove(pszArchiveInFileName, pszPrevDir + nToSkip, strlen(pszPrevDir + nToSkip) + 1);
else
memmove(pszPrevSlash + 1, pszPrevDir + nToSkip, strlen(pszPrevDir + nToSkip) + 1);
}
osFileInArchive = pszArchiveInFileName;
CPLFree(pszArchiveInFileName);
}
else
osFileInArchive = "";
/* Remove trailing slash */
if (osFileInArchive.size())
{
char lastC = osFileInArchive[strlen(osFileInArchive) - 1];
if (lastC == '\\' || lastC == '/')
osFileInArchive.resize(strlen(osFileInArchive) - 1);
}
return archiveFilename;
}
CPLFree(archiveFilename);
}
i++;
}
return NULL;
}
/**
* \brief Fetch a document from an url and return in a string.
*
* @param pszURL valid URL recognized by underlying download library (libcurl)
* @param papszOptions option list as a NULL-terminated array of strings. May be NULL.
* The following options are handled :
* <ul>
* <li>TIMEOUT=val, where val is in seconds</li>
* <li>HEADERS=val, where val is an extra header to use when getting a web page.
* For example "Accept: application/x-ogcwkt"
* <li>HTTPAUTH=[BASIC/NTLM/GSSNEGOTIATE/ANY] to specify an authentication scheme to use.
* <li>USERPWD=userid:password to specify a user and password for authentication
* <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the server
* with a POST request.
* <li>PROXY=val, to make requests go through a proxy server, where val is of the
* form proxy.server.com:port_number
* <li>PROXYUSERPWD=val, where val is of the form username:password
* <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL >= 1.9.0)
* </ul>
*
* Alternatively, if not defined in the papszOptions arguments, the PROXY and
* PROXYUSERPWD values are searched in the configuration options named
* GDAL_HTTP_PROXY and GDAL_HTTP_PROXYUSERPWD, as proxy configuration belongs
* to networking setup and makes more sense at the configuration option level
* than at the connection level.
*
* @return a CPLHTTPResult* structure that must be freed by
* CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
*/
CPLHTTPResult *CPLHTTPFetch( const char *pszURL, char **papszOptions )
{
#ifndef HAVE_CURL
(void) papszOptions;
(void) pszURL;
CPLError( CE_Failure, CPLE_NotSupported,
"GDAL/OGR not compiled with libcurl support, remote requests not supported." );
return NULL;
#else
/* -------------------------------------------------------------------- */
/* Are we using a persistent named session? If so, search for */
/* or create it. */
/* */
/* Currently this code does not attempt to protect against */
/* multiple threads asking for the same named session. If that */
/* occurs it will be in use in multiple threads at once which */
/* might have bad consequences depending on what guarantees */
/* libcurl gives - which I have not investigated. */
/* -------------------------------------------------------------------- */
CURL *http_handle = NULL;
const char *pszPersistent = CSLFetchNameValue( papszOptions, "PERSISTENT" );
const char *pszClosePersistent = CSLFetchNameValue( papszOptions, "CLOSE_PERSISTENT" );
if (pszPersistent)
{
CPLString osSessionName = pszPersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( oSessionMap.count( osSessionName ) == 0 )
{
oSessionMap[osSessionName] = curl_easy_init();
CPLDebug( "HTTP", "Establish persistent session named '%s'.",
osSessionName.c_str() );
}
http_handle = oSessionMap[osSessionName];
}
/* -------------------------------------------------------------------- */
/* Are we requested to close a persistent named session? */
/* -------------------------------------------------------------------- */
else if (pszClosePersistent)
{
CPLString osSessionName = pszClosePersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
std::map<CPLString,CURL*>::iterator oIter = oSessionMap.find( osSessionName );
if( oIter != oSessionMap.end() )
{
curl_easy_cleanup(oIter->second);
oSessionMap.erase(oIter);
CPLDebug( "HTTP", "Ended persistent session named '%s'.",
osSessionName.c_str() );
}
else
{
CPLDebug( "HTTP", "Could not find persistent session named '%s'.",
osSessionName.c_str() );
}
return NULL;
}
else
http_handle = curl_easy_init();
/* -------------------------------------------------------------------- */
/* Setup the request. */
/* -------------------------------------------------------------------- */
char szCurlErrBuf[CURL_ERROR_SIZE+1];
CPLHTTPResult *psResult;
struct curl_slist *headers=NULL;
const char* pszArobase = strchr(pszURL, '@');
const char* pszSlash = strchr(pszURL, '/');
const char* pszColon = (pszSlash) ? strchr(pszSlash, ':') : NULL;
if (pszArobase != NULL && pszColon != NULL && pszArobase - pszColon > 0)
{
/* http://user:[email protected] */
char* pszSanitizedURL = CPLStrdup(pszURL);
pszSanitizedURL[pszColon-pszURL] = 0;
CPLDebug( "HTTP", "Fetch(%s:#password#%s)", pszSanitizedURL, pszArobase );
CPLFree(pszSanitizedURL);
}
else
{
CPLDebug( "HTTP", "Fetch(%s)", pszURL );
}
psResult = (CPLHTTPResult *) CPLCalloc(1,sizeof(CPLHTTPResult));
curl_easy_setopt(http_handle, CURLOPT_URL, pszURL );
if (CSLTestBoolean(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
const char *pszHttpVersion = CSLFetchNameValue( papszOptions, "HTTP_VERSION");
if( pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0 )
curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0 );
/* Support control over HTTPAUTH */
const char *pszHttpAuth = CSLFetchNameValue( papszOptions, "HTTPAUTH" );
if( pszHttpAuth == NULL )
/* do nothing */;
/* CURLOPT_HTTPAUTH is defined in curl 7.11.0 or newer */
#if LIBCURL_VERSION_NUM >= 0x70B00
else if( EQUAL(pszHttpAuth,"BASIC") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC );
else if( EQUAL(pszHttpAuth,"NTLM") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_NTLM );
else if( EQUAL(pszHttpAuth,"ANY") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY );
#ifdef CURLAUTH_GSSNEGOTIATE
else if( EQUAL(pszHttpAuth,"NEGOTIATE") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_GSSNEGOTIATE );
#endif
else
{
CPLError( CE_Warning, CPLE_AppDefined,
"Unsupported HTTPAUTH value '%s', ignored.",
pszHttpAuth );
}
#else
else
{
/**
* \brief Fetch a document from an url and return in a string.
*
* @param pszURL valid URL recognized by underlying download library (libcurl)
* @param papszOptions option list as a NULL-terminated array of strings. May be NULL.
* The following options are handled :
* <ul>
* <li>TIMEOUT=val, where val is in seconds</li>
* <li>HEADERS=val, where val is an extra header to use when getting a web page.
* For example "Accept: application/x-ogcwkt"
* <li>HTTPAUTH=[BASIC/NTLM/ANY] to specify an authentication scheme to use.
* <li>USERPWD=userid:password to specify a user and password for authentication
* <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the server
* with a POST request.
* <li>PROXY=val, to make requests go through a proxy server, where val is of the
* form proxy.server.com:port_number
* <li>PROXYUSERPWD=val, where val is of the form username:password
* </ul>
*
* Alternatively, if not defined in the papszOptions arguments, the PROXY and PROXYUSERPWD
* values are searched in the configuration options named GDAL_HTTP_PROXY and GDAL_HTTP_PROXYUSERPWD,
* as proxy configuration belongs to networking setup and makes more sense at the configuration
* option level than at the connection level.
*
* @return a CPLHTTPResult* structure that must be freed by CPLHTTPDestroyResult(),
* or NULL if libcurl support is disabled
*/
CPLHTTPResult *CPLHTTPFetch( const char *pszURL, char **papszOptions )
{
#ifndef HAVE_CURL
(void) papszOptions;
(void) pszURL;
CPLError( CE_Failure, CPLE_NotSupported,
"GDAL/OGR not compiled with libcurl support, remote requests not supported." );
return NULL;
#else
/* -------------------------------------------------------------------- */
/* Are we using a persistent named session? If so, search for */
/* or create it. */
/* */
/* Currently this code does not attempt to protect against */
/* multiple threads asking for the same named session. If that */
/* occurs it will be in use in multiple threads at once which */
/* might have bad consequences depending on what guarantees */
/* libcurl gives - which I have not investigated. */
/* -------------------------------------------------------------------- */
CURL *http_handle = NULL;
const char *pszPersistent = CSLFetchNameValue( papszOptions, "PERSISTENT" );
if (pszPersistent)
{
CPLString osSessionName = pszPersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( oSessionMap.count( osSessionName ) == 0 )
{
oSessionMap[osSessionName] = curl_easy_init();
CPLDebug( "HTTP", "Establish persistent session named '%s'.",
osSessionName.c_str() );
}
http_handle = oSessionMap[osSessionName];
}
else
http_handle = curl_easy_init();
/* -------------------------------------------------------------------- */
/* Setup the request. */
/* -------------------------------------------------------------------- */
char szCurlErrBuf[CURL_ERROR_SIZE+1];
CPLHTTPResult *psResult;
struct curl_slist *headers=NULL;
CPLDebug( "HTTP", "Fetch(%s)", pszURL );
psResult = (CPLHTTPResult *) CPLCalloc(1,sizeof(CPLHTTPResult));
curl_easy_setopt(http_handle, CURLOPT_URL, pszURL );
const char *pszHttpVersion = CSLFetchNameValue( papszOptions, "HTTP_VERSION");
if( pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0 )
curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0 );
/* Support control over HTTPAUTH */
const char *pszHttpAuth = CSLFetchNameValue( papszOptions, "HTTPAUTH" );
if( pszHttpAuth == NULL )
/* do nothing */;
/* CURLOPT_HTTPAUTH is defined in curl 7.11.0 or newer */
#if LIBCURL_VERSION_NUM >= 0x70B00
else if( EQUAL(pszHttpAuth,"BASIC") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC );
else if( EQUAL(pszHttpAuth,"NTLM") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_NTLM );
else if( EQUAL(pszHttpAuth,"ANY") )
curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY );
else
{
CPLError( CE_Warning, CPLE_AppDefined,
"Unsupported HTTPAUTH value '%s', ignored.",
pszHttpAuth );
}
#else
else
{
void VSIUnixStdioFilesystemHandler::AddToTotal(vsi_l_offset nBytes)
{
CPLMutexHolder oHolder(&hMutex);
nTotalBytesRead += nBytes;
}
