void VSICurlStreamingHandle::DownloadInThread()
{
VSICurlSetOptions(hCurlHandle, pszURL);
static int bHasCheckVersion = FALSE;
static int bSupportGZip = FALSE;
if (!bHasCheckVersion)
{
bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
bHasCheckVersion = TRUE;
}
if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
{
curl_easy_setopt(hCurlHandle, CURLOPT_ENCODING, "gzip");
}
if (pabyHeaderData == NULL)
pabyHeaderData = (GByte*) CPLMalloc(HEADER_SIZE + 1);
nHeaderSize = 0;
nBodySize = 0;
nHTTPCode = 0;
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleReceivedBytesHeader);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleReceivedBytes);
char szCurlErrBuf[CURL_ERROR_SIZE+1];
szCurlErrBuf[0] = '\0';
curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
CURLcode eRet = curl_easy_perform(hCurlHandle);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
AcquireMutex();
if (!bAskDownloadEnd && eRet == 0 && !bHastComputedFileSize)
{
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
cachedFileProp->fileSize = fileSize = nBodySize;
cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
poFS->ReleaseMutex();
}
bDownloadInProgress = FALSE;
bDownloadStopped = TRUE;
/* Signal to the consumer that the download has ended */
CPLCondSignal(hCondProducer);
ReleaseMutex();
}
bool VSIDIRAz::IssueListDir()
{
WriteFuncStruct sWriteFuncData;
const CPLString l_osNextMarker(osNextMarker);
clear();
CPLString osMaxKeys = CPLGetConfigOption("AZURE_MAX_RESULTS", "");
const int AZURE_SERVER_LIMIT_SINGLE_REQUEST = 5000;
if( nMaxFiles > 0 && nMaxFiles < AZURE_SERVER_LIMIT_SINGLE_REQUEST &&
(osMaxKeys.empty() || nMaxFiles < atoi(osMaxKeys)) )
{
osMaxKeys.Printf("%d", nMaxFiles);
}
poHandleHelper->ResetQueryParameters();
CPLString osBaseURL(poHandleHelper->GetURL());
CURLM* hCurlMultiHandle = poFS->GetCurlMultiHandleFor(osBaseURL);
CURL* hCurlHandle = curl_easy_init();
poHandleHelper->AddQueryParameter("comp", "list");
if( !l_osNextMarker.empty() )
poHandleHelper->AddQueryParameter("marker", l_osNextMarker);
if( !osMaxKeys.empty() )
poHandleHelper->AddQueryParameter("maxresults", osMaxKeys);
if( !osBucket.empty() )
{
poHandleHelper->AddQueryParameter("restype", "container");
if( nRecurseDepth == 0 )
poHandleHelper->AddQueryParameter("delimiter", "/");
if( !osObjectKey.empty() )
poHandleHelper->AddQueryParameter("prefix", osObjectKey + "/");
}
struct curl_slist* headers =
VSICurlSetOptions(hCurlHandle, poHandleHelper->GetURL(), nullptr);
curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr);
VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION,
VSICurlHandleWriteFunc);
char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
headers = VSICurlMergeHeaders(headers,
poHandleHelper->GetCurlHeaders("GET", headers));
curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
MultiPerform(hCurlMultiHandle, hCurlHandle);
if( headers != nullptr )
curl_slist_free_all(headers);
if( sWriteFuncData.pBuffer == nullptr)
{
curl_easy_cleanup(hCurlHandle);
return false;
}
long response_code = 0;
curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
if( response_code != 200 )
{
CPLDebug("AZURE", "%s",
sWriteFuncData.pBuffer
? sWriteFuncData.pBuffer : "(null)");
CPLFree(sWriteFuncData.pBuffer);
curl_easy_cleanup(hCurlHandle);
return false;
}
else
{
bool ret = AnalyseAzureFileList( osBaseURL,
sWriteFuncData.pBuffer );
CPLFree(sWriteFuncData.pBuffer);
curl_easy_cleanup(hCurlHandle);
return ret;
}
}
vsi_l_offset VSICurlStreamingHandle::GetFileSize()
{
WriteFuncStruct sWriteFuncData;
WriteFuncStruct sWriteFuncHeaderData;
AcquireMutex();
if (bHastComputedFileSize)
{
vsi_l_offset nRet = fileSize;
ReleaseMutex();
return nRet;
}
ReleaseMutex();
#if LIBCURL_VERSION_NUM < 0x070B00
/* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
/* previously set, so we have to reinit the connection handle */
if (hCurlHandle)
{
curl_easy_cleanup(hCurlHandle);
hCurlHandle = curl_easy_init();
}
#endif
CURL* hLocalHandle = curl_easy_init();
VSICurlSetOptions(hLocalHandle, pszURL);
VSICURLStreamingInitWriteFuncStruct(&sWriteFuncHeaderData);
/* HACK for mbtiles driver: proper fix would be to auto-detect servers that don't accept HEAD */
/* http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so let's start a GET */
/* and interrupt is as soon as the header is found */
if (strstr(pszURL, ".tiles.mapbox.com/") != NULL)
{
curl_easy_setopt(hLocalHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
curl_easy_setopt(hLocalHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
}
else
{
curl_easy_setopt(hLocalHandle, CURLOPT_NOBODY, 1);
curl_easy_setopt(hLocalHandle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(hLocalHandle, CURLOPT_HEADER, 1);
}
/* We need that otherwise OSGEO4W's libcurl issue a dummy range request */
/* when doing a HEAD when recycling connections */
curl_easy_setopt(hLocalHandle, CURLOPT_RANGE, NULL);
/* Bug with older curl versions (<=7.16.4) and FTP. See http://curl.haxx.se/mail/lib-2007-08/0312.html */
VSICURLStreamingInitWriteFuncStruct(&sWriteFuncData);
curl_easy_setopt(hLocalHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
curl_easy_setopt(hLocalHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
char szCurlErrBuf[CURL_ERROR_SIZE+1];
szCurlErrBuf[0] = '\0';
curl_easy_setopt(hLocalHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
double dfSize = 0;
curl_easy_perform(hLocalHandle);
AcquireMutex();
eExists = EXIST_UNKNOWN;
bHastComputedFileSize = TRUE;
if (strncmp(pszURL, "ftp", 3) == 0)
{
if (sWriteFuncData.pBuffer != NULL &&
strncmp(sWriteFuncData.pBuffer, "Content-Length: ", strlen( "Content-Length: ")) == 0)
{
const char* pszBuffer = sWriteFuncData.pBuffer + strlen("Content-Length: ");
eExists = EXIST_YES;
fileSize = CPLScanUIntBig(pszBuffer, sWriteFuncData.nSize - strlen("Content-Length: "));
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
pszURL, fileSize);
}
}
if (eExists != EXIST_YES)
{
CURLcode code = curl_easy_getinfo(hLocalHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dfSize );
if (code == 0)
{
eExists = EXIST_YES;
if (dfSize < 0)
fileSize = 0;
else
fileSize = (GUIntBig)dfSize;
}
else
{
eExists = EXIST_NO;
fileSize = 0;
CPLError(CE_Failure, CPLE_AppDefined,
"VSICurlStreamingHandle::GetFileSize failed");
}
long response_code = 0;
curl_easy_getinfo(hLocalHandle, CURLINFO_HTTP_CODE, &response_code);
if (response_code != 200)
{
eExists = EXIST_NO;
fileSize = 0;
}
/* Try to guess if this is a directory. Generally if this is a directory, */
/* curl will retry with an URL with slash added */
char *pszEffectiveURL = NULL;
curl_easy_getinfo(hLocalHandle, CURLINFO_EFFECTIVE_URL, &pszEffectiveURL);
if (pszEffectiveURL != NULL &&
strncmp(pszURL, pszEffectiveURL, strlen(pszURL)) == 0 &&
pszEffectiveURL[strlen(pszURL)] == '/')
{
eExists = EXIST_YES;
fileSize = 0;
bIsDirectory = TRUE;
}
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
pszURL, fileSize, (int)response_code);
}
CPLFree(sWriteFuncData.pBuffer);
CPLFree(sWriteFuncHeaderData.pBuffer);
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
cachedFileProp->bHastComputedFileSize = TRUE;
#ifdef notdef
cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
#endif
cachedFileProp->fileSize = fileSize;
cachedFileProp->eExists = eExists;
cachedFileProp->bIsDirectory = bIsDirectory;
poFS->ReleaseMutex();
vsi_l_offset nRet = fileSize;
ReleaseMutex();
if (hCurlHandle == NULL)
hCurlHandle = hLocalHandle;
else
curl_easy_cleanup(hLocalHandle);
return nRet;
}
