void VSICurlStreamingHandle::StopDownload()
{
if (hThread)
{
//if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Stop download for %s", pszURL);
AcquireMutex();
/* Signal to the producer that we ask for download interruption */
bAskDownloadEnd = TRUE;
CPLCondSignal(hCondConsumer);
/* Wait for the producer to have finished */
while(bDownloadInProgress)
CPLCondWait(hCondProducer, hRingBufferMutex);
bAskDownloadEnd = FALSE;
ReleaseMutex();
CPLJoinThread(hThread);
hThread = NULL;
curl_easy_cleanup(hCurlHandle);
hCurlHandle = NULL;
}
oRingBuffer.Reset();
bDownloadStopped = FALSE;
}
/** Destroys a pool of worker threads.
*
* Any still pending job will be completed before the destructor returns.
*/
CPLWorkerThreadPool::~CPLWorkerThreadPool()
{
if( hCond )
{
WaitCompletion();
CPLAcquireMutex(hMutex, 1000.0);
eState = CPLWTS_STOP;
CPLReleaseMutex(hMutex);
for(size_t i=0;i<aWT.size();i++)
{
CPLAcquireMutex(aWT[i].hMutex, 1000.0);
CPLCondSignal(aWT[i].hCond);
CPLReleaseMutex(aWT[i].hMutex);
CPLJoinThread(aWT[i].hThread);
CPLDestroyCond(aWT[i].hCond);
CPLDestroyMutex(aWT[i].hMutex);
}
CPLListDestroy(psWaitingWorkerThreadsList);
CPLDestroyCond(hCond);
}
CPLDestroyMutex(hMutex);
}
void CPLWorkerThreadPool::DeclareJobFinished()
{
CPLAcquireMutex(hMutex, 1000.0);
nPendingJobs--;
CPLCondSignal(hCond);
CPLReleaseMutex(hMutex);
}
void VSICurlStreamingHandle::PutRingBufferInCache()
{
if (nRingBufferFileOffset >= BKGND_BUFFER_SIZE)
return;
AcquireMutex();
/* Cache any remaining bytes available in the ring buffer */
size_t nBufSize = oRingBuffer.GetSize();
if ( nBufSize > 0 )
{
if (nRingBufferFileOffset + nBufSize > BKGND_BUFFER_SIZE)
nBufSize = (size_t) (BKGND_BUFFER_SIZE - nRingBufferFileOffset);
GByte* pabyTmp = (GByte*) CPLMalloc(nBufSize);
oRingBuffer.Read(pabyTmp, nBufSize);
/* Signal to the producer that we have ingested some bytes */
CPLCondSignal(hCondConsumer);
AddRegion(nRingBufferFileOffset, nBufSize, pabyTmp);
nRingBufferFileOffset += nBufSize;
CPLFree(pabyTmp);
}
ReleaseMutex();
}
/** Queue a new job.
*
* @param pfnFunc Function to run for the job.
* @param pData User data to pass to the job function.
* @return true in case of success.
*/
bool CPLWorkerThreadPool::SubmitJob( CPLThreadFunc pfnFunc, void* pData )
{
CPLAssert( !aWT.empty() );
CPLWorkerThreadJob* psJob = static_cast<CPLWorkerThreadJob *>(
VSI_MALLOC_VERBOSE(sizeof(CPLWorkerThreadJob)));
if( psJob == nullptr )
return false;
psJob->pfnFunc = pfnFunc;
psJob->pData = pData;
CPLList* psItem =
static_cast<CPLList *>(VSI_MALLOC_VERBOSE(sizeof(CPLList)));
if( psItem == nullptr )
{
VSIFree(psJob);
return false;
}
psItem->pData = psJob;
CPLAcquireMutex(hMutex, 1000.0);
psItem->psNext = psJobQueue;
psJobQueue = psItem;
nPendingJobs++;
if( psWaitingWorkerThreadsList )
{
CPLWorkerThread* psWorkerThread =
static_cast<CPLWorkerThread *>(psWaitingWorkerThreadsList->pData);
CPLAssert( psWorkerThread->bMarkedAsWaiting );
psWorkerThread->bMarkedAsWaiting = FALSE;
CPLList* psNext = psWaitingWorkerThreadsList->psNext;
CPLList* psToFree = psWaitingWorkerThreadsList;
psWaitingWorkerThreadsList = psNext;
nWaitingWorkerThreads--;
// CPLAssert(
// CPLListCount(psWaitingWorkerThreadsList) == nWaitingWorkerThreads);
#if DEBUG_VERBOSE
CPLDebug("JOB", "Waking up %p", psWorkerThread);
#endif
CPLAcquireMutex(psWorkerThread->hMutex, 1000.0);
CPLReleaseMutex(hMutex);
CPLCondSignal(psWorkerThread->hCond);
CPLReleaseMutex(psWorkerThread->hMutex);
CPLFree(psToFree);
}
else
{
CPLReleaseMutex(hMutex);
}
return true;
}
void VSICurlStreamingHandle::DownloadInThread()
{
VSICurlSetOptions(hCurlHandle, pszURL);
static int bHasCheckVersion = FALSE;
static int bSupportGZip = FALSE;
if (!bHasCheckVersion)
{
bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
bHasCheckVersion = TRUE;
}
if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
{
curl_easy_setopt(hCurlHandle, CURLOPT_ENCODING, "gzip");
}
if (pabyHeaderData == NULL)
pabyHeaderData = (GByte*) CPLMalloc(HEADER_SIZE + 1);
nHeaderSize = 0;
nBodySize = 0;
nHTTPCode = 0;
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleReceivedBytesHeader);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleReceivedBytes);
char szCurlErrBuf[CURL_ERROR_SIZE+1];
szCurlErrBuf[0] = '\0';
curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
CURLcode eRet = curl_easy_perform(hCurlHandle);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
AcquireMutex();
if (!bAskDownloadEnd && eRet == 0 && !bHastComputedFileSize)
{
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
cachedFileProp->fileSize = fileSize = nBodySize;
cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
poFS->ReleaseMutex();
}
bDownloadInProgress = FALSE;
bDownloadStopped = TRUE;
/* Signal to the consumer that the download has ended */
CPLCondSignal(hCondProducer);
ReleaseMutex();
}
void GDALAbstractBandBlockCache::AddBlockToFreeList( GDALRasterBlock *poBlock )
{
CPLAssert(poBlock->poPrevious == NULL);
CPLAssert(poBlock->poNext == NULL);
{
#ifdef DEBUG_VERBOSE_ABBC
CPLAtomicInc(&nAllBandsKeptAlivedBlocks);
fprintf(stderr, "AddBlockToFreeList(): nAllBandsKeptAlivedBlocks=%d\n", nAllBandsKeptAlivedBlocks);
#endif
CPLLockHolderOptionalLockD(hSpinLock);
poBlock->poNext = psListBlocksToFree;
psListBlocksToFree = poBlock;
}
// If no more blocks in transient state, then warn WaitKeepAliveCounter()
CPLAcquireMutex(hCondMutex, 1000);
if( CPLAtomicDec(&nKeepAliveCounter) == 0 )
{
CPLCondSignal(hCond);
}
CPLReleaseMutex(hCondMutex);
}
CPLWorkerThreadJob *
CPLWorkerThreadPool::GetNextJob( CPLWorkerThread* psWorkerThread )
{
while(true)
{
CPLAcquireMutex(hMutex, 1000.0);
if( eState == CPLWTS_STOP )
{
CPLReleaseMutex(hMutex);
return nullptr;
}
CPLList* psTopJobIter = psJobQueue;
if( psTopJobIter )
{
psJobQueue = psTopJobIter->psNext;
#if DEBUG_VERBOSE
CPLDebug("JOB", "%p got a job", psWorkerThread);
#endif
CPLWorkerThreadJob* psJob =
static_cast<CPLWorkerThreadJob*>(psTopJobIter->pData);
CPLReleaseMutex(hMutex);
CPLFree(psTopJobIter);
return psJob;
}
if( !psWorkerThread->bMarkedAsWaiting )
{
psWorkerThread->bMarkedAsWaiting = TRUE;
nWaitingWorkerThreads++;
CPLAssert(nWaitingWorkerThreads <= static_cast<int>(aWT.size()));
CPLList* psItem =
static_cast<CPLList *>(VSI_MALLOC_VERBOSE(sizeof(CPLList)));
if( psItem == nullptr )
{
eState = CPLWTS_ERROR;
CPLCondSignal(hCond);
CPLReleaseMutex(hMutex);
return nullptr;
}
psItem->pData = psWorkerThread;
psItem->psNext = psWaitingWorkerThreadsList;
psWaitingWorkerThreadsList = psItem;
#if DEBUG_VERBOSE
CPLAssert(CPLListCount(psWaitingWorkerThreadsList) ==
nWaitingWorkerThreads);
#endif
}
CPLCondSignal(hCond);
CPLAcquireMutex(psWorkerThread->hMutex, 1000.0);
#if DEBUG_VERBOSE
CPLDebug("JOB", "%p sleeping", psWorkerThread);
#endif
CPLReleaseMutex(hMutex);
CPLCondWait( psWorkerThread->hCond, psWorkerThread->hMutex );
// TODO(rouault): Explain or delete.
// CPLWorkerThreadJob* psJob = psWorkerThread->psNextJob;
// psWorkerThread->psNextJob = nullptr;
CPLReleaseMutex(psWorkerThread->hMutex);
// TODO(rouault): Explain or delete.
// if( psJob )
// return psJob;
}
}
/** Queue several jobs
*
* @param pfnFunc Function to run for the job.
* @param apData User data instances to pass to the job function.
* @return true in case of success.
*/
bool CPLWorkerThreadPool::SubmitJobs(CPLThreadFunc pfnFunc,
const std::vector<void*>& apData)
{
CPLAssert( !aWT.empty() );
CPLAcquireMutex(hMutex, 1000.0);
CPLList* psJobQueueInit = psJobQueue;
bool bRet = true;
for(size_t i=0;i<apData.size();i++)
{
CPLWorkerThreadJob* psJob = static_cast<CPLWorkerThreadJob*>(
VSI_MALLOC_VERBOSE(sizeof(CPLWorkerThreadJob)));
if( psJob == nullptr )
{
bRet = false;
break;
}
psJob->pfnFunc = pfnFunc;
psJob->pData = apData[i];
CPLList* psItem =
static_cast<CPLList *>(VSI_MALLOC_VERBOSE(sizeof(CPLList)));
if( psItem == nullptr )
{
VSIFree(psJob);
bRet = false;
break;
}
psItem->pData = psJob;
psItem->psNext = psJobQueue;
psJobQueue = psItem;
nPendingJobs++;
}
if( !bRet )
{
for( CPLList* psIter = psJobQueue; psIter != psJobQueueInit; )
{
CPLList* psNext = psIter->psNext;
VSIFree(psIter->pData);
VSIFree(psIter);
nPendingJobs--;
psIter = psNext;
}
}
CPLReleaseMutex(hMutex);
if( !bRet )
return false;
for(size_t i=0;i<apData.size();i++)
{
CPLAcquireMutex(hMutex, 1000.0);
if( psWaitingWorkerThreadsList && psJobQueue )
{
CPLWorkerThread* psWorkerThread;
psWorkerThread = static_cast<CPLWorkerThread*>(psWaitingWorkerThreadsList->pData);
CPLAssert( psWorkerThread->bMarkedAsWaiting );
psWorkerThread->bMarkedAsWaiting = FALSE;
CPLList* psNext = psWaitingWorkerThreadsList->psNext;
CPLList* psToFree = psWaitingWorkerThreadsList;
psWaitingWorkerThreadsList = psNext;
nWaitingWorkerThreads--;
// CPLAssert(
// CPLListCount(psWaitingWorkerThreadsList) ==
// nWaitingWorkerThreads);
#if DEBUG_VERBOSE
CPLDebug("JOB", "Waking up %p", psWorkerThread);
#endif
CPLAcquireMutex(psWorkerThread->hMutex, 1000.0);
// CPLAssert(psWorkerThread->psNextJob == nullptr);
// psWorkerThread->psNextJob =
// (CPLWorkerThreadJob*)psJobQueue->pData;
// psNext = psJobQueue->psNext;
// CPLFree(psJobQueue);
// psJobQueue = psNext;
CPLReleaseMutex(hMutex);
CPLCondSignal(psWorkerThread->hCond);
CPLReleaseMutex(psWorkerThread->hMutex);
CPLFree(psToFree);
}
else
{
CPLReleaseMutex(hMutex);
break;
}
}
return true;
}
int VSICurlStreamingHandle::ReceivedBytes(GByte *buffer, size_t count, size_t nmemb)
{
size_t nSize = count * nmemb;
nBodySize += nSize;
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Receiving %d bytes...", (int)nSize);
if( bHasCandidateFileSize && bCanTrustCandidateFileSize && !bHastComputedFileSize )
{
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
cachedFileProp->fileSize = fileSize = nCandidateFileSize;
cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
poFS->ReleaseMutex();
}
AcquireMutex();
if (eExists == EXIST_UNKNOWN)
{
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
cachedFileProp->eExists = eExists = EXIST_YES;
poFS->ReleaseMutex();
}
else if (eExists == EXIST_NO)
{
ReleaseMutex();
return 0;
}
while(TRUE)
{
size_t nFree = oRingBuffer.GetCapacity() - oRingBuffer.GetSize();
if (nSize <= nFree)
{
oRingBuffer.Write(buffer, nSize);
/* Signal to the consumer that we have added bytes to the buffer */
CPLCondSignal(hCondProducer);
if (bAskDownloadEnd)
{
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Download interruption asked");
ReleaseMutex();
return 0;
}
break;
}
else
{
oRingBuffer.Write(buffer, nFree);
buffer += nFree;
nSize -= nFree;
/* Signal to the consumer that we have added bytes to the buffer */
CPLCondSignal(hCondProducer);
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Waiting for reader to consume some bytes...");
while(oRingBuffer.GetSize() == oRingBuffer.GetCapacity() && !bAskDownloadEnd)
{
CPLCondWait(hCondConsumer, hRingBufferMutex);
}
if (bAskDownloadEnd)
{
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Download interruption asked");
ReleaseMutex();
return 0;
}
}
}
ReleaseMutex();
return nmemb;
}
size_t VSICurlStreamingHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
{
GByte* pabyBuffer = (GByte*)pBuffer;
size_t nBufferRequestSize = nSize * nMemb;
if (nBufferRequestSize == 0)
return 0;
size_t nRemaining = nBufferRequestSize;
AcquireMutex();
int bHastComputedFileSizeLocal = bHastComputedFileSize;
vsi_l_offset fileSizeLocal = fileSize;
ReleaseMutex();
if (bHastComputedFileSizeLocal && curOffset >= fileSizeLocal)
{
CPLDebug("VSICURL", "Read attempt beyond end of file");
bEOF = TRUE;
}
if (bEOF)
return 0;
if (curOffset < nRingBufferFileOffset)
PutRingBufferInCache();
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Read [" CPL_FRMT_GUIB ", " CPL_FRMT_GUIB "[ in %s",
curOffset, curOffset + nBufferRequestSize, pszURL);
#ifdef notdef
if( pCachedData != NULL && nCachedSize >= 1024 &&
nRecomputedChecksumOfFirst1024Bytes == 0 )
{
for(size_t i = 0; i < 1024 / sizeof(int); i ++)
{
int nVal;
memcpy(&nVal, pCachedData + i * sizeof(int), sizeof(int));
nRecomputedChecksumOfFirst1024Bytes += nVal;
}
if( bHastComputedFileSizeLocal )
{
poFS->AcquireMutex();
CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
if( cachedFileProp->nChecksumOfFirst1024Bytes == 0 )
{
cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
}
else if( nRecomputedChecksumOfFirst1024Bytes != cachedFileProp->nChecksumOfFirst1024Bytes )
{
CPLDebug("VSICURL", "Invalidating previously cached file size. First bytes of file have changed!");
AcquireMutex();
bHastComputedFileSize = FALSE;
cachedFileProp->bHastComputedFileSize = FALSE;
cachedFileProp->nChecksumOfFirst1024Bytes = 0;
ReleaseMutex();
}
poFS->ReleaseMutex();
}
}
#endif
/* Can we use the cache ? */
if( pCachedData != NULL && curOffset < nCachedSize )
{
size_t nSz = MIN(nRemaining, (size_t)(nCachedSize - curOffset));
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
(int)curOffset, (int)(curOffset + nSz), pszURL);
memcpy(pabyBuffer, pCachedData + curOffset, nSz);
pabyBuffer += nSz;
curOffset += nSz;
nRemaining -= nSz;
}
/* Is the request partially covered by the cache and going beyond file size ? */
if ( pCachedData != NULL && bHastComputedFileSizeLocal &&
curOffset <= nCachedSize &&
curOffset + nRemaining > fileSizeLocal &&
fileSize == nCachedSize )
{
size_t nSz = (size_t) (nCachedSize - curOffset);
if (ENABLE_DEBUG && nSz != 0)
CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
(int)curOffset, (int)(curOffset + nSz), pszURL);
memcpy(pabyBuffer, pCachedData + curOffset, nSz);
pabyBuffer += nSz;
curOffset += nSz;
nRemaining -= nSz;
bEOF = TRUE;
}
/* Has a Seek() being done since the last Read() ? */
if (!bEOF && nRemaining > 0 && curOffset != nRingBufferFileOffset)
{
/* Backward seek : we need to restart the download from the start */
if (curOffset < nRingBufferFileOffset)
StopDownload();
StartDownload();
#define SKIP_BUFFER_SIZE 32768
GByte* pabyTmp = (GByte*)CPLMalloc(SKIP_BUFFER_SIZE);
CPLAssert(curOffset >= nRingBufferFileOffset);
vsi_l_offset nBytesToSkip = curOffset - nRingBufferFileOffset;
while(nBytesToSkip > 0)
{
vsi_l_offset nBytesToRead = nBytesToSkip;
AcquireMutex();
if (nBytesToRead > oRingBuffer.GetSize())
nBytesToRead = oRingBuffer.GetSize();
if (nBytesToRead > SKIP_BUFFER_SIZE)
nBytesToRead = SKIP_BUFFER_SIZE;
oRingBuffer.Read(pabyTmp, (size_t)nBytesToRead);
/* Signal to the producer that we have ingested some bytes */
CPLCondSignal(hCondConsumer);
ReleaseMutex();
if (nBytesToRead)
AddRegion(nRingBufferFileOffset, (size_t)nBytesToRead, pabyTmp);
nBytesToSkip -= nBytesToRead;
nRingBufferFileOffset += nBytesToRead;
if (nBytesToRead == 0 && nBytesToSkip != 0)
{
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
AcquireMutex();
while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
CPLCondWait(hCondProducer, hRingBufferMutex);
int bBufferEmpty = (oRingBuffer.GetSize() == 0);
ReleaseMutex();
if (bBufferEmpty && !bDownloadInProgress)
break;
}
}
CPLFree(pabyTmp);
if (nBytesToSkip != 0)
{
bEOF = TRUE;
return 0;
}
}
if (!bEOF && nRemaining > 0)
{
StartDownload();
CPLAssert(curOffset == nRingBufferFileOffset);
}
/* Fill the destination buffer from the ring buffer */
while(!bEOF && nRemaining > 0)
{
AcquireMutex();
size_t nToRead = oRingBuffer.GetSize();
if (nToRead > nRemaining)
nToRead = nRemaining;
oRingBuffer.Read(pabyBuffer, nToRead);
/* Signal to the producer that we have ingested some bytes */
CPLCondSignal(hCondConsumer);
ReleaseMutex();
if (nToRead)
AddRegion(curOffset, nToRead, pabyBuffer);
nRemaining -= nToRead;
pabyBuffer += nToRead;
curOffset += nToRead;
nRingBufferFileOffset += nToRead;
if (nToRead == 0 && nRemaining != 0)
{
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
AcquireMutex();
while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
CPLCondWait(hCondProducer, hRingBufferMutex);
int bBufferEmpty = (oRingBuffer.GetSize() == 0);
ReleaseMutex();
if (bBufferEmpty && !bDownloadInProgress)
break;
}
}
if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Read(%d) = %d",
(int)nBufferRequestSize, (int)(nBufferRequestSize - nRemaining));
size_t nRet = (nBufferRequestSize - nRemaining) / nSize;
if (nRet < nMemb)
bEOF = TRUE;
return nRet;
}