int main(int argc, char* argv[])
{
CPLJoinableThread* hThread;
printf("main thread %p\n", (void*)CPLGetPID());
argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
CPLSetConfigOption("GDAL_CACHEMAX", "0");
CPLSetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "ON");
MyDataset* poDS = new MyDataset();
char buf1[] = { 1 } ;
CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 0, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
hThread = CPLCreateJoinableThread(thread_func, NULL);
CPLSleep(0.3);
CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 1, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
GDALFlushCacheBlock();
CPLJoinThread(hThread);
delete poDS;
GDALDestroyDriverManager();
CSLDestroy( argv );
return 0;
}
void VSICurlStreamingHandle::StartDownload()
{
if (bDownloadInProgress || bDownloadStopped)
return;
//if (ENABLE_DEBUG)
CPLDebug("VSICURL", "Start download for %s", pszURL);
if (hCurlHandle == NULL)
hCurlHandle = curl_easy_init();
oRingBuffer.Reset();
bDownloadInProgress = TRUE;
nRingBufferFileOffset = 0;
hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
}
static int test_two_pages()
{
CPLVirtualMem* ctxt;
volatile char* addr;
void* hThread;
ctxt = CPLVirtualMemNew(3*MINIMUM_PAGE_SIZE,
MINIMUM_PAGE_SIZE,
MINIMUM_PAGE_SIZE,
FALSE,
VIRTUALMEM_READONLY,
test_two_pages_cbk,
NULL,
NULL, NULL);
if( ctxt == NULL )
return FALSE;
addr = (char*) CPLVirtualMemGetAddr(ctxt);
assert(CPLVirtualMemGetPageSize(ctxt) == MINIMUM_PAGE_SIZE);
assert(CPLVirtualMemIsAccessThreadSafe(ctxt));
/*fprintfstderr("main thread is %X, addr=%p\n", pthread_self(), addr);*/
hThread = CPLCreateJoinableThread(test_two_pages_thread, ctxt);
CPLVirtualMemDeclareThread(ctxt);
{
int i=0;
for(i=0; i<50*1000; i++)
{
char val = addr[MINIMUM_PAGE_SIZE * (i % 3)];
/*fprintfstderr("T1: val[%d] = %d\n", MINIMUM_PAGE_SIZE * (i % 2), val);*/
assert(val == (((i % 3) == 0) ? 0x3F : ((i % 3) == 1) ? 0x5F : 0x7F));
}
}
CPLVirtualMemUnDeclareThread(ctxt);
CPLJoinThread(hThread);
CPLVirtualMemFree(ctxt);
return TRUE;
}
int main(int argc, char* argv[])
{
int i;
int nThreads = CPLGetNumCPUs();
std::vector<CPLJoinableThread*> apsThreads;
Strategy eStrategy = STRATEGY_RANDOM;
int bNewDatasetOption = FALSE;
int nXSize = 5000;
int nYSize = 5000;
int nBands = 4;
char** papszOptions = NULL;
int bOnDisk = FALSE;
std::vector<ThreadDescription> asThreadDescription;
int bMemDriver = FALSE;
GDALDataset* poMEMDS = NULL;
int bMigrate = FALSE;
int nMaxRequests = -1;
argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
GDALAllRegister();
for(i = 1; i < argc; i++)
{
if( EQUAL(argv[i], "-threads") && i + 1 < argc)
{
i ++;
nThreads = atoi(argv[i]);
}
else if( EQUAL(argv[i], "-loops") && i + 1 < argc)
{
i ++;
nLoops = atoi(argv[i]);
if( nLoops <= 0 )
nLoops = INT_MAX;
}
else if( EQUAL(argv[i], "-max_requests") && i + 1 < argc)
{
i ++;
nMaxRequests = atoi(argv[i]);
}
else if( EQUAL(argv[i], "-strategy") && i + 1 < argc)
{
i ++;
if( EQUAL(argv[i], "random") )
eStrategy = STRATEGY_RANDOM;
else if( EQUAL(argv[i], "line") )
eStrategy = STRATEGY_LINE;
else if( EQUAL(argv[i], "block") )
eStrategy = STRATEGY_BLOCK;
else
Usage();
}
else if( EQUAL(argv[i], "-xsize") && i + 1 < argc)
{
i ++;
nXSize = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-ysize") && i + 1 < argc)
{
i ++;
nYSize = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-bands") && i + 1 < argc)
{
i ++;
nBands = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-co") && i + 1 < argc)
{
i ++;
papszOptions = CSLAddString(papszOptions, argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-ondisk"))
{
bOnDisk = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-check"))
{
bCheck = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-memdriver"))
{
bMemDriver = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-migrate"))
bMigrate = TRUE;
else if( argv[i][0] == '-' )
Usage();
else if( pszDataset == NULL )
pszDataset = argv[i];
else
{
Usage();
}
}
if( pszDataset != NULL && bNewDatasetOption )
Usage();
CPLDebug("TEST", "Using %d threads", nThreads);
int bCreatedDataset = FALSE;
if( pszDataset == NULL )
{
bCreatedDataset = TRUE;
if( bOnDisk )
pszDataset = "/tmp/tmp.tif";
else
pszDataset = "/vsimem/tmp.tif";
GDALDataset* poDS = ((GDALDriver*)GDALGetDriverByName((bMemDriver) ? "MEM" : "GTiff"))->Create(pszDataset,
nXSize, nYSize, nBands, GDT_Byte, papszOptions);
if( bCheck )
{
GByte* pabyLine = (GByte*) VSIMalloc(nBands * nXSize);
for(int iY=0;iY<nYSize;iY++)
{
for(int iX=0;iX<nXSize;iX++)
{
for(int iBand=0;iBand<nBands;iBand++)
{
unsigned long seed = iBand * nXSize * nYSize + iY * nXSize + iX;
pabyLine[iBand * nXSize + iX] = (GByte)(myrand_r(&seed) & 0xff);
}
}
CPL_IGNORE_RET_VAL(poDS->RasterIO(GF_Write, 0, iY, nXSize, 1,
pabyLine, nXSize, 1,
GDT_Byte,
nBands, NULL,
0, 0, 0
#ifdef GDAL_COMPILATION
, NULL
#endif
));
}
VSIFree(pabyLine);
}
if( bMemDriver )
poMEMDS = poDS;
else
GDALClose(poDS);
}
else
{
bCheck = FALSE;
}
CSLDestroy(papszOptions);
papszOptions = NULL;
Request* psGlobalRequestLast = NULL;
for(i = 0; i < nThreads; i++ )
{
GDALDataset* poDS;
// Since GDAL 2.0, the MEM driver is thread-safe, i.e. does not use the block
// cache, but only for operations not involving resampling, which is
// the case here
if( poMEMDS )
poDS = poMEMDS;
else
{
poDS = (GDALDataset*)GDALOpen(pszDataset, GA_ReadOnly);
if( poDS == NULL )
exit(1);
}
if( bMigrate )
{
Resource* psResource = (Resource*)CPLMalloc(sizeof(Resource));
psResource->poDS = poDS;
int nBufferSize;
if( eStrategy == STRATEGY_RANDOM )
nBufferSize = CreateRandomStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
else if( eStrategy == STRATEGY_LINE )
nBufferSize = CreateLineStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
else
nBufferSize = CreateBlockStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
psResource->pBuffer = CPLMalloc(nBufferSize);
PutResourceAtEnd(psResource);
}
else
{
ThreadDescription sThreadDescription;
sThreadDescription.poDS = poDS;
sThreadDescription.psRequestList = NULL;
Request* psRequestLast = NULL;
if( eStrategy == STRATEGY_RANDOM )
sThreadDescription.nBufferSize = CreateRandomStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
else if( eStrategy == STRATEGY_LINE )
sThreadDescription.nBufferSize = CreateLineStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
else
sThreadDescription.nBufferSize = CreateBlockStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
asThreadDescription.push_back(sThreadDescription);
}
}
if( bCreatedDataset && poMEMDS == NULL && bOnDisk )
{
CPLPushErrorHandler(CPLQuietErrorHandler);
VSIUnlink(pszDataset);
CPLPopErrorHandler();
}
if( bMigrate )
{
psLock = CPLCreateLock(LOCK_SPIN);
}
for(i = 0; i < nThreads; i++ )
{
CPLJoinableThread* pThread;
if( bMigrate )
pThread = CPLCreateJoinableThread(ThreadFuncWithMigration, NULL);
else
pThread = CPLCreateJoinableThread(ThreadFuncDedicatedDataset,
&(asThreadDescription[i]));
apsThreads.push_back(pThread);
}
for(i = 0; i < nThreads; i++ )
{
CPLJoinThread(apsThreads[i]);
if( !bMigrate && poMEMDS == NULL )
GDALClose(asThreadDescription[i].poDS);
}
while( psGlobalResourceList != NULL )
{
CPLFree( psGlobalResourceList->pBuffer);
if( poMEMDS == NULL )
GDALClose(psGlobalResourceList->poDS);
Resource* psNext = psGlobalResourceList->psNext;
CPLFree( psGlobalResourceList );
psGlobalResourceList = psNext;
}
if( psLock )
{
CPLDestroyLock( psLock );
}
if( bCreatedDataset && poMEMDS == NULL )
{
CPLPushErrorHandler(CPLQuietErrorHandler);
VSIUnlink(pszDataset);
CPLPopErrorHandler();
}
if( poMEMDS )
GDALClose(poMEMDS);
assert( GDALGetCacheUsed64() == 0 );
GDALDestroyDriverManager();
CSLDestroy( argv );
return 0;
}
void *GDALCreateTPSTransformerInt( int nGCPCount, const GDAL_GCP *pasGCPList,
int bReversed, char** papszOptions )
{
TPSTransformInfo *psInfo;
int iGCP;
/* -------------------------------------------------------------------- */
/* Allocate transform info. */
/* -------------------------------------------------------------------- */
psInfo = (TPSTransformInfo *) CPLCalloc(sizeof(TPSTransformInfo),1);
psInfo->pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPList );
psInfo->nGCPCount = nGCPCount;
psInfo->bReversed = bReversed;
psInfo->poForward = new VizGeorefSpline2D( 2 );
psInfo->poReverse = new VizGeorefSpline2D( 2 );
strcpy( psInfo->sTI.szSignature, "GTI" );
psInfo->sTI.pszClassName = "GDALTPSTransformer";
psInfo->sTI.pfnTransform = GDALTPSTransform;
psInfo->sTI.pfnCleanup = GDALDestroyTPSTransformer;
psInfo->sTI.pfnSerialize = GDALSerializeTPSTransformer;
/* -------------------------------------------------------------------- */
/* Attach all the points to the transformation. */
/* -------------------------------------------------------------------- */
for( iGCP = 0; iGCP < nGCPCount; iGCP++ )
{
double afPL[2], afXY[2];
afPL[0] = pasGCPList[iGCP].dfGCPPixel;
afPL[1] = pasGCPList[iGCP].dfGCPLine;
afXY[0] = pasGCPList[iGCP].dfGCPX;
afXY[1] = pasGCPList[iGCP].dfGCPY;
if( bReversed )
{
psInfo->poReverse->add_point( afPL[0], afPL[1], afXY );
psInfo->poForward->add_point( afXY[0], afXY[1], afPL );
}
else
{
psInfo->poForward->add_point( afPL[0], afPL[1], afXY );
psInfo->poReverse->add_point( afXY[0], afXY[1], afPL );
}
}
psInfo->nRefCount = 1;
int nThreads = 1;
if( nGCPCount > 100 )
{
const char* pszWarpThreads = CSLFetchNameValue(papszOptions, "NUM_THREADS");
if (pszWarpThreads == NULL)
pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
if (EQUAL(pszWarpThreads, "ALL_CPUS"))
nThreads = CPLGetNumCPUs();
else
nThreads = atoi(pszWarpThreads);
}
if( nThreads > 1 )
{
/* Compute direct and reverse transforms in parallel */
void* hThread = CPLCreateJoinableThread(GDALTPSComputeForwardInThread, psInfo);
psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
if( hThread != NULL )
CPLJoinThread(hThread);
else
psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
}
else
{
psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
}
if( !psInfo->bForwardSolved || !psInfo->bReverseSolved )
{
GDALDestroyTPSTransformer(psInfo);
return NULL;
}
return psInfo;
}
/** Setup the pool.
*
* @param nThreads Number of threads to launch
* @param pfnInitFunc Initialization function to run in each thread. May be NULL
* @param pasInitData Array of initialization data. Its length must be nThreads,
* or it should be NULL.
* @return true if initialization was successful.
*/
bool CPLWorkerThreadPool::Setup(int nThreads,
CPLThreadFunc pfnInitFunc,
void** pasInitData)
{
CPLAssert( nThreads > 0 );
hCond = CPLCreateCond();
if( hCond == nullptr )
return false;
bool bRet = true;
aWT.resize(nThreads);
for(int i=0;i<nThreads;i++)
{
aWT[i].pfnInitFunc = pfnInitFunc;
aWT[i].pInitData = pasInitData ? pasInitData[i] : nullptr;
aWT[i].poTP = this;
aWT[i].hMutex = CPLCreateMutexEx(CPL_MUTEX_REGULAR);
if( aWT[i].hMutex == nullptr )
{
nThreads = i;
aWT.resize(nThreads);
bRet = false;
break;
}
CPLReleaseMutex(aWT[i].hMutex);
aWT[i].hCond = CPLCreateCond();
if( aWT[i].hCond == nullptr )
{
CPLDestroyMutex(aWT[i].hMutex);
nThreads = i;
aWT.resize(nThreads);
bRet = false;
break;
}
aWT[i].bMarkedAsWaiting = FALSE;
// aWT[i].psNextJob = nullptr;
aWT[i].hThread =
CPLCreateJoinableThread(WorkerThreadFunction, &(aWT[i]));
if( aWT[i].hThread == nullptr )
{
nThreads = i;
aWT.resize(nThreads);
bRet = false;
break;
}
}
// Wait all threads to be started
while( true )
{
CPLAcquireMutex(hMutex, 1000.0);
int nWaitingWorkerThreadsLocal = nWaitingWorkerThreads;
if( nWaitingWorkerThreadsLocal < nThreads )
CPLCondWait(hCond, hMutex);
CPLReleaseMutex(hMutex);
if( nWaitingWorkerThreadsLocal == nThreads )
break;
}
if( eState == CPLWTS_ERROR )
bRet = false;
return bRet;
}
void *GDALCreateTPSTransformerInt( int nGCPCount, const GDAL_GCP *pasGCPList,
int bReversed, char** papszOptions )
{
TPSTransformInfo *psInfo;
int iGCP;
/* -------------------------------------------------------------------- */
/* Allocate transform info. */
/* -------------------------------------------------------------------- */
psInfo = (TPSTransformInfo *) CPLCalloc(sizeof(TPSTransformInfo),1);
psInfo->pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPList );
psInfo->nGCPCount = nGCPCount;
psInfo->bReversed = bReversed;
psInfo->poForward = new VizGeorefSpline2D( 2 );
psInfo->poReverse = new VizGeorefSpline2D( 2 );
memcpy( psInfo->sTI.abySignature, GDAL_GTI2_SIGNATURE, strlen(GDAL_GTI2_SIGNATURE) );
psInfo->sTI.pszClassName = "GDALTPSTransformer";
psInfo->sTI.pfnTransform = GDALTPSTransform;
psInfo->sTI.pfnCleanup = GDALDestroyTPSTransformer;
psInfo->sTI.pfnSerialize = GDALSerializeTPSTransformer;
psInfo->sTI.pfnCreateSimilar = GDALCreateSimilarTPSTransformer;
/* -------------------------------------------------------------------- */
/* Attach all the points to the transformation. */
/* -------------------------------------------------------------------- */
std::map< std::pair<double, double>, int > oMapPixelLineToIdx;
std::map< std::pair<double, double>, int > oMapXYToIdx;
for( iGCP = 0; iGCP < nGCPCount; iGCP++ )
{
double afPL[2], afXY[2];
afPL[0] = pasGCPList[iGCP].dfGCPPixel;
afPL[1] = pasGCPList[iGCP].dfGCPLine;
afXY[0] = pasGCPList[iGCP].dfGCPX;
afXY[1] = pasGCPList[iGCP].dfGCPY;
std::map< std::pair<double, double>, int >::iterator oIter;
oIter = oMapPixelLineToIdx.find( std::pair<double,double>(afPL[0], afPL[1]) );
if( oIter != oMapPixelLineToIdx.end() )
{
if( afXY[0] == pasGCPList[oIter->second].dfGCPX &&
afXY[1] == pasGCPList[oIter->second].dfGCPY )
{
continue;
}
else
{
CPLError(CE_Warning, CPLE_AppDefined,
"GCP %d and %d have same (pixel,line)=(%f,%f) but different (X,Y): (%f,%f) vs (%f,%f)",
iGCP + 1, oIter->second,
afPL[0], afPL[1],
afXY[0], afXY[1],
pasGCPList[oIter->second].dfGCPX, pasGCPList[oIter->second].dfGCPY);
}
}
else
{
oMapPixelLineToIdx[ std::pair<double,double>(afPL[0], afPL[1]) ] = iGCP;
}
oIter = oMapXYToIdx.find( std::pair<double,double>(afXY[0], afXY[1]) );
if( oIter != oMapXYToIdx.end() )
{
CPLError(CE_Warning, CPLE_AppDefined,
"GCP %d and %d have same (x,y)=(%f,%f) but different (pixel,line): (%f,%f) vs (%f,%f)",
iGCP + 1, oIter->second,
afXY[0], afXY[1],
afPL[0], afPL[1],
pasGCPList[oIter->second].dfGCPPixel, pasGCPList[oIter->second].dfGCPLine);
}
else
{
oMapXYToIdx[ std::pair<double,double>(afXY[0], afXY[1]) ] = iGCP;
}
bool bOK = true;
if( bReversed )
{
bOK &= psInfo->poReverse->add_point( afPL[0], afPL[1], afXY );
bOK &= psInfo->poForward->add_point( afXY[0], afXY[1], afPL );
}
else
{
bOK &= psInfo->poForward->add_point( afPL[0], afPL[1], afXY );
bOK &= psInfo->poReverse->add_point( afXY[0], afXY[1], afPL );
}
if( !bOK )
{
GDALDestroyTPSTransformer(psInfo);
return NULL;
}
}
psInfo->nRefCount = 1;
int nThreads = 1;
if( nGCPCount > 100 )
{
const char* pszWarpThreads = CSLFetchNameValue(papszOptions, "NUM_THREADS");
if (pszWarpThreads == NULL)
pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
if (EQUAL(pszWarpThreads, "ALL_CPUS"))
nThreads = CPLGetNumCPUs();
else
nThreads = atoi(pszWarpThreads);
}
if( nThreads > 1 )
{
/* Compute direct and reverse transforms in parallel */
CPLJoinableThread* hThread = CPLCreateJoinableThread(GDALTPSComputeForwardInThread, psInfo);
psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
if( hThread != NULL )
CPLJoinThread(hThread);
else
psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
}
else
{
psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
}
if( !psInfo->bForwardSolved || !psInfo->bReverseSolved )
{
GDALDestroyTPSTransformer(psInfo);
return NULL;
}
return psInfo;
}
/*
** Fetch a nomads forecast from the NWS servers. The forecasts consist of grib
** files, one forecast for each forecast hour. The time starts and the nearest
** forecast hour *before* the reference time passed. If no reference time is
** passed, or it is not valid, now() is used.
**
** The forecasts are downloaded into a temporary directory. If the entire
** download succeeds, then the files are copied into the path specified. If
** the path specified is a zip file (ends in *.zip), then the forecast files
** are zipped.
**
** Available compile time configuration options:
** NOMADS_USE_IP: Use the ip address instead of the hostname. It
** possibly goes around dns lookup, but it's doubtfull.
** NOMADS_ENABLE_ASYNC: Allow for asynchronous connections to the
** server.
** NOMADS_EXPER_FORECASTS: Compile in forecasts that may not work with
** the current configuration, essentially
** unsupported (ie NARRE, RTMA).
** NOMADS_USE_VSI_READ: Use the VSI*L api for downloading. This will
** allow definition of chunk sizes for download,
** although it is probably unnecessary. See
** NOMADS_VSI_BLOCK_SIZE below. If not enabled, a
** single fetch is made for each file, which is
** faster.
** Available runtime configuration options:
** NOMADS_THREAD_COUNT: Number of threads to use for downloads if
** NOMADS_ENABLE_ASYNC is set to ON during
** compilation. Default is 4.
** NOMADS_VSI_BLOCK_SIZE: Number of bytes to request at a time when
** downloading files if NOMADS_USE_VSI_READ is
** set to ON during compilation. Default is 512.
** NOMADS_MAX_FCST_REWIND: Number of forecast run time steps to go back
** to attempt to get a full time frame.
** GDAL_HTTP_TIMEOUT: Timeout for HTTP requests in seconds. We should
** be able to set this reasonably low.
**
** \param pszModelKey The name key of the model to use, ie "nam_conus". For a
** listing of models, see nomads.ncep.gov or /see nomads.h
**
** \param pszRefTime The reference time to begin searching for forecasts from.
** The search starts at refrence time, then goes back until
** it hits a valid forecast time. If the download cannot be
** complete, it will step back NOMADS_MAX_FCST_REWIND
** foreacst times to attempt to get a full forecast.
**
** \param nHours The extent of hours to download forecasts from the reference
** time. If we step back, we will still grab all forecasts up
** until nHours from the reference time, not the forecast time.
**
** \param nStride The number of forecasts to skip in time steps. For example,
** if 12 hours of gfs is requested (normally 5 files/steps (0,
** 3, 6, 9, 12) with a stride of 2, you'd get 0, 6, 12 forecast
** hours.
**
** \param padfBbox The bounding box of the request in WGS84 decimal degrees.
** Order is xmin, xmax, ymax, ymin.
**
** \param pszDstVsiPath The location to write the files. If the location
** has an extension of ".zip", then the output files are
** written as a zip archive, otherwise to a path.
**
** \param papszOptions List of key=value options, unused.
**
** \param pfnProgress Optional progress function.
**
** \return NOMADS_OK(0) on success, NOMADS_ERR(1) otherwise.
*/
int NomadsFetch( const char *pszModelKey, const char *pszRefTime,
int nHours, int nStride, double *padfBbox,
const char *pszDstVsiPath, char ** papszOptions,
GDALProgressFunc pfnProgress )
{
const char **ppszKey = NULL;
int nFcstHour = 0;
int *panRunHours = NULL;
int i = 0;
int j = 0;
int k = 0;
int t = 0;
int rc = 0;
char **papszDownloadUrls = NULL;
char **papszOutputFiles = NULL;
char **papszFinalFiles = NULL;
int nFilesToGet = 0;
const char *pszTmpDir;
const char *pszConfigOpt;
int nFcstTries;
int nMaxFcstRewind;
int nrc;
int bZip;
void **pThreads;
int nThreads;
const char *pszThreadCount;
NomadsThreadData *pasData;
nomads_utc *ref, *end, *fcst;
nrc = NOMADS_OK;
CPLDebug( "NOMADS", "Fetching data for bounding box: %lf, %lf, %lf, %lf",
padfBbox[0], padfBbox[1], padfBbox[2], padfBbox[3] );
ppszKey = NomadsFindModel( pszModelKey );
if( ppszKey == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not find model key in nomads data" );
return NOMADS_ERR;
}
NomadsUtcCreate( &ref );
NomadsUtcCreate( &end );
rc = NOMADS_OK;
if( pszRefTime )
{
rc = NomadsUtcFromIsoFrmt( ref, pszRefTime );
}
if( rc != NOMADS_OK || pszRefTime == NULL )
{
NomadsUtcNow( ref );
}
NomadsUtcCopy( end, ref );
NomadsUtcAddHours( end, nHours );
/* Disable unneeded reading of entire directories, good speedup */
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE" );
pszConfigOpt = CPLGetConfigOption( "NOMADS_HTTP_TIMEOUT", "20" );
if( pszConfigOpt != NULL )
{
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", pszConfigOpt );
}
nMaxFcstRewind = atoi( CPLGetConfigOption( "NOMADS_MAX_FCST_REWIND", "2" ) );
if( nMaxFcstRewind < 1 || nMaxFcstRewind > 24 )
{
nMaxFcstRewind = 2;
}
/* Go back at least 3 for rap, as it may not get updated all the time. */
if( EQUALN( pszModelKey, "rap", 3 ) || EQUALN( pszModelKey, "hrrr", 4 ) )
{
nMaxFcstRewind = nMaxFcstRewind > 3 ? nMaxFcstRewind : 3;
}
#ifdef NOMADS_ENABLE_ASYNC
pszThreadCount = CPLGetConfigOption( "NOMADS_THREAD_COUNT", "4" );
nThreads = atoi( pszThreadCount );
if( nThreads < 1 || nThreads > 96 )
{
nThreads = 4;
}
pThreads = CPLMalloc( sizeof( void * ) * nThreads );
pasData = CPLMalloc( sizeof( NomadsThreadData ) * nThreads );
#else /* NOMADS_ENABLE_ASYNC */
/* Unused variables, set to null to so free is no-op */
nThreads = 1;
pThreads = NULL;
pasData = NULL;
#endif /* NOMADS_ENABLE_ASYNC */
fcst = NULL;
nFcstTries = 0;
while( nFcstTries < nMaxFcstRewind )
{
nrc = NOMADS_OK;
fcst = NomadsSetForecastTime( ppszKey, ref, nFcstTries );
nFcstHour = fcst->ts->tm_hour;
CPLDebug( "WINDNINJA", "Generated forecast time in utc: %s",
NomadsUtcStrfTime( fcst, "%Y%m%dT%HZ" ) );
if( EQUAL( pszModelKey, "rtma_conus" ) )
{
panRunHours = (int*)CPLMalloc( sizeof( int ) );
nFilesToGet = 1;
}
else
{
nFilesToGet =
NomadsBuildForecastRunHours( ppszKey, fcst, end, nHours,
nStride, &panRunHours );
}
papszDownloadUrls =
NomadsBuildForecastFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, fcst, padfBbox );
if( papszDownloadUrls == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
pszTmpDir = CPLStrdup( CPLGenerateTempFilename( NULL ) );
CPLDebug( "WINDNINJA", "Creating Temp directory: %s", pszTmpDir );
VSIMkdir( pszTmpDir, 0777 );
papszOutputFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszTmpDir, FALSE );
if( papszOutputFiles == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
CPLAssert( CSLCount( papszDownloadUrls ) == nFilesToGet );
CPLAssert( CSLCount( papszOutputFiles ) == nFilesToGet );
if( pfnProgress )
{
pfnProgress( 0.0, "Starting download...", NULL );
}
/* Download one file and start over if it's not there. */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[0], papszOutputFiles[0] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[0], papszOutputFiles[0] );
#endif /* NOMADS_USE_VSI_READ */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
/*
** Don't explicitly break here. We'll skip the while loop because
** nrc != NOMADS_OK, and we can clean up memory and shift times in
** one spot to avoid duplicate code.
*/
}
/* Get the rest */
i = 1;
while( i < nFilesToGet && nrc == NOMADS_OK )
{
if( pfnProgress )
{
if( pfnProgress( (double)i / nFilesToGet,
CPLSPrintf( "Downloading %s...",
CPLGetFilename( papszOutputFiles[i] ) ),
NULL ) )
{
CPLError( CE_Failure, CPLE_UserInterrupt,
"Cancelled by user." );
nrc = NOMADS_ERR;
nFcstTries = nMaxFcstRewind;
break;
}
}
#ifdef NOMADS_ENABLE_ASYNC
k = i > nFilesToGet - nThreads ? (nFilesToGet - 1) % nThreads : nThreads;
for( t = 0; t < k; t++ )
{
pasData[t].pszUrl = papszDownloadUrls[i];
pasData[t].pszFilename = papszOutputFiles[i];
pThreads[t] =
CPLCreateJoinableThread( NomadsFetchAsync, &pasData[t] );
i++;
}
for( t = 0; t < k; t++ )
{
CPLJoinThread( pThreads[t] );
}
for( t = 0; t < k; t++ )
{
if( pasData[t].nErr )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Threaded download failed, attempting " \
"serial download for %s",
CPLGetFilename( pasData[t].pszFilename ) );
/* Try again, serially though */
if( CPLCheckForFile( (char *)pasData[t].pszFilename, NULL ) );
{
VSIUnlink( pasData[t].pszFilename );
}
#ifdef NOMADS_USE_VSI_READ
rc = NomadsFetchVsi( pasData[t].pszUrl,
pasData[t].pszFilename );
#else /* NOMADS_USE_VSI_READ */
rc = NomadsFetchHttp( pasData[t].pszUrl,
pasData[t].pszFilename );
#endif /* NOMADS_USE_VSI_READ */
if( rc != NOMADS_OK )
{
nrc = rc;
}
}
}
#else /* NOMADS_ENABLE_ASYNC */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[i], papszOutputFiles[i] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[i], papszOutputFiles[i] );
#endif /* NOMADS_USE_VSI_READ */
i++;
#endif /* NOMADS_ENABLE_ASYNC */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
break;
}
}
/*
** XXX Cleanup XXX
** After each loop we can get rid of the urls, but we can only get rid
** of the others if they are to be reallocated in the next loop. Any
** cleanup in the else nrc == NOMADS_OK clause should be cleaned up in
** the nrc == NOMADS_OK outside the loop. Those are held so we can
** process the output files and zip them into an archive.
*/
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
if( nrc == NOMADS_OK )
{
break;
}
else
{
CPLFree( (void*)panRunHours );
CSLDestroy( papszOutputFiles );
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
}
if( nrc == NOMADS_OK )
{
bZip = EQUAL( CPLGetExtension( pszDstVsiPath ), "zip" ) ? TRUE : FALSE;
papszFinalFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszDstVsiPath,
bZip );
CPLFree( (void*)panRunHours );
if( CPLCheckForFile( (char*)pszDstVsiPath, NULL ) )
{
CPLUnlinkTree( pszDstVsiPath );
}
if( !bZip )
{
VSIMkdir( pszDstVsiPath, 0777 );
}
nrc = NomadsZipFiles( papszOutputFiles, papszFinalFiles );
CSLDestroy( papszOutputFiles );
CSLDestroy( papszFinalFiles );
if( nrc != NOMADS_OK )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not copy files into path, unknown i/o failure" );
CPLUnlinkTree( pszDstVsiPath );
}
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
CPLFree( (void*)pasData );
CPLFree( (void**)pThreads );
NomadsUtcFree( ref );
NomadsUtcFree( end );
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", NULL );
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", NULL );
if( nrc == NOMADS_OK && pfnProgress )
{
pfnProgress( 1.0, NULL, NULL );
}
return nrc;
}