Пример #1
0
int main(int argc, char* argv[])
{
    CPLJoinableThread* hThread;

    printf("main thread %p\n", (void*)CPLGetPID());

    argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );

    CPLSetConfigOption("GDAL_CACHEMAX", "0");
    CPLSetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "ON");

    MyDataset* poDS = new MyDataset();

    char buf1[] = { 1 } ;
    CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 0, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));

    hThread = CPLCreateJoinableThread(thread_func, NULL);
    CPLSleep(0.3);
    CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 1, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
    GDALFlushCacheBlock();

    CPLJoinThread(hThread);

    delete poDS;
    GDALDestroyDriverManager();
    CSLDestroy( argv );

    return 0;
}
Пример #2
0
void VSICurlStreamingHandle::StartDownload()
{
    if (bDownloadInProgress || bDownloadStopped)
        return;

    //if (ENABLE_DEBUG)
        CPLDebug("VSICURL", "Start download for %s", pszURL);

    if (hCurlHandle == NULL)
        hCurlHandle = curl_easy_init();
    oRingBuffer.Reset();
    bDownloadInProgress = TRUE;
    nRingBufferFileOffset = 0;
    hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
}
Пример #3
0
static int test_two_pages()
{
    CPLVirtualMem* ctxt;
    volatile char* addr;
    void* hThread;

    ctxt = CPLVirtualMemNew(3*MINIMUM_PAGE_SIZE,
                            MINIMUM_PAGE_SIZE,
                            MINIMUM_PAGE_SIZE,
                            FALSE,
                            VIRTUALMEM_READONLY,
                            test_two_pages_cbk,
                            NULL,
                            NULL, NULL);
    if( ctxt == NULL )
        return FALSE;

    addr = (char*) CPLVirtualMemGetAddr(ctxt);
    assert(CPLVirtualMemGetPageSize(ctxt) == MINIMUM_PAGE_SIZE);
    assert(CPLVirtualMemIsAccessThreadSafe(ctxt));
    /*fprintfstderr("main thread is %X, addr=%p\n", pthread_self(), addr);*/
    hThread = CPLCreateJoinableThread(test_two_pages_thread, ctxt);
    CPLVirtualMemDeclareThread(ctxt);
    {
        int i=0;
        for(i=0; i<50*1000; i++)
        {
            char val = addr[MINIMUM_PAGE_SIZE * (i % 3)];
            /*fprintfstderr("T1: val[%d] = %d\n", MINIMUM_PAGE_SIZE * (i % 2), val);*/
            assert(val == (((i % 3) == 0) ? 0x3F : ((i % 3) == 1) ? 0x5F : 0x7F));
        }
    }
    CPLVirtualMemUnDeclareThread(ctxt);
    CPLJoinThread(hThread);
    CPLVirtualMemFree(ctxt);

    return TRUE;
}
Пример #4
0
int main(int argc, char* argv[])
{
    int i;
    int nThreads = CPLGetNumCPUs();
    std::vector<CPLJoinableThread*> apsThreads;
    Strategy eStrategy = STRATEGY_RANDOM;
    int bNewDatasetOption = FALSE;
    int nXSize = 5000;
    int nYSize = 5000;
    int nBands = 4;
    char** papszOptions = NULL;
    int bOnDisk = FALSE;
    std::vector<ThreadDescription> asThreadDescription;
    int bMemDriver = FALSE;
    GDALDataset* poMEMDS = NULL;
    int bMigrate = FALSE;
    int nMaxRequests = -1;

    argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );

    GDALAllRegister();

    for(i = 1; i < argc; i++)
    {
        if( EQUAL(argv[i], "-threads") && i + 1 < argc)
        {
            i ++;
            nThreads = atoi(argv[i]);
        }
        else if( EQUAL(argv[i], "-loops") && i + 1 < argc)
        {
            i ++;
            nLoops = atoi(argv[i]);
            if( nLoops <= 0 )
                nLoops = INT_MAX;
        }
        else if( EQUAL(argv[i], "-max_requests") && i + 1 < argc)
        {
            i ++;
            nMaxRequests = atoi(argv[i]);
        }
        else if( EQUAL(argv[i], "-strategy") && i + 1 < argc)
        {
            i ++;
            if( EQUAL(argv[i], "random") )
                eStrategy = STRATEGY_RANDOM;
            else if( EQUAL(argv[i], "line") )
                eStrategy = STRATEGY_LINE;
            else if( EQUAL(argv[i], "block") )
                eStrategy = STRATEGY_BLOCK;
            else
                Usage();
        }
        else if( EQUAL(argv[i], "-xsize") && i + 1 < argc)
        {
            i ++;
            nXSize = atoi(argv[i]);
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-ysize") && i + 1 < argc)
        {
            i ++;
            nYSize = atoi(argv[i]);
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-bands") && i + 1 < argc)
        {
            i ++;
            nBands = atoi(argv[i]);
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-co") && i + 1 < argc)
        {
            i ++;
            papszOptions = CSLAddString(papszOptions, argv[i]);
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-ondisk"))
        {
            bOnDisk = TRUE;
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-check"))
        {
            bCheck = TRUE;
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-memdriver"))
        {
            bMemDriver = TRUE;
            bNewDatasetOption = TRUE;
        }
        else if( EQUAL(argv[i], "-migrate"))
            bMigrate = TRUE;
        else if( argv[i][0] == '-' )
            Usage();
        else if( pszDataset == NULL )
            pszDataset = argv[i];
        else
        {
            Usage();
        }
    }

    if( pszDataset != NULL && bNewDatasetOption )
        Usage();

    CPLDebug("TEST", "Using %d threads", nThreads);

    int bCreatedDataset = FALSE;
    if( pszDataset == NULL )
    {
        bCreatedDataset = TRUE;
        if( bOnDisk )
            pszDataset = "/tmp/tmp.tif";
        else
            pszDataset = "/vsimem/tmp.tif";
        GDALDataset* poDS = ((GDALDriver*)GDALGetDriverByName((bMemDriver) ? "MEM" : "GTiff"))->Create(pszDataset,
                                nXSize, nYSize, nBands, GDT_Byte, papszOptions);
        if( bCheck )
        {
            GByte* pabyLine = (GByte*) VSIMalloc(nBands * nXSize);
            for(int iY=0;iY<nYSize;iY++)
            {
                for(int iX=0;iX<nXSize;iX++)
                {
                    for(int iBand=0;iBand<nBands;iBand++)
                    {
                        unsigned long seed = iBand * nXSize * nYSize + iY * nXSize + iX;
                        pabyLine[iBand * nXSize + iX] = (GByte)(myrand_r(&seed) & 0xff);
                    }
                }
                CPL_IGNORE_RET_VAL(poDS->RasterIO(GF_Write, 0, iY, nXSize, 1,
                               pabyLine, nXSize, 1,
                               GDT_Byte,
                               nBands, NULL,
                               0, 0, 0
#ifdef GDAL_COMPILATION
                               , NULL
#endif
                               ));
            }
            VSIFree(pabyLine);
        }
        if( bMemDriver ) 
            poMEMDS = poDS;
        else
            GDALClose(poDS);
    }
    else
    {
        bCheck = FALSE;
    }
    CSLDestroy(papszOptions);
    papszOptions = NULL;
    
    Request* psGlobalRequestLast = NULL;

    for(i = 0; i < nThreads; i++ )
    {
        GDALDataset* poDS;
        // Since GDAL 2.0, the MEM driver is thread-safe, i.e. does not use the block
        // cache, but only for operations not involving resampling, which is
        // the case here
        if( poMEMDS ) 
            poDS = poMEMDS;
        else
        {
            poDS = (GDALDataset*)GDALOpen(pszDataset, GA_ReadOnly);
            if( poDS == NULL )
                exit(1);
        }
        if( bMigrate )
        {
            Resource* psResource = (Resource*)CPLMalloc(sizeof(Resource));
            psResource->poDS = poDS;
            int nBufferSize;
            if( eStrategy == STRATEGY_RANDOM )
                nBufferSize = CreateRandomStrategyRequests(
                        poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
            else if( eStrategy == STRATEGY_LINE )
                nBufferSize = CreateLineStrategyRequests(
                        poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
            else
                nBufferSize = CreateBlockStrategyRequests(
                        poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
            psResource->pBuffer = CPLMalloc(nBufferSize);
            PutResourceAtEnd(psResource);
        }
        else
        {
            ThreadDescription sThreadDescription;
            sThreadDescription.poDS = poDS;
            sThreadDescription.psRequestList = NULL;
            Request* psRequestLast = NULL;
            if( eStrategy == STRATEGY_RANDOM )
                sThreadDescription.nBufferSize = CreateRandomStrategyRequests(
                        poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
            else if( eStrategy == STRATEGY_LINE )
                sThreadDescription.nBufferSize = CreateLineStrategyRequests(
                        poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
            else
                sThreadDescription.nBufferSize = CreateBlockStrategyRequests(
                        poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
            asThreadDescription.push_back(sThreadDescription);
        }
    }

    if( bCreatedDataset && poMEMDS == NULL && bOnDisk )
    {
        CPLPushErrorHandler(CPLQuietErrorHandler);
        VSIUnlink(pszDataset);
        CPLPopErrorHandler();
    }
    
    if( bMigrate )
    {
        psLock = CPLCreateLock(LOCK_SPIN);
    }

    for(i = 0; i < nThreads; i++ )
    {
        CPLJoinableThread* pThread;
        if( bMigrate )
            pThread = CPLCreateJoinableThread(ThreadFuncWithMigration, NULL);
        else
            pThread = CPLCreateJoinableThread(ThreadFuncDedicatedDataset,
                                              &(asThreadDescription[i]));
        apsThreads.push_back(pThread);
    }
    for(i = 0; i < nThreads; i++ )
    {
        CPLJoinThread(apsThreads[i]);
        if( !bMigrate && poMEMDS == NULL )
            GDALClose(asThreadDescription[i].poDS);
    }
    while( psGlobalResourceList != NULL )
    {
        CPLFree( psGlobalResourceList->pBuffer);
        if( poMEMDS == NULL )
            GDALClose(psGlobalResourceList->poDS);
        Resource* psNext = psGlobalResourceList->psNext;
        CPLFree( psGlobalResourceList );
        psGlobalResourceList = psNext;
    }

    if( psLock )
    {
        CPLDestroyLock( psLock );
    }

    if( bCreatedDataset && poMEMDS == NULL  )
    {
        CPLPushErrorHandler(CPLQuietErrorHandler);
        VSIUnlink(pszDataset);
        CPLPopErrorHandler();
    }
    if( poMEMDS )
        GDALClose(poMEMDS);

    assert( GDALGetCacheUsed64() == 0 );

    GDALDestroyDriverManager();
    CSLDestroy( argv );

    return 0;
}
void *GDALCreateTPSTransformerInt( int nGCPCount, const GDAL_GCP *pasGCPList,
                                   int bReversed, char** papszOptions )

{
    TPSTransformInfo *psInfo;
    int    iGCP;

    /* -------------------------------------------------------------------- */
    /*      Allocate transform info.                                        */
    /* -------------------------------------------------------------------- */
    psInfo = (TPSTransformInfo *) CPLCalloc(sizeof(TPSTransformInfo),1);

    psInfo->pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPList );
    psInfo->nGCPCount = nGCPCount;

    psInfo->bReversed = bReversed;
    psInfo->poForward = new VizGeorefSpline2D( 2 );
    psInfo->poReverse = new VizGeorefSpline2D( 2 );

    strcpy( psInfo->sTI.szSignature, "GTI" );
    psInfo->sTI.pszClassName = "GDALTPSTransformer";
    psInfo->sTI.pfnTransform = GDALTPSTransform;
    psInfo->sTI.pfnCleanup = GDALDestroyTPSTransformer;
    psInfo->sTI.pfnSerialize = GDALSerializeTPSTransformer;

    /* -------------------------------------------------------------------- */
    /*      Attach all the points to the transformation.                    */
    /* -------------------------------------------------------------------- */
    for( iGCP = 0; iGCP < nGCPCount; iGCP++ )
    {
        double    afPL[2], afXY[2];

        afPL[0] = pasGCPList[iGCP].dfGCPPixel;
        afPL[1] = pasGCPList[iGCP].dfGCPLine;
        afXY[0] = pasGCPList[iGCP].dfGCPX;
        afXY[1] = pasGCPList[iGCP].dfGCPY;

        if( bReversed )
        {
            psInfo->poReverse->add_point( afPL[0], afPL[1], afXY );
            psInfo->poForward->add_point( afXY[0], afXY[1], afPL );
        }
        else
        {
            psInfo->poForward->add_point( afPL[0], afPL[1], afXY );
            psInfo->poReverse->add_point( afXY[0], afXY[1], afPL );
        }
    }

    psInfo->nRefCount = 1;

    int nThreads = 1;
    if( nGCPCount > 100 )
    {
        const char* pszWarpThreads = CSLFetchNameValue(papszOptions, "NUM_THREADS");
        if (pszWarpThreads == NULL)
            pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
        if (EQUAL(pszWarpThreads, "ALL_CPUS"))
            nThreads = CPLGetNumCPUs();
        else
            nThreads = atoi(pszWarpThreads);
    }

    if( nThreads > 1 )
    {
        /* Compute direct and reverse transforms in parallel */
        void* hThread = CPLCreateJoinableThread(GDALTPSComputeForwardInThread, psInfo);
        psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
        if( hThread != NULL )
            CPLJoinThread(hThread);
        else
            psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
    }
    else
    {
        psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
        psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
    }

    if( !psInfo->bForwardSolved || !psInfo->bReverseSolved )
    {
        GDALDestroyTPSTransformer(psInfo);
        return NULL;
    }

    return psInfo;
}
Пример #6
0
/** Setup the pool.
 *
 * @param nThreads Number of threads to launch
 * @param pfnInitFunc Initialization function to run in each thread. May be NULL
 * @param pasInitData Array of initialization data. Its length must be nThreads,
 *                    or it should be NULL.
 * @return true if initialization was successful.
 */
bool CPLWorkerThreadPool::Setup(int nThreads,
                            CPLThreadFunc pfnInitFunc,
                            void** pasInitData)
{
    CPLAssert( nThreads > 0 );

    hCond = CPLCreateCond();
    if( hCond == nullptr )
        return false;

    bool bRet = true;
    aWT.resize(nThreads);
    for(int i=0;i<nThreads;i++)
    {
        aWT[i].pfnInitFunc = pfnInitFunc;
        aWT[i].pInitData = pasInitData ? pasInitData[i] : nullptr;
        aWT[i].poTP = this;

        aWT[i].hMutex = CPLCreateMutexEx(CPL_MUTEX_REGULAR);
        if( aWT[i].hMutex == nullptr )
        {
            nThreads = i;
            aWT.resize(nThreads);
            bRet = false;
            break;
        }
        CPLReleaseMutex(aWT[i].hMutex);
        aWT[i].hCond = CPLCreateCond();
        if( aWT[i].hCond == nullptr )
        {
            CPLDestroyMutex(aWT[i].hMutex);
            nThreads = i;
            aWT.resize(nThreads);
            bRet = false;
            break;
        }

        aWT[i].bMarkedAsWaiting = FALSE;
        // aWT[i].psNextJob = nullptr;

        aWT[i].hThread =
            CPLCreateJoinableThread(WorkerThreadFunction, &(aWT[i]));
        if( aWT[i].hThread == nullptr )
        {
            nThreads = i;
            aWT.resize(nThreads);
            bRet = false;
            break;
        }
    }

    // Wait all threads to be started
    while( true )
    {
        CPLAcquireMutex(hMutex, 1000.0);
        int nWaitingWorkerThreadsLocal = nWaitingWorkerThreads;
        if( nWaitingWorkerThreadsLocal < nThreads )
            CPLCondWait(hCond, hMutex);
        CPLReleaseMutex(hMutex);
        if( nWaitingWorkerThreadsLocal == nThreads )
            break;
    }

    if( eState == CPLWTS_ERROR )
        bRet = false;

    return bRet;
}
Пример #7
0
void *GDALCreateTPSTransformerInt( int nGCPCount, const GDAL_GCP *pasGCPList,
                                   int bReversed, char** papszOptions )

{
    TPSTransformInfo *psInfo;
    int    iGCP;

/* -------------------------------------------------------------------- */
/*      Allocate transform info.                                        */
/* -------------------------------------------------------------------- */
    psInfo = (TPSTransformInfo *) CPLCalloc(sizeof(TPSTransformInfo),1);

    psInfo->pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPList );
    psInfo->nGCPCount = nGCPCount;

    psInfo->bReversed = bReversed;
    psInfo->poForward = new VizGeorefSpline2D( 2 );
    psInfo->poReverse = new VizGeorefSpline2D( 2 );

    memcpy( psInfo->sTI.abySignature, GDAL_GTI2_SIGNATURE, strlen(GDAL_GTI2_SIGNATURE) );
    psInfo->sTI.pszClassName = "GDALTPSTransformer";
    psInfo->sTI.pfnTransform = GDALTPSTransform;
    psInfo->sTI.pfnCleanup = GDALDestroyTPSTransformer;
    psInfo->sTI.pfnSerialize = GDALSerializeTPSTransformer;
    psInfo->sTI.pfnCreateSimilar = GDALCreateSimilarTPSTransformer;

/* -------------------------------------------------------------------- */
/*      Attach all the points to the transformation.                    */
/* -------------------------------------------------------------------- */
    std::map< std::pair<double, double>, int > oMapPixelLineToIdx;
    std::map< std::pair<double, double>, int > oMapXYToIdx;
    for( iGCP = 0; iGCP < nGCPCount; iGCP++ )
    {
        double    afPL[2], afXY[2];

        afPL[0] = pasGCPList[iGCP].dfGCPPixel;
        afPL[1] = pasGCPList[iGCP].dfGCPLine;
        afXY[0] = pasGCPList[iGCP].dfGCPX;
        afXY[1] = pasGCPList[iGCP].dfGCPY;

        std::map< std::pair<double, double>, int >::iterator oIter;
        oIter = oMapPixelLineToIdx.find( std::pair<double,double>(afPL[0], afPL[1]) );
        if( oIter != oMapPixelLineToIdx.end() )
        {
            if( afXY[0] == pasGCPList[oIter->second].dfGCPX &&
                afXY[1] == pasGCPList[oIter->second].dfGCPY )
            {
                continue;
            }
            else
            {
                CPLError(CE_Warning, CPLE_AppDefined,
                         "GCP %d and %d have same (pixel,line)=(%f,%f) but different (X,Y): (%f,%f) vs (%f,%f)",
                         iGCP + 1, oIter->second,
                         afPL[0], afPL[1],
                         afXY[0], afXY[1],
                         pasGCPList[oIter->second].dfGCPX, pasGCPList[oIter->second].dfGCPY);
            }
        }
        else
        {
            oMapPixelLineToIdx[ std::pair<double,double>(afPL[0], afPL[1]) ] = iGCP;
        }

        oIter = oMapXYToIdx.find( std::pair<double,double>(afXY[0], afXY[1]) );
        if( oIter != oMapXYToIdx.end() )
        {
            CPLError(CE_Warning, CPLE_AppDefined,
                        "GCP %d and %d have same (x,y)=(%f,%f) but different (pixel,line): (%f,%f) vs (%f,%f)",
                        iGCP + 1, oIter->second,
                        afXY[0], afXY[1],
                        afPL[0], afPL[1],
                        pasGCPList[oIter->second].dfGCPPixel, pasGCPList[oIter->second].dfGCPLine);
        }
        else
        {
            oMapXYToIdx[ std::pair<double,double>(afXY[0], afXY[1]) ] = iGCP;
        }

        bool bOK = true;
        if( bReversed )
        {
            bOK &= psInfo->poReverse->add_point( afPL[0], afPL[1], afXY );
            bOK &= psInfo->poForward->add_point( afXY[0], afXY[1], afPL );
        }
        else
        {
            bOK &= psInfo->poForward->add_point( afPL[0], afPL[1], afXY );
            bOK &= psInfo->poReverse->add_point( afXY[0], afXY[1], afPL );
        }
        if( !bOK )
        {
            GDALDestroyTPSTransformer(psInfo);
            return NULL;
        }
    }

    psInfo->nRefCount = 1;

    int nThreads = 1;
    if( nGCPCount > 100 )
    {
        const char* pszWarpThreads = CSLFetchNameValue(papszOptions, "NUM_THREADS");
        if (pszWarpThreads == NULL)
            pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
        if (EQUAL(pszWarpThreads, "ALL_CPUS"))
            nThreads = CPLGetNumCPUs();
        else
            nThreads = atoi(pszWarpThreads);
    }

    if( nThreads > 1 )
    {
        /* Compute direct and reverse transforms in parallel */
        CPLJoinableThread* hThread = CPLCreateJoinableThread(GDALTPSComputeForwardInThread, psInfo);
        psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
        if( hThread != NULL )
            CPLJoinThread(hThread);
        else
            psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
    }
    else
    {
        psInfo->bForwardSolved = psInfo->poForward->solve() != 0;
        psInfo->bReverseSolved = psInfo->poReverse->solve() != 0;
    }

    if( !psInfo->bForwardSolved || !psInfo->bReverseSolved )
    {
        GDALDestroyTPSTransformer(psInfo);
        return NULL;
    }

    return psInfo;
}
Пример #8
0
/*
** Fetch a nomads forecast from the NWS servers.  The forecasts consist of grib
** files, one forecast for each forecast hour.  The time starts and the nearest
** forecast hour *before* the reference time passed.  If no reference time is
** passed, or it is not valid, now() is used.
**
** The forecasts are downloaded into a temporary directory.  If the entire
** download succeeds, then the files are copied into the path specified.  If
** the path specified is a zip file (ends in *.zip), then the forecast files
** are zipped.
**
** Available compile time configuration options:
**        NOMADS_USE_IP: Use the ip address instead of the hostname.  It
**                       possibly goes around dns lookup, but it's doubtfull.
**        NOMADS_ENABLE_ASYNC: Allow for asynchronous connections to the
**                             server.
**        NOMADS_EXPER_FORECASTS: Compile in forecasts that may not work with
**                                the current configuration, essentially
**                                unsupported (ie NARRE, RTMA).
**        NOMADS_USE_VSI_READ: Use the VSI*L api for downloading.  This will
**                             allow definition of chunk sizes for download,
**                             although it is probably unnecessary.  See
**                             NOMADS_VSI_BLOCK_SIZE below.  If not enabled, a
**                             single fetch is made for each file, which is
**                             faster.
** Available runtime configuration options:
**        NOMADS_THREAD_COUNT: Number of threads to use for downloads if
**                             NOMADS_ENABLE_ASYNC is set to ON during
**                             compilation.  Default is 4.
**        NOMADS_VSI_BLOCK_SIZE: Number of bytes to request at a time when
**                               downloading files if NOMADS_USE_VSI_READ is
**                               set to ON during compilation. Default is 512.
**        NOMADS_MAX_FCST_REWIND: Number of forecast run time steps to go back
**                                to attempt to get a full time frame.
**        GDAL_HTTP_TIMEOUT: Timeout for HTTP requests in seconds.  We should
**                           be able to set this reasonably low.
**
** \param pszModelKey The name key of the model to use, ie "nam_conus".  For a
**                    listing of models, see nomads.ncep.gov or /see nomads.h
**
** \param pszRefTime The reference time to begin searching for forecasts from.
**                   The search starts at refrence time, then goes back until
**                   it hits a valid forecast time.  If the download cannot be
**                   complete, it will step back NOMADS_MAX_FCST_REWIND
**                   foreacst times to attempt to get a full forecast.
**
** \param nHours The extent of hours to download forecasts from the reference
**               time.  If we step back, we will still grab all forecasts up
**               until nHours from the reference time, not the forecast time.
**
** \param nStride The number of forecasts to skip in time steps.  For example,
**                if 12 hours of gfs is requested (normally 5 files/steps (0,
**                3, 6, 9, 12) with a stride of 2, you'd get 0, 6, 12 forecast
**                hours.
**
** \param padfBbox The bounding box of the request in WGS84 decimal degrees.
**                 Order is xmin, xmax, ymax, ymin.
**
** \param pszDstVsiPath The location to write the files.  If the location
**                      has an extension of ".zip", then the output files are
**                      written as a zip archive, otherwise to a path.
**
** \param papszOptions List of key=value options, unused.
**
** \param pfnProgress Optional progress function.
**
** \return NOMADS_OK(0) on success, NOMADS_ERR(1) otherwise.
*/
int NomadsFetch( const char *pszModelKey, const char *pszRefTime,
                 int nHours, int nStride, double *padfBbox,
                 const char *pszDstVsiPath, char ** papszOptions,
                 GDALProgressFunc pfnProgress )
{
    const char **ppszKey = NULL;
    int nFcstHour = 0;
    int *panRunHours = NULL;
    int i = 0;
    int j = 0;
    int k = 0;
    int t = 0;
    int rc = 0;
    char **papszDownloadUrls = NULL;
    char **papszOutputFiles = NULL;
    char **papszFinalFiles = NULL;
    int nFilesToGet = 0;
    const char *pszTmpDir;
    const char *pszConfigOpt;
    int nFcstTries;
    int nMaxFcstRewind;
    int nrc;
    int bZip;
    void **pThreads;
    int nThreads;
    const char *pszThreadCount;

    NomadsThreadData *pasData;
    nomads_utc *ref, *end, *fcst;
    nrc = NOMADS_OK;

    CPLDebug( "NOMADS", "Fetching data for bounding box: %lf, %lf, %lf, %lf",
              padfBbox[0], padfBbox[1], padfBbox[2], padfBbox[3] );

    ppszKey = NomadsFindModel( pszModelKey );
    if( ppszKey == NULL )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Could not find model key in nomads data" );
        return NOMADS_ERR;
    }

    NomadsUtcCreate( &ref );
    NomadsUtcCreate( &end );
    rc = NOMADS_OK;
    if( pszRefTime )
    {
        rc = NomadsUtcFromIsoFrmt( ref, pszRefTime );
    }
    if( rc != NOMADS_OK || pszRefTime == NULL )
    {
        NomadsUtcNow( ref );
    }
    NomadsUtcCopy( end, ref );
    NomadsUtcAddHours( end, nHours );

    /* Disable unneeded reading of entire directories, good speedup */
    CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE" );

    pszConfigOpt = CPLGetConfigOption( "NOMADS_HTTP_TIMEOUT", "20" );
    if( pszConfigOpt != NULL )
    {
        CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", pszConfigOpt );
    }

    nMaxFcstRewind = atoi( CPLGetConfigOption( "NOMADS_MAX_FCST_REWIND", "2" ) );
    if( nMaxFcstRewind < 1 || nMaxFcstRewind > 24 )
    {
        nMaxFcstRewind = 2;
    }
    /* Go back at least 3 for rap, as it may not get updated all the time. */
    if( EQUALN( pszModelKey, "rap", 3 ) || EQUALN( pszModelKey, "hrrr", 4 ) )
    {
        nMaxFcstRewind = nMaxFcstRewind > 3 ? nMaxFcstRewind : 3;
    }

#ifdef NOMADS_ENABLE_ASYNC
    pszThreadCount = CPLGetConfigOption( "NOMADS_THREAD_COUNT", "4" );
    nThreads = atoi( pszThreadCount );
    if( nThreads < 1 || nThreads > 96 )
    {
        nThreads = 4;
    }
    pThreads = CPLMalloc( sizeof( void * ) * nThreads );
    pasData = CPLMalloc( sizeof( NomadsThreadData ) * nThreads );
#else /* NOMADS_ENABLE_ASYNC */
    /* Unused variables, set to null to so free is no-op */
    nThreads = 1;
    pThreads = NULL;
    pasData = NULL;
#endif /* NOMADS_ENABLE_ASYNC */

    fcst = NULL;
    nFcstTries = 0;
    while( nFcstTries < nMaxFcstRewind )
    {
        nrc = NOMADS_OK;
        fcst = NomadsSetForecastTime( ppszKey, ref, nFcstTries );
        nFcstHour = fcst->ts->tm_hour;
        CPLDebug( "WINDNINJA", "Generated forecast time in utc: %s",
                  NomadsUtcStrfTime( fcst, "%Y%m%dT%HZ" ) );

        if( EQUAL( pszModelKey, "rtma_conus" ) )
        {
            panRunHours = (int*)CPLMalloc( sizeof( int ) );
            nFilesToGet = 1;
        }
        else
        {
            nFilesToGet =
                NomadsBuildForecastRunHours( ppszKey, fcst, end, nHours,
                                             nStride, &panRunHours );
        }

        papszDownloadUrls =
            NomadsBuildForecastFileList( pszModelKey, nFcstHour, panRunHours,
                                         nFilesToGet, fcst, padfBbox );
        if( papszDownloadUrls == NULL )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not generate list of URLs to download, invalid data" );
            nFcstTries++;
            NomadsUtcFree( fcst );
            fcst = NULL;
            nrc = NOMADS_ERR;
            continue;
        }
        pszTmpDir = CPLStrdup( CPLGenerateTempFilename( NULL ) );
        CPLDebug( "WINDNINJA", "Creating Temp directory: %s", pszTmpDir );
        VSIMkdir( pszTmpDir, 0777 );
        papszOutputFiles =
            NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
                                       nFilesToGet, pszTmpDir, FALSE );
        if( papszOutputFiles == NULL )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not generate list of URLs to download, invalid data" );
            nFcstTries++;
            CSLDestroy( papszDownloadUrls );
            NomadsUtcFree( fcst );
            fcst = NULL;
            nrc = NOMADS_ERR;
            continue;
        }

        CPLAssert( CSLCount( papszDownloadUrls ) == nFilesToGet );
        CPLAssert( CSLCount( papszOutputFiles ) == nFilesToGet );

        if( pfnProgress )
        {
            pfnProgress( 0.0, "Starting download...", NULL );
        }

        /* Download one file and start over if it's not there. */
#ifdef NOMADS_USE_VSI_READ
        nrc = NomadsFetchVsi( papszDownloadUrls[0], papszOutputFiles[0] );
#else /* NOMADS_USE_VSI_READ */
        nrc = NomadsFetchHttp( papszDownloadUrls[0], papszOutputFiles[0] );
#endif /* NOMADS_USE_VSI_READ */
        if( nrc != NOMADS_OK )
        {
            CPLError( CE_Warning, CPLE_AppDefined,
                      "Failed to download forecast, " \
                      "stepping back one forecast run time step." );
            nFcstTries++;
            CPLSleep( 1 );
            /*
            ** Don't explicitly break here.  We'll skip the while loop because
            ** nrc != NOMADS_OK, and we can clean up memory and shift times in
            ** one spot to avoid duplicate code.
            */
        }
        /* Get the rest */
        i = 1;
        while( i < nFilesToGet && nrc == NOMADS_OK )
        {
            if( pfnProgress )
            {
                if( pfnProgress( (double)i / nFilesToGet,
                                 CPLSPrintf( "Downloading %s...",
                                             CPLGetFilename( papszOutputFiles[i] ) ),
                                 NULL ) )
                {
                    CPLError( CE_Failure, CPLE_UserInterrupt,
                              "Cancelled by user." );
                    nrc = NOMADS_ERR;
                    nFcstTries = nMaxFcstRewind;
                    break;
                }
            }
#ifdef NOMADS_ENABLE_ASYNC
            k = i > nFilesToGet - nThreads ? (nFilesToGet - 1) % nThreads : nThreads;
            for( t = 0; t < k; t++ )
            {
                pasData[t].pszUrl = papszDownloadUrls[i];
                pasData[t].pszFilename = papszOutputFiles[i];
                pThreads[t] =
                    CPLCreateJoinableThread( NomadsFetchAsync, &pasData[t] );
                i++;
            }
            for( t = 0; t < k; t++ )
            {
                CPLJoinThread( pThreads[t] );
            }
            for( t = 0; t < k; t++ )
            {
                if( pasData[t].nErr )
                {
                    CPLError( CE_Warning, CPLE_AppDefined,
                              "Threaded download failed, attempting " \
                              "serial download for %s",
                              CPLGetFilename( pasData[t].pszFilename ) );
                    /* Try again, serially though */
                    if( CPLCheckForFile( (char *)pasData[t].pszFilename, NULL ) );
                    {
                        VSIUnlink( pasData[t].pszFilename );
                    }
#ifdef NOMADS_USE_VSI_READ
                    rc = NomadsFetchVsi( pasData[t].pszUrl,
                                         pasData[t].pszFilename );
#else /* NOMADS_USE_VSI_READ */
                    rc = NomadsFetchHttp( pasData[t].pszUrl,
                                          pasData[t].pszFilename );
#endif /* NOMADS_USE_VSI_READ */
                    if( rc != NOMADS_OK )
                    {
                        nrc = rc;
                    }
                }
            }
#else /* NOMADS_ENABLE_ASYNC */
#ifdef NOMADS_USE_VSI_READ
            nrc = NomadsFetchVsi( papszDownloadUrls[i], papszOutputFiles[i] );
#else /* NOMADS_USE_VSI_READ */
            nrc = NomadsFetchHttp( papszDownloadUrls[i], papszOutputFiles[i] );
#endif /* NOMADS_USE_VSI_READ */
            i++;
#endif /* NOMADS_ENABLE_ASYNC */
            if( nrc != NOMADS_OK )
            {
                CPLError( CE_Warning, CPLE_AppDefined,
                          "Failed to download forecast, " \
                          "stepping back one forecast run time step." );
                nFcstTries++;
                CPLSleep( 1 );
                break;
            }
        }
        /*
        ** XXX Cleanup XXX
        ** After each loop we can get rid of the urls, but we can only get rid
        ** of the others if they are to be reallocated in the next loop.  Any
        ** cleanup in the else nrc == NOMADS_OK clause should be cleaned up in 
        ** the nrc == NOMADS_OK outside the loop.  Those are held so we can
        ** process the output files and zip them into an archive.
        */
        CSLDestroy( papszDownloadUrls );
        NomadsUtcFree( fcst );
        if( nrc == NOMADS_OK )
        {
            break;
        }
        else
        {
            CPLFree( (void*)panRunHours );
            CSLDestroy( papszOutputFiles );
            CPLUnlinkTree( pszTmpDir );
            CPLFree( (void*)pszTmpDir );
        }
    }
    if( nrc == NOMADS_OK )
    {
        bZip = EQUAL( CPLGetExtension( pszDstVsiPath ), "zip" ) ? TRUE : FALSE;
        papszFinalFiles =
            NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
                                       nFilesToGet, pszDstVsiPath,
                                       bZip );
        CPLFree( (void*)panRunHours );
        if( CPLCheckForFile( (char*)pszDstVsiPath, NULL ) )
        {
            CPLUnlinkTree( pszDstVsiPath );
        }
        if( !bZip )
        {
            VSIMkdir( pszDstVsiPath, 0777 );
        }
        nrc = NomadsZipFiles( papszOutputFiles, papszFinalFiles );
        CSLDestroy( papszOutputFiles );
        CSLDestroy( papszFinalFiles );
        if( nrc != NOMADS_OK )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not copy files into path, unknown i/o failure" );
            CPLUnlinkTree( pszDstVsiPath );
        }
        CPLUnlinkTree( pszTmpDir );
        CPLFree( (void*)pszTmpDir );
    }
    CPLFree( (void*)pasData );
    CPLFree( (void**)pThreads );

    NomadsUtcFree( ref );
    NomadsUtcFree( end );
    CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", NULL );
    CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", NULL );
    if( nrc == NOMADS_OK && pfnProgress )
    {
        pfnProgress( 1.0, NULL, NULL );
    }
    return nrc;
}