Exemplo n.º 1
0
OGRErr OGRCSVDriver::DeleteDataSource( const char *pszFilename )

{
    if( CPLUnlinkTree( pszFilename ) == 0 )
        return OGRERR_NONE;
    else
        return OGRERR_FAILURE;
}
Exemplo n.º 2
0
OGRErr FGdbDriver::RollbackTransaction(OGRDataSource*& poDSInOut, int& bOutHasReopenedDS)
{
    CPLMutexHolderOptionalLockD(hMutex);

    bOutHasReopenedDS = FALSE;

    OGRMutexedDataSource* poMutexedDS = (OGRMutexedDataSource*)poDSInOut;
    FGdbDataSource* poDS = (FGdbDataSource* )poMutexedDS->GetBaseDataSource();
    FGdbDatabaseConnection* pConnection = poDS->GetConnection();
    if( !pConnection->IsLocked() )
    {
        CPLError(CE_Failure, CPLE_NotSupported,
                 "No transaction in progress");
        return OGRERR_FAILURE;
    }

    bOutHasReopenedDS = TRUE;

    CPLString osName(poMutexedDS->GetName());
    CPLString osNameOri(osName);
    if( osName[osName.size()-1] == '/' || osName[osName.size()-1] == '\\' )
        osName.resize(osName.size()-1);

    //int bPerLayerCopyingForTransaction = poDS->HasPerLayerCopyingForTransaction();

    pConnection->m_nRefCount ++;
    delete poDSInOut;
    poDSInOut = NULL;
    poMutexedDS = NULL;
    poDS = NULL;

    pConnection->CloseGeodatabase();

    CPLString osEditedName(osName);
    osEditedName += ".ogredited";

    OGRErr eErr = OGRERR_NONE;
    if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") || 
        CPLUnlinkTree(osEditedName) != 0 )
    {
        CPLError(CE_Warning, CPLE_AppDefined,
                 "Cannot remove %s. Manual cleanup required", osEditedName.c_str());
        eErr = OGRERR_FAILURE;
    }

    pConnection->m_pGeodatabase = new Geodatabase;
    long hr = ::OpenGeodatabase(StringToWString(osName), *(pConnection->m_pGeodatabase));
    if (EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE2") ||
        FAILED(hr))
    {
        delete pConnection->m_pGeodatabase;
        pConnection->m_pGeodatabase = NULL;
        pConnection->SetLocked(FALSE);
        Release(osName);
        GDBErr(hr, "Failed to re-open Geodatabase. Dataset should be closed");
        return OGRERR_FAILURE;
    }

    FGdbDataSource* pDS = new FGdbDataSource(this, pConnection);
    pDS->Open(osNameOri, TRUE, NULL);
    //pDS->SetPerLayerCopyingForTransaction(bPerLayerCopyingForTransaction);
    poDSInOut = new OGRMutexedDataSource(pDS, TRUE, hMutex, TRUE);

    pConnection->SetLocked(FALSE);

    return eErr;
}
Exemplo n.º 3
0
OGRErr FGdbDriver::StartTransaction(OGRDataSource*& poDSInOut, int& bOutHasReopenedDS)
{
    CPLMutexHolderOptionalLockD(hMutex);

    bOutHasReopenedDS = FALSE;

    OGRMutexedDataSource* poMutexedDS = (OGRMutexedDataSource*)poDSInOut;
    FGdbDataSource* poDS = (FGdbDataSource* )poMutexedDS->GetBaseDataSource();
    if( !poDS->GetUpdate() )
        return OGRERR_FAILURE;
    FGdbDatabaseConnection* pConnection = poDS->GetConnection();
    if( pConnection->GetRefCount() != 1 )
    {
        CPLError(CE_Failure, CPLE_AppDefined,
                 "Cannot start transaction as database is opened in another connection");
        return OGRERR_FAILURE;
    }
    if( pConnection->IsLocked() )
    {
        CPLError(CE_Failure, CPLE_AppDefined,
                 "Transaction is already in progress");
        return OGRERR_FAILURE;
    }

    bOutHasReopenedDS = TRUE;

    CPLString osName(poMutexedDS->GetName());
    CPLString osNameOri(osName);
    if( osName[osName.size()-1] == '/' || osName[osName.size()-1] == '\\' )
        osName.resize(osName.size()-1);

#ifndef WIN32
    int bPerLayerCopyingForTransaction = poDS->HasPerLayerCopyingForTransaction();
#endif

    pConnection->m_nRefCount ++;
    delete poDSInOut;
    poDSInOut = NULL;
    poMutexedDS = NULL;
    poDS = NULL;

    pConnection->CloseGeodatabase();

    CPLString osEditedName(osName);
    osEditedName += ".ogredited";

    CPLPushErrorHandler(CPLQuietErrorHandler);
    CPL_IGNORE_RET_VAL(CPLUnlinkTree(osEditedName));
    CPLPopErrorHandler();

    OGRErr eErr = OGRERR_NONE;
    
    CPLString osDatabaseToReopen;
#ifndef WIN32
    if( bPerLayerCopyingForTransaction )
    {
        int bError = FALSE;
        
        if( VSIMkdir( osEditedName, 0755 ) != 0 )
        {
            CPLError( CE_Failure, CPLE_AppDefined, 
                      "Cannot create directory '%s'.",
                      osEditedName.c_str() );
            bError = TRUE;
        }

        // Only copy a0000000X.Y files with X >= 1 && X <= 8, gdb and timestamps
        // and symlink others
        char** papszFiles = VSIReadDir(osName);
        for(char** papszIter = papszFiles; !bError && *papszIter; ++papszIter)
        {
            if( strcmp(*papszIter, ".") == 0 || strcmp(*papszIter, "..") == 0 )
                continue;
            if( ((*papszIter)[0] == 'a' && atoi((*papszIter)+1) >= 1 &&
                 atoi((*papszIter)+1) <= 8) || EQUAL(*papszIter, "gdb") ||
                 EQUAL(*papszIter, "timestamps") )
            {
                if( CPLCopyFile(CPLFormFilename(osEditedName, *papszIter, NULL),
                                CPLFormFilename(osName, *papszIter, NULL)) != 0 )
                {
                    bError = TRUE;
                    CPLError(CE_Failure, CPLE_AppDefined,
                             "Cannot copy %s", *papszIter);
                }
            }
            else
            {
                CPLString osSourceFile;
                if( CPLIsFilenameRelative(osName) )
                    osSourceFile = CPLFormFilename(CPLSPrintf("../%s", CPLGetFilename(osName.c_str())), *papszIter, NULL);
                else
                    osSourceFile = osName;
                if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") ||
                    CPLSymlink( osSourceFile,
                                CPLFormFilename(osEditedName.c_str(), *papszIter, NULL),
                                NULL ) != 0 )
                {
                    bError = TRUE;
                    CPLError(CE_Failure, CPLE_AppDefined,
                             "Cannot symlink %s", *papszIter);
                }
            }
        }
        CSLDestroy(papszFiles);

        if( bError )
        {
            eErr = OGRERR_FAILURE;
            osDatabaseToReopen = osName;
        }
        else
            osDatabaseToReopen = osEditedName;
    }
    else
#endif
    {
        if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") ||
            CPLCopyTree( osEditedName, osName ) != 0 )
        {
            CPLError(CE_Failure, CPLE_AppDefined,
                    "Cannot backup geodatabase");
            eErr = OGRERR_FAILURE;
            osDatabaseToReopen = osName;
        }
        else
            osDatabaseToReopen = osEditedName;
    }

    pConnection->m_pGeodatabase = new Geodatabase;
    long hr = ::OpenGeodatabase(StringToWString(osDatabaseToReopen), *(pConnection->m_pGeodatabase));
    if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE2") || FAILED(hr))
    {
        delete pConnection->m_pGeodatabase;
        pConnection->m_pGeodatabase = NULL;
        Release(osName);
        GDBErr(hr, CPLSPrintf("Failed to open %s. Dataset should be closed",
                              osDatabaseToReopen.c_str()));

        return OGRERR_FAILURE;
    }

    FGdbDataSource* pDS = new FGdbDataSource(this, pConnection);
    pDS->Open(osDatabaseToReopen, TRUE, osNameOri);

#ifndef WIN32
    if( eErr == OGRERR_NONE && bPerLayerCopyingForTransaction )
    {
        pDS->SetPerLayerCopyingForTransaction(bPerLayerCopyingForTransaction);
        pDS->SetSymlinkFlagOnAllLayers();
    }
#endif

    poDSInOut = new OGRMutexedDataSource(pDS, TRUE, hMutex, TRUE);

    if( eErr == OGRERR_NONE )
        pConnection->SetLocked(TRUE);
    return eErr;
}
Exemplo n.º 4
0
OGRErr FGdbDriver::CommitTransaction(OGRDataSource*& poDSInOut, int& bOutHasReopenedDS)
{
    CPLMutexHolderOptionalLockD(hMutex);

    bOutHasReopenedDS = FALSE;


    OGRMutexedDataSource* poMutexedDS = (OGRMutexedDataSource*)poDSInOut;
    FGdbDataSource* poDS = (FGdbDataSource* )poMutexedDS->GetBaseDataSource();
    FGdbDatabaseConnection* pConnection = poDS->GetConnection();
    if( !pConnection->IsLocked() )
    {
        CPLError(CE_Failure, CPLE_NotSupported,
                 "No transaction in progress");
        return OGRERR_FAILURE;
    }

    bOutHasReopenedDS = TRUE;

    CPLString osName(poMutexedDS->GetName());
    CPLString osNameOri(osName);
    if( osName[osName.size()-1] == '/' || osName[osName.size()-1] == '\\' )
        osName.resize(osName.size()-1);

#ifndef WIN32
    int bPerLayerCopyingForTransaction = poDS->HasPerLayerCopyingForTransaction();
#endif

    pConnection->m_nRefCount ++;
    delete poDSInOut;
    poDSInOut = NULL;
    poMutexedDS = NULL;
    poDS = NULL;

    pConnection->CloseGeodatabase();

    CPLString osEditedName(osName);
    osEditedName += ".ogredited";
    
#ifndef WIN32
    if( bPerLayerCopyingForTransaction )
    {
        int bError = FALSE;
        char** papszFiles;
        std::vector<CPLString> aosTmpFilesToClean;
        
        // Check for files present in original copy that are not in edited copy
        // That is to say deleted layers
        papszFiles = VSIReadDir(osName);
        for(char** papszIter = papszFiles; !bError && *papszIter; ++papszIter)
        {
            if( strcmp(*papszIter, ".") == 0 || strcmp(*papszIter, "..") == 0 )
                continue;
            VSIStatBufL sStat;
            if( (*papszIter)[0] == 'a' &&
                VSIStatL( CPLFormFilename(osEditedName, *papszIter, NULL), &sStat ) != 0 )
            {
                if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") || 
                    VSIRename( CPLFormFilename(osName, *papszIter, NULL),
                               CPLFormFilename(osName, *papszIter, "tmp") ) != 0 )
                {
                    CPLError(CE_Failure, CPLE_AppDefined, "Cannot rename %s to %s",
                             CPLFormFilename(osName, *papszIter, NULL),
                             CPLFormFilename(osName, *papszIter, "tmp"));
                    bError = TRUE;
                }
                else
                    aosTmpFilesToClean.push_back(CPLFormFilename(osName, *papszIter, "tmp"));
            }
        }
        CSLDestroy(papszFiles);

        // Move modified files from edited directory to main directory
        papszFiles = VSIReadDir(osEditedName);
        for(char** papszIter = papszFiles; !bError && *papszIter; ++papszIter)
        {
            if( strcmp(*papszIter, ".") == 0 || strcmp(*papszIter, "..") == 0 )
                continue;
            struct stat sStat;
            if( lstat( CPLFormFilename(osEditedName, *papszIter, NULL), &sStat ) != 0 )
            {
                CPLError(CE_Failure, CPLE_AppDefined, "Cannot stat %s",
                         CPLFormFilename(osEditedName, *papszIter, NULL));
                bError = TRUE;
            }
            else if( !S_ISLNK(sStat.st_mode) )
            {
                // If there was such a file in original directory, first rename it
                // as a temporary file
                if( lstat( CPLFormFilename(osName, *papszIter, NULL), &sStat ) == 0 )
                {
                    if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE2") || 
                        VSIRename( CPLFormFilename(osName, *papszIter, NULL),
                                   CPLFormFilename(osName, *papszIter, "tmp") ) != 0 )
                    {
                        CPLError(CE_Failure, CPLE_AppDefined, "Cannot rename %s to %s",
                                 CPLFormFilename(osName, *papszIter, NULL),
                                 CPLFormFilename(osName, *papszIter, "tmp"));
                        bError = TRUE;
                    }
                    else
                        aosTmpFilesToClean.push_back(CPLFormFilename(osName, *papszIter, "tmp"));
                }
                if( !bError )
                {
                    if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE3") || 
                        CPLMoveFile( CPLFormFilename(osName, *papszIter, NULL),
                                     CPLFormFilename(osEditedName, *papszIter, NULL) ) != 0 )
                    {
                        CPLError(CE_Failure, CPLE_AppDefined, "Cannot move %s to %s",
                                 CPLFormFilename(osEditedName, *papszIter, NULL),
                                 CPLFormFilename(osName, *papszIter, NULL));
                        bError = TRUE;
                    }
                    else
                        CPLDebug("FileGDB", "Move %s to %s",
                                 CPLFormFilename(osEditedName, *papszIter, NULL),
                                 CPLFormFilename(osName, *papszIter, NULL));
                }
            }
        }
        CSLDestroy(papszFiles);

        if( !bError )
        {
            for(size_t i=0;i<aosTmpFilesToClean.size();i++)
            {
                if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE4") || 
                    VSIUnlink(aosTmpFilesToClean[i]) != 0 )
                {
                    CPLError(CE_Warning, CPLE_AppDefined,
                             "Cannot remove %s. Manual cleanup required", aosTmpFilesToClean[i].c_str());
                }
            }
        }

        if( bError )
        {
            CPLError(CE_Failure, CPLE_AppDefined,
                     "An error occurred while moving files from %s back to %s. "
                     "Manual cleaning must be done and dataset should be closed",
                     osEditedName.c_str(),
                     osName.c_str());
            pConnection->SetLocked(FALSE);
            Release(osName);
            return OGRERR_FAILURE;
        }
        else if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE5") || 
                 CPLUnlinkTree(osEditedName) != 0 )
        {
            CPLError(CE_Warning, CPLE_AppDefined,
                    "Cannot remove %s. Manual cleanup required", osEditedName.c_str());
        }
    }
    else
#endif
    {
        CPLString osTmpName(osName);
        osTmpName += ".ogrtmp";
        
        /* Install the backup copy as the main database in 3 steps : */
        /* first rename the main directory  in .tmp */
        /* then rename the edited copy under regular name */
        /* and finally dispose the .tmp directory */
        /* That way there's no risk definitely losing data */
        if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") || 
            VSIRename(osName, osTmpName) != 0 )
        {
            CPLError(CE_Failure, CPLE_AppDefined,
                    "Cannot rename %s to %s. Edited database during transaction is in %s"
                    "Dataset should be closed",
                    osName.c_str(), osTmpName.c_str(), osEditedName.c_str());
            pConnection->SetLocked(FALSE);
            Release(osName);
            return OGRERR_FAILURE;
        }
        
        if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE2") || 
            VSIRename(osEditedName, osName) != 0 )
        {
            CPLError(CE_Failure, CPLE_AppDefined,
                    "Cannot rename %s to %s. The original geodatabase is in '%s'. "
                    "Dataset should be closed",
                    osEditedName.c_str(), osName.c_str(), osTmpName.c_str());
            pConnection->SetLocked(FALSE);
            Release(osName);
            return OGRERR_FAILURE;
        }

        if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE3") || 
            CPLUnlinkTree(osTmpName) != 0 )
        {
            CPLError(CE_Warning, CPLE_AppDefined,
                    "Cannot remove %s. Manual cleanup required", osTmpName.c_str());
        }
    }

    pConnection->m_pGeodatabase = new Geodatabase;
    long hr = ::OpenGeodatabase(StringToWString(osName), *(pConnection->m_pGeodatabase));
    if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE_REOPEN") || FAILED(hr))
    {
        delete pConnection->m_pGeodatabase;
        pConnection->m_pGeodatabase = NULL;
        pConnection->SetLocked(FALSE);
        Release(osName);
        GDBErr(hr, "Failed to re-open Geodatabase. Dataset should be closed");
        return OGRERR_FAILURE;
    }

    FGdbDataSource* pDS = new FGdbDataSource(this, pConnection);
    pDS->Open(osNameOri, TRUE, NULL);
    //pDS->SetPerLayerCopyingForTransaction(bPerLayerCopyingForTransaction);
    poDSInOut = new OGRMutexedDataSource(pDS, TRUE, hMutex, TRUE);

    pConnection->SetLocked(FALSE);

    return OGRERR_NONE;
}
Exemplo n.º 5
0
/*
** Fetch a nomads forecast from the NWS servers.  The forecasts consist of grib
** files, one forecast for each forecast hour.  The time starts and the nearest
** forecast hour *before* the reference time passed.  If no reference time is
** passed, or it is not valid, now() is used.
**
** The forecasts are downloaded into a temporary directory.  If the entire
** download succeeds, then the files are copied into the path specified.  If
** the path specified is a zip file (ends in *.zip), then the forecast files
** are zipped.
**
** Available compile time configuration options:
**        NOMADS_USE_IP: Use the ip address instead of the hostname.  It
**                       possibly goes around dns lookup, but it's doubtfull.
**        NOMADS_ENABLE_ASYNC: Allow for asynchronous connections to the
**                             server.
**        NOMADS_EXPER_FORECASTS: Compile in forecasts that may not work with
**                                the current configuration, essentially
**                                unsupported (ie NARRE, RTMA).
**        NOMADS_USE_VSI_READ: Use the VSI*L api for downloading.  This will
**                             allow definition of chunk sizes for download,
**                             although it is probably unnecessary.  See
**                             NOMADS_VSI_BLOCK_SIZE below.  If not enabled, a
**                             single fetch is made for each file, which is
**                             faster.
** Available runtime configuration options:
**        NOMADS_THREAD_COUNT: Number of threads to use for downloads if
**                             NOMADS_ENABLE_ASYNC is set to ON during
**                             compilation.  Default is 4.
**        NOMADS_VSI_BLOCK_SIZE: Number of bytes to request at a time when
**                               downloading files if NOMADS_USE_VSI_READ is
**                               set to ON during compilation. Default is 512.
**        NOMADS_MAX_FCST_REWIND: Number of forecast run time steps to go back
**                                to attempt to get a full time frame.
**        GDAL_HTTP_TIMEOUT: Timeout for HTTP requests in seconds.  We should
**                           be able to set this reasonably low.
**
** \param pszModelKey The name key of the model to use, ie "nam_conus".  For a
**                    listing of models, see nomads.ncep.gov or /see nomads.h
**
** \param pszRefTime The reference time to begin searching for forecasts from.
**                   The search starts at refrence time, then goes back until
**                   it hits a valid forecast time.  If the download cannot be
**                   complete, it will step back NOMADS_MAX_FCST_REWIND
**                   foreacst times to attempt to get a full forecast.
**
** \param nHours The extent of hours to download forecasts from the reference
**               time.  If we step back, we will still grab all forecasts up
**               until nHours from the reference time, not the forecast time.
**
** \param nStride The number of forecasts to skip in time steps.  For example,
**                if 12 hours of gfs is requested (normally 5 files/steps (0,
**                3, 6, 9, 12) with a stride of 2, you'd get 0, 6, 12 forecast
**                hours.
**
** \param padfBbox The bounding box of the request in WGS84 decimal degrees.
**                 Order is xmin, xmax, ymax, ymin.
**
** \param pszDstVsiPath The location to write the files.  If the location
**                      has an extension of ".zip", then the output files are
**                      written as a zip archive, otherwise to a path.
**
** \param papszOptions List of key=value options, unused.
**
** \param pfnProgress Optional progress function.
**
** \return NOMADS_OK(0) on success, NOMADS_ERR(1) otherwise.
*/
int NomadsFetch( const char *pszModelKey, const char *pszRefTime,
                 int nHours, int nStride, double *padfBbox,
                 const char *pszDstVsiPath, char ** papszOptions,
                 GDALProgressFunc pfnProgress )
{
    const char **ppszKey = NULL;
    int nFcstHour = 0;
    int *panRunHours = NULL;
    int i = 0;
    int j = 0;
    int k = 0;
    int t = 0;
    int rc = 0;
    char **papszDownloadUrls = NULL;
    char **papszOutputFiles = NULL;
    char **papszFinalFiles = NULL;
    int nFilesToGet = 0;
    const char *pszTmpDir;
    const char *pszConfigOpt;
    int nFcstTries;
    int nMaxFcstRewind;
    int nrc;
    int bZip;
    void **pThreads;
    int nThreads;
    const char *pszThreadCount;

    NomadsThreadData *pasData;
    nomads_utc *ref, *end, *fcst;
    nrc = NOMADS_OK;

    CPLDebug( "NOMADS", "Fetching data for bounding box: %lf, %lf, %lf, %lf",
              padfBbox[0], padfBbox[1], padfBbox[2], padfBbox[3] );

    ppszKey = NomadsFindModel( pszModelKey );
    if( ppszKey == NULL )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Could not find model key in nomads data" );
        return NOMADS_ERR;
    }

    NomadsUtcCreate( &ref );
    NomadsUtcCreate( &end );
    rc = NOMADS_OK;
    if( pszRefTime )
    {
        rc = NomadsUtcFromIsoFrmt( ref, pszRefTime );
    }
    if( rc != NOMADS_OK || pszRefTime == NULL )
    {
        NomadsUtcNow( ref );
    }
    NomadsUtcCopy( end, ref );
    NomadsUtcAddHours( end, nHours );

    /* Disable unneeded reading of entire directories, good speedup */
    CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE" );

    pszConfigOpt = CPLGetConfigOption( "NOMADS_HTTP_TIMEOUT", "20" );
    if( pszConfigOpt != NULL )
    {
        CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", pszConfigOpt );
    }

    nMaxFcstRewind = atoi( CPLGetConfigOption( "NOMADS_MAX_FCST_REWIND", "2" ) );
    if( nMaxFcstRewind < 1 || nMaxFcstRewind > 24 )
    {
        nMaxFcstRewind = 2;
    }
    /* Go back at least 3 for rap, as it may not get updated all the time. */
    if( EQUALN( pszModelKey, "rap", 3 ) || EQUALN( pszModelKey, "hrrr", 4 ) )
    {
        nMaxFcstRewind = nMaxFcstRewind > 3 ? nMaxFcstRewind : 3;
    }

#ifdef NOMADS_ENABLE_ASYNC
    pszThreadCount = CPLGetConfigOption( "NOMADS_THREAD_COUNT", "4" );
    nThreads = atoi( pszThreadCount );
    if( nThreads < 1 || nThreads > 96 )
    {
        nThreads = 4;
    }
    pThreads = CPLMalloc( sizeof( void * ) * nThreads );
    pasData = CPLMalloc( sizeof( NomadsThreadData ) * nThreads );
#else /* NOMADS_ENABLE_ASYNC */
    /* Unused variables, set to null to so free is no-op */
    nThreads = 1;
    pThreads = NULL;
    pasData = NULL;
#endif /* NOMADS_ENABLE_ASYNC */

    fcst = NULL;
    nFcstTries = 0;
    while( nFcstTries < nMaxFcstRewind )
    {
        nrc = NOMADS_OK;
        fcst = NomadsSetForecastTime( ppszKey, ref, nFcstTries );
        nFcstHour = fcst->ts->tm_hour;
        CPLDebug( "WINDNINJA", "Generated forecast time in utc: %s",
                  NomadsUtcStrfTime( fcst, "%Y%m%dT%HZ" ) );

        if( EQUAL( pszModelKey, "rtma_conus" ) )
        {
            panRunHours = (int*)CPLMalloc( sizeof( int ) );
            nFilesToGet = 1;
        }
        else
        {
            nFilesToGet =
                NomadsBuildForecastRunHours( ppszKey, fcst, end, nHours,
                                             nStride, &panRunHours );
        }

        papszDownloadUrls =
            NomadsBuildForecastFileList( pszModelKey, nFcstHour, panRunHours,
                                         nFilesToGet, fcst, padfBbox );
        if( papszDownloadUrls == NULL )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not generate list of URLs to download, invalid data" );
            nFcstTries++;
            NomadsUtcFree( fcst );
            fcst = NULL;
            nrc = NOMADS_ERR;
            continue;
        }
        pszTmpDir = CPLStrdup( CPLGenerateTempFilename( NULL ) );
        CPLDebug( "WINDNINJA", "Creating Temp directory: %s", pszTmpDir );
        VSIMkdir( pszTmpDir, 0777 );
        papszOutputFiles =
            NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
                                       nFilesToGet, pszTmpDir, FALSE );
        if( papszOutputFiles == NULL )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not generate list of URLs to download, invalid data" );
            nFcstTries++;
            CSLDestroy( papszDownloadUrls );
            NomadsUtcFree( fcst );
            fcst = NULL;
            nrc = NOMADS_ERR;
            continue;
        }

        CPLAssert( CSLCount( papszDownloadUrls ) == nFilesToGet );
        CPLAssert( CSLCount( papszOutputFiles ) == nFilesToGet );

        if( pfnProgress )
        {
            pfnProgress( 0.0, "Starting download...", NULL );
        }

        /* Download one file and start over if it's not there. */
#ifdef NOMADS_USE_VSI_READ
        nrc = NomadsFetchVsi( papszDownloadUrls[0], papszOutputFiles[0] );
#else /* NOMADS_USE_VSI_READ */
        nrc = NomadsFetchHttp( papszDownloadUrls[0], papszOutputFiles[0] );
#endif /* NOMADS_USE_VSI_READ */
        if( nrc != NOMADS_OK )
        {
            CPLError( CE_Warning, CPLE_AppDefined,
                      "Failed to download forecast, " \
                      "stepping back one forecast run time step." );
            nFcstTries++;
            CPLSleep( 1 );
            /*
            ** Don't explicitly break here.  We'll skip the while loop because
            ** nrc != NOMADS_OK, and we can clean up memory and shift times in
            ** one spot to avoid duplicate code.
            */
        }
        /* Get the rest */
        i = 1;
        while( i < nFilesToGet && nrc == NOMADS_OK )
        {
            if( pfnProgress )
            {
                if( pfnProgress( (double)i / nFilesToGet,
                                 CPLSPrintf( "Downloading %s...",
                                             CPLGetFilename( papszOutputFiles[i] ) ),
                                 NULL ) )
                {
                    CPLError( CE_Failure, CPLE_UserInterrupt,
                              "Cancelled by user." );
                    nrc = NOMADS_ERR;
                    nFcstTries = nMaxFcstRewind;
                    break;
                }
            }
#ifdef NOMADS_ENABLE_ASYNC
            k = i > nFilesToGet - nThreads ? (nFilesToGet - 1) % nThreads : nThreads;
            for( t = 0; t < k; t++ )
            {
                pasData[t].pszUrl = papszDownloadUrls[i];
                pasData[t].pszFilename = papszOutputFiles[i];
                pThreads[t] =
                    CPLCreateJoinableThread( NomadsFetchAsync, &pasData[t] );
                i++;
            }
            for( t = 0; t < k; t++ )
            {
                CPLJoinThread( pThreads[t] );
            }
            for( t = 0; t < k; t++ )
            {
                if( pasData[t].nErr )
                {
                    CPLError( CE_Warning, CPLE_AppDefined,
                              "Threaded download failed, attempting " \
                              "serial download for %s",
                              CPLGetFilename( pasData[t].pszFilename ) );
                    /* Try again, serially though */
                    if( CPLCheckForFile( (char *)pasData[t].pszFilename, NULL ) );
                    {
                        VSIUnlink( pasData[t].pszFilename );
                    }
#ifdef NOMADS_USE_VSI_READ
                    rc = NomadsFetchVsi( pasData[t].pszUrl,
                                         pasData[t].pszFilename );
#else /* NOMADS_USE_VSI_READ */
                    rc = NomadsFetchHttp( pasData[t].pszUrl,
                                          pasData[t].pszFilename );
#endif /* NOMADS_USE_VSI_READ */
                    if( rc != NOMADS_OK )
                    {
                        nrc = rc;
                    }
                }
            }
#else /* NOMADS_ENABLE_ASYNC */
#ifdef NOMADS_USE_VSI_READ
            nrc = NomadsFetchVsi( papszDownloadUrls[i], papszOutputFiles[i] );
#else /* NOMADS_USE_VSI_READ */
            nrc = NomadsFetchHttp( papszDownloadUrls[i], papszOutputFiles[i] );
#endif /* NOMADS_USE_VSI_READ */
            i++;
#endif /* NOMADS_ENABLE_ASYNC */
            if( nrc != NOMADS_OK )
            {
                CPLError( CE_Warning, CPLE_AppDefined,
                          "Failed to download forecast, " \
                          "stepping back one forecast run time step." );
                nFcstTries++;
                CPLSleep( 1 );
                break;
            }
        }
        /*
        ** XXX Cleanup XXX
        ** After each loop we can get rid of the urls, but we can only get rid
        ** of the others if they are to be reallocated in the next loop.  Any
        ** cleanup in the else nrc == NOMADS_OK clause should be cleaned up in 
        ** the nrc == NOMADS_OK outside the loop.  Those are held so we can
        ** process the output files and zip them into an archive.
        */
        CSLDestroy( papszDownloadUrls );
        NomadsUtcFree( fcst );
        if( nrc == NOMADS_OK )
        {
            break;
        }
        else
        {
            CPLFree( (void*)panRunHours );
            CSLDestroy( papszOutputFiles );
            CPLUnlinkTree( pszTmpDir );
            CPLFree( (void*)pszTmpDir );
        }
    }
    if( nrc == NOMADS_OK )
    {
        bZip = EQUAL( CPLGetExtension( pszDstVsiPath ), "zip" ) ? TRUE : FALSE;
        papszFinalFiles =
            NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
                                       nFilesToGet, pszDstVsiPath,
                                       bZip );
        CPLFree( (void*)panRunHours );
        if( CPLCheckForFile( (char*)pszDstVsiPath, NULL ) )
        {
            CPLUnlinkTree( pszDstVsiPath );
        }
        if( !bZip )
        {
            VSIMkdir( pszDstVsiPath, 0777 );
        }
        nrc = NomadsZipFiles( papszOutputFiles, papszFinalFiles );
        CSLDestroy( papszOutputFiles );
        CSLDestroy( papszFinalFiles );
        if( nrc != NOMADS_OK )
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "Could not copy files into path, unknown i/o failure" );
            CPLUnlinkTree( pszDstVsiPath );
        }
        CPLUnlinkTree( pszTmpDir );
        CPLFree( (void*)pszTmpDir );
    }
    CPLFree( (void*)pasData );
    CPLFree( (void**)pThreads );

    NomadsUtcFree( ref );
    NomadsUtcFree( end );
    CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", NULL );
    CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", NULL );
    if( nrc == NOMADS_OK && pfnProgress )
    {
        pfnProgress( 1.0, NULL, NULL );
    }
    return nrc;
}