void *DTEDCreatePtStream( const char *pszPath, int nLevel )
{
DTEDPtStream *psStream;
int i;
VSIStatBuf sStat;
/* -------------------------------------------------------------------- */
/* Does the target directory already exist? If not try to */
/* create it. */
/* -------------------------------------------------------------------- */
if( CPLStat( pszPath, &sStat ) != 0 )
{
if( VSIMkdir( pszPath, 0755 ) != 0 )
{
#ifndef AVOID_CPL
CPLError( CE_Failure, CPLE_OpenFailed,
"Unable to find, or create directory `%s'.",
pszPath );
#endif
return NULL;
}
}
/* -------------------------------------------------------------------- */
/* Create the stream and initialize it. */
/* -------------------------------------------------------------------- */
psStream = (DTEDPtStream *) CPLCalloc( sizeof(DTEDPtStream), 1 );
psStream->nLevel = nLevel;
psStream->pszPath = CPLStrdup( pszPath );
psStream->nOpenFiles = 0;
psStream->pasCF = NULL;
psStream->nLastFile = -1;
for( i = 0; i < DTEDMD_MAX+1; i++ )
psStream->apszMetadata[i] = NULL;
if( nLevel == 0 )
psStream->dfPixelSize = 1.0 / 120.0;
else if( nLevel == 1 )
psStream->dfPixelSize = 1.0 / 1200.0;
else if( nLevel == 2 )
psStream->dfPixelSize = 1.0 / 3600.0;
else
psStream->dfPixelSize = 1.0 / 3600.0;
return (void *) psStream;
}
int OGRTigerDataSource::Create( const char *pszNameIn, char **papszOptionsIn )
{
VSIStatBufL stat;
/* -------------------------------------------------------------------- */
/* Try to create directory if it doesn't already exist. */
/* -------------------------------------------------------------------- */
if( VSIStatL( pszNameIn, &stat ) != 0 )
{
VSIMkdir( pszNameIn, 0755 );
}
if( VSIStatL( pszNameIn, &stat ) != 0 || !VSI_ISDIR(stat.st_mode) )
{
CPLError( CE_Failure, CPLE_AppDefined,
"%s is not a directory, nor can be directly created as one.",
pszNameIn );
return FALSE;
}
/* -------------------------------------------------------------------- */
/* Store various information. */
/* -------------------------------------------------------------------- */
pszPath = CPLStrdup( pszNameIn );
pszName = CPLStrdup( pszNameIn );
bWriteMode = true;
SetOptionList( papszOptionsIn );
/* -------------------------------------------------------------------- */
/* Work out the version. */
/* -------------------------------------------------------------------- */
// nVersionCode = 1000; /* census 2000 */
nVersionCode = 1002; /* census 2002 */
if( GetOption("VERSION") != nullptr )
{
nVersionCode = atoi(GetOption("VERSION"));
nVersionCode = std::max(0, std::min(9999, nVersionCode));
}
nVersion = TigerClassifyVersion(nVersionCode);
return TRUE;
}
OGRDataSource *OGRCSVDriver::CreateDataSource( const char * pszName,
char ** /* papszOptions */ )
{
/* -------------------------------------------------------------------- */
/* First, ensure there isn't any such file yet. */
/* -------------------------------------------------------------------- */
VSIStatBuf sStatBuf;
if( VSIStat( pszName, &sStatBuf ) == 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"It seems a file system object called '%s' already exists.",
pszName );
return NULL;
}
/* -------------------------------------------------------------------- */
/* Create a directory. */
/* -------------------------------------------------------------------- */
if( VSIMkdir( pszName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to create directory %s:\n%s",
pszName, VSIStrerror( errno ) );
return NULL;
}
/* -------------------------------------------------------------------- */
/* Force it to open as a datasource. */
/* -------------------------------------------------------------------- */
OGRCSVDataSource *poDS = new OGRCSVDataSource();
if( !poDS->Open( pszName, TRUE, TRUE ) )
{
delete poDS;
return NULL;
}
return poDS;
}
int OGRTABDataSource::Create( const char * pszName, char **papszOptions )
{
CPLAssert(m_pszName == nullptr);
m_pszName = CPLStrdup(pszName);
m_papszOptions = CSLDuplicate(papszOptions);
m_bUpdate = TRUE;
const char *pszOpt = CSLFetchNameValue(papszOptions, "FORMAT");
if( pszOpt != nullptr && EQUAL(pszOpt, "MIF") )
m_bCreateMIF = TRUE;
else if( EQUAL(CPLGetExtension(pszName),"mif") ||
EQUAL(CPLGetExtension(pszName),"mid") )
m_bCreateMIF = TRUE;
if( (pszOpt = CSLFetchNameValue(papszOptions,"SPATIAL_INDEX_MODE")) != nullptr )
{
if( EQUAL(pszOpt, "QUICK") )
m_bQuickSpatialIndexMode = TRUE;
else if( EQUAL(pszOpt, "OPTIMIZED") )
m_bQuickSpatialIndexMode = FALSE;
}
m_nBlockSize = atoi(CSLFetchNameValueDef(papszOptions, "BLOCKSIZE", "512"));
// Create a new empty directory.
VSIStatBufL sStat;
if( strlen(CPLGetExtension(pszName)) == 0 )
{
if( VSIStatL( pszName, &sStat ) == 0 )
{
if( !VSI_ISDIR(sStat.st_mode) )
{
CPLError(CE_Failure, CPLE_OpenFailed,
"Attempt to create dataset named %s,\n"
"but that is an existing file.",
pszName);
return FALSE;
}
}
else
{
if( VSIMkdir(pszName, 0755) != 0 )
{
CPLError(CE_Failure, CPLE_AppDefined,
"Unable to create directory %s.",
pszName);
return FALSE;
}
}
m_pszDirectory = CPLStrdup(pszName);
}
// Create a new single file.
else
{
IMapInfoFile *poFile = nullptr;
const char *pszEncoding( CSLFetchNameValue( papszOptions, "ENCODING" ) );
const char *pszCharset( IMapInfoFile::EncodingToCharset( pszEncoding ) );
if( m_bCreateMIF )
{
poFile = new MIFFile;
if( poFile->Open(m_pszName, TABWrite, FALSE, pszCharset) != 0 )
{
delete poFile;
return FALSE;
}
}
else
{
TABFile *poTabFile = new TABFile;
if( poTabFile->Open(m_pszName, TABWrite, FALSE,
m_nBlockSize, pszCharset) != 0 )
{
delete poTabFile;
return FALSE;
}
poFile = poTabFile;
}
m_nLayerCount = 1;
m_papoLayers = static_cast<IMapInfoFile **>(CPLMalloc(sizeof(void *)));
m_papoLayers[0] = poFile;
m_pszDirectory = CPLStrdup(CPLGetPath(pszName));
m_bSingleFile = TRUE;
}
return TRUE;
}
int OGRGeoconceptDataSource::Create( const char *pszName, char** papszOptions )
{
const char *pszConf;
const char *pszExtension;
if( _pszName ) CPLFree(_pszName);
_papszOptions = CSLDuplicate( papszOptions );
pszConf= CSLFetchNameValue(papszOptions,"CONFIG");
if( pszConf != NULL )
{
_pszGCT = CPLStrdup(pszConf);
}
_pszExt = (char *)CSLFetchNameValue(papszOptions,"EXTENSION");
pszExtension = CSLFetchNameValue(papszOptions,"EXTENSION");
if( pszExtension == NULL )
{
_pszExt = CPLStrdup(CPLGetExtension(pszName));
}
else
{
_pszExt = CPLStrdup(pszExtension);
}
if( strlen(_pszExt) == 0 )
{
if( VSIMkdir( pszName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Directory %s already exists"
" as geoconcept datastore or"
" is made up of a non existing list of directories.",
pszName );
return FALSE;
}
_pszDirectory = CPLStrdup( pszName );
CPLFree(_pszExt);
_pszExt = CPLStrdup("gxt");
char *pszbName = CPLStrdup(CPLGetBasename( pszName ));
if (strlen(pszbName)==0) {/* pszName ends with '/' */
CPLFree(pszbName);
char *pszNameDup= CPLStrdup(pszName);
pszNameDup[strlen(pszName)-2] = '\0';
pszbName = CPLStrdup(CPLGetBasename( pszNameDup ));
CPLFree(pszNameDup);
}
_pszName = CPLStrdup((char *)CPLFormFilename( _pszDirectory, pszbName, NULL ));
CPLFree(pszbName);
}
else
{
_pszDirectory = CPLStrdup( CPLGetPath(pszName) );
_pszName = CPLStrdup( pszName );
}
/* -------------------------------------------------------------------- */
/* Create a new single file. */
/* OGRGeoconceptDriver::CreateLayer() will do the job. */
/* -------------------------------------------------------------------- */
_bSingleNewFile = TRUE;
if( !LoadFile( "wt" ) )
{
CPLDebug( "GEOCONCEPT",
"Failed to create Geoconcept %s.",
pszName );
return FALSE;
}
return TRUE;
}
int OGRTABDataSource::Create( const char * pszName, char **papszOptions )
{
VSIStatBuf sStat;
const char *pszOpt;
CPLAssert( m_pszName == NULL );
m_pszName = CPLStrdup( pszName );
m_papszOptions = CSLDuplicate( papszOptions );
if( (pszOpt=CSLFetchNameValue(papszOptions,"FORMAT")) != NULL
&& EQUAL(pszOpt, "MIF") )
m_bCreateMIF = TRUE;
else if( EQUAL(CPLGetExtension(pszName),"mif")
|| EQUAL(CPLGetExtension(pszName),"mid") )
m_bCreateMIF = TRUE;
if( (pszOpt=CSLFetchNameValue(papszOptions,"SPATIAL_INDEX_MODE")) != NULL
&& EQUAL(pszOpt, "QUICK") )
m_bQuickSpatialIndexMode = TRUE;
/* -------------------------------------------------------------------- */
/* Create a new empty directory. */
/* -------------------------------------------------------------------- */
if( strlen(CPLGetExtension(pszName)) == 0 )
{
if( VSIStat( pszName, &sStat ) == 0 )
{
if( !VSI_ISDIR(sStat.st_mode) )
{
CPLError( CE_Failure, CPLE_OpenFailed,
"Attempt to create dataset named %s,\n"
"but that is an existing file.\n",
pszName );
return FALSE;
}
}
else
{
if( VSIMkdir( pszName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Unable to create directory %s.\n",
pszName );
return FALSE;
}
}
m_pszDirectory = CPLStrdup(pszName);
}
/* -------------------------------------------------------------------- */
/* Create a new single file. */
/* -------------------------------------------------------------------- */
else
{
IMapInfoFile *poFile;
if( m_bCreateMIF )
poFile = new MIFFile;
else
poFile = new TABFile;
if( poFile->Open( pszName, "wb", FALSE ) != 0 )
{
delete poFile;
return FALSE;
}
m_nLayerCount = 1;
m_papoLayers = (IMapInfoFile **) CPLMalloc(sizeof(void*));
m_papoLayers[0] = poFile;
m_pszDirectory = CPLStrdup( CPLGetPath(pszName) );
m_bSingleFile = TRUE;
}
return TRUE;
}
int GNMFileNetwork::CheckNetworkExist(const char *pszFilename, char **papszOptions)
{
// check if path exist
// if path exist check if network already present and OVERWRITE option
// else create the path
bool bOverwrite = CSLFetchBoolean(papszOptions, "OVERWRITE", FALSE);
if(m_soName.empty())
{
const char* pszNetworkName = CSLFetchNameValue(papszOptions, GNM_MD_NAME);
if( NULL != pszNetworkName )
{
m_soName = pszNetworkName;
}
}
if(FormPath(pszFilename, papszOptions) != CE_None)
{
return TRUE;
}
if (CPLCheckForFile((char*)m_soNetworkFullName.c_str(), NULL))
{
char **papszFiles = CPLReadDir( m_soNetworkFullName );
if( CSLCount(papszFiles) == 0 )
{
return FALSE;
}
// search for base GNM files
for(int i = 0; papszFiles[i] != NULL; i++ )
{
if( EQUAL(papszFiles[i],".") || EQUAL(papszFiles[i],"..") )
continue;
if( EQUAL(CPLGetBasename(papszFiles[i]), GNM_SYSLAYER_META) ||
EQUAL(CPLGetBasename(papszFiles[i]), GNM_SYSLAYER_GRAPH) ||
EQUAL(CPLGetBasename(papszFiles[i]), GNM_SYSLAYER_FEATURES) ||
EQUAL(papszFiles[i], GNM_SRSFILENAME) )
{
if(bOverwrite)
{
const char* pszDeleteFile = CPLFormFilename(
m_soNetworkFullName, papszFiles[i], NULL);
CPLDebug("GNM", "Delete file: %s", pszDeleteFile);
if( VSIUnlink(pszDeleteFile) != 0 )
{
return TRUE;
}
}
else
{
return TRUE;
}
}
}
CSLDestroy( papszFiles );
}
else
{
if (VSIMkdir(m_soNetworkFullName, 0755) != 0)
{
return TRUE;
}
}
return FALSE;
}
OGRDataSource *OGRShapeDriver::CreateDataSource( const char * pszName,
CPL_UNUSED char **papszOptions )
{
VSIStatBuf stat;
int bSingleNewFile = FALSE;
/* -------------------------------------------------------------------- */
/* Is the target a valid existing directory? */
/* -------------------------------------------------------------------- */
if( CPLStat( pszName, &stat ) == 0 )
{
if( !VSI_ISDIR(stat.st_mode) )
{
CPLError( CE_Failure, CPLE_AppDefined,
"%s is not a directory.\n",
pszName );
return NULL;
}
}
/* -------------------------------------------------------------------- */
/* Does it end in the extension .shp indicating the user likely */
/* wants to create a single file set? */
/* -------------------------------------------------------------------- */
else if( EQUAL(CPLGetExtension(pszName),"shp")
|| EQUAL(CPLGetExtension(pszName),"dbf") )
{
bSingleNewFile = TRUE;
}
/* -------------------------------------------------------------------- */
/* Otherwise try to create a new directory. */
/* -------------------------------------------------------------------- */
else
{
if( VSIMkdir( pszName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to create directory %s\n"
"for shapefile datastore.\n",
pszName );
return NULL;
}
}
/* -------------------------------------------------------------------- */
/* Return a new OGRDataSource() */
/* -------------------------------------------------------------------- */
OGRShapeDataSource *poDS = NULL;
poDS = new OGRShapeDataSource();
if( !poDS->Open( pszName, TRUE, FALSE, bSingleNewFile ) )
{
delete poDS;
return NULL;
}
else
return poDS;
}
OGRErr FGdbDriver::StartTransaction(OGRDataSource*& poDSInOut, int& bOutHasReopenedDS)
{
CPLMutexHolderOptionalLockD(hMutex);
bOutHasReopenedDS = FALSE;
OGRMutexedDataSource* poMutexedDS = (OGRMutexedDataSource*)poDSInOut;
FGdbDataSource* poDS = (FGdbDataSource* )poMutexedDS->GetBaseDataSource();
if( !poDS->GetUpdate() )
return OGRERR_FAILURE;
FGdbDatabaseConnection* pConnection = poDS->GetConnection();
if( pConnection->GetRefCount() != 1 )
{
CPLError(CE_Failure, CPLE_AppDefined,
"Cannot start transaction as database is opened in another connection");
return OGRERR_FAILURE;
}
if( pConnection->IsLocked() )
{
CPLError(CE_Failure, CPLE_AppDefined,
"Transaction is already in progress");
return OGRERR_FAILURE;
}
bOutHasReopenedDS = TRUE;
CPLString osName(poMutexedDS->GetName());
CPLString osNameOri(osName);
if( osName[osName.size()-1] == '/' || osName[osName.size()-1] == '\\' )
osName.resize(osName.size()-1);
#ifndef WIN32
int bPerLayerCopyingForTransaction = poDS->HasPerLayerCopyingForTransaction();
#endif
pConnection->m_nRefCount ++;
delete poDSInOut;
poDSInOut = NULL;
poMutexedDS = NULL;
poDS = NULL;
pConnection->CloseGeodatabase();
CPLString osEditedName(osName);
osEditedName += ".ogredited";
CPLPushErrorHandler(CPLQuietErrorHandler);
CPL_IGNORE_RET_VAL(CPLUnlinkTree(osEditedName));
CPLPopErrorHandler();
OGRErr eErr = OGRERR_NONE;
CPLString osDatabaseToReopen;
#ifndef WIN32
if( bPerLayerCopyingForTransaction )
{
int bError = FALSE;
if( VSIMkdir( osEditedName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Cannot create directory '%s'.",
osEditedName.c_str() );
bError = TRUE;
}
// Only copy a0000000X.Y files with X >= 1 && X <= 8, gdb and timestamps
// and symlink others
char** papszFiles = VSIReadDir(osName);
for(char** papszIter = papszFiles; !bError && *papszIter; ++papszIter)
{
if( strcmp(*papszIter, ".") == 0 || strcmp(*papszIter, "..") == 0 )
continue;
if( ((*papszIter)[0] == 'a' && atoi((*papszIter)+1) >= 1 &&
atoi((*papszIter)+1) <= 8) || EQUAL(*papszIter, "gdb") ||
EQUAL(*papszIter, "timestamps") )
{
if( CPLCopyFile(CPLFormFilename(osEditedName, *papszIter, NULL),
CPLFormFilename(osName, *papszIter, NULL)) != 0 )
{
bError = TRUE;
CPLError(CE_Failure, CPLE_AppDefined,
"Cannot copy %s", *papszIter);
}
}
else
{
CPLString osSourceFile;
if( CPLIsFilenameRelative(osName) )
osSourceFile = CPLFormFilename(CPLSPrintf("../%s", CPLGetFilename(osName.c_str())), *papszIter, NULL);
else
osSourceFile = osName;
if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") ||
CPLSymlink( osSourceFile,
CPLFormFilename(osEditedName.c_str(), *papszIter, NULL),
NULL ) != 0 )
{
bError = TRUE;
CPLError(CE_Failure, CPLE_AppDefined,
"Cannot symlink %s", *papszIter);
}
}
}
CSLDestroy(papszFiles);
if( bError )
{
eErr = OGRERR_FAILURE;
osDatabaseToReopen = osName;
}
else
osDatabaseToReopen = osEditedName;
}
else
#endif
{
if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE1") ||
CPLCopyTree( osEditedName, osName ) != 0 )
{
CPLError(CE_Failure, CPLE_AppDefined,
"Cannot backup geodatabase");
eErr = OGRERR_FAILURE;
osDatabaseToReopen = osName;
}
else
osDatabaseToReopen = osEditedName;
}
pConnection->m_pGeodatabase = new Geodatabase;
long hr = ::OpenGeodatabase(StringToWString(osDatabaseToReopen), *(pConnection->m_pGeodatabase));
if( EQUAL(CPLGetConfigOption("FGDB_SIMUL_FAIL", ""), "CASE2") || FAILED(hr))
{
delete pConnection->m_pGeodatabase;
pConnection->m_pGeodatabase = NULL;
Release(osName);
GDBErr(hr, CPLSPrintf("Failed to open %s. Dataset should be closed",
osDatabaseToReopen.c_str()));
return OGRERR_FAILURE;
}
FGdbDataSource* pDS = new FGdbDataSource(this, pConnection);
pDS->Open(osDatabaseToReopen, TRUE, osNameOri);
#ifndef WIN32
if( eErr == OGRERR_NONE && bPerLayerCopyingForTransaction )
{
pDS->SetPerLayerCopyingForTransaction(bPerLayerCopyingForTransaction);
pDS->SetSymlinkFlagOnAllLayers();
}
#endif
poDSInOut = new OGRMutexedDataSource(pDS, TRUE, hMutex, TRUE);
if( eErr == OGRERR_NONE )
pConnection->SetLocked(TRUE);
return eErr;
}
OGRDataSource *OGRCSVDriver::CreateDataSource( const char * pszName,
char **papszOptions )
{
/* -------------------------------------------------------------------- */
/* First, ensure there isn't any such file yet. */
/* -------------------------------------------------------------------- */
VSIStatBufL sStatBuf;
if (strcmp(pszName, "/dev/stdout") == 0)
pszName = "/vsistdout/";
if( VSIStatL( pszName, &sStatBuf ) == 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"It seems a file system object called '%s' already exists.",
pszName );
return NULL;
}
/* -------------------------------------------------------------------- */
/* If the target is not a simple .csv then create it as a */
/* directory. */
/* -------------------------------------------------------------------- */
CPLString osDirName;
if( EQUAL(CPLGetExtension(pszName),"csv") )
{
osDirName = CPLGetPath(pszName);
if( osDirName == "" )
osDirName = ".";
}
else
{
if( strncmp(pszName, "/vsizip/", 8) == 0)
{
/* do nothing */
}
else if( !EQUAL(pszName, "/vsistdout/") &&
VSIMkdir( pszName, 0755 ) != 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to create directory %s:\n%s",
pszName, VSIStrerror( errno ) );
return NULL;
}
osDirName = pszName;
}
/* -------------------------------------------------------------------- */
/* Force it to open as a datasource. */
/* -------------------------------------------------------------------- */
OGRCSVDataSource *poDS = new OGRCSVDataSource();
if( !poDS->Open( osDirName, TRUE, TRUE ) )
{
delete poDS;
return NULL;
}
if( osDirName != pszName )
poDS->SetDefaultCSVName( CPLGetFilename(pszName) );
return poDS;
}
/*
** Fetch a nomads forecast from the NWS servers. The forecasts consist of grib
** files, one forecast for each forecast hour. The time starts and the nearest
** forecast hour *before* the reference time passed. If no reference time is
** passed, or it is not valid, now() is used.
**
** The forecasts are downloaded into a temporary directory. If the entire
** download succeeds, then the files are copied into the path specified. If
** the path specified is a zip file (ends in *.zip), then the forecast files
** are zipped.
**
** Available compile time configuration options:
** NOMADS_USE_IP: Use the ip address instead of the hostname. It
** possibly goes around dns lookup, but it's doubtfull.
** NOMADS_ENABLE_ASYNC: Allow for asynchronous connections to the
** server.
** NOMADS_EXPER_FORECASTS: Compile in forecasts that may not work with
** the current configuration, essentially
** unsupported (ie NARRE, RTMA).
** NOMADS_USE_VSI_READ: Use the VSI*L api for downloading. This will
** allow definition of chunk sizes for download,
** although it is probably unnecessary. See
** NOMADS_VSI_BLOCK_SIZE below. If not enabled, a
** single fetch is made for each file, which is
** faster.
** Available runtime configuration options:
** NOMADS_THREAD_COUNT: Number of threads to use for downloads if
** NOMADS_ENABLE_ASYNC is set to ON during
** compilation. Default is 4.
** NOMADS_VSI_BLOCK_SIZE: Number of bytes to request at a time when
** downloading files if NOMADS_USE_VSI_READ is
** set to ON during compilation. Default is 512.
** NOMADS_MAX_FCST_REWIND: Number of forecast run time steps to go back
** to attempt to get a full time frame.
** GDAL_HTTP_TIMEOUT: Timeout for HTTP requests in seconds. We should
** be able to set this reasonably low.
**
** \param pszModelKey The name key of the model to use, ie "nam_conus". For a
** listing of models, see nomads.ncep.gov or /see nomads.h
**
** \param pszRefTime The reference time to begin searching for forecasts from.
** The search starts at refrence time, then goes back until
** it hits a valid forecast time. If the download cannot be
** complete, it will step back NOMADS_MAX_FCST_REWIND
** foreacst times to attempt to get a full forecast.
**
** \param nHours The extent of hours to download forecasts from the reference
** time. If we step back, we will still grab all forecasts up
** until nHours from the reference time, not the forecast time.
**
** \param nStride The number of forecasts to skip in time steps. For example,
** if 12 hours of gfs is requested (normally 5 files/steps (0,
** 3, 6, 9, 12) with a stride of 2, you'd get 0, 6, 12 forecast
** hours.
**
** \param padfBbox The bounding box of the request in WGS84 decimal degrees.
** Order is xmin, xmax, ymax, ymin.
**
** \param pszDstVsiPath The location to write the files. If the location
** has an extension of ".zip", then the output files are
** written as a zip archive, otherwise to a path.
**
** \param papszOptions List of key=value options, unused.
**
** \param pfnProgress Optional progress function.
**
** \return NOMADS_OK(0) on success, NOMADS_ERR(1) otherwise.
*/
int NomadsFetch( const char *pszModelKey, const char *pszRefTime,
int nHours, int nStride, double *padfBbox,
const char *pszDstVsiPath, char ** papszOptions,
GDALProgressFunc pfnProgress )
{
const char **ppszKey = NULL;
int nFcstHour = 0;
int *panRunHours = NULL;
int i = 0;
int j = 0;
int k = 0;
int t = 0;
int rc = 0;
char **papszDownloadUrls = NULL;
char **papszOutputFiles = NULL;
char **papszFinalFiles = NULL;
int nFilesToGet = 0;
const char *pszTmpDir;
const char *pszConfigOpt;
int nFcstTries;
int nMaxFcstRewind;
int nrc;
int bZip;
void **pThreads;
int nThreads;
const char *pszThreadCount;
NomadsThreadData *pasData;
nomads_utc *ref, *end, *fcst;
nrc = NOMADS_OK;
CPLDebug( "NOMADS", "Fetching data for bounding box: %lf, %lf, %lf, %lf",
padfBbox[0], padfBbox[1], padfBbox[2], padfBbox[3] );
ppszKey = NomadsFindModel( pszModelKey );
if( ppszKey == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not find model key in nomads data" );
return NOMADS_ERR;
}
NomadsUtcCreate( &ref );
NomadsUtcCreate( &end );
rc = NOMADS_OK;
if( pszRefTime )
{
rc = NomadsUtcFromIsoFrmt( ref, pszRefTime );
}
if( rc != NOMADS_OK || pszRefTime == NULL )
{
NomadsUtcNow( ref );
}
NomadsUtcCopy( end, ref );
NomadsUtcAddHours( end, nHours );
/* Disable unneeded reading of entire directories, good speedup */
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE" );
pszConfigOpt = CPLGetConfigOption( "NOMADS_HTTP_TIMEOUT", "20" );
if( pszConfigOpt != NULL )
{
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", pszConfigOpt );
}
nMaxFcstRewind = atoi( CPLGetConfigOption( "NOMADS_MAX_FCST_REWIND", "2" ) );
if( nMaxFcstRewind < 1 || nMaxFcstRewind > 24 )
{
nMaxFcstRewind = 2;
}
/* Go back at least 3 for rap, as it may not get updated all the time. */
if( EQUALN( pszModelKey, "rap", 3 ) || EQUALN( pszModelKey, "hrrr", 4 ) )
{
nMaxFcstRewind = nMaxFcstRewind > 3 ? nMaxFcstRewind : 3;
}
#ifdef NOMADS_ENABLE_ASYNC
pszThreadCount = CPLGetConfigOption( "NOMADS_THREAD_COUNT", "4" );
nThreads = atoi( pszThreadCount );
if( nThreads < 1 || nThreads > 96 )
{
nThreads = 4;
}
pThreads = CPLMalloc( sizeof( void * ) * nThreads );
pasData = CPLMalloc( sizeof( NomadsThreadData ) * nThreads );
#else /* NOMADS_ENABLE_ASYNC */
/* Unused variables, set to null to so free is no-op */
nThreads = 1;
pThreads = NULL;
pasData = NULL;
#endif /* NOMADS_ENABLE_ASYNC */
fcst = NULL;
nFcstTries = 0;
while( nFcstTries < nMaxFcstRewind )
{
nrc = NOMADS_OK;
fcst = NomadsSetForecastTime( ppszKey, ref, nFcstTries );
nFcstHour = fcst->ts->tm_hour;
CPLDebug( "WINDNINJA", "Generated forecast time in utc: %s",
NomadsUtcStrfTime( fcst, "%Y%m%dT%HZ" ) );
if( EQUAL( pszModelKey, "rtma_conus" ) )
{
panRunHours = (int*)CPLMalloc( sizeof( int ) );
nFilesToGet = 1;
}
else
{
nFilesToGet =
NomadsBuildForecastRunHours( ppszKey, fcst, end, nHours,
nStride, &panRunHours );
}
papszDownloadUrls =
NomadsBuildForecastFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, fcst, padfBbox );
if( papszDownloadUrls == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
pszTmpDir = CPLStrdup( CPLGenerateTempFilename( NULL ) );
CPLDebug( "WINDNINJA", "Creating Temp directory: %s", pszTmpDir );
VSIMkdir( pszTmpDir, 0777 );
papszOutputFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszTmpDir, FALSE );
if( papszOutputFiles == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
CPLAssert( CSLCount( papszDownloadUrls ) == nFilesToGet );
CPLAssert( CSLCount( papszOutputFiles ) == nFilesToGet );
if( pfnProgress )
{
pfnProgress( 0.0, "Starting download...", NULL );
}
/* Download one file and start over if it's not there. */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[0], papszOutputFiles[0] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[0], papszOutputFiles[0] );
#endif /* NOMADS_USE_VSI_READ */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
/*
** Don't explicitly break here. We'll skip the while loop because
** nrc != NOMADS_OK, and we can clean up memory and shift times in
** one spot to avoid duplicate code.
*/
}
/* Get the rest */
i = 1;
while( i < nFilesToGet && nrc == NOMADS_OK )
{
if( pfnProgress )
{
if( pfnProgress( (double)i / nFilesToGet,
CPLSPrintf( "Downloading %s...",
CPLGetFilename( papszOutputFiles[i] ) ),
NULL ) )
{
CPLError( CE_Failure, CPLE_UserInterrupt,
"Cancelled by user." );
nrc = NOMADS_ERR;
nFcstTries = nMaxFcstRewind;
break;
}
}
#ifdef NOMADS_ENABLE_ASYNC
k = i > nFilesToGet - nThreads ? (nFilesToGet - 1) % nThreads : nThreads;
for( t = 0; t < k; t++ )
{
pasData[t].pszUrl = papszDownloadUrls[i];
pasData[t].pszFilename = papszOutputFiles[i];
pThreads[t] =
CPLCreateJoinableThread( NomadsFetchAsync, &pasData[t] );
i++;
}
for( t = 0; t < k; t++ )
{
CPLJoinThread( pThreads[t] );
}
for( t = 0; t < k; t++ )
{
if( pasData[t].nErr )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Threaded download failed, attempting " \
"serial download for %s",
CPLGetFilename( pasData[t].pszFilename ) );
/* Try again, serially though */
if( CPLCheckForFile( (char *)pasData[t].pszFilename, NULL ) );
{
VSIUnlink( pasData[t].pszFilename );
}
#ifdef NOMADS_USE_VSI_READ
rc = NomadsFetchVsi( pasData[t].pszUrl,
pasData[t].pszFilename );
#else /* NOMADS_USE_VSI_READ */
rc = NomadsFetchHttp( pasData[t].pszUrl,
pasData[t].pszFilename );
#endif /* NOMADS_USE_VSI_READ */
if( rc != NOMADS_OK )
{
nrc = rc;
}
}
}
#else /* NOMADS_ENABLE_ASYNC */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[i], papszOutputFiles[i] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[i], papszOutputFiles[i] );
#endif /* NOMADS_USE_VSI_READ */
i++;
#endif /* NOMADS_ENABLE_ASYNC */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
break;
}
}
/*
** XXX Cleanup XXX
** After each loop we can get rid of the urls, but we can only get rid
** of the others if they are to be reallocated in the next loop. Any
** cleanup in the else nrc == NOMADS_OK clause should be cleaned up in
** the nrc == NOMADS_OK outside the loop. Those are held so we can
** process the output files and zip them into an archive.
*/
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
if( nrc == NOMADS_OK )
{
break;
}
else
{
CPLFree( (void*)panRunHours );
CSLDestroy( papszOutputFiles );
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
}
if( nrc == NOMADS_OK )
{
bZip = EQUAL( CPLGetExtension( pszDstVsiPath ), "zip" ) ? TRUE : FALSE;
papszFinalFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszDstVsiPath,
bZip );
CPLFree( (void*)panRunHours );
if( CPLCheckForFile( (char*)pszDstVsiPath, NULL ) )
{
CPLUnlinkTree( pszDstVsiPath );
}
if( !bZip )
{
VSIMkdir( pszDstVsiPath, 0777 );
}
nrc = NomadsZipFiles( papszOutputFiles, papszFinalFiles );
CSLDestroy( papszOutputFiles );
CSLDestroy( papszFinalFiles );
if( nrc != NOMADS_OK )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not copy files into path, unknown i/o failure" );
CPLUnlinkTree( pszDstVsiPath );
}
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
CPLFree( (void*)pasData );
CPLFree( (void**)pThreads );
NomadsUtcFree( ref );
NomadsUtcFree( end );
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", NULL );
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", NULL );
if( nrc == NOMADS_OK && pfnProgress )
{
pfnProgress( 1.0, NULL, NULL );
}
return nrc;
}
