int GDALRasterBlock::FlushCacheBlock( int bDirtyBlocksOnly )
{
GDALRasterBlock *poTarget;
{
INITIALIZE_LOCK;
poTarget = poOldest;
while( poTarget != NULL )
{
if( !bDirtyBlocksOnly || poTarget->GetDirty() )
{
if( CPLAtomicCompareAndExchange(
&(poTarget->nLockCount), 0, -1) )
break;
}
poTarget = poTarget->poPrevious;
}
if( poTarget == NULL )
return FALSE;
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption(
"GDAL_RB_FLUSHBLOCK_SLEEP_AFTER_DROP_LOCK", "0")));
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget);
}
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption("GDAL_RB_FLUSHBLOCK_SLEEP_AFTER_RB_LOCK", "0")));
if( poTarget->GetDirty() )
{
const CPLErr eErr = poTarget->Write();
if( eErr != CE_None )
{
// Save the error for later reporting.
poTarget->GetBand()->SetFlushBlockErr(eErr);
}
}
VSIFree(poTarget->pData);
poTarget->pData = NULL;
poTarget->GetBand()->AddBlockToFreeList(poTarget);
return TRUE;
}
bool OGRAmigoCloudDataSource::waitForJobToFinish(const char* jobId)
{
std::stringstream url;
url << std::string(GetAPIURL()) << "/me/jobs/" << std::string(jobId);
int count = 0;
while (count<5) {
count++;
json_object *result = RunGET(url.str().c_str());
if (result == nullptr) {
CPLError(CE_Failure, CPLE_AppDefined, "waitForJobToFinish failed.");
return false;
}
if (result != nullptr) {
int type = json_object_get_type(result);
if (type == json_type_object) {
json_object *poStatus = CPL_json_object_object_get(result, "status");
const char *status = json_object_get_string(poStatus);
if (status != nullptr) {
if (std::string(status) == "SUCCESS") {
return true;
} else if (std::string(status) == "FAILURE") {
CPLError(CE_Failure, CPLE_AppDefined, "Job failed : %s", json_object_get_string(result));
return false;
}
}
}
}
CPLSleep(1.0); // Sleep 1 sec.
}
return false;
}
int main(int argc, char* argv[])
{
CPLJoinableThread* hThread;
printf("main thread %p\n", (void*)CPLGetPID());
argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
CPLSetConfigOption("GDAL_CACHEMAX", "0");
CPLSetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "ON");
MyDataset* poDS = new MyDataset();
char buf1[] = { 1 } ;
CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 0, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
hThread = CPLCreateJoinableThread(thread_func, NULL);
CPLSleep(0.3);
CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 1, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
GDALFlushCacheBlock();
CPLJoinThread(hThread);
delete poDS;
GDALDestroyDriverManager();
CSLDestroy( argv );
return 0;
}
int GDALRasterBlock::TakeLock()
{
const int nLockVal = AddLock();
CPLAssert(nLockVal >= 0);
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption("GDAL_RB_TRYGET_SLEEP_AFTER_TAKE_LOCK", "0")));
if( nLockVal == 0 )
{
#ifdef DEBUG
CPLDebug(
"GDAL",
"TakeLock(%p): Block(%d,%d,%p) is being evicted while trying to "
"reacquire it.",
reinterpret_cast<void *>(CPLGetPID()), nXOff, nYOff, poBand );
#endif
// The block is being evicted by GDALRasterBlock::Internalize()
// or FlushCacheBlock(), so wait for this to be done before trying
// again.
DropLock();
// wait for the block having been unreferenced
TAKE_LOCK;
return FALSE;
}
Touch();
return TRUE;
}
CPLErr IWriteBlock(int nXBlock, int nYBlock, void*)
{
printf("Entering IWriteBlock(%d, %d)\n", nXBlock, nYBlock);
assert(!bBusy);
bBusy = TRUE;
CPLSleep(0.5);
bBusy = FALSE;
printf("Leaving IWriteBlock(%d, %d)\n", nXBlock, nYBlock);
return CE_None;
}
void *CPLLockFile( const char *pszPath, double dfWaitInSeconds )
{
FILE *fpLock;
char *pszLockFilename;
/* -------------------------------------------------------------------- */
/* We use a lock file with a name derived from the file we want */
/* to lock to represent the file being locked. Note that for */
/* the stub implementation the target file does not even need */
/* to exist to be locked. */
/* -------------------------------------------------------------------- */
pszLockFilename = (char *) CPLMalloc(strlen(pszPath) + 30);
sprintf( pszLockFilename, "%s.lock", pszPath );
fpLock = fopen( pszLockFilename, "r" );
while( fpLock != NULL && dfWaitInSeconds > 0.0 )
{
fclose( fpLock );
CPLSleep( MIN(dfWaitInSeconds,0.5) );
dfWaitInSeconds -= 0.5;
fpLock = fopen( pszLockFilename, "r" );
}
if( fpLock != NULL )
{
fclose( fpLock );
CPLFree( pszLockFilename );
return NULL;
}
fpLock = fopen( pszLockFilename, "w" );
if( fpLock == NULL )
{
CPLFree( pszLockFilename );
return NULL;
}
fwrite( "held\n", 1, 5, fpLock );
fclose( fpLock );
return pszLockFilename;
}
void *CPLLockFile( const char *pszPath, double dfWaitInSeconds )
{
char *pszLockFilename;
HANDLE hLockFile;
pszLockFilename = (char *) CPLMalloc(strlen(pszPath) + 30);
sprintf( pszLockFilename, "%s.lock", pszPath );
hLockFile =
CreateFile( pszLockFilename, GENERIC_WRITE, 0, NULL,CREATE_NEW,
FILE_ATTRIBUTE_NORMAL|FILE_FLAG_DELETE_ON_CLOSE, NULL );
while( GetLastError() == ERROR_ALREADY_EXISTS
&& dfWaitInSeconds > 0.0 )
{
CloseHandle( hLockFile );
CPLSleep( MIN(dfWaitInSeconds,0.125) );
dfWaitInSeconds -= 0.125;
hLockFile =
CreateFile( pszLockFilename, GENERIC_WRITE, 0, NULL, CREATE_NEW,
FILE_ATTRIBUTE_NORMAL|FILE_FLAG_DELETE_ON_CLOSE,
NULL );
}
CPLFree( pszLockFilename );
if( hLockFile == INVALID_HANDLE_VALUE )
return NULL;
if( GetLastError() == ERROR_ALREADY_EXISTS )
{
CloseHandle( hLockFile );
return NULL;
}
return (void *) hLockFile;
}
int CPLAcquireMutex( void *hMutexIn, double dfWaitInSeconds )
{
#ifdef USE_WIN32_MUTEX
HANDLE hMutex = (HANDLE) hMutexIn;
DWORD hr;
hr = WaitForSingleObject( hMutex, (int) (dfWaitInSeconds * 1000) );
return hr != WAIT_TIMEOUT;
#else
CRITICAL_SECTION *pcs = (CRITICAL_SECTION *)hMutexIn;
BOOL ret;
while( (ret = TryEnterCriticalSection(pcs)) == 0 && dfWaitInSeconds > 0.0 )
{
CPLSleep( MIN(dfWaitInSeconds,0.125) );
dfWaitInSeconds -= 0.125;
}
return ret;
#endif
}
GDALAsyncStatusType
ECWAsyncReader::GetNextUpdatedRegion( double dfTimeout,
int* pnXBufOff, int* pnYBufOff,
int* pnXBufSize, int* pnYBufSize )
{
CPLDebug( "ECW", "GetNextUpdatedRegion()" );
/* -------------------------------------------------------------------- */
/* We always mark the whole raster as updated since the ECW SDK */
/* does not have a concept of partial update notifications. */
/* -------------------------------------------------------------------- */
*pnXBufOff = 0;
*pnYBufOff = 0;
*pnXBufSize = nBufXSize;
*pnYBufSize = nBufYSize;
if( bComplete && !bUpdateReady )
{
CPLDebug( "ECW", "return GARIO_COMPLETE" );
return GARIO_COMPLETE;
}
/* -------------------------------------------------------------------- */
/* Wait till our timeout, or until we are notified there is */
/* data ready. We are trusting the CPLSleep() to be pretty */
/* accurate instead of keeping track of time elapsed ourselves */
/* - this is not necessarily a good approach. */
/* -------------------------------------------------------------------- */
if( dfTimeout < 0.0 )
dfTimeout = 100000.0;
while( !bUpdateReady && dfTimeout > 0.0 )
{
CPLSleep( MIN(0.1, dfTimeout) );
dfTimeout -= 0.1;
CPLDebug( "ECW", "wait..." );
}
if( !bUpdateReady )
{
CPLDebug( "ECW", "return GARIO_PENDING" );
return GARIO_PENDING;
}
bUpdateReady = FALSE;
/* -------------------------------------------------------------------- */
/* Acquire Mutex */
/* -------------------------------------------------------------------- */
if( !CPLAcquireMutex( hMutex, dfTimeout ) )
{
CPLDebug( "ECW", "return GARIO_PENDING" );
return GARIO_PENDING;
}
/* -------------------------------------------------------------------- */
/* Actually decode the imagery into our buffer. */
/* -------------------------------------------------------------------- */
NCSEcwReadStatus eRStatus = ReadToBuffer();
if( eRStatus != NCSECW_READ_OK )
{
CPLReleaseMutex( hMutex );
return GARIO_ERROR;
}
/* -------------------------------------------------------------------- */
/* Return indication of complete or just buffer updateded. */
/* -------------------------------------------------------------------- */
if( bComplete && !bUpdateReady )
{
CPLReleaseMutex( hMutex );
CPLDebug( "ECW", "return GARIO_COMPLETE" );
return GARIO_COMPLETE;
}
else
{
CPLReleaseMutex( hMutex );
CPLDebug( "ECW", "return GARIO_UPDATE" );
return GARIO_UPDATE;
}
}
bool OGRGMELayer::CreateTableIfNotCreated()
{
if (!bCreateTablePending || (osTableId.size() != 0)) {
CPLDebug("GME", "Not creating table since already created");
CPLDebug("GME", "bCreateTablePending = %d osTableId ='%s'",
bCreateTablePending, osTableId.c_str());
return true;
}
CPLDebug("GME", "Creating table...");
json_object *pjoCreateDoc = json_object_new_object();
json_object *pjoProjectId = json_object_new_string( osProjectId.c_str() );
json_object_object_add( pjoCreateDoc, "projectId", pjoProjectId );
json_object *pjoName = json_object_new_string( osTableName.c_str() );
json_object_object_add( pjoCreateDoc, "name", pjoName );
json_object *pjoDraftACL = json_object_new_string( osDraftACL.c_str() );
json_object_object_add( pjoCreateDoc, "draftAccessList", pjoDraftACL );
json_object *pjoPublishedACL = json_object_new_string( osPublishedACL.c_str() );
json_object_object_add( pjoCreateDoc, "publishedAccessList", pjoPublishedACL );
json_object *pjoSchema = json_object_new_object();
json_object *pjoColumns = json_object_new_array();
poFeatureDefn->SetGeomType( eGTypeForCreation );
json_object *pjoGeometryColumn = json_object_new_object();
json_object *pjoGeometryName = json_object_new_string( "geometry" );
json_object *pjoGeometryType;
switch(eGTypeForCreation) {
case wkbPoint:
case wkbPoint25D:
case wkbMultiPoint:
case wkbMultiPoint25D:
pjoGeometryType = json_object_new_string( "points" );
break;
case wkbLineString:
case wkbLineString25D:
case wkbMultiLineString:
case wkbLinearRing:
case wkbMultiLineString25D:
pjoGeometryType = json_object_new_string( "lineStrings" );
break;
case wkbPolygon:
case wkbPolygon25D:
case wkbMultiPolygon:
case wkbGeometryCollection:
case wkbMultiPolygon25D:
pjoGeometryType = json_object_new_string( "polygons" );
break;
case wkbGeometryCollection25D:
pjoGeometryType = json_object_new_string( "mixedGeometry" );
break;
default:
CPLError(CE_Failure, CPLE_AppDefined,
"Unsupported Geometry type. Defaulting to Points");
pjoGeometryType = json_object_new_string( "points" );
poFeatureDefn->SetGeomType( wkbPoint );
}
json_object_object_add( pjoGeometryColumn, "name", pjoGeometryName );
json_object_object_add( pjoGeometryColumn, "type", pjoGeometryType );
json_object_array_add( pjoColumns, pjoGeometryColumn );
for (int iOGRField = 0; iOGRField < poFeatureDefn->GetFieldCount(); iOGRField++ )
{
if ((iOGRField == iGxIdField) && (iGxIdField >= 0))
continue; // don't create the gx_id field.
const char *pszFieldName = poFeatureDefn->GetFieldDefn(iOGRField)->GetNameRef();
if (EQUAL(pszFieldName, "gx_id")) {
iGxIdField = iOGRField;
continue;
}
json_object *pjoColumn = json_object_new_object();
json_object *pjoFieldName =
json_object_new_string( pszFieldName );
json_object *pjoFieldType;
switch(poFeatureDefn->GetFieldDefn(iOGRField)->GetType()) {
case OFTInteger:
pjoFieldType = json_object_new_string( "integer" );
break;
case OFTReal:
pjoFieldType = json_object_new_string( "double" );
break;
default:
pjoFieldType = json_object_new_string( "string" );
}
json_object_object_add( pjoColumn, "name", pjoFieldName );
json_object_object_add( pjoColumn, "type", pjoFieldType );
json_object_array_add( pjoColumns, pjoColumn );
}
json_object_object_add( pjoSchema, "columns", pjoColumns );
json_object_object_add( pjoCreateDoc, "schema", pjoSchema );
const char *body =
json_object_to_json_string_ext(pjoCreateDoc,
JSON_C_TO_STRING_SPACED | JSON_C_TO_STRING_PRETTY);
CPLDebug("GME", "Create Table Doc:\n%s", body);
/* -------------------------------------------------------------------- */
/* POST changes */
/* -------------------------------------------------------------------- */
CPLString osRequest = "tables";
CPLHTTPResult *poCreateResult = poDS->PostRequest(osRequest, body);
if( poCreateResult == NULL || poCreateResult->pabyData == NULL )
{
CPLError(CE_Failure, CPLE_AppDefined,
"Table creation failed.");
if( poCreateResult )
CPLHTTPDestroyResult(poCreateResult);
return false;
}
CPLDebug("GME", "CreateTable returned %d\n%s", poCreateResult->nStatus,
poCreateResult->pabyData);
json_object *pjoResponseDoc = OGRGMEParseJSON((const char *) poCreateResult->pabyData);
osTableId = OGRGMEGetJSONString(pjoResponseDoc, "id", "");
CPLHTTPDestroyResult(poCreateResult);
if (osTableId.size() == 0) {
CPLError(CE_Failure, CPLE_AppDefined,
"Table creation failed, or could not find table id.");
return false;
}
/*
OGRFieldDefn *poGxIdField = new OGRFieldDefn("gx_id", OFTString);
poFeatureDefn->AddFieldDefn(poGxIdField);
iGxIdField = poFeatureDefn->GetFieldCount() - 1;
CPLDebug("GME", "create field %s(%d) of type %s",
"gx_id", iGxIdField, OGRFieldDefn::GetFieldTypeName(OFTString));
*/
bCreateTablePending = false;
CPLDebug("GME", "sleeping 3s to give GME time to create the table...");
CPLSleep( 3.0 );
return true;
}
SURF_FETCH_E LandfireClient::FetchBoundingBox( double *bbox, double resolution,
const char *filename,
char **options )
{
(void)resolution;
if( NULL == filename )
{
return SURF_FETCH_E_BAD_INPUT;
}
/*
** We have an arbitrary limit on request size, 0.001 degrees
*/
if( fabs( bbox[0] - bbox[2] ) < 0.001 || fabs( bbox[1] - bbox[3] ) < 0.001 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Bounding box too small, must be greater than " \
"0.001 x 0.001 degrees." );
return SURF_FETCH_E_BAD_INPUT;
}
/*-----------------------------------------------------------------------------
* Local Variable Declarations
*-----------------------------------------------------------------------------*/
int i = 0;
char *p;
int nMaxTries = atoi( CPLGetConfigOption( "LCP_MAX_DOWNLOAD_TRIES", "40" ) );
double dfWait = atof( CPLGetConfigOption( "LCP_DOWNLOAD_WAIT", "3" ) );
const char *pszProduct = CPLStrdup( CSLFetchNameValue( options, "PRODUCT" ) );
/*
** Stupidly simple heuristics to try to get the 'correct' product.
*/
if( pszProduct == NULL || EQUAL( pszProduct, "" ) )
{
if( EQUAL( pszProduct, "" ) )
CPLFree( (void*)pszProduct );
std::string osDataPath = FindDataPath( "landfire.zip" );
osDataPath = "/vsizip/" + osDataPath;
const char *pszGeom;
pszGeom = CPLSPrintf( "POLYGON((%lf %lf,%lf %lf,%lf %lf,%lf %lf,%lf %lf))",
bbox[1], bbox[0], bbox[3], bbox[0],
bbox[3], bbox[2], bbox[1], bbox[2],
bbox[1], bbox[0] );
CPLDebug( "LCP_CLIENT", "Testing if %s contains %s", osDataPath.c_str(),
pszGeom );
if( NinjaOGRContain( pszGeom, osDataPath.c_str(), "conus" ) )
{
pszProduct = CPLStrdup( "F4W21HZ" );
}
else if( NinjaOGRContain( pszGeom, osDataPath.c_str(), "ak" ) )
{
pszProduct = CPLStrdup( "F7C29HZ" );
}
else if( NinjaOGRContain( pszGeom, osDataPath.c_str(), "hi" ) )
{
pszProduct = CPLStrdup( "F4825HZ" );
}
/* Contiguous US */
//if( bbox[0] < 52 && bbox[1] < -60 && bbox[2] > 22 && bbox[3] > -136 )
// pszProduct = CPLStrdup( "F4W21HZ" );
/* Alaska */
//else if( bbox[0] < 75 && bbox[1] < -125 && bbox[2] > 50 && bbox[3] > -179 )
// pszProduct = CPLStrdup( "F7C29HZ" );
/* Hawaii */
//else if( bbox[0] < 25 && bbox[1] < -150 && bbox[2] > 15 && bbox[3] > -170 )
// pszProduct = CPLStrdup( "F4825HZ" );
else
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to locate product." );
return SURF_FETCH_E_BAD_INPUT;
}
}
CPLDebug( "LCP_CLIENT", "Using product: %s", pszProduct );
const char *pszUrl;
const char *pszTemp = CSLFetchNameValue( options, "OVERRIDE_BEST_UTM" );
int nEpsgCode = -1;
if( pszTemp == NULL )
{
nEpsgCode = BoundingBoxUtm( bbox );
}
else
{
nEpsgCode = atoi( pszTemp );
}
/*
** Better check?
*/
if( nEpsgCode < 1 )
{
CPLError( CE_Failure, CPLE_AppDefined, "Invalid EPSG code." );
CPLFree( (void*)pszProduct );
return SURF_FETCH_E_BAD_INPUT;
}
/*-----------------------------------------------------------------------------
* Request a Model via the landfire.cr.usgs.gov REST client
*-----------------------------------------------------------------------------*/
pszUrl = CPLSPrintf( LF_REQUEST_TEMPLATE, bbox[0], bbox[2], bbox[3],
bbox[1], pszProduct );
CPLFree( (void*)pszProduct );
m_poResult = CPLHTTPFetch( pszUrl, NULL );
CHECK_HTTP_RESULT( "Failed to get download URL" );
CPLDebug( "LCP_CLIENT", "Request URL: %s", pszUrl );
/*-----------------------------------------------------------------------------
* Parse the JSON result of the request
*-----------------------------------------------------------------------------*/
int nSize = strlen( (char*) m_poResult->pabyData );
//Create a buffer so we can use sscanf, couldn't find a CPL version
char *pszResponse = new char[ nSize + 1 ];
pszResponse[0] = '\0';
CPLDebug( "LCP_CLIENT", "JSON Response: %s", m_poResult->pabyData );
/*
** Regular expression support if we have C++11 support. isnan ambiguouity
** is causing non-C++11 compliance. Not tested or used, 0 disables.
*/
#if __cplusplus >= 201103 && 0
std::string s((const char*) m_poResult->pabyData );
std::smatch m;
std::regex e( "\\b(?:(?:https?)://|www\\.)[a-z-A-Z0-9+&@#/%=~_|$?!:,.]" \
"*[a-z-A-Z0-9+&@#/%=~_|$]" );
std::regex_search( s, m, e );
std::string url = m[0].str(); //retrieve first match
CPLStrlcpy( pszResponse, url.c_str(), nSize );
#else
char **papszTokens = NULL;
papszTokens = CSLTokenizeString2( (const char*)m_poResult->pabyData, ",:",
CSLT_HONOURSTRINGS | CSLT_PRESERVEESCAPES |
CSLT_STRIPENDSPACES | CSLT_STRIPLEADSPACES );
int nTokens = CSLCount( papszTokens );
if( nTokens < 2 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to generate valid URL for LCP download." );
delete [] pszResponse;
CPLHTTPDestroyResult( m_poResult );
CSLDestroy( papszTokens );
return SURF_FETCH_E_IO_ERR;
}
for( int i = 1; i < nTokens; i++ )
{
if( EQUALN( papszTokens[i], "https://", 6 ) &&
EQUAL( papszTokens[i - 1], "DOWNLOAD_URL" ) )
{
CPLStrlcpy( pszResponse, papszTokens[i], nSize );
break;
}
}
CSLDestroy( papszTokens );
#endif
//Grab the download URL from the JSON response, stores in pszResponse
//std::sscanf( (char*) m_poResult->pabyData, LF_REQUEST_RETURN_TEMPLATE, pszResponse);
CPLHTTPDestroyResult( m_poResult );
if( !EQUALN( pszResponse, "https://", 6 ) )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to generate valid URL for LCP download." );
delete [] pszResponse;
return SURF_FETCH_E_IO_ERR;
}
p = strstr( pszResponse, "}]" );
if( p )
*p = '\0';
CPLDebug( "LCP_CLIENT", "Download URL: %s", pszResponse );
// Fix the SRS
const char *pszNewUrl = ReplaceSRS( nEpsgCode, pszResponse );
CPLDebug( "LCP_CLIENT", "Sanitized SRS Download URL: %s", pszNewUrl );
/*-----------------------------------------------------------------------------
* Get the Job ID by visiting the download URL
*-----------------------------------------------------------------------------*/
m_poResult = CPLHTTPFetch( pszNewUrl, NULL );
CPLFree( (void*)pszNewUrl );
delete [] pszResponse;
CHECK_HTTP_RESULT( "Failed to get Job ID" );
nSize = strlen( (char*) m_poResult->pabyData );
pszResponse = new char[ nSize + 1 ];
//grabs the Job ID from the Download URL response
std::sscanf( (char*) m_poResult->pabyData, LF_INIT_RESPONSE_TEMPLATE, pszResponse);
CPLHTTPDestroyResult( m_poResult );
//store the Job Id into a class attribute, so we can reuse pszResponse, but keep
//the Job Id (needed for future parts)
m_JobId = std::string( pszResponse );
CPLDebug( "LCP_CLIENT", "Job id: %s", m_JobId.c_str() );
/*-----------------------------------------------------------------------------
* Initiate the download by using the obtained Job ID
*-----------------------------------------------------------------------------*/
pszUrl = CPLSPrintf( LF_INIT_DOWNLOAD_TEMPLATE, m_JobId.c_str() );
//fetch the response of download initiation
//note: for some reason it alway returns a key error, but download still works
m_poResult = CPLHTTPFetch( pszUrl, NULL );
CPLHTTPDestroyResult( m_poResult );
/*-----------------------------------------------------------------------------
* Check the status of the download, able to download when status=400
*-----------------------------------------------------------------------------*/
int dl_status = 0;
//Obtain the readiness status of the current job
pszUrl = CPLSPrintf( LF_GET_STATUS_TEMPLATE, m_JobId.c_str() );
CPLDebug( "LCP_CLIENT", "Status url: %s", pszUrl );
do
{
m_poResult = CPLHTTPFetch( pszUrl, NULL );
delete [] pszResponse;
CHECK_HTTP_RESULT( "Failed to get job status" );
nSize = strlen( (char*) m_poResult->pabyData );
pszResponse = new char[ nSize + 1 ];
std::sscanf( (char*) m_poResult->pabyData, LF_STATUS_RESPONSE_TEMPLATE,
&dl_status, pszResponse );
CPLHTTPDestroyResult( m_poResult );
i++;
CPLSleep( dfWait );
CPLDebug( "LCP_CLIENT", "Attempting to fetch LCP, try %d of %d, " \
"status: %d", i, nMaxTries, dl_status );
} while( dl_status < 400 && dl_status > 0 && i < nMaxTries );
delete [] pszResponse;
if( dl_status >= 900 && dl_status <= 902)
{
CPLError( CE_Warning, CPLE_AppDefined, "Failed to download lcp," \
"There was an extraction " \
"error on the server." );
return SURF_FETCH_E_IO_ERR;
}
else if( dl_status != 400 )
{
CPLError( CE_Warning, CPLE_AppDefined, "Failed to download lcp, timed " \
"out. Try increasing " \
"LCP_MAX_DOWNLOAD_TRIES or "
"LCP_DOWNLOAD_WAIT" );
return SURF_FETCH_E_TIMEOUT;
}
/*-----------------------------------------------------------------------------
* Download the landfire model
*-----------------------------------------------------------------------------*/
pszUrl = CPLSPrintf( LF_DOWNLOAD_JOB_TEMPLATE, m_JobId.c_str() );
m_poResult = CPLHTTPFetch( pszUrl, NULL );
CHECK_HTTP_RESULT( "Failed to get job status" );
/*
** Parse the URL from the returned string
*/
std::string ss((const char*) m_poResult->pabyData );
CPLHTTPDestroyResult( m_poResult );
std::size_t pos1 = ss.find("https://");
std::size_t pos2 = ss.find(".zip");
std::string url = ss.substr(pos1, (pos2+4-pos1));
pszUrl = url.c_str();
m_poResult = CPLHTTPFetch( pszUrl, NULL );
CHECK_HTTP_RESULT( "Failed to get job status" );
nSize = m_poResult->nDataLen;
VSILFILE *fout;
const char *pszTmpZip = CPLFormFilename( NULL,
CPLGenerateTempFilename( "NINJA_LCP_CLIENT" ),
".zip" );
fout = VSIFOpenL( pszTmpZip, "w+" );
if( NULL == fout )
{
CPLError( CE_Warning, CPLE_AppDefined, "Failed to create output file" );
CPLHTTPDestroyResult( m_poResult );
return SURF_FETCH_E_IO_ERR;
}
VSIFWriteL( m_poResult->pabyData, nSize, 1, fout );
VSIFCloseL( fout );
CPLHTTPDestroyResult( m_poResult );
/*
** Extract the lcp and the prj file, and 'save as'
*/
char **papszFileList = NULL;
std::string osPathInZip;
const char *pszVSIZip = CPLSPrintf( "/vsizip/%s", pszTmpZip );
CPLDebug( "LCP_CLIENT", "Extracting lcp from %s", pszVSIZip );
papszFileList = VSIReadDirRecursive( pszVSIZip );
int bFound = FALSE;
std::string osFullPath;
for( int i = 0; i < CSLCount( papszFileList ); i++ )
{
osFullPath = papszFileList[i];
if( osFullPath.find( "Landscape_1.lcp" ) != std::string::npos )
{
osPathInZip = CPLGetPath( papszFileList[i] );
CPLDebug( "LCP_CLIENT", "Found lcp in: %s", osPathInZip.c_str() );
bFound = TRUE;
break;
}
}
CSLDestroy( papszFileList );
if( !bFound )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to find lcp in archive" );
//VSIUnlink( pszTmpZip );
return SURF_FETCH_E_IO_ERR;
}
int nError = 0;
const char *pszFileToFind = CPLSPrintf( "%s/Landscape_1.lcp",
osPathInZip.c_str() );
nError = ExtractFileFromZip( pszTmpZip, pszFileToFind, filename );
if( nError )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to extract LCP from zip." );
VSIUnlink( pszTmpZip );
return SURF_FETCH_E_IO_ERR;
}
pszFileToFind = CPLSPrintf( "%s/Landscape_1.prj", osPathInZip.c_str() );
nError = ExtractFileFromZip( pszTmpZip, pszFileToFind,
CPLFormFilename( CPLGetPath( filename ),
CPLGetBasename( filename ),
".prj" ) );
if( nError )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Failed to extract PRJ from zip." );
return SURF_FETCH_E_IO_ERR;
}
if( !CSLTestBoolean( CPLGetConfigOption( "LCP_KEEP_ARCHIVE", "FALSE" ) ) )
{
VSIUnlink( pszTmpZip );
}
return SURF_FETCH_E_NONE;
}
CPLErr GDALRasterBlock::Internalize()
{
CPLAssert( pData == NULL );
void *pNewData = NULL;
// This call will initialize the hRBLock mutex. Other call places can
// only be called if we have go through there.
const GIntBig nCurCacheMax = GDALGetCacheMax64();
// No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo().
const int nSizeInBytes = GetBlockSize();
/* -------------------------------------------------------------------- */
/* Flush old blocks if we are nearing our memory limit. */
/* -------------------------------------------------------------------- */
bool bFirstIter = true;
bool bLoopAgain = false;
do
{
bLoopAgain = false;
GDALRasterBlock* apoBlocksToFree[64] = { NULL };
int nBlocksToFree = 0;
{
TAKE_LOCK;
if( bFirstIter )
nCacheUsed += nSizeInBytes;
GDALRasterBlock *poTarget = poOldest;
while( nCacheUsed > nCurCacheMax )
{
while( poTarget != NULL )
{
if( CPLAtomicCompareAndExchange(
&(poTarget->nLockCount), 0, -1) )
break;
poTarget = poTarget->poPrevious;
}
if( poTarget != NULL )
{
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption(
"GDAL_RB_INTERNALIZE_SLEEP_AFTER_DROP_LOCK",
"0")));
GDALRasterBlock* _poPrevious = poTarget->poPrevious;
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget);
apoBlocksToFree[nBlocksToFree++] = poTarget;
if( poTarget->GetDirty() )
{
// Only free one dirty block at a time so that
// other dirty blocks of other bands with the same
// coordinates can be found with TryGetLockedBlock()
bLoopAgain = nCacheUsed > nCurCacheMax;
break;
}
if( nBlocksToFree == 64 )
{
bLoopAgain = ( nCacheUsed > nCurCacheMax );
break;
}
poTarget = _poPrevious;
}
else
{
break;
}
}
/* ------------------------------------------------------------------ */
/* Add this block to the list. */
/* ------------------------------------------------------------------ */
if( !bLoopAgain )
Touch_unlocked();
}
bFirstIter = false;
// Now free blocks we have detached and removed from their band.
for( int i = 0; i < nBlocksToFree; ++i)
{
GDALRasterBlock * const poBlock = apoBlocksToFree[i];
if( poBlock->GetDirty() )
{
CPLErr eErr = poBlock->Write();
if( eErr != CE_None )
{
// Save the error for later reporting.
poBlock->GetBand()->SetFlushBlockErr(eErr);
}
}
// Try to recycle the data of an existing block.
void* pDataBlock = poBlock->pData;
if( pNewData == NULL && pDataBlock != NULL &&
poBlock->GetBlockSize() == nSizeInBytes )
{
pNewData = pDataBlock;
}
else
{
VSIFree(poBlock->pData);
}
poBlock->pData = NULL;
poBlock->GetBand()->AddBlockToFreeList(poBlock);
}
}
while(bLoopAgain);
if( pNewData == NULL )
{
pNewData = VSI_MALLOC_VERBOSE( nSizeInBytes );
if( pNewData == NULL )
{
return( CE_Failure );
}
}
pData = pNewData;
return CE_None;
}
/**
* \brief Fetch a document from an url and return in a string.
*
* @param pszURL valid URL recognized by underlying download library (libcurl)
* @param papszOptions option list as a NULL-terminated array of strings. May be NULL.
* The following options are handled :
* <ul>
* <li>TIMEOUT=val, where val is in seconds</li>
* <li>HEADERS=val, where val is an extra header to use when getting a web page.
* For example "Accept: application/x-ogcwkt"
* <li>HTTPAUTH=[BASIC/NTLM/GSSNEGOTIATE/ANY] to specify an authentication scheme to use.
* <li>USERPWD=userid:password to specify a user and password for authentication
* <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the server
* with a POST request.
* <li>PROXY=val, to make requests go through a proxy server, where val is of the
* form proxy.server.com:port_number
* <li>PROXYUSERPWD=val, where val is of the form username:password
* <li>PROXYAUTH=[BASIC/NTLM/DIGEST/ANY] to specify an proxy authentication scheme to use.
* <li>NETRC=[YES/NO] to enable or disable use of $HOME/.netrc, default YES.
* <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL >= 1.9.0)
* <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2; ...
* <li>MAX_RETRY=val, where val is the maximum number of retry attempts if a 503 or
* 504 HTTP error occurs. Default is 0. (GDAL >= 2.0)
* <li>RETRY_DELAY=val, where val is the number of seconds between retry attempts.
* Default is 30. (GDAL >= 2.0)
* </ul>
*
* Alternatively, if not defined in the papszOptions arguments, the PROXY,
* PROXYUSERPWD, PROXYAUTH, NETRC, MAX_RETRY and RETRY_DELAY values are searched in the configuration
* options named GDAL_HTTP_PROXY, GDAL_HTTP_PROXYUSERPWD, GDAL_PROXY_AUTH,
* GDAL_HTTP_NETRC, GDAL_HTTP_MAX_RETRY and GDAL_HTTP_RETRY_DELAY.
*
* @return a CPLHTTPResult* structure that must be freed by
* CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
*/
CPLHTTPResult *CPLHTTPFetch( const char *pszURL, char **papszOptions )
{
if( strncmp(pszURL, "/vsimem/", strlen("/vsimem/")) == 0 &&
/* Disabled by default for potential security issues */
CSLTestBoolean(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")) )
{
CPLString osURL(pszURL);
const char* pszPost = CSLFetchNameValue( papszOptions, "POSTFIELDS" );
if( pszPost != NULL ) /* Hack: we append post content to filename */
{
osURL += "&POSTFIELDS=";
osURL += pszPost;
}
vsi_l_offset nLength = 0;
CPLHTTPResult* psResult = (CPLHTTPResult* )CPLCalloc(1, sizeof(CPLHTTPResult));
GByte* pabyData = VSIGetMemFileBuffer( osURL, &nLength, FALSE );
if( pabyData == NULL )
{
CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
psResult->nStatus = 1;
psResult->pszErrBuf = CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
CPLError( CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf );
}
else if( nLength != 0 )
{
psResult->nDataLen = (size_t)nLength;
psResult->pabyData = (GByte*) CPLMalloc((size_t)nLength + 1);
memcpy(psResult->pabyData, pabyData, (size_t)nLength);
psResult->pabyData[(size_t)nLength] = 0;
}
if( psResult->pabyData != NULL &&
strncmp((const char*)psResult->pabyData, "Content-Type: ",
strlen("Content-Type: ")) == 0 )
{
const char* pszContentType = (const char*)psResult->pabyData + strlen("Content-type: ");
const char* pszEOL = strchr(pszContentType, '\r');
if( pszEOL )
pszEOL = strchr(pszContentType, '\n');
if( pszEOL )
{
int nLength = pszEOL - pszContentType;
psResult->pszContentType = (char*)CPLMalloc(nLength + 1);
memcpy(psResult->pszContentType, pszContentType, nLength);
psResult->pszContentType[nLength] = 0;
}
}
return psResult;
}
#ifndef HAVE_CURL
(void) papszOptions;
(void) pszURL;
CPLError( CE_Failure, CPLE_NotSupported,
"GDAL/OGR not compiled with libcurl support, remote requests not supported." );
return NULL;
#else
/* -------------------------------------------------------------------- */
/* Are we using a persistent named session? If so, search for */
/* or create it. */
/* */
/* Currently this code does not attempt to protect against */
/* multiple threads asking for the same named session. If that */
/* occurs it will be in use in multiple threads at once which */
/* might have bad consequences depending on what guarantees */
/* libcurl gives - which I have not investigated. */
/* -------------------------------------------------------------------- */
CURL *http_handle = NULL;
const char *pszPersistent = CSLFetchNameValue( papszOptions, "PERSISTENT" );
const char *pszClosePersistent = CSLFetchNameValue( papszOptions, "CLOSE_PERSISTENT" );
if (pszPersistent)
{
CPLString osSessionName = pszPersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( poSessionMap == NULL )
poSessionMap = new std::map<CPLString,CURL*>;
if( poSessionMap->count( osSessionName ) == 0 )
{
(*poSessionMap)[osSessionName] = curl_easy_init();
CPLDebug( "HTTP", "Establish persistent session named '%s'.",
osSessionName.c_str() );
}
http_handle = (*poSessionMap)[osSessionName];
}
/* -------------------------------------------------------------------- */
/* Are we requested to close a persistent named session? */
/* -------------------------------------------------------------------- */
else if (pszClosePersistent)
{
CPLString osSessionName = pszClosePersistent;
CPLMutexHolder oHolder( &hSessionMapMutex );
if( poSessionMap )
{
std::map<CPLString,CURL*>::iterator oIter = poSessionMap->find( osSessionName );
if( oIter != poSessionMap->end() )
{
curl_easy_cleanup(oIter->second);
poSessionMap->erase(oIter);
if( poSessionMap->size() == 0 )
{
delete poSessionMap;
poSessionMap = NULL;
}
CPLDebug( "HTTP", "Ended persistent session named '%s'.",
osSessionName.c_str() );
}
else
{
CPLDebug( "HTTP", "Could not find persistent session named '%s'.",
osSessionName.c_str() );
}
}
return NULL;
}
else
http_handle = curl_easy_init();
/* -------------------------------------------------------------------- */
/* Setup the request. */
/* -------------------------------------------------------------------- */
char szCurlErrBuf[CURL_ERROR_SIZE+1];
CPLHTTPResult *psResult;
struct curl_slist *headers=NULL;
const char* pszArobase = strchr(pszURL, '@');
const char* pszSlash = strchr(pszURL, '/');
const char* pszColon = (pszSlash) ? strchr(pszSlash, ':') : NULL;
if (pszArobase != NULL && pszColon != NULL && pszArobase - pszColon > 0)
{
/* http://user:[email protected] */
char* pszSanitizedURL = CPLStrdup(pszURL);
pszSanitizedURL[pszColon-pszURL] = 0;
CPLDebug( "HTTP", "Fetch(%s:#password#%s)", pszSanitizedURL, pszArobase );
CPLFree(pszSanitizedURL);
}
else
{
CPLDebug( "HTTP", "Fetch(%s)", pszURL );
}
psResult = (CPLHTTPResult *) CPLCalloc(1,sizeof(CPLHTTPResult));
curl_easy_setopt(http_handle, CURLOPT_URL, pszURL );
CPLHTTPSetOptions(http_handle, papszOptions);
// turn off SSL verification, accept all servers with ssl
curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, FALSE);
/* Set Headers.*/
const char *pszHeaders = CSLFetchNameValue( papszOptions, "HEADERS" );
if( pszHeaders != NULL ) {
CPLDebug ("HTTP", "These HTTP headers were set: %s", pszHeaders);
headers = curl_slist_append(headers, pszHeaders);
curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
}
// are we making a head request
const char* pszNoBody = NULL;
if ((pszNoBody = CSLFetchNameValue( papszOptions, "NO_BODY" )) != NULL)
{
if (CSLTestBoolean(pszNoBody))
{
CPLDebug ("HTTP", "HEAD Request: %s", pszURL);
curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
}
}
// capture response headers
curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION, CPLHdrWriteFct);
curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, psResult );
curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, CPLWriteFct );
szCurlErrBuf[0] = '\0';
curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
static int bHasCheckVersion = FALSE;
static int bSupportGZip = FALSE;
if (!bHasCheckVersion)
{
bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
bHasCheckVersion = TRUE;
}
int bGZipRequested = FALSE;
if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
{
bGZipRequested = TRUE;
curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
}
/* -------------------------------------------------------------------- */
/* If 502, 503 or 504 status code retry this HTTP call until max */
/* retry has been rearched */
/* -------------------------------------------------------------------- */
const char *pszRetryDelay = CSLFetchNameValue( papszOptions, "RETRY_DELAY" );
if( pszRetryDelay == NULL )
pszRetryDelay = CPLGetConfigOption( "GDAL_HTTP_RETRY_DELAY", "30" );
const char *pszMaxRetries = CSLFetchNameValue( papszOptions, "MAX_RETRY" );
if( pszMaxRetries == NULL )
pszMaxRetries = CPLGetConfigOption( "GDAL_HTTP_MAX_RETRY", "0" );
int nRetryDelaySecs = atoi(pszRetryDelay);
int nMaxRetries = atoi(pszMaxRetries);
int nRetryCount = 0;
bool bRequestRetry;
do
{
bRequestRetry = FALSE;
/* -------------------------------------------------------------------- */
/* Execute the request, waiting for results. */
/* -------------------------------------------------------------------- */
psResult->nStatus = (int) curl_easy_perform( http_handle );
/* -------------------------------------------------------------------- */
/* Fetch content-type if possible. */
/* -------------------------------------------------------------------- */
psResult->pszContentType = NULL;
curl_easy_getinfo( http_handle, CURLINFO_CONTENT_TYPE,
&(psResult->pszContentType) );
if( psResult->pszContentType != NULL )
psResult->pszContentType = CPLStrdup(psResult->pszContentType);
/* -------------------------------------------------------------------- */
/* Have we encountered some sort of error? */
/* -------------------------------------------------------------------- */
if( strlen(szCurlErrBuf) > 0 )
{
int bSkipError = FALSE;
/* Some servers such as http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities */
/* invalidly return Content-Length as the uncompressed size, with makes curl to wait for more data */
/* and time-out finally. If we got the expected data size, then we don't emit an error */
/* but turn off GZip requests */
if (bGZipRequested &&
strstr(szCurlErrBuf, "transfer closed with") &&
strstr(szCurlErrBuf, "bytes remaining to read"))
{
const char* pszContentLength =
CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
if (pszContentLength && psResult->nDataLen != 0 &&
atoi(pszContentLength) == psResult->nDataLen)
{
const char* pszCurlGZIPOption = CPLGetConfigOption("CPL_CURL_GZIP", NULL);
if (pszCurlGZIPOption == NULL)
{
CPLSetConfigOption("CPL_CURL_GZIP", "NO");
CPLDebug("HTTP", "Disabling CPL_CURL_GZIP, because %s doesn't support it properly",
pszURL);
}
psResult->nStatus = 0;
bSkipError = TRUE;
}
}
if (!bSkipError)
{
psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
CPLError( CE_Failure, CPLE_AppDefined,
"%s", szCurlErrBuf );
}
}
else
{
/* HTTP errors do not trigger curl errors. But we need to */
/* propagate them to the caller though */
long response_code = 0;
curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
if (response_code >= 400 && response_code < 600)
{
/* If HTTP 502, 503 or 504 gateway timeout error retry after a pause */
if ((response_code >= 502 && response_code <= 504) && nRetryCount < nMaxRetries)
{
CPLError(CE_Warning, CPLE_AppDefined,
"HTTP error code: %d - %s. Retrying again in %d secs",
(int)response_code, pszURL, nRetryDelaySecs);
CPLSleep(nRetryDelaySecs);
nRetryCount++;
CPLFree(psResult->pszContentType);
psResult->pszContentType = NULL;
CSLDestroy(psResult->papszHeaders);
psResult->papszHeaders = NULL;
CPLFree(psResult->pabyData);
psResult->pabyData = NULL;
psResult->nDataLen = 0;
psResult->nDataAlloc = 0;
bRequestRetry = TRUE;
}
else
{
psResult->pszErrBuf = CPLStrdup(CPLSPrintf("HTTP error code : %d", (int)response_code));
CPLError( CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf );
}
}
}
}
while (bRequestRetry);
if (!pszPersistent)
curl_easy_cleanup( http_handle );
curl_slist_free_all(headers);
return psResult;
#endif /* def HAVE_CURL */
}
int main( int argc, char ** argv )
{
int iArg;
/* -------------------------------------------------------------------- */
/* Process arguments. */
/* -------------------------------------------------------------------- */
argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
if( argc < 1 )
exit( -argc );
for( iArg = 1; iArg < argc; iArg++ )
{
if( EQUAL(argv[iArg],"-i") && iArg < argc-1 )
nIterations = atoi(argv[++iArg]);
else if( EQUAL(argv[iArg],"-oi") && iArg < argc-1 )
nOpenIterations = atoi(argv[++iArg]);
else if( EQUAL(argv[iArg],"-t") && iArg < argc-1 )
nThreadCount = atoi(argv[++iArg]);
else if( EQUAL(argv[iArg],"-nlo") )
bLockOnOpen = FALSE;
else if( pszFilename == NULL )
pszFilename = argv[iArg];
else
{
printf( "Unrecognised argument: %s\n", argv[iArg] );
Usage();
}
}
if( pszFilename == NULL )
{
printf( "Need a file to operate on.\n" );
Usage();
exit( 1 );
}
if( nOpenIterations > 0 )
bLockOnOpen = FALSE;
/* -------------------------------------------------------------------- */
/* Get the checksum of band1. */
/* -------------------------------------------------------------------- */
GDALDatasetH hDS;
GDALAllRegister();
hDS = GDALOpen( pszFilename, GA_ReadOnly );
if( hDS == NULL )
exit( 1 );
nChecksum = GDALChecksumImage( GDALGetRasterBand( hDS, 1 ),
0, 0,
GDALGetRasterXSize( hDS ),
GDALGetRasterYSize( hDS ) );
GDALClose( hDS );
printf( "Got checksum %d, launching %d worker threads on %s, %d iterations.\n",
nChecksum, nThreadCount, pszFilename, nIterations );
/* -------------------------------------------------------------------- */
/* Fire off worker threads. */
/* -------------------------------------------------------------------- */
int iThread;
pGlobalMutex = CPLCreateMutex();
CPLReleaseMutex( pGlobalMutex );
nPendingThreads = nThreadCount;
for( iThread = 0; iThread < nThreadCount; iThread++ )
{
if( CPLCreateThread( WorkerFunc, NULL ) == -1 )
{
printf( "CPLCreateThread() failed.\n" );
exit( 1 );
}
}
while( nPendingThreads > 0 )
CPLSleep( 0.5 );
CPLReleaseMutex( pGlobalMutex );
printf( "All threads complete.\n" );
CSLDestroy( argv );
GDALDestroyDriverManager();
return 0;
}
int main( int argc, char ** argv )
{
/* -------------------------------------------------------------------- */
/* Process arguments. */
/* -------------------------------------------------------------------- */
argc = GDALGeneralCmdLineProcessor(argc, &argv, 0);
if( argc < 1 )
exit(-argc);
int nThreadCount = 4;
bool bOpenInThreads = true;
for( int iArg = 1; iArg < argc; iArg++ )
{
if( iArg < argc-1 && EQUAL(argv[iArg], "-i") )
{
nIterations = atoi(argv[++iArg]);
}
else if( iArg < argc-1 && EQUAL(argv[iArg], "-oi") )
{
nOpenIterations = atoi(argv[++iArg]);
}
else if( iArg < argc-1 && EQUAL(argv[iArg], "-t") )
{
nThreadCount = atoi(argv[++iArg]);
}
else if( EQUAL(argv[iArg], "-lock_on_open") )
{
bLockOnOpen = true;
}
else if( EQUAL(argv[iArg], "-open_in_main") )
{
bOpenInThreads = false;
}
else if( pszFilename == nullptr )
{
pszFilename = argv[iArg];
}
else
{
printf("Unrecognized argument: %s\n", argv[iArg]);
Usage();
}
}
if( pszFilename == nullptr )
{
printf("Need a file to operate on.\n");
Usage();
exit(1);
}
if( nOpenIterations > 0 )
bLockOnOpen = false;
/* -------------------------------------------------------------------- */
/* Get the checksum of band1. */
/* -------------------------------------------------------------------- */
GDALDatasetH hDS = nullptr;
GDALAllRegister();
for( int i = 0; i < 2; i++ )
{
hDS = GDALOpen( pszFilename, GA_ReadOnly );
if( hDS == nullptr )
exit( 1 );
nChecksum = GDALChecksumImage(GDALGetRasterBand(hDS, 1),
0, 0,
GDALGetRasterXSize(hDS),
GDALGetRasterYSize(hDS));
GDALClose(hDS);
}
printf(
"Got checksum %d, launching %d worker threads on %s, %d iterations.\n",
nChecksum, nThreadCount, pszFilename, nIterations);
/* -------------------------------------------------------------------- */
/* Fire off worker threads. */
/* -------------------------------------------------------------------- */
pGlobalMutex = CPLCreateMutex();
CPLReleaseMutex(pGlobalMutex);
nPendingThreads = nThreadCount;
std::vector<GDALDatasetH> aoDS;
for( int iThread = 0; iThread < nThreadCount; iThread++ )
{
hDS = nullptr;
if( !bOpenInThreads )
{
hDS = GDALOpen(pszFilename, GA_ReadOnly);
if( !hDS )
{
printf("GDALOpen() failed.\n");
exit(1);
}
aoDS.push_back(hDS);
}
if( CPLCreateThread(WorkerFunc, hDS) == -1 )
{
printf("CPLCreateThread() failed.\n");
exit(1);
}
}
while( nPendingThreads > 0 )
CPLSleep(0.5);
CPLDestroyMutex(pGlobalMutex);
for( size_t i = 0; i < aoDS.size(); ++i )
GDALClose(aoDS[i]);
printf("All threads complete.\n");
CSLDestroy(argv);
GDALDestroyDriverManager();
return 0;
}
/*
** Fetch a nomads forecast from the NWS servers. The forecasts consist of grib
** files, one forecast for each forecast hour. The time starts and the nearest
** forecast hour *before* the reference time passed. If no reference time is
** passed, or it is not valid, now() is used.
**
** The forecasts are downloaded into a temporary directory. If the entire
** download succeeds, then the files are copied into the path specified. If
** the path specified is a zip file (ends in *.zip), then the forecast files
** are zipped.
**
** Available compile time configuration options:
** NOMADS_USE_IP: Use the ip address instead of the hostname. It
** possibly goes around dns lookup, but it's doubtfull.
** NOMADS_ENABLE_ASYNC: Allow for asynchronous connections to the
** server.
** NOMADS_EXPER_FORECASTS: Compile in forecasts that may not work with
** the current configuration, essentially
** unsupported (ie NARRE, RTMA).
** NOMADS_USE_VSI_READ: Use the VSI*L api for downloading. This will
** allow definition of chunk sizes for download,
** although it is probably unnecessary. See
** NOMADS_VSI_BLOCK_SIZE below. If not enabled, a
** single fetch is made for each file, which is
** faster.
** Available runtime configuration options:
** NOMADS_THREAD_COUNT: Number of threads to use for downloads if
** NOMADS_ENABLE_ASYNC is set to ON during
** compilation. Default is 4.
** NOMADS_VSI_BLOCK_SIZE: Number of bytes to request at a time when
** downloading files if NOMADS_USE_VSI_READ is
** set to ON during compilation. Default is 512.
** NOMADS_MAX_FCST_REWIND: Number of forecast run time steps to go back
** to attempt to get a full time frame.
** GDAL_HTTP_TIMEOUT: Timeout for HTTP requests in seconds. We should
** be able to set this reasonably low.
**
** \param pszModelKey The name key of the model to use, ie "nam_conus". For a
** listing of models, see nomads.ncep.gov or /see nomads.h
**
** \param pszRefTime The reference time to begin searching for forecasts from.
** The search starts at refrence time, then goes back until
** it hits a valid forecast time. If the download cannot be
** complete, it will step back NOMADS_MAX_FCST_REWIND
** foreacst times to attempt to get a full forecast.
**
** \param nHours The extent of hours to download forecasts from the reference
** time. If we step back, we will still grab all forecasts up
** until nHours from the reference time, not the forecast time.
**
** \param nStride The number of forecasts to skip in time steps. For example,
** if 12 hours of gfs is requested (normally 5 files/steps (0,
** 3, 6, 9, 12) with a stride of 2, you'd get 0, 6, 12 forecast
** hours.
**
** \param padfBbox The bounding box of the request in WGS84 decimal degrees.
** Order is xmin, xmax, ymax, ymin.
**
** \param pszDstVsiPath The location to write the files. If the location
** has an extension of ".zip", then the output files are
** written as a zip archive, otherwise to a path.
**
** \param papszOptions List of key=value options, unused.
**
** \param pfnProgress Optional progress function.
**
** \return NOMADS_OK(0) on success, NOMADS_ERR(1) otherwise.
*/
int NomadsFetch( const char *pszModelKey, const char *pszRefTime,
int nHours, int nStride, double *padfBbox,
const char *pszDstVsiPath, char ** papszOptions,
GDALProgressFunc pfnProgress )
{
const char **ppszKey = NULL;
int nFcstHour = 0;
int *panRunHours = NULL;
int i = 0;
int j = 0;
int k = 0;
int t = 0;
int rc = 0;
char **papszDownloadUrls = NULL;
char **papszOutputFiles = NULL;
char **papszFinalFiles = NULL;
int nFilesToGet = 0;
const char *pszTmpDir;
const char *pszConfigOpt;
int nFcstTries;
int nMaxFcstRewind;
int nrc;
int bZip;
void **pThreads;
int nThreads;
const char *pszThreadCount;
NomadsThreadData *pasData;
nomads_utc *ref, *end, *fcst;
nrc = NOMADS_OK;
CPLDebug( "NOMADS", "Fetching data for bounding box: %lf, %lf, %lf, %lf",
padfBbox[0], padfBbox[1], padfBbox[2], padfBbox[3] );
ppszKey = NomadsFindModel( pszModelKey );
if( ppszKey == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not find model key in nomads data" );
return NOMADS_ERR;
}
NomadsUtcCreate( &ref );
NomadsUtcCreate( &end );
rc = NOMADS_OK;
if( pszRefTime )
{
rc = NomadsUtcFromIsoFrmt( ref, pszRefTime );
}
if( rc != NOMADS_OK || pszRefTime == NULL )
{
NomadsUtcNow( ref );
}
NomadsUtcCopy( end, ref );
NomadsUtcAddHours( end, nHours );
/* Disable unneeded reading of entire directories, good speedup */
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "TRUE" );
pszConfigOpt = CPLGetConfigOption( "NOMADS_HTTP_TIMEOUT", "20" );
if( pszConfigOpt != NULL )
{
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", pszConfigOpt );
}
nMaxFcstRewind = atoi( CPLGetConfigOption( "NOMADS_MAX_FCST_REWIND", "2" ) );
if( nMaxFcstRewind < 1 || nMaxFcstRewind > 24 )
{
nMaxFcstRewind = 2;
}
/* Go back at least 3 for rap, as it may not get updated all the time. */
if( EQUALN( pszModelKey, "rap", 3 ) || EQUALN( pszModelKey, "hrrr", 4 ) )
{
nMaxFcstRewind = nMaxFcstRewind > 3 ? nMaxFcstRewind : 3;
}
#ifdef NOMADS_ENABLE_ASYNC
pszThreadCount = CPLGetConfigOption( "NOMADS_THREAD_COUNT", "4" );
nThreads = atoi( pszThreadCount );
if( nThreads < 1 || nThreads > 96 )
{
nThreads = 4;
}
pThreads = CPLMalloc( sizeof( void * ) * nThreads );
pasData = CPLMalloc( sizeof( NomadsThreadData ) * nThreads );
#else /* NOMADS_ENABLE_ASYNC */
/* Unused variables, set to null to so free is no-op */
nThreads = 1;
pThreads = NULL;
pasData = NULL;
#endif /* NOMADS_ENABLE_ASYNC */
fcst = NULL;
nFcstTries = 0;
while( nFcstTries < nMaxFcstRewind )
{
nrc = NOMADS_OK;
fcst = NomadsSetForecastTime( ppszKey, ref, nFcstTries );
nFcstHour = fcst->ts->tm_hour;
CPLDebug( "WINDNINJA", "Generated forecast time in utc: %s",
NomadsUtcStrfTime( fcst, "%Y%m%dT%HZ" ) );
if( EQUAL( pszModelKey, "rtma_conus" ) )
{
panRunHours = (int*)CPLMalloc( sizeof( int ) );
nFilesToGet = 1;
}
else
{
nFilesToGet =
NomadsBuildForecastRunHours( ppszKey, fcst, end, nHours,
nStride, &panRunHours );
}
papszDownloadUrls =
NomadsBuildForecastFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, fcst, padfBbox );
if( papszDownloadUrls == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
pszTmpDir = CPLStrdup( CPLGenerateTempFilename( NULL ) );
CPLDebug( "WINDNINJA", "Creating Temp directory: %s", pszTmpDir );
VSIMkdir( pszTmpDir, 0777 );
papszOutputFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszTmpDir, FALSE );
if( papszOutputFiles == NULL )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not generate list of URLs to download, invalid data" );
nFcstTries++;
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
fcst = NULL;
nrc = NOMADS_ERR;
continue;
}
CPLAssert( CSLCount( papszDownloadUrls ) == nFilesToGet );
CPLAssert( CSLCount( papszOutputFiles ) == nFilesToGet );
if( pfnProgress )
{
pfnProgress( 0.0, "Starting download...", NULL );
}
/* Download one file and start over if it's not there. */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[0], papszOutputFiles[0] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[0], papszOutputFiles[0] );
#endif /* NOMADS_USE_VSI_READ */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
/*
** Don't explicitly break here. We'll skip the while loop because
** nrc != NOMADS_OK, and we can clean up memory and shift times in
** one spot to avoid duplicate code.
*/
}
/* Get the rest */
i = 1;
while( i < nFilesToGet && nrc == NOMADS_OK )
{
if( pfnProgress )
{
if( pfnProgress( (double)i / nFilesToGet,
CPLSPrintf( "Downloading %s...",
CPLGetFilename( papszOutputFiles[i] ) ),
NULL ) )
{
CPLError( CE_Failure, CPLE_UserInterrupt,
"Cancelled by user." );
nrc = NOMADS_ERR;
nFcstTries = nMaxFcstRewind;
break;
}
}
#ifdef NOMADS_ENABLE_ASYNC
k = i > nFilesToGet - nThreads ? (nFilesToGet - 1) % nThreads : nThreads;
for( t = 0; t < k; t++ )
{
pasData[t].pszUrl = papszDownloadUrls[i];
pasData[t].pszFilename = papszOutputFiles[i];
pThreads[t] =
CPLCreateJoinableThread( NomadsFetchAsync, &pasData[t] );
i++;
}
for( t = 0; t < k; t++ )
{
CPLJoinThread( pThreads[t] );
}
for( t = 0; t < k; t++ )
{
if( pasData[t].nErr )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Threaded download failed, attempting " \
"serial download for %s",
CPLGetFilename( pasData[t].pszFilename ) );
/* Try again, serially though */
if( CPLCheckForFile( (char *)pasData[t].pszFilename, NULL ) );
{
VSIUnlink( pasData[t].pszFilename );
}
#ifdef NOMADS_USE_VSI_READ
rc = NomadsFetchVsi( pasData[t].pszUrl,
pasData[t].pszFilename );
#else /* NOMADS_USE_VSI_READ */
rc = NomadsFetchHttp( pasData[t].pszUrl,
pasData[t].pszFilename );
#endif /* NOMADS_USE_VSI_READ */
if( rc != NOMADS_OK )
{
nrc = rc;
}
}
}
#else /* NOMADS_ENABLE_ASYNC */
#ifdef NOMADS_USE_VSI_READ
nrc = NomadsFetchVsi( papszDownloadUrls[i], papszOutputFiles[i] );
#else /* NOMADS_USE_VSI_READ */
nrc = NomadsFetchHttp( papszDownloadUrls[i], papszOutputFiles[i] );
#endif /* NOMADS_USE_VSI_READ */
i++;
#endif /* NOMADS_ENABLE_ASYNC */
if( nrc != NOMADS_OK )
{
CPLError( CE_Warning, CPLE_AppDefined,
"Failed to download forecast, " \
"stepping back one forecast run time step." );
nFcstTries++;
CPLSleep( 1 );
break;
}
}
/*
** XXX Cleanup XXX
** After each loop we can get rid of the urls, but we can only get rid
** of the others if they are to be reallocated in the next loop. Any
** cleanup in the else nrc == NOMADS_OK clause should be cleaned up in
** the nrc == NOMADS_OK outside the loop. Those are held so we can
** process the output files and zip them into an archive.
*/
CSLDestroy( papszDownloadUrls );
NomadsUtcFree( fcst );
if( nrc == NOMADS_OK )
{
break;
}
else
{
CPLFree( (void*)panRunHours );
CSLDestroy( papszOutputFiles );
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
}
if( nrc == NOMADS_OK )
{
bZip = EQUAL( CPLGetExtension( pszDstVsiPath ), "zip" ) ? TRUE : FALSE;
papszFinalFiles =
NomadsBuildOutputFileList( pszModelKey, nFcstHour, panRunHours,
nFilesToGet, pszDstVsiPath,
bZip );
CPLFree( (void*)panRunHours );
if( CPLCheckForFile( (char*)pszDstVsiPath, NULL ) )
{
CPLUnlinkTree( pszDstVsiPath );
}
if( !bZip )
{
VSIMkdir( pszDstVsiPath, 0777 );
}
nrc = NomadsZipFiles( papszOutputFiles, papszFinalFiles );
CSLDestroy( papszOutputFiles );
CSLDestroy( papszFinalFiles );
if( nrc != NOMADS_OK )
{
CPLError( CE_Failure, CPLE_AppDefined,
"Could not copy files into path, unknown i/o failure" );
CPLUnlinkTree( pszDstVsiPath );
}
CPLUnlinkTree( pszTmpDir );
CPLFree( (void*)pszTmpDir );
}
CPLFree( (void*)pasData );
CPLFree( (void**)pThreads );
NomadsUtcFree( ref );
NomadsUtcFree( end );
CPLSetConfigOption( "GDAL_HTTP_TIMEOUT", NULL );
CPLSetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", NULL );
if( nrc == NOMADS_OK && pfnProgress )
{
pfnProgress( 1.0, NULL, NULL );
}
return nrc;
}
