int OGRCSVLayer::GetFeatureCount( int bForce )
{
if (bInWriteMode || m_poFilterGeom != NULL || m_poAttrQuery != NULL)
return OGRLayer::GetFeatureCount(bForce);
if (nTotalFeatures >= 0)
return nTotalFeatures;
if (fpCSV == NULL)
return 0;
ResetReading();
char **papszTokens;
nTotalFeatures = 0;
while(TRUE)
{
papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings );
if( papszTokens == NULL )
break;
if( papszTokens[0] != NULL )
nTotalFeatures ++;
CSLDestroy(papszTokens);
}
ResetReading();
return nTotalFeatures;
}
void OGRCSVLayer::ResetReading()
{
if (fpCSV)
VSIRewindL( fpCSV );
if( bHasFieldNames )
CSLDestroy( OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings ) );
bNeedRewindBeforeRead = FALSE;
nNextFID = 1;
}
int OGRCSVDataSource::OpenTable( const char * pszFilename,
char** papszOpenOptions,
const char* pszNfdcRunwaysGeomField,
const char* pszGeonamesGeomFieldPrefix)
{
/* -------------------------------------------------------------------- */
/* Open the file. */
/* -------------------------------------------------------------------- */
VSILFILE * fp;
if( bUpdate )
fp = VSIFOpenL( pszFilename, "rb+" );
else
fp = VSIFOpenL( pszFilename, "rb" );
if( fp == NULL )
{
CPLError( CE_Warning, CPLE_OpenFailed,
"Failed to open %s, %s.",
pszFilename, VSIStrerror( errno ) );
return FALSE;
}
if( !bUpdate && strstr(pszFilename, "/vsigzip/") == NULL &&
strstr(pszFilename, "/vsizip/") == NULL )
fp = (VSILFILE*) VSICreateBufferedReaderHandle((VSIVirtualHandle*)fp);
CPLString osLayerName = CPLGetBasename(pszFilename);
CPLString osExt = CPLGetExtension(pszFilename);
if( strncmp(pszFilename, "/vsigzip/", 9) == 0 && EQUAL(osExt, "gz") )
{
if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".csv.gz") )
{
osLayerName = osLayerName.substr(0, osLayerName.size() - 4);
osExt = "csv";
}
else if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".tsv.gz") )
{
osLayerName = osLayerName.substr(0, osLayerName.size() - 4);
osExt = "tsv";
}
}
/* -------------------------------------------------------------------- */
/* Read and parse a line. Did we get multiple fields? */
/* -------------------------------------------------------------------- */
const char* pszLine = CPLReadLineL( fp );
if (pszLine == NULL)
{
VSIFCloseL( fp );
return FALSE;
}
char chDelimiter = CSVDetectSeperator(pszLine);
if( chDelimiter != '\t' && strchr(pszLine, '\t') != NULL )
{
/* Force the delimiter to be TAB for a .tsv file that has a tabulation */
/* in its first line */
if( EQUAL(osExt, "tsv") )
{
chDelimiter = '\t';
}
else
{
for(int bDontHonourStrings=0; bDontHonourStrings<=1; bDontHonourStrings++)
{
// Read the first 2 lines to see if they have the same number of fields, if using tabulation
VSIRewindL( fp );
char** papszTokens = OGRCSVReadParseLineL( fp, '\t', bDontHonourStrings );
int nTokens1 = CSLCount(papszTokens);
CSLDestroy(papszTokens);
papszTokens = OGRCSVReadParseLineL( fp, '\t', bDontHonourStrings );
int nTokens2 = CSLCount(papszTokens);
CSLDestroy(papszTokens);
if( nTokens1 >= 2 && nTokens1 == nTokens2 )
{
chDelimiter = '\t';
break;
}
}
}
}
VSIRewindL( fp );
#if 0
const char *pszDelimiter = CSLFetchNameValueDef( papszOpenOptions, "SEPARATOR", "AUTO");
if( !EQUAL(pszDelimiter, "AUTO") )
{
if (EQUAL(pszDelimiter, "COMMA"))
chDelimiter = ',';
else if (EQUAL(pszDelimiter, "SEMICOLON"))
chDelimiter = ';';
else if (EQUAL(pszDelimiter, "TAB"))
chDelimiter = '\t';
else if (EQUAL(pszDelimiter, "SPACE"))
chDelimiter = ' ';
else
{
CPLError( CE_Warning, CPLE_AppDefined,
"SEPARATOR=%s not understood, use one of COMMA, SEMICOLON, SPACE or TAB.",
pszDelimiter );
}
}
#endif
/* GNIS specific */
if (pszGeonamesGeomFieldPrefix != NULL &&
strchr(pszLine, '|') != NULL)
chDelimiter = '|';
char **papszFields = OGRCSVReadParseLineL( fp, chDelimiter, FALSE );
if( CSLCount(papszFields) < 2 )
{
VSIFCloseL( fp );
CSLDestroy( papszFields );
return FALSE;
}
VSIRewindL( fp );
CSLDestroy( papszFields );
/* -------------------------------------------------------------------- */
/* Create a layer. */
/* -------------------------------------------------------------------- */
nLayers++;
papoLayers = (OGRCSVLayer **) CPLRealloc(papoLayers,
sizeof(void*) * nLayers);
if (pszNfdcRunwaysGeomField != NULL)
{
osLayerName += "_";
osLayerName += pszNfdcRunwaysGeomField;
}
else if (pszGeonamesGeomFieldPrefix != NULL &&
!EQUAL(pszGeonamesGeomFieldPrefix, ""))
{
osLayerName += "_";
osLayerName += pszGeonamesGeomFieldPrefix;
}
if (EQUAL(pszFilename, "/vsistdin/"))
osLayerName = "layer";
papoLayers[nLayers-1] =
new OGRCSVLayer( osLayerName, fp, pszFilename, FALSE, bUpdate,
chDelimiter );
papoLayers[nLayers-1]->BuildFeatureDefn( pszNfdcRunwaysGeomField,
pszGeonamesGeomFieldPrefix,
papszOpenOptions );
return TRUE;
}
int OGRCSVDataSource::OpenTable( const char * pszFilename,
const char* pszNfdcRunwaysGeomField,
const char* pszGeonamesGeomFieldPrefix)
{
/* -------------------------------------------------------------------- */
/* Open the file. */
/* -------------------------------------------------------------------- */
VSILFILE * fp;
if( bUpdate )
fp = VSIFOpenL( pszFilename, "rb+" );
else
fp = VSIFOpenL( pszFilename, "rb" );
if( fp == NULL )
{
CPLError( CE_Warning, CPLE_OpenFailed,
"Failed to open %s, %s.",
pszFilename, VSIStrerror( errno ) );
return FALSE;
}
if( !bUpdate && strstr(pszFilename, "/vsigzip/") == NULL &&
strstr(pszFilename, "/vsizip/") == NULL )
fp = (VSILFILE*) VSICreateBufferedReaderHandle((VSIVirtualHandle*)fp);
CPLString osLayerName = CPLGetBasename(pszFilename);
CPLString osExt = CPLGetExtension(pszFilename);
if( strncmp(pszFilename, "/vsigzip/", 9) == 0 && EQUAL(osExt, "gz") )
{
if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".csv.gz") )
{
osLayerName = osLayerName.substr(0, osLayerName.size() - 4);
osExt = "csv";
}
else if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".tsv.gz") )
{
osLayerName = osLayerName.substr(0, osLayerName.size() - 4);
osExt = "tsv";
}
}
/* -------------------------------------------------------------------- */
/* Read and parse a line. Did we get multiple fields? */
/* -------------------------------------------------------------------- */
const char* pszLine = CPLReadLineL( fp );
if (pszLine == NULL)
{
VSIFCloseL( fp );
return FALSE;
}
char chDelimiter = CSVDetectSeperator(pszLine);
/* Force the delimiter to be TAB for a .tsv file that has a tabulation */
/* in its first line */
if( EQUAL(osExt, "tsv") && chDelimiter != '\t' &&
strchr(pszLine, '\t') != NULL )
{
chDelimiter = '\t';
}
VSIRewindL( fp );
/* GNIS specific */
if (pszGeonamesGeomFieldPrefix != NULL &&
strchr(pszLine, '|') != NULL)
chDelimiter = '|';
char **papszFields = OGRCSVReadParseLineL( fp, chDelimiter, FALSE );
if( CSLCount(papszFields) < 2 )
{
VSIFCloseL( fp );
CSLDestroy( papszFields );
return FALSE;
}
VSIRewindL( fp );
CSLDestroy( papszFields );
/* -------------------------------------------------------------------- */
/* Create a layer. */
/* -------------------------------------------------------------------- */
nLayers++;
papoLayers = (OGRCSVLayer **) CPLRealloc(papoLayers,
sizeof(void*) * nLayers);
if (pszNfdcRunwaysGeomField != NULL)
{
osLayerName += "_";
osLayerName += pszNfdcRunwaysGeomField;
}
else if (pszGeonamesGeomFieldPrefix != NULL &&
!EQUAL(pszGeonamesGeomFieldPrefix, ""))
{
osLayerName += "_";
osLayerName += pszGeonamesGeomFieldPrefix;
}
if (EQUAL(pszFilename, "/vsistdin/"))
osLayerName = "layer";
papoLayers[nLayers-1] =
new OGRCSVLayer( osLayerName, fp, pszFilename, FALSE, bUpdate,
chDelimiter, pszNfdcRunwaysGeomField, pszGeonamesGeomFieldPrefix );
return TRUE;
}
OGRFeature * OGRCSVLayer::GetNextUnfilteredFeature()
{
if (fpCSV == NULL)
return NULL;
/* -------------------------------------------------------------------- */
/* Read the CSV record. */
/* -------------------------------------------------------------------- */
char **papszTokens;
while(TRUE)
{
papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings );
if( papszTokens == NULL )
return NULL;
if( papszTokens[0] != NULL )
break;
CSLDestroy(papszTokens);
}
/* -------------------------------------------------------------------- */
/* Create the OGR feature. */
/* -------------------------------------------------------------------- */
OGRFeature *poFeature;
poFeature = new OGRFeature( poFeatureDefn );
/* -------------------------------------------------------------------- */
/* Set attributes for any indicated attribute records. */
/* -------------------------------------------------------------------- */
int iAttr;
int nAttrCount = MIN(CSLCount(papszTokens),
poFeatureDefn->GetFieldCount() );
CPLValueType eType;
for( iAttr = 0; !bIsEurostatTSV && iAttr < nAttrCount; iAttr++)
{
if( iAttr == iWktGeomReadField && papszTokens[iAttr][0] != '\0' )
{
char *pszWKT = papszTokens[iAttr];
OGRGeometry *poGeom = NULL;
if( OGRGeometryFactory::createFromWkt( &pszWKT, NULL, &poGeom )
== OGRERR_NONE )
poFeature->SetGeometryDirectly( poGeom );
}
OGRFieldType eFieldType = poFeatureDefn->GetFieldDefn(iAttr)->GetType();
if ( eFieldType == OFTReal || eFieldType == OFTInteger )
{
if (chDelimiter == ';' && eFieldType == OFTReal)
{
char* chComma = strchr(papszTokens[iAttr], ',');
if (chComma)
*chComma = '.';
}
eType = CPLGetValueType(papszTokens[iAttr]);
if ( (papszTokens[iAttr][0] != '\0') &&
( eType == CPL_VALUE_INTEGER ||
eType == CPL_VALUE_REAL ) )
{
poFeature->SetField( iAttr, papszTokens[iAttr] );
}
}
else if (eFieldType != OFTString)
{
if (papszTokens[iAttr][0] != '\0')
poFeature->SetField( iAttr, papszTokens[iAttr] );
}
else
poFeature->SetField( iAttr, papszTokens[iAttr] );
}
/* -------------------------------------------------------------------- */
/* Eurostat TSV files. */
/* -------------------------------------------------------------------- */
for( iAttr = 0; bIsEurostatTSV && iAttr < nAttrCount; iAttr++)
{
if( iAttr == 0 )
{
char** papszDims = CSLTokenizeString2( papszTokens[0], ",", 0 );
if( CSLCount(papszDims) != nEurostatDims )
{
CSLDestroy(papszDims);
break;
}
for( int iSubAttr = 0; iSubAttr < nEurostatDims; iSubAttr ++ )
{
poFeature->SetField( iSubAttr, papszDims[iSubAttr] );
}
CSLDestroy(papszDims);
}
else
{
char** papszVals = CSLTokenizeString2( papszTokens[iAttr], " ", 0 );
eType = CPLGetValueType(papszVals[0]);
if ( (papszVals[0][0] != '\0') &&
( eType == CPL_VALUE_INTEGER ||
eType == CPL_VALUE_REAL ) )
{
poFeature->SetField( nEurostatDims + 2 * (iAttr - 1), papszVals[0] );
}
if( CSLCount(papszVals) == 2 )
{
poFeature->SetField( nEurostatDims + 2 * (iAttr - 1) + 1, papszVals[1] );
}
CSLDestroy(papszVals);
}
}
/* -------------------------------------------------------------------- */
/*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */
/* -------------------------------------------------------------------- */
if ( iNfdcLatitudeS != -1 &&
iNfdcLongitudeS != -1 &&
nAttrCount > iNfdcLatitudeS &&
nAttrCount > iNfdcLongitudeS &&
papszTokens[iNfdcLongitudeS][0] != 0 &&
papszTokens[iNfdcLatitudeS][0] != 0)
{
double dfLon = atof(papszTokens[iNfdcLongitudeS]) / 3600;
if (strchr(papszTokens[iNfdcLongitudeS], 'W'))
dfLon *= -1;
double dfLat = atof(papszTokens[iNfdcLatitudeS]) / 3600;
if (strchr(papszTokens[iNfdcLatitudeS], 'S'))
dfLat *= -1;
poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) );
}
/* -------------------------------------------------------------------- */
/* GNIS specific */
/* -------------------------------------------------------------------- */
else if ( iLatitudeField != -1 &&
iLongitudeField != -1 &&
nAttrCount > iLatitudeField &&
nAttrCount > iLongitudeField &&
papszTokens[iLongitudeField][0] != 0 &&
papszTokens[iLatitudeField][0] != 0)
{
/* Some records have dummy 0,0 value */
if (papszTokens[iLongitudeField][0] != '0' ||
papszTokens[iLongitudeField][1] != '\0' ||
papszTokens[iLatitudeField][0] != '0' ||
papszTokens[iLatitudeField][1] != '\0')
{
double dfLon = atof(papszTokens[iLongitudeField]);
double dfLat = atof(papszTokens[iLatitudeField]);
poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) );
}
}
CSLDestroy( papszTokens );
/* -------------------------------------------------------------------- */
/* Translate the record id. */
/* -------------------------------------------------------------------- */
poFeature->SetFID( nNextFID++ );
m_nFeaturesRead++;
return poFeature;
}
OGRCSVLayer::OGRCSVLayer( const char *pszLayerNameIn,
VSILFILE * fp, const char *pszFilename, int bNew, int bInWriteMode,
char chDelimiter, const char* pszNfdcGeomField,
const char* pszGeonamesGeomFieldPrefix)
{
fpCSV = fp;
iWktGeomReadField = -1;
iNfdcLatitudeS = iNfdcLongitudeS = -1;
iLatitudeField = iLongitudeField = -1;
this->bInWriteMode = bInWriteMode;
this->bNew = bNew;
this->pszFilename = CPLStrdup(pszFilename);
this->chDelimiter = chDelimiter;
bFirstFeatureAppendedDuringSession = TRUE;
bUseCRLF = FALSE;
bNeedRewindBeforeRead = FALSE;
eGeometryFormat = OGR_CSV_GEOM_NONE;
nNextFID = 1;
poFeatureDefn = new OGRFeatureDefn( pszLayerNameIn );
poFeatureDefn->Reference();
poFeatureDefn->SetGeomType( wkbNone );
bCreateCSVT = FALSE;
bDontHonourStrings = FALSE;
bWriteBOM = FALSE;
bIsEurostatTSV = FALSE;
nEurostatDims = 0;
nTotalFeatures = -1;
/* -------------------------------------------------------------------- */
/* If this is not a new file, read ahead to establish if it is */
/* already in CRLF (DOS) mode, or just a normal unix CR mode. */
/* -------------------------------------------------------------------- */
if( !bNew && bInWriteMode )
{
int nBytesRead = 0;
char chNewByte;
while( nBytesRead < 10000 && VSIFReadL( &chNewByte, 1, 1, fpCSV ) == 1 )
{
if( chNewByte == 13 )
{
bUseCRLF = TRUE;
break;
}
nBytesRead ++;
}
VSIRewindL( fpCSV );
}
/* -------------------------------------------------------------------- */
/* Check if the first record seems to be field definitions or */
/* not. We assume it is field definitions if none of the */
/* values are strictly numeric. */
/* -------------------------------------------------------------------- */
char **papszTokens = NULL;
int nFieldCount=0, iField;
CPLValueType eType;
if( !bNew )
{
const char *pszLine = NULL;
char szDelimiter[2];
szDelimiter[0] = chDelimiter; szDelimiter[1] = '\0';
pszLine = CPLReadLineL( fpCSV );
if ( pszLine != NULL )
{
/* Detect and remove UTF-8 BOM marker if found (#4623) */
if (pszLine[0] == (char)0xEF &&
pszLine[1] == (char)0xBB &&
pszLine[2] == (char)0xBF)
{
pszLine += 3;
}
/* tokenize the strings and preserve quotes, so we can separate string from numeric */
/* this is only used in the test for bHasFeldNames (bug #4361) */
papszTokens = CSLTokenizeString2( pszLine, szDelimiter,
(CSLT_HONOURSTRINGS |
CSLT_ALLOWEMPTYTOKENS |
CSLT_PRESERVEQUOTES) );
nFieldCount = CSLCount( papszTokens );
bHasFieldNames = TRUE;
for( iField = 0; iField < nFieldCount && bHasFieldNames; iField++ )
{
eType = CPLGetValueType(papszTokens[iField]);
if ( (eType == CPL_VALUE_INTEGER ||
eType == CPL_VALUE_REAL) ) {
/* we have a numeric field, therefore do not consider the first line as field names */
bHasFieldNames = FALSE;
}
}
CPLString osExt = OGRCSVDataSource::GetRealExtension(pszFilename);
/* Eurostat .tsv files */
if( EQUAL(osExt, "tsv") && nFieldCount > 1 &&
strchr(papszTokens[0], ',') != NULL && strchr(papszTokens[0], '\\') != NULL )
{
bHasFieldNames = TRUE;
bIsEurostatTSV = TRUE;
}
/* tokenize without quotes to get the actual values */
CSLDestroy( papszTokens );
// papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, FALSE );
papszTokens = CSLTokenizeString2( pszLine, szDelimiter,
(CSLT_HONOURSTRINGS |
CSLT_ALLOWEMPTYTOKENS));
nFieldCount = CSLCount( papszTokens );
}
}
else
bHasFieldNames = FALSE;
if( !bNew && !bHasFieldNames )
VSIRewindL( fpCSV );
/* -------------------------------------------------------------------- */
/* Check for geonames.org tables */
/* -------------------------------------------------------------------- */
if( !bHasFieldNames && nFieldCount == 19 )
{
if (CPLGetValueType(papszTokens[0]) == CPL_VALUE_INTEGER &&
CPLGetValueType(papszTokens[4]) == CPL_VALUE_REAL &&
CPLGetValueType(papszTokens[5]) == CPL_VALUE_REAL &&
CPLAtof(papszTokens[4]) >= -90 && CPLAtof(papszTokens[4]) <= 90 &&
CPLAtof(papszTokens[5]) >= -180 && CPLAtof(papszTokens[4]) <= 180)
{
bHasFieldNames = TRUE;
CSLDestroy(papszTokens);
papszTokens = NULL;
static const struct {
const char* pszName;
OGRFieldType eType;
}
asGeonamesFieldDesc[] =
{
{ "GEONAMEID", OFTString },
{ "NAME", OFTString },
{ "ASCIINAME", OFTString },
{ "ALTNAMES", OFTString },
{ "LATITUDE", OFTReal },
{ "LONGITUDE", OFTReal },
{ "FEATCLASS", OFTString },
{ "FEATCODE", OFTString },
{ "COUNTRY", OFTString },
{ "CC2", OFTString },
{ "ADMIN1", OFTString },
{ "ADMIN2", OFTString },
{ "ADMIN3", OFTString },
{ "ADMIN4", OFTString },
{ "POPULATION", OFTReal },
{ "ELEVATION", OFTInteger },
{ "GTOPO30", OFTInteger },
{ "TIMEZONE", OFTString },
{ "MODDATE", OFTString }
};
for(iField = 0; iField < nFieldCount; iField++)
{
OGRFieldDefn oFieldDefn(asGeonamesFieldDesc[iField].pszName,
asGeonamesFieldDesc[iField].eType);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
iLatitudeField = 4;
iLongitudeField = 5;
nFieldCount = 0;
}
}
/* -------------------------------------------------------------------- */
/* Search a csvt file for types */
/* -------------------------------------------------------------------- */
char** papszFieldTypes = NULL;
if (!bNew) {
char* dname = strdup(CPLGetDirname(pszFilename));
char* fname = strdup(CPLGetBasename(pszFilename));
VSILFILE* fpCSVT = VSIFOpenL(CPLFormFilename(dname, fname, ".csvt"), "r");
free(dname);
free(fname);
if (fpCSVT!=NULL) {
VSIRewindL(fpCSVT);
papszFieldTypes = OGRCSVReadParseLineL(fpCSVT, ',', FALSE);
VSIFCloseL(fpCSVT);
}
}
/* -------------------------------------------------------------------- */
/* Build field definitions. */
/* -------------------------------------------------------------------- */
for( iField = 0; !bIsEurostatTSV && iField < nFieldCount; iField++ )
{
char *pszFieldName = NULL;
char szFieldNameBuffer[100];
if( bHasFieldNames )
{
pszFieldName = papszTokens[iField];
// trim white space.
while( *pszFieldName == ' ' )
pszFieldName++;
while( pszFieldName[0] != '\0'
&& pszFieldName[strlen(pszFieldName)-1] == ' ' )
pszFieldName[strlen(pszFieldName)-1] = '\0';
if (*pszFieldName == '\0')
pszFieldName = NULL;
}
if (pszFieldName == NULL)
{
/* Re-read single column CSV files that have a trailing comma */
/* in the header line */
if( iField == 1 && nFieldCount == 2 && papszTokens[1][0] == '\0' )
{
nFieldCount = 1;
break;
}
pszFieldName = szFieldNameBuffer;
sprintf( szFieldNameBuffer, "field_%d", iField+1 );
}
OGRFieldDefn oField(pszFieldName, OFTString);
if (papszFieldTypes!=NULL && iField<CSLCount(papszFieldTypes)) {
char* pszLeftParenthesis = strchr(papszFieldTypes[iField], '(');
if (pszLeftParenthesis && pszLeftParenthesis != papszFieldTypes[iField] &&
pszLeftParenthesis[1] >= '0' && pszLeftParenthesis[1] <= '9')
{
int nWidth = 0;
int nPrecision = 0;
char* pszDot = strchr(pszLeftParenthesis, '.');
if (pszDot) *pszDot = 0;
*pszLeftParenthesis = 0;
if (pszLeftParenthesis[-1] == ' ')
pszLeftParenthesis[-1] = 0;
nWidth = atoi(pszLeftParenthesis+1);
if (pszDot)
nPrecision = atoi(pszDot+1);
oField.SetWidth(nWidth);
oField.SetPrecision(nPrecision);
}
if (EQUAL(papszFieldTypes[iField], "Integer"))
oField.SetType(OFTInteger);
else if (EQUAL(papszFieldTypes[iField], "Real"))
oField.SetType(OFTReal);
else if (EQUAL(papszFieldTypes[iField], "String"))
oField.SetType(OFTString);
else if (EQUAL(papszFieldTypes[iField], "Date"))
oField.SetType(OFTDate);
else if (EQUAL(papszFieldTypes[iField], "Time"))
oField.SetType(OFTTime);
else if (EQUAL(papszFieldTypes[iField], "DateTime"))
oField.SetType(OFTDateTime);
else
CPLError(CE_Warning, CPLE_NotSupported, "Unknown type : %s", papszFieldTypes[iField]);
}
if( EQUAL(oField.GetNameRef(),"WKT")
&& oField.GetType() == OFTString
&& iWktGeomReadField == -1 )
{
iWktGeomReadField = iField;
poFeatureDefn->SetGeomType( wkbUnknown );
}
/*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */
if ( pszNfdcGeomField != NULL &&
EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) &&
EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LatitudeS") )
iNfdcLatitudeS = iField;
else if ( pszNfdcGeomField != NULL &&
EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) &&
EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LongitudeS") )
iNfdcLongitudeS = iField;
/* GNIS specific */
else if ( pszGeonamesGeomFieldPrefix != NULL &&
EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) &&
(EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LAT_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE")) )
{
oField.SetType(OFTReal);
iLatitudeField = iField;
}
else if ( pszGeonamesGeomFieldPrefix != NULL &&
EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) &&
(EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONG_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE")) )
{
oField.SetType(OFTReal);
iLongitudeField = iField;
}
poFeatureDefn->AddFieldDefn( &oField );
}
if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 )
{
bDontHonourStrings = TRUE;
poFeatureDefn->SetGeomType( wkbPoint );
}
else if ( iLatitudeField != -1 && iLongitudeField != -1 )
{
poFeatureDefn->SetGeomType( wkbPoint );
}
/* -------------------------------------------------------------------- */
/* Build field definitions for Eurostat TSV files. */
/* -------------------------------------------------------------------- */
CPLString osSeqDim;
for( iField = 0; bIsEurostatTSV && iField < nFieldCount; iField++ )
{
if( iField == 0 )
{
char** papszDims = CSLTokenizeString2( papszTokens[0], ",\\", 0 );
nEurostatDims = CSLCount(papszDims) - 1;
for(int iSubField = 0; iSubField < nEurostatDims; iSubField++)
{
OGRFieldDefn oField(papszDims[iSubField], OFTString);
poFeatureDefn->AddFieldDefn( &oField );
}
osSeqDim = papszDims[nEurostatDims];
CSLDestroy(papszDims);
}
else
{
if( papszTokens[iField][0] != '\0'
&& papszTokens[iField][strlen(papszTokens[iField])-1] == ' ' )
papszTokens[iField][strlen(papszTokens[iField])-1] = '\0';
OGRFieldDefn oField(CPLSPrintf("%s_%s", osSeqDim.c_str(), papszTokens[iField]), OFTReal);
poFeatureDefn->AddFieldDefn( &oField );
OGRFieldDefn oField2(CPLSPrintf("%s_%s_flag", osSeqDim.c_str(), papszTokens[iField]), OFTString);
poFeatureDefn->AddFieldDefn( &oField2 );
}
}
/* -------------------------------------------------------------------- */
/* Cleanup. */
/* -------------------------------------------------------------------- */
CSLDestroy( papszTokens );
CSLDestroy( papszFieldTypes );
}
OGRCSVLayer::OGRCSVLayer( const char *pszLayerNameIn,
VSILFILE * fp, const char *pszFilename, int bNew, int bInWriteMode,
char chDelimiter, const char* pszNfdcGeomField,
const char* pszGeonamesGeomFieldPrefix)
{
fpCSV = fp;
iWktGeomReadField = -1;
iNfdcLatitudeS = iNfdcLongitudeS = -1;
iLatitudeField = iLongitudeField = -1;
this->bInWriteMode = bInWriteMode;
this->bNew = bNew;
this->pszFilename = CPLStrdup(pszFilename);
this->chDelimiter = chDelimiter;
bFirstFeatureAppendedDuringSession = TRUE;
bUseCRLF = FALSE;
bNeedRewindBeforeRead = FALSE;
eGeometryFormat = OGR_CSV_GEOM_NONE;
nNextFID = 1;
poFeatureDefn = new OGRFeatureDefn( pszLayerNameIn );
poFeatureDefn->Reference();
poFeatureDefn->SetGeomType( wkbNone );
bCreateCSVT = FALSE;
bDontHonourStrings = FALSE;
nTotalFeatures = -1;
/* -------------------------------------------------------------------- */
/* If this is not a new file, read ahead to establish if it is */
/* already in CRLF (DOS) mode, or just a normal unix CR mode. */
/* -------------------------------------------------------------------- */
if( !bNew && bInWriteMode )
{
int nBytesRead = 0;
char chNewByte;
while( nBytesRead < 10000 && VSIFReadL( &chNewByte, 1, 1, fpCSV ) == 1 )
{
if( chNewByte == 13 )
{
bUseCRLF = TRUE;
break;
}
}
VSIRewindL( fpCSV );
}
/* -------------------------------------------------------------------- */
/* Check if the first record seems to be field definitions or */
/* not. We assume it is field definitions if none of the */
/* values are strictly numeric. */
/* -------------------------------------------------------------------- */
char **papszTokens = NULL;
int nFieldCount=0, iField;
CPLValueType eType;
if( !bNew )
{
papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, FALSE );
nFieldCount = CSLCount( papszTokens );
bHasFieldNames = TRUE;
}
else
bHasFieldNames = FALSE;
for( iField = 0; iField < nFieldCount && bHasFieldNames; iField++ )
{
eType = CPLGetValueType(papszTokens[iField]);
if ( eType == CPL_VALUE_INTEGER ||
eType == CPL_VALUE_REAL ) {
/* we have a numeric field, therefore do not consider the first line as field names */
bHasFieldNames = FALSE;
}
}
if( !bNew && !bHasFieldNames )
VSIRewindL( fpCSV );
/* -------------------------------------------------------------------- */
/* Check for geonames.org tables */
/* -------------------------------------------------------------------- */
if( !bHasFieldNames && nFieldCount == 19 )
{
if (CPLGetValueType(papszTokens[0]) == CPL_VALUE_INTEGER &&
CPLGetValueType(papszTokens[4]) == CPL_VALUE_REAL &&
CPLGetValueType(papszTokens[5]) == CPL_VALUE_REAL &&
CPLAtof(papszTokens[4]) >= -90 && CPLAtof(papszTokens[4]) <= 90 &&
CPLAtof(papszTokens[5]) >= -180 && CPLAtof(papszTokens[4]) <= 180)
{
bHasFieldNames = TRUE;
CSLDestroy(papszTokens);
papszTokens = NULL;
static const struct {
const char* pszName;
OGRFieldType eType;
}
asGeonamesFieldDesc[] =
{
{ "GEONAMEID", OFTString },
{ "NAME", OFTString },
{ "ASCIINAME", OFTString },
{ "ALTNAMES", OFTString },
{ "LATITUDE", OFTReal },
{ "LONGITUDE", OFTReal },
{ "FEATCLASS", OFTString },
{ "FEATCODE", OFTString },
{ "COUNTRY", OFTString },
{ "CC2", OFTString },
{ "ADMIN1", OFTString },
{ "ADMIN2", OFTString },
{ "ADMIN3", OFTString },
{ "ADMIN4", OFTString },
{ "POPULATION", OFTReal },
{ "ELEVATION", OFTInteger },
{ "GTOPO30", OFTInteger },
{ "TIMEZONE", OFTString },
{ "MODDATE", OFTString }
};
for(iField = 0; iField < nFieldCount; iField++)
{
OGRFieldDefn oFieldDefn(asGeonamesFieldDesc[iField].pszName,
asGeonamesFieldDesc[iField].eType);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
iLatitudeField = 4;
iLongitudeField = 5;
nFieldCount = 0;
}
}
/* -------------------------------------------------------------------- */
/* Search a csvt file for types */
/* -------------------------------------------------------------------- */
char** papszFieldTypes = NULL;
if (!bNew) {
char* dname = strdup(CPLGetDirname(pszFilename));
char* fname = strdup(CPLGetBasename(pszFilename));
VSILFILE* fpCSVT = VSIFOpenL(CPLFormFilename(dname, fname, ".csvt"), "r");
free(dname);
free(fname);
if (fpCSVT!=NULL) {
VSIRewindL(fpCSVT);
papszFieldTypes = OGRCSVReadParseLineL(fpCSVT, ',', FALSE);
VSIFCloseL(fpCSVT);
}
}
/* -------------------------------------------------------------------- */
/* Build field definitions. */
/* -------------------------------------------------------------------- */
for( iField = 0; iField < nFieldCount; iField++ )
{
char *pszFieldName = NULL;
char szFieldNameBuffer[100];
if( bHasFieldNames )
{
pszFieldName = papszTokens[iField];
// trim white space.
while( *pszFieldName == ' ' )
pszFieldName++;
while( pszFieldName[0] != '\0'
&& pszFieldName[strlen(pszFieldName)-1] == ' ' )
pszFieldName[strlen(pszFieldName)-1] = '\0';
if (*pszFieldName == '\0')
pszFieldName = NULL;
}
if (pszFieldName == NULL)
{
pszFieldName = szFieldNameBuffer;
sprintf( szFieldNameBuffer, "field_%d", iField+1 );
}
OGRFieldDefn oField(pszFieldName, OFTString);
if (papszFieldTypes!=NULL && iField<CSLCount(papszFieldTypes)) {
char* pszLeftParenthesis = strchr(papszFieldTypes[iField], '(');
if (pszLeftParenthesis && pszLeftParenthesis != papszFieldTypes[iField] &&
pszLeftParenthesis[1] >= '0' && pszLeftParenthesis[1] <= '9')
{
int nWidth = 0;
int nPrecision = 0;
char* pszDot = strchr(pszLeftParenthesis, '.');
if (pszDot) *pszDot = 0;
*pszLeftParenthesis = 0;
if (pszLeftParenthesis[-1] == ' ')
pszLeftParenthesis[-1] = 0;
nWidth = atoi(pszLeftParenthesis+1);
if (pszDot)
nPrecision = atoi(pszDot+1);
oField.SetWidth(nWidth);
oField.SetPrecision(nPrecision);
}
if (EQUAL(papszFieldTypes[iField], "Integer"))
oField.SetType(OFTInteger);
else if (EQUAL(papszFieldTypes[iField], "Real"))
oField.SetType(OFTReal);
else if (EQUAL(papszFieldTypes[iField], "String"))
oField.SetType(OFTString);
else if (EQUAL(papszFieldTypes[iField], "Date"))
oField.SetType(OFTDate);
else if (EQUAL(papszFieldTypes[iField], "Time"))
oField.SetType(OFTTime);
else if (EQUAL(papszFieldTypes[iField], "DateTime"))
oField.SetType(OFTDateTime);
else
CPLError(CE_Warning, CPLE_NotSupported, "Unknown type : %s", papszFieldTypes[iField]);
}
if( EQUAL(oField.GetNameRef(),"WKT")
&& oField.GetType() == OFTString
&& iWktGeomReadField == -1 )
{
iWktGeomReadField = iField;
poFeatureDefn->SetGeomType( wkbUnknown );
}
/*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */
if ( pszNfdcGeomField != NULL &&
EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) &&
EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LatitudeS") )
iNfdcLatitudeS = iField;
else if ( pszNfdcGeomField != NULL &&
EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) &&
EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LongitudeS") )
iNfdcLongitudeS = iField;
/* GNIS specific */
else if ( pszGeonamesGeomFieldPrefix != NULL &&
EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) &&
(EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LAT_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE")) )
{
oField.SetType(OFTReal);
iLatitudeField = iField;
}
else if ( pszGeonamesGeomFieldPrefix != NULL &&
EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) &&
(EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONG_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE_DEC") ||
EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE")) )
{
oField.SetType(OFTReal);
iLongitudeField = iField;
}
poFeatureDefn->AddFieldDefn( &oField );
}
if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 )
{
bDontHonourStrings = TRUE;
poFeatureDefn->SetGeomType( wkbPoint );
}
else if ( iLatitudeField != -1 && iLongitudeField != -1 )
{
poFeatureDefn->SetGeomType( wkbPoint );
}
CSLDestroy( papszTokens );
CSLDestroy( papszFieldTypes );
}
OGRFeature * OGRCSVLayer::GetNextUnfilteredFeature()
{
if (fpCSV == NULL)
return NULL;
/* -------------------------------------------------------------------- */
/* Read the CSV record. */
/* -------------------------------------------------------------------- */
char **papszTokens;
while(TRUE)
{
papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings );
if( papszTokens == NULL )
return NULL;
if( papszTokens[0] != NULL )
break;
CSLDestroy(papszTokens);
}
/* -------------------------------------------------------------------- */
/* Create the OGR feature. */
/* -------------------------------------------------------------------- */
OGRFeature *poFeature;
poFeature = new OGRFeature( poFeatureDefn );
/* -------------------------------------------------------------------- */
/* Set attributes for any indicated attribute records. */
/* -------------------------------------------------------------------- */
int iAttr;
int nAttrCount = MIN(CSLCount(papszTokens),
poFeatureDefn->GetFieldCount() );
for( iAttr = 0; iAttr < nAttrCount; iAttr++)
{
if( iAttr == iWktGeomReadField && papszTokens[iAttr][0] != '\0' )
{
char *pszWKT = papszTokens[iAttr];
OGRGeometry *poGeom = NULL;
if( OGRGeometryFactory::createFromWkt( &pszWKT, NULL, &poGeom )
== OGRERR_NONE )
poFeature->SetGeometryDirectly( poGeom );
}
if (poFeatureDefn->GetFieldDefn(iAttr)->GetType() == OFTReal)
{
if (papszTokens[iAttr][0] != '\0')
poFeature->SetField( iAttr, CPLAtof(papszTokens[iAttr]) );
}
else if (poFeatureDefn->GetFieldDefn(iAttr)->GetType() != OFTString)
{
if (papszTokens[iAttr][0] != '\0')
poFeature->SetField( iAttr, papszTokens[iAttr] );
}
else
poFeature->SetField( iAttr, papszTokens[iAttr] );
}
/* -------------------------------------------------------------------- */
/*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */
/* -------------------------------------------------------------------- */
if ( iNfdcLatitudeS != -1 &&
iNfdcLongitudeS != -1 &&
nAttrCount > iNfdcLatitudeS &&
nAttrCount > iNfdcLongitudeS &&
papszTokens[iNfdcLongitudeS][0] != 0 &&
papszTokens[iNfdcLatitudeS][0] != 0)
{
double dfLon = atof(papszTokens[iNfdcLongitudeS]) / 3600;
if (strchr(papszTokens[iNfdcLongitudeS], 'W'))
dfLon *= -1;
double dfLat = atof(papszTokens[iNfdcLatitudeS]) / 3600;
if (strchr(papszTokens[iNfdcLatitudeS], 'S'))
dfLat *= -1;
poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) );
}
/* -------------------------------------------------------------------- */
/* GNIS specific */
/* -------------------------------------------------------------------- */
else if ( iLatitudeField != -1 &&
iLongitudeField != -1 &&
nAttrCount > iLatitudeField &&
nAttrCount > iLongitudeField &&
papszTokens[iLongitudeField][0] != 0 &&
papszTokens[iLatitudeField][0] != 0)
{
/* Some records have dummy 0,0 value */
if (papszTokens[iLongitudeField][0] != '0' ||
papszTokens[iLongitudeField][1] != '\0' ||
papszTokens[iLatitudeField][0] != '0' ||
papszTokens[iLatitudeField][1] != '\0')
{
double dfLon = atof(papszTokens[iLongitudeField]);
double dfLat = atof(papszTokens[iLatitudeField]);
poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) );
}
}
CSLDestroy( papszTokens );
/* -------------------------------------------------------------------- */
/* Translate the record id. */
/* -------------------------------------------------------------------- */
poFeature->SetFID( nNextFID++ );
m_nFeaturesRead++;
return poFeature;
}
