int OGRCSVLayer::GetFeatureCount( int bForce ) { if (bInWriteMode || m_poFilterGeom != NULL || m_poAttrQuery != NULL) return OGRLayer::GetFeatureCount(bForce); if (nTotalFeatures >= 0) return nTotalFeatures; if (fpCSV == NULL) return 0; ResetReading(); char **papszTokens; nTotalFeatures = 0; while(TRUE) { papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings ); if( papszTokens == NULL ) break; if( papszTokens[0] != NULL ) nTotalFeatures ++; CSLDestroy(papszTokens); } ResetReading(); return nTotalFeatures; }
void OGRCSVLayer::ResetReading() { if (fpCSV) VSIRewindL( fpCSV ); if( bHasFieldNames ) CSLDestroy( OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings ) ); bNeedRewindBeforeRead = FALSE; nNextFID = 1; }
int OGRCSVDataSource::OpenTable( const char * pszFilename, char** papszOpenOptions, const char* pszNfdcRunwaysGeomField, const char* pszGeonamesGeomFieldPrefix) { /* -------------------------------------------------------------------- */ /* Open the file. */ /* -------------------------------------------------------------------- */ VSILFILE * fp; if( bUpdate ) fp = VSIFOpenL( pszFilename, "rb+" ); else fp = VSIFOpenL( pszFilename, "rb" ); if( fp == NULL ) { CPLError( CE_Warning, CPLE_OpenFailed, "Failed to open %s, %s.", pszFilename, VSIStrerror( errno ) ); return FALSE; } if( !bUpdate && strstr(pszFilename, "/vsigzip/") == NULL && strstr(pszFilename, "/vsizip/") == NULL ) fp = (VSILFILE*) VSICreateBufferedReaderHandle((VSIVirtualHandle*)fp); CPLString osLayerName = CPLGetBasename(pszFilename); CPLString osExt = CPLGetExtension(pszFilename); if( strncmp(pszFilename, "/vsigzip/", 9) == 0 && EQUAL(osExt, "gz") ) { if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".csv.gz") ) { osLayerName = osLayerName.substr(0, osLayerName.size() - 4); osExt = "csv"; } else if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".tsv.gz") ) { osLayerName = osLayerName.substr(0, osLayerName.size() - 4); osExt = "tsv"; } } /* -------------------------------------------------------------------- */ /* Read and parse a line. Did we get multiple fields? */ /* -------------------------------------------------------------------- */ const char* pszLine = CPLReadLineL( fp ); if (pszLine == NULL) { VSIFCloseL( fp ); return FALSE; } char chDelimiter = CSVDetectSeperator(pszLine); if( chDelimiter != '\t' && strchr(pszLine, '\t') != NULL ) { /* Force the delimiter to be TAB for a .tsv file that has a tabulation */ /* in its first line */ if( EQUAL(osExt, "tsv") ) { chDelimiter = '\t'; } else { for(int bDontHonourStrings=0; bDontHonourStrings<=1; bDontHonourStrings++) { // Read the first 2 lines to see if they have the same number of fields, if using tabulation VSIRewindL( fp ); char** papszTokens = OGRCSVReadParseLineL( fp, '\t', bDontHonourStrings ); int nTokens1 = CSLCount(papszTokens); CSLDestroy(papszTokens); papszTokens = OGRCSVReadParseLineL( fp, '\t', bDontHonourStrings ); int nTokens2 = CSLCount(papszTokens); CSLDestroy(papszTokens); if( nTokens1 >= 2 && nTokens1 == nTokens2 ) { chDelimiter = '\t'; break; } } } } VSIRewindL( fp ); #if 0 const char *pszDelimiter = CSLFetchNameValueDef( papszOpenOptions, "SEPARATOR", "AUTO"); if( !EQUAL(pszDelimiter, "AUTO") ) { if (EQUAL(pszDelimiter, "COMMA")) chDelimiter = ','; else if (EQUAL(pszDelimiter, "SEMICOLON")) chDelimiter = ';'; else if (EQUAL(pszDelimiter, "TAB")) chDelimiter = '\t'; else if (EQUAL(pszDelimiter, "SPACE")) chDelimiter = ' '; else { CPLError( CE_Warning, CPLE_AppDefined, "SEPARATOR=%s not understood, use one of COMMA, SEMICOLON, SPACE or TAB.", pszDelimiter ); } } #endif /* GNIS specific */ if (pszGeonamesGeomFieldPrefix != NULL && strchr(pszLine, '|') != NULL) chDelimiter = '|'; char **papszFields = OGRCSVReadParseLineL( fp, chDelimiter, FALSE ); if( CSLCount(papszFields) < 2 ) { VSIFCloseL( fp ); CSLDestroy( papszFields ); return FALSE; } VSIRewindL( fp ); CSLDestroy( papszFields ); /* -------------------------------------------------------------------- */ /* Create a layer. */ /* -------------------------------------------------------------------- */ nLayers++; papoLayers = (OGRCSVLayer **) CPLRealloc(papoLayers, sizeof(void*) * nLayers); if (pszNfdcRunwaysGeomField != NULL) { osLayerName += "_"; osLayerName += pszNfdcRunwaysGeomField; } else if (pszGeonamesGeomFieldPrefix != NULL && !EQUAL(pszGeonamesGeomFieldPrefix, "")) { osLayerName += "_"; osLayerName += pszGeonamesGeomFieldPrefix; } if (EQUAL(pszFilename, "/vsistdin/")) osLayerName = "layer"; papoLayers[nLayers-1] = new OGRCSVLayer( osLayerName, fp, pszFilename, FALSE, bUpdate, chDelimiter ); papoLayers[nLayers-1]->BuildFeatureDefn( pszNfdcRunwaysGeomField, pszGeonamesGeomFieldPrefix, papszOpenOptions ); return TRUE; }
int OGRCSVDataSource::OpenTable( const char * pszFilename, const char* pszNfdcRunwaysGeomField, const char* pszGeonamesGeomFieldPrefix) { /* -------------------------------------------------------------------- */ /* Open the file. */ /* -------------------------------------------------------------------- */ VSILFILE * fp; if( bUpdate ) fp = VSIFOpenL( pszFilename, "rb+" ); else fp = VSIFOpenL( pszFilename, "rb" ); if( fp == NULL ) { CPLError( CE_Warning, CPLE_OpenFailed, "Failed to open %s, %s.", pszFilename, VSIStrerror( errno ) ); return FALSE; } if( !bUpdate && strstr(pszFilename, "/vsigzip/") == NULL && strstr(pszFilename, "/vsizip/") == NULL ) fp = (VSILFILE*) VSICreateBufferedReaderHandle((VSIVirtualHandle*)fp); CPLString osLayerName = CPLGetBasename(pszFilename); CPLString osExt = CPLGetExtension(pszFilename); if( strncmp(pszFilename, "/vsigzip/", 9) == 0 && EQUAL(osExt, "gz") ) { if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".csv.gz") ) { osLayerName = osLayerName.substr(0, osLayerName.size() - 4); osExt = "csv"; } else if( strlen(pszFilename) > 7 && EQUAL(pszFilename + strlen(pszFilename) - 7, ".tsv.gz") ) { osLayerName = osLayerName.substr(0, osLayerName.size() - 4); osExt = "tsv"; } } /* -------------------------------------------------------------------- */ /* Read and parse a line. Did we get multiple fields? */ /* -------------------------------------------------------------------- */ const char* pszLine = CPLReadLineL( fp ); if (pszLine == NULL) { VSIFCloseL( fp ); return FALSE; } char chDelimiter = CSVDetectSeperator(pszLine); /* Force the delimiter to be TAB for a .tsv file that has a tabulation */ /* in its first line */ if( EQUAL(osExt, "tsv") && chDelimiter != '\t' && strchr(pszLine, '\t') != NULL ) { chDelimiter = '\t'; } VSIRewindL( fp ); /* GNIS specific */ if (pszGeonamesGeomFieldPrefix != NULL && strchr(pszLine, '|') != NULL) chDelimiter = '|'; char **papszFields = OGRCSVReadParseLineL( fp, chDelimiter, FALSE ); if( CSLCount(papszFields) < 2 ) { VSIFCloseL( fp ); CSLDestroy( papszFields ); return FALSE; } VSIRewindL( fp ); CSLDestroy( papszFields ); /* -------------------------------------------------------------------- */ /* Create a layer. */ /* -------------------------------------------------------------------- */ nLayers++; papoLayers = (OGRCSVLayer **) CPLRealloc(papoLayers, sizeof(void*) * nLayers); if (pszNfdcRunwaysGeomField != NULL) { osLayerName += "_"; osLayerName += pszNfdcRunwaysGeomField; } else if (pszGeonamesGeomFieldPrefix != NULL && !EQUAL(pszGeonamesGeomFieldPrefix, "")) { osLayerName += "_"; osLayerName += pszGeonamesGeomFieldPrefix; } if (EQUAL(pszFilename, "/vsistdin/")) osLayerName = "layer"; papoLayers[nLayers-1] = new OGRCSVLayer( osLayerName, fp, pszFilename, FALSE, bUpdate, chDelimiter, pszNfdcRunwaysGeomField, pszGeonamesGeomFieldPrefix ); return TRUE; }
OGRFeature * OGRCSVLayer::GetNextUnfilteredFeature() { if (fpCSV == NULL) return NULL; /* -------------------------------------------------------------------- */ /* Read the CSV record. */ /* -------------------------------------------------------------------- */ char **papszTokens; while(TRUE) { papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings ); if( papszTokens == NULL ) return NULL; if( papszTokens[0] != NULL ) break; CSLDestroy(papszTokens); } /* -------------------------------------------------------------------- */ /* Create the OGR feature. */ /* -------------------------------------------------------------------- */ OGRFeature *poFeature; poFeature = new OGRFeature( poFeatureDefn ); /* -------------------------------------------------------------------- */ /* Set attributes for any indicated attribute records. */ /* -------------------------------------------------------------------- */ int iAttr; int nAttrCount = MIN(CSLCount(papszTokens), poFeatureDefn->GetFieldCount() ); CPLValueType eType; for( iAttr = 0; !bIsEurostatTSV && iAttr < nAttrCount; iAttr++) { if( iAttr == iWktGeomReadField && papszTokens[iAttr][0] != '\0' ) { char *pszWKT = papszTokens[iAttr]; OGRGeometry *poGeom = NULL; if( OGRGeometryFactory::createFromWkt( &pszWKT, NULL, &poGeom ) == OGRERR_NONE ) poFeature->SetGeometryDirectly( poGeom ); } OGRFieldType eFieldType = poFeatureDefn->GetFieldDefn(iAttr)->GetType(); if ( eFieldType == OFTReal || eFieldType == OFTInteger ) { if (chDelimiter == ';' && eFieldType == OFTReal) { char* chComma = strchr(papszTokens[iAttr], ','); if (chComma) *chComma = '.'; } eType = CPLGetValueType(papszTokens[iAttr]); if ( (papszTokens[iAttr][0] != '\0') && ( eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL ) ) { poFeature->SetField( iAttr, papszTokens[iAttr] ); } } else if (eFieldType != OFTString) { if (papszTokens[iAttr][0] != '\0') poFeature->SetField( iAttr, papszTokens[iAttr] ); } else poFeature->SetField( iAttr, papszTokens[iAttr] ); } /* -------------------------------------------------------------------- */ /* Eurostat TSV files. */ /* -------------------------------------------------------------------- */ for( iAttr = 0; bIsEurostatTSV && iAttr < nAttrCount; iAttr++) { if( iAttr == 0 ) { char** papszDims = CSLTokenizeString2( papszTokens[0], ",", 0 ); if( CSLCount(papszDims) != nEurostatDims ) { CSLDestroy(papszDims); break; } for( int iSubAttr = 0; iSubAttr < nEurostatDims; iSubAttr ++ ) { poFeature->SetField( iSubAttr, papszDims[iSubAttr] ); } CSLDestroy(papszDims); } else { char** papszVals = CSLTokenizeString2( papszTokens[iAttr], " ", 0 ); eType = CPLGetValueType(papszVals[0]); if ( (papszVals[0][0] != '\0') && ( eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL ) ) { poFeature->SetField( nEurostatDims + 2 * (iAttr - 1), papszVals[0] ); } if( CSLCount(papszVals) == 2 ) { poFeature->SetField( nEurostatDims + 2 * (iAttr - 1) + 1, papszVals[1] ); } CSLDestroy(papszVals); } } /* -------------------------------------------------------------------- */ /*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */ /* -------------------------------------------------------------------- */ if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 && nAttrCount > iNfdcLatitudeS && nAttrCount > iNfdcLongitudeS && papszTokens[iNfdcLongitudeS][0] != 0 && papszTokens[iNfdcLatitudeS][0] != 0) { double dfLon = atof(papszTokens[iNfdcLongitudeS]) / 3600; if (strchr(papszTokens[iNfdcLongitudeS], 'W')) dfLon *= -1; double dfLat = atof(papszTokens[iNfdcLatitudeS]) / 3600; if (strchr(papszTokens[iNfdcLatitudeS], 'S')) dfLat *= -1; poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) ); } /* -------------------------------------------------------------------- */ /* GNIS specific */ /* -------------------------------------------------------------------- */ else if ( iLatitudeField != -1 && iLongitudeField != -1 && nAttrCount > iLatitudeField && nAttrCount > iLongitudeField && papszTokens[iLongitudeField][0] != 0 && papszTokens[iLatitudeField][0] != 0) { /* Some records have dummy 0,0 value */ if (papszTokens[iLongitudeField][0] != '0' || papszTokens[iLongitudeField][1] != '\0' || papszTokens[iLatitudeField][0] != '0' || papszTokens[iLatitudeField][1] != '\0') { double dfLon = atof(papszTokens[iLongitudeField]); double dfLat = atof(papszTokens[iLatitudeField]); poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) ); } } CSLDestroy( papszTokens ); /* -------------------------------------------------------------------- */ /* Translate the record id. */ /* -------------------------------------------------------------------- */ poFeature->SetFID( nNextFID++ ); m_nFeaturesRead++; return poFeature; }
OGRCSVLayer::OGRCSVLayer( const char *pszLayerNameIn, VSILFILE * fp, const char *pszFilename, int bNew, int bInWriteMode, char chDelimiter, const char* pszNfdcGeomField, const char* pszGeonamesGeomFieldPrefix) { fpCSV = fp; iWktGeomReadField = -1; iNfdcLatitudeS = iNfdcLongitudeS = -1; iLatitudeField = iLongitudeField = -1; this->bInWriteMode = bInWriteMode; this->bNew = bNew; this->pszFilename = CPLStrdup(pszFilename); this->chDelimiter = chDelimiter; bFirstFeatureAppendedDuringSession = TRUE; bUseCRLF = FALSE; bNeedRewindBeforeRead = FALSE; eGeometryFormat = OGR_CSV_GEOM_NONE; nNextFID = 1; poFeatureDefn = new OGRFeatureDefn( pszLayerNameIn ); poFeatureDefn->Reference(); poFeatureDefn->SetGeomType( wkbNone ); bCreateCSVT = FALSE; bDontHonourStrings = FALSE; bWriteBOM = FALSE; bIsEurostatTSV = FALSE; nEurostatDims = 0; nTotalFeatures = -1; /* -------------------------------------------------------------------- */ /* If this is not a new file, read ahead to establish if it is */ /* already in CRLF (DOS) mode, or just a normal unix CR mode. */ /* -------------------------------------------------------------------- */ if( !bNew && bInWriteMode ) { int nBytesRead = 0; char chNewByte; while( nBytesRead < 10000 && VSIFReadL( &chNewByte, 1, 1, fpCSV ) == 1 ) { if( chNewByte == 13 ) { bUseCRLF = TRUE; break; } nBytesRead ++; } VSIRewindL( fpCSV ); } /* -------------------------------------------------------------------- */ /* Check if the first record seems to be field definitions or */ /* not. We assume it is field definitions if none of the */ /* values are strictly numeric. */ /* -------------------------------------------------------------------- */ char **papszTokens = NULL; int nFieldCount=0, iField; CPLValueType eType; if( !bNew ) { const char *pszLine = NULL; char szDelimiter[2]; szDelimiter[0] = chDelimiter; szDelimiter[1] = '\0'; pszLine = CPLReadLineL( fpCSV ); if ( pszLine != NULL ) { /* Detect and remove UTF-8 BOM marker if found (#4623) */ if (pszLine[0] == (char)0xEF && pszLine[1] == (char)0xBB && pszLine[2] == (char)0xBF) { pszLine += 3; } /* tokenize the strings and preserve quotes, so we can separate string from numeric */ /* this is only used in the test for bHasFeldNames (bug #4361) */ papszTokens = CSLTokenizeString2( pszLine, szDelimiter, (CSLT_HONOURSTRINGS | CSLT_ALLOWEMPTYTOKENS | CSLT_PRESERVEQUOTES) ); nFieldCount = CSLCount( papszTokens ); bHasFieldNames = TRUE; for( iField = 0; iField < nFieldCount && bHasFieldNames; iField++ ) { eType = CPLGetValueType(papszTokens[iField]); if ( (eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL) ) { /* we have a numeric field, therefore do not consider the first line as field names */ bHasFieldNames = FALSE; } } CPLString osExt = OGRCSVDataSource::GetRealExtension(pszFilename); /* Eurostat .tsv files */ if( EQUAL(osExt, "tsv") && nFieldCount > 1 && strchr(papszTokens[0], ',') != NULL && strchr(papszTokens[0], '\\') != NULL ) { bHasFieldNames = TRUE; bIsEurostatTSV = TRUE; } /* tokenize without quotes to get the actual values */ CSLDestroy( papszTokens ); // papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, FALSE ); papszTokens = CSLTokenizeString2( pszLine, szDelimiter, (CSLT_HONOURSTRINGS | CSLT_ALLOWEMPTYTOKENS)); nFieldCount = CSLCount( papszTokens ); } } else bHasFieldNames = FALSE; if( !bNew && !bHasFieldNames ) VSIRewindL( fpCSV ); /* -------------------------------------------------------------------- */ /* Check for geonames.org tables */ /* -------------------------------------------------------------------- */ if( !bHasFieldNames && nFieldCount == 19 ) { if (CPLGetValueType(papszTokens[0]) == CPL_VALUE_INTEGER && CPLGetValueType(papszTokens[4]) == CPL_VALUE_REAL && CPLGetValueType(papszTokens[5]) == CPL_VALUE_REAL && CPLAtof(papszTokens[4]) >= -90 && CPLAtof(papszTokens[4]) <= 90 && CPLAtof(papszTokens[5]) >= -180 && CPLAtof(papszTokens[4]) <= 180) { bHasFieldNames = TRUE; CSLDestroy(papszTokens); papszTokens = NULL; static const struct { const char* pszName; OGRFieldType eType; } asGeonamesFieldDesc[] = { { "GEONAMEID", OFTString }, { "NAME", OFTString }, { "ASCIINAME", OFTString }, { "ALTNAMES", OFTString }, { "LATITUDE", OFTReal }, { "LONGITUDE", OFTReal }, { "FEATCLASS", OFTString }, { "FEATCODE", OFTString }, { "COUNTRY", OFTString }, { "CC2", OFTString }, { "ADMIN1", OFTString }, { "ADMIN2", OFTString }, { "ADMIN3", OFTString }, { "ADMIN4", OFTString }, { "POPULATION", OFTReal }, { "ELEVATION", OFTInteger }, { "GTOPO30", OFTInteger }, { "TIMEZONE", OFTString }, { "MODDATE", OFTString } }; for(iField = 0; iField < nFieldCount; iField++) { OGRFieldDefn oFieldDefn(asGeonamesFieldDesc[iField].pszName, asGeonamesFieldDesc[iField].eType); poFeatureDefn->AddFieldDefn(&oFieldDefn); } iLatitudeField = 4; iLongitudeField = 5; nFieldCount = 0; } } /* -------------------------------------------------------------------- */ /* Search a csvt file for types */ /* -------------------------------------------------------------------- */ char** papszFieldTypes = NULL; if (!bNew) { char* dname = strdup(CPLGetDirname(pszFilename)); char* fname = strdup(CPLGetBasename(pszFilename)); VSILFILE* fpCSVT = VSIFOpenL(CPLFormFilename(dname, fname, ".csvt"), "r"); free(dname); free(fname); if (fpCSVT!=NULL) { VSIRewindL(fpCSVT); papszFieldTypes = OGRCSVReadParseLineL(fpCSVT, ',', FALSE); VSIFCloseL(fpCSVT); } } /* -------------------------------------------------------------------- */ /* Build field definitions. */ /* -------------------------------------------------------------------- */ for( iField = 0; !bIsEurostatTSV && iField < nFieldCount; iField++ ) { char *pszFieldName = NULL; char szFieldNameBuffer[100]; if( bHasFieldNames ) { pszFieldName = papszTokens[iField]; // trim white space. while( *pszFieldName == ' ' ) pszFieldName++; while( pszFieldName[0] != '\0' && pszFieldName[strlen(pszFieldName)-1] == ' ' ) pszFieldName[strlen(pszFieldName)-1] = '\0'; if (*pszFieldName == '\0') pszFieldName = NULL; } if (pszFieldName == NULL) { /* Re-read single column CSV files that have a trailing comma */ /* in the header line */ if( iField == 1 && nFieldCount == 2 && papszTokens[1][0] == '\0' ) { nFieldCount = 1; break; } pszFieldName = szFieldNameBuffer; sprintf( szFieldNameBuffer, "field_%d", iField+1 ); } OGRFieldDefn oField(pszFieldName, OFTString); if (papszFieldTypes!=NULL && iField<CSLCount(papszFieldTypes)) { char* pszLeftParenthesis = strchr(papszFieldTypes[iField], '('); if (pszLeftParenthesis && pszLeftParenthesis != papszFieldTypes[iField] && pszLeftParenthesis[1] >= '0' && pszLeftParenthesis[1] <= '9') { int nWidth = 0; int nPrecision = 0; char* pszDot = strchr(pszLeftParenthesis, '.'); if (pszDot) *pszDot = 0; *pszLeftParenthesis = 0; if (pszLeftParenthesis[-1] == ' ') pszLeftParenthesis[-1] = 0; nWidth = atoi(pszLeftParenthesis+1); if (pszDot) nPrecision = atoi(pszDot+1); oField.SetWidth(nWidth); oField.SetPrecision(nPrecision); } if (EQUAL(papszFieldTypes[iField], "Integer")) oField.SetType(OFTInteger); else if (EQUAL(papszFieldTypes[iField], "Real")) oField.SetType(OFTReal); else if (EQUAL(papszFieldTypes[iField], "String")) oField.SetType(OFTString); else if (EQUAL(papszFieldTypes[iField], "Date")) oField.SetType(OFTDate); else if (EQUAL(papszFieldTypes[iField], "Time")) oField.SetType(OFTTime); else if (EQUAL(papszFieldTypes[iField], "DateTime")) oField.SetType(OFTDateTime); else CPLError(CE_Warning, CPLE_NotSupported, "Unknown type : %s", papszFieldTypes[iField]); } if( EQUAL(oField.GetNameRef(),"WKT") && oField.GetType() == OFTString && iWktGeomReadField == -1 ) { iWktGeomReadField = iField; poFeatureDefn->SetGeomType( wkbUnknown ); } /*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */ if ( pszNfdcGeomField != NULL && EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) && EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LatitudeS") ) iNfdcLatitudeS = iField; else if ( pszNfdcGeomField != NULL && EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) && EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LongitudeS") ) iNfdcLongitudeS = iField; /* GNIS specific */ else if ( pszGeonamesGeomFieldPrefix != NULL && EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) && (EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LAT_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE")) ) { oField.SetType(OFTReal); iLatitudeField = iField; } else if ( pszGeonamesGeomFieldPrefix != NULL && EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) && (EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONG_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE")) ) { oField.SetType(OFTReal); iLongitudeField = iField; } poFeatureDefn->AddFieldDefn( &oField ); } if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 ) { bDontHonourStrings = TRUE; poFeatureDefn->SetGeomType( wkbPoint ); } else if ( iLatitudeField != -1 && iLongitudeField != -1 ) { poFeatureDefn->SetGeomType( wkbPoint ); } /* -------------------------------------------------------------------- */ /* Build field definitions for Eurostat TSV files. */ /* -------------------------------------------------------------------- */ CPLString osSeqDim; for( iField = 0; bIsEurostatTSV && iField < nFieldCount; iField++ ) { if( iField == 0 ) { char** papszDims = CSLTokenizeString2( papszTokens[0], ",\\", 0 ); nEurostatDims = CSLCount(papszDims) - 1; for(int iSubField = 0; iSubField < nEurostatDims; iSubField++) { OGRFieldDefn oField(papszDims[iSubField], OFTString); poFeatureDefn->AddFieldDefn( &oField ); } osSeqDim = papszDims[nEurostatDims]; CSLDestroy(papszDims); } else { if( papszTokens[iField][0] != '\0' && papszTokens[iField][strlen(papszTokens[iField])-1] == ' ' ) papszTokens[iField][strlen(papszTokens[iField])-1] = '\0'; OGRFieldDefn oField(CPLSPrintf("%s_%s", osSeqDim.c_str(), papszTokens[iField]), OFTReal); poFeatureDefn->AddFieldDefn( &oField ); OGRFieldDefn oField2(CPLSPrintf("%s_%s_flag", osSeqDim.c_str(), papszTokens[iField]), OFTString); poFeatureDefn->AddFieldDefn( &oField2 ); } } /* -------------------------------------------------------------------- */ /* Cleanup. */ /* -------------------------------------------------------------------- */ CSLDestroy( papszTokens ); CSLDestroy( papszFieldTypes ); }
OGRCSVLayer::OGRCSVLayer( const char *pszLayerNameIn, VSILFILE * fp, const char *pszFilename, int bNew, int bInWriteMode, char chDelimiter, const char* pszNfdcGeomField, const char* pszGeonamesGeomFieldPrefix) { fpCSV = fp; iWktGeomReadField = -1; iNfdcLatitudeS = iNfdcLongitudeS = -1; iLatitudeField = iLongitudeField = -1; this->bInWriteMode = bInWriteMode; this->bNew = bNew; this->pszFilename = CPLStrdup(pszFilename); this->chDelimiter = chDelimiter; bFirstFeatureAppendedDuringSession = TRUE; bUseCRLF = FALSE; bNeedRewindBeforeRead = FALSE; eGeometryFormat = OGR_CSV_GEOM_NONE; nNextFID = 1; poFeatureDefn = new OGRFeatureDefn( pszLayerNameIn ); poFeatureDefn->Reference(); poFeatureDefn->SetGeomType( wkbNone ); bCreateCSVT = FALSE; bDontHonourStrings = FALSE; nTotalFeatures = -1; /* -------------------------------------------------------------------- */ /* If this is not a new file, read ahead to establish if it is */ /* already in CRLF (DOS) mode, or just a normal unix CR mode. */ /* -------------------------------------------------------------------- */ if( !bNew && bInWriteMode ) { int nBytesRead = 0; char chNewByte; while( nBytesRead < 10000 && VSIFReadL( &chNewByte, 1, 1, fpCSV ) == 1 ) { if( chNewByte == 13 ) { bUseCRLF = TRUE; break; } } VSIRewindL( fpCSV ); } /* -------------------------------------------------------------------- */ /* Check if the first record seems to be field definitions or */ /* not. We assume it is field definitions if none of the */ /* values are strictly numeric. */ /* -------------------------------------------------------------------- */ char **papszTokens = NULL; int nFieldCount=0, iField; CPLValueType eType; if( !bNew ) { papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, FALSE ); nFieldCount = CSLCount( papszTokens ); bHasFieldNames = TRUE; } else bHasFieldNames = FALSE; for( iField = 0; iField < nFieldCount && bHasFieldNames; iField++ ) { eType = CPLGetValueType(papszTokens[iField]); if ( eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL ) { /* we have a numeric field, therefore do not consider the first line as field names */ bHasFieldNames = FALSE; } } if( !bNew && !bHasFieldNames ) VSIRewindL( fpCSV ); /* -------------------------------------------------------------------- */ /* Check for geonames.org tables */ /* -------------------------------------------------------------------- */ if( !bHasFieldNames && nFieldCount == 19 ) { if (CPLGetValueType(papszTokens[0]) == CPL_VALUE_INTEGER && CPLGetValueType(papszTokens[4]) == CPL_VALUE_REAL && CPLGetValueType(papszTokens[5]) == CPL_VALUE_REAL && CPLAtof(papszTokens[4]) >= -90 && CPLAtof(papszTokens[4]) <= 90 && CPLAtof(papszTokens[5]) >= -180 && CPLAtof(papszTokens[4]) <= 180) { bHasFieldNames = TRUE; CSLDestroy(papszTokens); papszTokens = NULL; static const struct { const char* pszName; OGRFieldType eType; } asGeonamesFieldDesc[] = { { "GEONAMEID", OFTString }, { "NAME", OFTString }, { "ASCIINAME", OFTString }, { "ALTNAMES", OFTString }, { "LATITUDE", OFTReal }, { "LONGITUDE", OFTReal }, { "FEATCLASS", OFTString }, { "FEATCODE", OFTString }, { "COUNTRY", OFTString }, { "CC2", OFTString }, { "ADMIN1", OFTString }, { "ADMIN2", OFTString }, { "ADMIN3", OFTString }, { "ADMIN4", OFTString }, { "POPULATION", OFTReal }, { "ELEVATION", OFTInteger }, { "GTOPO30", OFTInteger }, { "TIMEZONE", OFTString }, { "MODDATE", OFTString } }; for(iField = 0; iField < nFieldCount; iField++) { OGRFieldDefn oFieldDefn(asGeonamesFieldDesc[iField].pszName, asGeonamesFieldDesc[iField].eType); poFeatureDefn->AddFieldDefn(&oFieldDefn); } iLatitudeField = 4; iLongitudeField = 5; nFieldCount = 0; } } /* -------------------------------------------------------------------- */ /* Search a csvt file for types */ /* -------------------------------------------------------------------- */ char** papszFieldTypes = NULL; if (!bNew) { char* dname = strdup(CPLGetDirname(pszFilename)); char* fname = strdup(CPLGetBasename(pszFilename)); VSILFILE* fpCSVT = VSIFOpenL(CPLFormFilename(dname, fname, ".csvt"), "r"); free(dname); free(fname); if (fpCSVT!=NULL) { VSIRewindL(fpCSVT); papszFieldTypes = OGRCSVReadParseLineL(fpCSVT, ',', FALSE); VSIFCloseL(fpCSVT); } } /* -------------------------------------------------------------------- */ /* Build field definitions. */ /* -------------------------------------------------------------------- */ for( iField = 0; iField < nFieldCount; iField++ ) { char *pszFieldName = NULL; char szFieldNameBuffer[100]; if( bHasFieldNames ) { pszFieldName = papszTokens[iField]; // trim white space. while( *pszFieldName == ' ' ) pszFieldName++; while( pszFieldName[0] != '\0' && pszFieldName[strlen(pszFieldName)-1] == ' ' ) pszFieldName[strlen(pszFieldName)-1] = '\0'; if (*pszFieldName == '\0') pszFieldName = NULL; } if (pszFieldName == NULL) { pszFieldName = szFieldNameBuffer; sprintf( szFieldNameBuffer, "field_%d", iField+1 ); } OGRFieldDefn oField(pszFieldName, OFTString); if (papszFieldTypes!=NULL && iField<CSLCount(papszFieldTypes)) { char* pszLeftParenthesis = strchr(papszFieldTypes[iField], '('); if (pszLeftParenthesis && pszLeftParenthesis != papszFieldTypes[iField] && pszLeftParenthesis[1] >= '0' && pszLeftParenthesis[1] <= '9') { int nWidth = 0; int nPrecision = 0; char* pszDot = strchr(pszLeftParenthesis, '.'); if (pszDot) *pszDot = 0; *pszLeftParenthesis = 0; if (pszLeftParenthesis[-1] == ' ') pszLeftParenthesis[-1] = 0; nWidth = atoi(pszLeftParenthesis+1); if (pszDot) nPrecision = atoi(pszDot+1); oField.SetWidth(nWidth); oField.SetPrecision(nPrecision); } if (EQUAL(papszFieldTypes[iField], "Integer")) oField.SetType(OFTInteger); else if (EQUAL(papszFieldTypes[iField], "Real")) oField.SetType(OFTReal); else if (EQUAL(papszFieldTypes[iField], "String")) oField.SetType(OFTString); else if (EQUAL(papszFieldTypes[iField], "Date")) oField.SetType(OFTDate); else if (EQUAL(papszFieldTypes[iField], "Time")) oField.SetType(OFTTime); else if (EQUAL(papszFieldTypes[iField], "DateTime")) oField.SetType(OFTDateTime); else CPLError(CE_Warning, CPLE_NotSupported, "Unknown type : %s", papszFieldTypes[iField]); } if( EQUAL(oField.GetNameRef(),"WKT") && oField.GetType() == OFTString && iWktGeomReadField == -1 ) { iWktGeomReadField = iField; poFeatureDefn->SetGeomType( wkbUnknown ); } /*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */ if ( pszNfdcGeomField != NULL && EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) && EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LatitudeS") ) iNfdcLatitudeS = iField; else if ( pszNfdcGeomField != NULL && EQUALN(oField.GetNameRef(), pszNfdcGeomField, strlen(pszNfdcGeomField)) && EQUAL(oField.GetNameRef() + strlen(pszNfdcGeomField), "LongitudeS") ) iNfdcLongitudeS = iField; /* GNIS specific */ else if ( pszGeonamesGeomFieldPrefix != NULL && EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) && (EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LAT_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LATITUDE")) ) { oField.SetType(OFTReal); iLatitudeField = iField; } else if ( pszGeonamesGeomFieldPrefix != NULL && EQUALN(oField.GetNameRef(), pszGeonamesGeomFieldPrefix, strlen(pszGeonamesGeomFieldPrefix)) && (EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONG_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE_DEC") || EQUAL(oField.GetNameRef() + strlen(pszGeonamesGeomFieldPrefix), "_LONGITUDE")) ) { oField.SetType(OFTReal); iLongitudeField = iField; } poFeatureDefn->AddFieldDefn( &oField ); } if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 ) { bDontHonourStrings = TRUE; poFeatureDefn->SetGeomType( wkbPoint ); } else if ( iLatitudeField != -1 && iLongitudeField != -1 ) { poFeatureDefn->SetGeomType( wkbPoint ); } CSLDestroy( papszTokens ); CSLDestroy( papszFieldTypes ); }
OGRFeature * OGRCSVLayer::GetNextUnfilteredFeature() { if (fpCSV == NULL) return NULL; /* -------------------------------------------------------------------- */ /* Read the CSV record. */ /* -------------------------------------------------------------------- */ char **papszTokens; while(TRUE) { papszTokens = OGRCSVReadParseLineL( fpCSV, chDelimiter, bDontHonourStrings ); if( papszTokens == NULL ) return NULL; if( papszTokens[0] != NULL ) break; CSLDestroy(papszTokens); } /* -------------------------------------------------------------------- */ /* Create the OGR feature. */ /* -------------------------------------------------------------------- */ OGRFeature *poFeature; poFeature = new OGRFeature( poFeatureDefn ); /* -------------------------------------------------------------------- */ /* Set attributes for any indicated attribute records. */ /* -------------------------------------------------------------------- */ int iAttr; int nAttrCount = MIN(CSLCount(papszTokens), poFeatureDefn->GetFieldCount() ); for( iAttr = 0; iAttr < nAttrCount; iAttr++) { if( iAttr == iWktGeomReadField && papszTokens[iAttr][0] != '\0' ) { char *pszWKT = papszTokens[iAttr]; OGRGeometry *poGeom = NULL; if( OGRGeometryFactory::createFromWkt( &pszWKT, NULL, &poGeom ) == OGRERR_NONE ) poFeature->SetGeometryDirectly( poGeom ); } if (poFeatureDefn->GetFieldDefn(iAttr)->GetType() == OFTReal) { if (papszTokens[iAttr][0] != '\0') poFeature->SetField( iAttr, CPLAtof(papszTokens[iAttr]) ); } else if (poFeatureDefn->GetFieldDefn(iAttr)->GetType() != OFTString) { if (papszTokens[iAttr][0] != '\0') poFeature->SetField( iAttr, papszTokens[iAttr] ); } else poFeature->SetField( iAttr, papszTokens[iAttr] ); } /* -------------------------------------------------------------------- */ /*http://www.faa.gov/airports/airport_safety/airportdata_5010/menu/index.cfm specific */ /* -------------------------------------------------------------------- */ if ( iNfdcLatitudeS != -1 && iNfdcLongitudeS != -1 && nAttrCount > iNfdcLatitudeS && nAttrCount > iNfdcLongitudeS && papszTokens[iNfdcLongitudeS][0] != 0 && papszTokens[iNfdcLatitudeS][0] != 0) { double dfLon = atof(papszTokens[iNfdcLongitudeS]) / 3600; if (strchr(papszTokens[iNfdcLongitudeS], 'W')) dfLon *= -1; double dfLat = atof(papszTokens[iNfdcLatitudeS]) / 3600; if (strchr(papszTokens[iNfdcLatitudeS], 'S')) dfLat *= -1; poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) ); } /* -------------------------------------------------------------------- */ /* GNIS specific */ /* -------------------------------------------------------------------- */ else if ( iLatitudeField != -1 && iLongitudeField != -1 && nAttrCount > iLatitudeField && nAttrCount > iLongitudeField && papszTokens[iLongitudeField][0] != 0 && papszTokens[iLatitudeField][0] != 0) { /* Some records have dummy 0,0 value */ if (papszTokens[iLongitudeField][0] != '0' || papszTokens[iLongitudeField][1] != '\0' || papszTokens[iLatitudeField][0] != '0' || papszTokens[iLatitudeField][1] != '\0') { double dfLon = atof(papszTokens[iLongitudeField]); double dfLat = atof(papszTokens[iLatitudeField]); poFeature->SetGeometryDirectly( new OGRPoint(dfLon, dfLat) ); } } CSLDestroy( papszTokens ); /* -------------------------------------------------------------------- */ /* Translate the record id. */ /* -------------------------------------------------------------------- */ poFeature->SetFID( nNextFID++ ); m_nFeaturesRead++; return poFeature; }