QVariantMap QgsZonalHistogramAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
{
std::unique_ptr< QgsFeatureSource > zones( parameterAsSource( parameters, QStringLiteral( "INPUT_VECTOR" ), context ) );
if ( !zones )
throw QgsProcessingException( invalidSourceError( parameters, QStringLiteral( "INPUT_VECTOR" ) ) );
long count = zones->featureCount();
double step = count > 0 ? 100.0 / count : 1;
long current = 0;
QList< double > uniqueValues;
QMap< QgsFeatureId, QHash< double, qgssize > > featuresUniqueValues;
// First loop through the zones to build up a list of unique values across all zones to determine sink fields list
QgsFeatureRequest request;
request.setNoAttributes();
if ( zones->sourceCrs() != mCrs )
{
request.setDestinationCrs( mCrs, context.transformContext() );
}
QgsFeatureIterator it = zones->getFeatures( request );
QgsFeature f;
while ( it.nextFeature( f ) )
{
if ( feedback && feedback->isCanceled() )
{
break;
}
feedback->setProgress( current * step );
if ( !f.hasGeometry() )
{
current++;
continue;
}
QgsGeometry featureGeometry = f.geometry();
QgsRectangle featureRect = featureGeometry.boundingBox().intersect( mRasterExtent );
if ( featureRect.isEmpty() )
{
current++;
continue;
}
int nCellsX, nCellsY;
QgsRectangle rasterBlockExtent;
QgsRasterAnalysisUtils::cellInfoForBBox( mRasterExtent, featureRect, mCellSizeX, mCellSizeY, nCellsX, nCellsY, mNbCellsXProvider, mNbCellsYProvider, rasterBlockExtent );
QHash< double, qgssize > fUniqueValues;
QgsRasterAnalysisUtils::statisticsFromMiddlePointTest( mRasterInterface.get(), mRasterBand, featureGeometry, nCellsX, nCellsY, mCellSizeX, mCellSizeY,
rasterBlockExtent, [ &fUniqueValues]( double value ) { fUniqueValues[value]++; }, false );
if ( fUniqueValues.count() < 1 )
{
// The cell resolution is probably larger than the polygon area. We switch to slower precise pixel - polygon intersection in this case
// TODO: eventually deal with weight if needed
QgsRasterAnalysisUtils::statisticsFromPreciseIntersection( mRasterInterface.get(), mRasterBand, featureGeometry, nCellsX, nCellsY, mCellSizeX, mCellSizeY,
rasterBlockExtent, [ &fUniqueValues]( double value, double ) { fUniqueValues[value]++; }, false );
}
for ( auto it = fUniqueValues.constBegin(); it != fUniqueValues.constEnd(); ++it )
{
if ( uniqueValues.indexOf( it.key() ) == -1 )
{
uniqueValues << it.key();
}
featuresUniqueValues[f.id()][it.key()] += it.value();
}
current++;
}
std::sort( uniqueValues.begin(), uniqueValues.end() );
QString fieldPrefix = parameterAsString( parameters, QStringLiteral( "COLUMN_PREFIX" ), context );
QgsFields newFields;
for ( auto it = uniqueValues.constBegin(); it != uniqueValues.constEnd(); ++it )
{
newFields.append( QgsField( QStringLiteral( "%1%2" ).arg( fieldPrefix, mHasNoDataValue && *it == mNodataValue ? QStringLiteral( "NODATA" ) : QString::number( *it ) ), QVariant::LongLong, QString(), -1, 0 ) );
}
QgsFields fields = QgsProcessingUtils::combineFields( zones->fields(), newFields );
QString dest;
std::unique_ptr< QgsFeatureSink > sink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, dest, fields,
zones->wkbType(), zones->sourceCrs() ) );
if ( !sink )
throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "OUTPUT" ) ) );
it = zones->getFeatures( QgsFeatureRequest() );
while ( it.nextFeature( f ) )
{
QgsAttributes attributes = f.attributes();
QHash< double, qgssize > fUniqueValues = featuresUniqueValues.value( f.id() );
for ( auto it = uniqueValues.constBegin(); it != uniqueValues.constEnd(); ++it )
{
attributes += fUniqueValues.value( *it, 0 );
}
QgsFeature outputFeature;
outputFeature.setGeometry( f.geometry() );
outputFeature.setAttributes( attributes );
sink->addFeature( outputFeature, QgsFeatureSink::FastInsert );
}
QVariantMap outputs;
outputs.insert( QStringLiteral( "OUTPUT" ), dest );
return outputs;
}
bool QgsGeometryAnalyzer::convexHull( QgsVectorLayer* layer, const QString& shapefileName,
bool onlySelectedFeatures, int uniqueIdField, QProgressDialog* p )
{
if ( !layer )
{
return false;
}
QgsVectorDataProvider* dp = layer->dataProvider();
if ( !dp )
{
return false;
}
bool useField = false;
if ( uniqueIdField == -1 )
{
uniqueIdField = 0;
}
else
{
useField = true;
}
QgsFieldMap fields;
fields.insert( 0 , QgsField( QString( "UID" ), QVariant::String ) );
fields.insert( 1 , QgsField( QString( "AREA" ), QVariant::Double ) );
fields.insert( 2 , QgsField( QString( "PERIM" ), QVariant::Double ) );
QGis::WkbType outputType = QGis::WKBPolygon;
const QgsCoordinateReferenceSystem crs = layer->crs();
QgsVectorFileWriter vWriter( shapefileName, dp->encoding(), fields, outputType, &crs );
QgsFeature currentFeature;
QgsGeometry* dissolveGeometry = 0; //dissolve geometry
QMultiMap<QString, int> map;
if ( onlySelectedFeatures )
{
//use QgsVectorLayer::featureAtId
const QgsFeatureIds selection = layer->selectedFeaturesIds();
QgsFeatureIds::const_iterator it = selection.constBegin();
for ( ; it != selection.constEnd(); ++it )
{
#if 0
if ( p )
{
p->setValue( processedFeatures );
}
if ( p && p->wasCanceled() )
{
// break; // it may be better to do something else here?
return false;
}
#endif
if ( !layer->featureAtId( *it, currentFeature, true, true ) )
{
continue;
}
map.insert( currentFeature.attributeMap()[ uniqueIdField ].toString(), currentFeature.id() );
}
}
else
{
layer->select( layer->pendingAllAttributesList(), QgsRectangle(), true, false );
while ( layer->nextFeature( currentFeature ) )
{
#if 0
if ( p )
{
p->setValue( processedFeatures );
}
if ( p && p->wasCanceled() )
{
// break; // it may be better to do something else here?
return false;
}
#endif
map.insert( currentFeature.attributeMap()[ uniqueIdField ].toString(), currentFeature.id() );
}
}
QMultiMap<QString, int>::const_iterator jt = map.constBegin();
while ( jt != map.constEnd() )
{
QString currentKey = jt.key();
int processedFeatures = 0;
//take only selection
if ( onlySelectedFeatures )
{
//use QgsVectorLayer::featureAtId
const QgsFeatureIds selection = layer->selectedFeaturesIds();
if ( p )
{
p->setMaximum( selection.size() );
}
processedFeatures = 0;
while ( jt != map.constEnd() && ( jt.key() == currentKey || !useField ) )
{
if ( p && p->wasCanceled() )
{
break;
}
if ( selection.contains( jt.value() ) )
{
if ( p )
{
p->setValue( processedFeatures );
}
if ( !layer->featureAtId( jt.value(), currentFeature, true, true ) )
{
continue;
}
convexFeature( currentFeature, processedFeatures, &dissolveGeometry );
++processedFeatures;
}
++jt;
}
QList<double> values;
if ( !dissolveGeometry )
{
QgsDebugMsg( "no dissolved geometry - should not happen" );
return false;
}
dissolveGeometry = dissolveGeometry->convexHull();
values = simpleMeasure( dissolveGeometry );
QgsAttributeMap attributeMap;
attributeMap.insert( 0 , QVariant( currentKey ) );
attributeMap.insert( 1 , values[ 0 ] );
attributeMap.insert( 2 , values[ 1 ] );
QgsFeature dissolveFeature;
dissolveFeature.setAttributeMap( attributeMap );
dissolveFeature.setGeometry( dissolveGeometry );
vWriter.addFeature( dissolveFeature );
}
//take all features
else
{
int featureCount = layer->featureCount();
if ( p )
{
p->setMaximum( featureCount );
}
processedFeatures = 0;
while ( jt != map.constEnd() && ( jt.key() == currentKey || !useField ) )
{
if ( p )
{
p->setValue( processedFeatures );
}
if ( p && p->wasCanceled() )
{
break;
}
if ( !layer->featureAtId( jt.value(), currentFeature, true, true ) )
{
continue;
}
convexFeature( currentFeature, processedFeatures, &dissolveGeometry );
++processedFeatures;
++jt;
}
QList<double> values;
// QgsGeometry* tmpGeometry = 0;
if ( !dissolveGeometry )
{
QgsDebugMsg( "no dissolved geometry - should not happen" );
return false;
}
dissolveGeometry = dissolveGeometry->convexHull();
// values = simpleMeasure( tmpGeometry );
values = simpleMeasure( dissolveGeometry );
QgsAttributeMap attributeMap;
attributeMap.insert( 0 , QVariant( currentKey ) );
attributeMap.insert( 1 , QVariant( values[ 0 ] ) );
attributeMap.insert( 2 , QVariant( values[ 1 ] ) );
QgsFeature dissolveFeature;
dissolveFeature.setAttributeMap( attributeMap );
dissolveFeature.setGeometry( dissolveGeometry );
vWriter.addFeature( dissolveFeature );
}
}
return true;
}
// loadFields() gets the type from the field record
void QgsDb2Provider::loadFields()
{
mAttributeFields.clear();
//mDefaultValues.clear();
QString table = QStringLiteral( "%1.%2" ).arg( mSchemaName, mTableName );
// Use the Qt functionality to get the fields and their definitions.
QSqlRecord r = mDatabase.record( table );
int fieldCount = r.count();
for ( int i = 0; i < fieldCount; i++ )
{
QSqlField f = r.field( i );
int typeID = f.typeID(); // seems to be DB2 numeric type id (standard?)
QString sqlTypeName = db2TypeName( typeID );
QVariant::Type sqlType = f.type();
QgsDebugMsg( QString( "name: %1; length: %2; sqlTypeID: %3; sqlTypeName: %4" )
.arg( f.name() ).arg( f.length() ).arg( QString::number( typeID ), sqlTypeName ) );
if ( f.name() == mGeometryColName ) continue; // Got this with uri, just skip
if ( sqlType == QVariant::String )
{
mAttributeFields.append(
QgsField(
f.name(),
sqlType,
sqlTypeName,
f.length()
) );
}
else if ( sqlType == QVariant::Double )
{
mAttributeFields.append(
QgsField(
f.name(),
sqlType,
sqlTypeName,
f.length(),
f.precision()
) );
}
else
{
mAttributeFields.append(
QgsField(
f.name(),
sqlType,
sqlTypeName
) );
}
if ( !f.defaultValue().isNull() )
{
mDefaultValues.insert( i, f.defaultValue() );
}
// Hack to get primary key since the primaryIndex function above doesn't work
// on z/OS. Pick first integer column.
if ( mFidColName.isEmpty() &&
( sqlType == QVariant::LongLong || sqlType == QVariant::Int ) )
{
mFidColName = f.name();
}
}
if ( !mFidColName.isEmpty() )
{
mFidColIdx = mAttributeFields.indexFromName( mFidColName );
if ( mFidColIdx >= 0 )
{
// primary key has not null, unique constraints
QgsFieldConstraints constraints = mAttributeFields.at( mFidColIdx ).constraints();
constraints.setConstraint( QgsFieldConstraints::ConstraintNotNull, QgsFieldConstraints::ConstraintOriginProvider );
constraints.setConstraint( QgsFieldConstraints::ConstraintUnique, QgsFieldConstraints::ConstraintOriginProvider );
mAttributeFields[ mFidColIdx ].setConstraints( constraints );
}
}
}
bool QgsGeometryAnalyzer::extent( QgsVectorLayer* layer, const QString& shapefileName,
bool onlySelectedFeatures, QProgressDialog* p )
{
if ( !layer )
{
return false;
}
QgsVectorDataProvider* dp = layer->dataProvider();
if ( !dp )
{
return false;
}
QGis::WkbType outputType = QGis::WKBPolygon;
const QgsCoordinateReferenceSystem crs = layer->crs();
QgsFieldMap fields;
fields.insert( 0 , QgsField( QString( "MINX" ), QVariant::Double ) );
fields.insert( 1 , QgsField( QString( "MINY" ), QVariant::Double ) );
fields.insert( 2 , QgsField( QString( "MAXX" ), QVariant::Double ) );
fields.insert( 3 , QgsField( QString( "MAXY" ), QVariant::Double ) );
fields.insert( 4 , QgsField( QString( "CNTX" ), QVariant::Double ) );
fields.insert( 5 , QgsField( QString( "CNTY" ), QVariant::Double ) );
fields.insert( 6 , QgsField( QString( "AREA" ), QVariant::Double ) );
fields.insert( 7 , QgsField( QString( "PERIM" ), QVariant::Double ) );
fields.insert( 8 , QgsField( QString( "HEIGHT" ), QVariant::Double ) );
fields.insert( 9 , QgsField( QString( "WIDTH" ), QVariant::Double ) );
QgsVectorFileWriter vWriter( shapefileName, dp->encoding(), fields, outputType, &crs );
QgsRectangle rect;
if ( onlySelectedFeatures ) // take only selection
{
rect = layer->boundingBoxOfSelected();
}
else
{
rect = layer->extent();
}
double minx = rect.xMinimum();
double miny = rect.yMinimum();
double maxx = rect.xMaximum();
double maxy = rect.yMaximum();
double height = rect.height();
double width = rect.width();
double cntx = minx + ( width / 2.0 );
double cnty = miny + ( height / 2.0 );
double area = width * height;
double perim = ( 2 * width ) + ( 2 * height );
QgsFeature feat;
QgsAttributeMap map;
map.insert( 0 , QVariant( minx ) );
map.insert( 1 , QVariant( miny ) );
map.insert( 2 , QVariant( maxx ) );
map.insert( 3 , QVariant( maxy ) );
map.insert( 4 , QVariant( cntx ) );
map.insert( 5 , QVariant( cnty ) );
map.insert( 6 , QVariant( area ) );
map.insert( 7 , QVariant( perim ) );
map.insert( 8 , QVariant( height ) );
map.insert( 9 , QVariant( width ) );
feat.setAttributeMap( map );
feat.setGeometry( QgsGeometry::fromRect( rect ) );
vWriter.addFeature( feat );
return true;
}
QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
: QgsVectorDataProvider( uri ),
mXFieldIndex( -1 ), mYFieldIndex( -1 ),
mShowInvalidLines( true )
{
// Get the file name and mDelimiter out of the uri
mFileName = uri.left( uri.indexOf( "?" ) );
// split the string up on & to get the individual parameters
QStringList parameters = uri.mid( uri.indexOf( "?" ) ).split( "&", QString::SkipEmptyParts );
QgsDebugMsg( "Parameter count after split on &" + QString::number( parameters.size() ) );
// get the individual parameters and assign values
QStringList temp = parameters.filter( "delimiter=" );
mDelimiter = temp.size() ? temp[0].mid( temp[0].indexOf( "=" ) + 1 ) : "";
temp = parameters.filter( "delimiterType=" );
mDelimiterType = temp.size() ? temp[0].mid( temp[0].indexOf( "=" ) + 1 ) : "";
temp = parameters.filter( "xField=" );
QString xField = temp.size() ? temp[0].mid( temp[0].indexOf( "=" ) + 1 ) : "";
temp = parameters.filter( "yField=" );
QString yField = temp.size() ? temp[0].mid( temp[0].indexOf( "=" ) + 1 ) : "";
// Decode the parts of the uri. Good if someone entered '=' as a delimiter, for instance.
mFileName = QUrl::fromPercentEncoding( mFileName.toUtf8() );
mDelimiter = QUrl::fromPercentEncoding( mDelimiter.toUtf8() );
mDelimiterType = QUrl::fromPercentEncoding( mDelimiterType.toUtf8() );
xField = QUrl::fromPercentEncoding( xField.toUtf8() );
yField = QUrl::fromPercentEncoding( yField.toUtf8() );
QgsDebugMsg( "Data source uri is " + uri );
QgsDebugMsg( "Delimited text file is: " + mFileName );
QgsDebugMsg( "Delimiter is: " + mDelimiter );
QgsDebugMsg( "Delimiter type is: " + mDelimiterType );
QgsDebugMsg( "xField is: " + xField );
QgsDebugMsg( "yField is: " + yField );
// if delimiter contains some special characters, convert them
if ( mDelimiterType == "regexp" )
mDelimiterRegexp = QRegExp( mDelimiter );
else
mDelimiter.replace( "\\t", "\t" ); // replace "\t" with a real tabulator
// Set the selection rectangle to null
mSelectionRectangle = QgsRectangle();
// assume the layer is invalid until proven otherwise
mValid = false;
if ( mFileName.isEmpty() || mDelimiter.isEmpty() || xField.isEmpty() || yField.isEmpty() )
{
// uri is invalid so the layer must be too...
QString( "Data source is invalid" );
return;
}
// check to see that the file exists and perform some sanity checks
if ( !QFile::exists( mFileName ) )
{
QgsDebugMsg( "Data source " + dataSourceUri() + " doesn't exist" );
return;
}
// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure the the delimited file is properly formed.
mFile = new QFile( mFileName );
if ( !mFile->open( QIODevice::ReadOnly ) )
{
QgsDebugMsg( "Data source " + dataSourceUri() + " could not be opened" );
delete mFile;
return;
}
// now we have the file opened and ready for parsing
// set the initial extent
mExtent = QgsRectangle();
QMap<int, bool> couldBeInt;
QMap<int, bool> couldBeDouble;
mStream = new QTextStream( mFile );
QString line;
mNumberFeatures = 0;
int lineNumber = 0;
bool firstPoint = true;
bool hasFields = false;
while ( !mStream->atEnd() )
{
lineNumber++;
line = mStream->readLine(); // line of text excluding '\n', default local 8 bit encoding.
if ( !hasFields )
{
// Get the fields from the header row and store them in the
// fields vector
QgsDebugMsg( "Attempting to split the input line: " + line + " using delimiter " + mDelimiter );
QStringList fieldList;
if ( mDelimiterType == "regexp" )
fieldList = line.split( mDelimiterRegexp );
else
fieldList = line.split( mDelimiter );
QgsDebugMsg( "Split line into " + QString::number( fieldList.size() ) + " parts" );
// We don't know anything about a text based field other
// than its name. All fields are assumed to be text
int fieldPos = 0;
for ( QStringList::Iterator it = fieldList.begin();
it != fieldList.end(); ++it )
{
QString field = *it;
if ( field.length() > 0 )
{
// for now, let's set field type as text
attributeFields[fieldPos] = QgsField( *it, QVariant::String, "Text" );
// check to see if this field matches either the x or y field
if ( xField == *it )
{
QgsDebugMsg( "Found x field: " + ( *it ) );
mXFieldIndex = fieldPos;
}
else if ( yField == *it )
{
QgsDebugMsg( "Found y field: " + ( *it ) );
mYFieldIndex = fieldPos;
}
QgsDebugMsg( "Adding field: " + ( *it ) );
// assume that the field could be integer or double
couldBeInt.insert( fieldPos, true );
couldBeDouble.insert( fieldPos, true );
fieldPos++;
}
}
QgsDebugMsg( "Field count for the delimited text file is " + QString::number( attributeFields.size() ) );
hasFields = true;
}
else if ( mXFieldIndex != -1 && mYFieldIndex != -1 )
{
mNumberFeatures++;
// split the line on the delimiter
QStringList parts;
if ( mDelimiterType == "regexp" )
parts = line.split( mDelimiterRegexp );
else
parts = line.split( mDelimiter );
// Skip malformed lines silently. Report line number with nextFeature()
if ( attributeFields.size() != parts.size() )
{
continue;
}
// Get the x and y values, first checking to make sure they
// aren't null.
QString sX = parts[mXFieldIndex];
QString sY = parts[mYFieldIndex];
bool xOk = true;
bool yOk = true;
double x = sX.toDouble( &xOk );
double y = sY.toDouble( &yOk );
if ( xOk && yOk )
{
if ( !firstPoint )
{
mExtent.combineExtentWith( x, y );
}
else
{ // Extent for the first point is just the first point
mExtent.set( x, y, x, y );
firstPoint = false;
}
}
int i = 0;
for ( QStringList::iterator it = parts.begin(); it != parts.end(); ++it, ++i )
{
// try to convert attribute values to integer and double
if ( couldBeInt[i] )
{
it->toInt( &couldBeInt[i] );
}
if ( couldBeDouble[i] )
{
it->toDouble( &couldBeDouble[i] );
}
}
}
}
// now it's time to decide the types for the fields
for ( QgsFieldMap::iterator it = attributeFields.begin(); it != attributeFields.end(); ++it )
{
if ( couldBeInt[it.key()] )
{
it->setType( QVariant::Int );
it->setTypeName( "integer" );
}
else if ( couldBeDouble[it.key()] )
{
it->setType( QVariant::Double );
it->setTypeName( "double" );
}
}
if ( mXFieldIndex != -1 && mYFieldIndex != -1 )
{
QgsDebugMsg( "Data store is valid" );
QgsDebugMsg( "Number of features " + QString::number( mNumberFeatures ) );
QgsDebugMsg( "Extents " + mExtent.toString() );
mValid = true;
}
else
{
QgsDebugMsg( "Data store is invalid. Specified x,y fields do not match those in the database" );
}
QgsDebugMsg( "Done checking validity" );
}
void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
{
QStringList messages;
// assume the layer is invalid until proven otherwise
mLayerValid = false;
mValid = false;
mRescanRequired = false;
clearInvalidLines();
// Initiallize indexes
resetIndexes();
bool buildSpatialIndex = buildIndexes && mSpatialIndex != 0;
// No point building a subset index if there is no geometry, as all
// records will be included.
bool buildSubsetIndex = buildIndexes && mBuildSubsetIndex && mGeomRep != GeomNone;
if ( ! mFile->isValid() )
{
// uri is invalid so the layer must be too...
messages.append( tr( "File cannot be opened or delimiter parameters are not valid" ) );
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid - filename or delimiter parameters" );
return;
}
// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure that the delimited file is properly formed.
if ( mGeomRep == GeomAsWkt )
{
mWktFieldIndex = mFile->fieldIndex( mWktFieldName );
if ( mWktFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Wkt" ).arg( mWktFieldName ) );
}
}
else if ( mGeomRep == GeomAsXy )
{
mXFieldIndex = mFile->fieldIndex( mXFieldName );
mYFieldIndex = mFile->fieldIndex( mYFieldName );
if ( mXFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "X" ).arg( mWktFieldName ) );
}
if ( mYFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Y" ).arg( mWktFieldName ) );
}
}
if ( messages.size() > 0 )
{
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid - missing geometry fields" );
return;
}
// Scan the entire file to determine
// 1) the number of fields (this is handled by QgsDelimitedTextFile mFile
// 2) the number of valid features. Note that the selection of valid features
// should match the code in QgsDelimitedTextFeatureIterator
// 3) the geometric extents of the layer
// 4) the type of each field
//
// Also build subset and spatial indexes.
QStringList parts;
long nEmptyRecords = 0;
long nBadFormatRecords = 0;
long nIncompatibleGeometry = 0;
long nInvalidGeometry = 0;
long nEmptyGeometry = 0;
mNumberFeatures = 0;
mExtent = QgsRectangle();
QList<bool> isEmpty;
QList<bool> couldBeInt;
QList<bool> couldBeDouble;
while ( true )
{
QgsDelimitedTextFile::Status status = mFile->nextRecord( parts );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk )
{
nBadFormatRecords++;
recordInvalidLine( tr( "Invalid record format at line %1" ) );
continue;
}
// Skip over empty records
if ( recordIsEmpty( parts ) )
{
nEmptyRecords++;
continue;
}
// Check geometries are valid
bool geomValid = true;
if ( mGeomRep == GeomAsWkt )
{
if ( mWktFieldIndex >= parts.size() || parts[mWktFieldIndex].isEmpty() )
{
nEmptyGeometry++;
geomValid = false;
}
else
{
// Get the wkt - confirm it is valid, get the type, and
// if compatible with the rest of file, add to the extents
QString sWkt = parts[mWktFieldIndex];
QgsGeometry *geom = 0;
if ( !mWktHasPrefix && sWkt.indexOf( WktPrefixRegexp ) >= 0 )
mWktHasPrefix = true;
if ( !mWktHasZM && sWkt.indexOf( WktZMRegexp ) >= 0 )
mWktHasZM = true;
geom = geomFromWkt( sWkt, mWktHasPrefix, mWktHasZM );
if ( geom )
{
QGis::WkbType type = geom->wkbType();
if ( type != QGis::WKBNoGeometry )
{
if ( mGeometryType == QGis::UnknownGeometry || geom->type() == mGeometryType )
{
mGeometryType = geom->type();
if ( mNumberFeatures == 0 )
{
mNumberFeatures++;
mWkbType = type;
mExtent = geom->boundingBox();
}
else
{
mNumberFeatures++;
if ( geom->isMultipart() ) mWkbType = type;
QgsRectangle bbox( geom->boundingBox() );
mExtent.combineExtentWith( &bbox );
}
if ( buildSpatialIndex )
{
QgsFeature f;
f.setFeatureId( mFile->recordId() );
f.setGeometry( geom );
mSpatialIndex->insertFeature( f );
// Feature now has ownership of geometry, so set to null
// here to avoid deleting twice.
geom = 0;
}
}
else
{
nIncompatibleGeometry++;
geomValid = false;
}
}
if ( geom ) delete geom;
}
else
{
geomValid = false;
nInvalidGeometry++;
recordInvalidLine( tr( "Invalid WKT at line %1" ) );
}
}
}
else if ( mGeomRep == GeomAsXy )
{
// Get the x and y values, first checking to make sure they
// aren't null.
QString sX = mXFieldIndex < parts.size() ? parts[mXFieldIndex] : "";
QString sY = mYFieldIndex < parts.size() ? parts[mYFieldIndex] : "";
if ( sX.isEmpty() && sY.isEmpty() )
{
geomValid = false;
nEmptyGeometry++;
}
else
{
QgsPoint pt;
bool ok = pointFromXY( sX, sY, pt, mDecimalPoint, mXyDms );
if ( ok )
{
if ( mNumberFeatures > 0 )
{
mExtent.combineExtentWith( pt.x(), pt.y() );
}
else
{
// Extent for the first point is just the first point
mExtent.set( pt.x(), pt.y(), pt.x(), pt.y() );
mWkbType = QGis::WKBPoint;
mGeometryType = QGis::Point;
}
mNumberFeatures++;
if ( buildSpatialIndex && qIsFinite( pt.x() ) && qIsFinite( pt.y() ) )
{
QgsFeature f;
f.setFeatureId( mFile->recordId() );
f.setGeometry( QgsGeometry::fromPoint( pt ) );
mSpatialIndex->insertFeature( f );
}
}
else
{
geomValid = false;
nInvalidGeometry++;
recordInvalidLine( tr( "Invalid X or Y fields at line %1" ) );
}
}
}
else
{
mWkbType = QGis::WKBNoGeometry;
mNumberFeatures++;
}
if ( ! geomValid ) continue;
if ( buildSubsetIndex ) mSubsetIndex.append( mFile->recordId() );
// If we are going to use this record, then assess the potential types of each colum
for ( int i = 0; i < parts.size(); i++ )
{
QString &value = parts[i];
if ( value.isEmpty() )
continue;
// try to convert attribute values to integer and double
while ( couldBeInt.size() <= i )
{
isEmpty.append( true );
couldBeInt.append( false );
couldBeDouble.append( false );
}
if ( isEmpty[i] )
{
isEmpty[i] = false;
couldBeInt[i] = true;
couldBeDouble[i] = true;
}
if ( couldBeInt[i] )
{
value.toInt( &couldBeInt[i] );
}
if ( couldBeDouble[i] )
{
if ( ! mDecimalPoint.isEmpty() )
{
value.replace( mDecimalPoint, "." );
}
value.toDouble( &couldBeDouble[i] );
}
}
}
// Now create the attribute fields. Field types are integer by preference,
// failing that double, failing that text.
QStringList fieldNames = mFile->fieldNames();
mFieldCount = fieldNames.size();
attributeColumns.clear();
attributeFields.clear();
QString csvtMessage;
QStringList csvtTypes = readCsvtFieldTypes( mFile->fileName(), &csvtMessage );
for ( int i = 0; i < fieldNames.size(); i++ )
{
// Skip over WKT field ... don't want to display in attribute table
if ( i == mWktFieldIndex ) continue;
// Add the field index lookup for the column
attributeColumns.append( i );
QVariant::Type fieldType = QVariant::String;
QString typeName = "text";
if ( i < csvtTypes.size() )
{
if ( csvtTypes[i] == "integer" )
{
fieldType = QVariant::Int;
typeName = "integer";
}
else if ( csvtTypes[i] == "real" )
{
fieldType = QVariant::Double;
typeName = "double";
}
}
else if ( i < couldBeInt.size() )
{
if ( couldBeInt[i] )
{
fieldType = QVariant::Int;
typeName = "integer";
}
else if ( couldBeDouble[i] )
{
fieldType = QVariant::Double;
typeName = "double";
}
}
attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
}
QgsDebugMsg( "Field count for the delimited text file is " + QString::number( attributeFields.size() ) );
QgsDebugMsg( "geometry type is: " + QString::number( mWkbType ) );
QgsDebugMsg( "feature count is: " + QString::number( mNumberFeatures ) );
QStringList warnings;
if ( ! csvtMessage.isEmpty() ) warnings.append( csvtMessage );
if ( nBadFormatRecords > 0 )
warnings.append( tr( "%1 records discarded due to invalid format" ).arg( nBadFormatRecords ) );
if ( nEmptyGeometry > 0 )
warnings.append( tr( "%1 records discarded due to missing geometry definitions" ).arg( nEmptyGeometry ) );
if ( nInvalidGeometry > 0 )
warnings.append( tr( "%1 records discarded due to invalid geometry definitions" ).arg( nInvalidGeometry ) );
if ( nIncompatibleGeometry > 0 )
warnings.append( tr( "%1 records discarded due to incompatible geometry types" ).arg( nIncompatibleGeometry ) );
reportErrors( warnings );
// Decide whether to use subset ids to index records rather than simple iteration through all
// If more than 10% of records are being skipped, then use index. (Not based on any experimentation,
// could do with some analysis?)
if ( buildSubsetIndex )
{
long recordCount = mFile->recordCount();
recordCount -= recordCount / SUBSET_ID_THRESHOLD_FACTOR;
mUseSubsetIndex = mSubsetIndex.size() < recordCount;
if ( ! mUseSubsetIndex ) mSubsetIndex = QList<quintptr>();
}
mUseSpatialIndex = buildSpatialIndex;
mValid = mGeometryType != QGis::UnknownGeometry;
mLayerValid = mValid;
// If it is valid, then watch for changes to the file
connect( mFile, SIGNAL( fileUpdated() ), this, SLOT( onFileUpdated() ) );
}
void QgsGrassVectorMapLayer::load()
{
QgsDebugMsg( "entered" );
clear();
if ( !mMap )
{
return;
}
// Attributes are not loaded for topo layers in which case field == 0
if ( mField == 0 )
{
return;
}
QgsDebugMsg( QString( "cidxFieldIndex() = %1 cidxFieldNumCats() = %2" ).arg( cidxFieldIndex() ).arg( cidxFieldNumCats() ) );
mFieldInfo = Vect_get_field( mMap->map(), mField ); // should work also with field = 0
if ( !mFieldInfo )
{
QgsDebugMsg( "No field info -> no attribute table" );
}
else
{
QgsDebugMsg( "Field info found -> open database" );
QFileInfo di( mMap->grassObject().mapsetPath() + "/vector/" + mMap->grassObject().name() + "/dbln" );
mLastLoaded = di.lastModified();
QString error;
dbDriver *databaseDriver = openDriver( error );
if ( !databaseDriver || !error.isEmpty() )
{
QgsDebugMsg( error );
}
else
{
QgsDebugMsg( "Database opened -> open select cursor" );
QgsGrass::lock(); // not sure if lock is necessary
dbString dbstr;
db_init_string( &dbstr );
db_set_string( &dbstr, ( char * )"select * from " );
db_append_string( &dbstr, mFieldInfo->table );
QgsDebugMsg( QString( "SQL: %1" ).arg( db_get_string( &dbstr ) ) );
dbCursor databaseCursor;
if ( db_open_select_cursor( databaseDriver, &dbstr, &databaseCursor, DB_SCROLL ) != DB_OK )
{
db_close_database_shutdown_driver( databaseDriver );
QgsGrass::warning( "Cannot select attributes from table '" + QString( mFieldInfo->table ) + "'" );
}
else
{
#ifdef QGISDEBUG
int nRecords = db_get_num_rows( &databaseCursor );
QgsDebugMsg( QString( "Number of records: %1" ).arg( nRecords ) );
#endif
dbTable *databaseTable = db_get_cursor_table( &databaseCursor );
int nColumns = db_get_table_number_of_columns( databaseTable );
// Read columns' description
for ( int i = 0; i < nColumns; i++ )
{
QPair<double, double> minMax( DBL_MAX, -DBL_MAX );
dbColumn *column = db_get_table_column( databaseTable, i );
int ctype = db_sqltype_to_Ctype( db_get_column_sqltype( column ) );
QVariant::Type qtype = QVariant::String; //default to string
QgsDebugMsg( QString( "column = %1 ctype = %2" ).arg( db_get_column_name( column ) ).arg( ctype ) );
QString ctypeStr;
switch ( ctype )
{
case DB_C_TYPE_INT:
ctypeStr = "integer";
qtype = QVariant::Int;
break;
case DB_C_TYPE_DOUBLE:
ctypeStr = "double";
qtype = QVariant::Double;
break;
case DB_C_TYPE_STRING:
ctypeStr = "string";
qtype = QVariant::String;
break;
case DB_C_TYPE_DATETIME:
ctypeStr = "datetime";
qtype = QVariant::String;
break;
}
mTableFields.append( QgsField( db_get_column_name( column ), qtype, ctypeStr,
db_get_column_length( column ), db_get_column_precision( column ) ) );
mMinMax << minMax;
if ( G_strcasecmp( db_get_column_name( column ), mFieldInfo->key ) == 0 )
{
mKeyColumn = i;
}
}
if ( mKeyColumn < 0 )
{
mTableFields.clear();
QgsGrass::warning( QObject::tr( "Key column '%1' not found in the table '%2'" ).arg( mFieldInfo->key, mFieldInfo->table ) );
}
else
{
mHasTable = true;
// Read attributes to the memory
while ( true )
{
int more;
if ( db_fetch( &databaseCursor, DB_NEXT, &more ) != DB_OK )
{
QgsDebugMsg( "Cannot fetch DB record" );
break;
}
if ( !more )
{
break; // no more records
}
// Check cat value
dbColumn *column = db_get_table_column( databaseTable, mKeyColumn );
dbValue *value = db_get_column_value( column );
if ( db_test_value_isnull( value ) )
{
continue;
}
int cat = db_get_value_int( value );
if ( cat < 0 )
{
continue;
}
QList<QVariant> values;
for ( int i = 0; i < nColumns; i++ )
{
column = db_get_table_column( databaseTable, i );
int sqltype = db_get_column_sqltype( column );
int ctype = db_sqltype_to_Ctype( sqltype );
value = db_get_column_value( column );
db_convert_value_to_string( value, sqltype, &dbstr );
QgsDebugMsgLevel( QString( "column = %1 value = %2" ).arg( db_get_column_name( column ), db_get_string( &dbstr ) ), 3 );
QVariant variant;
if ( !db_test_value_isnull( value ) )
{
int iv;
double dv;
//layer.mAttributes[layer.nAttributes].values[i] = strdup( db_get_string( &dbstr ) );
switch ( ctype )
{
case DB_C_TYPE_INT:
iv = db_get_value_int( value );
variant = QVariant( iv );
mMinMax[i].first = qMin( mMinMax[i].first, ( double )iv );
mMinMax[i].second = qMin( mMinMax[i].second, ( double )iv );
break;
case DB_C_TYPE_DOUBLE:
dv = db_get_value_double( value );
variant = QVariant( dv );
mMinMax[i].first = qMin( mMinMax[i].first, dv );
mMinMax[i].second = qMin( mMinMax[i].second, dv );
break;
case DB_C_TYPE_STRING:
// Store as byte array so that codec may be used later
variant = QVariant( QByteArray( db_get_value_string( value ) ) );
break;
case DB_C_TYPE_DATETIME:
variant = QVariant( QByteArray( db_get_string( &dbstr ) ) );
break;
default:
variant = QVariant( QByteArray( db_get_string( &dbstr ) ) );
}
}
QgsDebugMsgLevel( QString( "column = %1 variant = %2" ).arg( db_get_column_name( column ), variant.toString() ), 3 );
values << variant;
}
mAttributes.insert( cat, values );
}
}
mValid = true;
db_close_cursor( &databaseCursor );
db_close_database_shutdown_driver( databaseDriver );
db_free_string( &dbstr );
QgsDebugMsg( QString( "mTableFields.size = %1" ).arg( mTableFields.size() ) );
QgsDebugMsg( QString( "number of attributes = %1" ).arg( mAttributes.size() ) );
}
QgsGrass::unlock();
}
}
// Add cat if no attribute fields exist (otherwise qgis crashes)
if ( mTableFields.size() == 0 )
{
mKeyColumn = 0;
mTableFields.append( QgsField( "cat", QVariant::Int, "integer" ) );
QPair<double, double> minMax( 0, 0 );
if ( cidxFieldIndex() >= 0 )
{
int ncats, cat, type, id;
ncats = Vect_cidx_get_num_cats_by_index( mMap->map(), cidxFieldIndex() );
if ( ncats > 0 )
{
Vect_cidx_get_cat_by_index( mMap->map(), cidxFieldIndex(), 0, &cat, &type, &id );
minMax.first = cat;
Vect_cidx_get_cat_by_index( mMap->map(), cidxFieldIndex(), ncats - 1, &cat, &type, &id );
minMax.second = cat;
}
}
mMinMax << minMax;
}
mFields = mTableFields;
mAttributeFields = mTableFields;
QgsDebugMsg( QString( "layer loaded mTableFields.size() = %1 mAttributes.size() = %2" ).arg( mTableFields.size() ).arg( mAttributes.size() ) );
mValid = true;
}
void TestQgsProjectStorage::testMemoryStorage()
{
QString dataDir( TEST_DATA_DIR ); // defined in CmakeLists.txt
QString layerPath = dataDir + "/points.shp";
QgsVectorLayer *layer1 = new QgsVectorLayer( layerPath, QStringLiteral( "points" ), QStringLiteral( "ogr" ) );
QVERIFY( layer1->isValid() );
MemoryStorage *memStorage = new MemoryStorage;
QgsApplication::projectStorageRegistry()->registerProjectStorage( memStorage );
QVERIFY( QgsApplication::projectStorageRegistry()->projectStorages().contains( memStorage ) );
QCOMPARE( memStorage->listProjects( QString() ).count(), 0 );
QgsProject prj1;
prj1.setTitle( "best project ever" );
prj1.addMapLayer( layer1 );
prj1.setFileName( "memory:project1" );
// let's use the aux storage as well - so that the project will include aux database as well
int fldCnt0 = layer1->fields().count();
QgsAuxiliaryLayer *layerAux = prj1.auxiliaryStorage()->createAuxiliaryLayer( layer1->fields().at( 0 ), layer1 );
layer1->setAuxiliaryLayer( layerAux );
layerAux->addAttribute( QgsField( "fld_aux", QVariant::Int ) );
layerAux->commitChanges();
QCOMPARE( fldCnt0, 6 );
QCOMPARE( layer1->fields().count(), 7 );
bool writeOk = prj1.write();
QVERIFY( writeOk );
QCOMPARE( memStorage->listProjects( QString() ).count(), 1 );
QVERIFY( prj1.absoluteFilePath().isEmpty() );
QCOMPARE( prj1.baseName(), QString( "project1" ) );
QVERIFY( prj1.lastModified().secsTo( QDateTime::currentDateTime() ) < 1 );
// read the project back
QgsProject prj2;
prj2.setFileName( "memory:project1" );
bool readOk = prj2.read();
QVERIFY( readOk );
QCOMPARE( prj2.mapLayers().count(), 1 );
QCOMPARE( prj2.title(), QString( "best project ever" ) );
// let's check that our stuff from auxiliary database got stored written and read
QgsVectorLayer *prj2layer1 = qobject_cast<QgsVectorLayer *>( prj2.mapLayers().constBegin().value() );
QVERIFY( prj2layer1 );
QCOMPARE( prj2layer1->fields().count(), 7 );
QCOMPARE( prj2layer1->fields().at( 6 ).name(), QString( "auxiliary_storage_fld_aux" ) );
// test project-related variables for project storage
QgsExpressionContext expressionContext;
expressionContext.appendScope( QgsExpressionContextUtils::projectScope( &prj2 ) );
QCOMPARE( QgsExpression( "@project_path" ).evaluate( &expressionContext ).toString(), QString( "memory:project1" ) );
QCOMPARE( QgsExpression( "@project_basename" ).evaluate( &expressionContext ).toString(), QString( "project1" ) );
// test access of non-existent project
QgsProject prj3;
prj3.setFileName( "memory:nooooooooo!" );
bool readInvalidOk = prj3.read();
QVERIFY( !readInvalidOk );
// test metadata access
QgsProjectStorage::Metadata meta1;
bool readMetaOk = memStorage->readProjectStorageMetadata( "memory:project1", meta1 );
QVERIFY( readMetaOk );
QCOMPARE( meta1.name, QString( "project1" ) );
QVERIFY( meta1.lastModified.secsTo( QDateTime::currentDateTime() ) < 1 );
QgsProjectStorage::Metadata metaX;
bool readMetaInvalidOk = memStorage->readProjectStorageMetadata( "memory:projectXYZ", metaX );
QVERIFY( !readMetaInvalidOk );
// test removal
bool removeInvalidOk = memStorage->removeProject( "memory:projectXYZ" );
QVERIFY( !removeInvalidOk );
QCOMPARE( memStorage->listProjects( QString() ).count(), 1 );
bool removeOk = memStorage->removeProject( "memory:project1" );
QVERIFY( removeOk );
QCOMPARE( memStorage->listProjects( QString() ).count(), 0 );
QgsApplication::projectStorageRegistry()->unregisterProjectStorage( memStorage );
}
