QgsFeatureList QgsOgrUtils::stringToFeatureList( const QString &string, const QgsFields &fields, QTextCodec *encoding )
{
QgsFeatureList features;
if ( string.isEmpty() )
return features;
QString randomFileName = QStringLiteral( "/vsimem/%1" ).arg( QUuid::createUuid().toString() );
// create memory file system object from string buffer
QByteArray ba = string.toUtf8();
VSIFCloseL( VSIFileFromMemBuffer( randomFileName.toUtf8().constData(), reinterpret_cast< GByte * >( ba.data() ),
static_cast< vsi_l_offset >( ba.size() ), FALSE ) );
gdal::ogr_datasource_unique_ptr hDS( OGROpen( randomFileName.toUtf8().constData(), false, nullptr ) );
if ( !hDS )
{
VSIUnlink( randomFileName.toUtf8().constData() );
return features;
}
OGRLayerH ogrLayer = OGR_DS_GetLayer( hDS.get(), 0 );
if ( !ogrLayer )
{
hDS.reset();
VSIUnlink( randomFileName.toUtf8().constData() );
return features;
}
gdal::ogr_feature_unique_ptr oFeat;
while ( oFeat.reset( OGR_L_GetNextFeature( ogrLayer ) ), oFeat )
{
QgsFeature feat = readOgrFeature( oFeat.get(), fields, encoding );
if ( feat.isValid() )
features << feat;
}
hDS.reset();
VSIUnlink( randomFileName.toUtf8().constData() );
return features;
}
QgsDataItem *QgsOgrDataItemProvider::createDataItem( const QString &pathIn, QgsDataItem *parentItem )
{
QString path( pathIn );
if ( path.isEmpty() )
return nullptr;
QgsDebugMsgLevel( "thePath: " + path, 2 );
// zip settings + info
QgsSettings settings;
QString scanZipSetting = settings.value( QStringLiteral( "qgis/scanZipInBrowser2" ), "basic" ).toString();
QString vsiPrefix = QgsZipItem::vsiPrefix( path );
bool is_vsizip = ( vsiPrefix == QLatin1String( "/vsizip/" ) );
bool is_vsigzip = ( vsiPrefix == QLatin1String( "/vsigzip/" ) );
bool is_vsitar = ( vsiPrefix == QLatin1String( "/vsitar/" ) );
// should we check ext. only?
// check if scanItemsInBrowser2 == extension or parent dir in scanItemsFastScanUris
// TODO - do this in dir item, but this requires a way to inform which extensions are supported by provider
// maybe a callback function or in the provider registry?
bool scanExtSetting = false;
if ( ( settings.value( QStringLiteral( "qgis/scanItemsInBrowser2" ),
"extension" ).toString() == QLatin1String( "extension" ) ) ||
( parentItem && settings.value( QStringLiteral( "qgis/scanItemsFastScanUris" ),
QStringList() ).toStringList().contains( parentItem->path() ) ) ||
( ( is_vsizip || is_vsitar ) && parentItem && parentItem->parent() &&
settings.value( QStringLiteral( "qgis/scanItemsFastScanUris" ),
QStringList() ).toStringList().contains( parentItem->parent()->path() ) ) )
{
scanExtSetting = true;
}
// get suffix, removing .gz if present
QString tmpPath = path; //path used for testing, not for layer creation
if ( is_vsigzip )
tmpPath.chop( 3 );
QFileInfo info( tmpPath );
QString suffix = info.suffix().toLower();
// extract basename with extension
info.setFile( path );
QString name = info.fileName();
// If a .tab exists, then the corresponding .map/.dat is very likely a
// side-car file of the .tab
if ( suffix == QLatin1String( "map" ) || suffix == QLatin1String( "dat" ) )
{
if ( QFileInfo( QDir( info.path() ), info.baseName() + ".tab" ).exists() )
return nullptr;
}
QgsDebugMsgLevel( "thePath= " + path + " tmpPath= " + tmpPath + " name= " + name
+ " suffix= " + suffix + " vsiPrefix= " + vsiPrefix, 3 );
QStringList myExtensions = fileExtensions();
QStringList dirExtensions = directoryExtensions();
// allow only normal files, supported directories, or VSIFILE items to continue
bool isOgrSupportedDirectory = info.isDir() && dirExtensions.contains( suffix );
if ( !isOgrSupportedDirectory && !info.isFile() && vsiPrefix.isEmpty() )
return nullptr;
// skip *.aux.xml files (GDAL auxiliary metadata files),
// *.shp.xml files (ESRI metadata) and *.tif.xml files (TIFF metadata)
// unless that extension is in the list (*.xml might be though)
if ( path.endsWith( QLatin1String( ".aux.xml" ), Qt::CaseInsensitive ) &&
!myExtensions.contains( QStringLiteral( "aux.xml" ) ) )
return nullptr;
if ( path.endsWith( QLatin1String( ".shp.xml" ), Qt::CaseInsensitive ) &&
!myExtensions.contains( QStringLiteral( "shp.xml" ) ) )
return nullptr;
if ( path.endsWith( QLatin1String( ".tif.xml" ), Qt::CaseInsensitive ) &&
!myExtensions.contains( QStringLiteral( "tif.xml" ) ) )
return nullptr;
// skip QGIS style xml files
if ( path.endsWith( QLatin1String( ".xml" ), Qt::CaseInsensitive ) &&
QgsStyle::isXmlStyleFile( path ) )
return nullptr;
// We have to filter by extensions, otherwise e.g. all Shapefile files are displayed
// because OGR drive can open also .dbf, .shx.
if ( myExtensions.indexOf( suffix ) < 0 && !dirExtensions.contains( suffix ) )
{
bool matches = false;
const auto constWildcards = wildcards();
for ( const QString &wildcard : constWildcards )
{
QRegExp rx( wildcard, Qt::CaseInsensitive, QRegExp::Wildcard );
if ( rx.exactMatch( info.fileName() ) )
{
matches = true;
break;
}
}
if ( !matches )
return nullptr;
}
// .dbf should probably appear if .shp is not present
if ( suffix == QLatin1String( "dbf" ) )
{
QString pathShp = path.left( path.count() - 4 ) + ".shp";
if ( QFileInfo::exists( pathShp ) )
return nullptr;
}
// fix vsifile path and name
if ( !vsiPrefix.isEmpty() )
{
// add vsiPrefix to path if needed
if ( !path.startsWith( vsiPrefix ) )
path = vsiPrefix + path;
// if this is a /vsigzip/path_to_zip.zip/file_inside_zip remove the full path from the name
// no need to change the name I believe
#if 0
if ( ( is_vsizip || is_vsitar ) && ( path != vsiPrefix + parentItem->path() ) )
{
name = path;
name = name.replace( vsiPrefix + parentItem->path() + '/', "" );
}
#endif
}
// Filters out the OGR/GDAL supported formats that can contain multiple layers
// and should be treated like a DB: GeoPackage and SQLite
// NOTE: this formats are scanned for rasters too and they must
// be skipped by "gdal" provider or the rasters will be listed
// twice. ogrSupportedDbLayersExtensions must be kept in sync
// with the companion variable (same name) in the gdal provider
// class
// TODO: add more OGR supported multiple layers formats here!
static QStringList sOgrSupportedDbLayersExtensions { QStringLiteral( "gpkg" ),
QStringLiteral( "sqlite" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ),
QStringLiteral( "kml" ) };
static QStringList sOgrSupportedDbDriverNames { QStringLiteral( "GPKG" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ) };
// these extensions are trivial to read, so there's no need to rely on
// the extension only scan here -- avoiding it always gives us the correct data type
// and sublayer visiblity
static QStringList sSkipFastTrackExtensions { QStringLiteral( "xlsx" ),
QStringLiteral( "ods" ),
QStringLiteral( "csv" ),
QStringLiteral( "nc" ) };
// Fast track: return item without testing if:
// scanExtSetting or zipfile and scan zip == "Basic scan"
// netCDF files can be both raster or vector, so fallback to opening
if ( ( scanExtSetting ||
( ( is_vsizip || is_vsitar ) && scanZipSetting == QLatin1String( "basic" ) ) ) &&
!sSkipFastTrackExtensions.contains( suffix ) )
{
// if this is a VRT file make sure it is vector VRT to avoid duplicates
if ( suffix == QLatin1String( "vrt" ) )
{
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
GDALDriverH hDriver = GDALIdentifyDriver( path.toUtf8().constData(), nullptr );
CPLPopErrorHandler();
if ( !hDriver || GDALGetDriverShortName( hDriver ) == QLatin1String( "VRT" ) )
{
QgsDebugMsgLevel( QStringLiteral( "Skipping VRT file because root is not a OGR VRT" ), 2 );
return nullptr;
}
}
// Handle collections
// Check if the layer has sublayers by comparing the extension
QgsDataItem *item = nullptr;
if ( ! sOgrSupportedDbLayersExtensions.contains( suffix ) )
{
item = new QgsOgrLayerItem( parentItem, name, path, path, QgsLayerItem::Vector );
}
else if ( suffix.compare( QLatin1String( "gpkg" ), Qt::CaseInsensitive ) == 0 )
{
item = new QgsGeoPackageCollectionItem( parentItem, name, path );
}
else
{
item = new QgsOgrDataCollectionItem( parentItem, name, path );
}
if ( item )
return item;
}
// Slow track: scan file contents
QgsDataItem *item = nullptr;
// test that file is valid with OGR
if ( OGRGetDriverCount() == 0 )
{
OGRRegisterAll();
}
// do not print errors, but write to debug
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
gdal::dataset_unique_ptr hDS( GDALOpenEx( path.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr ) );
CPLPopErrorHandler();
if ( ! hDS )
{
QgsDebugMsg( QStringLiteral( "GDALOpen error # %1 : %2 on %3" ).arg( CPLGetLastErrorNo() ).arg( CPLGetLastErrorMsg() ).arg( path ) );
return nullptr;
}
GDALDriverH hDriver = GDALGetDatasetDriver( hDS.get() );
QString driverName = GDALGetDriverShortName( hDriver );
QgsDebugMsgLevel( QStringLiteral( "GDAL Driver : %1" ).arg( driverName ), 2 );
int numLayers = GDALDatasetGetLayerCount( hDS.get() );
// GeoPackage needs a specialized data item, mainly because of raster deletion not
// yet implemented in GDAL (2.2.1)
if ( driverName == QLatin1String( "GPKG" ) )
{
item = new QgsGeoPackageCollectionItem( parentItem, name, path );
}
else if ( numLayers > 1 || sOgrSupportedDbDriverNames.contains( driverName ) )
{
item = new QgsOgrDataCollectionItem( parentItem, name, path );
}
else
{
item = dataItemForLayer( parentItem, name, path, hDS.get(), 0, false, true );
}
return item;
}
QVariantMap QgsPackageAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
{
bool overwrite = parameterAsBool( parameters, QStringLiteral( "OVERWRITE" ), context );
QString packagePath = parameterAsString( parameters, QStringLiteral( "OUTPUT" ), context );
if ( packagePath.isEmpty() )
throw QgsProcessingException( QObject::tr( "No output file specified." ) );
// delete existing geopackage if it exists
if ( overwrite && QFile::exists( packagePath ) )
{
feedback->pushInfo( QObject::tr( "Removing existing file '%1'" ).arg( packagePath ) );
if ( !QFile( packagePath ).remove() )
{
throw QgsProcessingException( QObject::tr( "Could not remove existing file '%1'" ) );
}
}
OGRSFDriverH hGpkgDriver = OGRGetDriverByName( "GPKG" );
if ( !hGpkgDriver )
{
throw QgsProcessingException( QObject::tr( "GeoPackage driver not found." ) );
}
gdal::ogr_datasource_unique_ptr hDS( OGR_Dr_CreateDataSource( hGpkgDriver, packagePath.toUtf8().constData(), nullptr ) );
if ( !hDS )
throw QgsProcessingException( QObject::tr( "Creation of database failed (OGR error: %1)" ).arg( QString::fromUtf8( CPLGetLastErrorMsg() ) ) );
bool errored = false;
const QList< QgsMapLayer * > layers = parameterAsLayerList( parameters, QStringLiteral( "LAYERS" ), context );
QgsProcessingMultiStepFeedback multiStepFeedback( layers.count(), feedback );
int i = 0;
for ( QgsMapLayer *layer : layers )
{
if ( feedback->isCanceled() )
break;
multiStepFeedback.setCurrentStep( i );
i++;
feedback->pushInfo( QObject::tr( "Packaging layer %1/%2: %3" ).arg( i ).arg( layers.count() ).arg( layer ? layer->name() : QString() ) );
if ( !layer )
{
// don't throw immediately - instead do what we can and error out later
feedback->pushDebugInfo( QObject::tr( "Error retrieving map layer." ) );
errored = true;
continue;
}
switch ( layer->type() )
{
case QgsMapLayer::VectorLayer:
{
if ( !packageVectorLayer( qobject_cast< QgsVectorLayer * >( layer ), packagePath,
context, &multiStepFeedback ) )
errored = true;
break;
}
case QgsMapLayer::RasterLayer:
{
//not supported
feedback->pushDebugInfo( QObject::tr( "Raster layers are not currently supported." ) );
errored = true;
break;
}
case QgsMapLayer::PluginLayer:
//not supported
feedback->pushDebugInfo( QObject::tr( "Packaging plugin layers is not supported." ) );
errored = true;
break;
}
}
if ( errored )
throw QgsProcessingException( QObject::tr( "Error obtained while packaging one or more layers." ) );
QVariantMap outputs;
outputs.insert( QStringLiteral( "OUTPUT" ), packagePath );
return outputs;
}
QList<QgsOgrDbLayerInfo *> QgsOgrLayerItem::subLayers( const QString &path, const QString &driver )
{
QList<QgsOgrDbLayerInfo *> children;
// Vector layers
const QgsVectorLayer::LayerOptions layerOptions { QgsProject::instance()->transformContext() };
QgsVectorLayer layer( path, QStringLiteral( "ogr_tmp" ), QStringLiteral( "ogr" ), layerOptions );
if ( ! layer.isValid( ) )
{
QgsDebugMsgLevel( QStringLiteral( "Layer is not a valid %1 Vector layer %2" ).arg( path ), 3 );
}
else
{
// Collect mixed-geom layers
QMultiMap<int, QStringList> subLayersMap;
QgsOgrProvider *ogrProvider = qobject_cast<QgsOgrProvider *>( layer.dataProvider() );
const QStringList subLayersList( ogrProvider->subLayersWithoutFeatureCount( ) );
QMap< QString, int > mapLayerNameToCount;
bool uniqueNames = true;
int prevIdx = -1;
for ( const QString &descriptor : subLayersList )
{
QStringList pieces = descriptor.split( QgsDataProvider::SUBLAYER_SEPARATOR );
int idx = pieces[0].toInt();
subLayersMap.insert( idx, pieces );
if ( pieces.count() >= 4 && idx != prevIdx )
{
QString layerName = pieces[1];
int count = ++mapLayerNameToCount[layerName];
if ( count > 1 || layerName.isEmpty() )
uniqueNames = false;
}
prevIdx = idx;
}
prevIdx = -1;
const auto subLayerKeys = subLayersMap.keys( );
for ( const int &idx : subLayerKeys )
{
if ( idx == prevIdx )
{
continue;
}
prevIdx = idx;
QList<QStringList> values = subLayersMap.values( idx );
for ( int i = 0; i < values.size(); ++i )
{
QStringList pieces = values.at( i );
QString layerId = pieces[0];
QString name = pieces[1];
// QString featuresCount = pieces[2]; // Not used
QString geometryType = pieces[3];
QString geometryColumn = pieces[4];
QgsLayerItem::LayerType layerType;
layerType = QgsOgrLayerItem::layerTypeFromDb( geometryType );
// example URI for mixed-geoms geoms: '/path/gdal_sample_v1.2_no_extensions.gpkg|layerid=7|geometrytype=Point'
// example URI for mixed-geoms attr table: '/path/gdal_sample_v1.2_no_extensions.gpkg|layername=MyLayer|layerid=7'
// example URI for single geoms: '/path/gdal_sample_v1.2_no_extensions.gpkg|layerid=6'
QString uri;
if ( layerType != QgsLayerItem::LayerType::NoType )
{
if ( geometryType.contains( QStringLiteral( "Collection" ), Qt::CaseInsensitive ) )
{
QgsDebugMsgLevel( QStringLiteral( "Layer %1 is a geometry collection: skipping %2" ).arg( name, path ), 3 );
}
else
{
if ( uniqueNames )
uri = QStringLiteral( "%1|layername=%2" ).arg( path, name );
else
uri = QStringLiteral( "%1|layerid=%2" ).arg( path, layerId );
if ( values.size() > 1 )
{
uri += QStringLiteral( "|geometrytype=" ) + geometryType;
}
QgsDebugMsgLevel( QStringLiteral( "Adding %1 Vector item %2 %3 %4" ).arg( driver, name, uri, geometryType ), 3 );
children.append( new QgsOgrDbLayerInfo( path, uri, name, geometryColumn, geometryType, layerType ) );
}
}
else
{
QgsDebugMsgLevel( QStringLiteral( "Layer type is not a supported %1 Vector layer %2" ).arg( driver, path ), 3 );
uri = QStringLiteral( "%1|layerid=%2|layername=%3" ).arg( path, layerId, name );
children.append( new QgsOgrDbLayerInfo( path, uri, name, geometryColumn, geometryType, QgsLayerItem::LayerType::TableLayer ) );
}
QgsDebugMsgLevel( QStringLiteral( "Adding %1 Vector item %2 %3 %4" ).arg( driver, name, uri, geometryType ), 3 );
}
}
}
// Raster layers
QgsRasterLayer::LayerOptions options;
options.loadDefaultStyle = false;
QgsRasterLayer rlayer( path, QStringLiteral( "gdal_tmp" ), QStringLiteral( "gdal" ), options );
if ( !rlayer.dataProvider()->subLayers( ).empty() )
{
const QStringList layers( rlayer.dataProvider()->subLayers( ) );
for ( const QString &uri : layers )
{
// Split on ':' since this is what comes out from the provider
QStringList pieces = uri.split( ':' );
QString name = pieces.value( pieces.length() - 1 );
QgsDebugMsgLevel( QStringLiteral( "Adding GeoPackage Raster item %1 %2 %3" ).arg( name, uri ), 3 );
children.append( new QgsOgrDbLayerInfo( path, uri, name, QString(), QStringLiteral( "Raster" ), QgsLayerItem::LayerType::Raster ) );
}
}
else if ( rlayer.isValid( ) )
{
// Get the identifier
GDALAllRegister();
// do not print errors, but write to debug
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
gdal::dataset_unique_ptr hDS( GDALOpen( path.toUtf8().constData(), GA_ReadOnly ) );
CPLPopErrorHandler();
if ( ! hDS )
{
QgsDebugMsg( QStringLiteral( "GDALOpen error # %1 : %2 " ).arg( CPLGetLastErrorNo() ).arg( CPLGetLastErrorMsg() ) );
}
else
{
QString uri( QStringLiteral( "%1:%2" ).arg( driver, path ) );
QString name = GDALGetMetadataItem( hDS.get(), "IDENTIFIER", nullptr );
hDS.reset();
// Fallback: will not be able to delete the table
if ( name.isEmpty() )
{
name = QFileInfo( path ).fileName();
}
else
{
uri += QStringLiteral( ":%1" ).arg( name );
}
QgsDebugMsgLevel( QStringLiteral( "Adding %1 Raster item %2 %3" ).arg( driver, name, path ), 3 );
children.append( new QgsOgrDbLayerInfo( path, uri, name, QString(), QStringLiteral( "Raster" ), QgsLayerItem::LayerType::Raster ) );
}
}
return children;
}
