int main(int argc,const char* argv[]) { _START_EASYLOGGINGPP(argc,argv); el::Loggers::addFlag(el::LoggingFlag::ColoredTerminalOutput); el::Loggers::reconfigureAllLoggers(el::ConfigurationType::Format, "%datetime : %msg"); /* parse command line */ LOG(INFO) << "Parsing command line arguments"; cmdargs_t args = parse_args(argc,argv); /* parse the collection */ LOG(INFO) << "Parsing collection directory " << args.collection_dir; collection col(args.collection_dir); /* load pattern file */ auto patterns = pattern_parser::parse_file<false>(args.pattern_file); LOG(INFO) << "Parsed " << patterns.size() << " patterns from file " << args.pattern_file; /* filter patterns */ if (args.patterns_per_bucket != 0) { std::sort(patterns.begin(), patterns.end(), [](const pattern_t& a, const pattern_t& b) { return a.bucket < b.bucket; }); size_t freq = 0; size_t cnt = 0; size_t bucket = patterns[0].bucket; auto itr = patterns.begin(); while (itr != patterns.end()) { if (itr->bucket != bucket) { LOG(INFO) << "bucket = " << bucket << " cnt = " << cnt; bucket = itr->bucket; freq = 0; cnt = 0; } if (freq >= args.patterns_per_bucket) { itr = patterns.erase(itr); } else { itr++; cnt++; } freq++; } LOG(INFO) << "bucket = " << bucket << " cnt = " << cnt; LOG(INFO) << "Filtered " << patterns.size() << " patterns"; } /* load indexes and test */ using invidx_typeA = index_invidx<uniform_eliasfano_list<128>,optpfor_list<128,false>>; invidx_typeA indexA(col); using invidx_typeB = index_invidx<eliasfano_skip_list<64,true>,optpfor_list<128,false>>; invidx_typeB indexB(col); compare_index(indexA,indexB,patterns); return 0; }
void QgsOverlayUtils::difference( const QgsFeatureSource &sourceA, const QgsFeatureSource &sourceB, QgsFeatureSink &sink, QgsProcessingContext &context, QgsProcessingFeedback *feedback, int &count, int totalCount, QgsOverlayUtils::DifferenceOutput outputAttrs ) { QgsFeatureRequest requestB; requestB.setSubsetOfAttributes( QgsAttributeList() ); if ( outputAttrs != OutputBA ) requestB.setDestinationCrs( sourceA.sourceCrs(), context.transformContext() ); QgsSpatialIndex indexB( sourceB.getFeatures( requestB ), feedback ); int fieldsCountA = sourceA.fields().count(); int fieldsCountB = sourceB.fields().count(); QgsAttributes attrs; attrs.resize( outputAttrs == OutputA ? fieldsCountA : ( fieldsCountA + fieldsCountB ) ); if ( totalCount == 0 ) totalCount = 1; // avoid division by zero QgsFeature featA; QgsFeatureRequest requestA; if ( outputAttrs == OutputBA ) requestA.setDestinationCrs( sourceB.sourceCrs(), context.transformContext() ); QgsFeatureIterator fitA = sourceA.getFeatures( requestA ); while ( fitA.nextFeature( featA ) ) { if ( feedback->isCanceled() ) break; if ( featA.hasGeometry() ) { QgsGeometry geom( featA.geometry() ); QgsFeatureIds intersects = indexB.intersects( geom.boundingBox() ).toSet(); QgsFeatureRequest request; request.setFilterFids( intersects ); request.setSubsetOfAttributes( QgsAttributeList() ); if ( outputAttrs != OutputBA ) request.setDestinationCrs( sourceA.sourceCrs(), context.transformContext() ); std::unique_ptr< QgsGeometryEngine > engine; if ( !intersects.isEmpty() ) { // use prepared geometries for faster intersection tests engine.reset( QgsGeometry::createGeometryEngine( geom.constGet() ) ); engine->prepareGeometry(); } QVector<QgsGeometry> geometriesB; QgsFeature featB; QgsFeatureIterator fitB = sourceB.getFeatures( request ); while ( fitB.nextFeature( featB ) ) { if ( feedback->isCanceled() ) break; if ( engine->intersects( featB.geometry().constGet() ) ) geometriesB << featB.geometry(); } if ( !geometriesB.isEmpty() ) { QgsGeometry geomB = QgsGeometry::unaryUnion( geometriesB ); geom = geom.difference( geomB ); } if ( !sanitizeDifferenceResult( geom ) ) continue; const QgsAttributes attrsA( featA.attributes() ); switch ( outputAttrs ) { case OutputA: attrs = attrsA; break; case OutputAB: for ( int i = 0; i < fieldsCountA; ++i ) attrs[i] = attrsA[i]; break; case OutputBA: for ( int i = 0; i < fieldsCountA; ++i ) attrs[i + fieldsCountB] = attrsA[i]; break; } QgsFeature outFeat; outFeat.setGeometry( geom ); outFeat.setAttributes( attrs ); sink.addFeature( outFeat, QgsFeatureSink::FastInsert ); } else { // TODO: should we write out features that do not have geometry? sink.addFeature( featA, QgsFeatureSink::FastInsert ); } ++count; feedback->setProgress( count / ( double ) totalCount * 100. ); } }
void QgsOverlayUtils::intersection( const QgsFeatureSource &sourceA, const QgsFeatureSource &sourceB, QgsFeatureSink &sink, QgsProcessingContext &context, QgsProcessingFeedback *feedback, int &count, int totalCount, const QList<int> &fieldIndicesA, const QList<int> &fieldIndicesB ) { QgsWkbTypes::GeometryType geometryType = QgsWkbTypes::geometryType( QgsWkbTypes::multiType( sourceA.wkbType() ) ); int attrCount = fieldIndicesA.count() + fieldIndicesB.count(); QgsFeatureRequest request; request.setSubsetOfAttributes( QgsAttributeList() ); request.setDestinationCrs( sourceA.sourceCrs(), context.transformContext() ); QgsFeature outFeat; QgsSpatialIndex indexB( sourceB.getFeatures( request ), feedback ); if ( totalCount == 0 ) totalCount = 1; // avoid division by zero QgsFeature featA; QgsFeatureIterator fitA = sourceA.getFeatures( QgsFeatureRequest().setSubsetOfAttributes( fieldIndicesA ) ); while ( fitA.nextFeature( featA ) ) { if ( feedback->isCanceled() ) break; if ( !featA.hasGeometry() ) continue; QgsGeometry geom( featA.geometry() ); QgsFeatureIds intersects = indexB.intersects( geom.boundingBox() ).toSet(); QgsFeatureRequest request; request.setFilterFids( intersects ); request.setDestinationCrs( sourceA.sourceCrs(), context.transformContext() ); request.setSubsetOfAttributes( fieldIndicesB ); std::unique_ptr< QgsGeometryEngine > engine; if ( !intersects.isEmpty() ) { // use prepared geometries for faster intersection tests engine.reset( QgsGeometry::createGeometryEngine( geom.constGet() ) ); engine->prepareGeometry(); } QgsAttributes outAttributes( attrCount ); const QgsAttributes attrsA( featA.attributes() ); for ( int i = 0; i < fieldIndicesA.count(); ++i ) outAttributes[i] = attrsA[fieldIndicesA[i]]; QgsFeature featB; QgsFeatureIterator fitB = sourceB.getFeatures( request ); while ( fitB.nextFeature( featB ) ) { if ( feedback->isCanceled() ) break; QgsGeometry tmpGeom( featB.geometry() ); if ( !engine->intersects( tmpGeom.constGet() ) ) continue; QgsGeometry intGeom = geom.intersection( tmpGeom ); if ( !sanitizeIntersectionResult( intGeom, geometryType ) ) continue; const QgsAttributes attrsB( featB.attributes() ); for ( int i = 0; i < fieldIndicesB.count(); ++i ) outAttributes[fieldIndicesA.count() + i] = attrsB[fieldIndicesB[i]]; outFeat.setGeometry( intGeom ); outFeat.setAttributes( outAttributes ); sink.addFeature( outFeat, QgsFeatureSink::FastInsert ); } ++count; feedback->setProgress( count / ( double ) totalCount * 100. ); } }