RubberSheet::RubberSheet() { _emptyMatch.score = 0.0; _emptyMatch.p = 0.0; _ref = ConfigOptions().getRubberSheetRef(); _debug = ConfigOptions().getRubberSheetDebug(); _minimumTies = ConfigOptions().getRubberSheetMinimumTies(); }
void setUp() { TestUtils::resetEnvironment(); conf().set(ConfigOptions().getMatchCreatorsKey(), "hoot::BuildingMatchCreator;hoot::PlacesPoiMatchCreator;hoot::PoiPolygonMatchCreator"); conf().set(ConfigOptions().getMergerCreatorsKey(), "hoot::BuildingMergerCreator;hoot::PlacesPoiMergerCreator;hoot::PoiPolygonMergerCreator"); }
void RemoveElementsVisitor::setConfiguration(const Settings& conf) { QString filterName = ConfigOptions(conf).getRemoveElementsVisitorFilter(); if (filterName.isEmpty() == false) { ElementCriterion* ef = Factory::getInstance().constructObject<ElementCriterion>(filterName); _filter.reset(ef); } _recursive = ConfigOptions(conf).getRemoveElementsVisitorRecursive(); }
ImplicitTagRawRulesDeriver::ImplicitTagRawRulesDeriver() : _statusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()), _countFileLineCtr(0), _sortParallelCount(QThread::idealThreadCount()), _skipFiltering(false), _keepTempFiles(false), _tempFileDir(ConfigOptions().getApidbBulkInserterTempFileDir()), _translateNamesToEnglish(true) { }
NetworkDetails::NetworkDetails(ConstOsmMapPtr map, ConstOsmNetworkPtr network) : _map(map), _network(network) { _sublineMatcher.reset( Factory::getInstance().constructObject<SublineStringMatcher>( ConfigOptions().getHighwaySublineStringMatcher())); _classifier.reset( Factory::getInstance().constructObject<HighwayClassifier>( ConfigOptions().getConflateMatchHighwayClassifier())); }
void testAll() { srand(0); OsmMap::resetCounters(); Settings::getInstance().clear(); conf().set(ConfigOptions().getUuidHelperRepeatableKey(), true); conf().set(ConfigOptions().getUnifyOptimizerTimeLimitKey(), -1); string outDir = "test-output/hadoop/HadoopTileWorkerTest/"; Hdfs fs; if (fs.exists(outDir)) { fs.deletePath(outDir, true); } fs.copyFromLocal("test-files/DcTigerRoads.pbf", outDir + "in1.pbf/DcTigerRoads.pbf"); fs.copyFromLocal("test-files/DcGisRoads.pbf", outDir + "in2.pbf/DcGisRoads.pbf"); shared_ptr<TileWorker> worker(new HadoopTileWorker()); TileConflator uut(worker); // ~240m uut.setBuffer(8.0 / 3600.0); uut.setMaxNodesPerBox(5000); uut.setSources(QString::fromStdString(outDir) + "in1.pbf", QString::fromStdString(outDir) + "in2.pbf"); uut.conflate(QString::fromStdString(outDir) + "HadoopTileWorkerTest.pbf"); shared_ptr<OsmMap> map(new OsmMap); PbfReader reader(true); reader.setUseFileStatus(true); std::vector<FileStatus> status = fs.listStatus(outDir + "HadoopTileWorkerTest.pbf", true); for (size_t i = 0; i < status.size(); i++) { const string& path = status[i].getPath(); LOG_INFO(path); if (QString::fromStdString(path).endsWith(".pbf")) { shared_ptr<istream> is(fs.open(path)); reader.parse(is.get(), map); } } QDir().mkpath(QString::fromStdString(outDir)); OsmWriter writer; writer.setIncludeHootInfo(true); writer.write(map, QString::fromStdString(outDir + "/result.osm")); HOOT_FILE_EQUALS("test-files/hadoop/HadoopTileWorkerTest/result.osm", "test-output/hadoop/HadoopTileWorkerTest/result.osm"); }
void ImplicitTagRawRulesDeriver::setConfiguration(const Settings& conf) { ConfigOptions options = ConfigOptions(conf); setSortParallelCount(options.getImplicitTaggingRawRulesDeriverSortParallelCount()); const int idealThreads = QThread::idealThreadCount(); LOG_VART(idealThreads); if (_sortParallelCount < 1 || _sortParallelCount > idealThreads) { setSortParallelCount(idealThreads); } setSkipFiltering(options.getImplicitTaggingRawRulesDeriverSkipFiltering()); setKeepTempFiles(options.getImplicitTaggingKeepTempFiles()); setTempFileDir(options.getApidbBulkInserterTempFileDir()); setTranslateNamesToEnglish(options.getImplicitTaggingDatabaseDeriverTranslateNamesToEnglish()); setElementCriterion(options.getImplicitTaggingElementCriterion()); if (_translateNamesToEnglish) { _translator.reset( Factory::getInstance().constructObject<ToEnglishTranslator>( options.getLanguageTranslationTranslator())); _translator->setConfiguration(conf); _translator->setSourceLanguages(options.getLanguageTranslationSourceLanguages()); _translator->setId("ImplicitTagRawRulesDeriver"); } }
shared_ptr<OsmMapWriter> OsmMapWriterFactory::createWriter(QString url) { QString writerOverride = ConfigOptions().getOsmMapWriterFactoryWriter(); shared_ptr<OsmMapWriter> writer; if (writerOverride != "") { writer.reset(Factory::getInstance().constructObject<OsmMapWriter>(writerOverride)); } vector<std::string> names = Factory::getInstance().getObjectNamesByBase(OsmMapWriter::className()); for (size_t i = 0; i < names.size() && !writer; ++i) { writer.reset(Factory::getInstance().constructObject<OsmMapWriter>(names[i])); if (writer->isSupported(url)) { LOG_DEBUG("Using writer: " << names[i]); } else { writer.reset(); } } if (!writer) { throw HootException("A valid writer could not be found for the URL: " + url); } return writer; }
ImplicitTagRulesDatabaseDeriver::ImplicitTagRulesDatabaseDeriver() : _statusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()), _minTagOccurrencesPerWord(1), _minWordLength(1), _useSchemaTagValuesForWordsOnly(true) { }
void UnifyingConflator::_addReviewTags(const OsmMapPtr& map, const vector<const Match*>& matches) { if (ConfigOptions(_settings).getConflateAddScoreTags()) { for (size_t i = 0; i < matches.size(); i++) { const Match* m = matches[i]; const MatchClassification& mc = m->getClassification(); set< pair<ElementId, ElementId> > pairs = m->getMatchPairs(); for (set< pair<ElementId, ElementId> >::const_iterator it = pairs.begin(); it != pairs.end(); ++it) { if (mc.getReviewP() > 0.0) { ElementPtr e1 = map->getElement(it->first); ElementPtr e2 = map->getElement(it->second); _addScoreTags(e1, mc); _addScoreTags(e2, mc); e1->getTags().appendValue(scoreUuidKey(), e2->getTags().getCreateUuid()); e2->getTags().appendValue(scoreUuidKey(), e1->getTags().getCreateUuid()); } } } } }
void PhoneNumberParser::setConfiguration(const Settings& conf) { ConfigOptions config = ConfigOptions(conf); setRegionCode(config.getPhoneNumberRegionCode()); setAdditionalTagKeys(config.getPhoneNumberAdditionalTagKeys()); setSearchInText(config.getPhoneNumberSearchInText()); }
void MapStatsWriter::writeStats(const QString& mapInputPath, const QString& statsOutputFilePath, QString sep) { LOG_INFO("Writing stats for map in file: " << mapInputPath << " to file: " << statsOutputFilePath); // read the conflation status from the file. conf().set(ConfigOptions().getReaderUseFileStatusKey(), true); shared_ptr<OsmMap> map(new OsmMap()); OsmUtils::loadMap(map, mapInputPath, true, Status::Invalid); MapReprojector::reprojectToPlanar(map); QList< QList<SingleStat> > allStats; shared_ptr<CalculateStatsOp> cso(new CalculateStatsOp()); cso->apply(map); allStats.append(cso->getStats()); QFile outputFile(statsOutputFilePath); if (outputFile.exists()) { outputFile.remove(); } if (outputFile.open(QFile::WriteOnly | QFile::Text)) { QTextStream out(&outputFile); //out << "Stat Name\t" << args.join(sep) << endl; out << statsToString(allStats, sep); outputFile.close(); } else { LOG_ERROR("Unable to write to output file."); } }
void OgrWriter::setConfiguration(const Settings& conf) { setCreateAllLayers(conf.getBool(createAllLayersKey(), false)); setScriptPath(conf.getString(scriptKey(), "")); setPrependLayerName(conf.getString(preLayerNameKey(), "")); _appendData = ConfigOptions(conf).getOgrAppendData(); QString strictStr = conf.getString(strictCheckingKey(), strictCheckingDefault()); if (strictStr == "on") { _strictChecking = StrictOn; } else if (strictStr == "off") { _strictChecking = StrictOff; } else if (strictStr == "warn") { _strictChecking = StrictWarn; } else { throw HootException("Error setting strict checking. Expected on/off/warn. got: " + strictStr); } }
void TagCategoryDifferencer::setConfiguration(const Settings& conf) { _category = OsmSchemaCategory::fromString(ConfigOptions(conf).getTagCategoryDifferencerName()); if (_category.toStringList().size() != 1) { throw HootException("The TagCategoryDifferencer can only filter on a single category."); } }
boost::shared_ptr<GDALDataset> OgrUtilities::openDataSource(const QString url, bool readonly) { /* Check for the correct driver name, if unknown try all drivers. * This can be an issue because drivers are tried in the order that they are * loaded which has been known to cause issues. */ OgrDriverInfo driverInfo = getDriverInfo(url, readonly); // With GDALOpenEx, we need to specify the GDAL_OF_UPDATE option or the dataset will get opened // Read Only. if (! readonly) { driverInfo._driverType = driverInfo._driverType | GDAL_OF_UPDATE; } LOG_VART(driverInfo._driverName); LOG_VART(driverInfo._driverType); LOG_VART(url.toUtf8().data()); const char* drivers[2] = { driverInfo._driverName, NULL }; // Setup read options for various file types OgrOptions options; if (QString(driverInfo._driverName) == "CSV") { options["X_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvLonfield(); options["Y_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvLatfield(); // options["Z_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvZfield(); options["KEEP_GEOM_COLUMNS"] = ConfigOptions().getOgrReaderCsvKeepGeomFields(); } if (QString(driverInfo._driverName) == "OGR_OGDI") { // From the GDAL docs: // From GDAL/OGR 1.8.0, setting the OGR_OGDI_LAUNDER_LAYER_NAMES configuration option // (or environment variable) to YES causes the layer names to be simplified. // For example : watrcrsl_hydro instead of 'watrcrsl@hydro(*)_line' options["OGR_OGDI_LAUNDER_LAYER_NAMES"] = ConfigOptions().getOgrReaderOgdiLaunderLayerNames(); } boost::shared_ptr<GDALDataset> result(static_cast<GDALDataset*>(GDALOpenEx(url.toUtf8().data(), driverInfo._driverType, (driverInfo._driverName != NULL ? drivers : NULL), options.getCrypticOptions(), NULL))); if (!result) throw HootException("Unable to open: " + url); return result; }
unsigned int Log::getWarnMessageLimit() { if (_warnMessageLimit == 0) { _warnMessageLimit = ConfigOptions().getLogWarnMessageLimit(); } return _warnMessageLimit; }
shared_ptr<IdGenerator> IdGenerator::getInstance() { if (!_theInstance) { _theInstance.reset(Factory::getInstance().constructObject<IdGenerator>( ConfigOptions().getIdGenerator())); } return _theInstance; }
void Log::init() { _logger = Logger::getLogger("hoot"); ConsoleAppender* ca = new ConsoleAppender( new PatternLayout(ConfigOptions().getLogFormat().toStdString())); _logger->removeAllAppenders(); _logger->addAppender(ca); _logger->setLevel(toLog4CxxLevel(Log::getLevel())); }
LevenshteinDistance::LevenshteinDistance(double alpha) { if (alpha == -1) { setAlpha(ConfigOptions().getLevenshteinDistanceAlpha()); } else { setAlpha(alpha); } }
boost::shared_ptr<const TagMerger> TagMergerFactory::getDefaultPtr() { if (!_default) { QString defaultName = ConfigOptions().getTagMergerDefault(); LOG_DEBUG("Default tag merger is: " << defaultName); _default = getMergerPtr(defaultName); } return _default; }
void PlacesPoiMerger::setConfiguration(const Settings& conf) { if (conf.hasKey(placesTagMerger())) { _merger = &TagMergerFactory::getInstance().getMerger(ConfigOptions(conf).getPlacesTagMerger()); } else { _merger = &TagMergerFactory::getInstance().getDefault(); } }
void TranslationOp::apply(shared_ptr<OsmMap>& map) { if (_translator.get() == 0) { throw Exception("The translation script must be set before the op can be applied. See " "the configuration key: " + ConfigOptions().getTranslationScriptKey()); } TranslationVisitor v(*_translator, _toOgr, map.get()); map->visitRw(v); }
void MostEnglishName::setConfiguration(const Settings& conf) { if (this == _theInstance.get()) { throw HootException("Please do not set the configuration on the singleton instance."); } _wordPaths = ConfigOptions(conf).getEnglishWordsFiles(); _tokenizer.setConfiguration(conf); _englishWords.clear(); _initialized = false; }
void AddressTranslator::setConfiguration(const Settings& conf) { ConfigOptions config = ConfigOptions(conf); if (!_translator) { _translator.reset( Factory::getInstance().constructObject<ToEnglishTranslator>( config.getLanguageTranslationTranslator())); _translator->setConfiguration(conf); _translator->setSourceLanguages(config.getLanguageTranslationSourceLanguages()); _translator->setId(QString::fromStdString(className())); } }
void ImplicitTagRulesDatabaseDeriver::_removeKvpsBelowOccurrenceThreshold(const QString input, const int minOccurrencesThreshold) { LOG_INFO( "Removing tags below minimum occurrence threshold of: " + QString::number(minOccurrencesThreshold) << "..."); _thresholdedCountFile.reset( new QTemporaryFile( ConfigOptions().getApidbBulkInserterTempFileDir() + "/implicit-tag-rules-deriver-temp-XXXXXX")); _thresholdedCountFile->setAutoRemove(!ConfigOptions().getImplicitTaggingKeepTempFiles()); if (!_thresholdedCountFile->open()) { throw HootException( QObject::tr("Error opening %1 for writing.").arg(_thresholdedCountFile->fileName())); } LOG_DEBUG("Opened thresholded temp file: " << _thresholdedCountFile->fileName()); if (ConfigOptions().getImplicitTaggingKeepTempFiles()) { LOG_WARN("Keeping temp file: " << _thresholdedCountFile->fileName()); } if (!QFile(input).exists()) { throw HootException("Unable to clean file; file doesn't exist."); } //This removes lines with occurrence counts below the specified threshold; not exactly sure why //one needs to be subtracted from the min occurrences here, though, for the result to be correct const QString cmd = "cat " + input + " | awk -v limit=" + QString::number(minOccurrencesThreshold - 1) + " '$1 > limit{print}' > " + _thresholdedCountFile->fileName(); LOG_DEBUG(cmd); if (std::system(cmd.toStdString().c_str()) != 0) { throw HootException("Unable to clean input file."); } _thresholdedCountFile->close(); }
void WayJoinDriver::joinPointsToWays(QString input, QString out) { // create a job Job job; job.setVerbose(Log::getInstance().getLevel() <= Log::Debug); // set the name job.setName("WayJoin2 " + input.toStdString()); // set the input/output LOG_INFO("input: " << input); job.setInput(input.toStdString()); job.setOutput(out.toStdString()); // Pass the min/max values as parameters to the job. MapStats stats; stats.readDir(input); stats.write(job.getConfiguration()); LOG_INFO("MapStats: " << stats.toString().toStdString()); // Pass the maximum way size as a parameter to the job. job.getConfiguration().setDouble("hoot.max.way.size", _maxWaySize); // be nice and don't start the reduce tasks until most of the map tasks are done. job.getConfiguration().setDouble("mapred.reduce.slowstart.completed.maps", 0.98); job.getConfiguration().setInt(WayJoin2Mapper::elementStatusKey(), _newStatus.getEnum()); job.getConfiguration().setLong(WayJoin2Mapper::nodeIdDeltaKey(), _nodeIdDelta); job.getConfiguration().setLong(WayJoin2Mapper::relationIdDeltaKey(), _relationIdDelta); job.getConfiguration().setLong(WayJoin2Mapper::wayIdDeltaKey(), _wayIdDelta); // setup the mapper and reducer classes. job.setMapperClass(WayJoin2Mapper::className()); job.setReducerClass(WayJoin2Reducer::className()); job.setInputFormatClass(WayJoin2InputFormat::className()); job.setRecordReaderClass(WayJoin2RecordReader::className()); job.setRecordWriterClass(PbfRecordWriter::className()); // Adds all libraries in this directory to the job. job.addLibraryDirs(ConfigOptions().getHootHadoopLibpath()); job.addFile(ConfPath::search("hoot.json").toStdString()); // This library will be used to provide mapper/reducer classes and anything else referenced // by the factory. job.addPlugin(getenv("HOOT_HOME") + string("/lib/libHootHadoop.so.1")); _addDefaultJobSettings(job); // run the job. job.run(); }
void OsmMapReaderFactory::_read(boost::shared_ptr<OsmMap> map, boost::shared_ptr<OsmMapReader> reader, const QString url) { boost::shared_ptr<Boundable> boundable = boost::dynamic_pointer_cast<Boundable>(reader); if (!ConfigOptions().getConvertBoundingBox().trimmed().isEmpty() && !boundable.get()) { throw IllegalArgumentException( ConfigOptions::getConvertBoundingBoxKey() + " configuration option used with unsupported reader for data source: " + url); } reader->open(url); reader->read(map); VALIDATE(map->validate(true)); }
void Hoot::reinit() { long max = _toBytes(ConfigOptions().getMaxMemoryUsage()); if (max > 0l) { struct rlimit rl; getrlimit(RLIMIT_AS, &rl); rl.rlim_cur = max; setrlimit(RLIMIT_AS, &rl); getrlimit(RLIMIT_AS, &rl); LOG_DEBUG("Set max memory usage to: " << rl.rlim_cur << "bytes."); } Log::getInstance().init(); }
Handle<Value> LogJs::log(const Arguments& args, Log::WarningLevel level) { HandleScope scope; Context::Scope context_scope(Context::GetCurrent()); if (level >= Log::getInstance().getLevel()) { Local<StackTrace> stack = StackTrace::CurrentStackTrace(1); Local<StackFrame> frame = stack->GetFrame(0); int lineNumber = -1; QString script("<unknown>"); QString functionName("<unknown>"); if (stack->GetFrameCount() >= 1) { lineNumber = frame->GetLineNumber(); script = toString(frame->GetScriptName()); functionName = toString(frame->GetFunctionName()); } std::stringstream rMessage; for (int i = 0; i < args.Length(); i++) { if (i != 0) { rMessage << " "; } rMessage << args[i]; } QString message = QString::fromUtf8(rMessage.str().data()); int logLimit = ConfigOptions().getOgrLogLimit(); int messageCount = getLogCount(message); if (messageCount == logLimit) { message = QString("Received %1 of the same message. Silencing: ").arg(messageCount) + message; } if (messageCount <= logLimit) { Log::getInstance().log(level, message, script, functionName, lineNumber); } } return scope.Close(Undefined()); }
bool NetworkDetails::isReversed(ConstNetworkEdgePtr e1, ConstNetworkEdgePtr e2) { assert(e1->getMembers().size() == 1); assert(e2->getMembers().size() == 1); ConstWayPtr w1 = dynamic_pointer_cast<const Way>(e1->getMembers()[0]); ConstWayPtr w2 = dynamic_pointer_cast<const Way>(e2->getMembers()[0]); // calculated the shared sublines WaySublineMatchString sublineMatch = _sublineMatcher->findMatch(_map, w1, w2, ConfigOptions().getSearchRadiusHighway()); if (sublineMatch.getReverseVector2().size() != 1) { throw NotImplementedException(); } return sublineMatch.getReverseVector2()[0]; }