RubberSheet::RubberSheet()
{
_emptyMatch.score = 0.0;
_emptyMatch.p = 0.0;
_ref = ConfigOptions().getRubberSheetRef();
_debug = ConfigOptions().getRubberSheetDebug();
_minimumTies = ConfigOptions().getRubberSheetMinimumTies();
}
void setUp()
{
TestUtils::resetEnvironment();
conf().set(ConfigOptions().getMatchCreatorsKey(),
"hoot::BuildingMatchCreator;hoot::PlacesPoiMatchCreator;hoot::PoiPolygonMatchCreator");
conf().set(ConfigOptions().getMergerCreatorsKey(),
"hoot::BuildingMergerCreator;hoot::PlacesPoiMergerCreator;hoot::PoiPolygonMergerCreator");
}
void RemoveElementsVisitor::setConfiguration(const Settings& conf)
{
QString filterName = ConfigOptions(conf).getRemoveElementsVisitorFilter();
if (filterName.isEmpty() == false)
{
ElementCriterion* ef = Factory::getInstance().constructObject<ElementCriterion>(filterName);
_filter.reset(ef);
}
_recursive = ConfigOptions(conf).getRemoveElementsVisitorRecursive();
}
ImplicitTagRawRulesDeriver::ImplicitTagRawRulesDeriver() :
_statusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()),
_countFileLineCtr(0),
_sortParallelCount(QThread::idealThreadCount()),
_skipFiltering(false),
_keepTempFiles(false),
_tempFileDir(ConfigOptions().getApidbBulkInserterTempFileDir()),
_translateNamesToEnglish(true)
{
}
NetworkDetails::NetworkDetails(ConstOsmMapPtr map, ConstOsmNetworkPtr network) :
_map(map),
_network(network)
{
_sublineMatcher.reset(
Factory::getInstance().constructObject<SublineStringMatcher>(
ConfigOptions().getHighwaySublineStringMatcher()));
_classifier.reset(
Factory::getInstance().constructObject<HighwayClassifier>(
ConfigOptions().getConflateMatchHighwayClassifier()));
}
void testAll()
{
srand(0);
OsmMap::resetCounters();
Settings::getInstance().clear();
conf().set(ConfigOptions().getUuidHelperRepeatableKey(), true);
conf().set(ConfigOptions().getUnifyOptimizerTimeLimitKey(), -1);
string outDir = "test-output/hadoop/HadoopTileWorkerTest/";
Hdfs fs;
if (fs.exists(outDir))
{
fs.deletePath(outDir, true);
}
fs.copyFromLocal("test-files/DcTigerRoads.pbf", outDir + "in1.pbf/DcTigerRoads.pbf");
fs.copyFromLocal("test-files/DcGisRoads.pbf", outDir + "in2.pbf/DcGisRoads.pbf");
shared_ptr<TileWorker> worker(new HadoopTileWorker());
TileConflator uut(worker);
// ~240m
uut.setBuffer(8.0 / 3600.0);
uut.setMaxNodesPerBox(5000);
uut.setSources(QString::fromStdString(outDir) + "in1.pbf",
QString::fromStdString(outDir) + "in2.pbf");
uut.conflate(QString::fromStdString(outDir) + "HadoopTileWorkerTest.pbf");
shared_ptr<OsmMap> map(new OsmMap);
PbfReader reader(true);
reader.setUseFileStatus(true);
std::vector<FileStatus> status = fs.listStatus(outDir + "HadoopTileWorkerTest.pbf", true);
for (size_t i = 0; i < status.size(); i++)
{
const string& path = status[i].getPath();
LOG_INFO(path);
if (QString::fromStdString(path).endsWith(".pbf"))
{
shared_ptr<istream> is(fs.open(path));
reader.parse(is.get(), map);
}
}
QDir().mkpath(QString::fromStdString(outDir));
OsmWriter writer;
writer.setIncludeHootInfo(true);
writer.write(map, QString::fromStdString(outDir + "/result.osm"));
HOOT_FILE_EQUALS("test-files/hadoop/HadoopTileWorkerTest/result.osm",
"test-output/hadoop/HadoopTileWorkerTest/result.osm");
}
void ImplicitTagRawRulesDeriver::setConfiguration(const Settings& conf)
{
ConfigOptions options = ConfigOptions(conf);
setSortParallelCount(options.getImplicitTaggingRawRulesDeriverSortParallelCount());
const int idealThreads = QThread::idealThreadCount();
LOG_VART(idealThreads);
if (_sortParallelCount < 1 || _sortParallelCount > idealThreads)
{
setSortParallelCount(idealThreads);
}
setSkipFiltering(options.getImplicitTaggingRawRulesDeriverSkipFiltering());
setKeepTempFiles(options.getImplicitTaggingKeepTempFiles());
setTempFileDir(options.getApidbBulkInserterTempFileDir());
setTranslateNamesToEnglish(options.getImplicitTaggingDatabaseDeriverTranslateNamesToEnglish());
setElementCriterion(options.getImplicitTaggingElementCriterion());
if (_translateNamesToEnglish)
{
_translator.reset(
Factory::getInstance().constructObject<ToEnglishTranslator>(
options.getLanguageTranslationTranslator()));
_translator->setConfiguration(conf);
_translator->setSourceLanguages(options.getLanguageTranslationSourceLanguages());
_translator->setId("ImplicitTagRawRulesDeriver");
}
}
shared_ptr<OsmMapWriter> OsmMapWriterFactory::createWriter(QString url)
{
QString writerOverride = ConfigOptions().getOsmMapWriterFactoryWriter();
shared_ptr<OsmMapWriter> writer;
if (writerOverride != "")
{
writer.reset(Factory::getInstance().constructObject<OsmMapWriter>(writerOverride));
}
vector<std::string> names =
Factory::getInstance().getObjectNamesByBase(OsmMapWriter::className());
for (size_t i = 0; i < names.size() && !writer; ++i)
{
writer.reset(Factory::getInstance().constructObject<OsmMapWriter>(names[i]));
if (writer->isSupported(url))
{
LOG_DEBUG("Using writer: " << names[i]);
}
else
{
writer.reset();
}
}
if (!writer)
{
throw HootException("A valid writer could not be found for the URL: " + url);
}
return writer;
}
ImplicitTagRulesDatabaseDeriver::ImplicitTagRulesDatabaseDeriver() :
_statusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()),
_minTagOccurrencesPerWord(1),
_minWordLength(1),
_useSchemaTagValuesForWordsOnly(true)
{
}
void UnifyingConflator::_addReviewTags(const OsmMapPtr& map, const vector<const Match*>& matches)
{
if (ConfigOptions(_settings).getConflateAddScoreTags())
{
for (size_t i = 0; i < matches.size(); i++)
{
const Match* m = matches[i];
const MatchClassification& mc = m->getClassification();
set< pair<ElementId, ElementId> > pairs = m->getMatchPairs();
for (set< pair<ElementId, ElementId> >::const_iterator it = pairs.begin();
it != pairs.end(); ++it)
{
if (mc.getReviewP() > 0.0)
{
ElementPtr e1 = map->getElement(it->first);
ElementPtr e2 = map->getElement(it->second);
_addScoreTags(e1, mc);
_addScoreTags(e2, mc);
e1->getTags().appendValue(scoreUuidKey(), e2->getTags().getCreateUuid());
e2->getTags().appendValue(scoreUuidKey(), e1->getTags().getCreateUuid());
}
}
}
}
}
void PhoneNumberParser::setConfiguration(const Settings& conf)
{
ConfigOptions config = ConfigOptions(conf);
setRegionCode(config.getPhoneNumberRegionCode());
setAdditionalTagKeys(config.getPhoneNumberAdditionalTagKeys());
setSearchInText(config.getPhoneNumberSearchInText());
}
void MapStatsWriter::writeStats(const QString& mapInputPath, const QString& statsOutputFilePath,
QString sep)
{
LOG_INFO("Writing stats for map in file: " << mapInputPath << " to file: " << statsOutputFilePath);
// read the conflation status from the file.
conf().set(ConfigOptions().getReaderUseFileStatusKey(), true);
shared_ptr<OsmMap> map(new OsmMap());
OsmUtils::loadMap(map, mapInputPath, true, Status::Invalid);
MapReprojector::reprojectToPlanar(map);
QList< QList<SingleStat> > allStats;
shared_ptr<CalculateStatsOp> cso(new CalculateStatsOp());
cso->apply(map);
allStats.append(cso->getStats());
QFile outputFile(statsOutputFilePath);
if (outputFile.exists())
{
outputFile.remove();
}
if (outputFile.open(QFile::WriteOnly | QFile::Text))
{
QTextStream out(&outputFile);
//out << "Stat Name\t" << args.join(sep) << endl;
out << statsToString(allStats, sep);
outputFile.close();
}
else
{
LOG_ERROR("Unable to write to output file.");
}
}
void OgrWriter::setConfiguration(const Settings& conf)
{
setCreateAllLayers(conf.getBool(createAllLayersKey(), false));
setScriptPath(conf.getString(scriptKey(), ""));
setPrependLayerName(conf.getString(preLayerNameKey(), ""));
_appendData = ConfigOptions(conf).getOgrAppendData();
QString strictStr = conf.getString(strictCheckingKey(), strictCheckingDefault());
if (strictStr == "on")
{
_strictChecking = StrictOn;
}
else if (strictStr == "off")
{
_strictChecking = StrictOff;
}
else if (strictStr == "warn")
{
_strictChecking = StrictWarn;
}
else
{
throw HootException("Error setting strict checking. Expected on/off/warn. got: " + strictStr);
}
}
void TagCategoryDifferencer::setConfiguration(const Settings& conf)
{
_category = OsmSchemaCategory::fromString(ConfigOptions(conf).getTagCategoryDifferencerName());
if (_category.toStringList().size() != 1)
{
throw HootException("The TagCategoryDifferencer can only filter on a single category.");
}
}
boost::shared_ptr<GDALDataset> OgrUtilities::openDataSource(const QString url, bool readonly)
{
/* Check for the correct driver name, if unknown try all drivers.
* This can be an issue because drivers are tried in the order that they are
* loaded which has been known to cause issues.
*/
OgrDriverInfo driverInfo = getDriverInfo(url, readonly);
// With GDALOpenEx, we need to specify the GDAL_OF_UPDATE option or the dataset will get opened
// Read Only.
if (! readonly)
{
driverInfo._driverType = driverInfo._driverType | GDAL_OF_UPDATE;
}
LOG_VART(driverInfo._driverName);
LOG_VART(driverInfo._driverType);
LOG_VART(url.toUtf8().data());
const char* drivers[2] = { driverInfo._driverName, NULL };
// Setup read options for various file types
OgrOptions options;
if (QString(driverInfo._driverName) == "CSV")
{
options["X_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvLonfield();
options["Y_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvLatfield();
// options["Z_POSSIBLE_NAMES"] = ConfigOptions().getOgrReaderCsvZfield();
options["KEEP_GEOM_COLUMNS"] = ConfigOptions().getOgrReaderCsvKeepGeomFields();
}
if (QString(driverInfo._driverName) == "OGR_OGDI")
{
// From the GDAL docs:
// From GDAL/OGR 1.8.0, setting the OGR_OGDI_LAUNDER_LAYER_NAMES configuration option
// (or environment variable) to YES causes the layer names to be simplified.
// For example : watrcrsl_hydro instead of 'watrcrsl@hydro(*)_line'
options["OGR_OGDI_LAUNDER_LAYER_NAMES"] = ConfigOptions().getOgrReaderOgdiLaunderLayerNames();
}
boost::shared_ptr<GDALDataset> result(static_cast<GDALDataset*>(GDALOpenEx(url.toUtf8().data(),
driverInfo._driverType, (driverInfo._driverName != NULL ? drivers : NULL), options.getCrypticOptions(), NULL)));
if (!result)
throw HootException("Unable to open: " + url);
return result;
}
unsigned int Log::getWarnMessageLimit()
{
if (_warnMessageLimit == 0)
{
_warnMessageLimit = ConfigOptions().getLogWarnMessageLimit();
}
return _warnMessageLimit;
}
shared_ptr<IdGenerator> IdGenerator::getInstance()
{
if (!_theInstance)
{
_theInstance.reset(Factory::getInstance().constructObject<IdGenerator>(
ConfigOptions().getIdGenerator()));
}
return _theInstance;
}
void Log::init()
{
_logger = Logger::getLogger("hoot");
ConsoleAppender* ca = new ConsoleAppender(
new PatternLayout(ConfigOptions().getLogFormat().toStdString()));
_logger->removeAllAppenders();
_logger->addAppender(ca);
_logger->setLevel(toLog4CxxLevel(Log::getLevel()));
}
LevenshteinDistance::LevenshteinDistance(double alpha)
{
if (alpha == -1)
{
setAlpha(ConfigOptions().getLevenshteinDistanceAlpha());
}
else
{
setAlpha(alpha);
}
}
boost::shared_ptr<const TagMerger> TagMergerFactory::getDefaultPtr()
{
if (!_default)
{
QString defaultName = ConfigOptions().getTagMergerDefault();
LOG_DEBUG("Default tag merger is: " << defaultName);
_default = getMergerPtr(defaultName);
}
return _default;
}
void PlacesPoiMerger::setConfiguration(const Settings& conf)
{
if (conf.hasKey(placesTagMerger()))
{
_merger = &TagMergerFactory::getInstance().getMerger(ConfigOptions(conf).getPlacesTagMerger());
}
else
{
_merger = &TagMergerFactory::getInstance().getDefault();
}
}
void TranslationOp::apply(shared_ptr<OsmMap>& map)
{
if (_translator.get() == 0)
{
throw Exception("The translation script must be set before the op can be applied. See "
"the configuration key: " + ConfigOptions().getTranslationScriptKey());
}
TranslationVisitor v(*_translator, _toOgr, map.get());
map->visitRw(v);
}
void MostEnglishName::setConfiguration(const Settings& conf)
{
if (this == _theInstance.get())
{
throw HootException("Please do not set the configuration on the singleton instance.");
}
_wordPaths = ConfigOptions(conf).getEnglishWordsFiles();
_tokenizer.setConfiguration(conf);
_englishWords.clear();
_initialized = false;
}
void AddressTranslator::setConfiguration(const Settings& conf)
{
ConfigOptions config = ConfigOptions(conf);
if (!_translator)
{
_translator.reset(
Factory::getInstance().constructObject<ToEnglishTranslator>(
config.getLanguageTranslationTranslator()));
_translator->setConfiguration(conf);
_translator->setSourceLanguages(config.getLanguageTranslationSourceLanguages());
_translator->setId(QString::fromStdString(className()));
}
}
void ImplicitTagRulesDatabaseDeriver::_removeKvpsBelowOccurrenceThreshold(const QString input,
const int minOccurrencesThreshold)
{
LOG_INFO(
"Removing tags below minimum occurrence threshold of: " +
QString::number(minOccurrencesThreshold) << "...");
_thresholdedCountFile.reset(
new QTemporaryFile(
ConfigOptions().getApidbBulkInserterTempFileDir() +
"/implicit-tag-rules-deriver-temp-XXXXXX"));
_thresholdedCountFile->setAutoRemove(!ConfigOptions().getImplicitTaggingKeepTempFiles());
if (!_thresholdedCountFile->open())
{
throw HootException(
QObject::tr("Error opening %1 for writing.").arg(_thresholdedCountFile->fileName()));
}
LOG_DEBUG("Opened thresholded temp file: " << _thresholdedCountFile->fileName());
if (ConfigOptions().getImplicitTaggingKeepTempFiles())
{
LOG_WARN("Keeping temp file: " << _thresholdedCountFile->fileName());
}
if (!QFile(input).exists())
{
throw HootException("Unable to clean file; file doesn't exist.");
}
//This removes lines with occurrence counts below the specified threshold; not exactly sure why
//one needs to be subtracted from the min occurrences here, though, for the result to be correct
const QString cmd =
"cat " + input + " | awk -v limit=" + QString::number(minOccurrencesThreshold - 1) +
" '$1 > limit{print}' > " + _thresholdedCountFile->fileName();
LOG_DEBUG(cmd);
if (std::system(cmd.toStdString().c_str()) != 0)
{
throw HootException("Unable to clean input file.");
}
_thresholdedCountFile->close();
}
void WayJoinDriver::joinPointsToWays(QString input, QString out)
{
// create a job
Job job;
job.setVerbose(Log::getInstance().getLevel() <= Log::Debug);
// set the name
job.setName("WayJoin2 " + input.toStdString());
// set the input/output
LOG_INFO("input: " << input);
job.setInput(input.toStdString());
job.setOutput(out.toStdString());
// Pass the min/max values as parameters to the job.
MapStats stats;
stats.readDir(input);
stats.write(job.getConfiguration());
LOG_INFO("MapStats: " << stats.toString().toStdString());
// Pass the maximum way size as a parameter to the job.
job.getConfiguration().setDouble("hoot.max.way.size", _maxWaySize);
// be nice and don't start the reduce tasks until most of the map tasks are done.
job.getConfiguration().setDouble("mapred.reduce.slowstart.completed.maps", 0.98);
job.getConfiguration().setInt(WayJoin2Mapper::elementStatusKey(), _newStatus.getEnum());
job.getConfiguration().setLong(WayJoin2Mapper::nodeIdDeltaKey(), _nodeIdDelta);
job.getConfiguration().setLong(WayJoin2Mapper::relationIdDeltaKey(), _relationIdDelta);
job.getConfiguration().setLong(WayJoin2Mapper::wayIdDeltaKey(), _wayIdDelta);
// setup the mapper and reducer classes.
job.setMapperClass(WayJoin2Mapper::className());
job.setReducerClass(WayJoin2Reducer::className());
job.setInputFormatClass(WayJoin2InputFormat::className());
job.setRecordReaderClass(WayJoin2RecordReader::className());
job.setRecordWriterClass(PbfRecordWriter::className());
// Adds all libraries in this directory to the job.
job.addLibraryDirs(ConfigOptions().getHootHadoopLibpath());
job.addFile(ConfPath::search("hoot.json").toStdString());
// This library will be used to provide mapper/reducer classes and anything else referenced
// by the factory.
job.addPlugin(getenv("HOOT_HOME") + string("/lib/libHootHadoop.so.1"));
_addDefaultJobSettings(job);
// run the job.
job.run();
}
void OsmMapReaderFactory::_read(boost::shared_ptr<OsmMap> map,
boost::shared_ptr<OsmMapReader> reader, const QString url)
{
boost::shared_ptr<Boundable> boundable = boost::dynamic_pointer_cast<Boundable>(reader);
if (!ConfigOptions().getConvertBoundingBox().trimmed().isEmpty() && !boundable.get())
{
throw IllegalArgumentException(
ConfigOptions::getConvertBoundingBoxKey() +
" configuration option used with unsupported reader for data source: " + url);
}
reader->open(url);
reader->read(map);
VALIDATE(map->validate(true));
}
void Hoot::reinit()
{
long max = _toBytes(ConfigOptions().getMaxMemoryUsage());
if (max > 0l)
{
struct rlimit rl;
getrlimit(RLIMIT_AS, &rl);
rl.rlim_cur = max;
setrlimit(RLIMIT_AS, &rl);
getrlimit(RLIMIT_AS, &rl);
LOG_DEBUG("Set max memory usage to: " << rl.rlim_cur << "bytes.");
}
Log::getInstance().init();
}
Handle<Value> LogJs::log(const Arguments& args, Log::WarningLevel level) {
HandleScope scope;
Context::Scope context_scope(Context::GetCurrent());
if (level >= Log::getInstance().getLevel())
{
Local<StackTrace> stack = StackTrace::CurrentStackTrace(1);
Local<StackFrame> frame = stack->GetFrame(0);
int lineNumber = -1;
QString script("<unknown>");
QString functionName("<unknown>");
if (stack->GetFrameCount() >= 1)
{
lineNumber = frame->GetLineNumber();
script = toString(frame->GetScriptName());
functionName = toString(frame->GetFunctionName());
}
std::stringstream rMessage;
for (int i = 0; i < args.Length(); i++)
{
if (i != 0)
{
rMessage << " ";
}
rMessage << args[i];
}
QString message = QString::fromUtf8(rMessage.str().data());
int logLimit = ConfigOptions().getOgrLogLimit();
int messageCount = getLogCount(message);
if (messageCount == logLimit)
{
message = QString("Received %1 of the same message. Silencing: ").arg(messageCount) + message;
}
if (messageCount <= logLimit)
{
Log::getInstance().log(level, message, script, functionName, lineNumber);
}
}
return scope.Close(Undefined());
}
bool NetworkDetails::isReversed(ConstNetworkEdgePtr e1, ConstNetworkEdgePtr e2)
{
assert(e1->getMembers().size() == 1);
assert(e2->getMembers().size() == 1);
ConstWayPtr w1 = dynamic_pointer_cast<const Way>(e1->getMembers()[0]);
ConstWayPtr w2 = dynamic_pointer_cast<const Way>(e2->getMembers()[0]);
// calculated the shared sublines
WaySublineMatchString sublineMatch = _sublineMatcher->findMatch(_map, w1, w2,
ConfigOptions().getSearchRadiusHighway());
if (sublineMatch.getReverseVector2().size() != 1)
{
throw NotImplementedException();
}
return sublineMatch.getReverseVector2()[0];
}
