void runTest() { DisableLog dl; OsmReader reader; shared_ptr<OsmMap> map(new OsmMap()); OsmMap::resetCounters(); reader.setDefaultStatus(Status::Unknown1); reader.read("test-files/ToyBuildingsTestA.osm", map); reader.setDefaultStatus(Status::Unknown2); reader.read("test-files/ToyBuildingsTestB.osm", map); // introduce a false positive in the test data. vector<long> wids = map->findWays("name", "Cheddar's Casual Cafe"); map->getWay(wids[0])->getTags()["REF1"] = "Bad REF1"; // introduce a false negative in the test data. wids = map->findWays("name", "Freddy's"); map->getWay(wids[0])->getTags()["REF1"] = "Biondi"; // add a uuid to all buildings. HasTagCriterion filter("REF1", "REF2"); AddUuidVisitor uuid("uuid"); FilteredVisitor v(filter, uuid); map->visitRw(v); shared_ptr<OsmMap> copy(new OsmMap(map)); /*#warning Remove this custom configuration that keeps the test from erroring out Settings testSettings = conf(); testSettings.set("conflate.match.threshold", QString::number(0.6)); testSettings.set("conflate.miss.threshold", QString::number(0.6)); testSettings.set("conflate.review.threshold", QString::number(0.6));*/ UnifyingConflator conflator; //conflator.setConfiguration(testSettings); conflator.apply(copy); MatchComparator comparator; double tpr = comparator.evaluateMatches(map, copy); LOG_INFO(comparator.toString()); // for debugging MapProjector::projectToWgs84(copy); QDir(".").mkpath("test-output/scoring"); OsmWriter writer; writer.write(copy, "test-output/scoring/MatchComparatorTest.osm"); CPPUNIT_ASSERT_DOUBLES_EQUAL(0.75, tpr, 0.001); CPPUNIT_ASSERT_EQUAL(6, comparator.getTp()); CPPUNIT_ASSERT_EQUAL(1, comparator.getFn()); CPPUNIT_ASSERT_EQUAL(1, comparator.getFp()); CPPUNIT_ASSERT_DOUBLES_EQUAL(0.857143, comparator.getPertyScore(), 0.000001); }
int ConflateCmd::runSimple(QStringList args) { Timer totalTime; Timer t; QList<SingleStat> stats; bool displayStats = false; //force stats to always be the last optional param so it can be followed by an optional //output file QString outputStatsFile; if (args.contains("--stats")) { if (args.endsWith("--stats")) { displayStats = true; //remove "--stats" from args list args.pop_back(); } else if (args[args.size() - 1] == "--stats") { displayStats = true; outputStatsFile = args[args.size() - 1]; //remove "--stats" and stats output file name from args list args.pop_back(); args.pop_back(); } } LOG_VARD(displayStats); LOG_VARD(outputStatsFile); DiffConflator diffConflator; bool isDiffConflate = false; if (args.contains("--differential")) { isDiffConflate = true; args.removeAt(args.indexOf("--differential")); // Check for tags argument "--Include-Tags" if (args.contains("--include-tags")) { diffConflator.enableTags(); args.removeAt(args.indexOf("--include-tags")); } } LOG_VARD(isDiffConflate); // Check for separate output files (for geometry & tags) bool separateOutput = false; if (args.contains("--separate-output")) { separateOutput = true; args.removeAt(args.indexOf("--separate-output")); } if (args.size() < 2 || args.size() > 3) { cout << getHelp() << endl << endl; throw HootException(QString("%1 takes two or three parameters.").arg(getName())); } QString input1 = args[0]; QString input2, output; if (args.size() == 3) { input2 = args[1]; output = args[2]; } else { output = args[1]; } QString msg = "Conflating " + input1.right(50) + " with " + input2.right(50) + " and writing the output to " + output.right(50); if (isDiffConflate) { msg = msg.prepend("Differentially "); } LOG_INFO(msg); double bytesRead = IoSingleStat(IoSingleStat::RChar).value; LOG_VART(bytesRead); QList<QList<SingleStat>> allStats; // read input 1 OsmMapPtr map(new OsmMap()); IoUtils::loadMap(map, input1, ConfigOptions().getReaderConflateUseDataSourceIds1(), Status::Unknown1); ChangesetProviderPtr pTagChanges; if (isDiffConflate) { // Store original IDs for tag diff diffConflator.storeOriginalMap(map); diffConflator.markInputElements(map); } // read input 2 if (!input2.isEmpty()) { IoUtils::loadMap(map, input2, ConfigOptions().getReaderConflateUseDataSourceIds2(), Status::Unknown2); } double inputBytes = IoSingleStat(IoSingleStat::RChar).value - bytesRead; LOG_VART(inputBytes); double elapsed = t.getElapsedAndRestart(); stats.append(SingleStat("Read Inputs Time (sec)", elapsed)); stats.append(SingleStat("(Dubious) Read Inputs Bytes", inputBytes)); stats.append(SingleStat("(Dubious) Read Inputs Bytes per Second", inputBytes / elapsed)); CalculateStatsOp input1Cso( ElementCriterionPtr(new StatusCriterion(Status::Unknown1)), "input map 1"); CalculateStatsOp input2Cso( ElementCriterionPtr(new StatusCriterion(Status::Unknown2)), "input map 2"); if (displayStats) { input1Cso.apply(map); allStats.append(input1Cso.getStats()); stats.append(SingleStat("Time to Calculate Stats for Input 1 (sec)", t.getElapsedAndRestart())); if (input2 != "") { input2Cso.apply(map); allStats.append(input2Cso.getStats()); stats.append(SingleStat("Time to Calculate Stats for Input 2 (sec)", t.getElapsedAndRestart())); } } size_t initialElementCount = map->getElementCount(); stats.append(SingleStat("Initial Element Count", initialElementCount)); OsmMapWriterFactory::writeDebugMap(map, "after-load"); LOG_INFO("Applying pre-conflation operations..."); NamedOp(ConfigOptions().getConflatePreOps()).apply(map); stats.append(SingleStat("Apply Named Ops Time (sec)", t.getElapsedAndRestart())); OsmMapWriterFactory::writeDebugMap(map, "after-pre-ops"); OsmMapPtr result = map; if (isDiffConflate) { // call the diff conflator diffConflator.apply(result); if (diffConflator.conflatingTags()) { pTagChanges = diffConflator.getTagDiff(); } stats.append(diffConflator.getStats()); stats.append(SingleStat("Conflation Time (sec)", t.getElapsedAndRestart())); } else { UnifyingConflator conflator; conflator.apply(result); stats.append(conflator.getStats()); stats.append(SingleStat("Conflation Time (sec)", t.getElapsedAndRestart())); } // Apply any user specified operations. _updateConfigOptionsForAttributeConflation(); LOG_INFO("Applying post-conflation operations..."); LOG_VART(ConfigOptions().getConflatePostOps()); NamedOp(ConfigOptions().getConflatePostOps()).apply(result); OsmMapWriterFactory::writeDebugMap(map, "after-post-ops"); // doing this after the conflate post ops, since some invalid reviews are removed by them CountUniqueReviewsVisitor countReviewsVis; result->visitRo(countReviewsVis); LOG_INFO("Generated " << countReviewsVis.getStat() << " feature reviews."); MapProjector::projectToWgs84(result); stats.append(SingleStat("Project to WGS84 Time (sec)", t.getElapsedAndRestart())); // Figure out what to write if (isDiffConflate && output.endsWith(".osc")) { diffConflator.writeChangeset(result, output, separateOutput); } else { // Write a map if (isDiffConflate && diffConflator.conflatingTags()) { // Add tag changes to our map diffConflator.addChangesToMap(result, pTagChanges); } IoUtils::saveMap(result, output); } // Do the tags if we need to if (isDiffConflate && diffConflator.conflatingTags()) { LOG_INFO("Generating tag changeset..."); // Write the file! QString outFileName = output; outFileName.replace(".osm", ""); } double timingOutput = t.getElapsedAndRestart(); if (displayStats) { CalculateStatsOp outputCso("output map", true); outputCso.apply(result); QList<SingleStat> outputStats = outputCso.getStats(); if (input2 != "") { ConflateStatsHelper(input1Cso.getStats(), input2Cso.getStats(), outputCso.getStats()) .updateStats( outputStats, outputCso.indexOfSingleStat("Total Unmatched Features")); } allStats.append(outputStats); stats.append(SingleStat("Time to Calculate Stats for Output (sec)", t.getElapsedAndRestart())); } double totalElapsed = totalTime.getElapsed(); stats.append(SingleStat("(Dubious) Initial Elements Processed per Second", initialElementCount / totalElapsed)); stats.append(SingleStat("(Dubious) Final Elements Processed per Second", result->getElementCount() / totalElapsed)); stats.append(SingleStat("Write Output Time (sec)", timingOutput)); stats.append(SingleStat("Final Element Count", result->getElementCount())); stats.append(SingleStat("Total Time Elapsed (sec)", totalElapsed)); stats.append(IoSingleStat(IoSingleStat::RChar)); stats.append(IoSingleStat(IoSingleStat::WChar)); stats.append(IoSingleStat(IoSingleStat::SysCr)); stats.append(IoSingleStat(IoSingleStat::SysCw)); stats.append(IoSingleStat(IoSingleStat::ReadBytes)); stats.append(IoSingleStat(IoSingleStat::WriteBytes)); stats.append(IoSingleStat(IoSingleStat::CancelledWriteBytes)); stats.append(SingleStat("(Dubious) Bytes Processed per Second", inputBytes / totalElapsed)); if (isDiffConflate) { diffConflator.calculateStats(result, stats); } if (displayStats) { if (outputStatsFile.isEmpty()) { allStats.append(stats); QString statsMsg = MapStatsWriter().statsToString(allStats, "\t"); cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat)\n" << statsMsg << endl; } else { allStats.append(stats); MapStatsWriter().writeStatsToJson(allStats, outputStatsFile); cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat) in file: " << outputStatsFile << endl; } } LOG_INFO("Conflation job completed."); return 0; }