void write_header(const osmium::io::Header& header) override final {
std::string out = "<?xml version='1.0' encoding='UTF-8'?>\n";
if (m_file.is_true("xml_change_format")) {
out += "<osmChange version=\"0.6\" generator=\"";
xml_string(out, header.get("generator").c_str());
out += "\">\n";
} else {
out += "<osm version=\"0.6\"";
std::string xml_josm_upload = header.get("xml_josm_upload");
if (xml_josm_upload == "true" || xml_josm_upload == "false") {
out += " upload=\"";
out += xml_josm_upload;
out += "\"";
}
out += " generator=\"";
xml_string(out, header.get("generator").c_str());
out += "\">\n";
}
for (const auto& box : header.boxes()) {
out += " <bounds";
oprintf(out, " minlon=\"%.7f\"", box.bottom_left().lon());
oprintf(out, " minlat=\"%.7f\"", box.bottom_left().lat());
oprintf(out, " maxlon=\"%.7f\"", box.top_right().lon());
oprintf(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat());
}
std::promise<std::string> promise;
m_output_queue.push(promise.get_future());
promise.set_value(std::move(out));
}
void CommandCat::setup_header(osmium::io::Header& header) const {
header.set("generator", m_generator);
for (const auto& h : m_output_headers) {
header.set(h);
}
}
explicit InputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
m_file(file),
m_read_which_entities(read_which_entities),
m_input_queue(input_queue) {
m_header.set_has_multiple_object_versions(m_file.has_multiple_object_versions());
}
Input(const osmium::io::File& file) :
m_file(file) {
m_header.has_multiple_object_versions(m_file.has_multiple_object_versions());
m_file.open_for_input();
}
/**
* TODO: Refactor this function into smaller functions for better readability.
*
* This function is the entry point for the whole extraction process. The goal of the extraction
* step is to filter and convert the OSM geometry to something more fitting for routing.
* That includes:
* - extracting turn restrictions
* - splitting ways into (directional!) edge segments
* - checking if nodes are barriers or traffic signal
* - discarding all tag information: All relevant type information for nodes/ways
* is extracted at this point.
*
* The result of this process are the following files:
* .names : Names of all streets, stored as long consecutive string with prefix sum based index
* .osrm : Nodes and edges in a intermediate format that easy to digest for osrm-prepare
* .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph
*
*/
int extractor::run(const ExtractorConfig &extractor_config)
{
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(extracting);
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
const auto number_of_threads =
std::min(recommended_num_threads, extractor_config.requested_num_threads);
tbb::task_scheduler_init init(number_of_threads);
SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string();
SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << number_of_threads;
// setup scripting environment
ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str());
ExtractionContainers extraction_containers;
auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers);
const osmium::io::File input_file(extractor_config.input_path.string());
osmium::io::Reader reader(input_file);
const osmium::io::Header header = reader.header();
std::atomic<unsigned> number_of_nodes{0};
std::atomic<unsigned> number_of_ways{0};
std::atomic<unsigned> number_of_relations{0};
std::atomic<unsigned> number_of_others{0};
SimpleLogger().Write() << "Parsing in progress..";
TIMER_START(parsing);
std::string generator = header.get("generator");
if (generator.empty())
{
generator = "unknown tool";
}
SimpleLogger().Write() << "input file generated by " << generator;
// write .timestamp data file
std::string timestamp = header.get("osmosis_replication_timestamp");
if (timestamp.empty())
{
timestamp = "n/a";
}
SimpleLogger().Write() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
timestamp_out.close();
// initialize vectors holding parsed objects
tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes;
tbb::concurrent_vector<std::pair<std::size_t, ExtractionWay>> resulting_ways;
tbb::concurrent_vector<mapbox::util::optional<InputRestrictionContainer>>
resulting_restrictions;
// setup restriction parser
const RestrictionParser restriction_parser(scripting_environment.get_lua_state());
while (const osmium::memory::Buffer buffer = reader.read())
{
// create a vector of iterators into the buffer
std::vector<osmium::memory::Buffer::const_iterator> osm_elements;
for (auto iter = std::begin(buffer); iter != std::end(buffer); ++iter)
{
osm_elements.push_back(iter);
}
// clear resulting vectors
resulting_nodes.clear();
resulting_ways.clear();
resulting_restrictions.clear();
// parse OSM entities in parallel, store in resulting vectors
tbb::parallel_for(
tbb::blocked_range<std::size_t>(0, osm_elements.size()),
[&](const tbb::blocked_range<std::size_t> &range)
{
ExtractionNode result_node;
ExtractionWay result_way;
lua_State *local_state = scripting_environment.get_lua_state();
for (auto x = range.begin(); x != range.end(); ++x)
{
const auto entity = osm_elements[x];
switch (entity->type())
{
case osmium::item_type::node:
result_node.clear();
++number_of_nodes;
luabind::call_function<void>(
local_state, "node_function",
boost::cref(static_cast<const osmium::Node &>(*entity)),
boost::ref(result_node));
resulting_nodes.push_back(std::make_pair(x, result_node));
break;
case osmium::item_type::way:
result_way.clear();
++number_of_ways;
luabind::call_function<void>(
local_state, "way_function",
boost::cref(static_cast<const osmium::Way &>(*entity)),
boost::ref(result_way));
resulting_ways.push_back(std::make_pair(x, result_way));
break;
case osmium::item_type::relation:
++number_of_relations;
resulting_restrictions.push_back(restriction_parser.TryParse(
static_cast<const osmium::Relation &>(*entity)));
break;
default:
++number_of_others;
break;
}
}
});
// put parsed objects thru extractor callbacks
for (const auto &result : resulting_nodes)
{
extractor_callbacks->ProcessNode(
static_cast<const osmium::Node &>(*(osm_elements[result.first])),
result.second);
}
for (const auto &result : resulting_ways)
{
extractor_callbacks->ProcessWay(
static_cast<const osmium::Way &>(*(osm_elements[result.first])), result.second);
}
for (const auto &result : resulting_restrictions)
{
extractor_callbacks->ProcessRestriction(result);
}
}
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
SimpleLogger().Write() << "Raw input contains " << number_of_nodes.load() << " nodes, "
<< number_of_ways.load() << " ways, and "
<< number_of_relations.load() << " relations, and "
<< number_of_others.load() << " unknown entities";
extractor_callbacks.reset();
if (extraction_containers.all_edges_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
extraction_containers.PrepareData(extractor_config.output_file_name,
extractor_config.restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << extractor_config.output_file_name
<< std::endl;
}
catch (std::exception &e)
{
SimpleLogger().Write(logWARNING) << e.what();
return 1;
}
return 0;
}