void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename,
const std::string &geometry_filename,
lua_State *lua_state)
{
TIMER_START(geometry);
CompressGeometry();
TIMER_STOP(geometry);
TIMER_START(renumber);
RenumberEdges();
TIMER_STOP(renumber);
TIMER_START(generate_nodes);
GenerateEdgeExpandedNodes();
TIMER_STOP(generate_nodes);
TIMER_START(generate_edges);
GenerateEdgeExpandedEdges(original_edge_data_filename, lua_state);
TIMER_STOP(generate_edges);
m_geometry_compressor.SerializeInternalVector(geometry_filename);
SimpleLogger().Write() << "Timing statistics for edge-expanded graph:";
SimpleLogger().Write() << "Geometry compression: " << TIMER_SEC(geometry) << "s";
SimpleLogger().Write() << "Renumbering edges: " << TIMER_SEC(renumber) << "s";
SimpleLogger().Write() << "Generating nodes: " << TIMER_SEC(generate_nodes) << "s";
SimpleLogger().Write() << "Generating edges: " << TIMER_SEC(generate_edges) << "s";
}
bool RequestManager::nextPreRenderRequest()
{
boost::mutex::scoped_lock preLock(preRJMutex);
if (preRenderRequests.empty())
return false;
shared_ptr<MetaIdentifier> mid = preRenderRequests.front();
preRenderRequests.pop();
currentPrerenderingThreads++;
preLock.unlock();
Job job(mid, config, shared_from_this());
// check if tiles are already in progress
if (running->start(&job, mid))
{
job.process();
running->finished(&job);
job.deliver();
}
if (!job.isEmpty() && mid->getZoom() < config->get<int>(opt::server::prerender_level)) {
std::vector<shared_ptr<MetaIdentifier>> children;
mid->getSubIdentifiers(children);
for (auto& c : children)
enqueue(c);
}
preLock.lock();
currentPrerenderingThreads--;
preLock.unlock();
if (currentPrerenderingThreads == 0 && preRenderRequests.size() == 0) {
TIMER_STOP(prerender);
log.info("Prerendering finished in %02i:%02i", (int) TIMER_MIN(prerender), ((int) TIMER_SEC(prerender)) % 60);
}
return true;
}
int Contractor::Run()
{
if (config.core_factor > 1.0 || config.core_factor < 0)
{
throw util::exception("Core factor must be between 0.0 to 1.0 (inclusive)" + SOURCE_REF);
}
if (config.use_cached_priority)
{
util::Log(logWARNING) << "Using cached priorities is deprecated and they will be ignored.";
}
TIMER_START(preparing);
util::Log() << "Reading node weights.";
std::vector<EdgeWeight> node_weights;
{
storage::io::FileReader reader(config.GetPath(".osrm.enw"),
storage::io::FileReader::VerifyFingerprint);
storage::serialization::read(reader, node_weights);
}
util::Log() << "Done reading node weights.";
util::Log() << "Loading edge-expanded graph representation";
std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
updater::Updater updater(config.updater_config);
EdgeID max_edge_id = updater.LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights);
// Contracting the edge-expanded graph
TIMER_START(contraction);
std::vector<std::vector<bool>> node_filters;
{
extractor::EdgeBasedNodeDataContainer node_data;
extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
extractor::ProfileProperties properties;
extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties);
node_filters = util::excludeFlagsToNodeFilter(max_edge_id + 1, node_data, properties);
}
RangebasedCRC32 crc32_calculator;
const unsigned checksum = crc32_calculator(edge_based_edge_list);
QueryGraph query_graph;
std::vector<std::vector<bool>> edge_filters;
std::vector<std::vector<bool>> cores;
std::tie(query_graph, edge_filters, cores) =
contractExcludableGraph(toContractorGraph(max_edge_id + 1, std::move(edge_based_edge_list)),
std::move(node_weights),
std::move(node_filters),
config.core_factor);
TIMER_STOP(contraction);
util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges.";
util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters);
files::writeCoreMarker(config.GetPath(".osrm.core"), cores);
TIMER_STOP(preparing);
util::Log() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
util::Log() << "finished preprocessing";
return 0;
}
int main(int argc, char *argv[])
{
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(preparing);
TIMER_START(expansion);
boost::filesystem::path config_file_path, input_path, restrictions_path, profile_path;
unsigned int requested_num_threads;
bool use_elevation;
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()("version,v", "Show version")("help,h",
"Show this help message")(
"config,c",
boost::program_options::value<boost::filesystem::path>(&config_file_path)
->default_value("contractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()(
"restrictions,r",
boost::program_options::value<boost::filesystem::path>(&restrictions_path),
"Restrictions file in .osrm.restrictions format")(
"profile,p",
boost::program_options::value<boost::filesystem::path>(&profile_path)
->default_value("profile.lua"),"Path to LUA routing profile")(
"elevation,e", boost::program_options::value<bool>(&use_elevation)->default_value(true),
"Process node elevations")(
"threads,t",
boost::program_options::value<unsigned int>(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be
// shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()(
"input,i",
boost::program_options::value<boost::filesystem::path>(&input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(
"Usage: " + boost::filesystem::basename(argv[0]) + " <input.osrm> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
.options(cmdline_options)
.positional(positional_options)
.run(),
option_variables);
if (option_variables.count("version"))
{
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return 0;
}
if (option_variables.count("help"))
{
SimpleLogger().Write() << "\n" << visible_options;
return 0;
}
boost::program_options::notify(option_variables);
if (!option_variables.count("restrictions"))
{
restrictions_path = std::string(input_path.string() + ".restrictions");
}
if (!option_variables.count("input"))
{
SimpleLogger().Write() << "\n" << visible_options;
return 0;
}
if (!boost::filesystem::is_regular_file(input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
<< " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
<< " not found!";
return 1;
}
if (1 > requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Using elevation: " << use_elevation;
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
LogPolicy::GetInstance().Unmute();
boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary);
TurnRestriction restriction;
FingerPrint fingerprint_loaded, fingerprint_orig;
unsigned number_of_usable_restrictions = 0;
restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestPrepare(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}
restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned));
restriction_list.resize(number_of_usable_restrictions);
if (number_of_usable_restrictions > 0)
{
restriction_stream.read((char *)&(restriction_list[0]),
number_of_usable_restrictions * sizeof(TurnRestriction));
}
restriction_stream.close();
boost::filesystem::ifstream in;
in.open(input_path, std::ios::in | std::ios::binary);
const std::string node_filename = input_path.string() + ".nodes";
const std::string edge_out = input_path.string() + ".edges";
const std::string geometry_filename = input_path.string() + ".geometry";
const std::string graphOut = input_path.string() + ".hsgr";
const std::string rtree_nodes_path = input_path.string() + ".ramIndex";
const std::string rtree_leafs_path = input_path.string() + ".fileIndex";
/*** Setup Scripting Environment ***/
// Create a new lua state
lua_State *lua_state = luaL_newstate();
// Connect LuaBind to this lua state
luabind::open(lua_state);
// open utility libraries string library;
luaL_openlibs(lua_state);
// adjust lua load path
luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str());
// Now call our function in a lua script
if (0 != luaL_dofile(lua_state, profile_path.string().c_str()))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return 1;
}
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return 1;
}
speed_profile.trafficSignalPenalty = 10 * lua_tointeger(lua_state, -1);
SimpleLogger().Write(logDEBUG)
<< "traffic_signal_penalty: " << speed_profile.trafficSignalPenalty;
if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n"))
{
std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl;
return 1;
}
speed_profile.uTurnPenalty = 10 * lua_tointeger(lua_state, -1);
speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function");
#ifdef WIN32
#pragma message ("Memory consumption on Windows can be higher due to memory alignment")
#else
static_assert(sizeof(ImportEdge) == 20,
"changing ImportEdge type has influence on memory consumption!");
#endif
std::vector<ImportEdge> edge_list;
NodeID number_of_node_based_nodes =
readBinaryOSRMGraphFromStream(in,
edge_list,
barrier_node_list,
traffic_light_list,
&internal_to_external_node_map,
restriction_list,
use_elevation);
in.close();
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
/***
* Building an edge-expanded graph from node-based input and turn restrictions
*/
SimpleLogger().Write() << "Generating edge-expanded graph representation";
std::shared_ptr<NodeBasedDynamicGraph> node_based_graph =
NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list);
std::unique_ptr<RestrictionMap> restriction_map =
std::unique_ptr<RestrictionMap>(new RestrictionMap(node_based_graph, restriction_list));
EdgeBasedGraphFactory *edge_based_graph_factor =
new EdgeBasedGraphFactory(node_based_graph,
std::move(restriction_map),
barrier_node_list,
traffic_light_list,
internal_to_external_node_map,
speed_profile);
edge_list.clear();
edge_list.shrink_to_fit();
edge_based_graph_factor->Run(edge_out, geometry_filename, lua_state);
restriction_list.clear();
restriction_list.shrink_to_fit();
barrier_node_list.clear();
barrier_node_list.shrink_to_fit();
traffic_light_list.clear();
traffic_light_list.shrink_to_fit();
unsigned number_of_edge_based_nodes = edge_based_graph_factor->GetNumberOfEdgeBasedNodes();
BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits<unsigned>::max());
DeallocatingVector<EdgeBasedEdge> edgeBasedEdgeList;
#ifndef WIN32
static_assert(sizeof(EdgeBasedEdge) == 16,
"changing ImportEdge type has influence on memory consumption!");
#endif
edge_based_graph_factor->GetEdgeBasedEdges(edgeBasedEdgeList);
std::vector<EdgeBasedNode> node_based_edge_list;
edge_based_graph_factor->GetEdgeBasedNodes(node_based_edge_list);
delete edge_based_graph_factor;
// TODO actually use scoping: Split this up in subfunctions
node_based_graph.reset();
TIMER_STOP(expansion);
// Building grid-like nearest-neighbor data structure
SimpleLogger().Write() << "building r-tree ...";
StaticRTree<EdgeBasedNode> *rtree =
new StaticRTree<EdgeBasedNode>(node_based_edge_list,
rtree_nodes_path.c_str(),
rtree_leafs_path.c_str(),
internal_to_external_node_map);
delete rtree;
IteratorbasedCRC32<std::vector<EdgeBasedNode>> crc32;
unsigned node_based_edge_list_CRC32 =
crc32(node_based_edge_list.begin(), node_based_edge_list.end());
node_based_edge_list.clear();
node_based_edge_list.shrink_to_fit();
SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32;
/***
* Writing info on original (node-based) nodes
*/
SimpleLogger().Write() << "writing node map ...";
boost::filesystem::ofstream node_stream(node_filename, std::ios::binary);
const unsigned size_of_mapping = internal_to_external_node_map.size();
node_stream.write((char *)&size_of_mapping, sizeof(unsigned));
if (size_of_mapping > 0)
{
node_stream.write((char *)&(internal_to_external_node_map[0]),
size_of_mapping * sizeof(NodeInfo));
}
node_stream.close();
internal_to_external_node_map.clear();
internal_to_external_node_map.shrink_to_fit();
/***
* Contracting the edge-expanded graph
*/
SimpleLogger().Write() << "initializing contractor";
Contractor *contractor = new Contractor(number_of_edge_based_nodes, edgeBasedEdgeList);
TIMER_START(contraction);
contractor->Run();
TIMER_STOP(contraction);
SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec";
DeallocatingVector<QueryEdge> contracted_edge_list;
contractor->GetEdges(contracted_edge_list);
delete contractor;
/***
* Sorting contracted edges in a way that the static query graph can read some in in-place.
*/
std::sort(contracted_edge_list.begin(), contracted_edge_list.end());
unsigned max_used_node_id = 0;
unsigned contracted_edge_count = contracted_edge_list.size();
SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
<< " edges";
boost::filesystem::ofstream hsgr_output_stream(graphOut, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint));
for (const QueryEdge &edge : contracted_edge_list)
{
BOOST_ASSERT(UINT_MAX != edge.source);
BOOST_ASSERT(UINT_MAX != edge.target);
max_used_node_id = std::max(max_used_node_id, edge.source);
max_used_node_id = std::max(max_used_node_id, edge.target);
}
SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes
<< " nodes";
SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes";
max_used_node_id += 1;
std::vector<StaticGraph<EdgeData>::NodeArrayEntry> node_array;
node_array.resize(number_of_edge_based_nodes + 1);
SimpleLogger().Write() << "Building node array";
StaticGraph<EdgeData>::EdgeIterator edge = 0;
StaticGraph<EdgeData>::EdgeIterator position = 0;
StaticGraph<EdgeData>::EdgeIterator last_edge = edge;
for (StaticGraph<EdgeData>::NodeIterator node = 0; node < max_used_node_id; ++node)
{
last_edge = edge;
while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node))
{
++edge;
}
node_array[node].first_edge = position; //=edge
position += edge - last_edge; // remove
}
for (unsigned sentinel_counter = max_used_node_id; sentinel_counter != node_array.size();
++sentinel_counter)
{
// sentinel element, guarded against underflow
node_array[sentinel_counter].first_edge = contracted_edge_count;
}
unsigned node_array_size = node_array.size();
// serialize crc32, aka checksum
hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned));
// serialize number of nodes
hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned));
// serialize number of edges
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned));
// serialize all nodes
if (node_array_size > 0)
{
hsgr_output_stream.write((char *)&node_array[0],
sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size);
}
// serialize all edges
SimpleLogger().Write() << "Building edge array";
edge = 0;
int number_of_used_edges = 0;
StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
for (unsigned edge = 0; edge < contracted_edge_list.size(); ++edge)
{
// no eigen loops
BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target);
current_edge.target = contracted_edge_list[edge].target;
current_edge.data = contracted_edge_list[edge].data;
// every target needs to be valid
BOOST_ASSERT(current_edge.target < max_used_node_id);
#ifndef NDEBUG
if (current_edge.data.distance <= 0)
{
SimpleLogger().Write(logWARNING)
<< "Edge: " << edge << ",source: " << contracted_edge_list[edge].source
<< ", target: " << contracted_edge_list[edge].target
<< ", dist: " << current_edge.data.distance;
SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node "
<< contracted_edge_list[edge].source << "/"
<< node_array.size() - 1;
return 1;
}
#endif
hsgr_output_stream.write((char *)¤t_edge,
sizeof(StaticGraph<EdgeData>::EdgeArrayEntry));
++number_of_used_edges;
}
hsgr_output_stream.close();
TIMER_STOP(preparing);
SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
SimpleLogger().Write() << "Expansion : "
<< (number_of_node_based_nodes / TIMER_SEC(expansion))
<< " nodes/sec and "
<< (number_of_edge_based_nodes / TIMER_SEC(expansion))
<< " edges/sec";
SimpleLogger().Write() << "Contraction: "
<< (number_of_edge_based_nodes / TIMER_SEC(contraction))
<< " nodes/sec and "
<< number_of_used_edges / TIMER_SEC(contraction)
<< " edges/sec";
node_array.clear();
SimpleLogger().Write() << "finished preprocessing";
}
catch (boost::program_options::too_many_positional_options_error &)
{
SimpleLogger().Write(logWARNING) << "Only one file can be specified";
return 1;
}
catch (boost::program_options::error &e)
{
SimpleLogger().Write(logWARNING) << e.what();
return 1;
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "Exception occured: " << e.what() << std::endl;
return 1;
}
return 0;
}
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
TIMER_STOP(sorting_used_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl;
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
TIMER_START(erasing_dups);
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
TIMER_STOP(erasing_dups);
std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl;
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
TIMER_START(sorting_nodes);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(),
stxxl_memory);
TIMER_STOP(sorting_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl;
std::cout << "[extractor] Sorting used ways ... " << std::flush;
TIMER_START(sort_ways);
stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(),
FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory);
TIMER_STOP(sort_ways);
std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl;
std::cout << "[extractor] Sorting " << restrictions_list.size()
<< " restrictions. by from... " << std::flush;
TIMER_START(sort_restrictions);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByFrom(), stxxl_memory);
TIMER_STOP(sort_restrictions);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
TIMER_START(fix_restriction_starts);
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.from.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_starts);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
TIMER_START(sort_restrictions_to);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByTo(), stxxl_memory);
TIMER_STOP(sort_restrictions_to);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
TIMER_START(fix_restriction_ends);
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.to.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_ends);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl;
// serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
const auto count_position = restrictions_out_stream.tellp();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
for (const auto &restriction_container : restrictions_list)
{
if (SPECIAL_NODEID != restriction_container.restriction.from.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
++written_restriction_count;
}
}
restrictions_out_stream.seekp(count_position);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
restrictions_out_stream.close();
SimpleLogger().Write() << "usable restrictions: " << written_restriction_count;
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->node_id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
TIMER_STOP(write_nodes);
std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
TIMER_START(sort_edges_by_start);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
TIMER_STOP(sort_edges_by_start);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl;
std::cout << "[extractor] Setting start coords ... " << std::flush;
TIMER_START(set_start_coords);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->start > node_iterator->node_id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->node_id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
TIMER_STOP(set_start_coords);
std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl;
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
TIMER_START(sort_edges_by_target);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(),
stxxl_memory);
TIMER_STOP(sort_edges_by_target);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl;
std::cout << "[extractor] Setting target coords ... " << std::flush;
TIMER_START(set_target_coords);
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->target > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->node_id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->speed != -1);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = coordinate_calculation::euclidean_distance(
edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon,
node_iterator->lat, node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1, (int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
const int integer_distance = std::max(1, (int)distance);
const short zero = 0;
const short one = 1;
const bool yes = true;
const bool no = false;
file_out_stream.write((char *)&edge_iterator->way_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw osrm::exception("edge has broken direction");
}
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
if (edge_iterator->is_roundabout)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_in_tiny_cc)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_access_restricted)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
// cannot take adress of bit field, so use local
const TravelMode travel_mode = edge_iterator->travel_mode;
file_out_stream.write((char *)&travel_mode, sizeof(TravelMode));
if (edge_iterator->is_split)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
++number_of_used_edges;
}
++edge_iterator;
}
TIMER_STOP(set_target_coords);
std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
std::cout << "[extractor] writing street name index ... " << std::flush;
TIMER_START(write_name_index);
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_lengths.push_back(string_length);
total_length += string_length;
}
RangeTable<> table(name_lengths);
name_file_stream << table;
name_file_stream.write((char *)&total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
TIMER_STOP(write_name_index);
std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e)
{
std::cerr << "Caught Execption:" << e.what() << std::endl;
}
}
int main(int argc, char *argv[])
{
#ifdef __FreeBSD__
SimpleLogger().Write() << "Not supported on FreeBSD";
return 0;
#endif
#ifdef _WIN32
SimpleLogger().Write() << "Not supported on Windows";
return 0;
#else
LogPolicy::GetInstance().Unmute();
boost::filesystem::path test_path;
try
{
SimpleLogger().Write() << "starting up engines, " << g_GIT_DESCRIPTION;
if (1 == argc)
{
SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " /path/on/device";
return -1;
}
test_path = boost::filesystem::path(argv[1]);
test_path /= "osrm.tst";
SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string();
// create files for testing
if (2 == argc)
{
// create file to test
if (boost::filesystem::exists(test_path))
{
throw osrm::exception("Data file already exists");
}
int *random_array = new int[number_of_elements];
std::generate(random_array, random_array + number_of_elements, std::rand);
#ifdef __APPLE__
FILE *fd = fopen(test_path.string().c_str(), "w");
fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0);
TIMER_START(write_1gb);
write(fileno(fd), (char *)random_array, number_of_elements * sizeof(unsigned));
TIMER_STOP(write_1gb);
fclose(fd);
#endif
#ifdef __linux__
int file_desc =
open(test_path.string().c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, S_IRWXU);
if (-1 == file_desc)
{
throw osrm::exception("Could not open random data file");
}
TIMER_START(write_1gb);
int ret = write(file_desc, random_array, number_of_elements * sizeof(unsigned));
if (0 > ret)
{
throw osrm::exception("could not write random data file");
}
TIMER_STOP(write_1gb);
close(file_desc);
#endif
delete[] random_array;
SimpleLogger().Write(logDEBUG) << "writing raw 1GB took " << TIMER_SEC(write_1gb)
<< "s";
SimpleLogger().Write() << "raw write performance: " << std::setprecision(5)
<< std::fixed << 1024 * 1024 / TIMER_SEC(write_1gb) << "MB/sec";
SimpleLogger().Write(logDEBUG)
<< "finished creation of random data. Flush disk cache now!";
}
else
{
// Run Non-Cached I/O benchmarks
if (!boost::filesystem::exists(test_path))
{
throw osrm::exception("data file does not exist");
}
// volatiles do not get optimized
Statistics stats;
#ifdef __APPLE__
volatile unsigned single_block[1024];
char *raw_array = new char[number_of_elements * sizeof(unsigned)];
FILE *fd = fopen(test_path.string().c_str(), "r");
fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0);
#endif
#ifdef __linux__
char *single_block = (char *)memalign(512, 1024 * sizeof(unsigned));
int file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
if (-1 == file_desc)
{
SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
return -1;
}
char *raw_array = (char *)memalign(512, number_of_elements * sizeof(unsigned));
#endif
TIMER_START(read_1gb);
#ifdef __APPLE__
read(fileno(fd), raw_array, number_of_elements * sizeof(unsigned));
close(fileno(fd));
fd = fopen(test_path.string().c_str(), "r");
#endif
#ifdef __linux__
int ret = read(file_desc, raw_array, number_of_elements * sizeof(unsigned));
SimpleLogger().Write(logDEBUG) << "read " << ret
<< " bytes, error: " << strerror(errno);
close(file_desc);
file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
#endif
TIMER_STOP(read_1gb);
SimpleLogger().Write(logDEBUG) << "reading raw 1GB took " << TIMER_SEC(read_1gb) << "s";
SimpleLogger().Write() << "raw read performance: " << std::setprecision(5) << std::fixed
<< 1024 * 1024 / TIMER_SEC(read_1gb) << "MB/sec";
std::vector<double> timing_results_raw_random;
SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB";
#ifdef __APPLE__
fseek(fd, 0, SEEK_SET);
#endif
#ifdef __linux__
lseek(file_desc, 0, SEEK_SET);
#endif
// make 1000 random access, time each I/O seperately
unsigned number_of_blocks = (number_of_elements * sizeof(unsigned) - 1) / 4096;
std::random_device rd;
std::default_random_engine e1(rd());
std::uniform_int_distribution<unsigned> uniform_dist(0, number_of_blocks - 1);
for (unsigned i = 0; i < 1000; ++i)
{
unsigned block_to_read = uniform_dist(e1);
off_t current_offset = block_to_read * 4096;
TIMER_START(random_access);
#ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char *)&single_block[0], 4096);
#endif
#ifdef __FreeBSD__
int ret1 = 0;
int ret2 = 0;
#endif
#ifdef __linux__
int ret1 = lseek(file_desc, current_offset, SEEK_SET);
int ret2 = read(file_desc, (char *)single_block, 4096);
#endif
TIMER_STOP(random_access);
if (((off_t)-1) == ret1)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
throw osrm::exception("seek error");
}
if (-1 == ret2)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
throw osrm::exception("read error");
}
timing_results_raw_random.push_back(TIMER_SEC(random_access));
}
// Do statistics
SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics";
std::ofstream random_csv("random.csv", std::ios::trunc);
for (unsigned i = 0; i < timing_results_raw_random.size(); ++i)
{
random_csv << i << ", " << timing_results_raw_random[i] << std::endl;
}
random_csv.close();
RunStatistics(timing_results_raw_random, stats);
SimpleLogger().Write() << "raw random I/O: " << std::setprecision(5) << std::fixed
<< "min: " << stats.min << "ms, "
<< "mean: " << stats.mean << "ms, "
<< "med: " << stats.med << "ms, "
<< "max: " << stats.max << "ms, "
<< "dev: " << stats.dev << "ms";
std::vector<double> timing_results_raw_seq;
#ifdef __APPLE__
fseek(fd, 0, SEEK_SET);
#endif
#ifdef __linux__
lseek(file_desc, 0, SEEK_SET);
#endif
// read every 100th block
for (unsigned i = 0; i < 1000; ++i)
{
off_t current_offset = i * 4096;
TIMER_START(read_every_100);
#ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char *)&single_block, 4096);
#endif
#ifdef __FreeBSD__
int ret1 = 0;
int ret2 = 0;
#endif
#ifdef __linux__
int ret1 = lseek(file_desc, current_offset, SEEK_SET);
int ret2 = read(file_desc, (char *)single_block, 4096);
#endif
TIMER_STOP(read_every_100);
if (((off_t)-1) == ret1)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
throw osrm::exception("seek error");
}
if (-1 == ret2)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
throw osrm::exception("read error");
}
timing_results_raw_seq.push_back(TIMER_SEC(read_every_100));
}
#ifdef __APPLE__
fclose(fd);
// free(single_element);
free(raw_array);
// free(single_block);
#endif
#ifdef __linux__
close(file_desc);
#endif
// Do statistics
SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics";
// print simple statistics: min, max, median, variance
std::ofstream seq_csv("sequential.csv", std::ios::trunc);
for (unsigned i = 0; i < timing_results_raw_seq.size(); ++i)
{
seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl;
}
seq_csv.close();
RunStatistics(timing_results_raw_seq, stats);
SimpleLogger().Write() << "raw sequential I/O: " << std::setprecision(5) << std::fixed
<< "min: " << stats.min << "ms, "
<< "mean: " << stats.mean << "ms, "
<< "med: " << stats.med << "ms, "
<< "max: " << stats.max << "ms, "
<< "dev: " << stats.dev << "ms";
if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path);
SimpleLogger().Write(logDEBUG) << "removing temporary files";
}
}
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "caught exception: " << e.what();
SimpleLogger().Write(logWARNING) << "cleaning up, and exiting";
if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path);
SimpleLogger().Write(logWARNING) << "removing temporary files";
}
return -1;
}
return 0;
#endif
}
RecursiveBisection::RecursiveBisection(BisectionGraph &bisection_graph_,
const std::size_t maximum_cell_size,
const double balance,
const double boundary_factor,
const std::size_t num_optimizing_cuts,
const std::size_t small_component_size)
: bisection_graph(bisection_graph_), internal_state(bisection_graph_)
{
auto components = internal_state.PrePartitionWithSCC(small_component_size);
BOOST_ASSERT(!components.empty());
// Parallelize recursive bisection trees. Root cut happens serially (well, this is a lie:
// since we handle big components in parallel, too. But we don't know this and
// don't have to. TBB's scheduler handles nested parallelism just fine).
//
// [ | ]
// / \ root cut
// [ | ] [ | ]
// / \ / \ descend, do cuts in parallel
//
// https://www.threadingbuildingblocks.org/docs/help/index.htm#reference/algorithms/parallel_do_func.html
struct TreeNode
{
GraphView graph;
std::uint64_t depth;
};
// Build a recursive bisection tree for all big components independently in parallel.
// Last GraphView is all small components: skip for bisection.
auto first = begin(components);
auto last = end(components) - 1;
// We construct the trees on the fly: the root node is the entry point.
// All tree branches depend on the actual cut and will be generated while descending.
std::vector<TreeNode> forest;
forest.reserve(last - first);
std::transform(first, last, std::back_inserter(forest), [this](auto graph) {
return TreeNode{std::move(graph), internal_state.SCCDepth()};
});
using Feeder = tbb::parallel_do_feeder<TreeNode>;
TIMER_START(bisection);
// Bisect graph into two parts. Get partition point and recurse left and right in parallel.
tbb::parallel_do(begin(forest), end(forest), [&](const TreeNode &node, Feeder &feeder) {
const auto cut =
computeInertialFlowCut(node.graph, num_optimizing_cuts, balance, boundary_factor);
const auto center = internal_state.ApplyBisection(
node.graph.Begin(), node.graph.End(), node.depth, cut.flags);
const auto terminal = [&](const auto &node) {
const auto maximum_depth = sizeof(BisectionID) * CHAR_BIT;
const auto too_small = node.graph.NumberOfNodes() < maximum_cell_size;
const auto too_deep = node.depth >= maximum_depth;
return too_small || too_deep;
};
GraphView left_graph{bisection_graph, node.graph.Begin(), center};
TreeNode left_node{std::move(left_graph), node.depth + 1};
if (!terminal(left_node))
feeder.add(std::move(left_node));
GraphView right_graph{bisection_graph, center, node.graph.End()};
TreeNode right_node{std::move(right_graph), node.depth + 1};
if (!terminal(right_node))
feeder.add(std::move(right_node));
});
TIMER_STOP(bisection);
util::Log() << "Full bisection done in " << TIMER_SEC(bisection) << "s";
}
int Extractor::Run(int argc, char *argv[])
{
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(extracting);
if (!ParseArguments(argc, argv))
return 0;
if (1 > requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
if (!boost::filesystem::is_regular_file(input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
<< " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
<< " not found!";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
/*** Setup Scripting Environment ***/
ScriptingEnvironment scripting_environment(profile_path.string().c_str());
GenerateOutputFilesNames();
std::unordered_map<std::string, NodeID> string_map;
ExtractionContainers extraction_containers;
string_map[""] = 0;
auto extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser;
if (file_has_pbf_format)
{
parser = new PBFParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment,
requested_num_threads);
}
else
{
parser = new XMLParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment);
}
if (!parser->ReadHeader())
{
throw OSRMException("Parser not initialized!");
}
SimpleLogger().Write() << "Parsing in progress..";
TIMER_START(parsing);
parser->Parse();
delete parser;
delete extractor_callbacks;
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
if (extraction_containers.all_edges_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
extraction_containers.PrepareData(output_file_name, restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << output_file_name << std::endl;
}
catch (boost::program_options::too_many_positional_options_error &)
{
SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
return 1;
}
catch (std::exception &e)
{
SimpleLogger().Write(logWARNING) << e.what();
return 1;
}
return 0;
}
int Prepare::Process(int argc, char *argv[])
{
LogPolicy::GetInstance().Unmute();
TIMER_START(preparing);
TIMER_START(expansion);
if (!ParseArguments(argc, argv))
{
return 0;
}
if (!boost::filesystem::is_regular_file(input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!";
return 1;
}
if (1 > requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
LogPolicy::GetInstance().Unmute();
FingerPrint fingerprint_orig;
CheckRestrictionsFile(fingerprint_orig);
boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary);
node_filename = input_path.string() + ".nodes";
edge_out = input_path.string() + ".edges";
geometry_filename = input_path.string() + ".geometry";
graph_out = input_path.string() + ".hsgr";
rtree_nodes_path = input_path.string() + ".ramIndex";
rtree_leafs_path = input_path.string() + ".fileIndex";
/*** Setup Scripting Environment ***/
// Create a new lua state
lua_State *lua_state = luaL_newstate();
// Connect LuaBind to this lua state
luabind::open(lua_state);
EdgeBasedGraphFactory::SpeedProfileProperties speed_profile;
if (!SetupScriptingEnvironment(lua_state, speed_profile))
{
return 1;
}
#ifdef WIN32
#pragma message("Memory consumption on Windows can be higher due to different bit packing")
#else
static_assert(sizeof(ImportEdge) == 20,
"changing ImportEdge type has influence on memory consumption!");
#endif
NodeID number_of_node_based_nodes =
readBinaryOSRMGraphFromStream(input_stream,
edge_list,
barrier_node_list,
traffic_light_list,
&internal_to_external_node_map,
restriction_list);
input_stream.close();
if (edge_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
SimpleLogger().Write() << restriction_list.size() << " restrictions, "
<< barrier_node_list.size() << " bollard nodes, "
<< traffic_light_list.size() << " traffic lights";
std::vector<EdgeBasedNode> node_based_edge_list;
unsigned number_of_edge_based_nodes = 0;
DeallocatingVector<EdgeBasedEdge> edge_based_edge_list;
// init node_based_edge_list, edge_based_edge_list by edgeList
number_of_edge_based_nodes = BuildEdgeExpandedGraph(lua_state,
number_of_node_based_nodes,
node_based_edge_list,
edge_based_edge_list,
speed_profile);
lua_close(lua_state);
TIMER_STOP(expansion);
BuildRTree(node_based_edge_list);
IteratorbasedCRC32<std::vector<EdgeBasedNode>> crc32;
const unsigned node_based_edge_list_CRC32 =
crc32(node_based_edge_list.begin(), node_based_edge_list.end());
node_based_edge_list.clear();
node_based_edge_list.shrink_to_fit();
SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32;
WriteNodeMapping();
/***
* Contracting the edge-expanded graph
*/
SimpleLogger().Write() << "initializing contractor";
Contractor *contractor = new Contractor(number_of_edge_based_nodes, edge_based_edge_list);
TIMER_START(contraction);
contractor->Run();
TIMER_STOP(contraction);
SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec";
DeallocatingVector<QueryEdge> contracted_edge_list;
contractor->GetEdges(contracted_edge_list);
delete contractor;
/***
* Sorting contracted edges in a way that the static query graph can read some in in-place.
*/
tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end());
const unsigned contracted_edge_count = contracted_edge_list.size();
SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count
<< " edges";
boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary);
hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint));
const unsigned max_used_node_id = 1 + [&contracted_edge_list]
{
unsigned tmp_max = 0;
for (const QueryEdge &edge : contracted_edge_list)
{
BOOST_ASSERT(SPECIAL_NODEID != edge.source);
BOOST_ASSERT(SPECIAL_NODEID != edge.target);
tmp_max = std::max(tmp_max, edge.source);
tmp_max = std::max(tmp_max, edge.target);
}
return tmp_max;
}();
SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes << " nodes";
SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes";
std::vector<StaticGraph<EdgeData>::NodeArrayEntry> node_array;
node_array.resize(number_of_edge_based_nodes + 1);
SimpleLogger().Write() << "Building node array";
StaticGraph<EdgeData>::EdgeIterator edge = 0;
StaticGraph<EdgeData>::EdgeIterator position = 0;
StaticGraph<EdgeData>::EdgeIterator last_edge = edge;
// initializing 'first_edge'-field of nodes:
for (const auto node : osrm::irange(0u, max_used_node_id))
{
last_edge = edge;
while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node))
{
++edge;
}
node_array[node].first_edge = position; //=edge
position += edge - last_edge; // remove
}
for (const auto sentinel_counter : osrm::irange<unsigned>(max_used_node_id, node_array.size()))
{
// sentinel element, guarded against underflow
node_array[sentinel_counter].first_edge = contracted_edge_count;
}
SimpleLogger().Write() << "Serializing node array";
const unsigned node_array_size = node_array.size();
// serialize crc32, aka checksum
hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned));
// serialize number of nodes
hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned));
// serialize number of edges
hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned));
// serialize all nodes
if (node_array_size > 0)
{
hsgr_output_stream.write((char *)&node_array[0],
sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size);
}
// serialize all edges
SimpleLogger().Write() << "Building edge array";
edge = 0;
int number_of_used_edges = 0;
StaticGraph<EdgeData>::EdgeArrayEntry current_edge;
for (const auto edge : osrm::irange<std::size_t>(0, contracted_edge_list.size()))
{
// no eigen loops
BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target);
current_edge.target = contracted_edge_list[edge].target;
current_edge.data = contracted_edge_list[edge].data;
// every target needs to be valid
BOOST_ASSERT(current_edge.target < max_used_node_id);
#ifndef NDEBUG
if (current_edge.data.distance <= 0)
{
SimpleLogger().Write(logWARNING) << "Edge: " << edge
<< ",source: " << contracted_edge_list[edge].source
<< ", target: " << contracted_edge_list[edge].target
<< ", dist: " << current_edge.data.distance;
SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node "
<< contracted_edge_list[edge].source << "/"
<< node_array.size() - 1;
return 1;
}
#endif
hsgr_output_stream.write((char *)¤t_edge,
sizeof(StaticGraph<EdgeData>::EdgeArrayEntry));
++number_of_used_edges;
}
hsgr_output_stream.close();
TIMER_STOP(preparing);
SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds";
SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion))
<< " nodes/sec and "
<< (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec";
SimpleLogger().Write() << "Contraction: "
<< (number_of_edge_based_nodes / TIMER_SEC(contraction))
<< " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction)
<< " edges/sec";
node_array.clear();
SimpleLogger().Write() << "finished preprocessing";
return 0;
}
void NodeBasedGraphFactory::CompressAnnotationData()
{
TIMER_START(compress_annotation);
/** Main idea, that we need to remove duplicated and unreferenced data
* For that:
* 1. We create set, that contains indecies of unique data items. Just create
* comparator, that compare data from annotation_data vector by passed index.
* 2. Create cached id's unordered_map, where key - stored id in set,
* value - index of item in a set from begin. We need that map, because
* std::distance(set.begin(), it) is too slow O(N). So any words in that step we reorder
* annotation data to the order it stored in a set. Apply new id's to edge data.
* 3. Remove unused anootation_data items.
* 4. At final step just need to sort result annotation_data in the same order as set.
* That makes id's stored in edge data valid.
*/
struct IndexComparator
{
IndexComparator(const std::vector<NodeBasedEdgeAnnotation> &annotation_data_)
: annotation_data(annotation_data_)
{
}
bool operator()(AnnotationID a, AnnotationID b) const
{
return annotation_data[a] < annotation_data[b];
}
private:
const std::vector<NodeBasedEdgeAnnotation> &annotation_data;
};
/** 1 */
IndexComparator comparator(annotation_data);
std::set<AnnotationID, IndexComparator> unique_annotations(comparator);
// first we mark entries, by setting their mapping to 0
for (const auto nbg_node_u : util::irange(0u, compressed_output_graph.GetNumberOfNodes()))
{
BOOST_ASSERT(nbg_node_u != SPECIAL_NODEID);
for (EdgeID nbg_edge_id : compressed_output_graph.GetAdjacentEdgeRange(nbg_node_u))
{
auto const &edge = compressed_output_graph.GetEdgeData(nbg_edge_id);
unique_annotations.insert(edge.annotation_data);
}
}
// make additional map, because std::distance of std::set seems is O(N)
// that very slow
/** 2 */
AnnotationID new_id = 0;
std::unordered_map<AnnotationID, AnnotationID> cached_ids;
for (auto id : unique_annotations)
cached_ids[id] = new_id++;
// apply the mapping
for (const auto nbg_node_u : util::irange(0u, compressed_output_graph.GetNumberOfNodes()))
{
BOOST_ASSERT(nbg_node_u != SPECIAL_NODEID);
for (EdgeID nbg_edge_id : compressed_output_graph.GetAdjacentEdgeRange(nbg_node_u))
{
auto &edge = compressed_output_graph.GetEdgeData(nbg_edge_id);
auto const it = unique_annotations.find(edge.annotation_data);
BOOST_ASSERT(it != unique_annotations.end());
auto const it2 = cached_ids.find(*it);
BOOST_ASSERT(it2 != cached_ids.end());
edge.annotation_data = it2->second;
}
}
/** 3 */
// mark unused references for remove
for (AnnotationID id = 0; id < annotation_data.size(); ++id)
{
auto const it = unique_annotations.find(id);
if (it == unique_annotations.end() || *it != id)
annotation_data[id].name_id = INVALID_NAMEID;
}
// remove unreferenced entries, shifting other entries to the front
const auto new_end =
std::remove_if(annotation_data.begin(), annotation_data.end(), [&](auto const &data) {
// both elements are considered equal (to remove the second
// one) if the annotation mapping of the second one is
// invalid
return data.name_id == INVALID_NAMEID;
});
const auto old_size = annotation_data.size();
// remove all remaining elements
annotation_data.erase(new_end, annotation_data.end());
// reorder data in the same order
/** 4 */
std::sort(annotation_data.begin(), annotation_data.end());
TIMER_STOP(compress_annotation);
util::Log() << " graph compression removed " << (old_size - annotation_data.size())
<< " annotations of " << old_size << " in " << TIMER_SEC(compress_annotation)
<< " seconds";
}
/**
* TODO: Refactor this function into smaller functions for better readability.
*
* This function is the entry point for the whole extraction process. The goal of the extraction
* step is to filter and convert the OSM geometry to something more fitting for routing.
* That includes:
* - extracting turn restrictions
* - splitting ways into (directional!) edge segments
* - checking if nodes are barriers or traffic signal
* - discarding all tag information: All relevant type information for nodes/ways
* is extracted at this point.
*
* The result of this process are the following files:
* .names : Names of all streets, stored as long consecutive string with prefix sum based index
* .osrm : Nodes and edges in a intermediate format that easy to digest for osrm-prepare
* .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph
*
*/
int extractor::run(const ExtractorConfig &extractor_config)
{
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(extracting);
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
const auto number_of_threads =
std::min(recommended_num_threads, extractor_config.requested_num_threads);
tbb::task_scheduler_init init(number_of_threads);
SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string();
SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << number_of_threads;
// setup scripting environment
ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str());
ExtractionContainers extraction_containers;
auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers);
const osmium::io::File input_file(extractor_config.input_path.string());
osmium::io::Reader reader(input_file);
const osmium::io::Header header = reader.header();
std::atomic<unsigned> number_of_nodes{0};
std::atomic<unsigned> number_of_ways{0};
std::atomic<unsigned> number_of_relations{0};
std::atomic<unsigned> number_of_others{0};
SimpleLogger().Write() << "Parsing in progress..";
TIMER_START(parsing);
std::string generator = header.get("generator");
if (generator.empty())
{
generator = "unknown tool";
}
SimpleLogger().Write() << "input file generated by " << generator;
// write .timestamp data file
std::string timestamp = header.get("osmosis_replication_timestamp");
if (timestamp.empty())
{
timestamp = "n/a";
}
SimpleLogger().Write() << "timestamp: " << timestamp;
boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name);
timestamp_out.write(timestamp.c_str(), timestamp.length());
timestamp_out.close();
// initialize vectors holding parsed objects
tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes;
tbb::concurrent_vector<std::pair<std::size_t, ExtractionWay>> resulting_ways;
tbb::concurrent_vector<mapbox::util::optional<InputRestrictionContainer>>
resulting_restrictions;
// setup restriction parser
const RestrictionParser restriction_parser(scripting_environment.get_lua_state());
while (const osmium::memory::Buffer buffer = reader.read())
{
// create a vector of iterators into the buffer
std::vector<osmium::memory::Buffer::const_iterator> osm_elements;
for (auto iter = std::begin(buffer); iter != std::end(buffer); ++iter)
{
osm_elements.push_back(iter);
}
// clear resulting vectors
resulting_nodes.clear();
resulting_ways.clear();
resulting_restrictions.clear();
// parse OSM entities in parallel, store in resulting vectors
tbb::parallel_for(
tbb::blocked_range<std::size_t>(0, osm_elements.size()),
[&](const tbb::blocked_range<std::size_t> &range)
{
ExtractionNode result_node;
ExtractionWay result_way;
lua_State *local_state = scripting_environment.get_lua_state();
for (auto x = range.begin(); x != range.end(); ++x)
{
const auto entity = osm_elements[x];
switch (entity->type())
{
case osmium::item_type::node:
result_node.clear();
++number_of_nodes;
luabind::call_function<void>(
local_state, "node_function",
boost::cref(static_cast<const osmium::Node &>(*entity)),
boost::ref(result_node));
resulting_nodes.push_back(std::make_pair(x, result_node));
break;
case osmium::item_type::way:
result_way.clear();
++number_of_ways;
luabind::call_function<void>(
local_state, "way_function",
boost::cref(static_cast<const osmium::Way &>(*entity)),
boost::ref(result_way));
resulting_ways.push_back(std::make_pair(x, result_way));
break;
case osmium::item_type::relation:
++number_of_relations;
resulting_restrictions.push_back(restriction_parser.TryParse(
static_cast<const osmium::Relation &>(*entity)));
break;
default:
++number_of_others;
break;
}
}
});
// put parsed objects thru extractor callbacks
for (const auto &result : resulting_nodes)
{
extractor_callbacks->ProcessNode(
static_cast<const osmium::Node &>(*(osm_elements[result.first])),
result.second);
}
for (const auto &result : resulting_ways)
{
extractor_callbacks->ProcessWay(
static_cast<const osmium::Way &>(*(osm_elements[result.first])), result.second);
}
for (const auto &result : resulting_restrictions)
{
extractor_callbacks->ProcessRestriction(result);
}
}
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
SimpleLogger().Write() << "Raw input contains " << number_of_nodes.load() << " nodes, "
<< number_of_ways.load() << " ways, and "
<< number_of_relations.load() << " relations, and "
<< number_of_others.load() << " unknown entities";
extractor_callbacks.reset();
if (extraction_containers.all_edges_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
extraction_containers.PrepareData(extractor_config.output_file_name,
extractor_config.restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << extractor_config.output_file_name
<< std::endl;
}
catch (std::exception &e)
{
SimpleLogger().Write(logWARNING) << e.what();
return 1;
}
return 0;
}
