void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, const std::string &geometry_filename, lua_State *lua_state) { TIMER_START(geometry); CompressGeometry(); TIMER_STOP(geometry); TIMER_START(renumber); RenumberEdges(); TIMER_STOP(renumber); TIMER_START(generate_nodes); GenerateEdgeExpandedNodes(); TIMER_STOP(generate_nodes); TIMER_START(generate_edges); GenerateEdgeExpandedEdges(original_edge_data_filename, lua_state); TIMER_STOP(generate_edges); m_geometry_compressor.SerializeInternalVector(geometry_filename); SimpleLogger().Write() << "Timing statistics for edge-expanded graph:"; SimpleLogger().Write() << "Geometry compression: " << TIMER_SEC(geometry) << "s"; SimpleLogger().Write() << "Renumbering edges: " << TIMER_SEC(renumber) << "s"; SimpleLogger().Write() << "Generating nodes: " << TIMER_SEC(generate_nodes) << "s"; SimpleLogger().Write() << "Generating edges: " << TIMER_SEC(generate_edges) << "s"; }
bool RequestManager::nextPreRenderRequest() { boost::mutex::scoped_lock preLock(preRJMutex); if (preRenderRequests.empty()) return false; shared_ptr<MetaIdentifier> mid = preRenderRequests.front(); preRenderRequests.pop(); currentPrerenderingThreads++; preLock.unlock(); Job job(mid, config, shared_from_this()); // check if tiles are already in progress if (running->start(&job, mid)) { job.process(); running->finished(&job); job.deliver(); } if (!job.isEmpty() && mid->getZoom() < config->get<int>(opt::server::prerender_level)) { std::vector<shared_ptr<MetaIdentifier>> children; mid->getSubIdentifiers(children); for (auto& c : children) enqueue(c); } preLock.lock(); currentPrerenderingThreads--; preLock.unlock(); if (currentPrerenderingThreads == 0 && preRenderRequests.size() == 0) { TIMER_STOP(prerender); log.info("Prerendering finished in %02i:%02i", (int) TIMER_MIN(prerender), ((int) TIMER_SEC(prerender)) % 60); } return true; }
int Contractor::Run() { if (config.core_factor > 1.0 || config.core_factor < 0) { throw util::exception("Core factor must be between 0.0 to 1.0 (inclusive)" + SOURCE_REF); } if (config.use_cached_priority) { util::Log(logWARNING) << "Using cached priorities is deprecated and they will be ignored."; } TIMER_START(preparing); util::Log() << "Reading node weights."; std::vector<EdgeWeight> node_weights; { storage::io::FileReader reader(config.GetPath(".osrm.enw"), storage::io::FileReader::VerifyFingerprint); storage::serialization::read(reader, node_weights); } util::Log() << "Done reading node weights."; util::Log() << "Loading edge-expanded graph representation"; std::vector<extractor::EdgeBasedEdge> edge_based_edge_list; updater::Updater updater(config.updater_config); EdgeID max_edge_id = updater.LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights); // Contracting the edge-expanded graph TIMER_START(contraction); std::vector<std::vector<bool>> node_filters; { extractor::EdgeBasedNodeDataContainer node_data; extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data); extractor::ProfileProperties properties; extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties); node_filters = util::excludeFlagsToNodeFilter(max_edge_id + 1, node_data, properties); } RangebasedCRC32 crc32_calculator; const unsigned checksum = crc32_calculator(edge_based_edge_list); QueryGraph query_graph; std::vector<std::vector<bool>> edge_filters; std::vector<std::vector<bool>> cores; std::tie(query_graph, edge_filters, cores) = contractExcludableGraph(toContractorGraph(max_edge_id + 1, std::move(edge_based_edge_list)), std::move(node_weights), std::move(node_filters), config.core_factor); TIMER_STOP(contraction); util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges."; util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec"; files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters); files::writeCoreMarker(config.GetPath(".osrm.core"), cores); TIMER_STOP(preparing); util::Log() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; util::Log() << "finished preprocessing"; return 0; }
int main(int argc, char *argv[]) { try { LogPolicy::GetInstance().Unmute(); TIMER_START(preparing); TIMER_START(expansion); boost::filesystem::path config_file_path, input_path, restrictions_path, profile_path; unsigned int requested_num_threads; bool use_elevation; // declare a group of options that will be allowed only on command line boost::program_options::options_description generic_options("Options"); generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")( "config,c", boost::program_options::value<boost::filesystem::path>(&config_file_path) ->default_value("contractor.ini"), "Path to a configuration file."); // declare a group of options that will be allowed both on command line and in config file boost::program_options::options_description config_options("Configuration"); config_options.add_options()( "restrictions,r", boost::program_options::value<boost::filesystem::path>(&restrictions_path), "Restrictions file in .osrm.restrictions format")( "profile,p", boost::program_options::value<boost::filesystem::path>(&profile_path) ->default_value("profile.lua"),"Path to LUA routing profile")( "elevation,e", boost::program_options::value<bool>(&use_elevation)->default_value(true), "Process node elevations")( "threads,t", boost::program_options::value<unsigned int>(&requested_num_threads)->default_value(tbb::task_scheduler_init::default_num_threads()), "Number of threads to use"); // hidden options, will be allowed both on command line and in config file, but will not be // shown to the user boost::program_options::options_description hidden_options("Hidden options"); hidden_options.add_options()( "input,i", boost::program_options::value<boost::filesystem::path>(&input_path), "Input file in .osm, .osm.bz2 or .osm.pbf format"); // positional option boost::program_options::positional_options_description positional_options; positional_options.add("input", 1); // combine above options for parsing boost::program_options::options_description cmdline_options; cmdline_options.add(generic_options).add(config_options).add(hidden_options); boost::program_options::options_description config_file_options; config_file_options.add(config_options).add(hidden_options); boost::program_options::options_description visible_options( "Usage: " + boost::filesystem::basename(argv[0]) + " <input.osrm> [options]"); visible_options.add(generic_options).add(config_options); // parse command line options boost::program_options::variables_map option_variables; boost::program_options::store(boost::program_options::command_line_parser(argc, argv) .options(cmdline_options) .positional(positional_options) .run(), option_variables); if (option_variables.count("version")) { SimpleLogger().Write() << g_GIT_DESCRIPTION; return 0; } if (option_variables.count("help")) { SimpleLogger().Write() << "\n" << visible_options; return 0; } boost::program_options::notify(option_variables); if (!option_variables.count("restrictions")) { restrictions_path = std::string(input_path.string() + ".restrictions"); } if (!option_variables.count("input")) { SimpleLogger().Write() << "\n" << visible_options; return 0; } if (!boost::filesystem::is_regular_file(input_path)) { SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; return 1; } if (!boost::filesystem::is_regular_file(profile_path)) { SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!"; return 1; } if (1 > requested_num_threads) { SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; return 1; } const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); SimpleLogger().Write() << "Input file: " << input_path.filename().string(); SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); SimpleLogger().Write() << "Using elevation: " << use_elevation; SimpleLogger().Write() << "Threads: " << requested_num_threads; if (recommended_num_threads != requested_num_threads) { SimpleLogger().Write(logWARNING) << "The recommended number of threads is " << recommended_num_threads << "! This setting may have performance side-effects."; } tbb::task_scheduler_init init(requested_num_threads); LogPolicy::GetInstance().Unmute(); boost::filesystem::ifstream restriction_stream(restrictions_path, std::ios::binary); TurnRestriction restriction; FingerPrint fingerprint_loaded, fingerprint_orig; unsigned number_of_usable_restrictions = 0; restriction_stream.read((char *)&fingerprint_loaded, sizeof(FingerPrint)); if (!fingerprint_loaded.TestPrepare(fingerprint_orig)) { SimpleLogger().Write(logWARNING) << ".restrictions was prepared with different build.\n" "Reprocess to get rid of this warning."; } restriction_stream.read((char *)&number_of_usable_restrictions, sizeof(unsigned)); restriction_list.resize(number_of_usable_restrictions); if (number_of_usable_restrictions > 0) { restriction_stream.read((char *)&(restriction_list[0]), number_of_usable_restrictions * sizeof(TurnRestriction)); } restriction_stream.close(); boost::filesystem::ifstream in; in.open(input_path, std::ios::in | std::ios::binary); const std::string node_filename = input_path.string() + ".nodes"; const std::string edge_out = input_path.string() + ".edges"; const std::string geometry_filename = input_path.string() + ".geometry"; const std::string graphOut = input_path.string() + ".hsgr"; const std::string rtree_nodes_path = input_path.string() + ".ramIndex"; const std::string rtree_leafs_path = input_path.string() + ".fileIndex"; /*** Setup Scripting Environment ***/ // Create a new lua state lua_State *lua_state = luaL_newstate(); // Connect LuaBind to this lua state luabind::open(lua_state); // open utility libraries string library; luaL_openlibs(lua_state); // adjust lua load path luaAddScriptFolderToLoadPath(lua_state, profile_path.string().c_str()); // Now call our function in a lua script if (0 != luaL_dofile(lua_state, profile_path.string().c_str())) { std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; return 1; } EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; if (0 != luaL_dostring(lua_state, "return traffic_signal_penalty\n")) { std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; return 1; } speed_profile.trafficSignalPenalty = 10 * lua_tointeger(lua_state, -1); SimpleLogger().Write(logDEBUG) << "traffic_signal_penalty: " << speed_profile.trafficSignalPenalty; if (0 != luaL_dostring(lua_state, "return u_turn_penalty\n")) { std::cerr << lua_tostring(lua_state, -1) << " occured in scripting block" << std::endl; return 1; } speed_profile.uTurnPenalty = 10 * lua_tointeger(lua_state, -1); speed_profile.has_turn_penalty_function = lua_function_exists(lua_state, "turn_function"); #ifdef WIN32 #pragma message ("Memory consumption on Windows can be higher due to memory alignment") #else static_assert(sizeof(ImportEdge) == 20, "changing ImportEdge type has influence on memory consumption!"); #endif std::vector<ImportEdge> edge_list; NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream(in, edge_list, barrier_node_list, traffic_light_list, &internal_to_external_node_map, restriction_list, use_elevation); in.close(); if (edge_list.empty()) { SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; return 1; } SimpleLogger().Write() << restriction_list.size() << " restrictions, " << barrier_node_list.size() << " bollard nodes, " << traffic_light_list.size() << " traffic lights"; /*** * Building an edge-expanded graph from node-based input and turn restrictions */ SimpleLogger().Write() << "Generating edge-expanded graph representation"; std::shared_ptr<NodeBasedDynamicGraph> node_based_graph = NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); std::unique_ptr<RestrictionMap> restriction_map = std::unique_ptr<RestrictionMap>(new RestrictionMap(node_based_graph, restriction_list)); EdgeBasedGraphFactory *edge_based_graph_factor = new EdgeBasedGraphFactory(node_based_graph, std::move(restriction_map), barrier_node_list, traffic_light_list, internal_to_external_node_map, speed_profile); edge_list.clear(); edge_list.shrink_to_fit(); edge_based_graph_factor->Run(edge_out, geometry_filename, lua_state); restriction_list.clear(); restriction_list.shrink_to_fit(); barrier_node_list.clear(); barrier_node_list.shrink_to_fit(); traffic_light_list.clear(); traffic_light_list.shrink_to_fit(); unsigned number_of_edge_based_nodes = edge_based_graph_factor->GetNumberOfEdgeBasedNodes(); BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits<unsigned>::max()); DeallocatingVector<EdgeBasedEdge> edgeBasedEdgeList; #ifndef WIN32 static_assert(sizeof(EdgeBasedEdge) == 16, "changing ImportEdge type has influence on memory consumption!"); #endif edge_based_graph_factor->GetEdgeBasedEdges(edgeBasedEdgeList); std::vector<EdgeBasedNode> node_based_edge_list; edge_based_graph_factor->GetEdgeBasedNodes(node_based_edge_list); delete edge_based_graph_factor; // TODO actually use scoping: Split this up in subfunctions node_based_graph.reset(); TIMER_STOP(expansion); // Building grid-like nearest-neighbor data structure SimpleLogger().Write() << "building r-tree ..."; StaticRTree<EdgeBasedNode> *rtree = new StaticRTree<EdgeBasedNode>(node_based_edge_list, rtree_nodes_path.c_str(), rtree_leafs_path.c_str(), internal_to_external_node_map); delete rtree; IteratorbasedCRC32<std::vector<EdgeBasedNode>> crc32; unsigned node_based_edge_list_CRC32 = crc32(node_based_edge_list.begin(), node_based_edge_list.end()); node_based_edge_list.clear(); node_based_edge_list.shrink_to_fit(); SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32; /*** * Writing info on original (node-based) nodes */ SimpleLogger().Write() << "writing node map ..."; boost::filesystem::ofstream node_stream(node_filename, std::ios::binary); const unsigned size_of_mapping = internal_to_external_node_map.size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) { node_stream.write((char *)&(internal_to_external_node_map[0]), size_of_mapping * sizeof(NodeInfo)); } node_stream.close(); internal_to_external_node_map.clear(); internal_to_external_node_map.shrink_to_fit(); /*** * Contracting the edge-expanded graph */ SimpleLogger().Write() << "initializing contractor"; Contractor *contractor = new Contractor(number_of_edge_based_nodes, edgeBasedEdgeList); TIMER_START(contraction); contractor->Run(); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; DeallocatingVector<QueryEdge> contracted_edge_list; contractor->GetEdges(contracted_edge_list); delete contractor; /*** * Sorting contracted edges in a way that the static query graph can read some in in-place. */ std::sort(contracted_edge_list.begin(), contracted_edge_list.end()); unsigned max_used_node_id = 0; unsigned contracted_edge_count = contracted_edge_list.size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; boost::filesystem::ofstream hsgr_output_stream(graphOut, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); for (const QueryEdge &edge : contracted_edge_list) { BOOST_ASSERT(UINT_MAX != edge.source); BOOST_ASSERT(UINT_MAX != edge.target); max_used_node_id = std::max(max_used_node_id, edge.source); max_used_node_id = std::max(max_used_node_id, edge.target); } SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes << " nodes"; SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes"; max_used_node_id += 1; std::vector<StaticGraph<EdgeData>::NodeArrayEntry> node_array; node_array.resize(number_of_edge_based_nodes + 1); SimpleLogger().Write() << "Building node array"; StaticGraph<EdgeData>::EdgeIterator edge = 0; StaticGraph<EdgeData>::EdgeIterator position = 0; StaticGraph<EdgeData>::EdgeIterator last_edge = edge; for (StaticGraph<EdgeData>::NodeIterator node = 0; node < max_used_node_id; ++node) { last_edge = edge; while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) { ++edge; } node_array[node].first_edge = position; //=edge position += edge - last_edge; // remove } for (unsigned sentinel_counter = max_used_node_id; sentinel_counter != node_array.size(); ++sentinel_counter) { // sentinel element, guarded against underflow node_array[sentinel_counter].first_edge = contracted_edge_count; } unsigned node_array_size = node_array.size(); // serialize crc32, aka checksum hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned)); // serialize number of nodes hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); // serialize number of edges hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned)); // serialize all nodes if (node_array_size > 0) { hsgr_output_stream.write((char *)&node_array[0], sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size); } // serialize all edges SimpleLogger().Write() << "Building edge array"; edge = 0; int number_of_used_edges = 0; StaticGraph<EdgeData>::EdgeArrayEntry current_edge; for (unsigned edge = 0; edge < contracted_edge_list.size(); ++edge) { // no eigen loops BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); current_edge.target = contracted_edge_list[edge].target; current_edge.data = contracted_edge_list[edge].data; // every target needs to be valid BOOST_ASSERT(current_edge.target < max_used_node_id); #ifndef NDEBUG if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge << ",source: " << contracted_edge_list[edge].source << ", target: " << contracted_edge_list[edge].target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " << contracted_edge_list[edge].source << "/" << node_array.size() - 1; return 1; } #endif hsgr_output_stream.write((char *)¤t_edge, sizeof(StaticGraph<EdgeData>::EdgeArrayEntry)); ++number_of_used_edges; } hsgr_output_stream.close(); TIMER_STOP(preparing); SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) << " nodes/sec and " << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; SimpleLogger().Write() << "Contraction: " << (number_of_edge_based_nodes / TIMER_SEC(contraction)) << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) << " edges/sec"; node_array.clear(); SimpleLogger().Write() << "finished preprocessing"; } catch (boost::program_options::too_many_positional_options_error &) { SimpleLogger().Write(logWARNING) << "Only one file can be specified"; return 1; } catch (boost::program_options::error &e) { SimpleLogger().Write(logWARNING) << e.what(); return 1; } catch (const std::exception &e) { SimpleLogger().Write(logWARNING) << "Exception occured: " << e.what() << std::endl; return 1; } return 0; }
void ExtractionContainers::PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name) { try { unsigned number_of_used_nodes = 0; unsigned number_of_used_edges = 0; std::cout << "[extractor] Sorting used nodes ... " << std::flush; TIMER_START(sorting_used_nodes); stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); TIMER_STOP(sorting_used_nodes); std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl; std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; TIMER_START(erasing_dups); auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end()); used_node_id_list.resize(new_end - used_node_id_list.begin()); TIMER_STOP(erasing_dups); std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl; std::cout << "[extractor] Sorting all nodes ... " << std::flush; TIMER_START(sorting_nodes); stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(), stxxl_memory); TIMER_STOP(sorting_nodes); std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; std::cout << "[extractor] Sorting used ways ... " << std::flush; TIMER_START(sort_ways); stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(), FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory); TIMER_STOP(sort_ways); std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl; std::cout << "[extractor] Sorting " << restrictions_list.size() << " restrictions. by from... " << std::flush; TIMER_START(sort_restrictions); stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom(), stxxl_memory); TIMER_STOP(sort_restrictions); std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl; std::cout << "[extractor] Fixing restriction starts ... " << std::flush; TIMER_START(fix_restriction_starts); auto restrictions_iterator = restrictions_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.cbegin(); while (way_start_and_end_iterator != way_start_end_id_list.cend() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) { ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == restrictions_iterator->restriction.from.way); const NodeID via_node_id = restrictions_iterator->restriction.via.node; if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { restrictions_iterator->restriction.from.node = way_start_and_end_iterator->first_segment_target_id; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { restrictions_iterator->restriction.from.node = way_start_and_end_iterator->last_segment_source_id; } ++restrictions_iterator; } TIMER_STOP(fix_restriction_starts); std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl; std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; TIMER_START(sort_restrictions_to); stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo(), stxxl_memory); TIMER_STOP(sort_restrictions_to); std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl; std::cout << "[extractor] Fixing restriction ends ... " << std::flush; TIMER_START(fix_restriction_ends); restrictions_iterator = restrictions_list.begin(); way_start_and_end_iterator = way_start_end_id_list.cbegin(); while (way_start_and_end_iterator != way_start_end_id_list.cend() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) { ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == restrictions_iterator->restriction.to.way); const NodeID via_node_id = restrictions_iterator->restriction.via.node; if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { restrictions_iterator->restriction.to.node = way_start_and_end_iterator->first_segment_target_id; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { restrictions_iterator->restriction.to.node = way_start_and_end_iterator->last_segment_source_id; } ++restrictions_iterator; } TIMER_STOP(fix_restriction_ends); std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl; // serialize restrictions std::ofstream restrictions_out_stream; unsigned written_restriction_count = 0; restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); const auto count_position = restrictions_out_stream.tellp(); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); for (const auto &restriction_container : restrictions_list) { if (SPECIAL_NODEID != restriction_container.restriction.from.node && SPECIAL_NODEID != restriction_container.restriction.to.node) { restrictions_out_stream.write((char *)&(restriction_container.restriction), sizeof(TurnRestriction)); ++written_restriction_count; } } restrictions_out_stream.seekp(count_position); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); restrictions_out_stream.close(); SimpleLogger().Write() << "usable restrictions: " << written_restriction_count; std::ofstream file_out_stream; file_out_stream.open(output_file_name.c_str(), std::ios::binary); file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; TIMER_START(write_nodes); // identify all used nodes by a merging step of two sorted lists auto node_iterator = all_nodes_list.begin(); auto node_id_iterator = used_node_id_list.begin(); while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) { if (*node_id_iterator < node_iterator->node_id) { ++node_id_iterator; continue; } if (*node_id_iterator > node_iterator->node_id) { ++node_iterator; continue; } BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); ++number_of_used_nodes; ++node_id_iterator; ++node_iterator; } TIMER_STOP(write_nodes); std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl; std::cout << "[extractor] setting number of nodes ... " << std::flush; std::ios::pos_type previous_file_position = file_out_stream.tellp(); file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); file_out_stream.seekp(previous_file_position); std::cout << "ok" << std::endl; // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; TIMER_START(sort_edges_by_start); stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); TIMER_STOP(sort_edges_by_start); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl; std::cout << "[extractor] Setting start coords ... " << std::flush; TIMER_START(set_start_coords); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); // Traverse list of edges and nodes in parallel and set start coord node_iterator = all_nodes_list.begin(); auto edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->start < node_iterator->node_id) { ++edge_iterator; continue; } if (edge_iterator->start > node_iterator->node_id) { node_iterator++; continue; } BOOST_ASSERT(edge_iterator->start == node_iterator->node_id); edge_iterator->source_coordinate.lat = node_iterator->lat; edge_iterator->source_coordinate.lon = node_iterator->lon; ++edge_iterator; } TIMER_STOP(set_start_coords); std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl; // Sort Edges by target std::cout << "[extractor] Sorting edges by target ... " << std::flush; TIMER_START(sort_edges_by_target); stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), stxxl_memory); TIMER_STOP(sort_edges_by_target); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl; std::cout << "[extractor] Setting target coords ... " << std::flush; TIMER_START(set_target_coords); // Traverse list of edges and nodes in parallel and set target coord node_iterator = all_nodes_list.begin(); edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->target < node_iterator->node_id) { ++edge_iterator; continue; } if (edge_iterator->target > node_iterator->node_id) { ++node_iterator; continue; } BOOST_ASSERT(edge_iterator->target == node_iterator->node_id); if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() && edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min()) { BOOST_ASSERT(edge_iterator->speed != -1); edge_iterator->target_coordinate.lat = node_iterator->lat; edge_iterator->target_coordinate.lon = node_iterator->lon; const double distance = coordinate_calculation::euclidean_distance( edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, node_iterator->lat, node_iterator->lon); const double weight = (distance * 10.) / (edge_iterator->speed / 3.6); int integer_weight = std::max( 1, (int)std::floor( (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5)); const int integer_distance = std::max(1, (int)distance); const short zero = 0; const short one = 1; const bool yes = true; const bool no = false; file_out_stream.write((char *)&edge_iterator->way_id, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned)); file_out_stream.write((char *)&integer_distance, sizeof(int)); switch (edge_iterator->direction) { case ExtractionWay::notSure: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::oneway: file_out_stream.write((char *)&one, sizeof(short)); break; case ExtractionWay::bidirectional: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::opposite: file_out_stream.write((char *)&one, sizeof(short)); break; default: throw osrm::exception("edge has broken direction"); } file_out_stream.write((char *)&integer_weight, sizeof(int)); file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned)); if (edge_iterator->is_roundabout) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } if (edge_iterator->is_in_tiny_cc) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } if (edge_iterator->is_access_restricted) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } // cannot take adress of bit field, so use local const TravelMode travel_mode = edge_iterator->travel_mode; file_out_stream.write((char *)&travel_mode, sizeof(TravelMode)); if (edge_iterator->is_split) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } ++number_of_used_edges; } ++edge_iterator; } TIMER_STOP(set_target_coords); std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl; std::cout << "[extractor] setting number of edges ... " << std::flush; file_out_stream.seekp(previous_file_position); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); file_out_stream.close(); std::cout << "ok" << std::endl; std::cout << "[extractor] writing street name index ... " << std::flush; TIMER_START(write_name_index); std::string name_file_streamName = (output_file_name + ".names"); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); unsigned total_length = 0; std::vector<unsigned> name_lengths; for (const std::string &temp_string : name_list) { const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u); name_lengths.push_back(string_length); total_length += string_length; } RangeTable<> table(name_lengths); name_file_stream << table; name_file_stream.write((char *)&total_length, sizeof(unsigned)); // write all chars consecutively for (const std::string &temp_string : name_list) { const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u); name_file_stream.write(temp_string.c_str(), string_length); } name_file_stream.close(); TIMER_STOP(write_name_index); std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl; SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and " << number_of_used_edges << " edges"; } catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; } }
int main(int argc, char *argv[]) { #ifdef __FreeBSD__ SimpleLogger().Write() << "Not supported on FreeBSD"; return 0; #endif #ifdef _WIN32 SimpleLogger().Write() << "Not supported on Windows"; return 0; #else LogPolicy::GetInstance().Unmute(); boost::filesystem::path test_path; try { SimpleLogger().Write() << "starting up engines, " << g_GIT_DESCRIPTION; if (1 == argc) { SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " /path/on/device"; return -1; } test_path = boost::filesystem::path(argv[1]); test_path /= "osrm.tst"; SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string(); // create files for testing if (2 == argc) { // create file to test if (boost::filesystem::exists(test_path)) { throw osrm::exception("Data file already exists"); } int *random_array = new int[number_of_elements]; std::generate(random_array, random_array + number_of_elements, std::rand); #ifdef __APPLE__ FILE *fd = fopen(test_path.string().c_str(), "w"); fcntl(fileno(fd), F_NOCACHE, 1); fcntl(fileno(fd), F_RDAHEAD, 0); TIMER_START(write_1gb); write(fileno(fd), (char *)random_array, number_of_elements * sizeof(unsigned)); TIMER_STOP(write_1gb); fclose(fd); #endif #ifdef __linux__ int file_desc = open(test_path.string().c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, S_IRWXU); if (-1 == file_desc) { throw osrm::exception("Could not open random data file"); } TIMER_START(write_1gb); int ret = write(file_desc, random_array, number_of_elements * sizeof(unsigned)); if (0 > ret) { throw osrm::exception("could not write random data file"); } TIMER_STOP(write_1gb); close(file_desc); #endif delete[] random_array; SimpleLogger().Write(logDEBUG) << "writing raw 1GB took " << TIMER_SEC(write_1gb) << "s"; SimpleLogger().Write() << "raw write performance: " << std::setprecision(5) << std::fixed << 1024 * 1024 / TIMER_SEC(write_1gb) << "MB/sec"; SimpleLogger().Write(logDEBUG) << "finished creation of random data. Flush disk cache now!"; } else { // Run Non-Cached I/O benchmarks if (!boost::filesystem::exists(test_path)) { throw osrm::exception("data file does not exist"); } // volatiles do not get optimized Statistics stats; #ifdef __APPLE__ volatile unsigned single_block[1024]; char *raw_array = new char[number_of_elements * sizeof(unsigned)]; FILE *fd = fopen(test_path.string().c_str(), "r"); fcntl(fileno(fd), F_NOCACHE, 1); fcntl(fileno(fd), F_RDAHEAD, 0); #endif #ifdef __linux__ char *single_block = (char *)memalign(512, 1024 * sizeof(unsigned)); int file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC); if (-1 == file_desc) { SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno); return -1; } char *raw_array = (char *)memalign(512, number_of_elements * sizeof(unsigned)); #endif TIMER_START(read_1gb); #ifdef __APPLE__ read(fileno(fd), raw_array, number_of_elements * sizeof(unsigned)); close(fileno(fd)); fd = fopen(test_path.string().c_str(), "r"); #endif #ifdef __linux__ int ret = read(file_desc, raw_array, number_of_elements * sizeof(unsigned)); SimpleLogger().Write(logDEBUG) << "read " << ret << " bytes, error: " << strerror(errno); close(file_desc); file_desc = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC); SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno); #endif TIMER_STOP(read_1gb); SimpleLogger().Write(logDEBUG) << "reading raw 1GB took " << TIMER_SEC(read_1gb) << "s"; SimpleLogger().Write() << "raw read performance: " << std::setprecision(5) << std::fixed << 1024 * 1024 / TIMER_SEC(read_1gb) << "MB/sec"; std::vector<double> timing_results_raw_random; SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB"; #ifdef __APPLE__ fseek(fd, 0, SEEK_SET); #endif #ifdef __linux__ lseek(file_desc, 0, SEEK_SET); #endif // make 1000 random access, time each I/O seperately unsigned number_of_blocks = (number_of_elements * sizeof(unsigned) - 1) / 4096; std::random_device rd; std::default_random_engine e1(rd()); std::uniform_int_distribution<unsigned> uniform_dist(0, number_of_blocks - 1); for (unsigned i = 0; i < 1000; ++i) { unsigned block_to_read = uniform_dist(e1); off_t current_offset = block_to_read * 4096; TIMER_START(random_access); #ifdef __APPLE__ int ret1 = fseek(fd, current_offset, SEEK_SET); int ret2 = read(fileno(fd), (char *)&single_block[0], 4096); #endif #ifdef __FreeBSD__ int ret1 = 0; int ret2 = 0; #endif #ifdef __linux__ int ret1 = lseek(file_desc, current_offset, SEEK_SET); int ret2 = read(file_desc, (char *)single_block, 4096); #endif TIMER_STOP(random_access); if (((off_t)-1) == ret1) { SimpleLogger().Write(logWARNING) << "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno); throw osrm::exception("seek error"); } if (-1 == ret2) { SimpleLogger().Write(logWARNING) << "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "read error " << strerror(errno); throw osrm::exception("read error"); } timing_results_raw_random.push_back(TIMER_SEC(random_access)); } // Do statistics SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics"; std::ofstream random_csv("random.csv", std::ios::trunc); for (unsigned i = 0; i < timing_results_raw_random.size(); ++i) { random_csv << i << ", " << timing_results_raw_random[i] << std::endl; } random_csv.close(); RunStatistics(timing_results_raw_random, stats); SimpleLogger().Write() << "raw random I/O: " << std::setprecision(5) << std::fixed << "min: " << stats.min << "ms, " << "mean: " << stats.mean << "ms, " << "med: " << stats.med << "ms, " << "max: " << stats.max << "ms, " << "dev: " << stats.dev << "ms"; std::vector<double> timing_results_raw_seq; #ifdef __APPLE__ fseek(fd, 0, SEEK_SET); #endif #ifdef __linux__ lseek(file_desc, 0, SEEK_SET); #endif // read every 100th block for (unsigned i = 0; i < 1000; ++i) { off_t current_offset = i * 4096; TIMER_START(read_every_100); #ifdef __APPLE__ int ret1 = fseek(fd, current_offset, SEEK_SET); int ret2 = read(fileno(fd), (char *)&single_block, 4096); #endif #ifdef __FreeBSD__ int ret1 = 0; int ret2 = 0; #endif #ifdef __linux__ int ret1 = lseek(file_desc, current_offset, SEEK_SET); int ret2 = read(file_desc, (char *)single_block, 4096); #endif TIMER_STOP(read_every_100); if (((off_t)-1) == ret1) { SimpleLogger().Write(logWARNING) << "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno); throw osrm::exception("seek error"); } if (-1 == ret2) { SimpleLogger().Write(logWARNING) << "offset: " << current_offset; SimpleLogger().Write(logWARNING) << "read error " << strerror(errno); throw osrm::exception("read error"); } timing_results_raw_seq.push_back(TIMER_SEC(read_every_100)); } #ifdef __APPLE__ fclose(fd); // free(single_element); free(raw_array); // free(single_block); #endif #ifdef __linux__ close(file_desc); #endif // Do statistics SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics"; // print simple statistics: min, max, median, variance std::ofstream seq_csv("sequential.csv", std::ios::trunc); for (unsigned i = 0; i < timing_results_raw_seq.size(); ++i) { seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl; } seq_csv.close(); RunStatistics(timing_results_raw_seq, stats); SimpleLogger().Write() << "raw sequential I/O: " << std::setprecision(5) << std::fixed << "min: " << stats.min << "ms, " << "mean: " << stats.mean << "ms, " << "med: " << stats.med << "ms, " << "max: " << stats.max << "ms, " << "dev: " << stats.dev << "ms"; if (boost::filesystem::exists(test_path)) { boost::filesystem::remove(test_path); SimpleLogger().Write(logDEBUG) << "removing temporary files"; } } } catch (const std::exception &e) { SimpleLogger().Write(logWARNING) << "caught exception: " << e.what(); SimpleLogger().Write(logWARNING) << "cleaning up, and exiting"; if (boost::filesystem::exists(test_path)) { boost::filesystem::remove(test_path); SimpleLogger().Write(logWARNING) << "removing temporary files"; } return -1; } return 0; #endif }
RecursiveBisection::RecursiveBisection(BisectionGraph &bisection_graph_, const std::size_t maximum_cell_size, const double balance, const double boundary_factor, const std::size_t num_optimizing_cuts, const std::size_t small_component_size) : bisection_graph(bisection_graph_), internal_state(bisection_graph_) { auto components = internal_state.PrePartitionWithSCC(small_component_size); BOOST_ASSERT(!components.empty()); // Parallelize recursive bisection trees. Root cut happens serially (well, this is a lie: // since we handle big components in parallel, too. But we don't know this and // don't have to. TBB's scheduler handles nested parallelism just fine). // // [ | ] // / \ root cut // [ | ] [ | ] // / \ / \ descend, do cuts in parallel // // https://www.threadingbuildingblocks.org/docs/help/index.htm#reference/algorithms/parallel_do_func.html struct TreeNode { GraphView graph; std::uint64_t depth; }; // Build a recursive bisection tree for all big components independently in parallel. // Last GraphView is all small components: skip for bisection. auto first = begin(components); auto last = end(components) - 1; // We construct the trees on the fly: the root node is the entry point. // All tree branches depend on the actual cut and will be generated while descending. std::vector<TreeNode> forest; forest.reserve(last - first); std::transform(first, last, std::back_inserter(forest), [this](auto graph) { return TreeNode{std::move(graph), internal_state.SCCDepth()}; }); using Feeder = tbb::parallel_do_feeder<TreeNode>; TIMER_START(bisection); // Bisect graph into two parts. Get partition point and recurse left and right in parallel. tbb::parallel_do(begin(forest), end(forest), [&](const TreeNode &node, Feeder &feeder) { const auto cut = computeInertialFlowCut(node.graph, num_optimizing_cuts, balance, boundary_factor); const auto center = internal_state.ApplyBisection( node.graph.Begin(), node.graph.End(), node.depth, cut.flags); const auto terminal = [&](const auto &node) { const auto maximum_depth = sizeof(BisectionID) * CHAR_BIT; const auto too_small = node.graph.NumberOfNodes() < maximum_cell_size; const auto too_deep = node.depth >= maximum_depth; return too_small || too_deep; }; GraphView left_graph{bisection_graph, node.graph.Begin(), center}; TreeNode left_node{std::move(left_graph), node.depth + 1}; if (!terminal(left_node)) feeder.add(std::move(left_node)); GraphView right_graph{bisection_graph, center, node.graph.End()}; TreeNode right_node{std::move(right_graph), node.depth + 1}; if (!terminal(right_node)) feeder.add(std::move(right_node)); }); TIMER_STOP(bisection); util::Log() << "Full bisection done in " << TIMER_SEC(bisection) << "s"; }
int Extractor::Run(int argc, char *argv[]) { try { LogPolicy::GetInstance().Unmute(); TIMER_START(extracting); if (!ParseArguments(argc, argv)) return 0; if (1 > requested_num_threads) { SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; return 1; } if (!boost::filesystem::is_regular_file(input_path)) { SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; return 1; } if (!boost::filesystem::is_regular_file(profile_path)) { SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!"; return 1; } const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); SimpleLogger().Write() << "Input file: " << input_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << requested_num_threads; if (recommended_num_threads != requested_num_threads) { SimpleLogger().Write(logWARNING) << "The recommended number of threads is " << recommended_num_threads << "! This setting may have performance side-effects."; } tbb::task_scheduler_init init(requested_num_threads); /*** Setup Scripting Environment ***/ ScriptingEnvironment scripting_environment(profile_path.string().c_str()); GenerateOutputFilesNames(); std::unordered_map<std::string, NodeID> string_map; ExtractionContainers extraction_containers; string_map[""] = 0; auto extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map); BaseParser *parser; if (file_has_pbf_format) { parser = new PBFParser(input_path.string().c_str(), extractor_callbacks, scripting_environment, requested_num_threads); } else { parser = new XMLParser(input_path.string().c_str(), extractor_callbacks, scripting_environment); } if (!parser->ReadHeader()) { throw OSRMException("Parser not initialized!"); } SimpleLogger().Write() << "Parsing in progress.."; TIMER_START(parsing); parser->Parse(); delete parser; delete extractor_callbacks; TIMER_STOP(parsing); SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds"; if (extraction_containers.all_edges_list.empty()) { SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; return 1; } extraction_containers.PrepareData(output_file_name, restriction_file_name); TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " << "./osrm-prepare " << output_file_name << std::endl; } catch (boost::program_options::too_many_positional_options_error &) { SimpleLogger().Write(logWARNING) << "Only one input file can be specified"; return 1; } catch (std::exception &e) { SimpleLogger().Write(logWARNING) << e.what(); return 1; } return 0; }
int Prepare::Process(int argc, char *argv[]) { LogPolicy::GetInstance().Unmute(); TIMER_START(preparing); TIMER_START(expansion); if (!ParseArguments(argc, argv)) { return 0; } if (!boost::filesystem::is_regular_file(input_path)) { SimpleLogger().Write(logWARNING) << "Input file " << input_path.string() << " not found!"; return 1; } if (!boost::filesystem::is_regular_file(profile_path)) { SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string() << " not found!"; return 1; } if (1 > requested_num_threads) { SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger"; return 1; } const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); SimpleLogger().Write() << "Input file: " << input_path.filename().string(); SimpleLogger().Write() << "Restrictions file: " << restrictions_path.filename().string(); SimpleLogger().Write() << "Profile: " << profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << requested_num_threads; if (recommended_num_threads != requested_num_threads) { SimpleLogger().Write(logWARNING) << "The recommended number of threads is " << recommended_num_threads << "! This setting may have performance side-effects."; } tbb::task_scheduler_init init(requested_num_threads); LogPolicy::GetInstance().Unmute(); FingerPrint fingerprint_orig; CheckRestrictionsFile(fingerprint_orig); boost::filesystem::ifstream input_stream(input_path, std::ios::in | std::ios::binary); node_filename = input_path.string() + ".nodes"; edge_out = input_path.string() + ".edges"; geometry_filename = input_path.string() + ".geometry"; graph_out = input_path.string() + ".hsgr"; rtree_nodes_path = input_path.string() + ".ramIndex"; rtree_leafs_path = input_path.string() + ".fileIndex"; /*** Setup Scripting Environment ***/ // Create a new lua state lua_State *lua_state = luaL_newstate(); // Connect LuaBind to this lua state luabind::open(lua_state); EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; if (!SetupScriptingEnvironment(lua_state, speed_profile)) { return 1; } #ifdef WIN32 #pragma message("Memory consumption on Windows can be higher due to different bit packing") #else static_assert(sizeof(ImportEdge) == 20, "changing ImportEdge type has influence on memory consumption!"); #endif NodeID number_of_node_based_nodes = readBinaryOSRMGraphFromStream(input_stream, edge_list, barrier_node_list, traffic_light_list, &internal_to_external_node_map, restriction_list); input_stream.close(); if (edge_list.empty()) { SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; return 1; } SimpleLogger().Write() << restriction_list.size() << " restrictions, " << barrier_node_list.size() << " bollard nodes, " << traffic_light_list.size() << " traffic lights"; std::vector<EdgeBasedNode> node_based_edge_list; unsigned number_of_edge_based_nodes = 0; DeallocatingVector<EdgeBasedEdge> edge_based_edge_list; // init node_based_edge_list, edge_based_edge_list by edgeList number_of_edge_based_nodes = BuildEdgeExpandedGraph(lua_state, number_of_node_based_nodes, node_based_edge_list, edge_based_edge_list, speed_profile); lua_close(lua_state); TIMER_STOP(expansion); BuildRTree(node_based_edge_list); IteratorbasedCRC32<std::vector<EdgeBasedNode>> crc32; const unsigned node_based_edge_list_CRC32 = crc32(node_based_edge_list.begin(), node_based_edge_list.end()); node_based_edge_list.clear(); node_based_edge_list.shrink_to_fit(); SimpleLogger().Write() << "CRC32: " << node_based_edge_list_CRC32; WriteNodeMapping(); /*** * Contracting the edge-expanded graph */ SimpleLogger().Write() << "initializing contractor"; Contractor *contractor = new Contractor(number_of_edge_based_nodes, edge_based_edge_list); TIMER_START(contraction); contractor->Run(); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; DeallocatingVector<QueryEdge> contracted_edge_list; contractor->GetEdges(contracted_edge_list); delete contractor; /*** * Sorting contracted edges in a way that the static query graph can read some in in-place. */ tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); const unsigned contracted_edge_count = contracted_edge_list.size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; boost::filesystem::ofstream hsgr_output_stream(graph_out, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint_orig, sizeof(FingerPrint)); const unsigned max_used_node_id = 1 + [&contracted_edge_list] { unsigned tmp_max = 0; for (const QueryEdge &edge : contracted_edge_list) { BOOST_ASSERT(SPECIAL_NODEID != edge.source); BOOST_ASSERT(SPECIAL_NODEID != edge.target); tmp_max = std::max(tmp_max, edge.source); tmp_max = std::max(tmp_max, edge.target); } return tmp_max; }(); SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes << " nodes"; SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes"; std::vector<StaticGraph<EdgeData>::NodeArrayEntry> node_array; node_array.resize(number_of_edge_based_nodes + 1); SimpleLogger().Write() << "Building node array"; StaticGraph<EdgeData>::EdgeIterator edge = 0; StaticGraph<EdgeData>::EdgeIterator position = 0; StaticGraph<EdgeData>::EdgeIterator last_edge = edge; // initializing 'first_edge'-field of nodes: for (const auto node : osrm::irange(0u, max_used_node_id)) { last_edge = edge; while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) { ++edge; } node_array[node].first_edge = position; //=edge position += edge - last_edge; // remove } for (const auto sentinel_counter : osrm::irange<unsigned>(max_used_node_id, node_array.size())) { // sentinel element, guarded against underflow node_array[sentinel_counter].first_edge = contracted_edge_count; } SimpleLogger().Write() << "Serializing node array"; const unsigned node_array_size = node_array.size(); // serialize crc32, aka checksum hsgr_output_stream.write((char *)&node_based_edge_list_CRC32, sizeof(unsigned)); // serialize number of nodes hsgr_output_stream.write((char *)&node_array_size, sizeof(unsigned)); // serialize number of edges hsgr_output_stream.write((char *)&contracted_edge_count, sizeof(unsigned)); // serialize all nodes if (node_array_size > 0) { hsgr_output_stream.write((char *)&node_array[0], sizeof(StaticGraph<EdgeData>::NodeArrayEntry) * node_array_size); } // serialize all edges SimpleLogger().Write() << "Building edge array"; edge = 0; int number_of_used_edges = 0; StaticGraph<EdgeData>::EdgeArrayEntry current_edge; for (const auto edge : osrm::irange<std::size_t>(0, contracted_edge_list.size())) { // no eigen loops BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); current_edge.target = contracted_edge_list[edge].target; current_edge.data = contracted_edge_list[edge].data; // every target needs to be valid BOOST_ASSERT(current_edge.target < max_used_node_id); #ifndef NDEBUG if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge << ",source: " << contracted_edge_list[edge].source << ", target: " << contracted_edge_list[edge].target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " << contracted_edge_list[edge].source << "/" << node_array.size() - 1; return 1; } #endif hsgr_output_stream.write((char *)¤t_edge, sizeof(StaticGraph<EdgeData>::EdgeArrayEntry)); ++number_of_used_edges; } hsgr_output_stream.close(); TIMER_STOP(preparing); SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) << " nodes/sec and " << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; SimpleLogger().Write() << "Contraction: " << (number_of_edge_based_nodes / TIMER_SEC(contraction)) << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) << " edges/sec"; node_array.clear(); SimpleLogger().Write() << "finished preprocessing"; return 0; }
void NodeBasedGraphFactory::CompressAnnotationData() { TIMER_START(compress_annotation); /** Main idea, that we need to remove duplicated and unreferenced data * For that: * 1. We create set, that contains indecies of unique data items. Just create * comparator, that compare data from annotation_data vector by passed index. * 2. Create cached id's unordered_map, where key - stored id in set, * value - index of item in a set from begin. We need that map, because * std::distance(set.begin(), it) is too slow O(N). So any words in that step we reorder * annotation data to the order it stored in a set. Apply new id's to edge data. * 3. Remove unused anootation_data items. * 4. At final step just need to sort result annotation_data in the same order as set. * That makes id's stored in edge data valid. */ struct IndexComparator { IndexComparator(const std::vector<NodeBasedEdgeAnnotation> &annotation_data_) : annotation_data(annotation_data_) { } bool operator()(AnnotationID a, AnnotationID b) const { return annotation_data[a] < annotation_data[b]; } private: const std::vector<NodeBasedEdgeAnnotation> &annotation_data; }; /** 1 */ IndexComparator comparator(annotation_data); std::set<AnnotationID, IndexComparator> unique_annotations(comparator); // first we mark entries, by setting their mapping to 0 for (const auto nbg_node_u : util::irange(0u, compressed_output_graph.GetNumberOfNodes())) { BOOST_ASSERT(nbg_node_u != SPECIAL_NODEID); for (EdgeID nbg_edge_id : compressed_output_graph.GetAdjacentEdgeRange(nbg_node_u)) { auto const &edge = compressed_output_graph.GetEdgeData(nbg_edge_id); unique_annotations.insert(edge.annotation_data); } } // make additional map, because std::distance of std::set seems is O(N) // that very slow /** 2 */ AnnotationID new_id = 0; std::unordered_map<AnnotationID, AnnotationID> cached_ids; for (auto id : unique_annotations) cached_ids[id] = new_id++; // apply the mapping for (const auto nbg_node_u : util::irange(0u, compressed_output_graph.GetNumberOfNodes())) { BOOST_ASSERT(nbg_node_u != SPECIAL_NODEID); for (EdgeID nbg_edge_id : compressed_output_graph.GetAdjacentEdgeRange(nbg_node_u)) { auto &edge = compressed_output_graph.GetEdgeData(nbg_edge_id); auto const it = unique_annotations.find(edge.annotation_data); BOOST_ASSERT(it != unique_annotations.end()); auto const it2 = cached_ids.find(*it); BOOST_ASSERT(it2 != cached_ids.end()); edge.annotation_data = it2->second; } } /** 3 */ // mark unused references for remove for (AnnotationID id = 0; id < annotation_data.size(); ++id) { auto const it = unique_annotations.find(id); if (it == unique_annotations.end() || *it != id) annotation_data[id].name_id = INVALID_NAMEID; } // remove unreferenced entries, shifting other entries to the front const auto new_end = std::remove_if(annotation_data.begin(), annotation_data.end(), [&](auto const &data) { // both elements are considered equal (to remove the second // one) if the annotation mapping of the second one is // invalid return data.name_id == INVALID_NAMEID; }); const auto old_size = annotation_data.size(); // remove all remaining elements annotation_data.erase(new_end, annotation_data.end()); // reorder data in the same order /** 4 */ std::sort(annotation_data.begin(), annotation_data.end()); TIMER_STOP(compress_annotation); util::Log() << " graph compression removed " << (old_size - annotation_data.size()) << " annotations of " << old_size << " in " << TIMER_SEC(compress_annotation) << " seconds"; }
/** * TODO: Refactor this function into smaller functions for better readability. * * This function is the entry point for the whole extraction process. The goal of the extraction * step is to filter and convert the OSM geometry to something more fitting for routing. * That includes: * - extracting turn restrictions * - splitting ways into (directional!) edge segments * - checking if nodes are barriers or traffic signal * - discarding all tag information: All relevant type information for nodes/ways * is extracted at this point. * * The result of this process are the following files: * .names : Names of all streets, stored as long consecutive string with prefix sum based index * .osrm : Nodes and edges in a intermediate format that easy to digest for osrm-prepare * .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph * */ int extractor::run(const ExtractorConfig &extractor_config) { try { LogPolicy::GetInstance().Unmute(); TIMER_START(extracting); const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads(); const auto number_of_threads = std::min(recommended_num_threads, extractor_config.requested_num_threads); tbb::task_scheduler_init init(number_of_threads); SimpleLogger().Write() << "Input file: " << extractor_config.input_path.filename().string(); SimpleLogger().Write() << "Profile: " << extractor_config.profile_path.filename().string(); SimpleLogger().Write() << "Threads: " << number_of_threads; // setup scripting environment ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str()); ExtractionContainers extraction_containers; auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers); const osmium::io::File input_file(extractor_config.input_path.string()); osmium::io::Reader reader(input_file); const osmium::io::Header header = reader.header(); std::atomic<unsigned> number_of_nodes{0}; std::atomic<unsigned> number_of_ways{0}; std::atomic<unsigned> number_of_relations{0}; std::atomic<unsigned> number_of_others{0}; SimpleLogger().Write() << "Parsing in progress.."; TIMER_START(parsing); std::string generator = header.get("generator"); if (generator.empty()) { generator = "unknown tool"; } SimpleLogger().Write() << "input file generated by " << generator; // write .timestamp data file std::string timestamp = header.get("osmosis_replication_timestamp"); if (timestamp.empty()) { timestamp = "n/a"; } SimpleLogger().Write() << "timestamp: " << timestamp; boost::filesystem::ofstream timestamp_out(extractor_config.timestamp_file_name); timestamp_out.write(timestamp.c_str(), timestamp.length()); timestamp_out.close(); // initialize vectors holding parsed objects tbb::concurrent_vector<std::pair<std::size_t, ExtractionNode>> resulting_nodes; tbb::concurrent_vector<std::pair<std::size_t, ExtractionWay>> resulting_ways; tbb::concurrent_vector<mapbox::util::optional<InputRestrictionContainer>> resulting_restrictions; // setup restriction parser const RestrictionParser restriction_parser(scripting_environment.get_lua_state()); while (const osmium::memory::Buffer buffer = reader.read()) { // create a vector of iterators into the buffer std::vector<osmium::memory::Buffer::const_iterator> osm_elements; for (auto iter = std::begin(buffer); iter != std::end(buffer); ++iter) { osm_elements.push_back(iter); } // clear resulting vectors resulting_nodes.clear(); resulting_ways.clear(); resulting_restrictions.clear(); // parse OSM entities in parallel, store in resulting vectors tbb::parallel_for( tbb::blocked_range<std::size_t>(0, osm_elements.size()), [&](const tbb::blocked_range<std::size_t> &range) { ExtractionNode result_node; ExtractionWay result_way; lua_State *local_state = scripting_environment.get_lua_state(); for (auto x = range.begin(); x != range.end(); ++x) { const auto entity = osm_elements[x]; switch (entity->type()) { case osmium::item_type::node: result_node.clear(); ++number_of_nodes; luabind::call_function<void>( local_state, "node_function", boost::cref(static_cast<const osmium::Node &>(*entity)), boost::ref(result_node)); resulting_nodes.push_back(std::make_pair(x, result_node)); break; case osmium::item_type::way: result_way.clear(); ++number_of_ways; luabind::call_function<void>( local_state, "way_function", boost::cref(static_cast<const osmium::Way &>(*entity)), boost::ref(result_way)); resulting_ways.push_back(std::make_pair(x, result_way)); break; case osmium::item_type::relation: ++number_of_relations; resulting_restrictions.push_back(restriction_parser.TryParse( static_cast<const osmium::Relation &>(*entity))); break; default: ++number_of_others; break; } } }); // put parsed objects thru extractor callbacks for (const auto &result : resulting_nodes) { extractor_callbacks->ProcessNode( static_cast<const osmium::Node &>(*(osm_elements[result.first])), result.second); } for (const auto &result : resulting_ways) { extractor_callbacks->ProcessWay( static_cast<const osmium::Way &>(*(osm_elements[result.first])), result.second); } for (const auto &result : resulting_restrictions) { extractor_callbacks->ProcessRestriction(result); } } TIMER_STOP(parsing); SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds"; SimpleLogger().Write() << "Raw input contains " << number_of_nodes.load() << " nodes, " << number_of_ways.load() << " ways, and " << number_of_relations.load() << " relations, and " << number_of_others.load() << " unknown entities"; extractor_callbacks.reset(); if (extraction_containers.all_edges_list.empty()) { SimpleLogger().Write(logWARNING) << "The input data is empty, exiting."; return 1; } extraction_containers.PrepareData(extractor_config.output_file_name, extractor_config.restriction_file_name); TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " << "./osrm-prepare " << extractor_config.output_file_name << std::endl; } catch (std::exception &e) { SimpleLogger().Write(logWARNING) << e.what(); return 1; } return 0; }