void ExtractionContainers::PrepareData(const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM) { try { unsigned usedNodeCounter = 0; unsigned usedEdgeCounter = 0; double time = get_timestamp(); boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024; std::cout << "[extractor] Sorting used nodes ... " << std::flush; stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; stxxl::vector<NodeID>::iterator NewEnd = std::unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ; usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() ); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Sorting all nodes ... " << std::flush; stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Sorting used ways ... " << std::flush; stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; std::cout << "[extractor] Sorting restrctns. by from... " << std::flush; stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; std::cout << "[extractor] Fixing restriction starts ... " << std::flush; STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin(); STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin(); while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) { if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){ ++wayStartAndEndEdgeIT; continue; } if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) { ++restrictionsIT; continue; } assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay); NodeID viaNode = restrictionsIT->restriction.viaNode; if(wayStartAndEndEdgeIT->firstStart == viaNode) { restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget; } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart; } else if(wayStartAndEndEdgeIT->lastStart == viaNode) { restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget; } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart; } ++restrictionsIT; } std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Sorting restrctns. by to ... " << std::flush; stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); unsigned usableRestrictionsCounter(0); std::cout << "[extractor] Fixing restriction ends ... " << std::flush; restrictionsIT = restrictionsVector.begin(); wayStartAndEndEdgeIT = wayStartEndVector.begin(); while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) { if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){ ++wayStartAndEndEdgeIT; continue; } if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) { ++restrictionsIT; continue; } NodeID viaNode = restrictionsIT->restriction.viaNode; if(wayStartAndEndEdgeIT->lastStart == viaNode) { restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget; } else if(wayStartAndEndEdgeIT->lastTarget == viaNode) { restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart; } else if(wayStartAndEndEdgeIT->firstStart == viaNode) { restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget; } else if(wayStartAndEndEdgeIT->firstTarget == viaNode) { restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart; } if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { ++usableRestrictionsCounter; } ++restrictionsIT; } std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; INFO("usable restrictions: " << usableRestrictionsCounter ); //serialize restrictions std::ofstream restrictionsOutstream; restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary); restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned)); for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) { if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) { restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction)); } } restrictionsOutstream.close(); std::ofstream fout; fout.open(outputFileName.c_str(), std::ios::binary); fout.write((char*)&usedNodeCounter, sizeof(unsigned)); time = get_timestamp(); std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; STXXLNodeVector::iterator nodesIT = allNodes.begin(); STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin(); while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) { if(*usedNodeIDsIT < nodesIT->id){ ++usedNodeIDsIT; continue; } if(*usedNodeIDsIT > nodesIT->id) { ++nodesIT; continue; } if(*usedNodeIDsIT == nodesIT->id) { fout.write((char*)&(*nodesIT), sizeof(_Node)); ++usedNodeCounter; ++usedNodeIDsIT; ++nodesIT; } } std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; std::cout << "[extractor] setting number of nodes ... " << std::flush; std::ios::pos_type positionInFile = fout.tellp(); fout.seekp(std::ios::beg); fout.write((char*)&usedNodeCounter, sizeof(unsigned)); fout.seekp(positionInFile); std::cout << "ok" << std::endl; time = get_timestamp(); // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Setting start coords ... " << std::flush; fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); // Traverse list of edges and nodes in parallel and set start coord nodesIT = allNodes.begin(); STXXLEdgeVector::iterator edgeIT = allEdges.begin(); while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { if(edgeIT->start < nodesIT->id){ ++edgeIT; continue; } if(edgeIT->start > nodesIT->id) { nodesIT++; continue; } if(edgeIT->start == nodesIT->id) { edgeIT->startCoord.lat = nodesIT->lat; edgeIT->startCoord.lon = nodesIT->lon; ++edgeIT; } } std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); // Sort Edges by target std::cout << "[extractor] Sorting edges by target ... " << std::flush; stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; time = get_timestamp(); std::cout << "[extractor] Setting target coords ... " << std::flush; // Traverse list of edges and nodes in parallel and set target coord nodesIT = allNodes.begin(); edgeIT = allEdges.begin(); while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) { if(edgeIT->target < nodesIT->id){ ++edgeIT; continue; } if(edgeIT->target > nodesIT->id) { ++nodesIT; continue; } if(edgeIT->target == nodesIT->id) { if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) { edgeIT->targetCoord.lat = nodesIT->lat; edgeIT->targetCoord.lon = nodesIT->lon; double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon); assert(edgeIT->speed != -1); double weight = ( distance * 10. ) / (edgeIT->speed / 3.6); int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) ); int intDist = std::max(1, (int)distance); short zero = 0; short one = 1; fout.write((char*)&edgeIT->start, sizeof(unsigned)); fout.write((char*)&edgeIT->target, sizeof(unsigned)); fout.write((char*)&intDist, sizeof(int)); switch(edgeIT->direction) { case ExtractionWay::oneway: fout.write((char*)&one, sizeof(short)); break; case ExtractionWay::bidirectional: fout.write((char*)&zero, sizeof(short)); break; case ExtractionWay::opposite: fout.write((char*)&one, sizeof(short)); break; default: std::cerr << "[error] edge with no direction: " << edgeIT->direction << std::endl; assert(false); break; } fout.write((char*)&intWeight, sizeof(int)); fout.write((char*)&edgeIT->nameID, sizeof(unsigned)); fout.write((char*)&edgeIT->isRoundabout, sizeof(bool)); fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool)); fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool)); fout.write((char*)&edgeIT->mode, sizeof(unsigned char)); } ++usedEdgeCounter; ++edgeIT; } } std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; std::cout << "[extractor] setting number of edges ... " << std::flush; std::cout << "[extractor] number of edges: " << usedEdgeCounter << std::flush; fout.seekp(positionInFile); fout.write((char*)&usedEdgeCounter, sizeof(unsigned)); fout.close(); std::cout << "ok" << std::endl; time = get_timestamp(); std::cout << "[extractor] writing street name index ... " << std::flush; std::string nameOutFileName = (outputFileName + ".names"); std::ofstream nameOutFile(nameOutFileName.c_str(), std::ios::binary); unsigned sizeOfNameIndex = nameVector.size(); nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned)); BOOST_FOREACH(const std::string & str, nameVector) { unsigned lengthOfRawString = strlen(str.c_str()); nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned)); nameOutFile.write(str.c_str(), lengthOfRawString); } nameOutFile.close(); std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl; // time = get_timestamp(); // cout << "[extractor] writing address list ... " << flush; // // adressFileName.append(".address"); // ofstream addressOutFile(adressFileName.c_str()); // for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) { // addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n"; // } // addressOutFile.close(); // cout << "ok, after " << get_timestamp() - time << "s" << endl; INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges"); } catch ( const std::exception& e ) {
void ExtractionContainers::PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name) { try { unsigned number_of_used_nodes = 0; unsigned number_of_used_edges = 0; std::cout << "[extractor] Sorting used nodes ... " << std::flush; TIMER_START(sorting_used_nodes); stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); TIMER_STOP(sorting_used_nodes); std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl; std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; TIMER_START(erasing_dups); auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end()); used_node_id_list.resize(new_end - used_node_id_list.begin()); TIMER_STOP(erasing_dups); std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl; std::cout << "[extractor] Sorting all nodes ... " << std::flush; TIMER_START(sorting_nodes); stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(), stxxl_memory); TIMER_STOP(sorting_nodes); std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; std::cout << "[extractor] Sorting used ways ... " << std::flush; TIMER_START(sort_ways); stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(), FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory); TIMER_STOP(sort_ways); std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl; std::cout << "[extractor] Sorting " << restrictions_list.size() << " restrictions. by from... " << std::flush; TIMER_START(sort_restrictions); stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom(), stxxl_memory); TIMER_STOP(sort_restrictions); std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl; std::cout << "[extractor] Fixing restriction starts ... " << std::flush; TIMER_START(fix_restriction_starts); auto restrictions_iterator = restrictions_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.cbegin(); while (way_start_and_end_iterator != way_start_end_id_list.cend() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way) { ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == restrictions_iterator->restriction.from.way); const NodeID via_node_id = restrictions_iterator->restriction.via.node; if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { restrictions_iterator->restriction.from.node = way_start_and_end_iterator->first_segment_target_id; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { restrictions_iterator->restriction.from.node = way_start_and_end_iterator->last_segment_source_id; } ++restrictions_iterator; } TIMER_STOP(fix_restriction_starts); std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl; std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; TIMER_START(sort_restrictions_to); stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo(), stxxl_memory); TIMER_STOP(sort_restrictions_to); std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl; std::cout << "[extractor] Fixing restriction ends ... " << std::flush; TIMER_START(fix_restriction_ends); restrictions_iterator = restrictions_list.begin(); way_start_and_end_iterator = way_start_end_id_list.cbegin(); while (way_start_and_end_iterator != way_start_end_id_list.cend() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way) { ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->way_id == restrictions_iterator->restriction.to.way); const NodeID via_node_id = restrictions_iterator->restriction.via.node; if (way_start_and_end_iterator->first_segment_source_id == via_node_id) { restrictions_iterator->restriction.to.node = way_start_and_end_iterator->first_segment_target_id; } else if (way_start_and_end_iterator->last_segment_target_id == via_node_id) { restrictions_iterator->restriction.to.node = way_start_and_end_iterator->last_segment_source_id; } ++restrictions_iterator; } TIMER_STOP(fix_restriction_ends); std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl; // serialize restrictions std::ofstream restrictions_out_stream; unsigned written_restriction_count = 0; restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); const auto count_position = restrictions_out_stream.tellp(); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); for (const auto &restriction_container : restrictions_list) { if (SPECIAL_NODEID != restriction_container.restriction.from.node && SPECIAL_NODEID != restriction_container.restriction.to.node) { restrictions_out_stream.write((char *)&(restriction_container.restriction), sizeof(TurnRestriction)); ++written_restriction_count; } } restrictions_out_stream.seekp(count_position); restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned)); restrictions_out_stream.close(); SimpleLogger().Write() << "usable restrictions: " << written_restriction_count; std::ofstream file_out_stream; file_out_stream.open(output_file_name.c_str(), std::ios::binary); file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; TIMER_START(write_nodes); // identify all used nodes by a merging step of two sorted lists auto node_iterator = all_nodes_list.begin(); auto node_id_iterator = used_node_id_list.begin(); while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) { if (*node_id_iterator < node_iterator->node_id) { ++node_id_iterator; continue; } if (*node_id_iterator > node_iterator->node_id) { ++node_iterator; continue; } BOOST_ASSERT(*node_id_iterator == node_iterator->node_id); file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); ++number_of_used_nodes; ++node_id_iterator; ++node_iterator; } TIMER_STOP(write_nodes); std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl; std::cout << "[extractor] setting number of nodes ... " << std::flush; std::ios::pos_type previous_file_position = file_out_stream.tellp(); file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); file_out_stream.seekp(previous_file_position); std::cout << "ok" << std::endl; // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; TIMER_START(sort_edges_by_start); stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); TIMER_STOP(sort_edges_by_start); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl; std::cout << "[extractor] Setting start coords ... " << std::flush; TIMER_START(set_start_coords); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); // Traverse list of edges and nodes in parallel and set start coord node_iterator = all_nodes_list.begin(); auto edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->start < node_iterator->node_id) { ++edge_iterator; continue; } if (edge_iterator->start > node_iterator->node_id) { node_iterator++; continue; } BOOST_ASSERT(edge_iterator->start == node_iterator->node_id); edge_iterator->source_coordinate.lat = node_iterator->lat; edge_iterator->source_coordinate.lon = node_iterator->lon; ++edge_iterator; } TIMER_STOP(set_start_coords); std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl; // Sort Edges by target std::cout << "[extractor] Sorting edges by target ... " << std::flush; TIMER_START(sort_edges_by_target); stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), stxxl_memory); TIMER_STOP(sort_edges_by_target); std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl; std::cout << "[extractor] Setting target coords ... " << std::flush; TIMER_START(set_target_coords); // Traverse list of edges and nodes in parallel and set target coord node_iterator = all_nodes_list.begin(); edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->target < node_iterator->node_id) { ++edge_iterator; continue; } if (edge_iterator->target > node_iterator->node_id) { ++node_iterator; continue; } BOOST_ASSERT(edge_iterator->target == node_iterator->node_id); if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() && edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min()) { BOOST_ASSERT(edge_iterator->speed != -1); edge_iterator->target_coordinate.lat = node_iterator->lat; edge_iterator->target_coordinate.lon = node_iterator->lon; const double distance = coordinate_calculation::euclidean_distance( edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, node_iterator->lat, node_iterator->lon); const double weight = (distance * 10.) / (edge_iterator->speed / 3.6); int integer_weight = std::max( 1, (int)std::floor( (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5)); const int integer_distance = std::max(1, (int)distance); const short zero = 0; const short one = 1; const bool yes = true; const bool no = false; file_out_stream.write((char *)&edge_iterator->way_id, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned)); file_out_stream.write((char *)&integer_distance, sizeof(int)); switch (edge_iterator->direction) { case ExtractionWay::notSure: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::oneway: file_out_stream.write((char *)&one, sizeof(short)); break; case ExtractionWay::bidirectional: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::opposite: file_out_stream.write((char *)&one, sizeof(short)); break; default: throw osrm::exception("edge has broken direction"); } file_out_stream.write((char *)&integer_weight, sizeof(int)); file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned)); if (edge_iterator->is_roundabout) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } if (edge_iterator->is_in_tiny_cc) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } if (edge_iterator->is_access_restricted) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } // cannot take adress of bit field, so use local const TravelMode travel_mode = edge_iterator->travel_mode; file_out_stream.write((char *)&travel_mode, sizeof(TravelMode)); if (edge_iterator->is_split) { file_out_stream.write((char *)&yes, sizeof(bool)); } else { file_out_stream.write((char *)&no, sizeof(bool)); } ++number_of_used_edges; } ++edge_iterator; } TIMER_STOP(set_target_coords); std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl; std::cout << "[extractor] setting number of edges ... " << std::flush; file_out_stream.seekp(previous_file_position); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); file_out_stream.close(); std::cout << "ok" << std::endl; std::cout << "[extractor] writing street name index ... " << std::flush; TIMER_START(write_name_index); std::string name_file_streamName = (output_file_name + ".names"); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); unsigned total_length = 0; std::vector<unsigned> name_lengths; for (const std::string &temp_string : name_list) { const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u); name_lengths.push_back(string_length); total_length += string_length; } RangeTable<> table(name_lengths); name_file_stream << table; name_file_stream.write((char *)&total_length, sizeof(unsigned)); // write all chars consecutively for (const std::string &temp_string : name_list) { const unsigned string_length = std::min(static_cast<unsigned>(temp_string.length()), 255u); name_file_stream.write(temp_string.c_str(), string_length); } name_file_stream.close(); TIMER_STOP(write_name_index); std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl; SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and " << number_of_used_edges << " edges"; } catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; } }
void ExtractionContainers::PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name) { try { unsigned number_of_used_nodes = 0; unsigned number_of_used_edges = 0; std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Sorting used nodes ... " << std::flush; stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now(); std::chrono::duration<double> elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush; auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end()); used_node_id_list.resize(new_end - used_node_id_list.begin()); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Sorting all nodes ... " << std::flush; stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Sorting used ways ... " << std::flush; stxxl::sort( way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "[extractor] Sorting restrictions. by from... " << std::flush; stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByFrom(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "[extractor] Fixing restriction starts ... " << std::flush; auto restrictions_iterator = restrictions_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.begin(); while (way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay) { ++restrictions_iterator; continue; } BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay); const NodeID via_node_id = restrictions_iterator->restriction.viaNode; if (way_start_and_end_iterator->firstStart == via_node_id) { restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstTarget; } else if (way_start_and_end_iterator->firstTarget == via_node_id) { restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->firstStart; } else if (way_start_and_end_iterator->lastStart == via_node_id) { restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastTarget; } else if (way_start_and_end_iterator->lastTarget == via_node_id) { restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart; } ++restrictions_iterator; } time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush; stxxl::sort(restrictions_list.begin(), restrictions_list.end(), CmpRestrictionContainerByTo(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); unsigned number_of_useable_restrictions = 0; std::cout << "[extractor] Fixing restriction ends ... " << std::flush; restrictions_iterator = restrictions_list.begin(); way_start_and_end_iterator = way_start_end_id_list.begin(); while (way_start_and_end_iterator != way_start_end_id_list.end() && restrictions_iterator != restrictions_list.end()) { if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay) { ++way_start_and_end_iterator; continue; } if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay) { ++restrictions_iterator; continue; } NodeID via_node_id = restrictions_iterator->restriction.viaNode; if (way_start_and_end_iterator->lastStart == via_node_id) { restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget; } else if (way_start_and_end_iterator->lastTarget == via_node_id) { restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart; } else if (way_start_and_end_iterator->firstStart == via_node_id) { restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget; } else if (way_start_and_end_iterator->firstTarget == via_node_id) { restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart; } if (std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.fromNode && std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.toNode) { ++number_of_useable_restrictions; } ++restrictions_iterator; } time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions; // serialize restrictions std::ofstream restrictions_out_stream; restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary); restrictions_out_stream.write((char *)&uuid, sizeof(UUID)); restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned)); // for (restrictions_iterator = restrictions_list.begin(); // restrictions_iterator != restrictions_list.end(); // ++restrictions_iterator) for(const auto & restriction_container : restrictions_list) { if (std::numeric_limits<unsigned>::max() != restriction_container.restriction.fromNode && std::numeric_limits<unsigned>::max() != restriction_container.restriction.toNode) { restrictions_out_stream.write((char *)&(restriction_container.restriction), sizeof(TurnRestriction)); } } restrictions_out_stream.close(); std::ofstream file_out_stream; file_out_stream.open(output_file_name.c_str(), std::ios::binary); file_out_stream.write((char *)&uuid, sizeof(UUID)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; // identify all used nodes by a merging step of two sorted lists auto node_iterator = all_nodes_list.begin(); auto node_id_iterator = used_node_id_list.begin(); while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) { if (*node_id_iterator < node_iterator->id) { ++node_id_iterator; continue; } if (*node_id_iterator > node_iterator->id) { ++node_iterator; continue; } BOOST_ASSERT(*node_id_iterator == node_iterator->id); file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); ++number_of_used_nodes; ++node_id_iterator; ++node_iterator; } time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "[extractor] setting number of nodes ... " << std::flush; std::ios::pos_type previous_file_position = file_out_stream.tellp(); file_out_stream.seekp(std::ios::beg + sizeof(UUID)); file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); file_out_stream.seekp(previous_file_position); std::cout << "ok" << std::endl; time1 = std::chrono::steady_clock::now(); // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Setting start coords ... " << std::flush; file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); // Traverse list of edges and nodes in parallel and set start coord node_iterator = all_nodes_list.begin(); auto edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->start < node_iterator->id) { ++edge_iterator; continue; } if (edge_iterator->start > node_iterator->id) { node_iterator++; continue; } BOOST_ASSERT(edge_iterator->start == node_iterator->id); edge_iterator->source_coordinate.lat = node_iterator->lat; edge_iterator->source_coordinate.lon = node_iterator->lon; ++edge_iterator; } time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); // Sort Edges by target std::cout << "[extractor] Sorting edges by target ... " << std::flush; stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), stxxl_memory); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] Setting target coords ... " << std::flush; // Traverse list of edges and nodes in parallel and set target coord node_iterator = all_nodes_list.begin(); edge_iterator = all_edges_list.begin(); while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->target < node_iterator->id) { ++edge_iterator; continue; } if (edge_iterator->target > node_iterator->id) { ++node_iterator; continue; } BOOST_ASSERT(edge_iterator->target == node_iterator->id); if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() && edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min()) { BOOST_ASSERT(edge_iterator->speed != -1); BOOST_ASSERT(edge_iterator->type >= 0); edge_iterator->target_coordinate.lat = node_iterator->lat; edge_iterator->target_coordinate.lon = node_iterator->lon; const double distance = FixedPointCoordinate::ApproximateEuclideanDistance( edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, node_iterator->lat, node_iterator->lon); const double weight = (distance * 10.) / (edge_iterator->speed / 3.6); int integer_weight = std::max( 1, (int)std::floor( (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5)); int integer_distance = std::max(1, (int)distance); short zero = 0; short one = 1; file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned)); file_out_stream.write((char *)&integer_distance, sizeof(int)); switch (edge_iterator->direction) { case ExtractionWay::notSure: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::oneway: file_out_stream.write((char *)&one, sizeof(short)); break; case ExtractionWay::bidirectional: file_out_stream.write((char *)&zero, sizeof(short)); break; case ExtractionWay::opposite: file_out_stream.write((char *)&one, sizeof(short)); break; default: throw OSRMException("edge has broken direction"); } file_out_stream.write((char *)&integer_weight, sizeof(int)); file_out_stream.write((char *)&edge_iterator->type, sizeof(short)); file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned)); file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool)); file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool)); file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool)); file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool)); file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool)); ++number_of_used_edges; } ++edge_iterator; } time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; std::cout << "[extractor] setting number of edges ... " << std::flush; file_out_stream.seekp(previous_file_position); file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned)); file_out_stream.close(); std::cout << "ok" << std::endl; time1 = std::chrono::steady_clock::now(); std::cout << "[extractor] writing street name index ... " << std::flush; std::string name_file_streamName = (output_file_name + ".names"); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); // write number of names const unsigned number_of_names = name_list.size() + 1; name_file_stream.write((char *)&(number_of_names), sizeof(unsigned)); // compute total number of chars unsigned total_number_of_chars = 0; for (const std::string &temp_string : name_list) { total_number_of_chars += temp_string.length(); } // write total number of chars name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned)); // write prefixe sums unsigned name_lengths_prefix_sum = 0; for (const std::string &temp_string : name_list) { name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned)); name_lengths_prefix_sum += temp_string.length(); } // duplicate on purpose! name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned)); // write all chars consecutively for (const std::string &temp_string : name_list) { const unsigned string_length = temp_string.length(); name_file_stream.write(temp_string.c_str(), string_length); } name_file_stream.close(); time2 = std::chrono::steady_clock::now(); elapsed_seconds = time2 - time1; std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl; SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and " << number_of_used_edges << " edges"; } catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; } }