void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
TIMER_START(sorting_used_nodes);
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
TIMER_STOP(sorting_used_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_used_nodes) << "s" << std::endl;
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
TIMER_START(erasing_dups);
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
TIMER_STOP(erasing_dups);
std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl;
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
TIMER_START(sorting_nodes);
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(),
stxxl_memory);
TIMER_STOP(sorting_nodes);
std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl;
std::cout << "[extractor] Sorting used ways ... " << std::flush;
TIMER_START(sort_ways);
stxxl::sort(way_start_end_id_list.begin(), way_start_end_id_list.end(),
FirstAndLastSegmentOfWayStxxlCompare(), stxxl_memory);
TIMER_STOP(sort_ways);
std::cout << "ok, after " << TIMER_SEC(sort_ways) << "s" << std::endl;
std::cout << "[extractor] Sorting " << restrictions_list.size()
<< " restrictions. by from... " << std::flush;
TIMER_START(sort_restrictions);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByFrom(), stxxl_memory);
TIMER_STOP(sort_restrictions);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
TIMER_START(fix_restriction_starts);
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.from.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.from.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.from.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_starts);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_starts) << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
TIMER_START(sort_restrictions_to);
stxxl::sort(restrictions_list.begin(), restrictions_list.end(),
CmpRestrictionContainerByTo(), stxxl_memory);
TIMER_STOP(sort_restrictions_to);
std::cout << "ok, after " << TIMER_SEC(sort_restrictions_to) << "s" << std::endl;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
TIMER_START(fix_restriction_ends);
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.cbegin();
while (way_start_and_end_iterator != way_start_end_id_list.cend() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->way_id > restrictions_iterator->restriction.to.way)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->way_id ==
restrictions_iterator->restriction.to.way);
const NodeID via_node_id = restrictions_iterator->restriction.via.node;
if (way_start_and_end_iterator->first_segment_source_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->first_segment_target_id;
}
else if (way_start_and_end_iterator->last_segment_target_id == via_node_id)
{
restrictions_iterator->restriction.to.node =
way_start_and_end_iterator->last_segment_source_id;
}
++restrictions_iterator;
}
TIMER_STOP(fix_restriction_ends);
std::cout << "ok, after " << TIMER_SEC(fix_restriction_ends) << "s" << std::endl;
// serialize restrictions
std::ofstream restrictions_out_stream;
unsigned written_restriction_count = 0;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
const auto count_position = restrictions_out_stream.tellp();
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
for (const auto &restriction_container : restrictions_list)
{
if (SPECIAL_NODEID != restriction_container.restriction.from.node &&
SPECIAL_NODEID != restriction_container.restriction.to.node)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
++written_restriction_count;
}
}
restrictions_out_stream.seekp(count_position);
restrictions_out_stream.write((char *)&written_restriction_count, sizeof(unsigned));
restrictions_out_stream.close();
SimpleLogger().Write() << "usable restrictions: " << written_restriction_count;
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char *)&fingerprint, sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
TIMER_START(write_nodes);
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->node_id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->node_id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
TIMER_STOP(write_nodes);
std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
TIMER_START(sort_edges_by_start);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
TIMER_STOP(sort_edges_by_start);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_start) << "s" << std::endl;
std::cout << "[extractor] Setting start coords ... " << std::flush;
TIMER_START(set_start_coords);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->start > node_iterator->node_id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->node_id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
TIMER_STOP(set_start_coords);
std::cout << "ok, after " << TIMER_SEC(set_start_coords) << "s" << std::endl;
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
TIMER_START(sort_edges_by_target);
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(),
stxxl_memory);
TIMER_STOP(sort_edges_by_target);
std::cout << "ok, after " << TIMER_SEC(sort_edges_by_target) << "s" << std::endl;
std::cout << "[extractor] Setting target coords ... " << std::flush;
TIMER_START(set_target_coords);
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->node_id)
{
++edge_iterator;
continue;
}
if (edge_iterator->target > node_iterator->node_id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->node_id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->speed != -1);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = coordinate_calculation::euclidean_distance(
edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon,
node_iterator->lat, node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1, (int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
const int integer_distance = std::max(1, (int)distance);
const short zero = 0;
const short one = 1;
const bool yes = true;
const bool no = false;
file_out_stream.write((char *)&edge_iterator->way_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw osrm::exception("edge has broken direction");
}
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
if (edge_iterator->is_roundabout)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_in_tiny_cc)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
if (edge_iterator->is_access_restricted)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
// cannot take adress of bit field, so use local
const TravelMode travel_mode = edge_iterator->travel_mode;
file_out_stream.write((char *)&travel_mode, sizeof(TravelMode));
if (edge_iterator->is_split)
{
file_out_stream.write((char *)&yes, sizeof(bool));
}
else
{
file_out_stream.write((char *)&no, sizeof(bool));
}
++number_of_used_edges;
}
++edge_iterator;
}
TIMER_STOP(set_target_coords);
std::cout << "ok, after " << TIMER_SEC(set_target_coords) << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
std::cout << "[extractor] writing street name index ... " << std::flush;
TIMER_START(write_name_index);
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
unsigned total_length = 0;
std::vector<unsigned> name_lengths;
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_lengths.push_back(string_length);
total_length += string_length;
}
RangeTable<> table(name_lengths);
name_file_stream << table;
name_file_stream.write((char *)&total_length, sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length =
std::min(static_cast<unsigned>(temp_string.length()), 255u);
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
TIMER_STOP(write_name_index);
std::cout << "ok, after " << TIMER_SEC(write_name_index) << "s" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e)
{
std::cerr << "Caught Execption:" << e.what() << std::endl;
}
}
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(
way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByFrom(),
stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.begin();
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByTo(),
stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin();
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator;
continue;
}
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
}
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
}
if (std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.fromNode &&
std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.toNode)
{
++number_of_useable_restrictions;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
// serialize restrictions
std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
// for (restrictions_iterator = restrictions_list.begin();
// restrictions_iterator != restrictions_list.end();
// ++restrictions_iterator)
for(const auto & restriction_container : restrictions_list)
{
if (std::numeric_limits<unsigned>::max() != restriction_container.restriction.fromNode &&
std::numeric_limits<unsigned>::max() != restriction_container.restriction.toNode)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
}
}
restrictions_out_stream.close();
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char *)&uuid, sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting start coords ... " << std::flush;
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->id)
{
++edge_iterator;
continue;
}
if (edge_iterator->start > node_iterator->id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->id)
{
++edge_iterator;
continue;
}
if (edge_iterator->target > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->speed != -1);
BOOST_ASSERT(edge_iterator->type >= 0);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance);
short zero = 0;
short one = 1;
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw OSRMException("edge has broken direction");
}
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
++number_of_used_edges;
}
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
// write number of names
const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
// compute total number of chars
unsigned total_number_of_chars = 0;
for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length();
}
// write total number of chars
name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
// write prefixe sums
unsigned name_lengths_prefix_sum = 0;
for (const std::string &temp_string : name_list)
{
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
name_lengths_prefix_sum += temp_string.length();
}
// duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
}
void ExtractionContainers::PrepareData(const std::string & outputFileName, const std::string restrictionsFileName, const unsigned amountOfRAM) {
try {
unsigned usedNodeCounter = 0;
unsigned usedEdgeCounter = 0;
double time = get_timestamp();
boost::uint64_t memory_to_use = static_cast<boost::uint64_t>(amountOfRAM) * 1024 * 1024 * 1024;
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(usedNodeIDs.begin(), usedNodeIDs.end(), Cmp(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
stxxl::vector<NodeID>::iterator NewEnd = std::unique ( usedNodeIDs.begin(),usedNodeIDs.end() ) ;
usedNodeIDs.resize ( NewEnd - usedNodeIDs.begin() );
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(allNodes.begin(), allNodes.end(), CmpNodeByID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(wayStartEndVector.begin(), wayStartEndVector.end(), CmpWayByID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
std::cout << "[extractor] Sorting restrctns. by from... " << std::flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByFrom(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
STXXLRestrictionsVector::iterator restrictionsIT = restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
++restrictionsIT;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart;
} else if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
++restrictionsIT;
}
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Sorting restrctns. by to ... " << std::flush;
stxxl::sort(restrictionsVector.begin(), restrictionsVector.end(), CmpRestrictionContainerByTo(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
unsigned usableRestrictionsCounter(0);
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictionsIT = restrictionsVector.begin();
wayStartAndEndEdgeIT = wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != wayStartEndVector.end() && restrictionsIT != restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
++wayStartAndEndEdgeIT;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
++restrictionsIT;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart;
} else if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart;
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
++usableRestrictionsCounter;
}
++restrictionsIT;
}
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
INFO("usable restrictions: " << usableRestrictionsCounter );
//serialize restrictions
std::ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), std::ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = restrictionsVector.begin(); restrictionsIT != restrictionsVector.end(); ++restrictionsIT) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
}
restrictionsOutstream.close();
std::ofstream fout;
fout.open(outputFileName.c_str(), std::ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
STXXLNodeVector::iterator nodesIT = allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = usedNodeIDs.begin();
while(usedNodeIDsIT != usedNodeIDs.end() && nodesIT != allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){
++usedNodeIDsIT;
continue;
}
if(*usedNodeIDsIT > nodesIT->id) {
++nodesIT;
continue;
}
if(*usedNodeIDsIT == nodesIT->id) {
fout.write((char*)&(*nodesIT), sizeof(_Node));
++usedNodeCounter;
++usedNodeIDsIT;
++nodesIT;
}
}
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type positionInFile = fout.tellp();
fout.seekp(std::ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile);
std::cout << "ok" << std::endl;
time = get_timestamp();
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByStartID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Setting start coords ... " << std::flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
nodesIT = allNodes.begin();
STXXLEdgeVector::iterator edgeIT = allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) {
if(edgeIT->start < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->start > nodesIT->id) {
nodesIT++;
continue;
}
if(edgeIT->start == nodesIT->id) {
edgeIT->startCoord.lat = nodesIT->lat;
edgeIT->startCoord.lon = nodesIT->lon;
++edgeIT;
}
}
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(allEdges.begin(), allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
time = get_timestamp();
std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord
nodesIT = allNodes.begin();
edgeIT = allEdges.begin();
while(edgeIT != allEdges.end() && nodesIT != allNodes.end()) {
if(edgeIT->target < nodesIT->id){
++edgeIT;
continue;
}
if(edgeIT->target > nodesIT->id) {
++nodesIT;
continue;
}
if(edgeIT->target == nodesIT->id) {
if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) {
edgeIT->targetCoord.lat = nodesIT->lat;
edgeIT->targetCoord.lon = nodesIT->lon;
double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon);
assert(edgeIT->speed != -1);
double weight = ( distance * 10. ) / (edgeIT->speed / 3.6);
int intWeight = std::max(1, (int)std::floor((edgeIT->isDurationSet ? edgeIT->speed : weight)+.5) );
int intDist = std::max(1, (int)distance);
short zero = 0;
short one = 1;
fout.write((char*)&edgeIT->start, sizeof(unsigned));
fout.write((char*)&edgeIT->target, sizeof(unsigned));
fout.write((char*)&intDist, sizeof(int));
switch(edgeIT->direction) {
case ExtractionWay::oneway:
fout.write((char*)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
fout.write((char*)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
fout.write((char*)&one, sizeof(short));
break;
default:
std::cerr << "[error] edge with no direction: " << edgeIT->direction << std::endl;
assert(false);
break;
}
fout.write((char*)&intWeight, sizeof(int));
fout.write((char*)&edgeIT->nameID, sizeof(unsigned));
fout.write((char*)&edgeIT->isRoundabout, sizeof(bool));
fout.write((char*)&edgeIT->ignoreInGrid, sizeof(bool));
fout.write((char*)&edgeIT->isAccessRestricted, sizeof(bool));
fout.write((char*)&edgeIT->mode, sizeof(unsigned char));
}
++usedEdgeCounter;
++edgeIT;
}
}
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
std::cout << "[extractor] number of edges: " << usedEdgeCounter << std::flush;
fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close();
std::cout << "ok" << std::endl;
time = get_timestamp();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string nameOutFileName = (outputFileName + ".names");
std::ofstream nameOutFile(nameOutFileName.c_str(), std::ios::binary);
unsigned sizeOfNameIndex = nameVector.size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
BOOST_FOREACH(const std::string & str, nameVector) {
unsigned lengthOfRawString = strlen(str.c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(str.c_str(), lengthOfRawString);
}
nameOutFile.close();
std::cout << "ok, after " << get_timestamp() - time << "s" << std::endl;
// time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush;
//
// adressFileName.append(".address");
// ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// }
// addressOutFile.close();
// cout << "ok, after " << get_timestamp() - time << "s" << endl;
INFO("Processed " << usedNodeCounter << " nodes and " << usedEdgeCounter << " edges");
} catch ( const std::exception& e ) {
int main (int argc, char *argv[]) {
if(argc <= 1) {
cerr << "usage: " << endl << argv[0] << " <file.osm/.osm.bz2/.osm.pbf>" << endl;
exit(-1);
}
cout << "[extractor] extracting data from input file " << argv[1] << endl;
bool isPBF = false;
string outputFileName(argv[1]);
string restrictionsFileName(argv[1]);
string::size_type pos = outputFileName.find(".osm.bz2");
if(pos==string::npos) {
pos = outputFileName.find(".osm.pbf");
if(pos!=string::npos) {
isPBF = true;
}
}
if(pos!=string::npos) {
outputFileName.replace(pos, 8, ".osrm");
restrictionsFileName.replace(pos, 8, ".osrm.restrictions");
} else {
pos=outputFileName.find(".osm");
if(pos!=string::npos) {
outputFileName.replace(pos, 5, ".osrm");
restrictionsFileName.replace(pos, 5, ".osrm.restrictions");
} else {
outputFileName.append(".osrm");
restrictionsFileName.append(".osrm.restrictions");
}
}
string adressFileName(outputFileName);
unsigned amountOfRAM = 1;
unsigned installedRAM = GetPhysicalmemory();
if(installedRAM < 2048264) {
cout << "[Warning] Machine has less than 2GB RAM." << endl;
}
if(testDataFile("extractor.ini")) {
ExtractorConfiguration extractorConfig("extractor.ini");
unsigned memoryAmountFromFile = atoi(extractorConfig.GetParameter("Memory").c_str());
if( memoryAmountFromFile != 0 && memoryAmountFromFile <= installedRAM/(1024*1024*1024))
amountOfRAM = memoryAmountFromFile;
cout << "[extractor] using " << amountOfRAM << " GB of RAM for buffers" << endl;
}
STXXLContainers externalMemory;
unsigned usedNodeCounter = 0;
unsigned usedEdgeCounter = 0;
StringMap * stringMap = new StringMap();
Settings settings;
settings.speedProfile.names.insert(settings.speedProfile.names.begin(), names, names+14);
settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+14);
double time = get_timestamp();
stringMap->set_empty_key(GetRandomString());
stringMap->insert(make_pair("", 0));
extractCallBacks = new ExtractorCallbacks(&externalMemory, settings, stringMap);
BaseParser<_Node, _RawRestrictionContainer, _Way> * parser;
if(isPBF) {
parser = new PBFParser(argv[1]);
} else {
parser = new XMLParser(argv[1]);
}
parser->RegisterCallbacks(&nodeFunction, &restrictionFunction, &wayFunction, &adressFunction);
if(parser->Init()) {
parser->Parse();
} else {
cerr << "[error] parser not initialized!" << endl;
exit(-1);
}
delete parser;
try {
// INFO("raw no. of names: " << externalMemory.nameVector.size());
// INFO("raw no. of nodes: " << externalMemory.allNodes.size());
// INFO("no. of used nodes: " << externalMemory.usedNodeIDs.size());
// INFO("raw no. of edges: " << externalMemory.allEdges.size());
// INFO("raw no. of ways: " << externalMemory.wayStartEndVector.size());
// INFO("raw no. of addresses: " << externalMemory.adressVector.size());
// INFO("raw no. of restrictions: " << externalMemory.restrictionsVector.size());
cout << "[extractor] parsing finished after " << get_timestamp() - time << "seconds" << endl;
time = get_timestamp();
uint64_t memory_to_use = static_cast<uint64_t>(amountOfRAM) * 1024 * 1024 * 1024;
cout << "[extractor] Sorting used nodes ... " << flush;
stxxl::sort(externalMemory.usedNodeIDs.begin(), externalMemory.usedNodeIDs.end(), Cmp(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Erasing duplicate nodes ... " << flush;
stxxl::vector<NodeID>::iterator NewEnd = unique ( externalMemory.usedNodeIDs.begin(),externalMemory.usedNodeIDs.end() ) ;
externalMemory.usedNodeIDs.resize ( NewEnd - externalMemory.usedNodeIDs.begin() );
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting all nodes ... " << flush;
stxxl::sort(externalMemory.allNodes.begin(), externalMemory.allNodes.end(), CmpNodeByID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting used ways ... " << flush;
stxxl::sort(externalMemory.wayStartEndVector.begin(), externalMemory.wayStartEndVector.end(), CmpWayStartAndEnd(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Sorting restrctns. by from... " << flush;
stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByFrom(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
cout << "[extractor] Fixing restriction starts ... " << flush;
STXXLRestrictionsVector::iterator restrictionsIT = externalMemory.restrictionsVector.begin();
STXXLWayIDStartEndVector::iterator wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() && restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->fromWay){
wayStartAndEndEdgeIT++;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->fromWay) {
restrictionsIT++;
continue;
}
assert(wayStartAndEndEdgeIT->wayID == restrictionsIT->fromWay);
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->firstStart;
} else if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.fromNode = wayStartAndEndEdgeIT->lastStart;
}
restrictionsIT++;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Sorting restrctns. by to ... " << flush;
stxxl::sort(externalMemory.restrictionsVector.begin(), externalMemory.restrictionsVector.end(), CmpRestrictionByTo(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
unsigned usableRestrictionsCounter(0);
cout << "[extractor] Fixing restriction ends ... " << flush;
restrictionsIT = externalMemory.restrictionsVector.begin();
wayStartAndEndEdgeIT = externalMemory.wayStartEndVector.begin();
while(wayStartAndEndEdgeIT != externalMemory.wayStartEndVector.end() &&
restrictionsIT != externalMemory.restrictionsVector.end()) {
if(wayStartAndEndEdgeIT->wayID < restrictionsIT->toWay){
wayStartAndEndEdgeIT++;
continue;
}
if(wayStartAndEndEdgeIT->wayID > restrictionsIT->toWay) {
restrictionsIT++;
continue;
}
NodeID viaNode = restrictionsIT->restriction.viaNode;
if(wayStartAndEndEdgeIT->lastStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastTarget;
} else if(wayStartAndEndEdgeIT->lastTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->lastStart;
} else if(wayStartAndEndEdgeIT->firstStart == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstTarget;
} else if(wayStartAndEndEdgeIT->firstTarget == viaNode) {
restrictionsIT->restriction.toNode = wayStartAndEndEdgeIT->firstStart;
}
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
usableRestrictionsCounter++;
}
restrictionsIT++;
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
//serialize restrictions
ofstream restrictionsOutstream;
restrictionsOutstream.open(restrictionsFileName.c_str(), ios::binary);
restrictionsOutstream.write((char*)&usableRestrictionsCounter, sizeof(unsigned));
for(restrictionsIT = externalMemory.restrictionsVector.begin(); restrictionsIT != externalMemory.restrictionsVector.end(); restrictionsIT++) {
if(UINT_MAX != restrictionsIT->restriction.fromNode && UINT_MAX != restrictionsIT->restriction.toNode) {
restrictionsOutstream.write((char *)&(restrictionsIT->restriction), sizeof(_Restriction));
}
}
restrictionsOutstream.close();
ofstream fout;
fout.open(outputFileName.c_str(), ios::binary);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
time = get_timestamp();
cout << "[extractor] Confirming used nodes ... " << flush;
STXXLNodeVector::iterator nodesIT = externalMemory.allNodes.begin();
STXXLNodeIDVector::iterator usedNodeIDsIT = externalMemory.usedNodeIDs.begin();
while(usedNodeIDsIT != externalMemory.usedNodeIDs.end() && nodesIT != externalMemory.allNodes.end()) {
if(*usedNodeIDsIT < nodesIT->id){
usedNodeIDsIT++;
continue;
}
if(*usedNodeIDsIT > nodesIT->id) {
nodesIT++;
continue;
}
if(*usedNodeIDsIT == nodesIT->id) {
fout.write((char*)&(nodesIT->id), sizeof(unsigned));
fout.write((char*)&(nodesIT->lon), sizeof(int));
fout.write((char*)&(nodesIT->lat), sizeof(int));
usedNodeCounter++;
usedNodeIDsIT++;
nodesIT++;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] setting number of nodes ... " << flush;
ios::pos_type positionInFile = fout.tellp();
fout.seekp(ios::beg);
fout.write((char*)&usedNodeCounter, sizeof(unsigned));
fout.seekp(positionInFile);
cout << "ok" << endl;
time = get_timestamp();
// Sort edges by start.
cout << "[extractor] Sorting edges by start ... " << flush;
stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByStartID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting start coords ... " << flush;
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
nodesIT = externalMemory.allNodes.begin();
STXXLEdgeVector::iterator edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->start < nodesIT->id){
edgeIT++;
continue;
}
if(edgeIT->start > nodesIT->id) {
nodesIT++;
continue;
}
if(edgeIT->start == nodesIT->id) {
edgeIT->startCoord.lat = nodesIT->lat;
edgeIT->startCoord.lon = nodesIT->lon;
edgeIT++;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
// Sort Edges by target
cout << "[extractor] Sorting edges by target ... " << flush;
stxxl::sort(externalMemory.allEdges.begin(), externalMemory.allEdges.end(), CmpEdgeByTargetID(), memory_to_use);
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] Setting target coords ... " << flush;
// Traverse list of edges and nodes in parallel and set target coord
nodesIT = externalMemory.allNodes.begin();
edgeIT = externalMemory.allEdges.begin();
while(edgeIT != externalMemory.allEdges.end() && nodesIT != externalMemory.allNodes.end()) {
if(edgeIT->target < nodesIT->id){
edgeIT++;
continue;
}
if(edgeIT->target > nodesIT->id) {
nodesIT++;
continue;
}
if(edgeIT->target == nodesIT->id) {
if(edgeIT->startCoord.lat != INT_MIN && edgeIT->startCoord.lon != INT_MIN) {
edgeIT->targetCoord.lat = nodesIT->lat;
edgeIT->targetCoord.lon = nodesIT->lon;
double distance = ApproximateDistance(edgeIT->startCoord.lat, edgeIT->startCoord.lon, nodesIT->lat, nodesIT->lon);
if(edgeIT->speed == -1)
edgeIT->speed = settings.speedProfile.speed[edgeIT->type];
double weight = ( distance * 10. ) / (edgeIT->speed / 3.6);
int intWeight = max(1, (int) weight);
int intDist = max(1, (int)distance);
int ferryIndex = settings.indexInAccessListOf("ferry");
assert(ferryIndex != -1);
short zero = 0;
short one = 1;
fout.write((char*)&edgeIT->start, sizeof(unsigned));
fout.write((char*)&edgeIT->target, sizeof(unsigned));
fout.write((char*)&intDist, sizeof(int));
switch(edgeIT->direction) {
case _Way::notSure:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::oneway:
fout.write((char*)&one, sizeof(short));
break;
case _Way::bidirectional:
fout.write((char*)&zero, sizeof(short));
break;
case _Way::opposite:
fout.write((char*)&one, sizeof(short));
break;
default:
cerr << "[error] edge with no direction: " << edgeIT->direction << endl;
assert(false);
break;
}
fout.write((char*)&intWeight, sizeof(int));
short edgeType = edgeIT->type;
fout.write((char*)&edgeType, sizeof(short));
fout.write((char*)&edgeIT->nameID, sizeof(unsigned));
}
usedEdgeCounter++;
edgeIT++;
}
}
cout << "ok, after " << get_timestamp() - time << "s" << endl;
time = get_timestamp();
cout << "[extractor] setting number of edges ... " << flush;
fout.seekp(positionInFile);
fout.write((char*)&usedEdgeCounter, sizeof(unsigned));
fout.close();
cout << "ok" << endl;
time = get_timestamp();
cout << "[extractor] writing street name index ... " << flush;
vector<unsigned> * nameIndex = new vector<unsigned>(externalMemory.nameVector.size()+1, 0);
outputFileName.append(".names");
ofstream nameOutFile(outputFileName.c_str(), ios::binary);
unsigned sizeOfNameIndex = nameIndex->size();
nameOutFile.write((char *)&(sizeOfNameIndex), sizeof(unsigned));
for(STXXLStringVector::iterator it = externalMemory.nameVector.begin(); it != externalMemory.nameVector.end(); it++) {
unsigned lengthOfRawString = strlen(it->c_str());
nameOutFile.write((char *)&(lengthOfRawString), sizeof(unsigned));
nameOutFile.write(it->c_str(), lengthOfRawString);
}
nameOutFile.close();
delete nameIndex;
cout << "ok, after " << get_timestamp() - time << "s" << endl;
// time = get_timestamp();
// cout << "[extractor] writing address list ... " << flush;
//
// adressFileName.append(".address");
// ofstream addressOutFile(adressFileName.c_str());
// for(STXXLAddressVector::iterator it = adressVector.begin(); it != adressVector.end(); it++) {
// addressOutFile << it->node.id << "|" << it->node.lat << "|" << it->node.lon << "|" << it->city << "|" << it->street << "|" << it->housenumber << "|" << it->state << "|" << it->country << "\n";
// }
// addressOutFile.close();
// cout << "ok, after " << get_timestamp() - time << "s" << endl;
} catch ( const exception& e ) {
cerr << "Caught Execption:" << e.what() << endl;
return false;
}
delete extractCallBacks;
cout << "[extractor] finished." << endl;
return 0;
}
