PBFParser::PBFParser(const char *fileName,
ExtractorCallbacks *extractor_callbacks,
ScriptingEnvironment &scripting_environment,
const bool use_elevation,
unsigned num_threads)
: BaseParser(extractor_callbacks, scripting_environment, use_elevation)
{
if (0 == num_threads)
{
num_parser_threads = tbb::task_scheduler_init::default_num_threads();
}
else
{
num_parser_threads = num_threads;
}
GOOGLE_PROTOBUF_VERIFY_VERSION;
// TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
// NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
// Max 2500 items in queue, hardcoded.
thread_data_queue = std::make_shared<ConcurrentQueue<ParserThreadData *>>(2500);
input.open(fileName, std::ios::in | std::ios::binary);
if (!input)
{
throw OSRMException("pbf file not found.");
}
block_count = 0;
group_count = 0;
}
void BaseParser::ReadUseRestrictionsSetting() {
if( 0 != luaL_dostring( lua_state, "return use_turn_restrictions\n") ) {
throw OSRMException("ERROR occured in scripting block");
}
if( lua_isboolean( lua_state, -1) ) {
use_turn_restrictions = lua_toboolean(lua_state, -1);
}
if( use_turn_restrictions ) {
SimpleLogger().Write() << "Using turn restrictions";
} else {
SimpleLogger().Write() << "Ignoring turn restrictions";
}
}
PBFParser::PBFParser(const char * fileName, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser( ec, se ) {
GOOGLE_PROTOBUF_VERIFY_VERSION;
//TODO: What is the bottleneck here? Filling the queue or reading the stuff from disk?
//NOTE: With Lua scripting, it is parsing the stuff. I/O is virtually for free.
threadDataQueue = boost::make_shared<ConcurrentQueue<_ThreadData*> >( 2500 ); /* Max 2500 items in queue, hardcoded. */
input.open(fileName, std::ios::in | std::ios::binary);
if (!input) {
throw OSRMException("pbf file not found.");
}
#ifndef NDEBUG
blockCount = 0;
groupCount = 0;
#endif
}
QueryObjectsStorage::QueryObjectsStorage(
const std::string & hsgrPath,
const std::string & ramIndexPath,
const std::string & fileIndexPath,
const std::string & nodesPath,
const std::string & edgesPath,
const std::string & namesPath,
const std::string & timestampPath
) {
if( hsgrPath.empty() ) {
throw OSRMException("no hsgr file given in ini file");
}
if( ramIndexPath.empty() ) {
throw OSRMException("no ram index file given in ini file");
}
if( fileIndexPath.empty() ) {
throw OSRMException("no mem index file given in ini file");
}
if( nodesPath.empty() ) {
throw OSRMException("no nodes file given in ini file");
}
if( edgesPath.empty() ) {
throw OSRMException("no edges file given in ini file");
}
if( namesPath.empty() ) {
throw OSRMException("no names file given in ini file");
}
SimpleLogger().Write() << "loading graph data";
//Deserialize road network graph
std::vector< QueryGraph::_StrNode> nodeList;
std::vector< QueryGraph::_StrEdge> edgeList;
const int n = readHSGRFromStream(
hsgrPath,
nodeList,
edgeList,
&checkSum
);
SimpleLogger().Write() << "Data checksum is " << checkSum;
graph = new QueryGraph(nodeList, edgeList);
assert(0 == nodeList.size());
assert(0 == edgeList.size());
if(timestampPath.length()) {
SimpleLogger().Write() << "Loading Timestamp";
std::ifstream timestampInStream(timestampPath.c_str());
if(!timestampInStream) {
SimpleLogger().Write(logWARNING) << timestampPath << " not found";
}
getline(timestampInStream, timestamp);
timestampInStream.close();
}
if(!timestamp.length()) {
timestamp = "n/a";
}
if(25 < timestamp.length()) {
timestamp.resize(25);
}
SimpleLogger().Write() << "Loading auxiliary information";
//Init nearest neighbor data structure
nodeHelpDesk = new NodeInformationHelpDesk(
ramIndexPath,
fileIndexPath,
nodesPath,
edgesPath,
n,
checkSum
);
//deserialize street name list
SimpleLogger().Write() << "Loading names index";
boost::filesystem::path names_file(namesPath);
if ( !boost::filesystem::exists( names_file ) ) {
throw OSRMException("names file does not exist");
}
if ( 0 == boost::filesystem::file_size( names_file ) ) {
throw OSRMException("names file is empty");
}
boost::filesystem::ifstream name_stream(names_file, std::ios::binary);
unsigned size = 0;
name_stream.read((char *)&size, sizeof(unsigned));
BOOST_ASSERT_MSG(0 != size, "name file empty");
char buf[1024];
for( unsigned i = 0; i < size; ++i ) {
unsigned size_of_string = 0;
name_stream.read((char *)&size_of_string, sizeof(unsigned));
buf[size_of_string] = '\0'; // instead of memset
name_stream.read(buf, size_of_string);
names.push_back(buf);
}
std::vector<std::string>(names).swap(names);
BOOST_ASSERT_MSG(0 != names.size(), "could not load any names");
name_stream.close();
SimpleLogger().Write() << "All query data structures loaded";
}
void ExtractionContainers::PrepareData(const std::string &output_file_name,
const std::string &restrictions_file_name)
{
try
{
unsigned number_of_used_nodes = 0;
unsigned number_of_used_edges = 0;
std::chrono::time_point<std::chrono::steady_clock> time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used nodes ... " << std::flush;
stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory);
std::chrono::time_point<std::chrono::steady_clock> time2 = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Erasing duplicate nodes ... " << std::flush;
auto new_end = std::unique(used_node_id_list.begin(), used_node_id_list.end());
used_node_id_list.resize(new_end - used_node_id_list.begin());
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting all nodes ... " << std::flush;
stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), CmpNodeByID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting used ways ... " << std::flush;
stxxl::sort(
way_start_end_id_list.begin(), way_start_end_id_list.end(), CmpWayByID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Sorting restrictions. by from... " << std::flush;
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByFrom(),
stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] Fixing restriction starts ... " << std::flush;
auto restrictions_iterator = restrictions_list.begin();
auto way_start_and_end_iterator = way_start_end_id_list.begin();
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->fromWay)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->wayID > restrictions_iterator->fromWay)
{
++restrictions_iterator;
continue;
}
BOOST_ASSERT(way_start_and_end_iterator->wayID == restrictions_iterator->fromWay);
const NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->firstStart;
}
else if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.fromNode =
way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.fromNode = way_start_and_end_iterator->lastStart;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Sorting restrictions. by to ... " << std::flush;
stxxl::sort(restrictions_list.begin(),
restrictions_list.end(),
CmpRestrictionContainerByTo(),
stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
unsigned number_of_useable_restrictions = 0;
std::cout << "[extractor] Fixing restriction ends ... " << std::flush;
restrictions_iterator = restrictions_list.begin();
way_start_and_end_iterator = way_start_end_id_list.begin();
while (way_start_and_end_iterator != way_start_end_id_list.end() &&
restrictions_iterator != restrictions_list.end())
{
if (way_start_and_end_iterator->wayID < restrictions_iterator->toWay)
{
++way_start_and_end_iterator;
continue;
}
if (way_start_and_end_iterator->wayID > restrictions_iterator->toWay)
{
++restrictions_iterator;
continue;
}
NodeID via_node_id = restrictions_iterator->restriction.viaNode;
if (way_start_and_end_iterator->lastStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastTarget;
}
else if (way_start_and_end_iterator->lastTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->lastStart;
}
else if (way_start_and_end_iterator->firstStart == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstTarget;
}
else if (way_start_and_end_iterator->firstTarget == via_node_id)
{
restrictions_iterator->restriction.toNode = way_start_and_end_iterator->firstStart;
}
if (std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.fromNode &&
std::numeric_limits<unsigned>::max() != restrictions_iterator->restriction.toNode)
{
++number_of_useable_restrictions;
}
++restrictions_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "usable restrictions: " << number_of_useable_restrictions;
// serialize restrictions
std::ofstream restrictions_out_stream;
restrictions_out_stream.open(restrictions_file_name.c_str(), std::ios::binary);
restrictions_out_stream.write((char *)&uuid, sizeof(UUID));
restrictions_out_stream.write((char *)&number_of_useable_restrictions, sizeof(unsigned));
// for (restrictions_iterator = restrictions_list.begin();
// restrictions_iterator != restrictions_list.end();
// ++restrictions_iterator)
for(const auto & restriction_container : restrictions_list)
{
if (std::numeric_limits<unsigned>::max() != restriction_container.restriction.fromNode &&
std::numeric_limits<unsigned>::max() != restriction_container.restriction.toNode)
{
restrictions_out_stream.write((char *)&(restriction_container.restriction),
sizeof(TurnRestriction));
}
}
restrictions_out_stream.close();
std::ofstream file_out_stream;
file_out_stream.open(output_file_name.c_str(), std::ios::binary);
file_out_stream.write((char *)&uuid, sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush;
// identify all used nodes by a merging step of two sorted lists
auto node_iterator = all_nodes_list.begin();
auto node_id_iterator = used_node_id_list.begin();
while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end())
{
if (*node_id_iterator < node_iterator->id)
{
++node_id_iterator;
continue;
}
if (*node_id_iterator > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(*node_id_iterator == node_iterator->id);
file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode));
++number_of_used_nodes;
++node_id_iterator;
++node_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of nodes ... " << std::flush;
std::ios::pos_type previous_file_position = file_out_stream.tellp();
file_out_stream.seekp(std::ios::beg + sizeof(UUID));
file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned));
file_out_stream.seekp(previous_file_position);
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort edges by start.
std::cout << "[extractor] Sorting edges by start ... " << std::flush;
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByStartID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting start coords ... " << std::flush;
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
// Traverse list of edges and nodes in parallel and set start coord
node_iterator = all_nodes_list.begin();
auto edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->start < node_iterator->id)
{
++edge_iterator;
continue;
}
if (edge_iterator->start > node_iterator->id)
{
node_iterator++;
continue;
}
BOOST_ASSERT(edge_iterator->start == node_iterator->id);
edge_iterator->source_coordinate.lat = node_iterator->lat;
edge_iterator->source_coordinate.lon = node_iterator->lon;
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
// Sort Edges by target
std::cout << "[extractor] Sorting edges by target ... " << std::flush;
stxxl::sort(all_edges_list.begin(), all_edges_list.end(), CmpEdgeByTargetID(), stxxl_memory);
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] Setting target coords ... " << std::flush;
// Traverse list of edges and nodes in parallel and set target coord
node_iterator = all_nodes_list.begin();
edge_iterator = all_edges_list.begin();
while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end())
{
if (edge_iterator->target < node_iterator->id)
{
++edge_iterator;
continue;
}
if (edge_iterator->target > node_iterator->id)
{
++node_iterator;
continue;
}
BOOST_ASSERT(edge_iterator->target == node_iterator->id);
if (edge_iterator->source_coordinate.lat != std::numeric_limits<int>::min() &&
edge_iterator->source_coordinate.lon != std::numeric_limits<int>::min())
{
BOOST_ASSERT(edge_iterator->speed != -1);
BOOST_ASSERT(edge_iterator->type >= 0);
edge_iterator->target_coordinate.lat = node_iterator->lat;
edge_iterator->target_coordinate.lon = node_iterator->lon;
const double distance = FixedPointCoordinate::ApproximateEuclideanDistance(
edge_iterator->source_coordinate.lat,
edge_iterator->source_coordinate.lon,
node_iterator->lat,
node_iterator->lon);
const double weight = (distance * 10.) / (edge_iterator->speed / 3.6);
int integer_weight = std::max(
1,
(int)std::floor(
(edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5));
int integer_distance = std::max(1, (int)distance);
short zero = 0;
short one = 1;
file_out_stream.write((char *)&edge_iterator->start, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->target, sizeof(unsigned));
file_out_stream.write((char *)&integer_distance, sizeof(int));
switch (edge_iterator->direction)
{
case ExtractionWay::notSure:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::oneway:
file_out_stream.write((char *)&one, sizeof(short));
break;
case ExtractionWay::bidirectional:
file_out_stream.write((char *)&zero, sizeof(short));
break;
case ExtractionWay::opposite:
file_out_stream.write((char *)&one, sizeof(short));
break;
default:
throw OSRMException("edge has broken direction");
}
file_out_stream.write((char *)&integer_weight, sizeof(int));
file_out_stream.write((char *)&edge_iterator->type, sizeof(short));
file_out_stream.write((char *)&edge_iterator->name_id, sizeof(unsigned));
file_out_stream.write((char *)&edge_iterator->is_roundabout, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_in_tiny_cc, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_access_restricted, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_contra_flow, sizeof(bool));
file_out_stream.write((char *)&edge_iterator->is_split, sizeof(bool));
++number_of_used_edges;
}
++edge_iterator;
}
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
std::cout << "[extractor] setting number of edges ... " << std::flush;
file_out_stream.seekp(previous_file_position);
file_out_stream.write((char *)&number_of_used_edges, sizeof(unsigned));
file_out_stream.close();
std::cout << "ok" << std::endl;
time1 = std::chrono::steady_clock::now();
std::cout << "[extractor] writing street name index ... " << std::flush;
std::string name_file_streamName = (output_file_name + ".names");
boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary);
// write number of names
const unsigned number_of_names = name_list.size() + 1;
name_file_stream.write((char *)&(number_of_names), sizeof(unsigned));
// compute total number of chars
unsigned total_number_of_chars = 0;
for (const std::string &temp_string : name_list)
{
total_number_of_chars += temp_string.length();
}
// write total number of chars
name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned));
// write prefixe sums
unsigned name_lengths_prefix_sum = 0;
for (const std::string &temp_string : name_list)
{
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
name_lengths_prefix_sum += temp_string.length();
}
// duplicate on purpose!
name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned));
// write all chars consecutively
for (const std::string &temp_string : name_list)
{
const unsigned string_length = temp_string.length();
name_file_stream.write(temp_string.c_str(), string_length);
}
name_file_stream.close();
time2 = std::chrono::steady_clock::now();
elapsed_seconds = time2 - time1;
std::cout << "ok, after " << elapsed_seconds.count() << "s" << std::endl;
SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes and "
<< number_of_used_edges << " edges";
}
catch (const std::exception &e) { std::cerr << "Caught Execption:" << e.what() << std::endl; }
}
inline void PBFParser::parseNode(ParserThreadData *)
{
throw OSRMException("Parsing of simple nodes not supported. PBF should use dense nodes");
}
int main(int argc, char *argv[])
{
LogPolicy::GetInstance().Unmute();
SimpleLogger().Write() << "starting up engines, " << g_GIT_DESCRIPTION << ", "
<< "compiled at " << __DATE__ << ", " __TIME__;
#ifdef __FreeBSD__
SimpleLogger().Write() << "Not supported on FreeBSD";
return 0;
#endif
#ifdef WIN32
SimpleLogger().Write() << "Not supported on Windows";
return 0;
#else
if (1 == argc)
{
SimpleLogger().Write(logWARNING) << "usage: " << argv[0] << " /path/on/device";
return -1;
}
boost::filesystem::path test_path = boost::filesystem::path(argv[1]);
test_path /= "osrm.tst";
SimpleLogger().Write(logDEBUG) << "temporary file: " << test_path.string();
try
{
// create files for testing
if (2 == argc)
{
// create file to test
if (boost::filesystem::exists(test_path))
{
throw OSRMException("Data file already exists");
}
int *random_array = new int[number_of_elements];
std::generate(random_array, random_array + number_of_elements, std::rand);
#ifdef __APPLE__
FILE *fd = fopen(test_path.string().c_str(), "w");
fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0);
TIMER_START(write_1gb);
write(fileno(fd), (char *)random_array, number_of_elements * sizeof(unsigned));
TIMER_STOP(write_1gb);
fclose(fd);
#endif
#ifdef __linux__
int f =
open(test_path.string().c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, S_IRWXU);
if (-1 == f)
{
throw OSRMException("Could not open random data file");
}
TIMER_START(write_1gb);
int ret = write(f, random_array, number_of_elements * sizeof(unsigned));
if (0 > ret)
{
throw OSRMException("could not write random data file");
}
TIMER_STOP(write_1gb);
close(f);
#endif
delete[] random_array;
SimpleLogger().Write(logDEBUG) << "writing raw 1GB took " << TIMER_SEC(write_1gb)
<< "s";
SimpleLogger().Write() << "raw write performance: " << std::setprecision(5)
<< std::fixed << 1024 * 1024 / TIMER_SEC(write_1gb)
<< "MB/sec";
SimpleLogger().Write(logDEBUG)
<< "finished creation of random data. Flush disk cache now!";
}
else
{
// Run Non-Cached I/O benchmarks
if (!boost::filesystem::exists(test_path))
{
throw OSRMException("data file does not exist");
}
// volatiles do not get optimized
Statistics stats;
#ifdef __APPLE__
volatile unsigned single_block[1024];
char *raw_array = new char[number_of_elements * sizeof(unsigned)];
FILE *fd = fopen(test_path.string().c_str(), "r");
fcntl(fileno(fd), F_NOCACHE, 1);
fcntl(fileno(fd), F_RDAHEAD, 0);
#endif
#ifdef __linux__
char *single_block = (char *)memalign(512, 1024 * sizeof(unsigned));
int f = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
if (-1 == f)
{
SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
return -1;
}
char *raw_array = (char *)memalign(512, number_of_elements * sizeof(unsigned));
#endif
TIMER_START(read_1gb);
#ifdef __APPLE__
read(fileno(fd), raw_array, number_of_elements * sizeof(unsigned));
close(fileno(fd));
fd = fopen(test_path.string().c_str(), "r");
#endif
#ifdef __linux__
int ret = read(f, raw_array, number_of_elements * sizeof(unsigned));
SimpleLogger().Write(logDEBUG) << "read " << ret
<< " bytes, error: " << strerror(errno);
close(f);
f = open(test_path.string().c_str(), O_RDONLY | O_DIRECT | O_SYNC);
SimpleLogger().Write(logDEBUG) << "opened, error: " << strerror(errno);
#endif
TIMER_STOP(read_1gb);
SimpleLogger().Write(logDEBUG) << "reading raw 1GB took " << TIMER_SEC(read_1gb)
<< "s";
SimpleLogger().Write() << "raw read performance: " << std::setprecision(5) << std::fixed
<< 1024 * 1024 / TIMER_SEC(read_1gb) << "MB/sec";
std::vector<double> timing_results_raw_random;
SimpleLogger().Write(logDEBUG) << "running 1000 random I/Os of 4KB";
#ifdef __APPLE__
fseek(fd, 0, SEEK_SET);
#endif
#ifdef __linux__
lseek(f, 0, SEEK_SET);
#endif
// make 1000 random access, time each I/O seperately
unsigned number_of_blocks = (number_of_elements * sizeof(unsigned) - 1) / 4096;
for (unsigned i = 0; i < 1000; ++i)
{
unsigned block_to_read = std::rand() % number_of_blocks;
off_t current_offset = block_to_read * 4096;
TIMER_START(random_access);
#ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char *)&single_block[0], 4096);
#endif
#ifdef __FreeBSD__
int ret1 = 0;
int ret2 = 0;
#endif
#ifdef __linux__
int ret1 = lseek(f, current_offset, SEEK_SET);
int ret2 = read(f, (char *)single_block, 4096);
#endif
TIMER_STOP(random_access);
if (((off_t) - 1) == ret1)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
throw OSRMException("seek error");
}
if (-1 == ret2)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
throw OSRMException("read error");
}
timing_results_raw_random.push_back(TIMER_SEC(random_access));
}
// Do statistics
SimpleLogger().Write(logDEBUG) << "running raw random I/O statistics";
std::ofstream random_csv("random.csv", std::ios::trunc);
for (unsigned i = 0; i < timing_results_raw_random.size(); ++i)
{
random_csv << i << ", " << timing_results_raw_random[i] << std::endl;
}
random_csv.close();
RunStatistics(timing_results_raw_random, stats);
SimpleLogger().Write() << "raw random I/O: " << std::setprecision(5) << std::fixed
<< "min: " << stats.min << "ms, "
<< "mean: " << stats.mean << "ms, "
<< "med: " << stats.med << "ms, "
<< "max: " << stats.max << "ms, "
<< "dev: " << stats.dev << "ms";
std::vector<double> timing_results_raw_seq;
#ifdef __APPLE__
fseek(fd, 0, SEEK_SET);
#endif
#ifdef __linux__
lseek(f, 0, SEEK_SET);
#endif
// read every 100th block
for (unsigned i = 0; i < 1000; ++i)
{
off_t current_offset = i * 4096;
TIMER_START(read_every_100);
#ifdef __APPLE__
int ret1 = fseek(fd, current_offset, SEEK_SET);
int ret2 = read(fileno(fd), (char *)&single_block, 4096);
#endif
#ifdef __FreeBSD__
int ret1 = 0;
int ret2 = 0;
#endif
#ifdef __linux__
int ret1 = lseek(f, current_offset, SEEK_SET);
int ret2 = read(f, (char *)single_block, 4096);
#endif
TIMER_STOP(read_every_100);
if (((off_t) - 1) == ret1)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "seek error " << strerror(errno);
throw OSRMException("seek error");
}
if (-1 == ret2)
{
SimpleLogger().Write(logWARNING) << "offset: " << current_offset;
SimpleLogger().Write(logWARNING) << "read error " << strerror(errno);
throw OSRMException("read error");
}
timing_results_raw_seq.push_back(TIMER_SEC(read_every_100));
}
#ifdef __APPLE__
fclose(fd);
// free(single_element);
free(raw_array);
// free(single_block);
#endif
#ifdef __linux__
close(f);
#endif
// Do statistics
SimpleLogger().Write(logDEBUG) << "running sequential I/O statistics";
// print simple statistics: min, max, median, variance
std::ofstream seq_csv("sequential.csv", std::ios::trunc);
for (unsigned i = 0; i < timing_results_raw_seq.size(); ++i)
{
seq_csv << i << ", " << timing_results_raw_seq[i] << std::endl;
}
seq_csv.close();
RunStatistics(timing_results_raw_seq, stats);
SimpleLogger().Write() << "raw sequential I/O: " << std::setprecision(5) << std::fixed
<< "min: " << stats.min << "ms, "
<< "mean: " << stats.mean << "ms, "
<< "med: " << stats.med << "ms, "
<< "max: " << stats.max << "ms, "
<< "dev: " << stats.dev << "ms";
if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path);
SimpleLogger().Write(logDEBUG) << "removing temporary files";
}
}
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "caught exception: " << e.what();
SimpleLogger().Write(logWARNING) << "cleaning up, and exiting";
if (boost::filesystem::exists(test_path))
{
boost::filesystem::remove(test_path);
SimpleLogger().Write(logWARNING) << "removing temporary files";
}
return -1;
}
return 0;
#endif
}
int main (int argc, char *argv[]) {
try {
LogPolicy::GetInstance().Unmute();
double startup_time = get_timestamp();
boost::filesystem::path config_file_path, input_path, profile_path;
int requested_num_threads;
// declare a group of options that will be allowed only on command line
boost::program_options::options_description generic_options("Options");
generic_options.add_options()
("version,v", "Show version")
("help,h", "Show this help message")
("config,c", boost::program_options::value<boost::filesystem::path>(&config_file_path)->default_value("extractor.ini"),
"Path to a configuration file.");
// declare a group of options that will be allowed both on command line and in config file
boost::program_options::options_description config_options("Configuration");
config_options.add_options()
("profile,p", boost::program_options::value<boost::filesystem::path>(&profile_path)->default_value("profile.lua"),
"Path to LUA routing profile")
("threads,t", boost::program_options::value<int>(&requested_num_threads)->default_value(8),
"Number of threads to use");
// hidden options, will be allowed both on command line and in config file, but will not be shown to the user
boost::program_options::options_description hidden_options("Hidden options");
hidden_options.add_options()
("input,i", boost::program_options::value<boost::filesystem::path>(&input_path),
"Input file in .osm, .osm.bz2 or .osm.pbf format");
// positional option
boost::program_options::positional_options_description positional_options;
positional_options.add("input", 1);
// combine above options for parsing
boost::program_options::options_description cmdline_options;
cmdline_options.add(generic_options).add(config_options).add(hidden_options);
boost::program_options::options_description config_file_options;
config_file_options.add(config_options).add(hidden_options);
boost::program_options::options_description visible_options(boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
visible_options.add(generic_options).add(config_options);
// parse command line options
boost::program_options::variables_map option_variables;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv).
options(cmdline_options).positional(positional_options).run(), option_variables);
if(option_variables.count("version")) {
SimpleLogger().Write() << g_GIT_DESCRIPTION;
return 0;
}
if(option_variables.count("help")) {
SimpleLogger().Write() << visible_options;
return 0;
}
boost::program_options::notify(option_variables);
// parse config file
if(boost::filesystem::is_regular_file(config_file_path)) {
SimpleLogger().Write() << "Reading options from: " << config_file_path.c_str();
std::string config_str;
PrepareConfigFile( config_file_path.c_str(), config_str );
std::stringstream config_stream( config_str );
boost::program_options::store(parse_config_file(config_stream, config_file_options), option_variables);
boost::program_options::notify(option_variables);
}
if(!option_variables.count("input")) {
SimpleLogger().Write(logWARNING) << "No input file specified";
SimpleLogger().Write() << visible_options;
return -1;
}
if(1 > requested_num_threads) {
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return -1;
}
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
/*** Setup Scripting Environment ***/
ScriptingEnvironment scriptingEnvironment(profile_path.c_str());
omp_set_num_threads( std::min( omp_get_num_procs(), requested_num_threads) );
bool file_has_pbf_format(false);
std::string output_file_name(input_path.c_str());
std::string restrictionsFileName(input_path.c_str());
std::string::size_type pos = output_file_name.find(".osm.bz2");
if(pos==std::string::npos) {
pos = output_file_name.find(".osm.pbf");
if(pos!=std::string::npos) {
file_has_pbf_format = true;
}
}
if(pos==std::string::npos) {
pos = output_file_name.find(".pbf");
if(pos!=std::string::npos) {
file_has_pbf_format = true;
}
}
if(pos!=std::string::npos) {
output_file_name.replace(pos, 8, ".osrm");
restrictionsFileName.replace(pos, 8, ".osrm.restrictions");
} else {
pos=output_file_name.find(".osm");
if(pos!=std::string::npos) {
output_file_name.replace(pos, 5, ".osrm");
restrictionsFileName.replace(pos, 5, ".osrm.restrictions");
} else {
output_file_name.append(".osrm");
restrictionsFileName.append(".osrm.restrictions");
}
}
StringMap stringMap;
ExtractionContainers externalMemory;
stringMap[""] = 0;
extractCallBacks = new ExtractorCallbacks(&externalMemory, &stringMap);
BaseParser* parser;
if(file_has_pbf_format) {
parser = new PBFParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
} else {
parser = new XMLParser(input_path.c_str(), extractCallBacks, scriptingEnvironment);
}
if(!parser->ReadHeader()) {
throw OSRMException("Parser not initialized!");
}
SimpleLogger().Write() << "Parsing in progress..";
double parsing_start_time = get_timestamp();
parser->Parse();
SimpleLogger().Write() << "Parsing finished after " <<
(get_timestamp() - parsing_start_time) <<
" seconds";
if( externalMemory.all_edges_list.empty() ) {
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return -1;
}
externalMemory.PrepareData(output_file_name, restrictionsFileName);
delete parser;
delete extractCallBacks;
SimpleLogger().Write() <<
"extraction finished after " << get_timestamp() - startup_time <<
"s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << output_file_name << std::endl;
} catch(boost::program_options::too_many_positional_options_error& e) {
SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
return -1;
} catch(boost::program_options::error& e) {
SimpleLogger().Write(logWARNING) << e.what();
return -1;
} catch(std::exception & e) {
SimpleLogger().Write(logWARNING) << "Unhandled exception: " << e.what();
return -1;
}
return 0;
}
int main(int argc, char *argv[])
{
LogPolicy::GetInstance().Unmute();
if (argc < 3)
{
SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0] << " <osrm> <osrm.restrictions>";
return -1;
}
try
{
SimpleLogger().Write() << "Using restrictions from file: " << argv[2];
std::ifstream restriction_ifstream(argv[2], std::ios::binary);
const FingerPrint fingerprint_orig;
FingerPrint fingerprint_loaded;
restriction_ifstream.read((char *)&fingerprint_loaded, sizeof(FingerPrint));
if (!fingerprint_loaded.TestGraphUtil(fingerprint_orig))
{
SimpleLogger().Write(logWARNING) << argv[2] << " was prepared with a different build. "
"Reprocess to get rid of this warning.";
}
if (!restriction_ifstream.good())
{
throw OSRMException("Could not access <osrm-restrictions> files");
}
uint32_t usable_restriction_count = 0;
restriction_ifstream.read((char *)&usable_restriction_count, sizeof(uint32_t));
restrictions_vector.resize(usable_restriction_count);
if (usable_restriction_count>0)
{
restriction_ifstream.read((char *)&(restrictions_vector[0]),
usable_restriction_count * sizeof(TurnRestriction));
}
restriction_ifstream.close();
std::ifstream input_stream;
input_stream.open(argv[1], std::ifstream::in | std::ifstream::binary);
if (!input_stream.is_open())
{
throw OSRMException("Cannot open osrm file");
}
std::vector<ImportEdge> edge_list;
const NodeID number_of_nodes = readBinaryOSRMGraphFromStream(input_stream,
edge_list,
bollard_node_IDs_vector,
traffic_light_node_IDs_vector,
&internal_to_external_node_map,
restrictions_vector);
input_stream.close();
BOOST_ASSERT_MSG(restrictions_vector.size() == usable_restriction_count,
"size of restrictions_vector changed");
SimpleLogger().Write() << restrictions_vector.size() << " restrictions, "
<< bollard_node_IDs_vector.size() << " bollard nodes, "
<< traffic_light_node_IDs_vector.size() << " traffic lights";
/***
* Building an edge-expanded graph from node-based input an turn
* restrictions
*/
SimpleLogger().Write() << "Starting SCC graph traversal";
std::shared_ptr<TarjanSCC> tarjan =
std::make_shared<TarjanSCC>(number_of_nodes,
edge_list,
bollard_node_IDs_vector,
traffic_light_node_IDs_vector,
restrictions_vector,
internal_to_external_node_map);
std::vector<ImportEdge>().swap(edge_list);
tarjan->Run();
SimpleLogger().Write() << "finished component analysis";
}
catch (const std::exception &e)
{
SimpleLogger().Write(logWARNING) << "[exception] " << e.what();
}
return 0;
}
int Extractor::Run(int argc, char *argv[])
{
try
{
LogPolicy::GetInstance().Unmute();
TIMER_START(extracting);
if (!ParseArguments(argc, argv))
return 0;
if (1 > requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
return 1;
}
if (!boost::filesystem::is_regular_file(input_path))
{
SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
<< " not found!";
return 1;
}
if (!boost::filesystem::is_regular_file(profile_path))
{
SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
<< " not found!";
return 1;
}
const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();
SimpleLogger().Write() << "Input file: " << input_path.filename().string();
SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
SimpleLogger().Write() << "Threads: " << requested_num_threads;
if (recommended_num_threads != requested_num_threads)
{
SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
<< recommended_num_threads
<< "! This setting may have performance side-effects.";
}
tbb::task_scheduler_init init(requested_num_threads);
/*** Setup Scripting Environment ***/
ScriptingEnvironment scripting_environment(profile_path.string().c_str());
GenerateOutputFilesNames();
std::unordered_map<std::string, NodeID> string_map;
ExtractionContainers extraction_containers;
string_map[""] = 0;
auto extractor_callbacks = new ExtractorCallbacks(extraction_containers, string_map);
BaseParser *parser;
if (file_has_pbf_format)
{
parser = new PBFParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment,
requested_num_threads);
}
else
{
parser = new XMLParser(input_path.string().c_str(),
extractor_callbacks,
scripting_environment);
}
if (!parser->ReadHeader())
{
throw OSRMException("Parser not initialized!");
}
SimpleLogger().Write() << "Parsing in progress..";
TIMER_START(parsing);
parser->Parse();
delete parser;
delete extractor_callbacks;
TIMER_STOP(parsing);
SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
if (extraction_containers.all_edges_list.empty())
{
SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
return 1;
}
extraction_containers.PrepareData(output_file_name, restriction_file_name);
TIMER_STOP(extracting);
SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
SimpleLogger().Write() << "To prepare the data for routing, run: "
<< "./osrm-prepare " << output_file_name << std::endl;
}
catch (boost::program_options::too_many_positional_options_error &)
{
SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
return 1;
}
catch (std::exception &e)
{
SimpleLogger().Write(logWARNING) << e.what();
return 1;
}
return 0;
}
ScriptingEnvironment::ScriptingEnvironment(const char * fileName) {
SimpleLogger().Write() << "Using script " << fileName;
// Create a new lua state
for(int i = 0; i < omp_get_max_threads(); ++i) {
luaStateVector.push_back(luaL_newstate());
}
// Connect LuaBind to this lua state for all threads
#pragma omp parallel
{
lua_State * myLuaState = getLuaStateForThreadID(omp_get_thread_num());
luabind::open(myLuaState);
//open utility libraries string library;
luaL_openlibs(myLuaState);
luaAddScriptFolderToLoadPath( myLuaState, fileName );
// Add our function to the state's global scope
luabind::module(myLuaState) [
luabind::def("print", LUA_print<std::string>),
luabind::def("parseMaxspeed", parseMaxspeed),
luabind::def("durationIsValid", durationIsValid),
luabind::def("parseDuration", parseDuration)
];
luabind::module(myLuaState) [
luabind::class_<HashTable<std::string, std::string> >("keyVals")
.def("Add", &HashTable<std::string, std::string>::Add)
.def("Find", &HashTable<std::string, std::string>::Find)
.def("Holds", &HashTable<std::string, std::string>::Holds)
];
luabind::module(myLuaState) [
luabind::class_<ImportNode>("Node")
.def(luabind::constructor<>())
.def_readwrite("lat", &ImportNode::lat)
.def_readwrite("lon", &ImportNode::lon)
.def_readwrite("id", &ImportNode::id)
.def_readwrite("bollard", &ImportNode::bollard)
.def_readwrite("traffic_light", &ImportNode::trafficLight)
.def_readwrite("tags", &ImportNode::keyVals)
];
luabind::module(myLuaState) [
luabind::class_<ExtractionWay>("Way")
.def(luabind::constructor<>())
.def_readwrite("name", &ExtractionWay::name)
.def_readwrite("speed", &ExtractionWay::speed)
.def_readwrite("backward_speed", &ExtractionWay::backward_speed)
.def_readwrite("duration", &ExtractionWay::duration)
.def_readwrite("type", &ExtractionWay::type)
.def_readwrite("access", &ExtractionWay::access)
.def_readwrite("roundabout", &ExtractionWay::roundabout)
.def_readwrite("is_access_restricted", &ExtractionWay::isAccessRestricted)
.def_readwrite("ignore_in_grid", &ExtractionWay::ignoreInGrid)
.def_readwrite("tags", &ExtractionWay::keyVals)
.def_readwrite("direction", &ExtractionWay::direction)
.enum_("constants") [
luabind::value("notSure", 0),
luabind::value("oneway", 1),
luabind::value("bidirectional", 2),
luabind::value("opposite", 3)
]
];
luabind::module(myLuaState) [
luabind::class_<std::vector<std::string> >("vector")
.def("Add", &std::vector<std::string>::push_back)
];
if(0 != luaL_dofile(myLuaState, fileName) ) {
throw OSRMException("ERROR occured in scripting block");
}
}
}
