void
QcOsmPbfReader::read_file(bool read_nodes, bool read_ways, bool read_relations, bool read_metadatas)
{
m_read_nodes = read_nodes;
m_read_ways = read_ways;
m_read_relations = read_relations;
m_read_metadatas = read_metadatas;
m_number_of_primitive_groups = 0;
m_number_of_node_primitive_groups = 0;
m_number_of_dense_node_primitive_groups = 0;
m_number_of_way_primitive_groups = 0;
m_number_of_relation_primitive_groups = 0;
m_number_of_nodes = 0;
m_number_of_ways = 0;
m_number_of_relations = 0;
m_buffer = new char[OSMPBF::max_uncompressed_blob_size];
m_unpack_buffer = new char[OSMPBF::max_uncompressed_blob_size];
QFile file(m_pbf_path);
if (!file.open(QIODevice::ReadOnly)) {
qCritical() << "can't open file" << m_pbf_path;
return ;
}
m_data_stream.setDevice(&file);
// Set network byte-order
m_data_stream.setByteOrder(QDataStream::BigEndian);
// read while the file has not reached its end
while (!file.atEnd()) {
/* A file contains a header followed by a sequence of fileblocks.
*
* The design is intended to allow future random-access to the
* contents of the file and skipping past not-understood or
* unwanted data.
*
* The format is a repeating sequence of:
* - int4: length of the BlobHeader message in network byte order
* - serialized BlobHeader message
* - serialized Blob message (size is given in the header)
*/
read_blob_header();
read_blob();
}
// clean up the protobuf lib
google::protobuf::ShutdownProtobufLibrary();
qDebug() << "File Statistics\n"
<< "Primitive Groups" << m_number_of_primitive_groups << "\n"
<< " for Node" << m_number_of_node_primitive_groups << "\n"
<< " for Dense Node" << m_number_of_dense_node_primitive_groups << "\n"
<< " for Way" << m_number_of_way_primitive_groups << "\n"
<< " for Relation" << m_number_of_relation_primitive_groups << "\n"
<< "Number of nodes" << m_number_of_nodes << "\n"
<< "Number of ways" << m_number_of_ways << "\n"
<< "Number of relations" << m_number_of_relations;
}
// Find position and size of all data blobs in the file
static VALUE find_all_blobs(VALUE obj)
{
FILE *input = DATA_PTR(obj);
long old_pos = ftell(input);
if (0 != fseek(input, 0, SEEK_SET)) {
rb_raise(rb_eIOError, "Unable to seek to beginning of file");
}
OSMPBF__BlobHeader *header;
VALUE blobs = rb_ary_new();
rb_iv_set(obj, "@blobs", blobs);
long pos = 0, data_pos = 0;
int32_t datasize;
while ((header = read_blob_header(input)) != NULL) {
datasize = header->datasize;
if (0 == strcmp(header->type, "OSMData")) {
VALUE blob_info = rb_hash_new();
data_pos = ftell(input);
// This is designed to be user-friendly, so I have chosen
// to make header_pos the position of the protobuf stream
// itself, in line with data_pos. However, internally, we
// subtract 4 when calling parse_osm_data().
rb_hash_aset(blob_info, STR2SYM("header_pos"),
LONG2NUM(pos + 4));
rb_hash_aset(blob_info, STR2SYM("header_size"),
LONG2NUM(data_pos - pos - 4));
rb_hash_aset(blob_info, STR2SYM("data_pos"),
LONG2NUM(data_pos));
rb_hash_aset(blob_info, STR2SYM("data_size"),
UINT2NUM(datasize));
rb_ary_push(blobs, blob_info);
}
osmpbf__blob_header__free_unpacked(header, NULL);
if (0 != fseek(input, datasize, SEEK_CUR)) {
break; // cut losses
}
pos = ftell(input);
}
// restore old position
if (0 != fseek(input, old_pos, SEEK_SET)) {
rb_raise(rb_eIOError, "Unable to restore old file position");
}
return blobs;
}
static VALUE parse_osm_data(VALUE obj)
{
FILE *input = DATA_PTR(obj);
OSMPBF__BlobHeader *header = read_blob_header(input);
if(header == NULL)
return Qfalse;
if(strcmp("OSMData", header->type) != 0)
rb_raise(rb_eIOError, "OSMData not found");
void *blob = NULL;
size_t blob_length = 0, datasize = header->datasize;
OSMPBF__PrimitiveBlock *primitive_block = NULL;
osmpbf__blob_header__free_unpacked(header, NULL);
blob = read_blob(input, datasize, &blob_length);
primitive_block = osmpbf__primitive_block__unpack(NULL, blob_length, blob);
free(blob);
if(primitive_block == NULL)
rb_raise(rb_eIOError, "Unable to unpack the PrimitiveBlock");
int64_t lat_offset, lon_offset, granularity;
int32_t ts_granularity;
lat_offset = primitive_block->lat_offset;
lon_offset = primitive_block->lon_offset;
granularity = primitive_block->granularity;
ts_granularity = primitive_block->date_granularity;
OSMPBF__StringTable *string_table = primitive_block->stringtable;
VALUE data = init_data_arr();
VALUE nodes = rb_hash_aref(data, STR2SYM("nodes"));
VALUE ways = rb_hash_aref(data, STR2SYM("ways"));
VALUE relations = rb_hash_aref(data, STR2SYM("relations"));
size_t i = 0;
for(i = 0; i < primitive_block->n_primitivegroup; i++)
{
OSMPBF__PrimitiveGroup *primitive_group = primitive_block->primitivegroup[i];
if(primitive_group->nodes)
process_nodes(nodes, primitive_group, string_table, lat_offset, lon_offset, granularity, ts_granularity);
if(primitive_group->dense)
process_dense_nodes(nodes, primitive_group->dense, string_table, lat_offset, lon_offset, granularity, ts_granularity);
if(primitive_group->ways)
process_ways(ways, primitive_group, string_table, ts_granularity);
if(primitive_group->relations)
process_relations(relations, primitive_group, string_table, ts_granularity);
}
rb_iv_set(obj, "@data", data);
osmpbf__primitive_block__free_unpacked(primitive_block, NULL);
// Increment position
rb_iv_set(obj, "@pos", INT2NUM(NUM2INT(rb_iv_get(obj, "@pos")) + 1));
return Qtrue;
}
static int parse_osm_header(VALUE obj, FILE *input)
{
OSMPBF__BlobHeader *header = read_blob_header(input);
// EOF reached
if(header == NULL)
rb_raise(rb_eEOFError, "EOF reached without finding data");
if(strcmp("OSMHeader", header->type) != 0)
rb_raise(rb_eIOError, "OSMHeader not found, probably the file is corrupt or invalid");
void *blob = NULL;
size_t blob_length = 0, datasize = header->datasize;
OSMPBF__HeaderBlock *header_block = NULL;
osmpbf__blob_header__free_unpacked(header, NULL);
blob = read_blob(input, datasize, &blob_length);
header_block = osmpbf__header_block__unpack(NULL, blob_length, blob);
free(blob);
if(header_block == NULL)
rb_raise(rb_eIOError, "Unable to unpack the HeaderBlock");
VALUE header_hash = rb_hash_new();
VALUE bbox_hash = rb_hash_new();
VALUE required_features = Qnil;
VALUE optional_features = Qnil;
VALUE writingprogram = Qnil;
VALUE source = Qnil;
VALUE osmosis_replication_timestamp = Qnil;
VALUE osmosis_replication_sequence_number = Qnil;
VALUE osmosis_replication_base_url = Qnil;
int i = 0;
if(header_block->n_required_features > 0)
{
required_features = rb_ary_new();
for(i = 0; i < (int)header_block->n_required_features; i++)
rb_ary_push(required_features, str_new(header_block->required_features[i]));
}
if(header_block->n_optional_features > 0)
{
optional_features = rb_ary_new();
for(i = 0; i < (int)header_block->n_optional_features; i++)
rb_ary_push(optional_features, str_new(header_block->optional_features[i]));
}
if(header_block->writingprogram)
writingprogram = str_new(header_block->writingprogram);
if(header_block->source)
source = str_new(header_block->source);
if(header_block->bbox)
{
rb_hash_aset(bbox_hash, STR2SYM("top"), rb_float_new(header_block->bbox->top * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("right"), rb_float_new(header_block->bbox->right * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("bottom"), rb_float_new(header_block->bbox->bottom * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("left"), rb_float_new(header_block->bbox->left * NANO_DEGREE));
}
if(header_block->has_osmosis_replication_timestamp)
osmosis_replication_timestamp = ULL2NUM(header_block->osmosis_replication_timestamp);
if(header_block->has_osmosis_replication_sequence_number)
osmosis_replication_sequence_number = ULL2NUM(header_block->osmosis_replication_sequence_number);
if(header_block->osmosis_replication_base_url)
osmosis_replication_base_url = str_new(header_block->osmosis_replication_base_url);
rb_hash_aset(header_hash, str_new("bbox"), bbox_hash);
rb_hash_aset(header_hash, str_new("required_features"), required_features);
rb_hash_aset(header_hash, str_new("optional_features"), optional_features);
rb_hash_aset(header_hash, str_new("writing_program"), writingprogram);
rb_hash_aset(header_hash, str_new("source"), source);
rb_hash_aset(header_hash, str_new("osmosis_replication_timestamp"), osmosis_replication_timestamp);
rb_hash_aset(header_hash, str_new("osmosis_replication_sequence_number"), osmosis_replication_sequence_number);
rb_hash_aset(header_hash, str_new("osmosis_replication_base_url"), osmosis_replication_base_url);
rb_iv_set(obj, "@header", header_hash);
osmpbf__header_block__free_unpacked(header_block, NULL);
return 1;
}
