// Find position and size of all data blobs in the file static VALUE find_all_blobs(VALUE obj) { FILE *input = DATA_PTR(obj); long old_pos = ftell(input); if (0 != fseek(input, 0, SEEK_SET)) { rb_raise(rb_eIOError, "Unable to seek to beginning of file"); } OSMPBF__BlobHeader *header; VALUE blobs = rb_ary_new(); rb_iv_set(obj, "@blobs", blobs); long pos = 0, data_pos = 0; int32_t datasize; while ((header = read_blob_header(input)) != NULL) { datasize = header->datasize; if (0 == strcmp(header->type, "OSMData")) { VALUE blob_info = rb_hash_new(); data_pos = ftell(input); // This is designed to be user-friendly, so I have chosen // to make header_pos the position of the protobuf stream // itself, in line with data_pos. However, internally, we // subtract 4 when calling parse_osm_data(). rb_hash_aset(blob_info, STR2SYM("header_pos"), LONG2NUM(pos + 4)); rb_hash_aset(blob_info, STR2SYM("header_size"), LONG2NUM(data_pos - pos - 4)); rb_hash_aset(blob_info, STR2SYM("data_pos"), LONG2NUM(data_pos)); rb_hash_aset(blob_info, STR2SYM("data_size"), UINT2NUM(datasize)); rb_ary_push(blobs, blob_info); } osmpbf__blob_header__free_unpacked(header, NULL); if (0 != fseek(input, datasize, SEEK_CUR)) { break; // cut losses } pos = ftell(input); } // restore old position if (0 != fseek(input, old_pos, SEEK_SET)) { rb_raise(rb_eIOError, "Unable to restore old file position"); } return blobs; }
static VALUE init_data_arr() { VALUE data = rb_hash_new(); rb_hash_aset(data, STR2SYM("nodes"), rb_ary_new()); rb_hash_aset(data, STR2SYM("ways"), rb_ary_new()); rb_hash_aset(data, STR2SYM("relations"), rb_ary_new()); return data; }
static VALUE seek_to_osm_data(VALUE obj, VALUE index) { FILE *input = DATA_PTR(obj); VALUE blobs = blobs_getter(obj); int index_raw = NUM2INT(index); // Normalise the index, otherwise #pos returns the wrong value if (index_raw < 0) { int size = NUM2INT(size_getter(obj)); // Only normalise if valid; otherwise, allow rb_ary_entry() to fail. if (index_raw + size >= 0) { index_raw += size; } } if (NUM2INT(rb_iv_get(obj, "@pos")) == index_raw) { return Qtrue; // already there } VALUE blob_info = rb_ary_entry(blobs, index_raw); if (!RTEST(blob_info)) { return Qfalse; // no such blob entry } long pos = NUM2LONG(rb_hash_aref(blob_info, STR2SYM("header_pos"))) - 4; if (0 != fseek(input, pos, SEEK_SET)) { rb_raise(rb_eIOError, "Unable to seek to file position"); } // Set position - incremented by parse_osm_data rb_iv_set(obj, "@pos", INT2NUM(index_raw - 1)); return parse_osm_data(obj); }
static void process_ways(VALUE out, OSMPBF__PrimitiveGroup *group, OSMPBF__StringTable *string_table, int32_t ts_granularity) { unsigned j, k; size_t i = 0; for(i = 0; i < group->n_ways; i++) { OSMPBF__Way *way = group->ways[i]; int64_t delta_refs = 0; VALUE way_out = rb_hash_new(); rb_hash_aset(way_out, STR2SYM("id"), LL2NUM(way->id)); // Extract tags VALUE tags = rb_hash_new(); for(j = 0; j < way->n_keys; j++) { char *key = parse_binary_str(string_table->s[way->keys[j]]); char *value = parse_binary_str(string_table->s[way->vals[j]]); rb_hash_aset(tags, str_new(key), str_new(value)); free(key); free(value); } // Extract refs VALUE refs = rb_ary_new(); for(k = 0; k < way->n_refs; k++) { delta_refs += way->refs[k]; rb_ary_push(refs, LL2NUM(delta_refs)); } // Extract info if(way->info) add_info(way_out, way->info, string_table, ts_granularity); rb_hash_aset(way_out, STR2SYM("tags"), tags); rb_hash_aset(way_out, STR2SYM("refs"), refs); rb_ary_push(out, way_out); } }
static void process_nodes(VALUE out, OSMPBF__PrimitiveGroup *group, OSMPBF__StringTable *string_table, int64_t lat_offset, int64_t lon_offset, int64_t granularity, int32_t ts_granularity) { double lat = 0; double lon = 0; unsigned j = 0; size_t i = 0; for(i = 0; i < group->n_nodes; i++) { OSMPBF__Node *node = group->nodes[i]; VALUE node_out = rb_hash_new(); lat = NANO_DEGREE * (lat_offset + (node->lat * granularity)); lon = NANO_DEGREE * (lon_offset + (node->lon * granularity)); rb_hash_aset(node_out, STR2SYM("id"), LL2NUM(node->id)); rb_hash_aset(node_out, STR2SYM("lat"), FIX7(rb_float_new(lat))); rb_hash_aset(node_out, STR2SYM("lon"), FIX7(rb_float_new(lon))); if(node->info) add_info(node_out, node->info, string_table, ts_granularity); VALUE tags = rb_hash_new(); for(j = 0; j < node->n_keys; j++) { char *key = parse_binary_str(string_table->s[node->keys[j]]); char *value = parse_binary_str(string_table->s[node->vals[j]]); rb_hash_aset(tags, str_new(key), str_new(value)); free(key); free(value); } rb_hash_aset(node_out, STR2SYM("tags"), tags); rb_ary_push(out, node_out); } }
static void add_info(VALUE hash, OSMPBF__Info *info, OSMPBF__StringTable *string_table, double ts_granularity) { VALUE version, timestamp, changeset, uid, user; version = info->version ? INT2NUM(info->version) : Qnil; timestamp = info->timestamp ? LL2NUM(info->timestamp * ts_granularity) : Qnil; changeset = info->changeset ? LL2NUM(info->changeset) : Qnil; uid = info->uid ? INT2NUM(info->uid) : Qnil; if(info->user_sid) { char *user_sid = parse_binary_str(string_table->s[info->user_sid]); user = str_new(user_sid); free(user_sid); } else user = Qnil; rb_hash_aset(hash, STR2SYM("version"), version); rb_hash_aset(hash, STR2SYM("timestamp"), timestamp); rb_hash_aset(hash, STR2SYM("changeset"), changeset); rb_hash_aset(hash, STR2SYM("uid"), uid); rb_hash_aset(hash, STR2SYM("user"), user); }
static VALUE relations_getter(VALUE obj) { VALUE data = rb_iv_get(obj, "@data"); return rb_hash_aref(data, STR2SYM("relations")); }
static VALUE nodes_getter(VALUE obj) { VALUE data = rb_iv_get(obj, "@data"); return rb_hash_aref(data, STR2SYM("nodes")); }
static VALUE parse_osm_data(VALUE obj) { FILE *input = DATA_PTR(obj); OSMPBF__BlobHeader *header = read_blob_header(input); if(header == NULL) return Qfalse; if(strcmp("OSMData", header->type) != 0) rb_raise(rb_eIOError, "OSMData not found"); void *blob = NULL; size_t blob_length = 0, datasize = header->datasize; OSMPBF__PrimitiveBlock *primitive_block = NULL; osmpbf__blob_header__free_unpacked(header, NULL); blob = read_blob(input, datasize, &blob_length); primitive_block = osmpbf__primitive_block__unpack(NULL, blob_length, blob); free(blob); if(primitive_block == NULL) rb_raise(rb_eIOError, "Unable to unpack the PrimitiveBlock"); int64_t lat_offset, lon_offset, granularity; int32_t ts_granularity; lat_offset = primitive_block->lat_offset; lon_offset = primitive_block->lon_offset; granularity = primitive_block->granularity; ts_granularity = primitive_block->date_granularity; OSMPBF__StringTable *string_table = primitive_block->stringtable; VALUE data = init_data_arr(); VALUE nodes = rb_hash_aref(data, STR2SYM("nodes")); VALUE ways = rb_hash_aref(data, STR2SYM("ways")); VALUE relations = rb_hash_aref(data, STR2SYM("relations")); size_t i = 0; for(i = 0; i < primitive_block->n_primitivegroup; i++) { OSMPBF__PrimitiveGroup *primitive_group = primitive_block->primitivegroup[i]; if(primitive_group->nodes) process_nodes(nodes, primitive_group, string_table, lat_offset, lon_offset, granularity, ts_granularity); if(primitive_group->dense) process_dense_nodes(nodes, primitive_group->dense, string_table, lat_offset, lon_offset, granularity, ts_granularity); if(primitive_group->ways) process_ways(ways, primitive_group, string_table, ts_granularity); if(primitive_group->relations) process_relations(relations, primitive_group, string_table, ts_granularity); } rb_iv_set(obj, "@data", data); osmpbf__primitive_block__free_unpacked(primitive_block, NULL); // Increment position rb_iv_set(obj, "@pos", INT2NUM(NUM2INT(rb_iv_get(obj, "@pos")) + 1)); return Qtrue; }
static void process_relations(VALUE out, OSMPBF__PrimitiveGroup *group, OSMPBF__StringTable *string_table, int32_t ts_granularity) { unsigned j, k; size_t i = 0; for(i = 0; i < group->n_relations; i++) { OSMPBF__Relation *relation = group->relations[i]; VALUE relation_out = rb_hash_new(); rb_hash_aset(relation_out, STR2SYM("id"), LL2NUM(relation->id)); // Extract tags VALUE tags = rb_hash_new(); for(j = 0; j < relation->n_keys; j++) { char *key = parse_binary_str(string_table->s[relation->keys[j]]); char *value = parse_binary_str(string_table->s[relation->vals[j]]); rb_hash_aset(tags, str_new(key), str_new(value)); free(key); free(value); } // Extract members VALUE members = rb_hash_new(); VALUE nodes = rb_ary_new(); VALUE ways = rb_ary_new(); VALUE relations = rb_ary_new(); int64_t delta_memids = 0; char *role; for(k = 0; k < relation->n_memids; k++) { VALUE member = rb_hash_new(); delta_memids += relation->memids[k]; rb_hash_aset(member, STR2SYM("id"), LL2NUM(delta_memids)); if(relation->roles_sid[k]) { role = parse_binary_str(string_table->s[relation->roles_sid[k]]); rb_hash_aset(member, STR2SYM("role"), str_new(role)); free(role); } switch(relation->types[k]) { case OSMPBF__RELATION__MEMBER_TYPE__NODE: rb_ary_push(nodes, member); break; case OSMPBF__RELATION__MEMBER_TYPE__WAY: rb_ary_push(ways, member); break; case OSMPBF__RELATION__MEMBER_TYPE__RELATION: rb_ary_push(relations, member); break; } } rb_hash_aset(members, STR2SYM("nodes"), nodes); rb_hash_aset(members, STR2SYM("ways"), ways); rb_hash_aset(members, STR2SYM("relations"), relations); // Extract info if(relation->info) add_info(relation_out, relation->info, string_table, ts_granularity); rb_hash_aset(relation_out, STR2SYM("tags"), tags); rb_hash_aset(relation_out, STR2SYM("members"), members); rb_ary_push(out, relation_out); } }
static void process_dense_nodes(VALUE out, OSMPBF__DenseNodes *dense_nodes, OSMPBF__StringTable *string_table, int64_t lat_offset, int64_t lon_offset, int64_t granularity, int32_t ts_granularity) { uint64_t node_id = 0; int64_t delta_lat = 0; int64_t delta_lon = 0; int64_t delta_timestamp = 0; int64_t delta_changeset = 0; int32_t delta_user_sid = 0; int32_t delta_uid = 0; double lat = 0; double lon = 0; unsigned j = 0; size_t i = 0; for(i = 0; i < dense_nodes->n_id; i++) { VALUE node = rb_hash_new(); node_id += dense_nodes->id[i]; delta_lat += dense_nodes->lat[i]; delta_lon += dense_nodes->lon[i]; lat = NANO_DEGREE * (lat_offset + (delta_lat * granularity)); lon = NANO_DEGREE * (lon_offset + (delta_lon * granularity)); rb_hash_aset(node, STR2SYM("id"), LL2NUM(node_id)); rb_hash_aset(node, STR2SYM("lat"), FIX7(rb_float_new(lat))); rb_hash_aset(node, STR2SYM("lon"), FIX7(rb_float_new(lon))); // Extract info if(dense_nodes->denseinfo) { delta_timestamp += dense_nodes->denseinfo->timestamp[i]; delta_changeset += dense_nodes->denseinfo->changeset[i]; delta_user_sid += dense_nodes->denseinfo->user_sid[i]; delta_uid += dense_nodes->denseinfo->uid[i]; OSMPBF__Info info = { .version = dense_nodes->denseinfo->version[i], .timestamp = delta_timestamp, .changeset = delta_changeset, .user_sid = delta_user_sid, .uid = delta_uid }; add_info(node, &info, string_table, ts_granularity); } // Extract tags VALUE tags = rb_hash_new(); if(j < dense_nodes->n_keys_vals) { while((dense_nodes->keys_vals[j] != 0) && (j < dense_nodes->n_keys_vals)) { char *key = parse_binary_str(string_table->s[dense_nodes->keys_vals[j]]); char *value = parse_binary_str(string_table->s[dense_nodes->keys_vals[j+1]]); rb_hash_aset(tags, str_new(key), str_new(value)); free(key); free(value); j += 2; } j += 1; } rb_hash_aset(node, STR2SYM("tags"), tags); rb_ary_push(out, node); } }
static int parse_osm_header(VALUE obj, FILE *input) { OSMPBF__BlobHeader *header = read_blob_header(input); // EOF reached if(header == NULL) rb_raise(rb_eEOFError, "EOF reached without finding data"); if(strcmp("OSMHeader", header->type) != 0) rb_raise(rb_eIOError, "OSMHeader not found, probably the file is corrupt or invalid"); void *blob = NULL; size_t blob_length = 0, datasize = header->datasize; OSMPBF__HeaderBlock *header_block = NULL; osmpbf__blob_header__free_unpacked(header, NULL); blob = read_blob(input, datasize, &blob_length); header_block = osmpbf__header_block__unpack(NULL, blob_length, blob); free(blob); if(header_block == NULL) rb_raise(rb_eIOError, "Unable to unpack the HeaderBlock"); VALUE header_hash = rb_hash_new(); VALUE bbox_hash = rb_hash_new(); VALUE required_features = Qnil; VALUE optional_features = Qnil; VALUE writingprogram = Qnil; VALUE source = Qnil; VALUE osmosis_replication_timestamp = Qnil; VALUE osmosis_replication_sequence_number = Qnil; VALUE osmosis_replication_base_url = Qnil; int i = 0; if(header_block->n_required_features > 0) { required_features = rb_ary_new(); for(i = 0; i < (int)header_block->n_required_features; i++) rb_ary_push(required_features, str_new(header_block->required_features[i])); } if(header_block->n_optional_features > 0) { optional_features = rb_ary_new(); for(i = 0; i < (int)header_block->n_optional_features; i++) rb_ary_push(optional_features, str_new(header_block->optional_features[i])); } if(header_block->writingprogram) writingprogram = str_new(header_block->writingprogram); if(header_block->source) source = str_new(header_block->source); if(header_block->bbox) { rb_hash_aset(bbox_hash, STR2SYM("top"), rb_float_new(header_block->bbox->top * NANO_DEGREE)); rb_hash_aset(bbox_hash, STR2SYM("right"), rb_float_new(header_block->bbox->right * NANO_DEGREE)); rb_hash_aset(bbox_hash, STR2SYM("bottom"), rb_float_new(header_block->bbox->bottom * NANO_DEGREE)); rb_hash_aset(bbox_hash, STR2SYM("left"), rb_float_new(header_block->bbox->left * NANO_DEGREE)); } if(header_block->has_osmosis_replication_timestamp) osmosis_replication_timestamp = ULL2NUM(header_block->osmosis_replication_timestamp); if(header_block->has_osmosis_replication_sequence_number) osmosis_replication_sequence_number = ULL2NUM(header_block->osmosis_replication_sequence_number); if(header_block->osmosis_replication_base_url) osmosis_replication_base_url = str_new(header_block->osmosis_replication_base_url); rb_hash_aset(header_hash, str_new("bbox"), bbox_hash); rb_hash_aset(header_hash, str_new("required_features"), required_features); rb_hash_aset(header_hash, str_new("optional_features"), optional_features); rb_hash_aset(header_hash, str_new("writing_program"), writingprogram); rb_hash_aset(header_hash, str_new("source"), source); rb_hash_aset(header_hash, str_new("osmosis_replication_timestamp"), osmosis_replication_timestamp); rb_hash_aset(header_hash, str_new("osmosis_replication_sequence_number"), osmosis_replication_sequence_number); rb_hash_aset(header_hash, str_new("osmosis_replication_base_url"), osmosis_replication_base_url); rb_iv_set(obj, "@header", header_hash); osmpbf__header_block__free_unpacked(header_block, NULL); return 1; }
/** * call-seq: * transaction( [ flags ] ){|transaction|...} → an_object * * Puts libalpm into transaction mode, i.e. allows you to add * and remove packaages by means of a transaction. The block * gets called with an instance of the (otherwise uninstanciatable, * this is a libalpm restriction) Transaction class, which you can * freely modify for your operations. When you added all packages * you want to add/remove to/from the system, call Transaction#prepare * in order to have libalpm resolve dependencies and other stuff. * You can then call Transaction#commit to execute your transaction. * * === Parameters * [flags ({})] * A hash with the following keys: * [:nodeps] * Ignore dependency checks. * [:force] * Ignore file conflicts and overwrite files. * [:nosave] * Delete files even if they are tagged as backup. * [:nodepversion] * Ignore version numbers when checking dependencies. * [:cascade] * Remove also any packages depending on a package being removed. * [:recurse] * Remove packages and their unneeded deps (not explicitely installed). * [:dbonly] * Modify database but do not commit changes to the filesystem. * [:alldeps] * Use ALPM_REASON_DEPEND when installing packages. * [:downloadonly] * Only download packages and do not actually install. * [:noscriptlet] * Do not execute install scriptlets after installing. * [:noconflicts] * Ignore dependency conflicts. * [:needed] * Do not install a package if it is already installed and up to date. * [:allexplicit] * Use ALPM_PKG_REASON_EXPLICIT when installing packages. * [:unneeded] * Do not remove a package if it is needed by another one. * [:recurseall] * Remove also explicitely installed unneeded deps (use with :recurse). * [:nolock] * Do not lock the database during the operation. * * === Return value * The result of the block’s last expression. * * === Remarks * Do not store the Transaction instance somewhere; this will give * you grief, because it is a transient object always referring to * the currently active transaction or bomb if there is none. */ static VALUE transaction(int argc, VALUE argv[], VALUE self) { VALUE transaction; VALUE result; alpm_handle_t* p_alpm = NULL; alpm_transflag_t flags = 0; Data_Get_Struct(self, alpm_handle_t, p_alpm); if (argc == 1) { if (TYPE(argv[0]) != T_HASH) rb_raise(rb_eTypeError, "Argument is not a hash."); if (rb_hash_aref(flags, STR2SYM("nodeps"))) flags |= ALPM_TRANS_FLAG_NODEPS; if (rb_hash_aref(flags, STR2SYM("force"))) flags |= ALPM_TRANS_FLAG_FORCE; if (rb_hash_aref(flags, STR2SYM("nosave"))) flags |= ALPM_TRANS_FLAG_NOSAVE; if (rb_hash_aref(flags, STR2SYM("nodepversion"))) flags |= ALPM_TRANS_FLAG_NODEPVERSION; if (rb_hash_aref(flags, STR2SYM("cascade"))) flags |= ALPM_TRANS_FLAG_CASCADE; if (rb_hash_aref(flags, STR2SYM("recurse"))) flags |= ALPM_TRANS_FLAG_RECURSE; if (rb_hash_aref(flags, STR2SYM("dbonly"))) flags |= ALPM_TRANS_FLAG_DBONLY; if (rb_hash_aref(flags, STR2SYM("alldeps"))) flags |= ALPM_TRANS_FLAG_ALLDEPS; if (rb_hash_aref(flags, STR2SYM("downloadonly"))) flags |= ALPM_TRANS_FLAG_DOWNLOADONLY; if (rb_hash_aref(flags, STR2SYM("noscriptlet"))) flags |= ALPM_TRANS_FLAG_NOSCRIPTLET; if (rb_hash_aref(flags, STR2SYM("noconflicts"))) flags |= ALPM_TRANS_FLAG_NOCONFLICTS; if (rb_hash_aref(flags, STR2SYM("needed"))) flags |= ALPM_TRANS_FLAG_NEEDED; if (rb_hash_aref(flags, STR2SYM("allexplicit"))) flags |= ALPM_TRANS_FLAG_ALLEXPLICIT; if (rb_hash_aref(flags, STR2SYM("unneeded"))) flags |= ALPM_TRANS_FLAG_UNNEEDED; if (rb_hash_aref(flags, STR2SYM("recurseall"))) flags |= ALPM_TRANS_FLAG_RECURSEALL; if (rb_hash_aref(flags, STR2SYM("nolock"))) flags |= ALPM_TRANS_FLAG_NOLOCK; } else { rb_raise(rb_eArgError, "Wrong number of arguments, expected 0..1, got %d.", argc); return Qnil; } /* Create the transaction */ if (alpm_trans_init(p_alpm, flags) < 0) return raise_last_alpm_error(p_alpm); /* Create an instance of Transaction. Note that alpm forces * you to only have *one* single Transaction instance, hence * there is no other way to instanciate this class apart from * this method. The user now modify and exute this sole * transaction. */ transaction = rb_obj_alloc(rb_cAlpm_Transaction); rb_iv_set(transaction, "@alpm", self); result = rb_yield(transaction); /* When we get here we assume the user is done with * his stuff. Clean up. */ if (alpm_trans_release(p_alpm) < 0) return raise_last_alpm_error(p_alpm); /* Return the last value from the block */ return result; }
/** Takes a Ruby array of Ruby Symbols and computes the C * alpm_siglevel_t from it. Raises an exception if `ary' * doesn’t respond to #to_ary. */ alpm_siglevel_t siglevel_from_ruby(VALUE ary) { alpm_siglevel_t level = 0; if (!(RTEST(ary = rb_check_array_type(ary)))) { /* Single = intended */ VALUE str = rb_inspect(level); rb_raise(rb_eTypeError, "Not an array (#to_ary): %s", StringValuePtr(str)); return Qnil; } if (rb_ary_includes(ary, STR2SYM("package"))) level |= ALPM_SIG_PACKAGE; if (rb_ary_includes(ary, STR2SYM("package_optional"))) level |= ALPM_SIG_PACKAGE_OPTIONAL; if (rb_ary_includes(ary, STR2SYM("package_marginal_ok"))) level |= ALPM_SIG_PACKAGE_MARGINAL_OK; if (rb_ary_includes(ary, STR2SYM("package_unknown_ok"))) level |= ALPM_SIG_PACKAGE_UNKNOWN_OK; if (rb_ary_includes(ary, STR2SYM("database"))) level |= ALPM_SIG_DATABASE; if (rb_ary_includes(ary, STR2SYM("database_optional"))) level |= ALPM_SIG_DATABASE_OPTIONAL; if (rb_ary_includes(ary, STR2SYM("database_marginal_ok"))) level |= ALPM_SIG_DATABASE_MARGINAL_OK; if (rb_ary_includes(ary, STR2SYM("database_unknown_ok"))) level |= ALPM_SIG_DATABASE_UNKNOWN_OK; if (rb_ary_includes(ary, STR2SYM("package_set"))) level |= ALPM_SIG_PACKAGE_SET; if (rb_ary_includes(ary, STR2SYM("package_trust_set"))) level |= ALPM_SIG_PACKAGE_TRUST_SET; if (rb_ary_includes(ary, STR2SYM("use_default"))) level |= ALPM_SIG_USE_DEFAULT; return level; }