static int
pollinbuf_blob_cb(struct pollfd_state *pfd){
pollinbuf *pibuf = get_pfd_icap(pfd)->pibuf;
typeof(pibuf->modestate.chunkdata) *chunk;
blob_read_res res;
uint32_t remains;
chunk = &pibuf->modestate.chunkdata;
remains = chunk->chunklen - chunk->chunkread;
res = read_blob(pfd->pfd.fd,&pibuf->cr,chunk->okey,&chunk->dumpoff,&remains);
if(res == BLOB_READ_SYSERR){
bitch("Error while reading %ub for chunk\n",remains);
return -1;
}
chunk->chunkread += chunk->chunklen - chunk->chunkread - remains;
if(res == BLOB_READ_NBLOCK){
return 0;
}
if(chunk->cb(pfd)){
return -1;
}
// For edge-triggered RX, we want to return 1 until a read() actually
// returns -1 + EAGAIN. For level-triggered, return 0 on
// BLOB_READ_SUCCESS. We're either that or BLOB_READ_MOREDATA.
return 1;
}
void
QcOsmPbfReader::read_file(bool read_nodes, bool read_ways, bool read_relations, bool read_metadatas)
{
m_read_nodes = read_nodes;
m_read_ways = read_ways;
m_read_relations = read_relations;
m_read_metadatas = read_metadatas;
m_number_of_primitive_groups = 0;
m_number_of_node_primitive_groups = 0;
m_number_of_dense_node_primitive_groups = 0;
m_number_of_way_primitive_groups = 0;
m_number_of_relation_primitive_groups = 0;
m_number_of_nodes = 0;
m_number_of_ways = 0;
m_number_of_relations = 0;
m_buffer = new char[OSMPBF::max_uncompressed_blob_size];
m_unpack_buffer = new char[OSMPBF::max_uncompressed_blob_size];
QFile file(m_pbf_path);
if (!file.open(QIODevice::ReadOnly)) {
qCritical() << "can't open file" << m_pbf_path;
return ;
}
m_data_stream.setDevice(&file);
// Set network byte-order
m_data_stream.setByteOrder(QDataStream::BigEndian);
// read while the file has not reached its end
while (!file.atEnd()) {
/* A file contains a header followed by a sequence of fileblocks.
*
* The design is intended to allow future random-access to the
* contents of the file and skipping past not-understood or
* unwanted data.
*
* The format is a repeating sequence of:
* - int4: length of the BlobHeader message in network byte order
* - serialized BlobHeader message
* - serialized Blob message (size is given in the header)
*/
read_blob_header();
read_blob();
}
// clean up the protobuf lib
google::protobuf::ShutdownProtobufLibrary();
qDebug() << "File Statistics\n"
<< "Primitive Groups" << m_number_of_primitive_groups << "\n"
<< " for Node" << m_number_of_node_primitive_groups << "\n"
<< " for Dense Node" << m_number_of_dense_node_primitive_groups << "\n"
<< " for Way" << m_number_of_way_primitive_groups << "\n"
<< " for Relation" << m_number_of_relation_primitive_groups << "\n"
<< "Number of nodes" << m_number_of_nodes << "\n"
<< "Number of ways" << m_number_of_ways << "\n"
<< "Number of relations" << m_number_of_relations;
}
static VALUE parse_osm_data(VALUE obj)
{
FILE *input = DATA_PTR(obj);
OSMPBF__BlobHeader *header = read_blob_header(input);
if(header == NULL)
return Qfalse;
if(strcmp("OSMData", header->type) != 0)
rb_raise(rb_eIOError, "OSMData not found");
void *blob = NULL;
size_t blob_length = 0, datasize = header->datasize;
OSMPBF__PrimitiveBlock *primitive_block = NULL;
osmpbf__blob_header__free_unpacked(header, NULL);
blob = read_blob(input, datasize, &blob_length);
primitive_block = osmpbf__primitive_block__unpack(NULL, blob_length, blob);
free(blob);
if(primitive_block == NULL)
rb_raise(rb_eIOError, "Unable to unpack the PrimitiveBlock");
int64_t lat_offset, lon_offset, granularity;
int32_t ts_granularity;
lat_offset = primitive_block->lat_offset;
lon_offset = primitive_block->lon_offset;
granularity = primitive_block->granularity;
ts_granularity = primitive_block->date_granularity;
OSMPBF__StringTable *string_table = primitive_block->stringtable;
VALUE data = init_data_arr();
VALUE nodes = rb_hash_aref(data, STR2SYM("nodes"));
VALUE ways = rb_hash_aref(data, STR2SYM("ways"));
VALUE relations = rb_hash_aref(data, STR2SYM("relations"));
size_t i = 0;
for(i = 0; i < primitive_block->n_primitivegroup; i++)
{
OSMPBF__PrimitiveGroup *primitive_group = primitive_block->primitivegroup[i];
if(primitive_group->nodes)
process_nodes(nodes, primitive_group, string_table, lat_offset, lon_offset, granularity, ts_granularity);
if(primitive_group->dense)
process_dense_nodes(nodes, primitive_group->dense, string_table, lat_offset, lon_offset, granularity, ts_granularity);
if(primitive_group->ways)
process_ways(ways, primitive_group, string_table, ts_granularity);
if(primitive_group->relations)
process_relations(relations, primitive_group, string_table, ts_granularity);
}
rb_iv_set(obj, "@data", data);
osmpbf__primitive_block__free_unpacked(primitive_block, NULL);
// Increment position
rb_iv_set(obj, "@pos", INT2NUM(NUM2INT(rb_iv_get(obj, "@pos")) + 1));
return Qtrue;
}
static int parse_osm_header(VALUE obj, FILE *input)
{
OSMPBF__BlobHeader *header = read_blob_header(input);
// EOF reached
if(header == NULL)
rb_raise(rb_eEOFError, "EOF reached without finding data");
if(strcmp("OSMHeader", header->type) != 0)
rb_raise(rb_eIOError, "OSMHeader not found, probably the file is corrupt or invalid");
void *blob = NULL;
size_t blob_length = 0, datasize = header->datasize;
OSMPBF__HeaderBlock *header_block = NULL;
osmpbf__blob_header__free_unpacked(header, NULL);
blob = read_blob(input, datasize, &blob_length);
header_block = osmpbf__header_block__unpack(NULL, blob_length, blob);
free(blob);
if(header_block == NULL)
rb_raise(rb_eIOError, "Unable to unpack the HeaderBlock");
VALUE header_hash = rb_hash_new();
VALUE bbox_hash = rb_hash_new();
VALUE required_features = Qnil;
VALUE optional_features = Qnil;
VALUE writingprogram = Qnil;
VALUE source = Qnil;
VALUE osmosis_replication_timestamp = Qnil;
VALUE osmosis_replication_sequence_number = Qnil;
VALUE osmosis_replication_base_url = Qnil;
int i = 0;
if(header_block->n_required_features > 0)
{
required_features = rb_ary_new();
for(i = 0; i < (int)header_block->n_required_features; i++)
rb_ary_push(required_features, str_new(header_block->required_features[i]));
}
if(header_block->n_optional_features > 0)
{
optional_features = rb_ary_new();
for(i = 0; i < (int)header_block->n_optional_features; i++)
rb_ary_push(optional_features, str_new(header_block->optional_features[i]));
}
if(header_block->writingprogram)
writingprogram = str_new(header_block->writingprogram);
if(header_block->source)
source = str_new(header_block->source);
if(header_block->bbox)
{
rb_hash_aset(bbox_hash, STR2SYM("top"), rb_float_new(header_block->bbox->top * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("right"), rb_float_new(header_block->bbox->right * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("bottom"), rb_float_new(header_block->bbox->bottom * NANO_DEGREE));
rb_hash_aset(bbox_hash, STR2SYM("left"), rb_float_new(header_block->bbox->left * NANO_DEGREE));
}
if(header_block->has_osmosis_replication_timestamp)
osmosis_replication_timestamp = ULL2NUM(header_block->osmosis_replication_timestamp);
if(header_block->has_osmosis_replication_sequence_number)
osmosis_replication_sequence_number = ULL2NUM(header_block->osmosis_replication_sequence_number);
if(header_block->osmosis_replication_base_url)
osmosis_replication_base_url = str_new(header_block->osmosis_replication_base_url);
rb_hash_aset(header_hash, str_new("bbox"), bbox_hash);
rb_hash_aset(header_hash, str_new("required_features"), required_features);
rb_hash_aset(header_hash, str_new("optional_features"), optional_features);
rb_hash_aset(header_hash, str_new("writing_program"), writingprogram);
rb_hash_aset(header_hash, str_new("source"), source);
rb_hash_aset(header_hash, str_new("osmosis_replication_timestamp"), osmosis_replication_timestamp);
rb_hash_aset(header_hash, str_new("osmosis_replication_sequence_number"), osmosis_replication_sequence_number);
rb_hash_aset(header_hash, str_new("osmosis_replication_base_url"), osmosis_replication_base_url);
rb_iv_set(obj, "@header", header_hash);
osmpbf__header_block__free_unpacked(header_block, NULL);
return 1;
}
/* This function converts the database format from version 3 to version 4.
* This function sets the KMO error string. It returns -1 on failure.
*/
static int maildb_convert_to_version_4(sqlite3 *db) {
sqlite3_stmt *stmt1 = NULL;
sqlite3_stmt *stmt2 = NULL;
kstr kstr_array[10];
int k;
for (k = 0; k < 10; k++) kstr_init(kstr_array + k);
begin_transaction(db);
/* Create the new table and update the maildb version. */
if (sqlite3_exec(db, "CREATE TABLE 'mail_eval_res3' "
"("
" 'entry_id' INTEGER PRIMARY KEY,"
" 'hash' BLOB (20),"
" 'ksn' BLOB (24),"
" 'status' INTEGER,"
" 'display_pref' TINYINT(1),"
" 'sig_msg' VARCHAR,"
" 'mid' INTEGER(20),"
" 'original_packaging' TINYINT(1),"
" 'mua' SMALLINT(3),"
" 'field_status' INTEGER(2),"
" 'att_plugin_nbr' TINYINT(1),"
" 'attachment_nbr' TINYINT(1),"
" 'attachment_status' BLOB,"
" 'sym_key' BLOB,"
" 'encryption_status' TINYINT(1),"
" 'decryption_error_msg' VARCHAR,"
" 'pod_status' TINYINT(1),"
" 'pod_msg' VARCHAR,"
" 'otut_status' TINYINT(1),"
" 'otut_string' BLOB,"
" 'otut_msg' VARCHAR,"
" 'kpg_addr' VARCHAR,"
" 'kpg_port' INTEGER(5)"
");"
"UPDATE maildb_version SET version = 4;", NULL, NULL, NULL)) goto ERR;
/* Transfer the data from mail_eval_res to mail_eval_res2. */
if (sqlite3_prepare(db, "SELECT entry_id, hash, ksn, status, display_pref, "
"sig_msg, mid, original_packaging, mua, field_status, "
"att_plugin_nbr, attachment_nbr, attachment_status, sym_key, encryption_status, "
"decryption_error_msg, pod_status, pod_msg, otut_status, otut_string, "
"otut_msg FROM mail_eval_res2;", -1, &stmt1, NULL)) goto ERR;
while (1) {
int i = 0;
int j = 1;
k = 0;
int res = sqlite3_step(stmt1);
if (res == SQLITE_DONE) break;
if (res != SQLITE_ROW) goto ERR;
if (sqlite3_prepare(db, "INSERT INTO mail_eval_res3 ("
"entry_id, hash, ksn, status, display_pref, sig_msg, mid, original_packaging, mua, "
"field_status, att_plugin_nbr, attachment_nbr, attachment_status, sym_key, "
"encryption_status, decryption_error_msg, pod_status, pod_msg, otut_status, "
"otut_string, otut_msg, kpg_addr, kpg_port) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
-1, &stmt2, NULL)) goto ERR;
if (sqlite3_bind_int64(stmt2, j++, sqlite3_column_int64(stmt1, i++))) goto ERR;
read_blob(stmt1, &kstr_array[k], i++); if (write_blob(stmt2, j++, &kstr_array[k])) goto ERR; k++;
read_blob(stmt1, &kstr_array[k], i++); if (write_blob(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
read_string(stmt1, &kstr_array[k], i++); if (write_string(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int64(stmt2, j++, sqlite3_column_int64(stmt1, i++))) goto ERR;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
if (sqlite3_bind_int64(stmt2, j++, sqlite3_column_int64(stmt1, i++))) goto ERR;
if (sqlite3_bind_int64(stmt2, j++, sqlite3_column_int64(stmt1, i++))) goto ERR;
if (sqlite3_bind_int64(stmt2, j++, sqlite3_column_int64(stmt1, i++))) goto ERR;
read_blob(stmt1, &kstr_array[k], i++); if (write_blob(stmt2, j++, &kstr_array[k])) goto ERR; k++;
read_blob(stmt1, &kstr_array[k], i++); if (write_blob(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
read_string(stmt1, &kstr_array[k], i++); if (write_string(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
read_string(stmt1, &kstr_array[k], i++); if (write_string(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int(stmt2, j++, sqlite3_column_int(stmt1, i++))) goto ERR;
read_blob(stmt1, &kstr_array[k], i++); if (write_blob(stmt2, j++, &kstr_array[k])) goto ERR; k++;
read_string(stmt1, &kstr_array[k], i++); if (write_string(stmt2, j++, &kstr_array[k])) goto ERR; k++;
/* Add KPG address and port. */
kstr_assign_cstr(&kstr_array[k], ""); if (write_string(stmt2, j++, &kstr_array[k])) goto ERR; k++;
if (sqlite3_bind_int(stmt2, j++, 0)) goto ERR;
if (sqlite3_step(stmt2) != SQLITE_DONE) goto ERR;
finalize_stmt(db, &stmt2);
}
finalize_stmt(db, &stmt1);
/* Delete the mail_eval_res table and commit. */
if (sqlite3_exec(db, "DROP TABLE mail_eval_res2; COMMIT;", NULL, NULL, NULL)) goto ERR;
/* Success. */
for (k = 0; k < 10; k++) kstr_free(kstr_array + k);
finalize_stmt(db, &stmt1);
finalize_stmt(db, &stmt2);
return 0;
ERR:
kmo_seterror(sqlite3_errmsg(db));
for (k = 0; k < 10; k++) kstr_free(kstr_array + k);
finalize_stmt(db, &stmt1);
finalize_stmt(db, &stmt2);
rollback_transaction(db);
return -1;
}
/* This function returns the mail info having the entry ID specified.
* This function sets the KMO error string. It returns -1 on general failure,
* -2 if not found.
*/
static int maildb_sqlite_get_mail_info_from_entry_id(maildb *mdb, maildb_mail_info *mail_info, int64_t entry_id) {
sqlite3 *db = (sqlite3 *) mdb->db;
sqlite3_stmt *stmt = NULL;
int i = 0;
/* Clear the mail info. */
maildb_clear_mail_info(mail_info);
/* No such mail. */
if (entry_id == 0) {
return -2;
}
/* Not a Kryptiva mail. */
if (entry_id == -1) {
mail_info->status = 3;
return 0;
}
assert(entry_id > 0);
/* It's a Kryptiva mail. Read the mail_eval_res info. */
if (sqlite3_prepare (db,
"SELECT hash, ksn, status, display_pref, sig_msg, mid, original_packaging, mua, field_status, "
"att_plugin_nbr, attachment_nbr, attachment_status, sym_key, encryption_status, "
"decryption_error_msg, pod_status, pod_msg, otut_status, otut_string, "
"otut_msg, kpg_addr, kpg_port "
"FROM mail_eval_res3 WHERE mail_eval_res3.entry_id = ?;",
-1, &stmt, NULL)) goto ERR;
if (sqlite3_bind_int64(stmt, 1, entry_id)) goto ERR;
if (sqlite3_step(stmt) != SQLITE_ROW) goto ERR;
i = 0;
read_blob(stmt, &mail_info->hash, i++);
read_blob(stmt, &mail_info->ksn, i++);
mail_info->status = sqlite3_column_int(stmt, i++);
mail_info->display_pref = sqlite3_column_int(stmt, i++);
read_string(stmt, &mail_info->sig_msg, i++);
mail_info->mid = sqlite3_column_int64(stmt, i++);
mail_info->original_packaging = sqlite3_column_int(stmt, i++);
mail_info->mua = sqlite3_column_int(stmt, i++);
mail_info->field_status = sqlite3_column_int(stmt, i++);
mail_info->att_plugin_nbr = sqlite3_column_int(stmt, i++);
mail_info->attachment_nbr = sqlite3_column_int(stmt, i++);
read_blob(stmt, &mail_info->attachment_status, i++);
read_blob(stmt, &mail_info->sym_key, i++);
mail_info->encryption_status = sqlite3_column_int(stmt, i++);
read_string(stmt, &mail_info->decryption_error_msg, i++);
mail_info->pod_status = sqlite3_column_int(stmt, i++);
read_string(stmt, &mail_info->pod_msg, i++);
mail_info->otut_status = sqlite3_column_int(stmt, i++);
read_blob(stmt, &mail_info->otut_string, i++);
read_string(stmt, &mail_info->otut_msg, i++);
read_string(stmt, &mail_info->kpg_addr, i++);
mail_info->kpg_port = sqlite3_column_int(stmt, i++);
if (sqlite3_step(stmt) != SQLITE_DONE) goto ERR;
finalize_stmt(db, &stmt);
/* Set the entry_id field of the mail_info object. */
mail_info->entry_id = entry_id;
return 0;
ERR:
kmo_seterror(sqlite3_errmsg(db));
finalize_stmt(db, &stmt);
return -1;
}
void transform(int deviceNumber, int levels, unsigned w, unsigned h, unsigned bits)
{
uint64_t cbinput=uint64_t(w)*bits/8;
std::vector<uint64_t> input(2*(cbinput/8));
std::vector<uint64_t> output(2*(cbinput/8));
std::vector<uint32_t> unpackedInput(2*(cbinput/4));
std::vector<uint32_t> unpackedOutput(2*(cbinput/4));
uint64_t* inputWriteptr = &input[0];
uint64_t* inputReadptr = &input[cbinput/8];
uint32_t* unpackedInputReadptr = &unpackedInput[cbinput/4];
uint32_t* unpackedInputWriteptr = &unpackedInput[0];
uint32_t* unpackedOutputReadptr = &unpackedOutput[cbinput/4];
uint32_t* unpackedOutputWriteptr = &unpackedOutput[0];
uint64_t* outputWriteptr = &output[0];
uint64_t* outputReadptr = &output[cbinput/8];
std::vector<uint32_t> gpuReadOffsets(2*levels+1,0);
std::vector<uint32_t> gpuWriteOffsets(2*levels+1,0);
std::vector<uint32_t> aboveOverrides(2*levels,0);
std::vector<uint32_t> belowOverrides(2*levels,0);
int j=0;
for (int i=0; i<2*levels+1; i++)
{
gpuWriteOffsets[i] = j;
j+= 2;
j = j - (4 & -(j >= 4));
j = j + (4 & -(j < 0));
gpuReadOffsets[i] = j;
}
auto cl_instance = init_cl(levels,w,h,bits,"pipeline_kernels.cl",deviceNumber);
tbb::task_group group;
bool finished = false;
int fullness = 0;
bool full = false;
int tailEnd = 4*levels+5;
int image_line = 0;
while(1){
for (int i=0; i<levels; i++) {
if (image_line == 4 + 2*i) aboveOverrides[i] = 0x0 ^ -(levels < 0);
else aboveOverrides[i] = 0xFFFFFFFF ^ -(levels < 0);
if (image_line == 3 + 2*i) belowOverrides[i] = 0x0 ^ -(levels < 0);
else belowOverrides[i] = 0xFFFFFFFF ^ -(levels < 0);
}
for (int i=levels; i<2*levels; i++) {
if (image_line == 4 + 2*i) aboveOverrides[i] = 0xFFFFFFFF ^ -(levels < 0);
else aboveOverrides[i] = 0x0 ^ -(levels < 0);
if (image_line == 3 + 2*i) belowOverrides[i] = 0xFFFFFFFF ^ -(levels < 0);
else belowOverrides[i] = 0x0 ^ -(levels < 0);
}
group.run([&](){
if(!finished && !read_blob(STDIN_FILENO, cbinput, inputWriteptr)) finished = true;
unpack_blob_32(cbinput, inputReadptr, unpackedInputWriteptr);
pack_blob_32(cbinput, unpackedOutputReadptr, outputWriteptr);
if (fullness >= 4*levels+6 || full)
{
full = true;
write_blob(STDOUT_FILENO, cbinput, outputReadptr);
}
else
{
fullness++;
}
});
group.run([&](){
process_opencl_packed_line(levels, w, bits, gpuReadOffsets, gpuWriteOffsets, unpackedInputReadptr, unpackedOutputWriteptr, aboveOverrides, belowOverrides, cl_instance);
for (int i=0; i<2*levels+1; i++)
{
int j = gpuWriteOffsets[i];
j++;
j = j - (4 & -(j >= 4));
gpuWriteOffsets[i] = j;
j = gpuReadOffsets[i];
j++;
j = j - (4 & -(j >= 4));
gpuReadOffsets[i] = j;
}
});
group.wait();
if (tailEnd == 0) {
break;
}
if (finished) tailEnd--;
std::swap(inputReadptr, inputWriteptr);
std::swap(unpackedInputReadptr, unpackedInputWriteptr);
std::swap(unpackedOutputReadptr, unpackedOutputWriteptr);
std::swap(outputReadptr, outputWriteptr);
image_line++;
if (image_line == h) image_line = 0;
}
}