void show_tree_range(struct btree *btree, tuxkey_t start, unsigned count)
{
if(DEBUG_MODE_K==1)
{
printf("\t\t\t\t%25s[K] %25s %4d #in\n",__FILE__,__func__,__LINE__);
}
__tux3_dbg("%i level btree at %Li:\n",
btree->root.depth, btree->root.block);
if (!has_root(btree))
return;
struct cursor *cursor = alloc_cursor(btree, 0);
if (!cursor) {
tux3_err(btree->sb, "out of memory");
return;
}
if (btree_probe(cursor, start)) {
tux3_fs_error(btree->sb, "tell me why!!!");
goto out;
}
struct buffer_head *buffer;
do {
buffer = cursor_leafbuf(cursor);
assert((btree->ops->leaf_sniff)(btree, bufdata(buffer)));
(btree->ops->leaf_dump)(btree, bufdata(buffer));
} while (--count && cursor_advance(cursor));
out:
free_cursor(cursor);
}
int main (int argc, const char * argv[]) {
FILE *fl = fopen(argv[1], "r");
FILE *out = fopen("/tmp/out.json", "w");
fseek(fl, 0, SEEK_END);
int file_size = ftell(fl);
printf("File size: %d\n", file_size);
fseek(fl, 0, SEEK_SET);
int index = 0;
Cursor* cursor = alloc_cursor();
clear_cursor(cursor);
cursor->position = -1;
read_osm_header(cursor, fl);
OsmItem* item;
do {
item = read_osm_item(cursor, fl, file_size);
if (item) {
char* json_item_txt = encode_item(item);
fputs(json_item_txt, out);
free(json_item_txt);
fputs("\n", out);
}
index += 1;
} while (item != NULL);
free_cursor(cursor);
fclose(fl);
fclose(out);
}
int db__driver_execute_immediate(dbString * sql)
{
char *s, msg[OD_MSG];
cursor *c;
SQLRETURN ret;
SQLINTEGER err;
s = db_get_string(sql);
/* allocate cursor */
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
ret = SQLExecDirect(c->stmt, s, SQL_NTS);
if ((ret != SQL_SUCCESS) && (ret != SQL_SUCCESS_WITH_INFO)) {
SQLGetDiagRec(SQL_HANDLE_STMT, c->stmt, 1, NULL, &err, msg,
sizeof(msg), NULL);
db_d_append_error("SQLExecDirect():\n%s\n%s (%d)\n", s, msg,
(int)err);
db_d_report_error();
return DB_FAILED;
}
free_cursor(c);
return DB_OK;
}
int db__driver_open_select_cursor(dbString * sel, dbCursor * dbc, int mode)
{
cursor *c;
dbTable *table;
char *str;
/* allocate cursor */
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
db_set_cursor_mode(dbc, mode);
db_set_cursor_type_readonly(dbc);
/* \ must be escaped, see explanation in
* db_driver_execute_immediate() */
str = G_str_replace(db_get_string(sel), "\\", "\\\\");
G_debug(3, "Escaped SQL: %s", str);
if (mysql_query(connection, str) != 0) {
db_d_append_error("%s\n%s\n%s",
_("Unable to select data:"),
db_get_string(sel),
mysql_error(connection));
if (str)
G_free(str);
db_d_report_error();
return DB_FAILED;
}
if (str)
G_free(str);
c->res = mysql_store_result(connection);
if (c->res == NULL) {
db_d_append_error("%s\n%s",
db_get_string(sel),
mysql_error(connection));
db_d_report_error();
return DB_FAILED;
}
if (describe_table(c->res, &table, c) == DB_FAILED) {
db_d_append_error(_("Unable to describe table."));
db_d_report_error();
mysql_free_result(c->res);
return DB_FAILED;
}
c->nrows = (int)mysql_num_rows(c->res);
/* record table with dbCursor */
db_set_cursor_table(dbc, table);
/* set dbCursor's token for my cursor */
db_set_cursor_token(dbc, c->token);
return DB_OK;
}
void show_tree_range(struct btree *btree, tuxkey_t start, unsigned count)
{
printf("%i level btree at %Li:\n", btree->root.depth, (L)btree->root.block);
struct cursor *cursor = alloc_cursor(btree, 0);
if (!cursor)
error("out of memory");
if (probe(btree, start, cursor))
error("tell me why!!!");
struct buffer_head *buffer;
do {
buffer = cursor_leafbuf(cursor);
assert((btree->ops->leaf_sniff)(btree, bufdata(buffer)));
(btree->ops->leaf_dump)(btree, bufdata(buffer));
//tuxkey_t *next = pnext_key(cursor, btree->depth);
//printf("next key = %Lx:\n", next ? (L)*next : 0);
} while (--count && advance(btree, cursor));
free_cursor(cursor);
}
/*!
\brief Open select cursor
\param sel select statement (given as dbString)
\param[out] dbc pointer to dbCursor
\param mode open mode
\return DB_OK on success
\return DB_FAILED on failure
*/
int db__driver_open_select_cursor(dbString * sel, dbCursor * dbc, int mode)
{
cursor *c;
dbTable *table;
init_error();
/* allocate cursor */
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
db_set_cursor_mode(dbc, mode);
db_set_cursor_type_readonly(dbc);
G_debug(3, "SQL: '%s'", db_get_string(sel));
c->hLayer = OGR_DS_ExecuteSQL(hDs, db_get_string(sel), NULL, NULL);
if (c->hLayer == NULL) {
append_error(_("Unable to select: \n"));
append_error(db_get_string(sel));
append_error("\n");
report_error();
return DB_FAILED;
}
if (describe_table(c->hLayer, &table, c) == DB_FAILED) {
append_error(_("Unable to describe table\n"));
report_error();
OGR_DS_ReleaseResultSet(hDs, c->hLayer);
return DB_FAILED;
}
/* record table with dbCursor */
db_set_cursor_table(dbc, table);
/* set dbCursor's token for my cursor */
db_set_cursor_token(dbc, c->token);
return DB_OK;
}
int db__driver_create_table(dbTable * table)
{
dbString sql;
cursor *c;
char msg[OD_MSG];
char *emsg = NULL;
SQLRETURN ret;
SQLINTEGER err;
G_debug(3, "db__driver_create_table()");
db_init_string(&sql);
db_table_to_sql(table, &sql);
G_debug(3, " SQL: %s", db_get_string(&sql));
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
ret = SQLExecDirect(c->stmt, db_get_string(&sql), SQL_NTS);
if ((ret != SQL_SUCCESS) && (ret != SQL_SUCCESS_WITH_INFO)) {
SQLGetDiagRec(SQL_HANDLE_STMT, c->stmt, 1, NULL, &err, msg,
sizeof(msg), NULL);
G_asprintf(&emsg, "SQLExecDirect():\n%s\n%s (%d)\n",
db_get_string(&sql), msg, (int)err);
report_error(emsg);
G_free(emsg);
return DB_FAILED;
}
free_cursor(c);
return DB_OK;
}
/*
* This is range deletion. So, instead of adjusting balance of the
* space on sibling nodes for each change, this just removes the range
* and merges from right to left even if it is not same parent.
*
* +--------------- (A, B, C)--------------------+
* | | |
* +-- (AA, AB, AC) -+ +- (BA, BB, BC) -+ + (CA, CB, CC) +
* | | | | | | | | |
* (AAA,AAB)(ABA,ABB)(ACA,ACB) (BAA,BAB)(BBA)(BCA,BCB) (CAA)(CBA,CBB)(CCA)
*
* [less : A, AA, AAA, AAB, AB, ABA, ABB, AC, ACA, ACB, B, BA ... : greater]
*
* If we merged from cousin (or re-distributed), we may have to update
* the index until common parent. (e.g. removed (ACB), then merged
* from (BAA,BAB) to (ACA), we have to adjust B in root node to BB)
*
* See, adjust_parent_sep().
*
* FIXME: no re-distribute. so, we don't guarantee above than 50%
* space efficiency. And if range is end of key (truncate() case), we
* don't need to merge, and adjust_parent_sep().
*
* FIXME2: we may want to split chop work for each step. instead of
* blocking for a long time.
*/
int btree_chop(struct btree *btree, tuxkey_t start, u64 len)
{
if(DEBUG_MODE_K==1)
{
printf("\t\t\t\t%25s[K] %25s %4d #in\n",__FILE__,__func__,__LINE__);
}
struct sb *sb = btree->sb;
struct btree_ops *ops = btree->ops;
struct buffer_head **prev, *leafprev = NULL;
struct chopped_index_info *cii;
struct cursor *cursor;
tuxkey_t limit;
int ret, done = 0;
if (!has_root(btree))
return 0;
/* Chop all range if len >= TUXKEY_LIMIT */
limit = (len >= TUXKEY_LIMIT) ? TUXKEY_LIMIT : start + len;
prev = malloc(sizeof(*prev) * btree->root.depth);
if (prev == NULL)
return -ENOMEM;
memset(prev, 0, sizeof(*prev) * btree->root.depth);
cii = malloc(sizeof(*cii) * btree->root.depth);
if (cii == NULL) {
ret = -ENOMEM;
goto error_cii;
}
memset(cii, 0, sizeof(*cii) * btree->root.depth);
cursor = alloc_cursor(btree, 0);
if (!cursor) {
ret = -ENOMEM;
goto error_alloc_cursor;
}
down_write(&btree->lock);
ret = btree_probe(cursor, start);
if (ret)
goto error_btree_probe;
/* Walk leaves */
while (1) {
struct buffer_head *leafbuf;
tuxkey_t this_key;
/*
* FIXME: If leaf was merged and freed later, we don't
* need to redirect leaf and leaf_chop()
*/
if ((ret = cursor_redirect(cursor)))
goto out;
leafbuf = cursor_pop(cursor);
/* Adjust start and len for this leaf */
this_key = cursor_level_this_key(cursor);
if (start < this_key) {
if (limit < TUXKEY_LIMIT)
len -= this_key - start;
start = this_key;
}
ret = ops->leaf_chop(btree, start, len, bufdata(leafbuf));
if (ret) {
if (ret < 0) {
blockput(leafbuf);
goto out;
}
mark_buffer_dirty_non(leafbuf);
}
/* Try to merge this leaf with prev */
if (leafprev) {
if (try_leaf_merge(btree, leafprev, leafbuf)) {
trace(">>> can merge leaf %p into leaf %p", leafbuf, leafprev);
remove_index(cursor, cii);
mark_buffer_dirty_non(leafprev);
blockput_free(sb, leafbuf);
goto keep_prev_leaf;
}
blockput(leafprev);
}
leafprev = leafbuf;
keep_prev_leaf:
if (cursor_level_next_key(cursor) >= limit)
done = 1;
/* Pop and try to merge finished nodes */
while (done || cursor_level_finished(cursor)) {
struct buffer_head *buf;
int level = cursor->level;
struct chopped_index_info *ciil = &cii[level];
/* Get merge src buffer, and go parent level */
buf = cursor_pop(cursor);
/*
* Logging chopped indexes
* FIXME: If node is freed later (e.g. merged),
* we dont't need to log this
*/
if (ciil->count) {
log_bnode_del(sb, bufindex(buf), ciil->start,
ciil->count);
}
memset(ciil, 0, sizeof(*ciil));
/* Try to merge node with prev */
if (prev[level]) {
assert(level);
if (try_bnode_merge(sb, prev[level], buf)) {
trace(">>> can merge node %p into node %p", buf, prev[level]);
remove_index(cursor, cii);
mark_buffer_unify_non(prev[level]);
blockput_free_unify(sb, buf);
goto keep_prev_node;
}
blockput(prev[level]);
}
prev[level] = buf;
keep_prev_node:
if (!level)
goto chop_root;
}
/* Push back down to leaf level */
do {
ret = cursor_advance_down(cursor);
if (ret < 0)
goto out;
} while (ret);
}
chop_root:
/* Remove depth if possible */
while (btree->root.depth > 1 && bcount(bufdata(prev[0])) == 1) {
trace("drop btree level");
btree->root.block = bufindex(prev[1]);
btree->root.depth--;
tux3_mark_btree_dirty(btree);
/*
* We know prev[0] is redirected and dirty. So, in
* here, we can just cancel bnode_redirect by bfree(),
* instead of defered_bfree()
* FIXME: we can optimize freeing bnode without
* bnode_redirect, and if we did, this is not true.
*/
bfree(sb, bufindex(prev[0]), 1);
log_bnode_free(sb, bufindex(prev[0]));
blockput_free_unify(sb, prev[0]);
vecmove(prev, prev + 1, btree->root.depth);
}
ret = 0;
out:
if (leafprev)
blockput(leafprev);
for (int i = 0; i < btree->root.depth; i++) {
if (prev[i])
blockput(prev[i]);
}
release_cursor(cursor);
error_btree_probe:
up_write(&btree->lock);
free_cursor(cursor);
error_alloc_cursor:
free(cii);
error_cii:
free(prev);
return ret;
}
int db__driver_open_select_cursor(dbString * sel, dbCursor * dbc, int mode)
{
PGresult *res;
cursor *c;
dbTable *table;
char *str;
/* Set datetime style */
res = PQexec(pg_conn, "SET DATESTYLE TO ISO");
if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) {
db_d_append_error(_("Unable set DATESTYLE"));
db_d_report_error();
PQclear(res);
return DB_FAILED;
}
PQclear(res);
/* allocate cursor */
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
db_set_cursor_mode(dbc, mode);
db_set_cursor_type_readonly(dbc);
/* \ must be escaped, see explanation in db_driver_execute_immediate() */
str = G_str_replace(db_get_string(sel), "\\", "\\\\");
G_debug(3, "Escaped SQL: %s", str);
c->res = PQexec(pg_conn, str);
if (!c->res || PQresultStatus(c->res) != PGRES_TUPLES_OK) {
db_d_append_error("%s\n%s\n%s",
_("Unable to select:"),
db_get_string(sel),
PQerrorMessage(pg_conn));
db_d_report_error();
PQclear(c->res);
if (str)
G_free(str);
return DB_FAILED;
}
if (str)
G_free(str);
if (describe_table(c->res, &table, c) == DB_FAILED) {
db_d_append_error(_("Unable to describe table"));
db_d_report_error();
PQclear(res);
return DB_FAILED;
}
c->nrows = PQntuples(c->res);
c->row = -1;
/* record table with dbCursor */
db_set_cursor_table(dbc, table);
/* set dbCursor's token for my cursor */
db_set_cursor_token(dbc, c->token);
return DB_OK;
}
int db__driver_drop_table(dbString * name)
{
char cmd[200];
cursor *c;
SQLRETURN ret;
char msg[OD_MSG];
char *emsg = NULL;
SQLINTEGER err;
SQLCHAR ttype[50], *tname;
SQLINTEGER nrow = 0;
/* allocate cursor */
c = alloc_cursor();
if (c == NULL)
return DB_FAILED;
tname = db_get_string(name);
ret = SQLTables(c->stmt, NULL, 0, NULL, 0, tname, sizeof(tname), NULL, 0);
if ((ret != SQL_SUCCESS) && (ret != SQL_SUCCESS_WITH_INFO)) {
report_error("SQLTables()");
return DB_FAILED;
}
/* Get number of rows */
ret = SQLRowCount(c->stmt, &nrow);
if ((ret != SQL_SUCCESS) && (ret != SQL_SUCCESS_WITH_INFO)) {
report_error("SQLRowCount()");
return DB_FAILED;
}
if (nrow == 0) {
G_asprintf(&emsg, "Table %s doesn't exist\n", tname);
report_error(emsg);
G_free(emsg);
return DB_FAILED;
}
ret = SQLFetchScroll(c->stmt, SQL_FETCH_NEXT, 0);
ret = SQLGetData(c->stmt, 4, SQL_C_CHAR, ttype, sizeof(ttype), NULL);
if (strcmp(ttype, "TABLE") == 0) {
sprintf(cmd, "DROP TABLE %s", tname);
}
else if (strcmp(ttype, "VIEW") == 0) {
sprintf(cmd, "DROP VIEW %s", tname);
}
else {
G_asprintf(&emsg, "Table %s isn't 'TABLE' or 'VIEW' but %s\n", tname,
ttype);
report_error(emsg);
G_free(emsg);
return DB_FAILED;
}
SQLCloseCursor(c->stmt);
ret = SQLExecDirect(c->stmt, cmd, SQL_NTS);
if ((ret != SQL_SUCCESS) && (ret != SQL_SUCCESS_WITH_INFO)) {
SQLGetDiagRec(SQL_HANDLE_STMT, c->stmt, 1, NULL, &err, msg,
sizeof(msg), NULL);
G_asprintf(&emsg, "SQLExecDirect():\n%s\n%s (%d)\n", cmd, msg,
(int)err);
report_error(emsg);
G_free(emsg);
return DB_FAILED;
}
free_cursor(c);
return DB_OK;
}
int tree_chop(struct btree *btree, struct delete_info *info, millisecond_t deadline)
{
int depth = btree->root.depth, level = depth - 1, suspend = 0;
struct cursor *cursor;
struct buffer_head *leafbuf, **prev, *leafprev = NULL;
struct btree_ops *ops = btree->ops;
struct sb *sb = btree->sb;
int ret;
cursor = alloc_cursor(btree, 0);
prev = malloc(sizeof(*prev) * depth);
memset(prev, 0, sizeof(*prev) * depth);
down_write(&btree->lock);
probe(btree, info->resume, cursor);
leafbuf = level_pop(cursor);
/* leaf walk */
while (1) {
ret = (ops->leaf_chop)(btree, info->key, bufdata(leafbuf));
if (ret) {
mark_buffer_dirty(leafbuf);
if (ret < 0)
goto error_leaf_chop;
}
/* try to merge this leaf with prev */
if (leafprev) {
struct vleaf *this = bufdata(leafbuf);
struct vleaf *that = bufdata(leafprev);
/* try to merge leaf with prev */
if ((ops->leaf_need)(btree, this) <= (ops->leaf_free)(btree, that)) {
trace(">>> can merge leaf %p into leaf %p", leafbuf, leafprev);
(ops->leaf_merge)(btree, that, this);
remove_index(cursor, level);
mark_buffer_dirty(leafprev);
brelse_free(btree, leafbuf);
//dirty_buffer_count_check(sb);
goto keep_prev_leaf;
}
brelse(leafprev);
}
leafprev = leafbuf;
keep_prev_leaf:
//nanosleep(&(struct timespec){ 0, 50 * 1000000 }, NULL);
//printf("time remaining: %Lx\n", deadline - gettime());
// if (deadline && gettime() > deadline)
// suspend = -1;
if (info->blocks && info->freed >= info->blocks)
suspend = -1;
/* pop and try to merge finished nodes */
while (suspend || level_finished(cursor, level)) {
/* try to merge node with prev */
if (prev[level]) {
assert(level); /* node has no prev */
struct bnode *this = cursor_node(cursor, level);
struct bnode *that = bufdata(prev[level]);
trace_off("check node %p against %p", this, that);
trace_off("this count = %i prev count = %i", bcount(this), bcount(that));
/* try to merge with node to left */
if (bcount(this) <= sb->entries_per_node - bcount(that)) {
trace(">>> can merge node %p into node %p", this, that);
merge_nodes(that, this);
remove_index(cursor, level - 1);
mark_buffer_dirty(prev[level]);
brelse_free(btree, level_pop(cursor));
//dirty_buffer_count_check(sb);
goto keep_prev_node;
}
brelse(prev[level]);
}
prev[level] = level_pop(cursor);
keep_prev_node:
/* deepest key in the cursor is the resume address */
if (suspend == -1 && !level_finished(cursor, level)) {
suspend = 1; /* only set resume once */
info->resume = from_be_u64((cursor->path[level].next)->key);
}
if (!level) { /* remove depth if possible */
while (depth > 1 && bcount(bufdata(prev[0])) == 1) {
trace("drop btree level");
btree->root.block = bufindex(prev[1]);
mark_btree_dirty(btree);
brelse_free(btree, prev[0]);
//dirty_buffer_count_check(sb);
depth = --btree->root.depth;
vecmove(prev, prev + 1, depth);
//set_sb_dirty(sb);
}
//sb->snapmask &= ~snapmask; delete_snapshot_from_disk();
//set_sb_dirty(sb);
//save_sb(sb);
ret = suspend;
goto out;
}
level--;
trace_off(printf("pop to level %i, block %Lx, %i of %i nodes\n", level, bufindex(cursor->path[level].buffer), cursor->path[level].next - cursor_node(cursor, level)->entries, bcount(cursor_node(cursor, level))););
}
/* push back down to leaf level */
while (level < depth - 1) {
struct buffer_head *buffer = sb_bread(vfs_sb(sb), from_be_u64(cursor->path[level++].next++->block));
if (!buffer) {
ret = -EIO;
goto out;
}
level_push(cursor, buffer, ((struct bnode *)bufdata(buffer))->entries);
trace_off(printf("push to level %i, block %Lx, %i nodes\n", level, bufindex(buffer), bcount(cursor_node(cursor, level))););
}
