static void
sv_mergeiter_merge_dup_a_chain(void)
{
stlist vlista;
stlist vlistb;
st_listinit(&vlista, 0);
st_listinit(&vlistb, 0);
int key = 7;
int i = 0;
int lsn = 5;
while (i < 5)
{
st_sv(&st_r.g, &vlista, lsn, 0 | ((i > 0) ? SVDUP: 0), key);
i++;
lsn--;
}
ssiter ita;
ss_iterinit(ss_bufiterref, &ita);
ss_iteropen(ss_bufiterref, &ita, &vlista.list, sizeof(sv*));
ssiter itb;
ss_iterinit(ss_bufiterref, &itb);
ss_iteropen(ss_bufiterref, &itb, &vlistb.list, sizeof(sv*));
svmerge m;
sv_mergeinit(&m);
sv_mergeprepare(&m, &st_r.r, 2);
svmergesrc *s = sv_mergeadd(&m, NULL);
t(s != NULL);
s->src = ita;
s = sv_mergeadd(&m, NULL);
t(s != NULL);
s->src = itb;
ssiter merge;
ss_iterinit(sv_mergeiter, &merge);
ss_iteropen(sv_mergeiter, &merge, &st_r.r, &m, SS_GTE);
i = 0;
while (ss_iteratorhas(&merge)) {
sv *v = (sv*)ss_iteratorof(&merge);
t( *(int*)sv_key(v, &st_r.r, 0) == key );
if (i == 0) {
t( sv_flags(v) == 0 );
} else {
t( (sv_flags(v) | sv_mergeisdup(&merge)) == (0|SVDUP) );
}
ss_iteratornext(&merge);
i++;
}
t( i == 5 );
ss_iteratorclose(&merge);
sv_mergefree(&m, &st_r.a);
st_listfree(&vlista, &st_r.r);
st_listfree(&vlistb, &st_r.r);
}
static void
sv_mergeiter_merge_ba(void)
{
stlist vlista;
stlist vlistb;
st_listinit(&vlista, 0);
st_listinit(&vlistb, 0);
int i = 0;
while (i < 5)
{
st_sv(&st_r.g, &vlista, i, 0, i);
i++;
}
while (i < 10)
{
st_sv(&st_r.g, &vlistb, i, 0, i);
i++;
}
ssiter ita;
ss_iterinit(ss_bufiterref, &ita);
ss_iteropen(ss_bufiterref, &ita, &vlista.list, sizeof(sv*));
ssiter itb;
ss_iterinit(ss_bufiterref, &itb);
ss_iteropen(ss_bufiterref, &itb, &vlistb.list, sizeof(sv*));
svmerge m;
sv_mergeinit(&m);
sv_mergeprepare(&m, &st_r.r, 3);
svmergesrc *s = sv_mergeadd(&m, NULL);
t(s != NULL);
s->src = ita;
s = sv_mergeadd(&m, NULL);
t(s != NULL);
s->src = itb;
ssiter merge;
ss_iterinit(sv_mergeiter, &merge);
ss_iteropen(sv_mergeiter, &merge, &st_r.r, &m, SS_GTE);
i = 0;
while (ss_iteratorhas(&merge)) {
sv *v = (sv*)ss_iteratorof(&merge);
t( *(int*)sv_key(v, &st_r.r, 0) == i );
t( sv_lsn(v) == i );
t( sv_flags(v) == 0 );
ss_iteratornext(&merge);
i++;
}
t( i == 10 );
ss_iteratorclose(&merge);
sv_mergefree(&m, &st_r.a);
st_listfree(&vlista, &st_r.r);
st_listfree(&vlistb, &st_r.r);
}
int si_compact(si *index, sr *r, sdc *c, siplan *plan, uint64_t vlsn)
{
sinode *node = plan->node;
assert(node->flags & SI_LOCK);
/* read node file */
sd_creset(c);
int rc = si_noderead(r, &c->c, node);
if (ssunlikely(rc == -1))
return -1;
/* prepare for compaction */
rc = sd_censure(c, r, node->branch_count);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "%s", "memory allocation failed");
return -1;
}
svmerge merge;
sv_mergeinit(&merge);
rc = sv_mergeprepare(&merge, r, node->branch_count);
if (ssunlikely(rc == -1))
return -1;
uint32_t size_stream = 0;
sdcbuf *cbuf = c->head;
sibranch *b = node->branch;
while (b) {
svmergessc *s = sv_mergeadd(&merge, NULL);
rc = ss_bufensure(&cbuf->b, r->a, b->index.h->sizevmax);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "%s", "memory allocation failed");
return -1;
}
size_stream += sd_indextotal(&b->index);
ss_iterinit(sd_iter, &s->ssc);
ss_iteropen(sd_iter, &s->ssc, r, &b->index, c->c.s, 0,
index->scheme->compression, &cbuf->a, &cbuf->b);
cbuf = cbuf->next;
b = b->next;
}
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, r, &merge, SS_GTE);
rc = si_compaction(index, r, c, vlsn, node, &i, size_stream);
if (ssunlikely(rc == -1)) {
sv_mergefree(&merge, r->a);
return -1;
}
sv_mergefree(&merge, r->a);
return 0;
}
static inline sibranch*
si_branchcreate(si *index, sdc *c, sinode *parent, svindex *vindex, uint64_t vlsn)
{
sr *r = index->r;
svmerge vmerge;
sv_mergeinit(&vmerge);
int rc = sv_mergeprepare(&vmerge, r, 1);
if (ssunlikely(rc == -1))
return NULL;
svmergesrc *s = sv_mergeadd(&vmerge, NULL);
ss_iterinit(sv_indexiterraw, &s->src);
ss_iteropen(sv_indexiterraw, &s->src, vindex);
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, r, &vmerge, SS_GTE, 1);
/* merge iter is not used */
sdmergeconf mergeconf = {
.size_stream = UINT32_MAX,
.size_node = UINT64_MAX,
.size_page = index->scheme->node_page_size,
.checksum = index->scheme->node_page_checksum,
.compression = index->scheme->compression,
.compression_key = index->scheme->compression_key,
.offset = parent->file.size,
.vlsn = vlsn,
.save_delete = 1
};
sdmerge merge;
sd_mergeinit(&merge, r, &i, &c->build, &mergeconf);
rc = sd_merge(&merge);
if (ssunlikely(rc == -1)) {
sv_mergefree(&vmerge, r->a);
sr_oom_malfunction(r->e);
goto error;
}
assert(rc == 1);
sv_mergefree(&vmerge, r->a);
sibranch *branch = si_branchnew(r);
if (ssunlikely(branch == NULL))
goto error;
sdid id = {
.parent = parent->self.id.id,
.flags = SD_IDBRANCH,
.id = sr_seq(r->seq, SR_NSNNEXT)
};
rc = sd_mergecommit(&merge, &id);
if (ssunlikely(rc == -1))
goto error;
si_branchset(branch, &merge.index);
rc = sd_commit(&c->build, r, &branch->index, &parent->file);
if (ssunlikely(rc == -1)) {
si_branchfree(branch, r);
return NULL;
}
SS_INJECTION(r->i, SS_INJECTION_SI_BRANCH_0,
sr_malfunction(r->e, "%s", "error injection");
si_branchfree(branch, r);
return NULL);
if (index->scheme->sync) {
rc = si_nodesync(parent, r);
if (ssunlikely(rc == -1)) {
si_branchfree(branch, r);
return NULL;
}
}
if (index->scheme->mmap) {
ss_mmapinit(&parent->map_swap);
rc = ss_mmap(&parent->map_swap, parent->file.fd,
parent->file.size, 1);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "db file '%s' mmap error: %s",
parent->file.file, strerror(errno));
return NULL;
}
}
return branch;
error:
sd_mergefree(&merge);
return NULL;
}
int si_branch(si *index, sdc *c, siplan *plan, uint64_t vlsn)
{
sr *r = index->r;
sinode *n = plan->node;
assert(n->flags & SI_LOCK);
si_lock(index);
if (ssunlikely(n->used == 0)) {
si_nodeunlock(n);
si_unlock(index);
return 0;
}
svindex *i;
i = si_noderotate(n);
si_unlock(index);
sd_creset(c);
sibranch *branch = si_branchcreate(index, c, n, i, vlsn);
if (ssunlikely(branch == NULL))
return -1;
/* commit */
si_lock(index);
branch->next = n->branch;
n->branch = branch;
n->branch_count++;
uint32_t used = sv_indexused(i);
n->used -= used;
ss_quota(r->quota, SS_QREMOVE, used);
svindex swap = *i;
si_nodeunrotate(n);
si_nodeunlock(n);
si_plannerupdate(&index->p, SI_BRANCH|SI_COMPACT, n);
ssmmap swap_map = n->map;
n->map = n->map_swap;
memset(&n->map_swap, 0, sizeof(n->map_swap));
si_unlock(index);
/* gc */
if (index->scheme->mmap) {
int rc = ss_munmap(&swap_map);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "db file '%s' munmap error: %s",
n->file.file, strerror(errno));
return -1;
}
}
si_nodegc_index(r, &swap);
return 1;
}
static inline char*
si_noderead(si *index, ssbuf *dest, sinode *node)
{
sr *r = index->r;
if (index->scheme->mmap) {
return node->map.p;
}
int rc = ss_bufensure(dest, r->a, node->file.size);
if (ssunlikely(rc == -1)) {
sr_oom_malfunction(r->e);
return NULL;
}
rc = ss_filepread(&node->file, 0, dest->s, node->file.size);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "db file '%s' read error: %s",
node->file.file, strerror(errno));
return NULL;
}
ss_bufadvance(dest, node->file.size);
return dest->s;
}
int si_compaction(si *index, sdc *c, siplan *plan, uint64_t vlsn)
{
sr *r = &index->r;
sinode *node = plan->node;
assert(node->flags & SI_LOCK);
si_lock(index);
svindex *vindex;
vindex = si_noderotate(node);
si_unlock(index);
uint64_t size_stream = vindex->used;
ssiter vindex_iter;
ss_iterinit(sv_indexiter, &vindex_iter);
ss_iteropen(sv_indexiter, &vindex_iter, &index->r, vindex, SS_GTE, NULL);
/* prepare direct_io stream */
int rc;
if (index->scheme.direct_io) {
rc = sd_ioprepare(&c->io, r,
index->scheme.direct_io,
index->scheme.direct_io_page_size,
index->scheme.direct_io_buffer_size);
if (ssunlikely(rc == -1))
return sr_oom(r->e);
}
/* prepare for compaction */
svmerge merge;
sv_mergeinit(&merge);
rc = sv_mergeprepare(&merge, r, 1 + 1);
if (ssunlikely(rc == -1))
return -1;
svmergesrc *s;
s = sv_mergeadd(&merge, &vindex_iter);
sdcbuf *cbuf = &c->e;
s = sv_mergeadd(&merge, NULL);
sdreadarg arg = {
.from_compaction = 1,
.io = &c->io,
.index = &node->index,
.buf = &cbuf->a,
.buf_read = &c->d,
.index_iter = &cbuf->index_iter,
.page_iter = &cbuf->page_iter,
.use_mmap = index->scheme.mmap,
.use_mmap_copy = 0,
.use_compression = index->scheme.compression,
.use_direct_io = index->scheme.direct_io,
.direct_io_page_size = index->scheme.direct_io_page_size,
.compression_if = index->scheme.compression_if,
.has = 0,
.has_vlsn = 0,
.o = SS_GTE,
.mmap = &node->map,
.file = &node->file,
.r = r
};
ss_iterinit(sd_read, &s->src);
rc = ss_iteropen(sd_read, &s->src, &arg, NULL);
if (ssunlikely(rc == -1))
return -1;
size_stream += sd_indextotal(&node->index);
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, r, &merge, SS_GTE);
rc = si_merge(index, c, node, vlsn, &i, size_stream,
sd_indexkeys(&node->index));
sv_mergefree(&merge, r->a);
return rc;
}
static inline int
si_rangebranch(siread *q, sinode *n, sibranch *b, svmerge *m)
{
sicachebranch *c = si_cachefollow(q->cache);
assert(c->branch == b);
/* iterate cache */
if (ss_iterhas(sd_read, &c->i)) {
svmergesrc *s = sv_mergeadd(m, &c->i);
si_readstat(q, 1, n, 1);
s->ptr = c;
return 1;
}
if (c->open) {
return 1;
}
if (q->cache_only) {
return 2;
}
c->open = 1;
/* choose compression type */
int compression;
ssfilterif *compression_if;
if (! si_branchis_root(b)) {
compression = q->index->scheme->compression_branch;
compression_if = q->index->scheme->compression_branch_if;
} else {
compression = q->index->scheme->compression;
compression_if = q->index->scheme->compression_if;
}
sdreadarg arg = {
.index = &b->index,
.buf = &c->buf_a,
.buf_xf = &c->buf_b,
.buf_read = &q->index->readbuf,
.index_iter = &c->index_iter,
.page_iter = &c->page_iter,
.use_memory = n->in_memory,
.use_mmap = q->index->scheme->mmap,
.use_mmap_copy = 1,
.use_compression = compression,
.compression_if = compression_if,
.has = 0,
.has_vlsn = 0,
.o = q->order,
.memory = &b->copy,
.mmap = &n->map,
.file = &n->file,
.r = q->r
};
ss_iterinit(sd_read, &c->i);
int rc = ss_iteropen(sd_read, &c->i, &arg, q->key, q->keysize);
int reads = sd_read_stat(&c->i);
si_readstat(q, 0, n, reads);
if (ssunlikely(rc == -1))
return -1;
if (ssunlikely(! ss_iterhas(sd_read, &c->i)))
return 0;
svmergesrc *s = sv_mergeadd(m, &c->i);
s->ptr = c;
return 1;
}
static inline int
si_range(siread *q)
{
assert(q->has == 0);
ssiter i;
ss_iterinit(si_iter, &i);
ss_iteropen(si_iter, &i, q->r, q->index, q->order, q->key, q->keysize);
sinode *node;
next_node:
node = ss_iterof(si_iter, &i);
if (ssunlikely(node == NULL))
return 0;
si_txtrack(q->x, node);
/* prepare sources */
svmerge *m = &q->merge;
int count = node->branch_count + 2 + 1;
int rc = sv_mergeprepare(m, q->r, count);
if (ssunlikely(rc == -1)) {
sr_errorreset(q->r->e);
return -1;
}
/* external source (upsert) */
svmergesrc *s;
sv upbuf_reserve;
ssbuf upbuf;
if (ssunlikely(q->upsert_v && q->upsert_v->v)) {
ss_bufinit_reserve(&upbuf, &upbuf_reserve, sizeof(upbuf_reserve));
ss_bufadd(&upbuf, NULL, (void*)&q->upsert_v, sizeof(sv*));
s = sv_mergeadd(m, NULL);
ss_iterinit(ss_bufiterref, &s->src);
ss_iteropen(ss_bufiterref, &s->src, &upbuf, sizeof(sv*));
}
/* in-memory indexes */
svindex *second;
svindex *first = si_nodeindex_priority(node, &second);
if (first->count) {
s = sv_mergeadd(m, NULL);
ss_iterinit(sv_indexiter, &s->src);
ss_iteropen(sv_indexiter, &s->src, q->r, first, q->order,
q->key, q->keysize);
}
if (ssunlikely(second && second->count)) {
s = sv_mergeadd(m, NULL);
ss_iterinit(sv_indexiter, &s->src);
ss_iteropen(sv_indexiter, &s->src, q->r, second, q->order,
q->key, q->keysize);
}
/* cache and branches */
rc = si_cachevalidate(q->cache, node);
if (ssunlikely(rc == -1)) {
sr_oom(q->r->e);
return -1;
}
sibranch *b = node->branch;
while (b) {
rc = si_rangebranch(q, node, b, m);
if (ssunlikely(rc == -1 || rc == 2))
return rc;
b = b->next;
}
/* merge and filter data stream */
ssiter j;
ss_iterinit(sv_mergeiter, &j);
ss_iteropen(sv_mergeiter, &j, q->r, m, q->order);
ssiter k;
ss_iterinit(sv_readiter, &k);
ss_iteropen(sv_readiter, &k, q->r, &j, &q->index->u, q->vlsn, 0);
sv *v = ss_iterof(sv_readiter, &k);
if (ssunlikely(v == NULL)) {
sv_mergereset(&q->merge);
ss_iternext(si_iter, &i);
goto next_node;
}
rc = 1;
/* convert upsert search to SS_EQ */
if (q->upsert_eq) {
rc = sr_compare(q->r->scheme, sv_pointer(v), sv_size(v),
q->key, q->keysize);
rc = rc == 0;
}
/* do prefix search */
if (q->prefix && rc) {
rc = sr_compareprefix(q->r->scheme, q->prefix, q->prefixsize,
sv_pointer(v),
sv_size(v));
}
if (sslikely(rc == 1)) {
if (ssunlikely(si_readdup(q, v) == -1))
return -1;
}
/* skip a possible duplicates from data sources */
ss_iternext(sv_readiter, &k);
return rc;
}
static inline int
si_getbranch(siread *q, sinode *n, sibranch *b)
{
sicachebranch *c = si_cachefollow(q->cache);
assert(c->branch == b);
/* choose compression type */
int compression;
ssfilterif *compression_if;
if (! si_branchis_root(b)) {
compression = q->index->scheme->compression_branch;
compression_if = q->index->scheme->compression_branch_if;
} else {
compression = q->index->scheme->compression;
compression_if = q->index->scheme->compression_if;
}
sdreadarg arg = {
.index = &b->index,
.buf = &c->buf_a,
.buf_xf = &c->buf_b,
.buf_read = &q->index->readbuf,
.index_iter = &c->index_iter,
.page_iter = &c->page_iter,
.use_memory = n->in_memory,
.use_mmap = q->index->scheme->mmap,
.use_mmap_copy = 0,
.use_compression = compression,
.compression_if = compression_if,
.has = q->has,
.has_vlsn = q->vlsn,
.o = SS_GTE,
.mmap = &n->map,
.memory = &b->copy,
.file = &n->file,
.r = q->r
};
ss_iterinit(sd_read, &c->i);
int rc = ss_iteropen(sd_read, &c->i, &arg, q->key, q->keysize);
int reads = sd_read_stat(&c->i);
si_readstat(q, 0, n, reads);
if (ssunlikely(rc <= 0))
return rc;
/* prepare sources */
sv_mergereset(&q->merge);
sv_mergeadd(&q->merge, &c->i);
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, q->r, &q->merge, SS_GTE);
uint64_t vlsn = q->vlsn;
if (ssunlikely(q->has))
vlsn = UINT64_MAX;
ssiter j;
ss_iterinit(sv_readiter, &j);
ss_iteropen(sv_readiter, &j, q->r, &i, &q->index->u, vlsn, 1);
sv *v = ss_iterof(sv_readiter, &j);
if (ssunlikely(v == NULL))
return 0;
return si_getresult(q, v, 1);
}
static inline int
si_get(siread *q)
{
ssiter i;
ss_iterinit(si_iter, &i);
ss_iteropen(si_iter, &i, q->r, q->index, SS_GTE, q->key, q->keysize);
sinode *node;
node = ss_iterof(si_iter, &i);
assert(node != NULL);
si_txtrack(q->x, node);
/* search in memory */
int rc;
rc = si_getindex(q, node);
if (rc != 0)
return rc;
if (q->cache_only)
return 2;
/* */
rc = si_cachevalidate(q->cache, node);
if (ssunlikely(rc == -1)) {
sr_oom(q->r->e);
return -1;
}
svmerge *m = &q->merge;
rc = sv_mergeprepare(m, q->r, 1);
assert(rc == 0);
/* search on disk */
sibranch *b = node->branch;
while (b) {
rc = si_getbranch(q, node, b);
if (rc != 0)
return rc;
b = b->next;
}
return 0;
}
static inline void
si_qrangebranch(siquery *q, sinode *n, sibranch *b, svmerge *m)
{
sicachebranch *cb = si_cachefollow(q->cache);
assert(cb->branch == b);
/* iterate cache */
if (ss_iterhas(si_read, &cb->i)) {
svmergesrc *s = sv_mergeadd(m, &cb->i);
q->index->read_cache++;
s->ptr = cb;
return;
}
if (cb->open) {
return;
}
cb->open = 1;
sireadarg arg = {
.scheme = q->index->scheme,
.index = q->index,
.n = n,
.b = b,
.buf = &cb->buf_a,
.buf_xf = &cb->buf_b,
.buf_read = &q->index->readbuf,
.index_iter = &cb->index_iter,
.page_iter = &cb->page_iter,
.vlsn = q->vlsn,
.has = 0,
.mmap_copy = 1,
.o = q->order,
.r = q->r
};
ss_iterinit(si_read, &cb->i);
int rc = ss_iteropen(si_read, &cb->i, &arg, q->key, q->keysize);
if (ssunlikely(rc == -1))
return;
if (ssunlikely(! ss_iterhas(si_read, &cb->i)))
return;
svmergesrc *s = sv_mergeadd(m, &cb->i);
s->ptr = cb;
}
static inline int
si_qrange(siquery *q)
{
ssiter i;
ss_iterinit(si_iter, &i);
ss_iteropen(si_iter, &i, q->r, q->index, q->order, q->key, q->keysize);
sinode *node;
next_node:
node = ss_iterof(si_iter, &i);
if (ssunlikely(node == NULL))
return 0;
/* prepare sources */
svmerge *m = &q->merge;
int count = node->branch_count + 2 + 1;
int rc = sv_mergeprepare(m, q->r, count);
if (ssunlikely(rc == -1)) {
sr_errorreset(q->r->e);
return -1;
}
/* external source (update) */
svmergesrc *s;
sv upbuf_reserve;
ssbuf upbuf;
if (ssunlikely(q->update_v && q->update_v->v)) {
ss_bufinit_reserve(&upbuf, &upbuf_reserve, sizeof(upbuf_reserve));
ss_bufadd(&upbuf, NULL, (void*)&q->update_v, sizeof(sv*));
s = sv_mergeadd(m, NULL);
ss_iterinit(ss_bufiterref, &s->src);
ss_iteropen(ss_bufiterref, &s->src, &upbuf, sizeof(sv*));
}
/* in-memory indexes */
svindex *second;
svindex *first = si_nodeindex_priority(node, &second);
if (first->count) {
s = sv_mergeadd(m, NULL);
ss_iterinit(sv_indexiter, &s->src);
ss_iteropen(sv_indexiter, &s->src, q->r, first, q->order,
q->key, q->keysize);
}
if (ssunlikely(second && second->count)) {
s = sv_mergeadd(m, NULL);
ss_iterinit(sv_indexiter, &s->src);
ss_iteropen(sv_indexiter, &s->src, q->r, second, q->order,
q->key, q->keysize);
}
/* cache and branches */
rc = si_cachevalidate(q->cache, node);
if (ssunlikely(rc == -1)) {
sr_oom(q->r->e);
return -1;
}
sibranch *b = node->branch;
while (b) {
si_qrangebranch(q, node, b, m);
b = b->next;
}
/* merge and filter data stream */
ssiter j;
ss_iterinit(sv_mergeiter, &j);
ss_iteropen(sv_mergeiter, &j, q->r, m, q->order);
ssiter k;
ss_iterinit(sv_readiter, &k);
ss_iteropen(sv_readiter, &k, q->r, &j, &q->index->u, q->vlsn, 0);
sv *v = ss_iterof(sv_readiter, &k);
if (ssunlikely(v == NULL)) {
sv_mergereset(&q->merge);
ss_iternext(si_iter, &i);
goto next_node;
}
rc = 1;
/* convert update search to SS_EQ */
if (q->update_eq) {
rc = sr_compare(q->r->scheme, sv_pointer(v), sv_size(v),
q->key, q->keysize);
rc = rc == 0;
}
/* do prefix search */
if (q->prefix && rc) {
rc = sr_compareprefix(q->r->scheme, q->prefix, q->prefixsize,
sv_pointer(v),
sv_size(v));
}
if (sslikely(rc == 1)) {
if (ssunlikely(si_querydup(q, v) == -1))
return -1;
}
/* skip a possible duplicates from data sources */
ss_iternext(sv_readiter, &k);
return rc;
}
static inline int
si_qgetbranch(siquery *q, sinode *n, sibranch *b)
{
sicachebranch *cb = si_cachefollow(q->cache);
assert(cb->branch == b);
sireadarg arg = {
.scheme = q->index->scheme,
.index = q->index,
.n = n,
.b = b,
.buf = &cb->buf_a,
.buf_xf = &cb->buf_b,
.buf_read = &q->index->readbuf,
.index_iter = &cb->index_iter,
.page_iter = &cb->page_iter,
.vlsn = q->vlsn,
.has = q->has,
.mmap_copy = 0,
.o = SS_GTE,
.r = q->r
};
ss_iterinit(si_read, &cb->i);
int rc = ss_iteropen(si_read, &cb->i, &arg, q->key, q->keysize);
if (ssunlikely(rc <= 0))
return rc;
uint64_t vlsn = q->vlsn;
if (q->has) {
vlsn = UINT64_MAX;
}
/* prepare sources */
sv_mergereset(&q->merge);
sv_mergeadd(&q->merge, &cb->i);
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, q->r, &q->merge, SS_GTE);
ssiter j;
ss_iterinit(sv_readiter, &j);
ss_iteropen(sv_readiter, &j, q->r, &i, &q->index->u, vlsn, 1);
sv *v = ss_iterof(sv_readiter, &j);
if (ssunlikely(v == NULL))
return 0;
return si_qgetresult(q, v, 1);
}
static inline int
si_qget(siquery *q)
{
ssiter i;
ss_iterinit(si_iter, &i);
ss_iteropen(si_iter, &i, q->r, q->index, SS_GTE, q->key, q->keysize);
sinode *node;
node = ss_iterof(si_iter, &i);
assert(node != NULL);
/* search in memory */
int rc;
rc = si_qgetindex(q, node);
if (rc != 0)
return rc;
/* */
rc = si_cachevalidate(q->cache, node);
if (ssunlikely(rc == -1)) {
sr_oom(q->r->e);
return -1;
}
svmerge *m = &q->merge;
rc = sv_mergeprepare(m, q->r, 1);
assert(rc == 0);
/* search on disk */
sibranch *b = node->branch;
while (b) {
rc = si_qgetbranch(q, node, b);
if (rc != 0)
return rc;
b = b->next;
}
return 0;
}
static inline sibranch*
si_branchcreate(si *index, sdc *c, sinode *parent, svindex *vindex, uint64_t vlsn)
{
sr *r = index->r;
sibranch *branch = NULL;
/* in-memory mode blob */
int rc;
ssblob copy, *blob = NULL;
if (parent->in_memory) {
ss_blobinit(©, r->vfs);
rc = ss_blobensure(©, 10ULL * 1024 * 1024);
if (ssunlikely(rc == -1)) {
sr_oom_malfunction(r->e);
return NULL;
}
blob = ©
}
svmerge vmerge;
sv_mergeinit(&vmerge);
rc = sv_mergeprepare(&vmerge, r, 1);
if (ssunlikely(rc == -1))
return NULL;
svmergesrc *s = sv_mergeadd(&vmerge, NULL);
ss_iterinit(sv_indexiter, &s->src);
ss_iteropen(sv_indexiter, &s->src, r, vindex, SS_GTE, NULL, 0);
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, r, &vmerge, SS_GTE);
/* merge iter is not used */
sdmergeconf mergeconf = {
.size_stream = UINT32_MAX,
.size_node = UINT64_MAX,
.size_page = index->scheme->node_page_size,
.checksum = index->scheme->node_page_checksum,
.compression_key = index->scheme->compression_key,
.compression = index->scheme->compression_branch,
.compression_if = index->scheme->compression_branch_if,
.vlsn = vlsn,
.vlsn_lru = 0,
.save_delete = 1,
.save_upsert = 1
};
sdmerge merge;
sd_mergeinit(&merge, r, &i, &c->build, &c->upsert, &mergeconf);
while ((rc = sd_merge(&merge)) > 0)
{
assert(branch == NULL);
/* write open seal */
uint64_t seal = parent->file.size;
rc = sd_writeseal(r, &parent->file, blob);
if (ssunlikely(rc == -1))
goto e0;
/* write pages */
uint64_t offset = parent->file.size;
while ((rc = sd_mergepage(&merge, offset)) == 1)
{
rc = sd_writepage(r, &parent->file, blob, merge.build);
if (ssunlikely(rc == -1))
goto e0;
offset = parent->file.size;
}
if (ssunlikely(rc == -1))
goto e0;
sdid id = {
.parent = parent->self.id.id,
.flags = SD_IDBRANCH,
.id = sr_seq(r->seq, SR_NSNNEXT)
};
rc = sd_mergecommit(&merge, &id, parent->file.size);
if (ssunlikely(rc == -1))
goto e0;
/* write index */
rc = sd_writeindex(r, &parent->file, blob, &merge.index);
if (ssunlikely(rc == -1))
goto e0;
if (index->scheme->sync) {
rc = ss_filesync(&parent->file);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "file '%s' sync error: %s",
ss_pathof(&parent->file.path),
strerror(errno));
goto e0;
}
}
SS_INJECTION(r->i, SS_INJECTION_SI_BRANCH_0,
sd_mergefree(&merge);
sr_malfunction(r->e, "%s", "error injection");
return NULL);
/* seal the branch */
rc = sd_seal(r, &parent->file, blob, &merge.index, seal);
if (ssunlikely(rc == -1))
goto e0;
if (index->scheme->sync == 2) {
rc = ss_filesync(&parent->file);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "file '%s' sync error: %s",
ss_pathof(&parent->file.path),
strerror(errno));
goto e0;
}
}
/* create new branch object */
branch = si_branchnew(r);
if (ssunlikely(branch == NULL))
goto e0;
si_branchset(branch, &merge.index);
}
sv_mergefree(&vmerge, r->a);
if (ssunlikely(rc == -1)) {
sr_oom_malfunction(r->e);
goto e0;
}
assert(branch != NULL);
/* in-memory mode support */
if (blob) {
rc = ss_blobfit(blob);
if (ssunlikely(rc == -1)) {
ss_blobfree(blob);
goto e1;
}
branch->copy = copy;
}
/* mmap support */
if (index->scheme->mmap) {
ss_mmapinit(&parent->map_swap);
rc = ss_vfsmmap(r->vfs, &parent->map_swap, parent->file.fd,
parent->file.size, 1);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "db file '%s' mmap error: %s",
ss_pathof(&parent->file.path),
strerror(errno));
goto e1;
}
}
return branch;
e0:
sd_mergefree(&merge);
if (blob)
ss_blobfree(blob);
return NULL;
e1:
si_branchfree(branch, r);
return NULL;
}
int si_branch(si *index, sdc *c, siplan *plan, uint64_t vlsn)
{
sr *r = index->r;
sinode *n = plan->node;
assert(n->flags & SI_LOCK);
si_lock(index);
if (ssunlikely(n->used == 0)) {
si_nodeunlock(n);
si_unlock(index);
return 0;
}
svindex *i;
i = si_noderotate(n);
si_unlock(index);
sibranch *branch = si_branchcreate(index, c, n, i, vlsn);
if (ssunlikely(branch == NULL))
return -1;
/* commit */
si_lock(index);
branch->next = n->branch;
n->branch->link = branch;
n->branch = branch;
n->branch_count++;
uint32_t used = sv_indexused(i);
n->used -= used;
ss_quota(r->quota, SS_QREMOVE, used);
index->size +=
sd_indexsize(branch->index.h) +
sd_indextotal(&branch->index);
svindex swap = *i;
si_nodeunrotate(n);
si_nodeunlock(n);
si_plannerupdate(&index->p, SI_BRANCH|SI_COMPACT, n);
ssmmap swap_map = n->map;
n->map = n->map_swap;
memset(&n->map_swap, 0, sizeof(n->map_swap));
si_unlock(index);
/* gc */
if (index->scheme->mmap) {
int rc = ss_vfsmunmap(r->vfs, &swap_map);
if (ssunlikely(rc == -1)) {
sr_malfunction(r->e, "db file '%s' munmap error: %s",
ss_pathof(&n->file.path),
strerror(errno));
return -1;
}
}
si_nodegc_index(r, &swap);
return 1;
}
int si_compact(si *index, sdc *c, siplan *plan,
uint64_t vlsn,
uint64_t vlsn_lru,
ssiter *vindex,
uint64_t vindex_used)
{
sr *r = index->r;
sinode *node = plan->node;
assert(node->flags & SI_LOCK);
/* prepare for compaction */
int rc;
rc = sd_censure(c, r, node->branch_count);
if (ssunlikely(rc == -1))
return sr_oom_malfunction(r->e);
svmerge merge;
sv_mergeinit(&merge);
rc = sv_mergeprepare(&merge, r, node->branch_count + 1);
if (ssunlikely(rc == -1))
return -1;
/* read node file into memory */
int use_mmap = index->scheme->mmap;
ssmmap *map = &node->map;
ssmmap preload;
if (index->scheme->node_compact_load) {
rc = si_noderead(node, r, &c->c);
if (ssunlikely(rc == -1))
return -1;
preload.p = c->c.s;
preload.size = ss_bufused(&c->c);
map = &preload;
use_mmap = 1;
}
/* include vindex into merge process */
svmergesrc *s;
uint64_t size_stream = 0;
if (vindex) {
s = sv_mergeadd(&merge, vindex);
size_stream = vindex_used;
}
sdcbuf *cbuf = c->head;
sibranch *b = node->branch;
while (b) {
s = sv_mergeadd(&merge, NULL);
/* choose compression type */
int compression;
ssfilterif *compression_if;
if (! si_branchis_root(b)) {
compression = index->scheme->compression_branch;
compression_if = index->scheme->compression_branch_if;
} else {
compression = index->scheme->compression;
compression_if = index->scheme->compression_if;
}
sdreadarg arg = {
.index = &b->index,
.buf = &cbuf->a,
.buf_xf = &cbuf->b,
.buf_read = &c->d,
.index_iter = &cbuf->index_iter,
.page_iter = &cbuf->page_iter,
.use_memory = node->in_memory,
.use_mmap = use_mmap,
.use_mmap_copy = 0,
.use_compression = compression,
.compression_if = compression_if,
.has = 0,
.has_vlsn = 0,
.o = SS_GTE,
.memory = &b->copy,
.mmap = map,
.file = &node->file,
.r = r
};
ss_iterinit(sd_read, &s->src);
int rc = ss_iteropen(sd_read, &s->src, &arg, NULL, 0);
if (ssunlikely(rc == -1))
return sr_oom_malfunction(r->e);
size_stream += sd_indextotal(&b->index);
cbuf = cbuf->next;
b = b->next;
}
ssiter i;
ss_iterinit(sv_mergeiter, &i);
ss_iteropen(sv_mergeiter, &i, r, &merge, SS_GTE);
rc = si_merge(index, c, node, vlsn, vlsn_lru, &i, size_stream);
sv_mergefree(&merge, r->a);
return rc;
}
int si_compact_index(si *index, sdc *c, siplan *plan,
uint64_t vlsn,
uint64_t vlsn_lru)
{
sinode *node = plan->node;
si_lock(index);
if (ssunlikely(node->used == 0)) {
si_nodeunlock(node);
si_unlock(index);
return 0;
}
svindex *vindex;
vindex = si_noderotate(node);
si_unlock(index);
uint64_t size_stream = sv_indexused(vindex);
ssiter i;
ss_iterinit(sv_indexiter, &i);
ss_iteropen(sv_indexiter, &i, index->r, vindex, SS_GTE, NULL, 0);
return si_compact(index, c, plan, vlsn, vlsn_lru, &i, size_stream);
}
