static inline int si_recovercomplete(sitrack *track, sr *r, si *index, ssbuf *buf) { /* prepare and build primary index */ ss_bufreset(buf); ssrbnode *p = ss_rbmin(&track->i); while (p) { sinode *n = sscast(p, sinode, node); int rc = ss_bufadd(buf, r->a, &n, sizeof(sinode*)); if (ssunlikely(rc == -1)) return sr_oom_malfunction(r->e); p = ss_rbnext(&track->i, p); } ssiter i; ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, buf, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { sinode *n = ss_iterof(ss_bufiterref, &i); if (n->recover & SI_RDB_REMOVE) { int rc = si_nodefree(n, r, 1); if (ssunlikely(rc == -1)) return -1; ss_iternext(ss_bufiterref, &i); continue; } n->recover = SI_RDB; si_insert(index, n); si_plannerupdate(&index->p, SI_COMPACT|SI_BRANCH|SI_TEMP, n); ss_iternext(ss_bufiterref, &i); } return 0; }
static int si_splitfree(ssbuf *result, sr *r) { ssiter i; ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { sinode *p = ss_iterof(ss_bufiterref, &i); si_nodefree(p, r, 0); ss_iternext(ss_bufiterref, &i); } return 0; }
static inline int si_tracksnapshot(sitrack *track, sr *r, si *i, sdsnapshot *s) { /* read snapshot */ ssiter iter; ss_iterinit(sd_snapshotiter, &iter); int rc; rc = ss_iteropen(sd_snapshotiter, &iter, r, s); if (ssunlikely(rc == -1)) return -1; for (; ss_iterhas(sd_snapshotiter, &iter); ss_iternext(sd_snapshotiter, &iter)) { sdsnapshotnode *n = ss_iterof(sd_snapshotiter, &iter); /* skip updated nodes */ sspath path; ss_path(&path, i->scheme->path, n->id, ".db"); rc = ss_vfsexists(r->vfs, path.path); if (! rc) continue; uint64_t size = ss_vfssize(r->vfs, path.path); if (size != n->size_file) continue; /* recover node */ sinode *node = si_nodenew(r); if (ssunlikely(node == NULL)) return -1; node->recover = SI_RDB; rc = si_nodeopen(node, r, i->scheme, &path, n); if (ssunlikely(rc == -1)) { si_nodefree(node, r, 0); return -1; } si_trackmetrics(track, node); si_trackset(track, node); } /* recover index temperature (read stats) */ sdsnapshotheader *h = sd_snapshot_header(s); i->read_cache = h->read_cache; i->read_disk = h->read_disk; i->lru_v = h->lru_v; i->lru_steps = h->lru_steps; return 0; }
sinode *si_bootstrap(si *i, uint64_t parent) { sr *r = i->r; /* create node */ sinode *n = si_nodenew(r); if (ssunlikely(n == NULL)) return NULL; sdid id = { .parent = parent, .flags = 0, .id = sr_seq(r->seq, SR_NSNNEXT) }; int rc; rc = si_nodecreate(n, r, i->scheme, &id); if (ssunlikely(rc == -1)) goto e0; n->branch = &n->self; n->branch_count++; /* in-memory mode support */ ssblob *blob = NULL; if (i->scheme->storage == SI_SIN_MEMORY) { blob = &n->self.copy; rc = ss_blobensure(blob, 4096); if (ssunlikely(rc == -1)) goto e0; n->in_memory = 1; } /* create index with one empty page */ sdindex index; sd_indexinit(&index); rc = sd_indexbegin(&index, r); if (ssunlikely(rc == -1)) goto e0; ssqf f, *qf = NULL; ss_qfinit(&f); sdbuild build; sd_buildinit(&build); rc = sd_buildbegin(&build, r, i->scheme->node_page_checksum, i->scheme->compression_key, i->scheme->compression, i->scheme->compression_if); if (ssunlikely(rc == -1)) goto e1; sd_buildend(&build, r); rc = sd_indexadd(&index, r, &build, sizeof(sdseal)); if (ssunlikely(rc == -1)) goto e1; /* write seal */ uint64_t seal = n->file.size; rc = sd_writeseal(r, &n->file, blob); if (ssunlikely(rc == -1)) goto e1; /* write page */ rc = sd_writepage(r, &n->file, blob, &build); if (ssunlikely(rc == -1)) goto e1; /* amqf */ if (i->scheme->amqf) { rc = ss_qfensure(&f, r->a, 0); if (ssunlikely(rc == -1)) goto e1; qf = &f; } rc = sd_indexcommit(&index, r, &id, qf, n->file.size); if (ssunlikely(rc == -1)) goto e1; ss_qffree(&f, r->a); /* write index */ rc = sd_writeindex(r, &n->file, blob, &index); if (ssunlikely(rc == -1)) goto e1; /* close seal */ rc = sd_seal(r, &n->file, blob, &index, seal); if (ssunlikely(rc == -1)) goto e1; if (blob) { rc = ss_blobfit(blob); if (ssunlikely(rc == -1)) goto e1; } if (i->scheme->mmap) { rc = si_nodemap(n, r); if (ssunlikely(rc == -1)) goto e1; } si_branchset(&n->self, &index); sd_buildcommit(&build, r); sd_buildfree(&build, r); return n; e1: ss_qffree(&f, r->a); sd_indexfree(&index, r); sd_buildfree(&build, r); e0: si_nodefree(n, r, 0); return NULL; } static inline int si_deploy(si *i, sr *r, int create_directory) { /* create directory */ int rc; if (sslikely(create_directory)) { rc = ss_vfsmkdir(r->vfs, i->scheme->path, 0755); if (ssunlikely(rc == -1)) { sr_malfunction(r->e, "directory '%s' create error: %s", i->scheme->path, strerror(errno)); return -1; } } /* create scheme file */ rc = si_schemedeploy(i->scheme, r); if (ssunlikely(rc == -1)) { sr_malfunction_set(r->e); return -1; } /* create initial node */ sinode *n = si_bootstrap(i, 0); if (ssunlikely(n == NULL)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_RECOVER_0, si_nodefree(n, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); rc = si_nodecomplete(n, r, i->scheme); if (ssunlikely(rc == -1)) { si_nodefree(n, r, 1); return -1; } si_insert(i, n); si_plannerupdate(&i->p, SI_COMPACT|SI_BRANCH|SI_TEMP, n); i->size = si_nodesize(n); return 1; }
static inline int si_trackdir(sitrack *track, sr *r, si *i) { DIR *dir = opendir(i->scheme->path); if (ssunlikely(dir == NULL)) { sr_malfunction(r->e, "directory '%s' open error: %s", i->scheme->path, strerror(errno)); return -1; } struct dirent *de; while ((de = readdir(dir))) { if (ssunlikely(de->d_name[0] == '.')) continue; uint64_t id_parent = 0; uint64_t id = 0; int rc = si_process(de->d_name, &id, &id_parent); if (ssunlikely(rc == -1)) continue; /* skip unknown file */ si_tracknsn(track, id_parent); si_tracknsn(track, id); sinode *head, *node; sspath path; switch (rc) { case SI_RDB_DBI: case SI_RDB_DBSEAL: { /* find parent node and mark it as having * incomplete compaction process */ head = si_trackget(track, id_parent); if (sslikely(head == NULL)) { head = si_nodenew(r); if (ssunlikely(head == NULL)) goto error; head->self.id.id = id_parent; head->recover = SI_RDB_UNDEF; si_trackset(track, head); } head->recover |= rc; /* remove any incomplete file made during compaction */ if (rc == SI_RDB_DBI) { ss_pathcompound(&path, i->scheme->path, id_parent, id, ".db.incomplete"); rc = ss_vfsunlink(r->vfs, path.path); if (ssunlikely(rc == -1)) { sr_malfunction(r->e, "db file '%s' unlink error: %s", path.path, strerror(errno)); goto error; } continue; } assert(rc == SI_RDB_DBSEAL); /* recover 'sealed' node */ node = si_nodenew(r); if (ssunlikely(node == NULL)) goto error; node->recover = SI_RDB_DBSEAL; ss_pathcompound(&path, i->scheme->path, id_parent, id, ".db.seal"); rc = si_nodeopen(node, r, i->scheme, &path, NULL); if (ssunlikely(rc == -1)) { si_nodefree(node, r, 0); goto error; } si_trackset(track, node); si_trackmetrics(track, node); continue; } } assert(rc == SI_RDB); head = si_trackget(track, id); if (head != NULL && (head->recover & SI_RDB)) { /* loaded by snapshot */ continue; } /* recover node */ node = si_nodenew(r); if (ssunlikely(node == NULL)) goto error; node->recover = SI_RDB; ss_path(&path, i->scheme->path, id, ".db"); rc = si_nodeopen(node, r, i->scheme, &path, NULL); if (ssunlikely(rc == -1)) { si_nodefree(node, r, 0); goto error; } si_trackmetrics(track, node); /* track node */ if (sslikely(head == NULL)) { si_trackset(track, node); } else { /* replace a node previously created by a * incomplete compaction */ si_trackreplace(track, head, node); head->recover &= ~SI_RDB_UNDEF; node->recover |= head->recover; si_nodefree(head, r, 0); } } closedir(dir); return 0; error: closedir(dir); return -1; }
static inline int si_split(si *index, sdc *c, ssbuf *result, sinode *parent, ssiter *i, uint64_t size_node, uint32_t size_stream, uint64_t vlsn) { sr *r = index->r; int count = 0; int rc; sdmergeconf mergeconf = { .size_stream = size_stream, .size_node = size_node, .size_page = index->scheme->node_page_size, .checksum = index->scheme->node_page_checksum, .compression = index->scheme->compression, .compression_key = index->scheme->compression_key, .offset = 0, .vlsn = vlsn, .save_delete = 0, .save_update = 0 }; sdmerge merge; sd_mergeinit(&merge, r, i, &c->build, &c->update, &mergeconf); while ((rc = sd_merge(&merge)) > 0) { sinode *n = si_nodenew(r); if (ssunlikely(n == NULL)) goto error; sdid id = { .parent = parent->self.id.id, .flags = 0, .id = sr_seq(index->r->seq, SR_NSNNEXT) }; rc = sd_mergecommit(&merge, &id); if (ssunlikely(rc == -1)) goto error; rc = si_nodecreate(n, r, index->scheme, &id, &merge.index, &c->build); if (ssunlikely(rc == -1)) goto error; rc = ss_bufadd(result, index->r->a, &n, sizeof(sinode*)); if (ssunlikely(rc == -1)) { sr_oom_malfunction(index->r->e); si_nodefree(n, r, 1); goto error; } sd_buildreset(&c->build); count++; } if (ssunlikely(rc == -1)) goto error; return 0; error: si_splitfree(result, r); sd_mergefree(&merge); return -1; } int si_compaction(si *index, sdc *c, uint64_t vlsn, sinode *node, ssiter *stream, uint32_t size_stream) { sr *r = index->r; ssbuf *result = &c->a; ssiter i; /* begin compaction. * * split merge stream into a number * of a new nodes. */ int rc; rc = si_split(index, c, result, node, stream, index->scheme->node_size, size_stream, vlsn); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_0, si_splitfree(result, r); sr_malfunction(r->e, "%s", "error injection"); return -1); /* mask removal of a single node as a * single node update */ int count = ss_bufused(result) / sizeof(sinode*); int count_index; si_lock(index); count_index = index->n; si_unlock(index); sinode *n; if (ssunlikely(count == 0 && count_index == 1)) { n = si_bootstrap(index, node->self.id.id); if (ssunlikely(n == NULL)) return -1; rc = ss_bufadd(result, r->a, &n, sizeof(sinode*)); if (ssunlikely(rc == -1)) { sr_oom_malfunction(r->e); si_nodefree(n, r, 1); return -1; } count++; } /* commit compaction changes */ si_lock(index); svindex *j = si_nodeindex(node); si_plannerremove(&index->p, SI_COMPACT|SI_BRANCH, node); switch (count) { case 0: /* delete */ si_remove(index, node); si_redistribute_index(index, r, c, node); uint32_t used = sv_indexused(j); if (used) { ss_quota(r->quota, SS_QREMOVE, used); } break; case 1: /* self update */ n = *(sinode**)result->s; n->i0 = *j; n->used = sv_indexused(j); si_nodelock(n); si_replace(index, node, n); si_plannerupdate(&index->p, SI_COMPACT|SI_BRANCH, n); break; default: /* split */ rc = si_redistribute(index, r, c, node, result); if (ssunlikely(rc == -1)) { si_unlock(index); si_splitfree(result, r); return -1; } ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); n = ss_iterof(ss_bufiterref, &i); n->used = sv_indexused(&n->i0); si_nodelock(n); si_replace(index, node, n); si_plannerupdate(&index->p, SI_COMPACT|SI_BRANCH, n); for (ss_iternext(ss_bufiterref, &i); ss_iterhas(ss_bufiterref, &i); ss_iternext(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); n->used = sv_indexused(&n->i0); si_nodelock(n); si_insert(index, n); si_plannerupdate(&index->p, SI_COMPACT|SI_BRANCH, n); } break; } sv_indexinit(j); si_unlock(index); /* compaction completion */ /* seal nodes */ ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); if (index->scheme->sync) { rc = si_nodesync(n, r); if (ssunlikely(rc == -1)) return -1; } rc = si_nodeseal(n, r, index->scheme); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_3, si_nodefree(node, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); ss_iternext(ss_bufiterref, &i); } SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_1, si_nodefree(node, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); /* gc old node */ rc = si_nodefree(node, r, 1); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_2, sr_malfunction(r->e, "%s", "error injection"); return -1); /* complete new nodes */ ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); rc = si_nodecomplete(n, r, index->scheme); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_4, sr_malfunction(r->e, "%s", "error injection"); return -1); ss_iternext(ss_bufiterref, &i); } /* unlock */ si_lock(index); ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); si_nodeunlock(n); ss_iternext(ss_bufiterref, &i); } si_unlock(index); return 0; }
static inline int si_split(si *index, sdc *c, ssbuf *result, sinode *parent, ssiter *i, uint64_t size_node, uint64_t size_stream, uint32_t stream, uint64_t vlsn) { sr *r = &index->r; uint32_t timestamp = ss_timestamp(); int rc; sdmergeconf mergeconf = { .stream = stream, .size_stream = size_stream, .size_node = size_node, .size_page = index->scheme.compaction.node_page_size, .checksum = index->scheme.compaction.node_page_checksum, .expire = index->scheme.expire, .timestamp = timestamp, .compression = index->scheme.compression, .compression_if = index->scheme.compression_if, .direct_io = index->scheme.direct_io, .direct_io_page_size = index->scheme.direct_io_page_size, .vlsn = vlsn }; sinode *n = NULL; sdmerge merge; rc = sd_mergeinit(&merge, r, i, &c->build, &c->build_index, &c->upsert, &mergeconf); if (ssunlikely(rc == -1)) return -1; while ((rc = sd_merge(&merge)) > 0) { /* create new node */ uint64_t id = sr_seq(index->r.seq, SR_NSNNEXT); n = si_nodenew(r, id, parent->id); if (ssunlikely(n == NULL)) goto error; rc = si_nodecreate(n, r, &index->scheme); if (ssunlikely(rc == -1)) goto error; /* write pages */ uint64_t offset; offset = sd_iosize(&c->io, &n->file); while ((rc = sd_mergepage(&merge, offset)) == 1) { rc = sd_writepage(r, &n->file, &c->io, merge.build); if (ssunlikely(rc == -1)) goto error; offset = sd_iosize(&c->io, &n->file); } if (ssunlikely(rc == -1)) goto error; offset = sd_iosize(&c->io, &n->file); rc = sd_mergeend(&merge, offset); if (ssunlikely(rc == -1)) goto error; /* write index */ rc = sd_writeindex(r, &n->file, &c->io, &merge.index); if (ssunlikely(rc == -1)) goto error; /* mmap mode */ if (index->scheme.mmap) { rc = si_nodemap(n, r); if (ssunlikely(rc == -1)) goto error; } /* add node to the list */ rc = ss_bufadd(result, index->r.a, &n, sizeof(sinode*)); if (ssunlikely(rc == -1)) { sr_oom_malfunction(index->r.e); goto error; } n->index = merge.index; } if (ssunlikely(rc == -1)) goto error; return 0; error: if (n) si_nodefree(n, r, 0); sd_mergefree(&merge); si_splitfree(result, r); return -1; } static int si_merge(si *index, sdc *c, sinode *node, uint64_t vlsn, ssiter *stream, uint64_t size_stream, uint32_t n_stream) { sr *r = &index->r; ssbuf *result = &c->a; ssiter i; /* begin compaction. * * Split merge stream into a number of * a new nodes. */ int rc; rc = si_split(index, c, result, node, stream, index->scheme.compaction.node_size, size_stream, n_stream, vlsn); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_0, si_splitfree(result, r); sr_malfunction(r->e, "%s", "error injection"); return -1); /* mask removal of a single node as a * single node update */ int count = ss_bufused(result) / sizeof(sinode*); int count_index; si_lock(index); count_index = index->n; si_unlock(index); sinode *n; if (ssunlikely(count == 0 && count_index == 1)) { n = si_bootstrap(index, node->id); if (ssunlikely(n == NULL)) return -1; rc = ss_bufadd(result, r->a, &n, sizeof(sinode*)); if (ssunlikely(rc == -1)) { sr_oom_malfunction(r->e); si_nodefree(n, r, 1); return -1; } count++; } /* commit compaction changes */ si_lock(index); svindex *j = si_nodeindex(node); si_plannerremove(&index->p, node); si_nodesplit(node); switch (count) { case 0: /* delete */ si_remove(index, node); si_redistribute_index(index, r, c, node); break; case 1: /* self update */ n = *(sinode**)result->s; n->i0 = *j; n->used = j->used; si_nodelock(n); si_replace(index, node, n); si_plannerupdate(&index->p, n); break; default: /* split */ rc = si_redistribute(index, r, c, node, result); if (ssunlikely(rc == -1)) { si_unlock(index); si_splitfree(result, r); return -1; } ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); n = ss_iterof(ss_bufiterref, &i); n->used = n->i0.used; si_nodelock(n); si_replace(index, node, n); si_plannerupdate(&index->p, n); for (ss_iternext(ss_bufiterref, &i); ss_iterhas(ss_bufiterref, &i); ss_iternext(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); n->used = n->i0.used; si_nodelock(n); si_insert(index, n); si_plannerupdate(&index->p, n); } break; } sv_indexinit(j); si_unlock(index); /* compaction completion */ /* seal nodes */ ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); if (index->scheme.sync) { rc = ss_filesync(&n->file); if (ssunlikely(rc == -1)) { sr_malfunction(r->e, "db file '%s' sync error: %s", ss_pathof(&n->file.path), strerror(errno)); return -1; } } rc = si_noderename_seal(n, r, &index->scheme); if (ssunlikely(rc == -1)) { si_nodefree(node, r, 0); return -1; } SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_3, si_nodefree(node, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); ss_iternext(ss_bufiterref, &i); } SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_1, si_nodefree(node, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); /* gc node */ uint16_t refs = si_noderefof(node); if (sslikely(refs == 0)) { rc = si_nodefree(node, r, 1); if (ssunlikely(rc == -1)) return -1; } else { /* node concurrently being read, schedule for * delayed removal */ si_nodegc(node, r, &index->scheme); si_lock(index); ss_listappend(&index->gc, &node->gc); index->gc_count++; si_unlock(index); } SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_2, sr_malfunction(r->e, "%s", "error injection"); return -1); /* complete new nodes */ ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); rc = si_noderename_complete(n, r, &index->scheme); if (ssunlikely(rc == -1)) return -1; SS_INJECTION(r->i, SS_INJECTION_SI_COMPACTION_4, sr_malfunction(r->e, "%s", "error injection"); return -1); ss_iternext(ss_bufiterref, &i); } /* unlock */ si_lock(index); ss_iterinit(ss_bufiterref, &i); ss_iteropen(ss_bufiterref, &i, result, sizeof(sinode*)); while (ss_iterhas(ss_bufiterref, &i)) { n = ss_iterof(ss_bufiterref, &i); si_nodeunlock(n); ss_iternext(ss_bufiterref, &i); } si_unlock(index); return 0; }
sinode *si_bootstrap(si *i, sr *r, uint32_t parent) { sinode *n = si_nodenew(r); if (srunlikely(n == NULL)) return NULL; sdid id = { .parent = parent, .flags = 0, .id = sr_seq(r->seq, SR_NSNNEXT) }; sdindex index; sd_indexinit(&index); int rc = sd_indexbegin(&index, r, 0); if (srunlikely(rc == -1)) { si_nodefree(n, r, 0); return NULL; } sdbuild build; sd_buildinit(&build); rc = sd_buildbegin(&build, r, i->conf->node_page_checksum, i->conf->compression); if (srunlikely(rc == -1)) { sd_indexfree(&index, r); sd_buildfree(&build, r); si_nodefree(n, r, 0); return NULL; } sd_buildend(&build, r); rc = sd_indexadd(&index, r, &build); if (srunlikely(rc == -1)) { sd_indexfree(&index, r); si_nodefree(n, r, 0); return NULL; } sd_buildcommit(&build); sd_indexcommit(&index, r, &id); rc = si_nodecreate(n, r, i->conf, &id, &index, &build); sd_buildfree(&build, r); if (srunlikely(rc == -1)) { si_nodefree(n, r, 1); return NULL; } return n; } static inline int si_deploy(si *i, sr *r) { int rc = sr_filemkdir(i->conf->path); if (srunlikely(rc == -1)) { sr_malfunction(r->e, "directory '%s' create error: %s", i->conf->path, strerror(errno)); return -1; } sinode *n = si_bootstrap(i, r, 0); if (srunlikely(n == NULL)) return -1; SR_INJECTION(r->i, SR_INJECTION_SI_RECOVER_0, si_nodefree(n, r, 0); sr_malfunction(r->e, "%s", "error injection"); return -1); rc = si_nodecomplete(n, r, i->conf); if (srunlikely(rc == -1)) { si_nodefree(n, r, 1); return -1; } si_insert(i, r, n); si_plannerupdate(&i->p, SI_COMPACT|SI_BRANCH, n); return 1; }