/* Search a q-gram database.
`wdb' specifies the q-gram database object.
`word' specifies the string of the word to be matched to.
`np' specifies the pointer to the variable into which the number of elements of the return
value is assigned.
If successful, the return value is the pointer to an array of ID numbers of the corresponding
records. */
static uint64_t *tcwdbsearchimpl(TCWDB *wdb, const char *word, int *np){
assert(wdb && word && np);
int wlen = strlen(word);
if(wlen > WDBMAXWORDLEN){
tcbdbsetecode(wdb->idx, TCEINVALID, __FILE__, __LINE__, __func__);
return NULL;
}
int vsiz;
const char *vbuf = tcbdbget3(wdb->idx, word, wlen, &vsiz);
if(!vbuf){
vbuf = "";
vsiz = 0;
}
uint64_t *res = tcmalloc(WDBRESUNIT * sizeof(*res));
int rnum = 0;
int ranum = WDBRESUNIT;
while(vsiz > 0){
int step;
uint64_t id;
TDREADVNUMBUF64(vbuf, id, step);
vbuf += step;
vsiz -= step;
if(rnum >= ranum){
ranum *= 2;
res = tcrealloc(res, ranum * sizeof(*res));
}
res[rnum++] = id;
}
*np = rnum;
return res;
}
dict_oid_t
dict_get_sym(dict_t d, const char *sym)
{
const dict_oid_t *rp;
const size_t ssz = strlen(sym);
int rz[1];
if (UNLIKELY((rp = tcbdbget3(d, sym, ssz, rz)) == NULL)) {
return NUL_OID;
} else if (UNLIKELY(*rz != sizeof(*rp))) {
return NUL_OID;
}
return *rp;
}
/* perform btread command */
int dobtread(char *name, int rnum, int rnd){
TCBDB *bdb;
int i, err, len, vlen;
const char *val;
char buf[RECBUFSIZ];
if(showprgr) printf("<Reading Test of B+ Tree>\n name=%s rnum=%d\n\n", name, rnum);
/* open a database */
bdb = tcbdbnew();
if(rnd){
tcbdbsetcache(bdb, rnum / 77 + 1, -1);
} else {
tcbdbsetcache(bdb, 256, 256);
}
tcbdbsetxmsiz(bdb, rnum * 32);
if(!tcbdbopen(bdb, name, BDBOREADER)){
fprintf(stderr, "tcbdbopen failed\n");
tcbdbdel(bdb);
return 1;
}
err = FALSE;
/* loop for each record */
for(i = 1; i <= rnum; i++){
/* store a record */
len = sprintf(buf, "%08d", rnd ? myrand() % rnum + 1 : i);
if(!(val = tcbdbget3(bdb, buf, len, &vlen))){
fprintf(stderr, "tdbdbget3 failed\n");
err = TRUE;
break;
}
/* print progression */
if(showprgr && rnum > 250 && i % (rnum / 250) == 0){
putchar('.');
fflush(stdout);
if(i == rnum || i % (rnum / 10) == 0){
printf(" (%08d)\n", i);
fflush(stdout);
}
}
}
/* close the database */
if(!tcbdbclose(bdb)){
fprintf(stderr, "tcbdbclose failed\n");
tcbdbdel(bdb);
return 1;
}
tcbdbdel(bdb);
if(showprgr && !err) printf("ok\n\n");
return err ? 1 : 0;
}
static int control(ErlDrvData handle, unsigned int command, char* buf, int count, char** res, int res_size) {
tcbdb_drv_t *driver = (tcbdb_drv_t*)handle;
TCBDB *bdb = driver->bdb;
int index = 1, tp, sz, version;
char key[MAX_KEY_SIZE], value[MAX_VALUE_SIZE];
long key_size, value_size;
long long long_tmp;
bool rs;
const char *value_tmp = NULL;
ei_x_buff x;
ei_x_new_with_version(&x);
if (bdb == NULL && command != OPEN)
return ei_error(&x, "database_not_opened", res, res_size);
ei_decode_version(buf, &index, &version);
switch (command) {
case OPEN:
// open(Filepath::string())
if (bdb == NULL) {
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_STRING_EXT)
return ei_error(&x, "invalid_argument", res, res_size);
TCBDB *bdb = tcbdbnew();
tcbdbsetmutex(bdb);
tcbdbsetcache(bdb, 104800, 51200);
tcbdbsetxmsiz(bdb, 1048576);
tcbdbtune(bdb, 0, 0, 0, 7, -1, BDBTLARGE);
char *file = driver_alloc(sz + 1);
ei_decode_string(buf, &index, file);
rs = tcbdbopen(bdb, file, BDBOWRITER | BDBOCREAT);
driver_free(file);
if (!rs)
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
driver->bdb = bdb;
ei_x_encode_atom(&x, "ok");
} else
return ei_error(&x, "database already opened", res, res_size);
break;
case CLOSE:
tcbdbclose(bdb);
tcbdbdel(bdb);
driver->bdb = NULL;
ei_x_encode_atom(&x, "ok");
break;
case PUT:
case PUTDUP:
case PUTCAT:
case PUTKEEP:
// put({Key::binary(), Value::binary()})
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_SMALL_TUPLE_EXT || sz != 2)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_tuple_header(buf, &index, &sz);
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &key[0], &key_size);
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT)
return ei_error(&x, "invalid_argument", res, res_size);
if (sz <= MAX_VALUE_SIZE) {
ei_decode_binary(buf, &index, &value[0], &value_size);
switch (command) {
case PUT: rs = tcbdbput(bdb, &key[0], key_size, &value[0], value_size); break;
case PUTDUP: rs = tcbdbputdup(bdb, &key[0], key_size, &value[0], value_size); break;
case PUTCAT: rs = tcbdbputcat(bdb, &key[0], key_size, &value[0], value_size); break;
default: rs = tcbdbputkeep(bdb, &key[0], key_size, &value[0], value_size);
}
} else {
void *p = driver_alloc(sz);
if (p) {
ei_decode_binary(buf, &index, p, &value_size);
switch (command) {
case PUT: rs = tcbdbput(bdb, &key[0], key_size, p, value_size); break;
case PUTDUP: rs = tcbdbputdup(bdb, &key[0], key_size, p, value_size); break;
case PUTCAT: rs = tcbdbputcat(bdb, &key[0], key_size, p, value_size); break;
default: rs = tcbdbputkeep(bdb, &key[0], key_size, p, value_size);
}
driver_free(p);
} else
return ei_error(&x, "too long value", res, res_size);
};
if (!rs)
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
ei_x_encode_atom(&x, "ok");
break;
case GET:
// get(Key::binary()) -> {ok, Value} | {error, not_found}
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &key[0], &key_size);
value_tmp = tcbdbget3(bdb, &key[0], key_size, &sz);
ei_x_encode_tuple_header(&x, 2);
if (value_tmp != NULL) {
ei_x_encode_atom(&x, "ok");
ei_x_encode_binary(&x, value_tmp, sz);
} else {
ei_x_encode_atom(&x, "error");
ei_x_encode_atom(&x, "not_found");
}
break;
case GETDUP:
// get(Key::binary()) -> {ok, Values}
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &key[0], &key_size);
TCLIST *vals = tcbdbget4(bdb, key, key_size);
if (vals) {
ei_x_encode_tuple_header(&x, 2);
ei_x_encode_atom(&x, "ok");
int j;
for (j=0; j<tclistnum(vals); j++) {
value_tmp = tclistval(vals, j, &sz);
ei_x_encode_list_header(&x, 1);
ei_x_encode_binary(&x, value_tmp, sz);
}
tclistdel(vals);
ei_x_encode_empty_list(&x);
} else {
ei_x_encode_tuple_header(&x, 2);
ei_x_encode_atom(&x, "ok");
ei_x_encode_empty_list(&x);
}
break;
case REMOVE:
// remove(Keys::list())
// remove(Key::binary())
// remove({Key::binary(), Value::binary()})
ei_get_type(buf, &index, &tp, &sz);
if (tp == ERL_LIST_EXT) {
int count, j;
ei_decode_list_header(buf, &index, &count);
for (j=0; j<count; j++) {
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &key[0], &key_size);
if (!tcbdbout3(bdb, &key[0], key_size))
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
}
ei_x_encode_atom(&x, "ok");
} else if (tp == ERL_BINARY_EXT && sz <= MAX_KEY_SIZE) {
ei_decode_binary(buf, &index, &key[0], &key_size);
tcbdbout3(bdb, &key[0], key_size);
ei_x_encode_atom(&x, "ok");
} else if (tp == ERL_SMALL_TUPLE_EXT && sz == 2) {
ei_decode_tuple_header(buf, &index, &sz);
// get key
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &key[0], &key_size);
// get value
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_VALUE_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &value[0], &value_size);
// remove by key&value
BDBCUR *cur = tcbdbcurnew(bdb);
if (!tcbdbcurjump(cur, &key[0], key_size))
return ei_error(&x, "record_not_found", res, res_size);
bool removed = false, not_found = false;
while (!removed && !not_found) {
int cur_key_size, cur_val_size;
const void *curkey = tcbdbcurkey3(cur, &cur_key_size);
if (cur_key_size == key_size && memcmp(curkey, key, key_size) == 0) {
const void *curval = tcbdbcurval3(cur, &cur_val_size);
if (cur_val_size == value_size && memcmp(curval, value, value_size) == 0) {
tcbdbcurout(cur);
removed = true;
} else
if (!tcbdbcurnext(cur))
not_found = true;
} else not_found = true;
}
if (not_found) ei_x_encode_atom(&x, "not_found");
else ei_x_encode_atom(&x, "ok");
} else
return ei_error(&x, "invalid_argument", res, res_size);
break;
case RANGE:
/*
* range({Prefix::binary(), limit:integer()})
* range({StartKey::binary(), BeginInclusion::boolean(), EndKey::binary(), EndInclusion::binary(), limit:integer()})
*/
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_SMALL_TUPLE_EXT || sz < 2)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_tuple_header(buf, &index, &sz);
ei_get_type(buf, &index, &tp, &sz);
if (tp == ERL_BINARY_EXT && sz <= MAX_KEY_SIZE) {
char keys[MAX_KEY_SIZE], keyf[MAX_KEY_SIZE];
long keys_size, keyf_size;
int keys_inc, keyf_inc;
long max = -1;
TCLIST *range;
ei_decode_binary(buf, &index, &keys[0], &keys_size);
ei_get_type(buf, &index, &tp, &sz);
if (tp == ERL_ATOM_EXT) {
// range
ei_decode_boolean(buf, &index, &keys_inc);
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_BINARY_EXT || sz > MAX_KEY_SIZE)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_binary(buf, &index, &keyf[0], &keyf_size);
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_ATOM_EXT)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_boolean(buf, &index, &keyf_inc);
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_INTEGER_EXT)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_long(buf, &index, &max);
range = tcbdbrange(bdb, &keys[0], keys_size, keys_inc == 1, &keyf[0], keyf_size, keyf_inc == 1, max);
} else if (tp == ERL_INTEGER_EXT) {
// prefix
ei_get_type(buf, &index, &tp, &sz);
if (tp != ERL_INTEGER_EXT)
return ei_error(&x, "invalid_argument", res, res_size);
ei_decode_long(buf, &index, &max);
range = tcbdbfwmkeys(bdb, &keys[0], keys_size, max);
} else
return ei_error(&x, "invalid_argument", res, res_size);
const char *key;
int key_size, value_size;
int idx, cnt = 0, rcount = tclistnum(range);
ei_x_encode_tuple_header(&x, 2);
ei_x_encode_atom(&x, "ok");
BDBCUR *cur = tcbdbcurnew(bdb);
for (idx=0; idx<rcount; idx++) {
key = tclistval(range, idx, &key_size);
TCLIST *vals = tcbdbget4(bdb, key, key_size);
if (vals) {
int j;
for (j=0; j<tclistnum(vals); j++) {
ei_x_encode_list_header(&x, 1);
value_tmp = tclistval(vals, j, &value_size);
ei_x_encode_binary(&x, value_tmp, value_size);
if (max >= 0 && ++cnt >= max) break;
}
tclistdel(vals);
}
idx++;
}
tcbdbcurdel(cur);
tclistdel(range);
ei_x_encode_empty_list(&x);
} else
return ei_error(&x, "invalid_argument", res, res_size);
break;
case SYNC:
// sync()
if (!tcbdbsync(bdb))
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
ei_x_encode_atom(&x, "ok");
break;
case INFO:
// info()
ei_x_encode_tuple_header(&x, 3);
ei_x_encode_atom(&x, "ok");
long_tmp = tcbdbrnum(bdb);
ei_x_encode_longlong(&x, long_tmp);
long_tmp = tcbdbfsiz(bdb);
ei_x_encode_longlong(&x, long_tmp);
break;
case ITERATE:
// Read(none) -> {ok, Key} | {error, not_found}
// Read(Key::binary()) -> {ok, Key} | {error, not_found}
ei_get_type(buf, &index, &tp, &sz);
BDBCUR *cur = tcbdbcurnew(bdb);
if (tp == ERL_BINARY_EXT && sz <= MAX_KEY_SIZE) {
ei_decode_binary(buf, &index, &key[0], &key_size);
rs = tcbdbcurjump(cur, &key[0], key_size) && tcbdbcurnext(cur);
} else
rs = tcbdbcurfirst(cur);
if (rs) {
int key_size;
const char *key = tcbdbcurkey3(cur, &key_size);
ei_x_encode_tuple_header(&x, 2);
ei_x_encode_atom(&x, "ok");
ei_x_encode_binary(&x, key, key_size);
tcbdbcurdel(cur);
} else {
tcbdbcurdel(cur);
return ei_error(&x, "not_found", res, res_size);
}
break;
case VANISH:
// vanish() -> ok
if (!tcbdbvanish(bdb))
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
ei_x_encode_atom(&x, "ok");
break;
case BACKUP:
// backup(path::string()) -> ok | {error, Reason}
ei_get_type(buf, &index, &tp, &sz);
if (tp == ERL_STRING_EXT) {
char *file = driver_alloc(sz + 1);
ei_decode_string(buf, &index, file);
if (tcbdbcopy(driver->bdb, file))
ei_x_encode_atom(&x, "ok");
else
return ei_error(&x, tcbdberrmsg(tcbdbecode(bdb)), res, res_size);
} else
return ei_error(&x, "invalid_argument", res, res_size);
break;
default:
return ei_error(&x, "invalid_command", res, res_size);
}
if (res_size < x.index)
(*res) = (char *)driver_alloc(x.index);
int n = x.index;
memcpy(*res, x.buff, x.index);
ei_x_free(&x);
return n;
};
/* Synchronize updating contents on memory of a word database object. */
bool tcwdbmemsync(TCWDB *wdb, int level){
assert(wdb);
if(!wdb->open || !wdb->cc){
tcbdbsetecode(wdb->idx, TCEINVALID, __FILE__, __LINE__, __func__);
return false;
}
bool err = false;
bool (*synccb)(int, int, const char *, void *) = wdb->synccb;
void *syncopq = wdb->syncopq;
bool (*addcb)(const char *, void *) = wdb->addcb;
void *addopq = wdb->addopq;
TCBDB *idx = wdb->idx;
TCMAP *cc = wdb->cc;
if(synccb && !synccb(0, 0, "started", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
return false;
}
if(tcmaprnum(cc) > 0){
if(synccb && !synccb(0, 0, "getting tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
return false;
}
int kn;
const char **keys = tcmapkeys2(cc, &kn);
if(synccb && !synccb(kn, 0, "sorting tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
tcfree(keys);
return false;
}
qsort(keys, kn, sizeof(*keys), (int(*)(const void *, const void *))tccmpwords);
for(int i = 0; i < kn; i++){
if(synccb && !synccb(kn, i + 1, "storing tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
tcfree(keys);
return false;
}
const char *kbuf = keys[i];
int ksiz = strlen(kbuf);
int vsiz;
const char *vbuf = tcmapget(cc, kbuf, ksiz, &vsiz);
if(!tcbdbputcat(idx, kbuf, ksiz, vbuf, vsiz)) err = true;
}
if(addcb){
if(synccb && !synccb(0, 0, "storing keyword list", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
tcfree(keys);
return false;
}
for(int i = 0; i < kn; i++){
if(!addcb(keys[i], addopq)){
tcfree(keys);
return false;
}
}
}
tcfree(keys);
tcmapclear(cc);
}
TCMAP *dtokens = wdb->dtokens;
TCIDSET *dids = wdb->dids;
if(tcmaprnum(dtokens) > 0){
if(synccb && !synccb(0, 0, "getting deleted tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
return false;
}
int kn;
const char **keys = tcmapkeys2(dtokens, &kn);
if(synccb && !synccb(kn, 0, "sorting deleted tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
tcfree(keys);
return false;
}
qsort(keys, kn, sizeof(*keys), (int(*)(const void *, const void *))tccmpwords);
for(int i = 0; i < kn; i++){
if(synccb && !synccb(kn, i + 1, "storing deleted tokens", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
tcfree(keys);
return false;
}
const char *kbuf = keys[i];
int ksiz = strlen(kbuf);
int vsiz;
const char *vbuf = tcbdbget3(idx, kbuf, ksiz, &vsiz);
if(!vbuf) continue;
char *nbuf = tcmalloc(vsiz + 1);
char *wp = nbuf;
const char *pv;
while(vsiz > 0){
pv = vbuf;
int step;
uint64_t id;
TDREADVNUMBUF64(vbuf, id, step);
vbuf += step;
vsiz -= step;
if(!tcidsetcheck(dids, id)){
int len = vbuf - pv;
memcpy(wp, pv, len);
wp += len;
}
}
int nsiz = wp - nbuf;
if(nsiz > 0){
if(!tcbdbput(idx, kbuf, ksiz, nbuf, nsiz)) err = true;
} else {
if(!tcbdbout(idx, kbuf, ksiz)) err = true;
}
tcfree(nbuf);
}
tcfree(keys);
tcmapclear(dtokens);
tcidsetclear(dids);
}
if(level > 0){
if(synccb && !synccb(0, 0, "synchronizing database", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
return false;
}
if(!tcbdbmemsync(idx, level > 1)) err = true;
}
if(synccb && !synccb(0, 0, "finished", syncopq)){
tcbdbsetecode(wdb->idx, TCEMISC, __FILE__, __LINE__, __func__);
return false;
}
return !err;
}
