static void putlist_test(void *db, const char *command, int num, int vsiz,
int batch, unsigned int seed)
{
TCADB *adb = db;
struct keygen keygen;
char *value = xmalloc(vsiz);
TCLIST *list = tclistnew();
int i;
keygen_init(&keygen, seed);
for (i = 0; i < num; i++) {
tclistpush2(list, keygen_next_key(&keygen));
tclistpush(list, value, vsiz);
if (tclistnum(list) / 2 >= batch) {
tclistdel(do_tcadbmisc(adb, command, list));
tclistclear(list);
}
}
if (tclistnum(list))
tclistdel(do_tcadbmisc(adb, command, list));
tclistdel(list);
free(value);
}
static void getlist_test(void *db, const char *command, int num, int vsiz,
int batch, unsigned int seed)
{
TCADB *adb = db;
struct keygen keygen;
struct keygen keygen_for_check;
TCLIST *list = tclistnew();
TCLIST *recs;
int i;
keygen_init(&keygen, seed);
keygen_init(&keygen_for_check, seed);
for (i = 0; i < num; i++) {
tclistpush2(list, keygen_next_key(&keygen));
if (tclistnum(list) >= batch) {
recs = do_tcadbmisc(adb, command, list);
check_records(recs, &keygen_for_check, vsiz,
tclistnum(list));
tclistdel(recs);
tclistclear(list);
}
}
if (tclistnum(list)) {
recs = do_tcadbmisc(adb, command, list);
check_records(recs, &keygen_for_check, vsiz, tclistnum(list));
tclistdel(recs);
}
tclistdel(list);
}
static void rangeout_test(void *db, const char *command, int num, int vsiz,
int batch, unsigned int seed)
{
TCADB *adb = db;
struct keygen keygen;
TCLIST *args = tclistnew();
char start_key[KEYGEN_PREFIX_SIZE + 1];
char max[100];
char end_key[KEYGEN_PREFIX_SIZE + 1];
char binc[2];
keygen_init(&keygen, seed);
keygen_prefix(&keygen, start_key);
sprintf(max, "%d", batch);
keygen_prefix(&keygen, end_key);
end_key[KEYGEN_PREFIX_SIZE - 1] = '-' + 1;
sprintf(binc, "0");
tclistpush2(args, start_key);
tclistpush2(args, max);
tclistpush2(args, end_key);
tclistpush2(args, binc);
while (1) {
TCLIST *recs;
recs = do_tcadbmisc(adb, command, args);
if (tclistnum(recs) == 0)
break;
if (debug) {
const char *num_recs = tclistval2(recs, 0);
num -= atoi(num_recs);
if (num != 0 && atoi(num_recs) != batch)
die("Unexpected number of records are deleted");
}
tclistdel(recs);
}
if (debug && num != 0)
die("Unexpected number of records are deleted");
tclistdel(args);
}
static void range_atomic_test(void *db, int num, int vsiz, int batch,
unsigned int seed)
{
TCADB *adb = db;
struct keygen keygen;
TCLIST *args = tclistnew();
char start_key[KEYGEN_PREFIX_SIZE + 1];
char max[100];
char end_key[KEYGEN_PREFIX_SIZE + 1];
char binc[2];
keygen_init(&keygen, seed);
keygen_prefix(&keygen, start_key);
sprintf(max, "%d", batch);
keygen_prefix(&keygen, end_key);
end_key[KEYGEN_PREFIX_SIZE - 1] = '-' + 1;
sprintf(binc, "0");
tclistpush2(args, start_key);
tclistpush2(args, max);
tclistpush2(args, end_key);
tclistpush2(args, binc);
while (1) {
TCLIST *recs;
int num_recs;
recs = do_tcadbmisc(adb, "range_atomic", args);
num_recs = tclistnum(recs) / 2;
if (!num_recs)
break;
check_records(recs, &keygen, vsiz, num < batch ? num : batch);
tclistover2(args, 0, tclistval2(recs, 2 * (num_recs - 1)));
tclistdel(recs);
num -= num_recs;
}
if (debug && num)
die("Unexpected record num: %d", num);
tclistdel(args);
}
/* parse arguments of import command */
static int runimport(int argc, char **argv){
char *dbpath = NULL;
TCLIST *files = tcmpoollistnew(tcmpoolglobal());
TCLIST *sufs = tcmpoollistnew(tcmpoolglobal());
for(int i = 2; i < argc; i++){
if(!dbpath && argv[i][0] == '-'){
if(!strcmp(argv[i], "-suf")){
if(++i >= argc) usage();
tclistpush2(sufs, argv[i]);
} else {
usage();
}
} else if(!dbpath){
dbpath = argv[i];
} else {
tclistpush2(files, argv[i]);
}
}
if(!dbpath || tclistnum(files) < 1) usage();
tclistpush2(sufs, ".tpw");
int rv = procimport(dbpath, files, sufs);
return rv;
}
TCLIST *explode(const char *delimiter , char *str)
{
TCLIST *data = NULL;
char *tmp = NULL;
char *next = NULL;
data = tclistnew2(30);
next = strtok_r(str, delimiter, &tmp);
while (next != NULL)
{
tclistpush2(data, next);
next = strtok_r(NULL, delimiter, &tmp);
}
return data;
}
/* parse arguments of misc command */
static int runmisc(int argc, char **argv){
char *name = NULL;
char *func = NULL;
TCLIST *args = tcmpoollistnew(tcmpoolglobal());
bool sx = false;
int sep = -1;
bool px = false;
for(int i = 2; i < argc; i++){
if(!name && argv[i][0] == '-'){
if(!strcmp(argv[i], "-sx")){
sx = true;
} else if(!strcmp(argv[i], "-sep")){
if(++i >= argc) usage();
sep = sepstrtochr(argv[i]);
} else if(!strcmp(argv[i], "-px")){
px = true;
} else {
usage();
}
} else if(!name){
name = argv[i];
} else if(!func){
func = argv[i];
} else {
if(sx){
int size;
char *buf = tchexdecode(argv[i], &size);
tclistpush(args, buf, size);
tcfree(buf);
} else if(sep > 0){
int size;
char *buf = strtozsv(argv[i], sep, &size);
tclistpush(args, buf, size);
tcfree(buf);
} else {
tclistpush2(args, argv[i]);
}
}
}
if(!name || !func) usage();
int rv = procmisc(name, func, args, sep, px);
return rv;
}
void testBSONExportImport(void) {
EJDB *jb = ejdbnew();
CU_ASSERT_TRUE_FATAL(ejdbopen(jb, "dbt4_export", JBOWRITER | JBOCREAT | JBOTRUNC));
EJCOLL *coll = ejdbcreatecoll(jb, "col1", NULL);
if (!coll) {
eprint(jb, __LINE__, "testBSONExportImport");
}
CU_ASSERT_TRUE(coll != NULL);
bson_oid_t oid;
bson bv1;
bson_init(&bv1);
bson_append_int(&bv1, "a", 1);
bson_append_string(&bv1, "c", "d");
bson_finish(&bv1);
ejdbsavebson(coll, &bv1, &oid);
bson_destroy(&bv1);
EJCOLLOPTS copts = {0};
copts.large = true;
copts.records = 200000;
coll = ejdbcreatecoll(jb, "col2", &copts);
if (!coll) {
eprint(jb, __LINE__, "testBSONExportImport");
}
CU_ASSERT_TRUE(coll != NULL);
CU_ASSERT_TRUE(ejdbsetindex(coll, "f", JBIDXSTR | JBIDXNUM));
bson_init(&bv1);
bson_append_int(&bv1, "e", 1);
bson_append_string(&bv1, "f", "g");
bson_finish(&bv1);
ejdbsavebson(coll, &bv1, &oid);
bson_destroy(&bv1);
bson_init(&bv1);
bson_append_int(&bv1, "e", 2);
bson_append_string(&bv1, "f", "g2");
bson_finish(&bv1);
ejdbsavebson(coll, &bv1, &oid);
bson_destroy(&bv1);
TCXSTR *log = tcxstrnew();
TCLIST *cnames = tclistnew();
tclistpush2(cnames, "col1");
tclistpush2(cnames, "col2");
bool rv = ejdbexport(jb, "testBSONExportImport", NULL, 0, log);
if (!rv) {
eprint(jb, __LINE__, "testBSONExportImport");
}
CU_ASSERT_TRUE(rv);
bson *ometa = ejdbmeta(jb);
CU_ASSERT_TRUE_FATAL(ometa != NULL);
ejdbclose(jb);
ejdbdel(jb);
//Restore data:
jb = ejdbnew();
CU_ASSERT_TRUE_FATAL(ejdbopen(jb, "dbt4_export", JBOWRITER | JBOCREAT));
coll = ejdbgetcoll(jb, "col1");
CU_ASSERT_PTR_NOT_NULL_FATAL(coll);
bson_init(&bv1);
bson_append_int(&bv1, "e", 2);
bson_finish(&bv1);
CU_ASSERT_TRUE(ejdbsavebson(coll, &bv1, &oid));
bson_destroy(&bv1);
rv = ejdbimport(jb, "testBSONExportImport", cnames, JBIMPORTREPLACE, log);
CU_ASSERT_TRUE(rv);
//fprintf(stderr, "\n\n%s", TCXSTRPTR(log));
CU_ASSERT_PTR_NOT_NULL(strstr(TCXSTRPTR(log), "Replacing all data in 'col1'"));
CU_ASSERT_PTR_NOT_NULL(strstr(TCXSTRPTR(log), "1 objects imported into 'col1'"));
CU_ASSERT_PTR_NOT_NULL(strstr(TCXSTRPTR(log), "2 objects imported into 'col2'"));
bson *nmeta = ejdbmeta(jb);
CU_ASSERT_TRUE_FATAL(nmeta != NULL);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.0.name", strlen("collections.0.name")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.0.records", strlen("collections.0.records")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.name", strlen("collections.1.name")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.records", strlen("collections.1.records")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.options.buckets", strlen("collections.1.options.buckets")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.options.large", strlen("collections.1.options.large")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.0.field", strlen("collections.1.indexes.0.field")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.0.type", strlen("collections.1.indexes.0.type")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.0.records", strlen("collections.1.indexes.0.records")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.1.field", strlen("collections.1.indexes.1.field")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.1.type", strlen("collections.1.indexes.1.type")) == 0);
CU_ASSERT_TRUE(bson_compare(bson_data(ometa), bson_data(nmeta), "collections.1.indexes.1.records", strlen("collections.1.indexes.1.records")) == 0);
ejdbclose(jb);
ejdbdel(jb);
jb = ejdbnew();
CU_ASSERT_TRUE_FATAL(ejdbopen(jb, "dbt4_export", JBOWRITER | JBOCREAT | JBOTRUNC));
coll = ejdbcreatecoll(jb, "col1", NULL);
CU_ASSERT_PTR_NOT_NULL_FATAL(coll);
bson_init(&bv1);
bson_append_int(&bv1, "e", 2);
bson_finish(&bv1);
CU_ASSERT_TRUE(ejdbsavebson(coll, &bv1, &oid));
EJQ *q = ejdbcreatequery(jb, &bv1, NULL, 0, NULL);
CU_ASSERT_PTR_NOT_NULL_FATAL(q);
uint32_t count = 0;
ejdbqryexecute(coll, q, &count, JBQRYCOUNT, NULL);
CU_ASSERT_EQUAL(count, 1);
rv = ejdbimport(jb, "testBSONExportImport", NULL, JBIMPORTUPDATE, NULL);
CU_ASSERT_TRUE(rv);
coll = ejdbcreatecoll(jb, "col1", NULL);
ejdbqryexecute(coll, q, &count, JBQRYCOUNT, NULL);
CU_ASSERT_EQUAL(count, 1);
ejdbquerydel(q);
bson_destroy(&bv1);
ejdbclose(jb);
ejdbdel(jb);
bson_del(ometa);
bson_del(nmeta);
tcxstrdel(log);
tclistdel(cnames);
}
/* parse arguments of search command */
static int runsearch(int argc, char **argv){
char *path = NULL;
TCLIST *conds = tcmpoollistnew(tcmpoolglobal());
char *oname = NULL;
char *otype = NULL;
int omode = 0;
int max = -1;
int skip = -1;
bool pv = false;
bool px = false;
bool kw = false;
bool ph = false;
int bt = 0;
bool rm = false;
char *mtype = NULL;
for(int i = 2; i < argc; i++){
if(!path && argv[i][0] == '-'){
if(!strcmp(argv[i], "-nl")){
omode |= TDBONOLCK;
} else if(!strcmp(argv[i], "-nb")){
omode |= TDBOLCKNB;
} else if(!strcmp(argv[i], "-ord")){
if(++i >= argc) usage();
oname = argv[i];
if(++i >= argc) usage();
otype = argv[i];
} else if(!strcmp(argv[i], "-m")){
if(++i >= argc) usage();
max = tcatoix(argv[i]);
} else if(!strcmp(argv[i], "-sk")){
if(++i >= argc) usage();
skip = tcatoix(argv[i]);
} else if(!strcmp(argv[i], "-kw")){
kw = true;
} else if(!strcmp(argv[i], "-pv")){
pv = true;
} else if(!strcmp(argv[i], "-px")){
px = true;
} else if(!strcmp(argv[i], "-ph")){
ph = true;
} else if(!strcmp(argv[i], "-bt")){
if(++i >= argc) usage();
bt = tcatoix(argv[i]);
} else if(!strcmp(argv[i], "-rm")){
rm = true;
} else if(!strcmp(argv[i], "-ms")){
if(++i >= argc) usage();
mtype = argv[i];
} else {
usage();
}
} else if(!path){
path = argv[i];
} else {
tclistpush2(conds, argv[i]);
}
}
if(!path || tclistnum(conds) % 3 != 0) usage();
int rv = procsearch(path, conds, oname, otype, omode, max, skip,
pv, px, kw, ph, bt, rm, mtype);
return rv;
}
/* parse arguments of put command */
static int runput(int argc, char **argv){
char *path = NULL;
char *pkey = NULL;
TCLIST *vals = tcmpoollistnew(tcmpoolglobal());
int omode = 0;
int dmode = 0;
bool sx = false;
for(int i = 2; i < argc; i++){
if(!path && argv[i][0] == '-'){
if(!strcmp(argv[i], "-nl")){
omode |= TDBONOLCK;
} else if(!strcmp(argv[i], "-nb")){
omode |= TDBOLCKNB;
} else if(!strcmp(argv[i], "-dk")){
dmode = -1;
} else if(!strcmp(argv[i], "-dc")){
dmode = 1;
} else if(!strcmp(argv[i], "-dai")){
dmode = 10;
} else if(!strcmp(argv[i], "-dad")){
dmode = 11;
} else if(!strcmp(argv[i], "-sx")){
sx = true;
} else {
usage();
}
} else if(!path){
path = argv[i];
} else if(!pkey){
pkey = argv[i];
} else {
tclistpush2(vals, argv[i]);
}
}
if(!path || !pkey) usage();
TCMAP *cols = tcmapnew();
char *pkbuf;
int pksiz;
if(sx){
pkbuf = tchexdecode(pkey, &pksiz);
for(int i = 0; i < tclistnum(vals) - 1; i += 2){
const char *name = tclistval2(vals, i);
const char *value = tclistval2(vals, i + 1);
int nsiz;
char *nbuf = tchexdecode(name, &nsiz);
int vsiz;
char *vbuf = tchexdecode(value, &vsiz);
tcmapput(cols, nbuf, nsiz, vbuf, vsiz);
tcfree(vbuf);
tcfree(nbuf);
}
} else {
pksiz = strlen(pkey);
pkbuf = tcmemdup(pkey, pksiz);
for(int i = 0; i < tclistnum(vals) - 1; i += 2){
const char *name = tclistval2(vals, i);
const char *value = tclistval2(vals, i + 1);
tcmapput2(cols, name, value);
}
}
int rv = procput(path, pkbuf, pksiz, cols, omode, dmode);
tcmapdel(cols);
tcfree(pkbuf);
return rv;
}
/* perform import command */
static int procimport(const char *dbpath, TCLIST *files, TCLIST *sufs){
TCTDB *tdb = tctdbnew();
if(!tctdbtune(tdb, TUNEBNUM, TUNEAPOW, TUNEFPOW, 0)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
if(!tctdbopen(tdb, dbpath, TDBOWRITER | TDBOCREAT)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(!tctdbsetindex(tdb, "name", TDBITLEXICAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "cdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "mdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "xdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
tclistinvert(files);
char *fpath;
while((fpath = tclistpop2(files)) != NULL){
TCLIST *cfiles = tcreaddir(fpath);
if(cfiles){
tclistsort(cfiles);
for(int i = tclistnum(cfiles) - 1; i >= 0; i--){
const char *cfile = tclistval2(cfiles, i);
bool hit = false;
for(int j = 0; j < tclistnum(sufs); j++){
if(tcstribwm(cfile, tclistval2(sufs, j))){
hit = true;
break;
}
}
if(!hit) continue;
char *lpath = tcsprintf("%s/%s", fpath, cfile);
tclistpush2(files, lpath);
tcfree(lpath);
}
tclistdel(cfiles);
} else {
int isiz;
char *ibuf = tcreadfile(fpath, IOMAXSIZ, &isiz);
if(ibuf){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
const char *name = tcmapget2(cols, "name");
if(name && *name != '\0'){
int64_t id = tcatoi(tcmapget4(cols, "id", ""));
if(dbputart(tdb, id, cols)){
id = tcatoi(tcmapget4(cols, "id", ""));
printf("%s: imported: id=%lld name=%s\n", fpath, (long long)id, name);
} else {
printdberr(tdb);
err = true;
}
} else {
printf("%s: ignored because there is no name\n", fpath);
}
tcmapdel(cols);
tcfree(ibuf);
}
}
tcfree(fpath);
}
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
