XTDBError
XTDBInitHandle(const char* dbName,const char* dir,XTDBHandle** out) {
XTDBError rv;
XTDBHandle* h = malloc(sizeof(XTDBHandle));
int rc;
//InitQueryModule();
if (!h) {
return XTDB_NO_MEM;
}
memset(h,0,sizeof(*h));
StrInit(&h->dbName);
if (StrAppend(&h->dbName,dbName)) {
rv = XTDB_NO_MEM;
goto error;
}
StrInit(&h->dataDir);
if (StrAppend(&h->dataDir,dir)) {
rv = XTDB_NO_MEM;
goto error;
}
StrInit(&h->descName);
if (StrAppendFmt(&h->descName,"%s/%s.desc",dir,dbName)) {
rv = XTDB_NO_MEM;
goto error;
}
StrInit(&h->mainDbFileName);
if (StrAppendFmt(&h->mainDbFileName,"%s/%s.main.data",dir,dbName)) {
rv = XTDB_NO_MEM;
goto error;
}
h->indexes = tcmapnew2(23);
if (!h->indexes) {
rv = XTDB_NO_MEM;
goto error;
}
//h->indexes = hash_table_new(stringhash,string_equal);
//hash_table_register_free_functions(h->indexes,free,NULL);
if (!(h->mainDB = DBInit(h->mainDbFileName.ptr,BDBOCREAT|BDBOWRITER))) {
rv = XTDB_NO_MEM;
goto error;
}
if (!(rc=DBOpen(h->mainDB,BDBOCREAT|BDBOWRITER))) {
rv = DBGetLastError(h->mainDB);
goto error;
}
if (!XTDBLoadIndexes(h)) {
rv = h->error;
goto error;
}
*out = h;
return XTDB_OK;
error:
XTDBFreeHandle(h);
*out = NULL;
return rv;
}
void *utf8fan2jian_init(const char *unihan)
{
char *data,*hzbuf,*hz,*hzfan,*hzjian;
TCMAP *map;
int hznum,hzlen;
data = loadfile(unihan,NULL);
if(!data) return NULL;
hznum = getlinenum(data,"\n");
map = tcmapnew2(hznum);
hzbuf = data;
while((hz = getaline(hzbuf,"\n", &hzlen)) != NULL) {
hzjian = strchr(hz,'=');
if(!hzjian) return NULL;
*hzjian++ = 0;
hzfan = hz;
trim(hzfan);
trim(hzjian);
tcmapput2(map,hzfan,hzjian);
hzbuf = hz + hzlen + 1;
}
xfree(data);
return map;
}
int sandbox_init()
{
if (_g_sandbox)
return 0;
_g_sandbox = (struct _sandbox *)malloc(sizeof(struct _sandbox));
memset(_g_sandbox, 0, sizeof(struct _sandbox));
_g_sandbox->event_listeners = tcmapnew2(32);
_g_sandbox->call_listeners = tcmapnew2(32);
_g_sandbox->call = sandbox_call;
_g_sandbox->notify = sandbox_notify;
_g_sandbox->call_listen = sandbox_call_listen;
_g_sandbox->listen = sandbox_listen;
_g_sandbox->listen2 = sandbox_listen2;
return 0;
}
/* putimpl */
JNIEXPORT jboolean JNICALL Java_tokyotyrant_TCRDB_putimpl
(JNIEnv *env, jobject self, jbyteArray pkey, jobjectArray cols, jint dmode){
if(!pkey || !cols){
throwillarg(env);
return false;
}
TCRDB *tcrdb = (TCRDB *)(intptr_t)(*env)->GetLongField(env, self, tcrdb_fid_ptr);
jboolean ick;
jbyte *kbuf = (*env)->GetByteArrayElements(env, pkey, &ick);
if(!kbuf){
throwoutmem(env);
return false;
}
int ksiz = (*env)->GetArrayLength(env, pkey);
jsize cnum = (*env)->GetArrayLength(env, cols);
TCMAP *tcols = tcmapnew2(cnum + 1);
cnum--;
for(int i = 0; i < cnum; i += 2){
jobject name = (*env)->GetObjectArrayElement(env, cols, i);
jboolean icn;
jbyte *nbuf = (*env)->GetByteArrayElements(env, name, &icn);
if(!nbuf){
throwoutmem(env);
return false;
}
int nsiz = (*env)->GetArrayLength(env, name);
jobject val = (*env)->GetObjectArrayElement(env, cols, i + 1);
jboolean icv;
jbyte *vbuf = (*env)->GetByteArrayElements(env, val, &icv);
if(!vbuf){
throwoutmem(env);
return false;
}
int vsiz = (*env)->GetArrayLength(env, val);
tcmapputkeep(tcols, nbuf, nsiz, vbuf, vsiz);
if(icv) (*env)->ReleaseByteArrayElements(env, val, vbuf, JNI_ABORT);
if(icn) (*env)->ReleaseByteArrayElements(env, name, nbuf, JNI_ABORT);
}
bool rv;
switch(dmode){
case 0:
rv = tcrdbtblput(tcrdb, kbuf, ksiz, tcols);
break;
case 1:
rv = tcrdbtblputkeep(tcrdb, kbuf, ksiz, tcols);
break;
case 2:
rv = tcrdbtblputcat(tcrdb, kbuf, ksiz, tcols);
break;
default:
rv = false;
break;
}
tcmapdel(tcols);
if(ick) (*env)->ReleaseByteArrayElements(env, pkey, kbuf, JNI_ABORT);
return rv;
}
/* Open a word database object.
`wdb' specifies the word database object.
`path' specifies the path of the database file.
`omode' specifies the connection mode.
If successful, the return value is true, else, it is false. */
static bool tcwdbopenimpl(TCWDB *wdb, const char *path, int omode){
assert(wdb && path);
int bomode = BDBOREADER;
if(omode & WDBOWRITER){
bomode = BDBOWRITER;
if(omode & WDBOCREAT) bomode |= BDBOCREAT;
if(omode & WDBOTRUNC) bomode |= BDBOTRUNC;
int64_t bnum = (wdb->etnum / WDBLMEMB) * 2 + 1;
int bopts = 0;
if(wdb->opts & WDBTLARGE) bopts |= BDBTLARGE;
if(wdb->opts & WDBTDEFLATE) bopts |= BDBTDEFLATE;
if(wdb->opts & WDBTBZIP) bopts |= BDBTBZIP;
if(wdb->opts & WDBTTCBS) bopts |= BDBTTCBS;
if(!tcbdbtune(wdb->idx, WDBLMEMB, WDBNMEMB, bnum, WDBAPOW, WDBFPOW, bopts)) return false;
if(!tcbdbsetlsmax(wdb->idx, WDBLSMAX)) return false;
}
if(wdb->lcnum > 0){
if(!tcbdbsetcache(wdb->idx, wdb->lcnum, wdb->lcnum / 4 + 1)) return false;
} else {
if(!tcbdbsetcache(wdb->idx, (omode & WDBOWRITER) ? WDBLCNUMW : WDBLCNUMR, WDBNCNUM))
return false;
}
if(omode & WDBONOLCK) bomode |= BDBONOLCK;
if(omode & WDBOLCKNB) bomode |= BDBOLCKNB;
if(!tcbdbopen(wdb->idx, path, bomode)) return false;
if((omode & WDBOWRITER) && tcbdbrnum(wdb->idx) < 1){
memcpy(tcbdbopaque(wdb->idx), WDBMAGICDATA, strlen(WDBMAGICDATA));
} else if(!(omode & WDBONOLCK) &&
memcmp(tcbdbopaque(wdb->idx), WDBMAGICDATA, strlen(WDBMAGICDATA))){
tcbdbclose(wdb->idx);
tcbdbsetecode(wdb->idx, TCEMETA, __FILE__, __LINE__, __func__);
return 0;
}
if(omode & WDBOWRITER){
wdb->cc = tcmapnew2(WDBCCBNUM);
wdb->dtokens = tcmapnew2(WDBDTKNBNUM);
wdb->dids = tcidsetnew(WDBDIDSBNUM);
}
wdb->open = true;
return true;
}
extern TCMAP *vhashtomap(VALUE vhash){
VALUE vkeys, vkey, vval;
TCMAP *map;
int i, num;
map = tcmapnew2(31);
vkeys = rb_funcall(vhash, rb_intern("keys"), 0);
num = RARRAY_LEN(vkeys);
for(i = 0; i < num; i++){
vkey = rb_ary_entry(vkeys, i);
vval = rb_hash_aref(vhash, vkey);
vkey = StringValueEx(vkey);
vval = StringValueEx(vval);
tcmapput(map, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vval), RSTRING_LEN(vval));
}
return map;
}
/* perform convert command */
static int procconvert(const char *ibuf, int isiz, int fmt,
const char *buri, const char *duri, bool page){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
if(fmt == FMTWIKI){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
wikidump(rbuf, cols);
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTTEXT){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page)
tcxstrprintf(rbuf, "------------------------ Tokyo Promenade ------------------------\n");
wikidumptext(rbuf, cols);
if(page)
tcxstrprintf(rbuf, "-----------------------------------------------------------------\n");
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTHTML){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page){
const char *name = tcmapget2(cols, "name");
tcxstrprintf(rbuf, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
tcxstrprintf(rbuf, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\""
" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
tcxstrprintf(rbuf, "<html xmlns=\"http://www.w3.org/1999/xhtml\""
" xml:lang=\"en\" lang=\"en\">\n");
tcxstrprintf(rbuf, "<head>\n");
tcxstrprintf(rbuf, "<meta http-equiv=\"Content-Type\""
" content=\"text/html; charset=UTF-8\" />\n");
tcxstrprintf(rbuf, "<link rel=\"contents\" href=\"%@\" />\n", buri);
tcxstrprintf(rbuf, "<title>%@</title>\n", name ? name : "Tokyo Promenade");
tcxstrprintf(rbuf, "</head>\n");
tcxstrprintf(rbuf, "<body>\n");
}
wikidumphtml(rbuf, cols, buri, 0, duri);
if(page){
tcxstrprintf(rbuf, "</body>\n");
tcxstrprintf(rbuf, "</html>\n");
}
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
}
tcmapdel(cols);
return 0;
}
/* Store a record into a q-gram database object.
`wdb' specifies the q-gram database object.
`id' specifies the ID number of the record.
`words' specifies a list object contains the words of the record.
If successful, the return value is true, else, it is false. */
static bool tcwdbputimpl(TCWDB *wdb, int64_t id, const TCLIST *words){
assert(wdb && id > 0 && words);
char idbuf[TDNUMBUFSIZ*2];
int idsiz;
TDSETVNUMBUF64(idsiz, idbuf, id);
TCMAP *cc = wdb->cc;
int wn = tclistnum(words);
TCMAP *uniq = tcmapnew2(wn + 1);
for(int i = 0; i < wn; i++){
int wsiz;
const char *word = tclistval(words, i, &wsiz);
if(!tcmapputkeep(uniq, word, wsiz, "", 0)) continue;
if(*word != '\0') tcmapputcat(cc, word, wsiz, idbuf, idsiz);
}
tcmapdel(uniq);
bool err = false;
if(tcmapmsiz(cc) >= wdb->icsiz && !tcwdbmemsync(wdb, 1)) err = true;
return !err;
}
/* perform update command */
static int procupdate(const char *dbpath, int64_t id, const char *wiki){
TCTDB *tdb = tctdbnew();
if(!tctdbopen(tdb, dbpath, TDBOWRITER)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, wiki);
if(!dbputart(tdb, id, cols)){
printdberr(tdb);
err = true;
}
tcmapdel(cols);
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* kwicimpl */
JNIEXPORT jobjectArray JNICALL Java_tokyocabinet_TDBQRY_kwicimpl
(JNIEnv *env, jobject self, jobjectArray cols, jstring name, jint width, jint opts){
if(!cols){
throwillarg(env);
return NULL;
}
jclass clsstring = name ?
(*env)->GetObjectClass(env, name) : (*env)->FindClass(env, CLSSTRING);
TDBQRY *qry = (TDBQRY *)(intptr_t)(*env)->GetLongField(env, self, tdbqry_fid_ptr);
jsize cnum = (*env)->GetArrayLength(env, cols);
TCMAP *tcols = tcmapnew2(cnum + 1);
jboolean icc = false;
const char *cbuf = NULL;
int csiz = 0;
if(name){
cbuf = (*env)->GetStringUTFChars(env, name, &icc);
if(!cbuf){
throwoutmem(env);
return NULL;
}
csiz = strlen(cbuf);
}
cnum--;
for(int i = 0; i < cnum; i += 2){
name = (*env)->GetObjectArrayElement(env, cols, i);
jboolean icn;
jbyte *nbuf = (*env)->GetByteArrayElements(env, name, &icn);
if(!nbuf){
throwoutmem(env);
return NULL;
}
int nsiz = (*env)->GetArrayLength(env, name);
jobject val = (*env)->GetObjectArrayElement(env, cols, i + 1);
jboolean icv;
jbyte *vbuf = (*env)->GetByteArrayElements(env, val, &icv);
if(!vbuf){
throwoutmem(env);
return false;
}
int vsiz = (*env)->GetArrayLength(env, val);
if(!cbuf || (nsiz == csiz && !memcmp(nbuf, cbuf, nsiz)))
tcmapputkeep(tcols, nbuf, nsiz, vbuf, vsiz);
if(icv) (*env)->ReleaseByteArrayElements(env, val, vbuf, JNI_ABORT);
if(icn) (*env)->ReleaseByteArrayElements(env, name, nbuf, JNI_ABORT);
(*env)->DeleteLocalRef(env, val);
(*env)->DeleteLocalRef(env, name);
}
if(width < 0){
width = 1 << 30;
opts |= TCKWNOOVER | TCKWPULEAD;
}
TCLIST *texts = tctdbqrykwic(qry, tcols, cbuf, width, opts);
int tnum = tclistnum(texts);
jobjectArray ary = (*env)->NewObjectArray(env, tnum, clsstring, NULL);
for(int i = 0; i < tnum; i++){
jobject text = (*env)->NewStringUTF(env, tclistval2(texts, i));
(*env)->SetObjectArrayElement(env, ary, i, text);
(*env)->DeleteLocalRef(env, text);
}
tclistdel(texts);
if(icc) (*env)->ReleaseStringUTFChars(env, name, cbuf);
tcmapdel(tcols);
return ary;
}
/* thread the typical function */
static void *threadtypical(void *targ){
TCFDB *fdb = ((TARGTYPICAL *)targ)->fdb;
int rnum = ((TARGTYPICAL *)targ)->rnum;
bool nc = ((TARGTYPICAL *)targ)->nc;
int rratio = ((TARGTYPICAL *)targ)->rratio;
int id = ((TARGTYPICAL *)targ)->id;
bool err = false;
TCMAP *map = (!nc && id == 0) ? tcmapnew2(rnum + 1) : NULL;
int base = id * rnum;
int mrange = tclmax(50 + rratio, 100);
int width = tcfdbwidth(fdb);
for(int i = 1; !err && i <= rnum; i++){
char buf[RECBUFSIZ];
int len = sprintf(buf, "%08d", base + myrandnd(i) + 1);
int rnd = myrand(mrange);
if(rnd < 10){
if(!tcfdbput2(fdb, buf, len, buf, len)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
if(map) tcmapput(map, buf, len, buf, len);
} else if(rnd < 15){
if(!tcfdbputkeep2(fdb, buf, len, buf, len) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep2");
err = true;
}
if(map) tcmapputkeep(map, buf, len, buf, len);
} else if(rnd < 20){
if(!tcfdbputcat2(fdb, buf, len, buf, len)){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
}
if(map) tcmapputcat(map, buf, len, buf, len);
} else if(rnd < 25){
if(!tcfdbout2(fdb, buf, len) && tcfdbecode(fdb) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
}
if(map) tcmapout(map, buf, len);
} else if(rnd < 26){
if(myrand(10) == 0 && !tcfdbiterinit(fdb) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
for(int j = 0; !err && j < 10; j++){
if(tcfdbiternext(fdb) < 1 &&
tcfdbecode(fdb) != TCEINVALID && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbiternext");
err = true;
}
}
} else {
int vsiz;
char *vbuf = tcfdbget2(fdb, buf, len, &vsiz);
if(vbuf){
if(map){
int msiz = 0;
const char *mbuf = tcmapget(map, buf, len, &msiz);
if(msiz > width) msiz = width;
if(!mbuf || msiz != vsiz || memcmp(mbuf, vbuf, vsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
}
tcfree(vbuf);
} else {
if(tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
}
if(map && tcmapget(map, buf, len, &vsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
}
}
if(id == 0 && rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) iprintf(" (%08d)\n", i);
}
}
if(map){
tcmapiterinit(map);
int ksiz;
const char *kbuf;
while(!err && (kbuf = tcmapiternext(map, &ksiz)) != NULL){
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
if(vbuf){
int msiz = 0;
const char *mbuf = tcmapget(map, kbuf, ksiz, &msiz);
if(msiz > width) msiz = width;
if(!mbuf || msiz != vsiz || memcmp(mbuf, vbuf, vsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
tcfree(vbuf);
} else {
eprint(fdb, __LINE__, "(validation)");
err = true;
}
}
tcmapdel(map);
}
return err ? "error" : NULL;
}
/* thread the typical function */
static void *threadtypical(void *targ){
TCMDB *mdb = ((TARGTYPICAL *)targ)->mdb;
TCNDB *ndb = ((TARGCOMBO *)targ)->ndb;
int rnum = ((TARGTYPICAL *)targ)->rnum;
bool nc = ((TARGTYPICAL *)targ)->nc;
int rratio = ((TARGTYPICAL *)targ)->rratio;
int id = ((TARGTYPICAL *)targ)->id;
bool err = false;
TCMAP *map = (!nc && id == 0) ? tcmapnew2(rnum + 1) : NULL;
int base = id * rnum;
int mrange = tclmax(50 + rratio, 100);
for(int i = 1; !err && i <= rnum; i++){
char buf[RECBUFSIZ];
int len = sprintf(buf, "%08d", base + myrandnd(i));
int rnd = myrand(mrange);
if(rnd < 10){
if(ndb){
tcndbput(ndb, buf, len, buf, len);
} else {
tcmdbput(mdb, buf, len, buf, len);
}
if(map) tcmapput(map, buf, len, buf, len);
} else if(rnd < 15){
if(ndb){
tcndbputkeep(ndb, buf, len, buf, len);
} else {
tcmdbputkeep(mdb, buf, len, buf, len);
}
if(map) tcmapputkeep(map, buf, len, buf, len);
} else if(rnd < 20){
if(ndb){
tcndbputcat(ndb, buf, len, buf, len);
} else {
tcmdbputcat(mdb, buf, len, buf, len);
}
if(map) tcmapputcat(map, buf, len, buf, len);
} else if(rnd < 30){
if(ndb){
tcndbout(ndb, buf, len);
} else {
tcmdbout(mdb, buf, len);
}
if(map) tcmapout(map, buf, len);
} else if(rnd < 31){
if(myrand(10) == 0) tcmdbiterinit(mdb);
for(int j = 0; !err && j < 10; j++){
int ksiz;
char *kbuf = ndb ? tcndbiternext(ndb, &ksiz) : tcmdbiternext(mdb, &ksiz);
if(kbuf) tcfree(kbuf);
}
} else {
int vsiz;
char *vbuf = ndb ? tcndbget(ndb, buf, len, &vsiz) : tcmdbget(mdb, buf, len, &vsiz);
if(vbuf){
if(map){
int msiz;
const char *mbuf = tcmapget(map, buf, len, &msiz);
if(msiz != vsiz || memcmp(mbuf, vbuf, vsiz)){
eprint(__LINE__, "(validation)");
err = true;
}
}
tcfree(vbuf);
} else {
if(map && tcmapget(map, buf, len, &vsiz)){
eprint(__LINE__, "(validation)");
err = true;
}
}
}
if(id == 0 && rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) iprintf(" (%08d)\n", i);
}
}
if(map){
tcmapiterinit(map);
int ksiz;
const char *kbuf;
while(!err && (kbuf = tcmapiternext(map, &ksiz)) != NULL){
int vsiz;
char *vbuf = ndb ? tcndbget(ndb, kbuf, ksiz, &vsiz) : tcmdbget(mdb, kbuf, ksiz, &vsiz);
if(vbuf){
int msiz;
const char *mbuf = tcmapget(map, kbuf, ksiz, &msiz);
if(!mbuf || msiz != vsiz || memcmp(mbuf, vbuf, vsiz)){
eprint(__LINE__, "(validation)");
err = true;
}
tcfree(vbuf);
} else {
eprint(__LINE__, "(validation)");
err = true;
}
}
tcmapdel(map);
}
return err ? "error" : NULL;
}
/* perform tblwrite command */
int dotblwrite(char *name, int rnum){
TCTDB *tdb;
int i, err, pksiz, vsiz;
char pkbuf[RECBUFSIZ], vbuf[RECBUFSIZ];
TCMAP *cols;
if(showprgr) printf("<Writing Test of Table>\n name=%s rnum=%d\n\n", name, rnum);
/* open a database */
tdb = tctdbnew();
tctdbtune(tdb, rnum * 3, 0, 0, 0);
tctdbsetxmsiz(tdb, rnum * 80);
tctdbsetcache(tdb, -1, rnum / 100, -1);
if(!tctdbopen(tdb, name, TDBOWRITER | TDBOCREAT | TDBOTRUNC)){
fprintf(stderr, "tctdbopen failed\n");
tctdbdel(tdb);
return 1;
}
if(!tctdbsetindex(tdb, "s", TDBITLEXICAL)){
fprintf(stderr, "tctdbsetindex failed\n");
tctdbdel(tdb);
return 1;
}
if(!tctdbsetindex(tdb, "n", TDBITDECIMAL)){
fprintf(stderr, "tctdbsetindex failed\n");
tctdbdel(tdb);
return 1;
}
err = FALSE;
/* loop for each record */
for(i = 1; i <= rnum; i++){
/* store a record */
pksiz = sprintf(pkbuf, "%d", i);
cols = tcmapnew2(7);
vsiz = sprintf(vbuf, "%08d", i);
tcmapput(cols, "s", 1, vbuf, vsiz);
vsiz = sprintf(vbuf, "%08d", myrand() % i);
tcmapput(cols, "n", 1, vbuf, vsiz);
vsiz = sprintf(vbuf, "%08d", i);
tcmapput(cols, "t", 1, vbuf, vsiz);
vsiz = sprintf(vbuf, "%08d", myrand() % rnum);
tcmapput(cols, "f", 1, vbuf, vsiz);
if(!tctdbput(tdb, pkbuf, pksiz, cols)){
fprintf(stderr, "tctdbput failed\n");
err = TRUE;
break;
}
tcmapdel(cols);
/* print progression */
if(showprgr && rnum > 250 && i % (rnum / 250) == 0){
putchar('.');
fflush(stdout);
if(i == rnum || i % (rnum / 10) == 0){
printf(" (%08d)\n", i);
fflush(stdout);
}
}
}
/* close the database */
if(!tctdbclose(tdb)){
fprintf(stderr, "tctdbclose failed\n");
tctdbdel(tdb);
return 1;
}
tctdbdel(tdb);
if(showprgr && !err) printf("ok\n\n");
return err ? 1 : 0;
}
/* perform import command */
static int procimport(const char *dbpath, TCLIST *files, TCLIST *sufs){
TCTDB *tdb = tctdbnew();
if(!tctdbtune(tdb, TUNEBNUM, TUNEAPOW, TUNEFPOW, 0)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
if(!tctdbopen(tdb, dbpath, TDBOWRITER | TDBOCREAT)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(!tctdbsetindex(tdb, "name", TDBITLEXICAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "cdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "mdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "xdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
tclistinvert(files);
char *fpath;
while((fpath = tclistpop2(files)) != NULL){
TCLIST *cfiles = tcreaddir(fpath);
if(cfiles){
tclistsort(cfiles);
for(int i = tclistnum(cfiles) - 1; i >= 0; i--){
const char *cfile = tclistval2(cfiles, i);
bool hit = false;
for(int j = 0; j < tclistnum(sufs); j++){
if(tcstribwm(cfile, tclistval2(sufs, j))){
hit = true;
break;
}
}
if(!hit) continue;
char *lpath = tcsprintf("%s/%s", fpath, cfile);
tclistpush2(files, lpath);
tcfree(lpath);
}
tclistdel(cfiles);
} else {
int isiz;
char *ibuf = tcreadfile(fpath, IOMAXSIZ, &isiz);
if(ibuf){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
const char *name = tcmapget2(cols, "name");
if(name && *name != '\0'){
int64_t id = tcatoi(tcmapget4(cols, "id", ""));
if(dbputart(tdb, id, cols)){
id = tcatoi(tcmapget4(cols, "id", ""));
printf("%s: imported: id=%lld name=%s\n", fpath, (long long)id, name);
} else {
printdberr(tdb);
err = true;
}
} else {
printf("%s: ignored because there is no name\n", fpath);
}
tcmapdel(cols);
tcfree(ibuf);
}
}
tcfree(fpath);
}
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* perform wicked command */
static int procwicked(const char *path, int rnum, bool mt, int omode){
my_my_my_iprintf("<Wicked Writing Test>\n seed=%u path=%s rnum=%d mt=%d omode=%d\n\n",
g_randseed, path, rnum, mt, omode);
bool err = false;
double stime = tctime();
TCFDB *fdb = tcfdbnew();
if(g_dbgfd >= 0) tcfdbsetdbgfd(fdb, g_dbgfd);
if(mt && !tcfdbsetmutex(fdb)){
eprint(fdb, __LINE__, "tcfdbsetmutex");
err = true;
}
if(!tcfdbtune(fdb, RECBUFSIZ * 2, EXHEADSIZ + (RECBUFSIZ * 2 + sizeof(int)) * rnum)){
eprint(fdb, __LINE__, "tcfdbtune");
err = true;
}
if(!tcfdbopen(fdb, path, FDBOWRITER | FDBOCREAT | FDBOTRUNC | omode)){
eprint(fdb, __LINE__, "tcfdbopen");
err = true;
}
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
TCMAP *map = tcmapnew2(rnum / 5);
for(int i = 1; i <= rnum && !err; i++){
uint64_t id = myrand(rnum) + 1;
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%llu", (unsigned long long)id);
char vbuf[RECBUFSIZ];
int vsiz = myrand(RECBUFSIZ);
memset(vbuf, '*', vsiz);
vbuf[vsiz] = '\0';
char *rbuf;
switch(myrand(16)){
case 0:
iputchar('0');
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
tcmapput(map, kbuf, ksiz, vbuf, vsiz);
break;
case 1:
iputchar('1');
if(!tcfdbput3(fdb, kbuf, vbuf)){
eprint(fdb, __LINE__, "tcfdbput3");
err = true;
}
tcmapput2(map, kbuf, vbuf);
break;
case 2:
iputchar('2');
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep2");
err = true;
}
tcmapputkeep(map, kbuf, ksiz, vbuf, vsiz);
break;
case 3:
iputchar('3');
if(!tcfdbputkeep3(fdb, kbuf, vbuf) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep3");
err = true;
}
tcmapputkeep2(map, kbuf, vbuf);
break;
case 4:
iputchar('4');
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
}
tcmapputcat(map, kbuf, ksiz, vbuf, vsiz);
break;
case 5:
iputchar('5');
if(!tcfdbputcat3(fdb, kbuf, vbuf)){
eprint(fdb, __LINE__, "tcfdbputcat3");
err = true;
}
tcmapputcat2(map, kbuf, vbuf);
break;
case 6:
iputchar('6');
if(myrand(10) == 0){
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
tcmapout(map, kbuf, ksiz);
}
break;
case 7:
iputchar('7');
if(myrand(10) == 0){
if(!tcfdbout3(fdb, kbuf) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout3");
err = true;
}
tcmapout2(map, kbuf);
}
break;
case 8:
iputchar('8');
if(!(rbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz))){
if(tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget2");
err = true;
}
rbuf = tcsprintf("[%d]", myrand(i + 1));
vsiz = strlen(rbuf);
}
vsiz += myrand(vsiz);
rbuf = tcrealloc(rbuf, vsiz + 1);
for(int j = 0; j < vsiz; j++){
rbuf[j] = myrand(0x100);
}
if(!tcfdbput2(fdb, kbuf, ksiz, rbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
tcmapput(map, kbuf, ksiz, rbuf, vsiz);
tcfree(rbuf);
break;
case 9:
iputchar('9');
if(!(rbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz)) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget2");
err = true;
}
tcfree(rbuf);
break;
case 10:
iputchar('A');
if(!(rbuf = tcfdbget3(fdb, kbuf)) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget3");
err = true;
}
tcfree(rbuf);
break;
case 11:
iputchar('B');
if(myrand(1) == 0) vsiz = 1;
if((vsiz = tcfdbget4(fdb, id, vbuf, vsiz)) < 0 && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget4");
err = true;
}
break;
case 12:
iputchar('C');
if(myrand(rnum / 128) == 0){
if(myrand(2) == 0){
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
} else {
if(!tcfdbiterinit2(fdb, myrand(rnum) + 1) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbiterinit2");
err = true;
}
}
}
for(int j = myrand(rnum) / 1000 + 1; j >= 0; j--){
if(tcfdbiternext(fdb) < 0){
int ecode = tcfdbecode(fdb);
if(ecode != TCEINVALID && ecode != TCENOREC){
eprint(fdb, __LINE__, "tcfdbiternext");
err = true;
}
}
}
break;
default:
iputchar('@');
if(myrand(10000) == 0) srand((unsigned int)(tctime() * 1000) % UINT_MAX);
if(myrand(rnum / 16 + 1) == 0){
int cnt = myrand(30);
for(int j = 0; j < rnum && !err; j++){
ksiz = sprintf(kbuf, "%d", i + j);
if(tcfdbout2(fdb, kbuf, ksiz)){
cnt--;
} else if(tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
tcmapout(map, kbuf, ksiz);
if(cnt < 0) break;
}
}
break;
}
if(i % 50 == 0) my_my_my_iprintf(" (%08d)\n", i);
if(i == rnum / 2){
if(!tcfdbclose(fdb)){
eprint(fdb, __LINE__, "tcfdbclose");
err = true;
}
if(!tcfdbopen(fdb, path, FDBOWRITER | omode)){
eprint(fdb, __LINE__, "tcfdbopen");
err = true;
}
} else if(i == rnum / 4){
char *npath = tcsprintf("%s-tmp", path);
if(!tcfdbcopy(fdb, npath)){
eprint(fdb, __LINE__, "tcfdbcopy");
err = true;
}
TCFDB *nfdb = tcfdbnew();
if(!tcfdbopen(nfdb, npath, FDBOREADER | omode)){
eprint(nfdb, __LINE__, "tcfdbopen");
err = true;
}
tcfdbdel(nfdb);
unlink(npath);
tcfree(npath);
if(!tcfdboptimize(fdb, RECBUFSIZ, -1)){
eprint(fdb, __LINE__, "tcfdboptimize");
err = true;
}
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
} else if(i == rnum / 8){
if(!tcfdbtranbegin(fdb)){
eprint(fdb, __LINE__, "tcfdbtranbegin");
err = true;
}
} else if(i == rnum / 8 + rnum / 16){
if(!tcfdbtrancommit(fdb)){
eprint(fdb, __LINE__, "tcfdbtrancommit");
err = true;
}
}
}
if(rnum % 50 > 0) my_my_my_iprintf(" (%08d)\n", rnum);
if(!tcfdbsync(fdb)){
eprint(fdb, __LINE__, "tcfdbsync");
err = true;
}
if(tcfdbrnum(fdb) != tcmaprnum(map)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
for(int i = 1; i <= rnum && !err; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", i);
int vsiz;
const char *vbuf = tcmapget(map, kbuf, ksiz, &vsiz);
int rsiz;
char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz);
if(vbuf){
iputchar('.');
if(vsiz > RECBUFSIZ) vsiz = RECBUFSIZ;
if(!rbuf){
eprint(fdb, __LINE__, "tcfdbget2");
err = true;
} else if(rsiz != vsiz || memcmp(rbuf, vbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
} else {
iputchar('*');
if(rbuf || tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
}
tcfree(rbuf);
if(i % 50 == 0) my_my_my_iprintf(" (%08d)\n", i);
}
if(rnum % 50 > 0) my_my_my_iprintf(" (%08d)\n", rnum);
tcmapiterinit(map);
int ksiz;
const char *kbuf;
for(int i = 1; (kbuf = tcmapiternext(map, &ksiz)) != NULL; i++){
iputchar('+');
int vsiz;
const char *vbuf = tcmapiterval(kbuf, &vsiz);
if(vsiz > tcfdbwidth(fdb)) vsiz = tcfdbwidth(fdb);
int rsiz;
char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz);
if(!rbuf){
eprint(fdb, __LINE__, "tcfdbget2");
err = true;
} else if(rsiz != vsiz || memcmp(rbuf, vbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
tcfree(rbuf);
if(!tcfdbout2(fdb, kbuf, ksiz)){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
if(i % 50 == 0) my_my_my_iprintf(" (%08d)\n", i);
}
int mrnum = tcmaprnum(map);
if(mrnum % 50 > 0) my_my_my_iprintf(" (%08d)\n", mrnum);
if(tcfdbrnum(fdb) != 0){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
my_my_my_iprintf("record number: %llu\n", (unsigned long long)tcfdbrnum(fdb));
my_my_my_iprintf("size: %llu\n", (unsigned long long)tcfdbfsiz(fdb));
mprint(fdb);
sysprint();
tcmapdel(map);
if(!tcfdbclose(fdb)){
eprint(fdb, __LINE__, "tcfdbclose");
err = true;
}
tcfdbdel(fdb);
my_my_my_iprintf("time: %.3f\n", tctime() - stime);
my_my_my_iprintf("%s\n\n", err ? "error" : "ok");
return err ? 1 : 0;
}