/* perform list command */
static int proclist(const char *path, int omode, bool pv, bool px){
TCHDB *hdb = tchdbnew();
if(g_dbgfd >= 0) tchdbsetdbgfd(hdb, g_dbgfd);
if(!tchdbopen(hdb, path, HDBOREADER | omode)){
printerr(hdb);
tchdbdel(hdb);
return 1;
}
bool err = false;
if(!tchdbiterinit(hdb)){
printerr(hdb);
err = true;
}
TCXSTR *key = tcxstrnew();
TCXSTR *val = tcxstrnew();
while(tchdbiternext3(hdb, key, val)){
printdata(tcxstrptr(key), tcxstrsize(key), px);
if(pv){
putchar('\t');
printdata(tcxstrptr(val), tcxstrsize(val), px);
}
putchar('\n');
}
tcxstrdel(val);
tcxstrdel(key);
if(!tchdbclose(hdb)){
if(!err) printerr(hdb);
err = true;
}
tchdbdel(hdb);
return err ? 1 : 0;
}
static PyObject *
list2(PyObject *self,PyObject *args){
const char *dbname;
if (!PyArg_ParseTuple(args, "s", &dbname))
return NULL;
TCADB *adb = openadb(dbname);
PyObject* pList = PyList_New(0);
if(!tcadbiterinit(adb)){
return NULL;
}
BDBCUR *cur = tcbdbcurnew(adb->bdb);
TCXSTR *key = tcxstrnew();
TCXSTR *val = tcxstrnew();
if(!tcbdbcurfirst(cur) && tcbdbecode(adb->bdb) != TCENOREC){
return NULL;
}
while(tcbdbcurrec(cur, key, val)){
PyObject* pList1 = PyList_New(0);
PyList_Append(pList1,Py_BuildValue("s",tcxstrptr(key)));
PyList_Append(pList1,Py_BuildValue("s",tcxstrptr(val)));
PyList_Append(pList,pList1);
if(!tcbdbcurnext(cur) && tcbdbecode(adb->bdb) != TCENOREC){
return NULL;
}
}
tcxstrdel(val);
tcxstrdel(key);
tcbdbcurdel(cur);
closeadb(adb);
return Py_BuildValue("O",pList);
}
static void do_mc_list(TTSOCK *sock, TASKARG *arg, TTREQ *req, char **tokens, int tnum){
ttservlog(g_serv, TTLOGDEBUG, "doing mc_list command");
TCADB *adb = arg->adb;
if(tnum < 1){
ttsockprintf(sock, "CLIENT_ERROR error\r\n");
return;
}
TCXSTR *xstr = tcxstrnew();
TCXSTR * head = tcxstrnew();
pthread_cleanup_push((void (*)(void *))tcxstrdel, xstr);
pthread_cleanup_push((void (*)(void *))tcxstrdel, head);
tcadbiterinit(adb);
char * key;
int list_size;
while((key=tcadbiternext2(adb))!=NULL){
tcxstrcat(xstr, key, strlen(key));
tcxstrcat(xstr,"\r\n",2);
}
list_size=tcxstrsize(xstr);
tcxstrprintf(head,"LIST %d\r\n",list_size);
tcxstrcat(xstr, "END\r\n",5);
if( ttsocksend(sock, tcxstrptr(head), tcxstrsize(head)) && ttsocksend(sock,tcxstrptr(xstr),tcxstrsize(xstr)) ){
req->keep = true;
} else {
ttservlog(g_serv, TTLOGINFO, "do_mc_list: response failed");
return ;
}
pthread_cleanup_pop(1);
pthread_cleanup_pop(1);
}
static gearman_return_t _libtokyocabinet_add(gearman_server_st *server, void *context,
const void *unique,
size_t unique_size,
const void *function_name,
size_t function_name_size,
const void *data, size_t data_size,
gearman_job_priority_t priority)
{
gearman_queue_libtokyocabinet_st *queue= (gearman_queue_libtokyocabinet_st *)context;
bool rc;
TCXSTR *key;
TCXSTR *job_data;
gearman_log_debug(server->gearman, "libtokyocabinet add: %.*s", (uint32_t)unique_size, (char *)unique);
char key_str[GEARMAN_QUEUE_TOKYOCABINET_MAX_KEY_LEN];
size_t key_length= (size_t)snprintf(key_str, GEARMAN_QUEUE_TOKYOCABINET_MAX_KEY_LEN, "%.*s-%.*s",
(int)function_name_size,
(const char *)function_name, (int)unique_size,
(const char *)unique);
key= tcxstrnew();
tcxstrcat(key, key_str, (int)key_length);
gearman_log_debug(server->gearman, "libtokyocabinet key: %.*s", (int)key_length, key_str);
job_data= tcxstrnew();
tcxstrcat(job_data, (const char *)function_name, (int)function_name_size);
tcxstrcat(job_data, "\0", 1);
tcxstrcat(job_data, (const char *)unique, (int)unique_size);
tcxstrcat(job_data, "\0", 1);
switch (priority)
{
case GEARMAN_JOB_PRIORITY_HIGH:
case GEARMAN_JOB_PRIORITY_MAX:
tcxstrcat2(job_data,"0");
break;
case GEARMAN_JOB_PRIORITY_LOW:
tcxstrcat2(job_data,"2");
break;
case GEARMAN_JOB_PRIORITY_NORMAL:
default:
tcxstrcat2(job_data,"1");
}
tcxstrcat(job_data, (const char *)data, (int)data_size);
rc= tcadbput(queue->db, tcxstrptr(key), tcxstrsize(key),
tcxstrptr(job_data), tcxstrsize(job_data));
tcxstrdel(key);
tcxstrdel(job_data);
if (!rc)
return GEARMAN_QUEUE_ERROR;
return GEARMAN_SUCCESS;
}
/* perform list command */
static int proclist(const char *path, int omode, int max, bool pv, bool px, const char *fmstr){
TCHDB *hdb = tchdbnew();
if(g_dbgfd >= 0) tchdbsetdbgfd(hdb, g_dbgfd);
if(!tchdbsetcodecfunc(hdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(hdb);
if(!tchdbopen(hdb, path, HDBOREADER | omode)){
printerr(hdb);
tchdbdel(hdb);
return 1;
}
bool err = false;
if(fmstr){
TCLIST *keys = tchdbfwmkeys2(hdb, fmstr, max);
for(int i = 0; i < tclistnum(keys); i++){
int ksiz;
const char *kbuf = tclistval(keys, i, &ksiz);
printdata(kbuf, ksiz, px);
if(pv){
int vsiz;
char *vbuf = tchdbget(hdb, kbuf, ksiz, &vsiz);
if(vbuf){
putchar('\t');
printdata(vbuf, vsiz, px);
tcfree(vbuf);
}
}
putchar('\n');
}
tclistdel(keys);
} else {
if(!tchdbiterinit(hdb)){
printerr(hdb);
err = true;
}
TCXSTR *key = tcxstrnew();
TCXSTR *val = tcxstrnew();
int cnt = 0;
while(tchdbiternext3(hdb, key, val)){
printdata(tcxstrptr(key), tcxstrsize(key), px);
if(pv){
putchar('\t');
printdata(tcxstrptr(val), tcxstrsize(val), px);
}
putchar('\n');
if(max >= 0 && ++cnt >= max) break;
}
tcxstrdel(val);
tcxstrdel(key);
}
if(!tchdbclose(hdb)){
if(!err) printerr(hdb);
err = true;
}
tchdbdel(hdb);
return err ? 1 : 0;
}
void ajax_pager(tpl_t *tpl, double total, double slot, uint pos, uint limit[2])
{
TCXSTR *str;
uint n, i, cur;
str = tcxstrnew();
n = (uint)ceil(total / slot);
if(pos < 1)
pos = 1;
if(pos > n)
pos = n;
limit[1] = slot;
limit[0] = (pos == 1) ? 0 : (pos-1) * limit[1];
for(i=0; i<n; i++)
{
cur = (i+1);
if(cur == pos)
tcxstrprintf(str, "<a class='page_link' href='javascript:void(0);'>%d</a> ", cur);
else
tcxstrprintf(str, "<a href='#' onclick='return loading(%d)'>%d</a> ", cur, cur);
}
tpl_set_field_global(tpl, "page", tcxstrptr(str), tcxstrsize(str));
tcxstrdel(str);
}
/* perform convert command */
static int procconvert(const char *ibuf, int isiz, int fmt,
const char *buri, const char *duri, bool page){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
if(fmt == FMTWIKI){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
wikidump(rbuf, cols);
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTTEXT){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page)
tcxstrprintf(rbuf, "------------------------ Tokyo Promenade ------------------------\n");
wikidumptext(rbuf, cols);
if(page)
tcxstrprintf(rbuf, "-----------------------------------------------------------------\n");
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTHTML){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page){
const char *name = tcmapget2(cols, "name");
tcxstrprintf(rbuf, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
tcxstrprintf(rbuf, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\""
" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
tcxstrprintf(rbuf, "<html xmlns=\"http://www.w3.org/1999/xhtml\""
" xml:lang=\"en\" lang=\"en\">\n");
tcxstrprintf(rbuf, "<head>\n");
tcxstrprintf(rbuf, "<meta http-equiv=\"Content-Type\""
" content=\"text/html; charset=UTF-8\" />\n");
tcxstrprintf(rbuf, "<link rel=\"contents\" href=\"%@\" />\n", buri);
tcxstrprintf(rbuf, "<title>%@</title>\n", name ? name : "Tokyo Promenade");
tcxstrprintf(rbuf, "</head>\n");
tcxstrprintf(rbuf, "<body>\n");
}
wikidumphtml(rbuf, cols, buri, 0, duri);
if(page){
tcxstrprintf(rbuf, "</body>\n");
tcxstrprintf(rbuf, "</html>\n");
}
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
}
tcmapdel(cols);
return 0;
}
bool open_tyrant_db() {
int ecode;
/* open the database */
if (!tcrdbopen(tdb, tcxstrptr(tdb_host), tdb_port)) {
ecode = tchdbecode(tdb);
fprintf(stdout, "Failed to open database, error: %s\n", tcrdberrmsg(ecode));
return false;
}
return true;
}
void driveInterctiveLoop(FILE *fpin, FILE *fpout, int evalLine){
int maxSize = 10000;
char inputs[maxSize];
if(evalLine){
while(fgets(inputs, maxSize, fpin) != NULL){
printf("input: %s\n", inputs);
fprintExp(eval(readScheme(inputs), sGlobalEnvironment), fpout);
}
}else{
TCXSTR *temp = tcxstrnew();
while(fread(inputs, sizeof(char), maxSize, fpin) > 0){
tcxstrcat2(temp, inputs);
}
fprintExp(eval(readScheme(tcxstrptr(temp)), sGlobalEnvironment), fpout);
}
}
sExpression *lookupVariable(sSymbol *symbol, sEnvironment *env){
int i = 0;
int keySize = strlen(symbol->name) * sizeof(char);
TCXSTR *str = getTypeTag(symbol->name);
void *result = tcmapget(env->varmap, symbol->name, keySize, &i);
if(i == 0){
//fail find
if(isNoParent(env)){
return &sNull;
}else{
return lookupVariable(symbol, env->parent);
}
}else{
//success find
sType *type = (sType *)tcmapget(env->varmap, tcxstrptr(str), tcxstrsize(str), &i);
sExpression *resultExp = newExp(result, *type);
return resultExp;
}
}
bool open_cabinet_db(int64_t bnum, bool optimize) {
int ecode;
if (optimize) fprintf(stdout, "Opening database optimized for %d items\n", bnum);
/* tune up the db*/
if (optimize && !tchdbtune(hdb, bnum, -1, -1, HDBTLARGE)) {
ecode = tchdbecode(hdb);
fprintf(stdout, "Failed to tune up database, error: %s\n", tchdberrmsg(ecode));
return false;
}
/* open the database */
if (!tchdbopen(hdb, tcxstrptr(db_file), HDBOWRITER | HDBOCREAT)) {
ecode = tchdbecode(hdb);
fprintf(stdout, "Failed to open database, error: %s\n", tchdberrmsg(ecode));
return false;
}
return true;
}
static void putVars(sList *parameterNames, sList *arguments, TCMAP *mapdb){
sExpression *firstP = car(parameterNames);
sExpression *firstA = car(arguments);
char *pName;
void *aPointer;
void *pPointer;
int aSize;
int pSize;
if(isNull(firstP)){
return;
}
if(isSymbol(firstP)){
pName = toSymb(firstP)->name;
}else{
return;
}
pPointer = pName;
pSize = strlen(pName) * sizeof(char);
aPointer = firstA->value;
aSize = firstA->valueSize;
TCXSTR *str = getTypeTag(pName);
tcmapput(mapdb, pPointer, pSize, aPointer, aSize);
tcmapput(mapdb, (char *)tcxstrptr(str), tcxstrsize(str), &(firstA->type), sizeof(sType));
sExpression *remainP = cdr(parameterNames);
sExpression *remainA = cdr(arguments);
if(isList(remainP) && isList(remainA)){
return putVars(toList(remainP), toList(remainA), mapdb);
}else if(isSymbol(remainP)){
sList *lastOneParameter = toList(cons(remainP, &sNull));
sList *lastOneArgument = toList(cons(remainA, &sNull));
return putVars(lastOneParameter, lastOneArgument, mapdb);
}
return;
}
/* perform list command */
static int proclist(const char *path, TCCMP cmp, int omode, int max, bool pv, bool px, bool bk,
const char *jstr, const char *bstr, const char *estr, const char *fmstr) {
TCBDB *bdb = tcbdbnew();
if (!INVALIDHANDLE(g_dbgfd)) tcbdbsetdbgfd(bdb, g_dbgfd);
if (cmp && !tcbdbsetcmpfunc(bdb, cmp, NULL)) printerr(bdb);
if (!tcbdbsetcodecfunc(bdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(bdb);
if (!tcbdbopen(bdb, path, BDBOREADER | omode)) {
printerr(bdb);
tcbdbdel(bdb);
return 1;
}
bool err = false;
if (bstr || fmstr) {
TCLIST *keys = fmstr ? tcbdbfwmkeys2(bdb, fmstr, max) :
tcbdbrange(bdb, bstr, strlen(bstr), true, estr, strlen(estr), true, max);
int cnt = 0;
for (int i = 0; i < tclistnum(keys); i++) {
int ksiz;
const char *kbuf = tclistval(keys, i, &ksiz);
if (pv) {
TCLIST *vals = tcbdbget4(bdb, kbuf, ksiz);
if (vals) {
for (int j = 0; j < tclistnum(vals); j++) {
int vsiz;
const char *vbuf = tclistval(vals, j, &vsiz);
printdata(kbuf, ksiz, px);
putchar('\t');
printdata(vbuf, vsiz, px);
putchar('\n');
if (max >= 0 && ++cnt >= max) break;
}
tclistdel(vals);
}
} else {
int num = tcbdbvnum(bdb, kbuf, ksiz);
for (int j = 0; j < num; j++) {
printdata(kbuf, ksiz, px);
putchar('\n');
if (max >= 0 && ++cnt >= max) break;
}
}
if (max >= 0 && cnt >= max) break;
}
tclistdel(keys);
} else {
BDBCUR *cur = tcbdbcurnew(bdb);
if (bk) {
if (jstr) {
if (!tcbdbcurjumpback(cur, jstr, strlen(jstr)) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
} else {
if (!tcbdbcurlast(cur) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
}
} else {
if (jstr) {
if (!tcbdbcurjump(cur, jstr, strlen(jstr)) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
} else {
if (!tcbdbcurfirst(cur) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
}
}
TCXSTR *key = tcxstrnew();
TCXSTR *val = tcxstrnew();
int cnt = 0;
while (tcbdbcurrec(cur, key, val)) {
printdata(tcxstrptr(key), tcxstrsize(key), px);
if (pv) {
putchar('\t');
printdata(tcxstrptr(val), tcxstrsize(val), px);
}
putchar('\n');
if (bk) {
if (!tcbdbcurprev(cur) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
} else {
if (!tcbdbcurnext(cur) && tcbdbecode(bdb) != TCENOREC) {
printerr(bdb);
err = true;
}
}
if (max >= 0 && ++cnt >= max) break;
}
tcxstrdel(val);
tcxstrdel(key);
tcbdbcurdel(cur);
}
if (!tcbdbclose(bdb)) {
if (!err) printerr(bdb);
err = true;
}
tcbdbdel(bdb);
return err ? 1 : 0;
}
int main(int argc, char **argv) {
TCXSTR *pattern, *key_root;
int ecode;
bool verbose, test, create, extract, optimize, resume;
int o;
verbose = test = create = extract = optimize = resume = false;
pattern = tcxstrnew();
key_root = tcxstrnew();
db_file = tcxstrnew();
tdb_host = tcxstrnew();
tdb_port = 0;
while (-1 != (o = getopt(argc, argv, "cxtvrof:p:k:H:P:"))) {
switch (o) {
case 'f':
use_cabinet_lib = true;
tcxstrcat2(db_file, optarg);
/* create the object */
hdb = tchdbnew();
break;
case 'H':
use_cabinet_lib = false;
tcxstrcat2(tdb_host, optarg);
tdb = tcrdbnew();
break;
case 'P':
tdb_port = strtol(optarg, NULL, 0);
break;
case 'k':
tcxstrcat2(key_root, optarg);
// add trailing slash if needed
char *last_c;
last_c = tcxstrptr(key_root) + tcxstrsize(key_root) - 1;
if (*last_c != '/') {
tcxstrcat2(key_root, "/");
}
break;
case 'p':
tcxstrcat2(pattern, optarg);
break;
case 'v':
verbose = true;
break;
case 't':
test = true;
break;
case 'c':
create = true;
break;
case 'x':
extract = true;
break;
case 'r':
resume = true;
break;
case 'o':
optimize = true;
case '?':
default:
break;
}
}
if (!create && !extract) {
fprintf(stdout, "No action specifed. Use -c to create DB and -x to extract files from dbfile.tch\n");
return 1;
}
if ((use_cabinet_lib && NULL == hdb)
||
(!use_cabinet_lib && NULL == tdb)) {
fprintf(stdout, "No database specifed. Use -f dbfile.tch\n");
return 1;
}
if (0 == tcxstrsize(pattern)) {
fprintf(stdout, "No pattern is given. Using *. Use -p <glob_pattern> to override\n");
tcxstrcat2(pattern, "*");
}
if (create) {
glob_t gtree;
glob(tcxstrptr(pattern), GLOB_NOSORT, NULL, >ree);
size_t found = gtree.gl_pathc;
if (use_cabinet_lib && !open_cabinet_db((int64_t) found, optimize)) return 2;
if (!use_cabinet_lib && !open_tyrant_db()) return 3;
int i;
for (i = 0; i < found; i++) {
char * fname = gtree.gl_pathv[i];
if (verbose || test) fprintf(stdout, "\n%d of %d - packing file: %s ...", i, found, fname);
if (!test) pack_file(fname, key_root, resume);
}
fprintf(stdout, "Finished. Processed %d items\n", (int) found);
globfree(>ree);
} else if (extract) {
if (!open_cabinet_db(0, false)) return 2;
int count;
count = unpack_files(NULL, verbose, test);
fprintf(stdout, "Finished. Processed %d items\n", count);
}
/* close the database */
close_db();
/* delete the objects */
tcxstrdel(pattern);
tcxstrdel(key_root);
return 0;
}
int pack_file(char *file, TCXSTR *root_key, bool resume) {
struct stat st;
int ecode;
TCXSTR *key;
key = tcxstrdup(root_key);
size_t file_path_len;
file_path_len = strlen(file);
char *file_name;
file_name = get_file_name(file, file_path_len);
tcxstrcat2(key, file_name);
if (resume && key_exists(tcxstrptr(key))) {
fprintf(stdout, "already exists");
return;
}
if (-1 == stat(file, &st)) {
return 1;
}
if (!S_ISREG(st.st_mode)) {
fprintf(stdout, "Not regular file: %s", file);
return 2;
}
int fd;
if (-1 == (fd = open(file, O_RDONLY))) {
fprintf(stdout, "***Failed open file %s", file);
return 1;
}
void *fmap;
if (MAP_FAILED == (fmap = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0))) {
fprintf(stdout, "mmaping failed for %s", file);
close(fd);
return -1;
}
/* store records */
if (use_cabinet_lib) {
if (!tchdbput(hdb, tcxstrptr(key), tcxstrsize(key), fmap, st.st_size)) {
ecode = tchdbecode(hdb);
fprintf(stdout, "put error: %s", tchdberrmsg(ecode));
}
} else {
if (!tcrdbput(tdb, tcxstrptr(key), tcxstrsize(key), fmap, st.st_size)) {
ecode = tcrdbecode(tdb);
fprintf(stdout, "put error: %s", tcrdberrmsg(ecode));
}
}
fprintf(stdout, "%d bytes", st.st_size);
munmap(fmap, st.st_size);
close(fd);
tcxstrdel(key);
}
/* perform export command */
static int procexport(const char *dbpath, int64_t id, const char *dirpath){
TCTDB *tdb = tctdbnew();
if(!tctdbopen(tdb, dbpath, TDBOREADER)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(id > 0){
char pkbuf[NUMBUFSIZ];
int pksiz = sprintf(pkbuf, "%lld", (long long)id);
TCMAP *cols = tctdbget(tdb, pkbuf, pksiz);
if(cols){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
wikidump(rbuf, cols);
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
tcmapdel(cols);
} else {
printdberr(tdb);
err = true;
}
} else {
if(!dirpath) dirpath = ".";
if(!tctdbiterinit(tdb)){
printdberr(tdb);
err = true;
}
char *pkbuf;
int pksiz;
while((pkbuf = tctdbiternext(tdb, &pksiz)) != NULL){
TCMAP *cols = tctdbget(tdb, pkbuf, pksiz);
if(cols){
char *name = tcstrdup(tcmapget4(cols, "name", ""));
tcstrcututf(name, 32);
char *enc = pathencode(name);
char *path = tcsprintf("%s/%s-%s.tpw", dirpath, pkbuf, enc);
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
wikidump(rbuf, cols);
if(tcwritefile(path, tcxstrptr(rbuf), tcxstrsize(rbuf))){
printf("%s: exported: id=%s name=%s\n", path, pkbuf, name);
} else {
printf("%s: writing failed\n", path);
err = true;
}
tcxstrdel(rbuf);
tcfree(path);
tcfree(enc);
tcfree(name);
tcmapdel(cols);
} else {
printdberr(tdb);
err = true;
}
tcfree(pkbuf);
}
}
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}