/* perform import command */
static int procimport(const char *upath, uint64_t lim){
TCULOG *ulog = tculognew();
if(!tculogopen(ulog, upath, lim)){
printerr("tculogopen");
return 1;
}
bool err = false;
char *line;
while(!err && (line = mygetline(stdin)) != NULL){
uint64_t ts = ttstrtots(line);
char *pv = strchr(line, '\t');
if(!pv || ts < 1){
tcfree(line);
continue;
}
pv++;
uint32_t sid = tcatoi(pv);
char *mp = strchr(pv, ':');
pv = strchr(pv, '\t');
if(!pv){
tcfree(line);
continue;
}
pv++;
uint32_t mid = 0;
if(mp && mp < pv) mid = tcatoi(mp + 1);
pv = strchr(pv, '\t');
if(!pv){
tcfree(line);
continue;
}
pv++;
int osiz;
unsigned char *obj = (unsigned char *)tchexdecode(pv, &osiz);
if(!obj || osiz < 3 || *obj != TTMAGICNUM){
tcfree(obj);
tcfree(line);
continue;
}
if(!tculogwrite(ulog, ts, sid, mid, obj, osiz)){
printerr("tculogwrite");
err = true;
}
tcfree(obj);
tcfree(line);
}
if(!tculogclose(ulog)){
printerr("tculogclose");
err = true;
}
tculogdel(ulog);
return err ? 1 : 0;
}
/* perform put command */
static int procput(const char *path, const char *pkbuf, int pksiz, TCMAP *cols,
int omode, int dmode){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
char pknumbuf[TCNUMBUFSIZ];
if(pksiz < 1){
pksiz = sprintf(pknumbuf, "%lld", (long long)tctdbgenuid(tdb));
pkbuf = pknumbuf;
}
const char *vbuf;
switch(dmode){
case -1:
if(!tctdbputkeep(tdb, pkbuf, pksiz, cols)){
printerr(tdb);
err = true;
}
break;
case 1:
if(!tctdbputcat(tdb, pkbuf, pksiz, cols)){
printerr(tdb);
err = true;
}
break;
case 10:
vbuf = tcmapget2(cols, "_num");
if(!vbuf) vbuf = "1";
if(tctdbaddint(tdb, pkbuf, pksiz, tcatoi(vbuf)) == INT_MIN){
printerr(tdb);
err = true;
}
break;
case 11:
vbuf = tcmapget2(cols, "_num");
if(!vbuf) vbuf = "1.0";
if(isnan(tctdbadddouble(tdb, pkbuf, pksiz, tcatof(vbuf)))){
printerr(tdb);
err = true;
}
break;
default:
if(!tctdbput(tdb, pkbuf, pksiz, cols)){
printerr(tdb);
err = true;
}
break;
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* perform setindex command */
static int procsetindex(const char *path, const char *name, int omode, int type){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(!tctdbsetindex(tdb, name, type)){
printerr(tdb);
err = true;
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* parse arguments of export command */
static int runexport(int argc, char **argv){
char *upath = NULL;
uint64_t ts = 0;
uint32_t sid = UINT32_MAX;
for(int i = 2; i < argc; i++){
if(!upath && argv[i][0] == '-'){
if(!strcmp(argv[i], "-ts")){
if(++i >= argc) usage();
ts = ttstrtots(argv[i]);
} else if(!strcmp(argv[i], "-sid")){
if(++i >= argc) usage();
sid = tcatoi(argv[i]);
} else {
usage();
}
} else if(!upath){
upath = argv[i];
} else {
usage();
}
}
if(!upath) usage();
int rv = procexport(upath, ts, sid);
return rv;
}
/* parse arguments of update command */
static int runupdate(int argc, char **argv){
char *dbpath = NULL;
char *idstr = NULL;
char *file = NULL;
for(int i = 2; i < argc; i++){
if(!dbpath && argv[i][0] == '-'){
usage();
} else if(!dbpath){
dbpath = argv[i];
} else if(!idstr){
idstr = argv[i];
} else if(!file){
file = argv[i];
} else {
usage();
}
}
if(!dbpath || !idstr) usage();
int64_t id = tcatoi(idstr);
char *ibuf;
int isiz;
if(file && file[0] == '@'){
isiz = strlen(file) - 1;
ibuf = tcmemdup(file + 1, isiz);
} else {
ibuf = tcreadfile(file, -1, &isiz);
}
if(!ibuf){
eprintf("%s: cannot open", file ? file : "(stdin)");
return 1;
}
int rv = procupdate(dbpath, id, ibuf);
tcfree(ibuf);
return rv;
}
/* parse arguments of export command */
static int runexport(int argc, char **argv){
char *dbpath = NULL;
char *idstr = NULL;
char *dirpath = NULL;
for(int i = 2; i < argc; i++){
if(!dbpath && argv[i][0] == '-'){
if(!strcmp(argv[i], "-dir")){
if(++i >= argc) usage();
dirpath = argv[i];
} else {
usage();
}
} else if(!dbpath){
dbpath = argv[i];
} else if(!idstr){
idstr = argv[i];
} else {
usage();
}
}
if(!dbpath) usage();
int64_t id = idstr ? tcatoi(idstr) : 0;
int rv = procexport(dbpath, id, dirpath);
return rv;
}
/* perform put command */
static int procput(const char *path, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
int omode, int dmode) {
TCHDB *hdb = tchdbnew();
if (!INVALIDHANDLE(g_dbgfd)) tchdbsetdbgfd(hdb, g_dbgfd);
if (!tchdbsetcodecfunc(hdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(hdb);
if (!tchdbopen(hdb, path, HDBOWRITER | omode)) {
printerr(hdb);
tchdbdel(hdb);
return 1;
}
bool err = false;
int inum;
double dnum;
switch (dmode) {
case -1:
if (!tchdbputkeep(hdb, kbuf, ksiz, vbuf, vsiz)) {
printerr(hdb);
err = true;
}
break;
case 1:
if (!tchdbputcat(hdb, kbuf, ksiz, vbuf, vsiz)) {
printerr(hdb);
err = true;
}
break;
case 10:
inum = tchdbaddint(hdb, kbuf, ksiz, tcatoi(vbuf));
if (inum == INT_MIN) {
printerr(hdb);
err = true;
} else {
printf("%d\n", inum);
}
break;
case 11:
dnum = tchdbadddouble(hdb, kbuf, ksiz, tcatof(vbuf));
if (isnan(dnum)) {
printerr(hdb);
err = true;
} else {
printf("%.6f\n", dnum);
}
break;
default:
if (!tchdbput(hdb, kbuf, ksiz, vbuf, vsiz)) {
printerr(hdb);
err = true;
}
break;
}
if (!tchdbclose(hdb)) {
if (!err) printerr(hdb);
err = true;
}
tchdbdel(hdb);
return err ? 1 : 0;
}
/* perform put command */
static int procput(const char *path, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
int omode, int dmode){
TCFDB *fdb = tcfdbnew();
if(g_dbgfd >= 0) tcfdbsetdbgfd(fdb, g_dbgfd);
if(!tcfdbopen(fdb, path, FDBOWRITER | omode)){
printerr(fdb);
tcfdbdel(fdb);
return 1;
}
bool err = false;
int inum;
double dnum;
switch(dmode){
case -1:
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz)){
printerr(fdb);
err = true;
}
break;
case 1:
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz)){
printerr(fdb);
err = true;
}
break;
case 10:
inum = tcfdbaddint(fdb, tcfdbkeytoid(kbuf, ksiz), tcatoi(vbuf));
if(inum == INT_MIN){
printerr(fdb);
err = true;
} else {
printf("%d\n", inum);
}
break;
case 11:
dnum = tcfdbadddouble(fdb, tcfdbkeytoid(kbuf, ksiz), tcatof(vbuf));
if(isnan(dnum)){
printerr(fdb);
err = true;
} else {
printf("%.6f\n", dnum);
}
break;
default:
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
printerr(fdb);
err = true;
}
break;
}
if(!tcfdbclose(fdb)){
if(!err) printerr(fdb);
err = true;
}
tcfdbdel(fdb);
return err ? 1 : 0;
}
int
Server_react(Server *server, void *socket, zmq_msg_t *msg)
{
size_t cmdlen, msglen, datalen;
char *eol, *message, *command;
TCLIST *args;
Command *cmd;
void *data = NULL;
message = zmq_msg_data(msg);
msglen = zmq_msg_size(msg);
zmq_msg_close(msg);
eol = strchr(message, '\n');
if (eol) {
cmdlen = 1 + (eol - message);
command = (char *)malloc((cmdlen + 1) * sizeof(char));
memset(command, 0, cmdlen + 1);
memcpy(command, message, cmdlen);
args = tcstrsplit(tcstrsqzspc(command), " \t");
cmd = lookup_command(tclistval2(args, 0));
if (cmd == NULL) {
ERROR("protocol error: %s", command);
send(socket, "-INVALID_COMMAND");
return 0;
}
if (msglen > cmdlen) {
/* data size will always be last arg */
datalen = tcatoi((char *)tclistval2(args, tclistnum(args) - 1));
if ((datalen + cmdlen) != msglen) {
ERROR("error receiving data or invalid data size");
send(socket, "-INCORRECT_DATA_SIZE");
return 0;
}
data = malloc(sizeof(char) * (datalen + 1));
if (data == NULL) {
ERROR("out of memory");
send(socket, "-OUT_OF_MEMORY");
return 0;
}
memset(data + datalen, 0, 1);
memcpy(data, message + cmdlen, datalen);
DEBUG("data received: %s", data);
}
cmd->fn(server, socket, args, data);
return 1;
}
else {
ERROR("request missing eol");
send(socket, "-MISSING_EOL");
}
return 0;
}
int
main(int argc, char **argv)
{
int i, errCode;
double magic, rlat, s, c;
int lat;
g_progname = argv[0];
for (i=1; i < argc; i++) {
if(!strcmp(argv[i], "-tchost")) {
if(++i >= argc) usage();
db_host = argv[i];
} else if(!strcmp(argv[i], "-tcport")) {
if(++i >= argc) usage();
db_port = tcatoi(argv[i]);
} else if (!strcmp(argv[i], "-help")) {
usage();
}
}
pi = atan(1.0)*4;
magic = cos(pi/180.0);
for (lat = 0; lat < 181; ++lat) {
rlat = lat*pi/180;
s = sin(rlat);
c = cos(rlat);
longDistance[lat] = radius*acos((s*s)+(magic*c*c));
}
memset(&db_status, -1, sizeof(db_status));
simplehttp_init();
db_reconnect(0, 0, NULL);
simplehttp_set_cb("/search*", search_cb, NULL);
simplehttp_set_cb("/put*", put_cb, NULL);
simplehttp_set_cb("/get*", get_cb, NULL);
simplehttp_set_cb("/del*", del_cb, NULL);
simplehttp_set_cb("/distance*", distance_cb, NULL);
simplehttp_set_cb("/box*", box_cb, NULL);
simplehttp_main(argc, argv);
if (!tcrdbclose(rdb)) {
errCode = tcrdbecode(rdb);
fprintf(stderr, "close error: %s\n", tcrdberrmsg(errCode));
}
tcrdbdel(rdb);
return 0;
}
/* parse arguments of remove command */
static int runremove(int argc, char **argv){
char *dbpath = NULL;
char *idstr = NULL;
for(int i = 2; i < argc; i++){
if(!dbpath && argv[i][0] == '-'){
usage();
} else if(!dbpath){
dbpath = argv[i];
} else if(!idstr){
idstr = argv[i];
} else {
usage();
}
}
if(!dbpath || !idstr) usage();
int64_t id = tcatoi(idstr);
if(id < 1) usage();
int rv = procremove(dbpath, id);
return rv;
}
/* perform optimize command */
static int procoptimize(const char *path, int bnum, int apow, int fpow, int opts, int omode,
bool df){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(df){
if(!tctdbdefrag(tdb, INT64_MAX)){
printerr(tdb);
err = true;
}
} else {
if(!tctdboptimize(tdb, bnum, apow, fpow, opts)){
printerr(tdb);
err = true;
}
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* perform put command */
static int procput(const char *name, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
int dmode){
TCADB *adb = tcadbnew();
ADBSKEL skel;
if(*name == '@'){
setskeltran(&skel);
if(!tcadbsetskel(adb, &skel)){
printerr(adb);
skel.del(skel.opq);
tcadbdel(adb);
return 1;
}
name++;
} else if(*name == '%'){
if(!tcadbsetskelmulti(adb, 8)){
printerr(adb);
tcadbdel(adb);
return 1;
}
name++;
}
if(!tcadbopen(adb, name)){
printerr(adb);
tcadbdel(adb);
return 1;
}
bool err = false;
switch(dmode){
case -1:
if(!tcadbputkeep(adb, kbuf, ksiz, vbuf, vsiz)){
printerr(adb);
err = true;
}
break;
case 1:
if(!tcadbputcat(adb, kbuf, ksiz, vbuf, vsiz)){
printerr(adb);
err = true;
}
break;
case 10:
if(tcadbaddint(adb, kbuf, ksiz, tcatoi(vbuf)) == INT_MIN){
printerr(adb);
err = true;
}
break;
case 11:
if(isnan(tcadbadddouble(adb, kbuf, ksiz, tcatof(vbuf)))){
printerr(adb);
err = true;
}
break;
default:
if(!tcadbput(adb, kbuf, ksiz, vbuf, vsiz)){
printerr(adb);
err = true;
}
break;
}
if(!tcadbclose(adb)){
if(!err) printerr(adb);
err = true;
}
tcadbdel(adb);
return err ? 1 : 0;
}
/* perform put command */
static int procput(const char *path, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
TCCMP cmp, int omode, int dmode){
TCBDB *bdb = tcbdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tcbdbsetdbgfd(bdb, g_dbgfd);
if(cmp && !tcbdbsetcmpfunc(bdb, cmp, NULL)) printerr(bdb);
if(!tcbdbsetcodecfunc(bdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(bdb);
if(!tcbdbopen(bdb, path, BDBOWRITER | omode)){
printerr(bdb);
tcbdbdel(bdb);
return 1;
}
bool err = false;
int inum;
double dnum;
switch(dmode){
case -1:
if(!tcbdbputkeep(bdb, kbuf, ksiz, vbuf, vsiz)){
printerr(bdb);
err = true;
}
break;
case 1:
if(!tcbdbputcat(bdb, kbuf, ksiz, vbuf, vsiz)){
printerr(bdb);
err = true;
}
break;
case 2:
if(!tcbdbputdup(bdb, kbuf, ksiz, vbuf, vsiz)){
printerr(bdb);
err = true;
}
break;
case 3:
if(!tcbdbputdupback(bdb, kbuf, ksiz, vbuf, vsiz)){
printerr(bdb);
err = true;
}
break;
case 10:
inum = tcbdbaddint(bdb, kbuf, ksiz, tcatoi(vbuf));
if(inum == INT_MIN){
printerr(bdb);
err = true;
} else {
printf("%d\n", inum);
}
break;
case 11:
dnum = tcbdbadddouble(bdb, kbuf, ksiz, tcatof(vbuf));
if(isnan(dnum)){
printerr(bdb);
err = true;
} else {
printf("%.6f\n", dnum);
}
break;
default:
if(!tcbdbput(bdb, kbuf, ksiz, vbuf, vsiz)){
printerr(bdb);
err = true;
}
break;
}
if(!tcbdbclose(bdb)){
if(!err) printerr(bdb);
err = true;
}
tcbdbdel(bdb);
return err ? 1 : 0;
}
/* perform importtsv command */
static int procimporttsv(const char *path, const char *file, int omode, bool sc){
FILE *ifp = file ? fopen(file, "rb") : stdin;
if(!ifp){
fprintf(stderr, "%s: could not open\n", file ? file : "(stdin)");
return 1;
}
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | TDBOCREAT | omode)){
printerr(tdb);
tctdbdel(tdb);
if(ifp != stdin) fclose(ifp);
return 1;
}
bool err = false;
char *line, numbuf[TCNUMBUFSIZ];
int cnt = 0;
while(!err && (line = mygetline(ifp)) != NULL){
char *pv = strchr(line, '\t');
if(!pv){
tcfree(line);
continue;
}
*pv = '\0';
if(sc) tcstrutfnorm(line, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
const char *pkey;
if(*line == '\0'){
sprintf(numbuf, "%lld", (long long)tctdbgenuid(tdb));
pkey = numbuf;
} else {
pkey = line;
}
if(!tctdbput3(tdb, pkey, pv + 1)){
printerr(tdb);
err = true;
}
tcfree(line);
if(cnt > 0 && cnt % 100 == 0){
putchar('.');
fflush(stdout);
if(cnt % 5000 == 0) printf(" (%08d)\n", cnt);
}
cnt++;
}
printf(" (%08d)\n", cnt);
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
if(ifp != stdin) fclose(ifp);
return err ? 1 : 0;
}
/* perform import command */
static int procimport(const char *dbpath, TCLIST *files, TCLIST *sufs){
TCTDB *tdb = tctdbnew();
if(!tctdbtune(tdb, TUNEBNUM, TUNEAPOW, TUNEFPOW, 0)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
if(!tctdbopen(tdb, dbpath, TDBOWRITER | TDBOCREAT)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(!tctdbsetindex(tdb, "name", TDBITLEXICAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "cdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "mdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
if(!tctdbsetindex(tdb, "xdate", TDBITDECIMAL | TDBITKEEP) && tctdbecode(tdb) != TCEKEEP){
printdberr(tdb);
err = true;
}
tclistinvert(files);
char *fpath;
while((fpath = tclistpop2(files)) != NULL){
TCLIST *cfiles = tcreaddir(fpath);
if(cfiles){
tclistsort(cfiles);
for(int i = tclistnum(cfiles) - 1; i >= 0; i--){
const char *cfile = tclistval2(cfiles, i);
bool hit = false;
for(int j = 0; j < tclistnum(sufs); j++){
if(tcstribwm(cfile, tclistval2(sufs, j))){
hit = true;
break;
}
}
if(!hit) continue;
char *lpath = tcsprintf("%s/%s", fpath, cfile);
tclistpush2(files, lpath);
tcfree(lpath);
}
tclistdel(cfiles);
} else {
int isiz;
char *ibuf = tcreadfile(fpath, IOMAXSIZ, &isiz);
if(ibuf){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
const char *name = tcmapget2(cols, "name");
if(name && *name != '\0'){
int64_t id = tcatoi(tcmapget4(cols, "id", ""));
if(dbputart(tdb, id, cols)){
id = tcatoi(tcmapget4(cols, "id", ""));
printf("%s: imported: id=%lld name=%s\n", fpath, (long long)id, name);
} else {
printdberr(tdb);
err = true;
}
} else {
printf("%s: ignored because there is no name\n", fpath);
}
tcmapdel(cols);
tcfree(ibuf);
}
}
tcfree(fpath);
}
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
