/* perform importtsv command */
static int procimporttsv(const char *path, const char *file, int omode, bool sc){
FILE *ifp = file ? fopen(file, "rb") : stdin;
if(!ifp){
fprintf(stderr, "%s: could not open\n", file ? file : "(stdin)");
return 1;
}
TCHDB *hdb = tchdbnew();
if(g_dbgfd >= 0) tchdbsetdbgfd(hdb, g_dbgfd);
if(!tchdbsetcodecfunc(hdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(hdb);
if(!tchdbopen(hdb, path, HDBOWRITER | HDBOCREAT | omode)){
printerr(hdb);
tchdbdel(hdb);
if(ifp != stdin) fclose(ifp);
return 1;
}
bool err = false;
char *line;
int cnt = 0;
while(!err && (line = mygetline(ifp)) != NULL){
char *pv = strchr(line, '\t');
if(!pv){
tcfree(line);
continue;
}
*pv = '\0';
if(sc) tcstrutfnorm(line, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
if(!tchdbput2(hdb, line, pv + 1)){
printerr(hdb);
err = true;
}
tcfree(line);
if(cnt > 0 && cnt % 100 == 0){
putchar('.');
fflush(stdout);
if(cnt % 5000 == 0) printf(" (%08d)\n", cnt);
}
cnt++;
}
printf(" (%08d)\n", cnt);
if(!tchdbclose(hdb)){
if(!err) printerr(hdb);
err = true;
}
tchdbdel(hdb);
if(ifp != stdin) fclose(ifp);
return err ? 1 : 0;
}
/* perform importtsv command */
static int procimporttsv(const char *path, const char *file, int omode, bool sc){
FILE *ifp = file ? fopen(file, "rb") : stdin;
if(!ifp){
fprintf(stderr, "%s: could not open\n", file ? file : "(stdin)");
return 1;
}
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | TDBOCREAT | omode)){
printerr(tdb);
tctdbdel(tdb);
if(ifp != stdin) fclose(ifp);
return 1;
}
bool err = false;
char *line, numbuf[TCNUMBUFSIZ];
int cnt = 0;
while(!err && (line = mygetline(ifp)) != NULL){
char *pv = strchr(line, '\t');
if(!pv){
tcfree(line);
continue;
}
*pv = '\0';
if(sc) tcstrutfnorm(line, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
const char *pkey;
if(*line == '\0'){
sprintf(numbuf, "%lld", (long long)tctdbgenuid(tdb));
pkey = numbuf;
} else {
pkey = line;
}
if(!tctdbput3(tdb, pkey, pv + 1)){
printerr(tdb);
err = true;
}
tcfree(line);
if(cnt > 0 && cnt % 100 == 0){
putchar('.');
fflush(stdout);
if(cnt % 5000 == 0) printf(" (%08d)\n", cnt);
}
cnt++;
}
printf(" (%08d)\n", cnt);
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
if(ifp != stdin) fclose(ifp);
return err ? 1 : 0;
}
