/* RTDB_tp_as_mapping.mp_ass_subscript */
static int
RTDB_SetItem(RTDB *self, PyObject *pykey, PyObject *pyvalue)
{
void *key;
int key_size;
TCMAP *value;
bool result;
RDBBase *rdbbase = (RDBBase *)self;
if (bytes_to_void(pykey, &key, &key_size)) {
return -1;
}
if (pyvalue) {
value = dict_to_tcmap(pyvalue);
if (!value) {
return -1;
}
Py_BEGIN_ALLOW_THREADS
result = tcrdbtblput(rdbbase->rdb, key, key_size, value);
Py_END_ALLOW_THREADS
if (!result) {
tcmapdel(value);
set_rdb_error(rdbbase->rdb, NULL);
return -1;
}
tcmapdel(value);
}
else {
/* Close a word database object.
`wdb' specifies the word database object.
If successful, the return value is true, else, it is false. */
static bool tcwdbcloseimpl(TCWDB *wdb){
assert(wdb);
bool err = false;
if(wdb->cc){
if((tcmaprnum(wdb->cc) > 0 || tcmaprnum(wdb->dtokens) > 0) && !tcwdbmemsync(wdb, 0))
err = true;
tcidsetdel(wdb->dids);
tcmapdel(wdb->dtokens);
tcmapdel(wdb->cc);
wdb->cc = NULL;
}
if(!tcbdbclose(wdb->idx)) err = true;
wdb->open = false;
return !err;
}
static VALUE cQuery_get(VALUE vself){
int i, num, ksiz;
const char *name, *col;
VALUE vqry, vary, vcols;
RDBQRY *qry;
TCLIST *res;
TCMAP *cols;
vqry = rb_iv_get(vself, RDBQRYVNDATA);
Data_Get_Struct(vqry, RDBQRY, qry);
res = tcrdbqrysearchget(qry);
num = tclistnum(res);
vary = rb_ary_new2(num);
for(i = 0; i < num; i++){
vcols = rb_hash_new();
cols = tcrdbqryrescols(res, i);
if(cols){
tcmapiterinit(cols);
while((name = tcmapiternext(cols, &ksiz)) != NULL){
col = tcmapget2(cols, name);
if (ksiz == 0) name = "__id";
rb_hash_aset(vcols, ID2SYM(rb_intern(name)), rb_str_new2(col));
}
}
tcmapdel(cols);
rb_ary_push(vary, vcols);
}
tclistdel(res);
return vary;
}
/* perform setindex command */
static int procsetindex(const char *path, const char *name, int omode, int type){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(!tctdbsetindex(tdb, name, type)){
printerr(tdb);
err = true;
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
void
XTDBFreeHandle(XTDBHandle* handle) {
//PrintProfile(handle);
if (handle->indexes) {
XTDBForAllIDX(handle,NULL,DBClose,DBFree,NULL);
tcmapdel(handle->indexes);
}
/* HashTableIterator iter;
hash_table_iterate(handle->indexes,&iter);
while (hash_table_iter_has_more(&iter)) {
DataBaseBE* be = hash_table_iter_next(&iter);
printf("Closing db %p \n",be);
DBClose(be);
DBFree(be);
}*/
if (handle->mainDB) {
DBClose(handle->mainDB);
DBFree(handle->mainDB);
}
if (handle->descDB) {
DBClose(handle->descDB);
DBFree(handle->descDB);
}
//hash_table_free(handle->indexes);
StrFree(&handle->dbName);
StrFree(&handle->dataDir);
StrFree(&handle->descName);
StrFree(&handle->mainDbFileName);
memset(handle,0,sizeof(*handle));
free(handle);
}
/* RTDBIterValuesValsType.tp_iternext */
static PyObject *
RTDBIterValuesVals_tp_iternext(DBIter *self)
{
RDBBase *rdbbase = (RDBBase *)self->db;
void *key;
int key_size;
TCMAP *value;
TCLIST *valuevals;
PyObject *pyvaluevals;
if (rdbbase->changed) {
return set_error(Error, "DB changed during iteration");
}
Py_BEGIN_ALLOW_THREADS
key = tcrdbiternext(rdbbase->rdb, &key_size);
if (key) {
value = tcrdbtblget(rdbbase->rdb, key, key_size);
}
Py_END_ALLOW_THREADS
if (!key) {
if (tcrdbecode(rdbbase->rdb) == TTENOREC) {
return set_stopiteration_error();
}
return set_rdb_error(rdbbase->rdb, NULL);
}
valuevals = tcmapvals(value);
pyvaluevals = tclist_to_tuple(valuevals);
tcmapdel(value);
tclistdel(valuevals);
return pyvaluevals;
}
/* RTDBIterItemsType.tp_iternext */
static PyObject *
RTDBIterItems_tp_iternext(DBIter *self)
{
RDBBase *rdbbase = (RDBBase *)self->db;
void *key;
int key_size;
TCMAP *value;
PyObject *pykey, *pyvalue, *pyresult = NULL;
if (rdbbase->changed) {
return set_error(Error, "DB changed during iteration");
}
Py_BEGIN_ALLOW_THREADS
key = tcrdbiternext(rdbbase->rdb, &key_size);
if (key) {
value = tcrdbtblget(rdbbase->rdb, key, key_size);
}
Py_END_ALLOW_THREADS
if (!key) {
if (tcrdbecode(rdbbase->rdb) == TTENOREC) {
return set_stopiteration_error();
}
return set_rdb_error(rdbbase->rdb, NULL);
}
pykey = void_to_bytes(key, key_size);
pyvalue = tcmap_to_dict(value);
if (pykey && pyvalue) {
pyresult = PyTuple_Pack(2, pykey, pyvalue);
}
Py_XDECREF(pykey);
Py_XDECREF(pyvalue);
tcfree(key);
tcmapdel(value);
return pyresult;
}
/* perform get command */
static int procget(const char *path, const char *pkbuf, int pksiz, int omode, bool px, bool pz){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOREADER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
TCMAP *cols = tctdbget(tdb, pkbuf, pksiz);
if(cols){
tcmapiterinit(cols);
const char *kbuf;
int ksiz;
while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
int vsiz;
const char *vbuf = tcmapiterval(kbuf, &vsiz);
printdata(kbuf, ksiz, px);
putchar('\t');
printdata(vbuf, vsiz, px);
putchar(pz ? '\t' : '\n');
}
tcmapdel(cols);
} else {
printerr(tdb);
err = true;
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
static VALUE cDB_mget(int argc, VALUE *argv, VALUE vself){
VALUE vkeys, vhash, vvalue;
TCRDB *db;
TCMAP *recs;
Data_Get_Struct(rb_iv_get(vself, RDBVNDATA), TCRDB, db);
rb_scan_args(argc, argv, "*", &vkeys);
// I really hope there is a better way to do this
if (RARRAY_LEN(vkeys) == 1) {
vvalue = rb_ary_entry(vkeys, 0);
switch (TYPE(vvalue)){
case T_STRING:
case T_FIXNUM:
break;
case T_ARRAY:
vkeys = vvalue;
break;
case T_OBJECT:
vkeys = rb_convert_type(vvalue, T_ARRAY, "Array", "to_a");
break;
}
}
Check_Type(vkeys, T_ARRAY);
recs = varytomap(vkeys);
if(!tcrdbget3(db, recs)) return Qnil;
vhash = maptovhash(recs);
tcmapdel(recs);
return vhash;
}
void get_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
const char *value, *name;
char *uri, *json, *hash, *key;
struct evkeyvalq args;
struct json_object *jsobj, *jsobj2, *jsobj3;
TCMAP *cols;
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
uri = evhttp_decode_uri(req->uri);
evhttp_parse_query(uri, &args);
free(uri);
hash = (char *)evhttp_find_header(&args, "hash");
key = (char *)evhttp_find_header(&args, "key");
if (hash == NULL) {
evhttp_send_error(req, 400, "hash is required");
evhttp_clear_headers(&args);
return;
}
jsobj = json_object_new_object();
jsobj2 = json_object_new_object();
cols = tcrdbtblget(rdb, hash, sizeof(hash));
if (cols) {
tcmapiterinit(cols);
jsobj3 = json_object_new_object();
if (key) {
value = tcmapget2(cols, key);
if (!value) {
value = "";
}
json_object_object_add(jsobj2, key, json_object_new_string(value));
} else {
while ((name = tcmapiternext2(cols)) != NULL) {
json_object_object_add(jsobj2, name, json_object_new_string(tcmapget2(cols, name)));
}
}
json_object_object_add(jsobj, "status", json_object_new_string("ok"));
json_object_object_add(jsobj, "results", jsobj2);
tcmapdel(cols);
} else {
json_object_object_add(jsobj, "status", json_object_new_string("error"));
}
finalize_json(req, evb, &args, jsobj);
}
/*
* Create a new user session. This is done upon each successful login.
*/
void create_session(unsigned long long sid)
{
char session_id[SID_LEN + 1];
char restrict_ip[2] = "0\0";
char pkbuf[256];
char timestamp[21];
char ssid[21];
char tenant[TENANT_MAX + 1];
char *username;
int primary_key_size;
MYSQL_RES *res;
TCTDB *tdb;
TCMAP *cols;
GHashTable *db_row = NULL;
username = make_mysql_safe_string(get_var(qvars, "username"));
res = sql_query("SELECT uid, name, capabilities FROM passwd WHERE "
"username = '******'", username);
db_row = get_dbrow(res);
get_tenant(env_vars.host, tenant);
generate_hash(session_id, SHA256);
if (strcmp(get_var(qvars, "restrict_ip"), "true") == 0) {
d_fprintf(debug_log, "Restricting session to origin ip "
"address\n");
restrict_ip[0] = '1';
}
tdb = tctdbnew();
tctdbopen(tdb, SESSION_DB, TDBOWRITER | TDBOCREAT);
primary_key_size = sprintf(pkbuf, "%ld", (long)tctdbgenuid(tdb));
snprintf(timestamp, sizeof(timestamp), "%ld", (long)time(NULL));
snprintf(ssid, sizeof(ssid), "%llu", sid);
cols = tcmapnew3("tenant", tenant,
"sid", ssid,
"uid", get_var(db_row, "uid"),
"username", get_var(qvars, "username"),
"name", get_var(db_row, "name"),
"login_at", timestamp,
"last_seen", timestamp,
"origin_ip", env_vars.remote_addr,
"client_id", env_vars.http_user_agent,
"session_id", session_id,
"csrf_token", "\0",
"restrict_ip", restrict_ip,
"capabilities", get_var(db_row, "capabilities"),
NULL);
tctdbput(tdb, pkbuf, primary_key_size, cols);
tcmapdel(cols);
tctdbclose(tdb);
tctdbdel(tdb);
fcgx_p("Set-Cookie: session_id=%s; path=/; httponly\r\n", session_id);
mysql_free_result(res);
free_vars(db_row);
free(username);
}
/* perform tblread command */
int dotblread(char *name, int rnum){
TCTDB *tdb;
int i, j, err, pksiz, rsiz;
char pkbuf[RECBUFSIZ];
const char *rbuf;
TCMAP *cols;
TDBQRY *qry;
TCLIST *res;
if(showprgr) printf("<Reading Test of Table>\n name=%s rnum=%d\n\n", name, rnum);
/* open a database */
tdb = tctdbnew();
tctdbsetxmsiz(tdb, rnum * 80);
tctdbsetcache(tdb, -1, rnum / 100, -1);
if(!tctdbopen(tdb, name, TDBOREADER)){
fprintf(stderr, "tctdbopen failed\n");
tctdbdel(tdb);
return 1;
}
err = FALSE;
/* loop for each record */
for(i = 1; i <= rnum; i++){
/* search for a record */
pksiz = sprintf(pkbuf, "%08d", i);
qry = tctdbqrynew(tdb);
tctdbqryaddcond(qry, "s", TDBQCSTREQ, pkbuf);
res = tctdbqrysearch(qry);
for(j = 0; j < tclistnum(res); j++){
rbuf = tclistval(res, j, &rsiz);
cols = tctdbget(tdb, rbuf, rsiz);
if(cols){
tcmapdel(cols);
} else {
fprintf(stderr, "tctdbget failed\n");
err = TRUE;
break;
}
}
tclistdel(res);
tctdbqrydel(qry);
/* print progression */
if(showprgr && rnum > 250 && i % (rnum / 250) == 0){
putchar('.');
fflush(stdout);
if(i == rnum || i % (rnum / 10) == 0){
printf(" (%08d)\n", i);
fflush(stdout);
}
}
}
/* close the database */
if(!tctdbclose(tdb)){
fprintf(stderr, "tctdbclose failed\n");
tctdbdel(tdb);
return 1;
}
tctdbdel(tdb);
if(showprgr && !err) printf("ok\n\n");
return err ? 1 : 0;
}
/* putimpl */
JNIEXPORT jboolean JNICALL Java_tokyotyrant_TCRDB_putimpl
(JNIEnv *env, jobject self, jbyteArray pkey, jobjectArray cols, jint dmode){
if(!pkey || !cols){
throwillarg(env);
return false;
}
TCRDB *tcrdb = (TCRDB *)(intptr_t)(*env)->GetLongField(env, self, tcrdb_fid_ptr);
jboolean ick;
jbyte *kbuf = (*env)->GetByteArrayElements(env, pkey, &ick);
if(!kbuf){
throwoutmem(env);
return false;
}
int ksiz = (*env)->GetArrayLength(env, pkey);
jsize cnum = (*env)->GetArrayLength(env, cols);
TCMAP *tcols = tcmapnew2(cnum + 1);
cnum--;
for(int i = 0; i < cnum; i += 2){
jobject name = (*env)->GetObjectArrayElement(env, cols, i);
jboolean icn;
jbyte *nbuf = (*env)->GetByteArrayElements(env, name, &icn);
if(!nbuf){
throwoutmem(env);
return false;
}
int nsiz = (*env)->GetArrayLength(env, name);
jobject val = (*env)->GetObjectArrayElement(env, cols, i + 1);
jboolean icv;
jbyte *vbuf = (*env)->GetByteArrayElements(env, val, &icv);
if(!vbuf){
throwoutmem(env);
return false;
}
int vsiz = (*env)->GetArrayLength(env, val);
tcmapputkeep(tcols, nbuf, nsiz, vbuf, vsiz);
if(icv) (*env)->ReleaseByteArrayElements(env, val, vbuf, JNI_ABORT);
if(icn) (*env)->ReleaseByteArrayElements(env, name, nbuf, JNI_ABORT);
}
bool rv;
switch(dmode){
case 0:
rv = tcrdbtblput(tcrdb, kbuf, ksiz, tcols);
break;
case 1:
rv = tcrdbtblputkeep(tcrdb, kbuf, ksiz, tcols);
break;
case 2:
rv = tcrdbtblputcat(tcrdb, kbuf, ksiz, tcols);
break;
default:
rv = false;
break;
}
tcmapdel(tcols);
if(ick) (*env)->ReleaseByteArrayElements(env, pkey, kbuf, JNI_ABORT);
return rv;
}
static PyObject *
search(PyObject *self, PyObject *args){
TCTDB *tdb;
int ecode, i, rsiz;
const char *rbuf, *name;
TCMAP *cols;
TDBQRY *qry;
TCLIST *res;
const char *dbname;
const char *sfield;
const char *stext;
const int *max;
PyObject* pDict = PyDict_New();
PyObject* pList = PyList_New(0);
if (!PyArg_ParseTuple(args, "sssi", &dbname, &sfield, &stext,&max))
return NULL;
tdb = tctdbnew();
if(!tctdbopen(tdb, dbname, TDBONOLCK | TDBOREADER)){
ecode = tctdbecode(tdb);
fprintf(stderr, "open error: %s\n", tctdberrmsg(ecode));
}
qry = tctdbqrynew(tdb);
tctdbqryaddcond(qry, sfield, TDBQCSTREQ, stext);
tctdbqrysetorder(qry, "savedate", TDBQOSTRDESC);
tctdbqrysetlimit(qry, max, 0);
res = tctdbqrysearch(qry);
for(i = 0; i < tclistnum(res); i++){
rbuf = tclistval(res, i, &rsiz);
cols = tctdbget(tdb, rbuf, rsiz);
if(cols){
tcmapiterinit(cols);
PyDict_SetItemString(pDict, "kid", Py_BuildValue("s",rbuf));
while((name = tcmapiternext2(cols)) != NULL){
PyDict_SetItemString(pDict, name, Py_BuildValue("s", tcmapget2(cols, name)));
}
PyList_Append(pList,pDict);
pDict = PyDict_New();
tcmapdel(cols);
}
}
tclistdel(res);
tctdbqrydel(qry);
if(!tctdbclose(tdb)){
ecode = tctdbecode(tdb);
fprintf(stderr, "close error: %s\n", tctdberrmsg(ecode));
}
tctdbdel(tdb);
return Py_BuildValue("O",pList);
}
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *uri, *id, *data, *json, *key, *value;
double lat, lng;
int x, y;
char buf[16];
struct evkeyvalq args;
struct json_object *jsobj;
TCMAP *cols;
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
uri = evhttp_decode_uri(req->uri);
evhttp_parse_query(uri, &args);
free(uri);
argtof(&args, "lat", &lat, 0);
argtof(&args, "lng", &lng, 0);
id = (char *)evhttp_find_header(&args, "id");
data = (char *)evhttp_find_header(&args, "data");
if (id == NULL) {
evhttp_send_error(req, 400, "id is required");
evhttp_clear_headers(&args);
return;
}
x = (lat * 10000) + 1800000;
y = (lng * 10000) + 1800000;
cols = tcmapnew();
tcmapput2(cols, "data", data);
sprintf(buf, "%d", x);
tcmapput2(cols, "x", buf);
sprintf(buf, "%d", y);
tcmapput2(cols, "y", buf);
sprintf(buf, "%f", lat);
tcmapput2(cols, "lat", buf);
sprintf(buf, "%f", lng);
tcmapput2(cols, "lng", buf);
jsobj = json_object_new_object();
if (tcrdbtblput(rdb, id, strlen(id), cols)) {
json_object_object_add(jsobj, "status", json_object_new_string("ok"));
} else {
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
tcmapdel(cols);
finalize_json(req, evb, &args, jsobj);
}
/*
* This checks if a user is currently logged in. It is called at the start
* of each request.
*
* There are upto three checks performed:
*
* 1) The session_id cookie from the browser is checked with the stored
* session_id generated at login.
* 2) The client_id from the browser (currently the user agent string) is
* checked against the stored client_id.
*
* 4) Optionally (enabled by default on the login screen) a check is made
* on the requesting ip address against the stored origin_ip that was
* used at login.
*
* If any of these checks fail, the request is denied and the user is
* punted to the login screen.
*/
bool is_logged_in(void)
{
char session_id[SID_LEN + 1];
TCTDB *tdb;
TDBQRY *qry;
TCLIST *res;
TCMAP *cols;
int rsize;
const char *rbuf;
bool login_ok = false;
if (!env_vars.http_cookie)
goto out3;
snprintf(session_id, sizeof(session_id), "%s",
env_vars.http_cookie + 11);
tdb = tctdbnew();
tctdbopen(tdb, SESSION_DB, TDBOREADER);
qry = tctdbqrynew(tdb);
tctdbqryaddcond(qry, "session_id", TDBQCSTREQ, session_id);
res = tctdbqrysearch(qry);
if (tclistnum(res) == 0)
goto out2;
rbuf = tclistval(res, 0, &rsize);
cols = tctdbget(tdb, rbuf, rsize);
tcmapiterinit(cols);
/* restrict_ip */
if (atoi(tcmapget2(cols, "restrict_ip")) == 1) {
/* origin_ip */
if (strcmp(tcmapget2(cols, "origin_ip"),
env_vars.remote_addr) != 0)
goto out;
}
/* client_id */
if (strcmp(tcmapget2(cols, "client_id"),
env_vars.http_user_agent) != 0)
goto out;
/* We got here, all checks are OK */
login_ok = true;
out:
tcmapdel(cols);
out2:
tctdbqrydel(qry);
tclistdel(res);
tctdbclose(tdb);
tctdbdel(tdb);
out3:
return login_ok;
}
bool QCrawlerDB::storeRecord(QCrawlerRecord &rec) {
QString url = rec.crawl_url().url();
QString host = rec.crawl_url().host();
QString url_md5 = md5_hash(url);
QString parent_url_md5 = rec.crawl_url().parent_url_md5();
int crawl_level = rec.crawl_url().crawl_level();
QString anchor_text = rec.crawl_url().anchor_text();
QString raw_html = rec.raw_html();
// QString raw_title = rec.raw_title();
// QString raw_content = rec.raw_content();
QString raw_content_md5 = md5_hash(rec.raw_content());
int links_size = rec.raw_sub_links().size();
int download_time = rec.download_time();
int last_modified = rec.last_modified();
int loading_time = rec.loading_time();
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", url.toUtf8().constData());
tcmapput2(cols, "host", host.toUtf8().constData());
tcmapput2(cols, "url_md5", url_md5.toUtf8().constData());
tcmapput2(cols, "parent_url_md5", parent_url_md5.toUtf8().constData());
tcmapput2(cols, "crawl_level", QByteArray::number(crawl_level).constData());
tcmapput2(cols, "anchor_text", anchor_text.toUtf8().constData());
// tcmapput2(cols, "raw_html", raw_html.toUtf8().constData());
// tcmapput2(cols, "raw_title", raw_title.toUtf8().constData());
// tcmapput2(cols, "raw_content", rec.raw_content().toUtf8().constData());
// tcmapput2(cols, "raw_content_md5", md5_hash(rec.raw_content()).toUtf8().constData());
tcmapput2(cols, "title", rec.title().toUtf8().constData());
tcmapput2(cols, "content", rec.content().toUtf8().constData());
tcmapput2(cols, "links_size", QByteArray::number(links_size).constData());
tcmapput2(cols, "download_time", QByteArray::number(download_time).constData());
tcmapput2(cols, "last_modified", QByteArray::number(last_modified).constData());
tcmapput2(cols, "loading_time", QByteArray::number(loading_time).constData());
bool status = true;
if(!tcrdbtblput(record_db, url_md5.toUtf8().constData(), url_md5.toUtf8().size(), cols)){
int ecode = tcrdbecode(record_db);
fprintf(stderr, "store record put error: %s\n", tcrdberrmsg(ecode));
status = false;
}
tcmapdel(cols);
// raw_html store
if (!tcrdbput2(html_record_db, url_md5.toUtf8().constData(), raw_html.toUtf8().constData())) {
int ecode = tcrdbecode(url_hash_db);
fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode));
return false;
}
return status;
}
/* print system information */
static void sysprint(void){
TCMAP *info = tcsysinfo();
if(info){
tcmapiterinit(info);
const char *kbuf;
while((kbuf = tcmapiternext2(info)) != NULL){
iprintf("sys_%s: %s\n", kbuf, tcmapiterval2(kbuf));
}
tcmapdel(info);
}
}
void parse_conf()
{
FILE *fp;
TCLIST *data;
TCMAP *map;
char *key, *val, buf[256];
const char *value;
fp = fopen("./conf/blog.conf", "r");
if(!fp) exit(0) ;
map = tcmapnew();
memset(buf, 0, sizeof(buf));
while ((fgets(buf, 255, fp)) != NULL)
{
trim(buf);
if(strlen(buf) == 0) continue;
if(buf[0] == '#') continue;
data = explode("=", buf);
if(tclistnum(data) == 2)
{
key = strdup(tclistval2(data, 0));
val = strdup(tclistval2(data, 1));
trim(key);
trim(val);
tcmapput(map, key, strlen(key), val, strlen(val));
safe_free(key);
safe_free(val);
}
tclistdel(data);
memset(buf, 0, sizeof(buf));
}
fclose(fp);
VALIDATE_INT("blog_page_size", value, conf.page.blog)
VALIDATE_INT("admin_page_size", value, conf.page.admin)
VALIDATE_INT("comment_page_size", value, conf.page.comment)
VALIDATE_STRINT("db", value, conf.path)
VALIDATE_STRINT("username", value, conf.username)
VALIDATE_STRINT("password", value, conf.password)
tcmapdel(map);
}
/* getimpl */
JNIEXPORT jobjectArray JNICALL Java_tokyotyrant_TCRDB_getimpl
(JNIEnv *env, jobject self, jbyteArray pkey){
if(!pkey){
throwillarg(env);
return 0;
}
TCRDB *tcrdb = (TCRDB *)(intptr_t)(*env)->GetLongField(env, self, tcrdb_fid_ptr);
jboolean ick;
jbyte *kbuf = (*env)->GetByteArrayElements(env, pkey, &ick);
if(!kbuf){
throwoutmem(env);
return NULL;
}
int ksiz = (*env)->GetArrayLength(env, pkey);
jobjectArray ary = NULL;
TCMAP *tcols = tcrdbtblget(tcrdb, kbuf, ksiz);
if(tcols){
int anum = tcmaprnum(tcols) * 2;
ary = (*env)->NewObjectArray(env, anum, (*env)->GetObjectClass(env, pkey), NULL);
anum = 0;
tcmapiterinit(tcols);
const char *nbuf;
int nsiz;
while((nbuf = tcmapiternext(tcols, &nsiz)) != NULL){
int vsiz;
const char *vbuf = tcmapiterval(nbuf, &vsiz);
jbyteArray nary = (*env)->NewByteArray(env, nsiz);
if(!nary){
throwoutmem(env);
return NULL;
}
jbyteArray vary = (*env)->NewByteArray(env, vsiz);
if(!vary){
throwoutmem(env);
return NULL;
}
(*env)->SetByteArrayRegion(env, nary, 0, nsiz, (jbyte *)nbuf);
(*env)->SetByteArrayRegion(env, vary, 0, vsiz, (jbyte *)vbuf);
(*env)->SetObjectArrayElement(env, ary, anum++, nary);
(*env)->SetObjectArrayElement(env, ary, anum++, vary);
(*env)->DeleteLocalRef(env, vary);
(*env)->DeleteLocalRef(env, nary);
}
tcmapdel(tcols);
}
if(ick) (*env)->ReleaseByteArrayElements(env, pkey, kbuf, JNI_ABORT);
return ary;
}
/* perform convert command */
static int procconvert(const char *ibuf, int isiz, int fmt,
const char *buri, const char *duri, bool page){
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, ibuf);
if(fmt == FMTWIKI){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
wikidump(rbuf, cols);
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTTEXT){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page)
tcxstrprintf(rbuf, "------------------------ Tokyo Promenade ------------------------\n");
wikidumptext(rbuf, cols);
if(page)
tcxstrprintf(rbuf, "-----------------------------------------------------------------\n");
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
} else if(fmt == FMTHTML){
TCXSTR *rbuf = tcxstrnew3(IOBUFSIZ);
if(page){
const char *name = tcmapget2(cols, "name");
tcxstrprintf(rbuf, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
tcxstrprintf(rbuf, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\""
" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
tcxstrprintf(rbuf, "<html xmlns=\"http://www.w3.org/1999/xhtml\""
" xml:lang=\"en\" lang=\"en\">\n");
tcxstrprintf(rbuf, "<head>\n");
tcxstrprintf(rbuf, "<meta http-equiv=\"Content-Type\""
" content=\"text/html; charset=UTF-8\" />\n");
tcxstrprintf(rbuf, "<link rel=\"contents\" href=\"%@\" />\n", buri);
tcxstrprintf(rbuf, "<title>%@</title>\n", name ? name : "Tokyo Promenade");
tcxstrprintf(rbuf, "</head>\n");
tcxstrprintf(rbuf, "<body>\n");
}
wikidumphtml(rbuf, cols, buri, 0, duri);
if(page){
tcxstrprintf(rbuf, "</body>\n");
tcxstrprintf(rbuf, "</html>\n");
}
fwrite(tcxstrptr(rbuf), 1, tcxstrsize(rbuf), stdout);
tcxstrdel(rbuf);
}
tcmapdel(cols);
return 0;
}
/* Store a record into a q-gram database object.
`wdb' specifies the q-gram database object.
`id' specifies the ID number of the record.
`words' specifies a list object contains the words of the record.
If successful, the return value is true, else, it is false. */
static bool tcwdbputimpl(TCWDB *wdb, int64_t id, const TCLIST *words){
assert(wdb && id > 0 && words);
char idbuf[TDNUMBUFSIZ*2];
int idsiz;
TDSETVNUMBUF64(idsiz, idbuf, id);
TCMAP *cc = wdb->cc;
int wn = tclistnum(words);
TCMAP *uniq = tcmapnew2(wn + 1);
for(int i = 0; i < wn; i++){
int wsiz;
const char *word = tclistval(words, i, &wsiz);
if(!tcmapputkeep(uniq, word, wsiz, "", 0)) continue;
if(*word != '\0') tcmapputcat(cc, word, wsiz, idbuf, idsiz);
}
tcmapdel(uniq);
bool err = false;
if(tcmapmsiz(cc) >= wdb->icsiz && !tcwdbmemsync(wdb, 1)) err = true;
return !err;
}
static void sandbox_map_free(TCMAP *map)
{
int sp;
TCLIST **p;
const char *pkbuf;
// FIXME: free listeners
tcmapiterinit(map);
pkbuf = tcmapiternext2(map);
while (pkbuf) {
p = (TCLIST **)tcmapget(map, pkbuf, strlen(pkbuf), &sp);
if (p) {
tclistdel(*p);
//free(p);
}
pkbuf = tcmapiternext2(map);
}
tcmapdel(map);
}
/* RTDB_tp_as_mapping.mp_subscript */
static PyObject *
RTDB_GetItem(RTDB *self, PyObject *pykey)
{
void *key;
int key_size;
TCMAP *value;
PyObject *pyvalue;
RDBBase *rdbbase = (RDBBase *)self;
if (bytes_to_void(pykey, &key, &key_size)) {
return NULL;
}
Py_BEGIN_ALLOW_THREADS
value = tcrdbtblget(rdbbase->rdb, key, key_size);
Py_END_ALLOW_THREADS
if (!value) {
return set_rdb_error(rdbbase->rdb, key);
}
pyvalue = tcmap_to_dict(value);
tcmapdel(value);
return pyvalue;
}
static TCMAP *
pydict2tcmap(PyObject *dict)
{
if (!PyDict_Check(dict))
{
PyErr_SetString(PyExc_TypeError, "Argument is not a dict.");
return NULL;
}
PyObject *key, *value;
Py_ssize_t pos = 0;
const char *kstr, *vstr;
TCMAP *map;
map = tcmapnew();
if (map == NULL)
{
PyErr_SetString(PyExc_MemoryError, "Could not allocate map.");
return NULL;
}
while (PyDict_Next(dict, &pos, &key, &value))
{
if (!PyString_Check(value))
{
tcmapdel(map);
PyErr_SetString(PyExc_TypeError, "All values must be strings.");
return NULL;
}
kstr = PyString_AsString(key);
vstr = PyString_AsString(value);
tcmapput2(map, kstr, vstr);
}
return map;
}
/* perform update command */
static int procupdate(const char *dbpath, int64_t id, const char *wiki){
TCTDB *tdb = tctdbnew();
if(!tctdbopen(tdb, dbpath, TDBOWRITER)){
printdberr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
TCMAP *cols = tcmapnew2(TINYBNUM);
wikiload(cols, wiki);
if(!dbputart(tdb, id, cols)){
printdberr(tdb);
err = true;
}
tcmapdel(cols);
if(!tctdbclose(tdb)){
printdberr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* perform optimize command */
static int procoptimize(const char *path, int bnum, int apow, int fpow, int opts, int omode,
bool df){
TCTDB *tdb = tctdbnew();
if(g_dbgfd != INVALID_HANDLE_VALUE) tctdbsetdbgfd(tdb, g_dbgfd);
if(!tctdbsetcodecfunc(tdb, _tc_recencode, NULL, _tc_recdecode, NULL)) printerr(tdb);
int64_t msiz = 0;
TCMAP *info = tcsysinfo();
if(info){
msiz = tcatoi(tcmapget4(info, "total", "0"));
tcmapdel(info);
}
if(!tctdbsetinvcache(tdb, msiz >= (1 << 30) ? msiz / 4 : 0, 1.0)) printerr(tdb);
if(!tctdbopen(tdb, path, TDBOWRITER | omode)){
printerr(tdb);
tctdbdel(tdb);
return 1;
}
bool err = false;
if(df){
if(!tctdbdefrag(tdb, INT64_MAX)){
printerr(tdb);
err = true;
}
} else {
if(!tctdboptimize(tdb, bnum, apow, fpow, opts)){
printerr(tdb);
err = true;
}
}
if(!tctdbclose(tdb)){
if(!err) printerr(tdb);
err = true;
}
tctdbdel(tdb);
return err ? 1 : 0;
}
/* kwicimpl */
JNIEXPORT jobjectArray JNICALL Java_tokyocabinet_TDBQRY_kwicimpl
(JNIEnv *env, jobject self, jobjectArray cols, jstring name, jint width, jint opts){
if(!cols){
throwillarg(env);
return NULL;
}
jclass clsstring = name ?
(*env)->GetObjectClass(env, name) : (*env)->FindClass(env, CLSSTRING);
TDBQRY *qry = (TDBQRY *)(intptr_t)(*env)->GetLongField(env, self, tdbqry_fid_ptr);
jsize cnum = (*env)->GetArrayLength(env, cols);
TCMAP *tcols = tcmapnew2(cnum + 1);
jboolean icc = false;
const char *cbuf = NULL;
int csiz = 0;
if(name){
cbuf = (*env)->GetStringUTFChars(env, name, &icc);
if(!cbuf){
throwoutmem(env);
return NULL;
}
csiz = strlen(cbuf);
}
cnum--;
for(int i = 0; i < cnum; i += 2){
name = (*env)->GetObjectArrayElement(env, cols, i);
jboolean icn;
jbyte *nbuf = (*env)->GetByteArrayElements(env, name, &icn);
if(!nbuf){
throwoutmem(env);
return NULL;
}
int nsiz = (*env)->GetArrayLength(env, name);
jobject val = (*env)->GetObjectArrayElement(env, cols, i + 1);
jboolean icv;
jbyte *vbuf = (*env)->GetByteArrayElements(env, val, &icv);
if(!vbuf){
throwoutmem(env);
return false;
}
int vsiz = (*env)->GetArrayLength(env, val);
if(!cbuf || (nsiz == csiz && !memcmp(nbuf, cbuf, nsiz)))
tcmapputkeep(tcols, nbuf, nsiz, vbuf, vsiz);
if(icv) (*env)->ReleaseByteArrayElements(env, val, vbuf, JNI_ABORT);
if(icn) (*env)->ReleaseByteArrayElements(env, name, nbuf, JNI_ABORT);
(*env)->DeleteLocalRef(env, val);
(*env)->DeleteLocalRef(env, name);
}
if(width < 0){
width = 1 << 30;
opts |= TCKWNOOVER | TCKWPULEAD;
}
TCLIST *texts = tctdbqrykwic(qry, tcols, cbuf, width, opts);
int tnum = tclistnum(texts);
jobjectArray ary = (*env)->NewObjectArray(env, tnum, clsstring, NULL);
for(int i = 0; i < tnum; i++){
jobject text = (*env)->NewStringUTF(env, tclistval2(texts, i));
(*env)->SetObjectArrayElement(env, ary, i, text);
(*env)->DeleteLocalRef(env, text);
}
tclistdel(texts);
if(icc) (*env)->ReleaseStringUTFChars(env, name, cbuf);
tcmapdel(tcols);
return ary;
}
void
xtc_mapdel(void* cols)
{
tcmapdel(cols);
}
int QStore::process(QContentRecord &record)
{
if (record.want_type == mimetype::image) {
assert(need_media_db);
const std::string &url_md5 = record.url_md5;
std::string media_key = url_md5 + ".i";
if (!tcrdbput(media_db, media_key.c_str(), media_key.size(), record.raw_content.c_str(), record.raw_content.size())) {
int ecode = tcrdbecode(media_db);
// TODO log
LOG(ERROR) << "put media url " << record.url << " error " << tcrdberrmsg(ecode);
return -1;
}
} else { // default text/html
assert(need_html_db && need_record_db);
std::string tmps;
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", record.url.c_str());
tcmapput2(cols, "host", record.host.c_str());
tcmapput2(cols, "url_md5", record.url_md5.c_str());
tcmapput2(cols, "parent_url_md5", record.parent_url_md5.c_str());
strtk::type_to_string(record.crawl_level, tmps);
tcmapput2(cols, "crawl_level", tmps.c_str());
strtk::type_to_string(record.find_time, tmps);
tcmapput2(cols, "find_time", tmps.c_str());
tcmapput2(cols, "anchor_text", record.anchor_text.c_str());
tcmapput2(cols, "crawl_tag", record.crawl_tag.c_str());
// last download_time
strtk::type_to_string(record.download_time, tmps);
tcmapput2(cols, "download_time", tmps.c_str());
strtk::type_to_string(record.http_code, tmps);
tcmapput2(cols, "http_code", tmps.c_str());
if (record.is_list) {
tcmapput2(cols, "is_list", "1");
} else {
tcmapput2(cols, "is_list", "0");
}
if (record.crawled_okay) {
tcmapput2(cols, "crawled_okay", "1");
tcmapput2(cols, "raw_title", record.raw_title.c_str());
tcmapput2(cols, "title", record.title.c_str());
tcmapput2(cols, "keywords", record.keywords.c_str());
tcmapput2(cols, "description", record.description.c_str());
tcmapput2(cols, "content", record.content.c_str());
strtk::type_to_string(record.publish_time, tmps);
tcmapput2(cols, "publish_time", tmps.c_str());
tcmapput2(cols, "images", record.images.c_str());
if (record.is_redirect) {
tcmapput2(cols, "is_redirect", "1");
} else {
tcmapput2(cols, "is_redirect", "0");
}
tcmapput2(cols, "redirect_url", record.redirect_url.c_str());
strtk::type_to_string(record.content_confidence, tmps);
tcmapput2(cols, "content_confidence", tmps.c_str());
strtk::type_to_string(record.list_confidence, tmps);
tcmapput2(cols, "list_confidence", tmps.c_str());
strtk::type_to_string(record.links_size, tmps);
tcmapput2(cols, "links_size", tmps.c_str());
strtk::type_to_string(record.last_modified, tmps);
tcmapput2(cols, "last_modified", tmps.c_str());
strtk::type_to_string(record.loading_time, tmps);
tcmapput2(cols, "loading_time", tmps.c_str());
strtk::type_to_string(record.new_links_size, tmps);
tcmapput2(cols, "new_links_size", tmps.c_str());
} else {
tcmapput2(cols, "crawled_okay", "0");
}
if(!tcrdbtblput(record_db, record.url_md5.c_str(), record.url_md5.size(), cols)){
int ecode = tcrdbecode(record_db);
LOG(ERROR) << "put record error " << tcrdberrmsg(ecode);
tcmapdel(cols);
return -1;
}
tcmapdel(cols);
std::string url_md5_vdom = record.url_md5 + ".v";
memcached_return_t rc = memcached_set(html_memc, url_md5_vdom.c_str(), url_md5_vdom.size(),
record.vdom.c_str(), record.vdom.size(), 0, 0);
if (rc != MEMCACHED_SUCCESS) {
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size()
<< " error: " << memcached_strerror(html_memc, rc);
return -1;
}
/*
if (!tcrdbput(html_db, url_md5_vdom.c_str(), url_md5_vdom.size(), record.vdom.c_str(), record.vdom.size())) {
int ecode = tcrdbecode(html_db);
// TODO log
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size() << " error " << tcrdberrmsg(ecode);
return -1;
}
*/
}
return 0;
}
