TCMAP * redis_keys_to_tcmap(char *keys) {
TCMAP *map;
redisReply *rr;
int i;
xlog("redis_keys_to_tcmap", "keys=%s\n", keys);
rr = redisCommand(dfs.redis.rc, "KEYS %s", keys);
if(!rr) return NULL;
map = tcmapnew();
if(!map) goto cleanup;
for(i = 0; i < rr->elements; i++) {
char * str = rr->element[i]->str;
tcmapput(map, str, strlen(str), str, strlen(str));
}
cleanup:
if(rr)
freeReplyObject(rr);
xlog("MAP", "%i\n", rr->elements);
return map;
}
bool QCrawlerDB::storeRecord(QCrawlerRecord &rec) {
QString url = rec.crawl_url().url();
QString host = rec.crawl_url().host();
QString url_md5 = md5_hash(url);
QString parent_url_md5 = rec.crawl_url().parent_url_md5();
int crawl_level = rec.crawl_url().crawl_level();
QString anchor_text = rec.crawl_url().anchor_text();
QString raw_html = rec.raw_html();
// QString raw_title = rec.raw_title();
// QString raw_content = rec.raw_content();
QString raw_content_md5 = md5_hash(rec.raw_content());
int links_size = rec.raw_sub_links().size();
int download_time = rec.download_time();
int last_modified = rec.last_modified();
int loading_time = rec.loading_time();
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", url.toUtf8().constData());
tcmapput2(cols, "host", host.toUtf8().constData());
tcmapput2(cols, "url_md5", url_md5.toUtf8().constData());
tcmapput2(cols, "parent_url_md5", parent_url_md5.toUtf8().constData());
tcmapput2(cols, "crawl_level", QByteArray::number(crawl_level).constData());
tcmapput2(cols, "anchor_text", anchor_text.toUtf8().constData());
// tcmapput2(cols, "raw_html", raw_html.toUtf8().constData());
// tcmapput2(cols, "raw_title", raw_title.toUtf8().constData());
// tcmapput2(cols, "raw_content", rec.raw_content().toUtf8().constData());
// tcmapput2(cols, "raw_content_md5", md5_hash(rec.raw_content()).toUtf8().constData());
tcmapput2(cols, "title", rec.title().toUtf8().constData());
tcmapput2(cols, "content", rec.content().toUtf8().constData());
tcmapput2(cols, "links_size", QByteArray::number(links_size).constData());
tcmapput2(cols, "download_time", QByteArray::number(download_time).constData());
tcmapput2(cols, "last_modified", QByteArray::number(last_modified).constData());
tcmapput2(cols, "loading_time", QByteArray::number(loading_time).constData());
bool status = true;
if(!tcrdbtblput(record_db, url_md5.toUtf8().constData(), url_md5.toUtf8().size(), cols)){
int ecode = tcrdbecode(record_db);
fprintf(stderr, "store record put error: %s\n", tcrdberrmsg(ecode));
status = false;
}
tcmapdel(cols);
// raw_html store
if (!tcrdbput2(html_record_db, url_md5.toUtf8().constData(), raw_html.toUtf8().constData())) {
int ecode = tcrdbecode(url_hash_db);
fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode));
return false;
}
return status;
}
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *uri, *id, *data, *json, *key, *value;
double lat, lng;
int x, y;
char buf[16];
struct evkeyvalq args;
struct json_object *jsobj;
TCMAP *cols;
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
uri = evhttp_decode_uri(req->uri);
evhttp_parse_query(uri, &args);
free(uri);
argtof(&args, "lat", &lat, 0);
argtof(&args, "lng", &lng, 0);
id = (char *)evhttp_find_header(&args, "id");
data = (char *)evhttp_find_header(&args, "data");
if (id == NULL) {
evhttp_send_error(req, 400, "id is required");
evhttp_clear_headers(&args);
return;
}
x = (lat * 10000) + 1800000;
y = (lng * 10000) + 1800000;
cols = tcmapnew();
tcmapput2(cols, "data", data);
sprintf(buf, "%d", x);
tcmapput2(cols, "x", buf);
sprintf(buf, "%d", y);
tcmapput2(cols, "y", buf);
sprintf(buf, "%f", lat);
tcmapput2(cols, "lat", buf);
sprintf(buf, "%f", lng);
tcmapput2(cols, "lng", buf);
jsobj = json_object_new_object();
if (tcrdbtblput(rdb, id, strlen(id), cols)) {
json_object_object_add(jsobj, "status", json_object_new_string("ok"));
} else {
db_status = tcrdbecode(rdb);
db_error_to_json(db_status, jsobj);
}
tcmapdel(cols);
finalize_json(req, evb, &args, jsobj);
}
extern TCMAP *varytomap(VALUE vary){
int i;
TCLIST *keys;
TCMAP *recs = tcmapnew();
keys = varytolist(vary);
for(i = 0; i < tclistnum(keys); i++){
int ksiz;
const char *kbuf = tclistval(keys, i, &ksiz);
tcmapput(recs, kbuf, ksiz, "", 0);
}
tclistdel(keys);
return recs;
}
void parse_conf()
{
FILE *fp;
TCLIST *data;
TCMAP *map;
char *key, *val, buf[256];
const char *value;
fp = fopen("./conf/blog.conf", "r");
if(!fp) exit(0) ;
map = tcmapnew();
memset(buf, 0, sizeof(buf));
while ((fgets(buf, 255, fp)) != NULL)
{
trim(buf);
if(strlen(buf) == 0) continue;
if(buf[0] == '#') continue;
data = explode("=", buf);
if(tclistnum(data) == 2)
{
key = strdup(tclistval2(data, 0));
val = strdup(tclistval2(data, 1));
trim(key);
trim(val);
tcmapput(map, key, strlen(key), val, strlen(val));
safe_free(key);
safe_free(val);
}
tclistdel(data);
memset(buf, 0, sizeof(buf));
}
fclose(fp);
VALIDATE_INT("blog_page_size", value, conf.page.blog)
VALIDATE_INT("admin_page_size", value, conf.page.admin)
VALIDATE_INT("comment_page_size", value, conf.page.comment)
VALIDATE_STRINT("db", value, conf.path)
VALIDATE_STRINT("username", value, conf.username)
VALIDATE_STRINT("password", value, conf.password)
tcmapdel(map);
}
Server *
Server_create(void)
{
Server *server = NULL;
server = (Server *)malloc(sizeof(Server));
if (server == NULL) {
ERROR("memory for server not allocated");
return 0;
}
server->config_file = "config.lua";
server->key_count = 0;
server->ttl_extension = 3600;
if ((L = luaL_newstate()) == NULL) {
ERROR("could not initiate scripting environment");
return 0;
}
luaL_openlibs(L);
server->book = tcmapnew();
return server;
}
static TCMAP *
pydict2tcmap(PyObject *dict)
{
if (!PyDict_Check(dict))
{
PyErr_SetString(PyExc_TypeError, "Argument is not a dict.");
return NULL;
}
PyObject *key, *value;
Py_ssize_t pos = 0;
const char *kstr, *vstr;
TCMAP *map;
map = tcmapnew();
if (map == NULL)
{
PyErr_SetString(PyExc_MemoryError, "Could not allocate map.");
return NULL;
}
while (PyDict_Next(dict, &pos, &key, &value))
{
if (!PyString_Check(value))
{
tcmapdel(map);
PyErr_SetString(PyExc_TypeError, "All values must be strings.");
return NULL;
}
kstr = PyString_AsString(key);
vstr = PyString_AsString(value);
tcmapput2(map, kstr, vstr);
}
return map;
}
void*
xtc_mapnew()
{
return tcmapnew();
}
int QStore::process(QContentRecord &record)
{
if (record.want_type == mimetype::image) {
assert(need_media_db);
const std::string &url_md5 = record.url_md5;
std::string media_key = url_md5 + ".i";
if (!tcrdbput(media_db, media_key.c_str(), media_key.size(), record.raw_content.c_str(), record.raw_content.size())) {
int ecode = tcrdbecode(media_db);
// TODO log
LOG(ERROR) << "put media url " << record.url << " error " << tcrdberrmsg(ecode);
return -1;
}
} else { // default text/html
assert(need_html_db && need_record_db);
std::string tmps;
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", record.url.c_str());
tcmapput2(cols, "host", record.host.c_str());
tcmapput2(cols, "url_md5", record.url_md5.c_str());
tcmapput2(cols, "parent_url_md5", record.parent_url_md5.c_str());
strtk::type_to_string(record.crawl_level, tmps);
tcmapput2(cols, "crawl_level", tmps.c_str());
strtk::type_to_string(record.find_time, tmps);
tcmapput2(cols, "find_time", tmps.c_str());
tcmapput2(cols, "anchor_text", record.anchor_text.c_str());
tcmapput2(cols, "crawl_tag", record.crawl_tag.c_str());
// last download_time
strtk::type_to_string(record.download_time, tmps);
tcmapput2(cols, "download_time", tmps.c_str());
strtk::type_to_string(record.http_code, tmps);
tcmapput2(cols, "http_code", tmps.c_str());
if (record.is_list) {
tcmapput2(cols, "is_list", "1");
} else {
tcmapput2(cols, "is_list", "0");
}
if (record.crawled_okay) {
tcmapput2(cols, "crawled_okay", "1");
tcmapput2(cols, "raw_title", record.raw_title.c_str());
tcmapput2(cols, "title", record.title.c_str());
tcmapput2(cols, "keywords", record.keywords.c_str());
tcmapput2(cols, "description", record.description.c_str());
tcmapput2(cols, "content", record.content.c_str());
strtk::type_to_string(record.publish_time, tmps);
tcmapput2(cols, "publish_time", tmps.c_str());
tcmapput2(cols, "images", record.images.c_str());
if (record.is_redirect) {
tcmapput2(cols, "is_redirect", "1");
} else {
tcmapput2(cols, "is_redirect", "0");
}
tcmapput2(cols, "redirect_url", record.redirect_url.c_str());
strtk::type_to_string(record.content_confidence, tmps);
tcmapput2(cols, "content_confidence", tmps.c_str());
strtk::type_to_string(record.list_confidence, tmps);
tcmapput2(cols, "list_confidence", tmps.c_str());
strtk::type_to_string(record.links_size, tmps);
tcmapput2(cols, "links_size", tmps.c_str());
strtk::type_to_string(record.last_modified, tmps);
tcmapput2(cols, "last_modified", tmps.c_str());
strtk::type_to_string(record.loading_time, tmps);
tcmapput2(cols, "loading_time", tmps.c_str());
strtk::type_to_string(record.new_links_size, tmps);
tcmapput2(cols, "new_links_size", tmps.c_str());
} else {
tcmapput2(cols, "crawled_okay", "0");
}
if(!tcrdbtblput(record_db, record.url_md5.c_str(), record.url_md5.size(), cols)){
int ecode = tcrdbecode(record_db);
LOG(ERROR) << "put record error " << tcrdberrmsg(ecode);
tcmapdel(cols);
return -1;
}
tcmapdel(cols);
std::string url_md5_vdom = record.url_md5 + ".v";
memcached_return_t rc = memcached_set(html_memc, url_md5_vdom.c_str(), url_md5_vdom.size(),
record.vdom.c_str(), record.vdom.size(), 0, 0);
if (rc != MEMCACHED_SUCCESS) {
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size()
<< " error: " << memcached_strerror(html_memc, rc);
return -1;
}
/*
if (!tcrdbput(html_db, url_md5_vdom.c_str(), url_md5_vdom.size(), record.vdom.c_str(), record.vdom.size())) {
int ecode = tcrdbecode(html_db);
// TODO log
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size() << " error " << tcrdberrmsg(ecode);
return -1;
}
*/
}
return 0;
}
/* perform misc command */
static int procmisc(const char *path, int rnum, bool mt, int omode){
my_my_my_iprintf("<Miscellaneous Test>\n seed=%u path=%s rnum=%d mt=%d omode=%d\n\n",
g_randseed, path, rnum, mt, omode);
bool err = false;
double stime = tctime();
TCFDB *fdb = tcfdbnew();
if(g_dbgfd >= 0) tcfdbsetdbgfd(fdb, g_dbgfd);
if(mt && !tcfdbsetmutex(fdb)){
eprint(fdb, __LINE__, "tcfdbsetmutex");
err = true;
}
if(!tcfdbtune(fdb, RECBUFSIZ, EXHEADSIZ + (RECBUFSIZ + sizeof(int)) * rnum)){
eprint(fdb, __LINE__, "tcfdbtune");
err = true;
}
if(!tcfdbopen(fdb, path, FDBOWRITER | FDBOCREAT | FDBOTRUNC | omode)){
eprint(fdb, __LINE__, "tcfdbopen");
err = true;
}
if(TCUSEPTHREAD){
TCFDB *fdbdup = tcfdbnew();
if(tcfdbopen(fdbdup, path, FDBOREADER)){
eprint(fdb, __LINE__, "(validation)");
err = true;
} else if(tcfdbecode(fdbdup) != TCETHREAD){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
tcfdbdel(fdbdup);
}
my_my_my_iprintf("writing:\n");
for(int i = 1; i <= rnum; i++){
char buf[RECBUFSIZ];
int len = sprintf(buf, "%08d", i);
if(!tcfdbputkeep2(fdb, buf, len, buf, len)){
eprint(fdb, __LINE__, "tcfdbputkeep");
err = true;
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("reading:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%08d", i);
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
if(!vbuf){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
break;
} else if(vsiz != ksiz || memcmp(vbuf, kbuf, vsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
tcfree(vbuf);
break;
}
tcfree(vbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(tcfdbrnum(fdb) != rnum){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
my_my_my_iprintf("random writing:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
char vbuf[RECBUFSIZ];
int vsiz = myrand(RECBUFSIZ);
memset(vbuf, '*', vsiz);
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput");
err = true;
break;
}
int rsiz;
char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz);
if(!rbuf){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
break;
}
if(rsiz != vsiz || memcmp(rbuf, vbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
tcfree(rbuf);
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
tcfree(rbuf);
}
my_my_my_iprintf("random erasing:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("writing:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "[%d]", i);
char vbuf[RECBUFSIZ];
int vsiz = i % RECBUFSIZ;
memset(vbuf, '*', vsiz);
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep");
err = true;
break;
}
if(vsiz < 1){
char tbuf[PATH_MAX];
for(int j = 0; j < PATH_MAX; j++){
tbuf[j] = myrand(0x100);
}
if(!tcfdbput2(fdb, kbuf, ksiz, tbuf, PATH_MAX)){
eprint(fdb, __LINE__, "tcfdbput");
err = true;
break;
}
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("erasing:\n");
for(int i = 1; i <= rnum; i++){
if(i % 2 == 1){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "[%d]", i);
if(!tcfdbout2(fdb, kbuf, ksiz)){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
break;
}
if(tcfdbout2(fdb, kbuf, ksiz) || tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
break;
}
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("random writing and reopening:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
int vsiz = myrand(RECBUFSIZ);
char *vbuf = tcmalloc(vsiz + 1);
memset(vbuf, '*', vsiz);
switch(myrand(3)){
case 0:
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput");
err = true;
}
break;
case 1:
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbputcat");
err = true;
}
break;
case 2:
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
}
break;
}
tcfree(vbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(!tcfdbclose(fdb)){
eprint(fdb, __LINE__, "tcfdbclose");
err = true;
}
if(!tcfdbopen(fdb, path, FDBOWRITER | omode)){
eprint(fdb, __LINE__, "tcfdbopen");
err = true;
}
my_my_my_iprintf("checking:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "[%d]", i);
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
if(vbuf){
tcfree(vbuf);
} else if(tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("writing:\n");
for(int i = 1; i <= rnum; i++){
char buf[RECBUFSIZ];
int len = sprintf(buf, "%08d", i);
if(!tcfdbput2(fdb, buf, len, buf, len)){
eprint(fdb, __LINE__, "tcfdbput");
err = true;
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("reading:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%08d", i);
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
if(!vbuf){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
break;
} else if(vsiz != ksiz || memcmp(vbuf, kbuf, vsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
tcfree(vbuf);
break;
}
tcfree(vbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("checking iterator:\n");
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
char *kbuf;
int ksiz;
int inum = 0;
for(int i = 1; (kbuf = tcfdbiternext2(fdb, &ksiz)) != NULL; i++, inum++){
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
if(!vbuf){
eprint(fdb, __LINE__, "tcfdbget2");
err = true;
tcfree(kbuf);
break;
}
tcfree(vbuf);
tcfree(kbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(tcfdbecode(fdb) != TCENOREC || inum != tcfdbrnum(fdb)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
my_my_my_iprintf("iteration updating:\n");
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
inum = 0;
for(int i = 1; (kbuf = tcfdbiternext2(fdb, &ksiz)) != NULL; i++, inum++){
if(myrand(2) == 0){
if(!tcfdbputcat2(fdb, kbuf, ksiz, "0123456789", 10)){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
tcfree(kbuf);
break;
}
} else {
if(!tcfdbout2(fdb, kbuf, ksiz)){
eprint(fdb, __LINE__, "tcfdbout");
err = true;
tcfree(kbuf);
break;
}
}
tcfree(kbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(tcfdbecode(fdb) != TCENOREC || inum < tcfdbrnum(fdb)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
if(myrand(10) == 0 && !tcfdbsync(fdb)){
eprint(fdb, __LINE__, "tcfdbsync");
err = true;
}
if(!tcfdbvanish(fdb)){
eprint(fdb, __LINE__, "tcfdbvanish");
err = true;
}
TCMAP *map = tcmapnew();
my_my_my_iprintf("random writing:\n");
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
char vbuf[RECBUFSIZ];
int vsiz = sprintf(vbuf, "%d", myrand(rnum) + 1);
switch(myrand(7)){
case 0:
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
tcmapput(map, kbuf, ksiz, vbuf, vsiz);
break;
case 1:
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep2");
err = true;
}
tcmapputkeep(map, kbuf, ksiz, vbuf, vsiz);
break;
case 2:
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
}
tcmapputcat(map, kbuf, ksiz, vbuf, vsiz);
break;
case 3:
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
tcmapout(map, kbuf, ksiz);
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
my_my_my_iprintf("checking transaction commit:\n");
if(!tcfdbtranbegin(fdb)){
eprint(fdb, __LINE__, "tcfdbtranbegin");
err = true;
}
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
char vbuf[RECBUFSIZ];
int vsiz = sprintf(vbuf, "[%d]", myrand(rnum) + 1);
switch(myrand(7)){
case 0:
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
tcmapput(map, kbuf, ksiz, vbuf, vsiz);
break;
case 1:
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep2");
err = true;
}
tcmapputkeep(map, kbuf, ksiz, vbuf, vsiz);
break;
case 2:
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
}
tcmapputcat(map, kbuf, ksiz, vbuf, vsiz);
break;
case 3:
if(tcfdbaddint(fdb, tcfdbkeytoid(kbuf, ksiz), 1) == INT_MIN &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbaddint");
err = true;
}
tcmapaddint(map, kbuf, ksiz, 1);
break;
case 4:
if(isnan(tcfdbadddouble(fdb, tcfdbkeytoid(kbuf, ksiz), 1.0)) &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbadddouble");
err = true;
}
tcmapadddouble(map, kbuf, ksiz, 1.0);
break;
case 5:
if(myrand(2) == 0){
void *op = (void *)(intptr_t)(myrand(3) + 1);
if(!tcfdbputproc(fdb, tcfdbkeytoid(kbuf, ksiz), vbuf, vsiz, pdprocfunc, op) &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputproc");
err = true;
}
tcmapputproc(map, kbuf, ksiz, vbuf, vsiz, pdprocfunc, op);
} else {
vsiz = myrand(10);
void *op = (void *)(intptr_t)(myrand(3) + 1);
if(!tcfdbputproc(fdb, tcfdbkeytoid(kbuf, ksiz), NULL, vsiz, pdprocfunc, op) &&
tcfdbecode(fdb) != TCEKEEP && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbputproc");
err = true;
}
tcmapputproc(map, kbuf, ksiz, NULL, vsiz, pdprocfunc, op);
}
break;
case 6:
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
tcmapout(map, kbuf, ksiz);
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(!tcfdbtrancommit(fdb)){
eprint(fdb, __LINE__, "tcfdbtrancommit");
err = true;
}
my_my_my_iprintf("checking transaction abort:\n");
uint64_t ornum = tcfdbrnum(fdb);
uint64_t ofsiz = tcfdbfsiz(fdb);
if(!tcfdbtranbegin(fdb)){
eprint(fdb, __LINE__, "tcfdbtranbegin");
err = true;
}
for(int i = 1; i <= rnum; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", myrand(rnum) + 1);
char vbuf[RECBUFSIZ];
int vsiz = sprintf(vbuf, "[%d]", myrand(rnum) + 1);
switch(myrand(7)){
case 0:
if(!tcfdbput2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
break;
case 1:
if(!tcfdbputkeep2(fdb, kbuf, ksiz, vbuf, vsiz) && tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputkeep2");
err = true;
}
break;
case 2:
if(!tcfdbputcat2(fdb, kbuf, ksiz, vbuf, vsiz)){
eprint(fdb, __LINE__, "tcfdbputcat2");
err = true;
}
break;
case 3:
if(tcfdbaddint(fdb, tcfdbkeytoid(kbuf, ksiz), 1) == INT_MIN &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbaddint");
err = true;
}
break;
case 4:
if(isnan(tcfdbadddouble(fdb, tcfdbkeytoid(kbuf, ksiz), 1.0)) &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbadddouble");
err = true;
}
break;
case 5:
if(myrand(2) == 0){
void *op = (void *)(intptr_t)(myrand(3) + 1);
if(!tcfdbputproc(fdb, tcfdbkeytoid(kbuf, ksiz), vbuf, vsiz, pdprocfunc, op) &&
tcfdbecode(fdb) != TCEKEEP){
eprint(fdb, __LINE__, "tcfdbputproc");
err = true;
}
} else {
vsiz = myrand(10);
void *op = (void *)(intptr_t)(myrand(3) + 1);
if(!tcfdbputproc(fdb, tcfdbkeytoid(kbuf, ksiz), NULL, vsiz, pdprocfunc, op) &&
tcfdbecode(fdb) != TCEKEEP && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbputproc");
err = true;
}
}
break;
case 6:
if(!tcfdbout2(fdb, kbuf, ksiz) && tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "tcfdbout2");
err = true;
}
break;
}
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(!tcfdbtranabort(fdb)){
eprint(fdb, __LINE__, "tcfdbtranabort");
err = true;
}
my_my_my_iprintf("checking consistency:\n");
if(tcfdbrnum(fdb) != ornum || tcfdbfsiz(fdb) != ofsiz || tcfdbrnum(fdb) != tcmaprnum(map)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
inum = 0;
tcmapiterinit(map);
const char *tkbuf;
int tksiz;
for(int i = 1; (tkbuf = tcmapiternext(map, &tksiz)) != NULL; i++, inum++){
int tvsiz;
const char *tvbuf = tcmapiterval(tkbuf, &tvsiz);
if(tvsiz > RECBUFSIZ) tvsiz = RECBUFSIZ;
int rsiz;
char *rbuf = tcfdbget2(fdb, tkbuf, tksiz, &rsiz);
if(!rbuf || rsiz != tvsiz || memcmp(rbuf, tvbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
tcfree(rbuf);
break;
}
tcfree(rbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(rnum > 250) my_my_my_iprintf(" (%08d)\n", inum);
inum = 0;
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
for(int i = 1; (kbuf = tcfdbiternext2(fdb, &ksiz)) != NULL; i++, inum++){
int vsiz;
char *vbuf = tcfdbget2(fdb, kbuf, ksiz, &vsiz);
int rsiz;
const char *rbuf = tcmapget(map, kbuf, ksiz, &rsiz);
if(rsiz > RECBUFSIZ) rsiz = RECBUFSIZ;
if(!rbuf || rsiz != vsiz || memcmp(rbuf, vbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
tcfree(vbuf);
tcfree(kbuf);
break;
}
tcfree(vbuf);
tcfree(kbuf);
if(rnum > 250 && i % (rnum / 250) == 0){
iputchar('.');
if(i == rnum || i % (rnum / 10) == 0) my_my_my_iprintf(" (%08d)\n", i);
}
}
if(rnum > 250) my_my_my_iprintf(" (%08d)\n", inum);
tcmapdel(map);
if(!tcfdbvanish(fdb)){
eprint(fdb, __LINE__, "tcfdbvanish");
err = true;
}
if(rnum >= 100){
if(!tcfdbtranbegin(fdb)){
eprint(fdb, __LINE__, "tcfdbtranbegin");
err = true;
}
if(!tcfdbput3(fdb, "99", "hirabayashi")){
eprint(fdb, __LINE__, "tcfdbput3");
err = true;
}
for(int i = 0; i < 10; i++){
char buf[RECBUFSIZ];
int size = sprintf(buf, "%d", myrand(rnum) + 1);
if(!tcfdbput2(fdb, buf, size, buf, size)){
eprint(fdb, __LINE__, "tcfdbput2");
err = true;
}
}
for(int i = myrand(3) + 1; i < PATH_MAX; i = i * 2 + myrand(3)){
char vbuf[i];
memset(vbuf, '@', i - 1);
vbuf[i-1] = '\0';
if(!tcfdbput3(fdb, "99", vbuf)){
eprint(fdb, __LINE__, "tcfdbput3");
err = true;
}
}
if(!tcfdbforeach(fdb, iterfunc, NULL)){
eprint(fdb, __LINE__, "tcfdbforeach");
err = true;
}
}
my_my_my_iprintf("record number: %llu\n", (unsigned long long)tcfdbrnum(fdb));
my_my_my_iprintf("size: %llu\n", (unsigned long long)tcfdbfsiz(fdb));
mprint(fdb);
sysprint();
if(!tcfdbclose(fdb)){
eprint(fdb, __LINE__, "tcfdbclose");
err = true;
}
tcfdbdel(fdb);
my_my_my_iprintf("time: %.3f\n", tctime() - stime);
my_my_my_iprintf("%s\n\n", err ? "error" : "ok");
return err ? 1 : 0;
}
/* perform wicked command */
static int procwicked(const char *path, int tnum, int rnum, int omode, bool nc){
iprintf("<Writing Test>\n seed=%u path=%s tnum=%d rnum=%d omode=%d nc=%d\n\n",
g_randseed, path, tnum, rnum, omode, nc);
bool err = false;
double stime = tctime();
TCFDB *fdb = tcfdbnew();
if(g_dbgfd >= 0) tcfdbsetdbgfd(fdb, g_dbgfd);
if(!tcfdbsetmutex(fdb)){
eprint(fdb, __LINE__, "tcfdbsetmutex");
err = true;
}
if(!tcfdbtune(fdb, RECBUFSIZ * 2, EXHEADSIZ + (RECBUFSIZ * 2 + sizeof(int)) * rnum * tnum)){
eprint(fdb, __LINE__, "tcfdbtune");
err = true;
}
if(!tcfdbopen(fdb, path, FDBOWRITER | FDBOCREAT | FDBOTRUNC | omode)){
eprint(fdb, __LINE__, "tcfdbopen");
err = true;
}
if(!tcfdbiterinit(fdb)){
eprint(fdb, __LINE__, "tcfdbiterinit");
err = true;
}
TARGWICKED targs[tnum];
pthread_t threads[tnum];
TCMAP *map = tcmapnew();
if(tnum == 1){
targs[0].fdb = fdb;
targs[0].rnum = rnum;
targs[0].nc = nc;
targs[0].id = 0;
targs[0].map = map;
if(threadwicked(targs) != NULL) err = true;
} else {
for(int i = 0; i < tnum; i++){
targs[i].fdb = fdb;
targs[i].rnum = rnum;
targs[i].nc = nc;
targs[i].id = i;
targs[i].map = map;
if(pthread_create(threads + i, NULL, threadwicked, targs + i) != 0){
eprint(fdb, __LINE__, "pthread_create");
targs[i].id = -1;
err = true;
}
}
for(int i = 0; i < tnum; i++){
if(targs[i].id == -1) continue;
void *rv;
if(pthread_join(threads[i], &rv) != 0){
eprint(fdb, __LINE__, "pthread_join");
err = true;
} else if(rv){
err = true;
}
}
}
if(!nc){
if(!tcfdbsync(fdb)){
eprint(fdb, __LINE__, "tcfdbsync");
err = true;
}
if(tcfdbrnum(fdb) != tcmaprnum(map)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
int end = rnum * tnum;
for(int i = 1; i <= end && !err; i++){
char kbuf[RECBUFSIZ];
int ksiz = sprintf(kbuf, "%d", i);
int vsiz;
const char *vbuf = tcmapget(map, kbuf, ksiz, &vsiz);
int rsiz;
char *rbuf = tcfdbget2(fdb, kbuf, ksiz, &rsiz);
if(vbuf){
iputchar('.');
if(vsiz > tcfdbwidth(fdb)) vsiz = tcfdbwidth(fdb);
if(!rbuf){
eprint(fdb, __LINE__, "tcfdbget");
err = true;
} else if(rsiz != vsiz || memcmp(rbuf, vbuf, rsiz)){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
} else {
iputchar('*');
if(rbuf || tcfdbecode(fdb) != TCENOREC){
eprint(fdb, __LINE__, "(validation)");
err = true;
}
}
tcfree(rbuf);
if(i % 50 == 0) iprintf(" (%08d)\n", i);
}
if(rnum % 50 > 0) iprintf(" (%08d)\n", rnum);
}
tcmapdel(map);
iprintf("record number: %llu\n", (unsigned long long)tcfdbrnum(fdb));
iprintf("size: %llu\n", (unsigned long long)tcfdbfsiz(fdb));
mprint(fdb);
sysprint();
if(!tcfdbclose(fdb)){
eprint(fdb, __LINE__, "tcfdbclose");
err = true;
}
tcfdbdel(fdb);
iprintf("time: %.3f\n", tctime() - stime);
iprintf("%s\n\n", err ? "error" : "ok");
return err ? 1 : 0;
}
/* parse arguments of put command */
static int runput(int argc, char **argv){
char *path = NULL;
char *pkey = NULL;
TCLIST *vals = tcmpoollistnew(tcmpoolglobal());
int omode = 0;
int dmode = 0;
bool sx = false;
for(int i = 2; i < argc; i++){
if(!path && argv[i][0] == '-'){
if(!strcmp(argv[i], "-nl")){
omode |= TDBONOLCK;
} else if(!strcmp(argv[i], "-nb")){
omode |= TDBOLCKNB;
} else if(!strcmp(argv[i], "-dk")){
dmode = -1;
} else if(!strcmp(argv[i], "-dc")){
dmode = 1;
} else if(!strcmp(argv[i], "-dai")){
dmode = 10;
} else if(!strcmp(argv[i], "-dad")){
dmode = 11;
} else if(!strcmp(argv[i], "-sx")){
sx = true;
} else {
usage();
}
} else if(!path){
path = argv[i];
} else if(!pkey){
pkey = argv[i];
} else {
tclistpush2(vals, argv[i]);
}
}
if(!path || !pkey) usage();
TCMAP *cols = tcmapnew();
char *pkbuf;
int pksiz;
if(sx){
pkbuf = tchexdecode(pkey, &pksiz);
for(int i = 0; i < tclistnum(vals) - 1; i += 2){
const char *name = tclistval2(vals, i);
const char *value = tclistval2(vals, i + 1);
int nsiz;
char *nbuf = tchexdecode(name, &nsiz);
int vsiz;
char *vbuf = tchexdecode(value, &vsiz);
tcmapput(cols, nbuf, nsiz, vbuf, vsiz);
tcfree(vbuf);
tcfree(nbuf);
}
} else {
pksiz = strlen(pkey);
pkbuf = tcmemdup(pkey, pksiz);
for(int i = 0; i < tclistnum(vals) - 1; i += 2){
const char *name = tclistval2(vals, i);
const char *value = tclistval2(vals, i + 1);
tcmapput2(cols, name, value);
}
}
int rv = procput(path, pkbuf, pksiz, cols, omode, dmode);
tcmapdel(cols);
tcfree(pkbuf);
return rv;
}
void search_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
char *uri, *json;
double lat, lng, distance, minlat, minlng, maxlat, maxlng, miles, lat2, lng2;
int x1, x2, y1, y2, id, max;
int total;
struct evkeyvalq args;
int ecode, pksiz, i, rsiz;
char pkbuf[256];
char minx[8];
char miny[8];
char maxx[8];
char maxy[8];
const char *rbuf, *name, *buf;
RDBQRY *query;
TCLIST *result;
TCMAP *cols;
Geo_Result *georesultPtr, **georesults;
struct json_object *jsobj, *jsobj2, *jsarr;
if (rdb == NULL) {
evhttp_send_error(req, 503, "database not connected");
return;
}
uri = evhttp_decode_uri(req->uri);
evhttp_parse_query(uri, &args);
free(uri);
argtof(&args, "lat", &lat, 0);
argtof(&args, "lng", &lng, 0);
argtof(&args, "miles", &miles, 0);
argtoi(&args, "max", &max, 1);
geo_box(lat, lng, miles, &minlat, &minlng, &maxlat, &maxlng);
x1 = (minlat * 10000) + 1800000;
y1 = (minlng * 10000) + 1800000;
x2 = (maxlat * 10000) + 1800000;
y2 = (maxlng * 10000) + 1800000;
sprintf(minx, "%d", x1);
sprintf(miny, "%d", y1);
sprintf(maxx, "%d", x2);
sprintf(maxy, "%d", y2);
query = tcrdbqrynew(rdb);
tcrdbqryaddcond(query, "x", RDBQCNUMGT, minx);
tcrdbqryaddcond(query, "x", RDBQCNUMLT, maxx);
tcrdbqryaddcond(query, "y", RDBQCNUMGT, miny);
tcrdbqryaddcond(query, "y", RDBQCNUMLT, maxy);
tcrdbqrysetorder(query, "x", RDBQONUMASC);
cols = tcmapnew();
result = tcrdbqrysearch(query);
total = tclistnum(result);
georesults = malloc(sizeof(Geo_Result *) * total);
for(i = 0; i < total; i++){
rbuf = tclistval(result, i, &rsiz);
cols = tcrdbtblget(rdb, rbuf, rsiz);
if (cols) {
georesultPtr = malloc(sizeof(*georesultPtr));
tcmapiterinit(cols);
buf = tcmapget2(cols, "lat");
lat2 = atof(buf);
georesultPtr->latitude = lat2;
buf = tcmapget2(cols, "lng");
lng2 = atof(buf);
georesultPtr->longitude = lng2;
id = atoi(rbuf);
georesultPtr->id = id;
georesultPtr->data = strdup(tcmapget2(cols, "data"));
distance = geo_distance(lat, lng, lat2, lng2);
georesultPtr->distance = distance;
georesults[i] = georesultPtr;
tcmapdel(cols);
}
}
tclistdel(result);
tcrdbqrydel(query);
qsort(georesults, total, sizeof(Geo_Result *), CmpElem);
jsobj = json_object_new_object();
jsarr = json_object_new_array();
for(i = 0; i < total; i++){
georesultPtr = georesults[i];
if (i < max) {
jsobj2 = json_object_new_object();
json_object_object_add(jsobj2, "id", json_object_new_int(georesultPtr->id));
json_object_object_add(jsobj2, "data", json_object_new_string(georesultPtr->data));
json_object_object_add(jsobj2, "latitude", json_object_new_double(georesultPtr->latitude));
json_object_object_add(jsobj2, "longitude", json_object_new_double(georesultPtr->longitude));
json_object_object_add(jsobj2, "distance", json_object_new_double(georesultPtr->distance));
json_object_array_add(jsarr, jsobj2);
}
free(georesultPtr->data);
free(georesultPtr);
}
free(georesults);
json_object_object_add(jsobj, "total", json_object_new_int(total));
json_object_object_add(jsobj, "results", jsarr);
finalize_json(req, evb, &args, jsobj);
}
sEnvironment *newEnvironment(sList *parameterNames, sList *arguments){
sEnvironment *env = (sEnvironment *)malloc(sizeof(sEnvironment));
env->varmap = tcmapnew();
putVars(parameterNames, arguments, env->varmap);
return env;
}
