char *iod_get(char *key,int *nv) { int ecode; int nk = strlen(key); char *value; TCRDB * rdb = tcrdbnew(); int db_port = get_db_port(IO_DATA); printf("db_port -- %d\n",db_port); if(!tcrdbopen(rdb,LOCALHOST,db_port)) { ecode = tcrdbecode(rdb); fprintf(stderr,"iod_get:open %s error -- %s\n",db_port,tcrdberrmsg(ecode)); return NULL; }else{ printf("db open success!\n"); if((value = tcrdbget(rdb,key,nk,nv)) == NULL) { ecode = tcrdbecode(rdb); fprintf(stderr,"(key=%s) tt_get error:%s\n",key,tcrdberrmsg(ecode)); return NULL; }else{ printf("(key=%s) tt_get success!\n",key); printf("%d bytes read\n",*nv); } } tcrdbdel(rdb); return value; }
QStore::~QStore() { if (need_url_seen_db) { memcached_free(url_seen_memc); } if (need_html_db) { memcached_free(html_memc); } if (need_record_db) { if(!tcrdbclose(record_db)){ int ecode = tcrdbecode(record_db); fprintf(stderr, "close record db error: %s\n", tcrdberrmsg(ecode)); } tcrdbdel(record_db); } if (need_media_db) { if(!tcrdbclose(media_db)){ int ecode = tcrdbecode(media_db); fprintf(stderr, "close media db error: %s\n", tcrdberrmsg(ecode)); } tcrdbdel(media_db); } }
int tt_put(char * key,char * zv,int nv,value_t vt) { int ecode; int rt = 0; int nk = strlen(key); TCRDB * rdb = tcrdbnew(); int db_port = get_db_port(vt); printf("db_port -- %d\n",db_port); if(!tcrdbopen(rdb,LOCALHOST,db_port)) { ecode = tcrdbecode(rdb); fprintf(stderr,"tt_put:open %s error -- %s\n",db_port,tcrdberrmsg(ecode)); rt = 1; } else { printf("db open success!\n"); if(!tcrdbput(rdb,key,nk,zv,nv)) { ecode = tcrdbecode(rdb); fprintf(stderr,"(key=%s) tt_put error:%s\n",key,tcrdberrmsg(ecode)); rt = 2; } else { printf("(key=%s) tt_put success!\n",key); } } tcrdbdel(rdb); return rt; }
int open_db(char *addr, int port, TCRDB **rdb) { int ecode=0; if (*rdb != NULL) { if(!tcrdbclose(*rdb)){ ecode = tcrdbecode(*rdb); fprintf(stderr, "close error: %s\n", tcrdberrmsg(ecode)); } tcrdbdel(*rdb); *rdb = NULL; } *rdb = tcrdbnew(); if(!tcrdbopen(*rdb, addr, port)){ ecode = tcrdbecode(*rdb); fprintf(stderr, "open error(%s:%d): %s\n", addr, port, tcrdberrmsg(ecode)); *rdb = NULL; } else { char *status = tcrdbstat(*rdb); printf("adding indices\n---------------------\n"); tcrdbtblsetindex(*rdb, "x", RDBITDECIMAL); tcrdbtblsetindex(*rdb, "y", RDBITDECIMAL); printf("%s---------------------\n", status); if (status) free(status); } return ecode; }
bool QCrawlerDB::storeRecord(QCrawlerRecord &rec) { QString url = rec.crawl_url().url(); QString host = rec.crawl_url().host(); QString url_md5 = md5_hash(url); QString parent_url_md5 = rec.crawl_url().parent_url_md5(); int crawl_level = rec.crawl_url().crawl_level(); QString anchor_text = rec.crawl_url().anchor_text(); QString raw_html = rec.raw_html(); // QString raw_title = rec.raw_title(); // QString raw_content = rec.raw_content(); QString raw_content_md5 = md5_hash(rec.raw_content()); int links_size = rec.raw_sub_links().size(); int download_time = rec.download_time(); int last_modified = rec.last_modified(); int loading_time = rec.loading_time(); TCMAP *cols = tcmapnew(); tcmapput2(cols, "url", url.toUtf8().constData()); tcmapput2(cols, "host", host.toUtf8().constData()); tcmapput2(cols, "url_md5", url_md5.toUtf8().constData()); tcmapput2(cols, "parent_url_md5", parent_url_md5.toUtf8().constData()); tcmapput2(cols, "crawl_level", QByteArray::number(crawl_level).constData()); tcmapput2(cols, "anchor_text", anchor_text.toUtf8().constData()); // tcmapput2(cols, "raw_html", raw_html.toUtf8().constData()); // tcmapput2(cols, "raw_title", raw_title.toUtf8().constData()); // tcmapput2(cols, "raw_content", rec.raw_content().toUtf8().constData()); // tcmapput2(cols, "raw_content_md5", md5_hash(rec.raw_content()).toUtf8().constData()); tcmapput2(cols, "title", rec.title().toUtf8().constData()); tcmapput2(cols, "content", rec.content().toUtf8().constData()); tcmapput2(cols, "links_size", QByteArray::number(links_size).constData()); tcmapput2(cols, "download_time", QByteArray::number(download_time).constData()); tcmapput2(cols, "last_modified", QByteArray::number(last_modified).constData()); tcmapput2(cols, "loading_time", QByteArray::number(loading_time).constData()); bool status = true; if(!tcrdbtblput(record_db, url_md5.toUtf8().constData(), url_md5.toUtf8().size(), cols)){ int ecode = tcrdbecode(record_db); fprintf(stderr, "store record put error: %s\n", tcrdberrmsg(ecode)); status = false; } tcmapdel(cols); // raw_html store if (!tcrdbput2(html_record_db, url_md5.toUtf8().constData(), raw_html.toUtf8().constData())) { int ecode = tcrdbecode(url_hash_db); fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode)); return false; } return status; }
void del_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { char *uri, *json, *id; struct evkeyvalq args; struct json_object *jsobj; if (rdb == NULL) { evhttp_send_error(req, 503, "database not connected"); return; } uri = evhttp_decode_uri(req->uri); evhttp_parse_query(uri, &args); free(uri); id = (char *)evhttp_find_header(&args, "id"); if (id == NULL) { evhttp_send_error(req, 400, "id is required"); evhttp_clear_headers(&args); return; } jsobj = json_object_new_object(); if (tcrdbtblout(rdb, id, sizeof(id))) { json_object_object_add(jsobj, "status", json_object_new_string("ok")); } else { db_status = tcrdbecode(rdb); db_error_to_json(db_status, jsobj); } finalize_json(req, evb, &args, jsobj); }
extern void mTokyoMessenger_exception(VALUE vself){ int ecode; TCRDB *db = mTokyoMessenger_getdb(vself); ecode = tcrdbecode(db); rb_raise(eTokyoMessengerError, tcrdberrmsg(ecode)); }
static VALUE cDB_put_method(VALUE vself, VALUE vkey, VALUE vstr, int method){ int ecode; bool res; TCRDB *db; Data_Get_Struct(rb_iv_get(vself, RDBVNDATA), TCRDB, db); vkey = StringValueEx(vkey); vstr = StringValueEx(vstr); switch(method){ case TTPUT: res = tcrdbput2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr)); break; case TTPUTKEEP: res = tcrdbputkeep2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr)); break; case TTPUTCAT: res = tcrdbputcat2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr)); break; case TTPUTNR: res = tcrdbputnr2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr)); break; default: res = false; break; } if(!res){ ecode = tcrdbecode(db); rb_raise(eTokyoTyrantError, "put error: %s", tcrdberrmsg(ecode)); } return Qtrue; }
/* RTDBIterItemsType.tp_iternext */ static PyObject * RTDBIterItems_tp_iternext(DBIter *self) { RDBBase *rdbbase = (RDBBase *)self->db; void *key; int key_size; TCMAP *value; PyObject *pykey, *pyvalue, *pyresult = NULL; if (rdbbase->changed) { return set_error(Error, "DB changed during iteration"); } Py_BEGIN_ALLOW_THREADS key = tcrdbiternext(rdbbase->rdb, &key_size); if (key) { value = tcrdbtblget(rdbbase->rdb, key, key_size); } Py_END_ALLOW_THREADS if (!key) { if (tcrdbecode(rdbbase->rdb) == TTENOREC) { return set_stopiteration_error(); } return set_rdb_error(rdbbase->rdb, NULL); } pykey = void_to_bytes(key, key_size); pyvalue = tcmap_to_dict(value); if (pykey && pyvalue) { pyresult = PyTuple_Pack(2, pykey, pyvalue); } Py_XDECREF(pykey); Py_XDECREF(pyvalue); tcfree(key); tcmapdel(value); return pyresult; }
string TTClient::get(string key){ LOG_DEBUG("TTClient::get => key=: "<<key); char* value = NULL; string ret; value = tcrdbget2(tcrdbPtr_, key.c_str()); if(value){ LOG_DEBUG("TTClient::tcrdbget2 ==> key: "<<key<<" return value : ["<<value<<"]"); ret = value; delete value; } else { LOG_DEBUG("TTClient::tcrdbget2 ==> key: "<<key<<" return null value "); int code = tcrdbecode(tcrdbPtr_); if(code != TTESUCCESS){ //网络异常,重试 LOG_ERROR("TTClient::tcrdbecode => key: "<<key<<" failure : recv error code: "<<code); reOpen(); ret = reGetFromTT(key); //重新获取 } else //从TT返回成功,不做任何处理 { LOG_DEBUG("TTClient::tcrdbecode ==> key: "<<key<<"return code == "<<code); } } return ret; }
bool QCrawlerDB::getUrlStatus(QString url, QCrawlerUrl::Status *url_status) { bool ret = false; int status = 0; char *value; value = tcrdbget2(url_hash_db, url.toUtf8().constData()); if(value){ status = atoi(value); if (status < -5) { status = -5; } *url_status = (QCrawlerUrl::Status)status; ret = true; free(value); } else { int ecode = tcrdbecode(url_hash_db); if (ecode == 7) { *url_status = QCrawlerUrl::NOT_EXIST; ret = true; } else { fprintf(stderr, "get url status error: %s", tcrdberrmsg(ecode)); ret = false; } } return ret; }
bool MemCacheServerHandler::UpdatePlatInfo(const string& platID,std::string& dist) { if (g_pInst == NULL || !g_pInst->CanUse()) { return false; } time_t time_first = Clock::getUTime(); int dbid = g_pInst->GetServerId(platID); TCRDB* pConn = g_pInst->GetDB(dbid); if (pConn == NULL) { g_pInst->SetEnbale(false); LOG4CXX_ERROR(logger,"memcache_put_connect_null_error"); return false; } const char* buf = dist.c_str(); int len = dist.length(); if(!tcrdbput(pConn, platID.c_str(), platID.size(), buf, len)) { int ecode = tcrdbecode(pConn); g_pInst->SetEnbale(false); LOG4CXX_ERROR(logger,"memcache_update_put_error"<<tcrdberrmsg(ecode) << platID); return false; } return true; }
/* RTDBIterValuesValsType.tp_iternext */ static PyObject * RTDBIterValuesVals_tp_iternext(DBIter *self) { RDBBase *rdbbase = (RDBBase *)self->db; void *key; int key_size; TCMAP *value; TCLIST *valuevals; PyObject *pyvaluevals; if (rdbbase->changed) { return set_error(Error, "DB changed during iteration"); } Py_BEGIN_ALLOW_THREADS key = tcrdbiternext(rdbbase->rdb, &key_size); if (key) { value = tcrdbtblget(rdbbase->rdb, key, key_size); } Py_END_ALLOW_THREADS if (!key) { if (tcrdbecode(rdbbase->rdb) == TTENOREC) { return set_stopiteration_error(); } return set_rdb_error(rdbbase->rdb, NULL); } valuevals = tcmapvals(value); pyvaluevals = tclist_to_tuple(valuevals); tcmapdel(value); tclistdel(valuevals); return pyvaluevals; }
static VALUE mTokyoMessenger_close(VALUE vself){ int ecode; TCRDB *db = mTokyoMessenger_getdb(vself); if(!tcrdbclose(db)){ ecode = tcrdbecode(db); rb_raise(eTokyoMessengerError, "close error: %s", tcrdberrmsg(ecode)); } return Qtrue; }
void put_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx) { char *uri, *id, *data, *json, *key, *value; double lat, lng; int x, y; char buf[16]; struct evkeyvalq args; struct json_object *jsobj; TCMAP *cols; if (rdb == NULL) { evhttp_send_error(req, 503, "database not connected"); return; } uri = evhttp_decode_uri(req->uri); evhttp_parse_query(uri, &args); free(uri); argtof(&args, "lat", &lat, 0); argtof(&args, "lng", &lng, 0); id = (char *)evhttp_find_header(&args, "id"); data = (char *)evhttp_find_header(&args, "data"); if (id == NULL) { evhttp_send_error(req, 400, "id is required"); evhttp_clear_headers(&args); return; } x = (lat * 10000) + 1800000; y = (lng * 10000) + 1800000; cols = tcmapnew(); tcmapput2(cols, "data", data); sprintf(buf, "%d", x); tcmapput2(cols, "x", buf); sprintf(buf, "%d", y); tcmapput2(cols, "y", buf); sprintf(buf, "%f", lat); tcmapput2(cols, "lat", buf); sprintf(buf, "%f", lng); tcmapput2(cols, "lng", buf); jsobj = json_object_new_object(); if (tcrdbtblput(rdb, id, strlen(id), cols)) { json_object_object_add(jsobj, "status", json_object_new_string("ok")); } else { db_status = tcrdbecode(rdb); db_error_to_json(db_status, jsobj); } tcmapdel(cols); finalize_json(req, evb, &args, jsobj); }
static VALUE mTokyoMessenger_errmsg(int argc, VALUE *argv, VALUE vself){ VALUE vecode; const char *msg; int ecode; TCRDB *db = mTokyoMessenger_getdb(vself); rb_scan_args(argc, argv, "01", &vecode); ecode = (vecode == Qnil) ? tcrdbecode(db) : NUM2INT(vecode); msg = tcrdberrmsg(ecode); return rb_str_new2(msg); }
bool QCrawlerDB::updateUrlStatus(const QString &url, int status) { char status_str[40]; sprintf(status_str, "%d", status); if(!tcrdbput2(url_hash_db, url.toUtf8().constData(), status_str)) { int ecode = tcrdbecode(url_hash_db); fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode)); return false; } return true; }
bool QStore::store_thumb(const std::string &thumb_key, const std::string &thumb) { assert(need_media_db); if (!tcrdbput(media_db, thumb_key.c_str(), thumb_key.size(), thumb.c_str(), thumb.size())) { int ecode = tcrdbecode(media_db); // TODO log LOG(ERROR) << "store thumb for thumb key " << thumb_key << " error " << tcrdberrmsg(ecode); return false; } return true; }
void TTClient::open(){ for(size_t i = 0; i < serverList_.size(); i++){ if(tcrdbopen(tcrdbPtr_, serverList_[i].first.c_str(), serverList_[i].second)){ LOG_INFO("TTClient::open => success : server = " << serverList_[i].first << ":" << serverList_[i].second); break; } else { int code = tcrdbecode(tcrdbPtr_); LOG_ERROR("TTClient::open => failure : " << serverList_[i].first << ":" << serverList_[i].second << " code = " << code << " msg = " << tcrdberrmsg(code)); } } }
int tt_get(char * key,char * zv,value_t vt) { int ecode; int nv; int rt = 0; int nk = strlen(key); char * value; TCRDB * rdb = tcrdbnew(); int db_port = get_db_port(vt); printf("db_port -- %d\n",db_port); if(!tcrdbopen(rdb,LOCALHOST,db_port)) { ecode = tcrdbecode(rdb); fprintf(stderr,"tt_get:open %s error -- %s\n",db_port,tcrdberrmsg(ecode)); rt = 1; } else { printf("db open success!\n"); if((value = tcrdbget(rdb,key,nk,&nv)) == NULL) { ecode = tcrdbecode(rdb); fprintf(stderr,"(key=%s) tt_get error:%s\n",key,tcrdberrmsg(ecode)); rt = 2; } else { printf("(key=%s) tt_get success!\n",key); printf("%d bytes read\n",nv); memcpy(zv,value,nv); free(value); } } tcrdbdel(rdb); return rt; }
static VALUE cDB_putshl(VALUE vself, VALUE vkey, VALUE vstr, VALUE vwidth){ int ecode; TCRDB *db; Data_Get_Struct(rb_iv_get(vself, RDBVNDATA), TCRDB, db); vkey = StringValueEx(vkey); vstr = StringValueEx(vstr); if(!tcrdbputshl2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr), FIXNUM_P(vwidth))){ ecode = tcrdbecode(db); rb_raise(eTokyoTyrantError, "put error: %s", tcrdberrmsg(ecode)); } return Qtrue; }
int main(int argc, char **argv) { int i, errCode; double magic, rlat, s, c; int lat; g_progname = argv[0]; for (i=1; i < argc; i++) { if(!strcmp(argv[i], "-tchost")) { if(++i >= argc) usage(); db_host = argv[i]; } else if(!strcmp(argv[i], "-tcport")) { if(++i >= argc) usage(); db_port = tcatoi(argv[i]); } else if (!strcmp(argv[i], "-help")) { usage(); } } pi = atan(1.0)*4; magic = cos(pi/180.0); for (lat = 0; lat < 181; ++lat) { rlat = lat*pi/180; s = sin(rlat); c = cos(rlat); longDistance[lat] = radius*acos((s*s)+(magic*c*c)); } memset(&db_status, -1, sizeof(db_status)); simplehttp_init(); db_reconnect(0, 0, NULL); simplehttp_set_cb("/search*", search_cb, NULL); simplehttp_set_cb("/put*", put_cb, NULL); simplehttp_set_cb("/get*", get_cb, NULL); simplehttp_set_cb("/del*", del_cb, NULL); simplehttp_set_cb("/distance*", distance_cb, NULL); simplehttp_set_cb("/box*", box_cb, NULL); simplehttp_main(argc, argv); if (!tcrdbclose(rdb)) { errCode = tcrdbecode(rdb); fprintf(stderr, "close error: %s\n", tcrdberrmsg(errCode)); } tcrdbdel(rdb); return 0; }
bool MemCacheServerHandler::GetPlatInfo(const string& platID, std::string& dist) { if (g_pInst == NULL || !g_pInst->CanUse()) { return false; } // if (uid != 2928562105553271) // { // return false; // } int dbid = g_pInst->GetServerId(platID); TCRDB* pConn = g_pInst->GetDB(dbid); if (pConn == NULL) { g_pInst->SetEnbale(false); LOG4CXX_ERROR(logger,"memcache_get_connect_null_error"); return false; } int len = 0; int klen = platID.length(); char* buffer = (char*)tcrdbget(pConn, platID.c_str(), klen, &len); if (buffer == NULL) { int ecode = tcrdbecode(pConn); if (ecode != TTENOREC) { g_pInst->SetEnbale(false); LOG4CXX_ERROR(logger,"memcache_get_ecode_error"); } return false; } std::string outbuf(buffer, len); dist = outbuf; free(buffer); //g_pInst->GetCounter().increase("mem_get_user"); return true; }
//从TT重新获取一次secketkey string TTClient::reGetFromTT(const string& key) { string strTT; char* value = tcrdbget2(tcrdbPtr_, key.c_str()); if(value){ strTT = value; delete value; LOG_DEBUG("TTClient::reGetFromTT ==> key: "<<key<<" return code : ["<<strTT<<"]"); } else { int code = tcrdbecode(tcrdbPtr_); //获取code码 if(code != TTESUCCESS){ LOG_ERROR("TTClient::reGetFromTT => recv error code : "<<code); } else //返回成功,不做任何处理 { LOG_DEBUG("TTClient::reGetFromTT ==> key: "<<key<<" return code == "<<code); } } return strTT; }
static VALUE mTokyoMessenger_connect(VALUE vself){ VALUE host, port, timeout, retry, server; int ecode; TCRDB *db = mTokyoMessenger_getdb(vself); host = rb_iv_get(vself, "@host"); port = rb_iv_get(vself, "@port"); timeout = rb_iv_get(vself, "@timeout"); retry = rb_iv_get(vself, "@retry"); if((!tcrdbtune(db, NUM2DBL(timeout), retry == Qtrue ? RDBTRECON : 0)) || (!tcrdbopen(db, RSTRING_PTR(host), FIX2INT(port)))){ ecode = tcrdbecode(db); rb_raise(eTokyoMessengerError, "open error: %s", tcrdberrmsg(ecode)); } server = rb_str_new2(tcrdbexpr(db)); rb_iv_set(vself, "@server", server); return Qtrue; }
static void printerr(void) { int ecode = tcrdbecode(rdb); ERROR("tokyotyrant plugin: error: %d, %s", ecode, tcrdberrmsg(ecode)); }
int xtcrdb_ecode(void* rdb) { return tcrdbecode(rdb); }
static int rdb_error(lua_State *L, TCRDB *rdb){ int ecode = tcrdbecode(rdb); return bool_error(L, ecode, tcrdberrmsg(ecode)); }
void QStore::init(bool need_url_seen_db, const std::string &url_seen_server, bool need_html_db, const std::string &html_db_server, bool need_record_db, const std::string &record_db_server, bool need_media_db, const std::string &media_db_server) { if (need_url_seen_db) { url_seen_memc = memcached_create(NULL); url_seen_servers = memcached_servers_parse(url_seen_server.c_str()); if (url_seen_servers != NULL) { memcached_return_t rc = memcached_server_push(url_seen_memc, url_seen_servers); memcached_server_list_free(url_seen_servers); if (rc != MEMCACHED_SUCCESS) { LOG(FATAL) << "open url seen db " << url_seen_server << " fail " << memcached_strerror(url_seen_memc, rc); } } else { LOG(FATAL) << "url seen servers server config error " << url_seen_server; } } /* if (need_html_db) { get_host_port_from_server(html_db_server, 9860, html_db_host, html_db_port); html_db = tcrdbnew(); if(!tcrdbopen(html_db, html_db_host.c_str(), html_db_port)){ int ecode = tcrdbecode(html_db); LOG(FATAL) << "open html db " << html_db_server << " fail " << tcrdberrmsg(ecode); } else { LOG(INFO) << "open html db " << html_db_server << " okay"; } } */ if (need_html_db) { html_memc = memcached_create(NULL); html_servers = memcached_servers_parse(html_db_server.c_str()); if (html_servers != NULL) { memcached_return_t rc = memcached_server_push(html_memc, html_servers); memcached_server_list_free(html_servers); if (rc != MEMCACHED_SUCCESS) { LOG(FATAL) << "open html db " << html_db_server << " fail " << memcached_strerror(html_memc, rc); } } else { LOG(FATAL) << "html servers server config error " << html_db_server; } } if (need_record_db) { get_host_port_from_server(record_db_server, 9870, record_db_host, record_db_port); record_db = tcrdbnew(); if(!tcrdbopen(record_db, record_db_host.c_str(), record_db_port)){ int ecode = tcrdbecode(record_db); LOG(FATAL) << "open record db " << record_db_server << " fail " << tcrdberrmsg(ecode); } else { LOG(INFO) << "open record db " << record_db_server << " okay"; } } if (need_media_db) { get_host_port_from_server(media_db_server, 9880, media_db_host, media_db_port); media_db = tcrdbnew(); if(!tcrdbopen(media_db, media_db_host.c_str(), media_db_port)){ int ecode = tcrdbecode(media_db); LOG(FATAL) << "open media db " << media_db_server << " fail " << tcrdberrmsg(ecode); } else { LOG(INFO) << "open media db " << media_db_server << " okay"; } } }
int QStore::process(QContentRecord &record) { if (record.want_type == mimetype::image) { assert(need_media_db); const std::string &url_md5 = record.url_md5; std::string media_key = url_md5 + ".i"; if (!tcrdbput(media_db, media_key.c_str(), media_key.size(), record.raw_content.c_str(), record.raw_content.size())) { int ecode = tcrdbecode(media_db); // TODO log LOG(ERROR) << "put media url " << record.url << " error " << tcrdberrmsg(ecode); return -1; } } else { // default text/html assert(need_html_db && need_record_db); std::string tmps; TCMAP *cols = tcmapnew(); tcmapput2(cols, "url", record.url.c_str()); tcmapput2(cols, "host", record.host.c_str()); tcmapput2(cols, "url_md5", record.url_md5.c_str()); tcmapput2(cols, "parent_url_md5", record.parent_url_md5.c_str()); strtk::type_to_string(record.crawl_level, tmps); tcmapput2(cols, "crawl_level", tmps.c_str()); strtk::type_to_string(record.find_time, tmps); tcmapput2(cols, "find_time", tmps.c_str()); tcmapput2(cols, "anchor_text", record.anchor_text.c_str()); tcmapput2(cols, "crawl_tag", record.crawl_tag.c_str()); // last download_time strtk::type_to_string(record.download_time, tmps); tcmapput2(cols, "download_time", tmps.c_str()); strtk::type_to_string(record.http_code, tmps); tcmapput2(cols, "http_code", tmps.c_str()); if (record.is_list) { tcmapput2(cols, "is_list", "1"); } else { tcmapput2(cols, "is_list", "0"); } if (record.crawled_okay) { tcmapput2(cols, "crawled_okay", "1"); tcmapput2(cols, "raw_title", record.raw_title.c_str()); tcmapput2(cols, "title", record.title.c_str()); tcmapput2(cols, "keywords", record.keywords.c_str()); tcmapput2(cols, "description", record.description.c_str()); tcmapput2(cols, "content", record.content.c_str()); strtk::type_to_string(record.publish_time, tmps); tcmapput2(cols, "publish_time", tmps.c_str()); tcmapput2(cols, "images", record.images.c_str()); if (record.is_redirect) { tcmapput2(cols, "is_redirect", "1"); } else { tcmapput2(cols, "is_redirect", "0"); } tcmapput2(cols, "redirect_url", record.redirect_url.c_str()); strtk::type_to_string(record.content_confidence, tmps); tcmapput2(cols, "content_confidence", tmps.c_str()); strtk::type_to_string(record.list_confidence, tmps); tcmapput2(cols, "list_confidence", tmps.c_str()); strtk::type_to_string(record.links_size, tmps); tcmapput2(cols, "links_size", tmps.c_str()); strtk::type_to_string(record.last_modified, tmps); tcmapput2(cols, "last_modified", tmps.c_str()); strtk::type_to_string(record.loading_time, tmps); tcmapput2(cols, "loading_time", tmps.c_str()); strtk::type_to_string(record.new_links_size, tmps); tcmapput2(cols, "new_links_size", tmps.c_str()); } else { tcmapput2(cols, "crawled_okay", "0"); } if(!tcrdbtblput(record_db, record.url_md5.c_str(), record.url_md5.size(), cols)){ int ecode = tcrdbecode(record_db); LOG(ERROR) << "put record error " << tcrdberrmsg(ecode); tcmapdel(cols); return -1; } tcmapdel(cols); std::string url_md5_vdom = record.url_md5 + ".v"; memcached_return_t rc = memcached_set(html_memc, url_md5_vdom.c_str(), url_md5_vdom.size(), record.vdom.c_str(), record.vdom.size(), 0, 0); if (rc != MEMCACHED_SUCCESS) { LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size() << " error: " << memcached_strerror(html_memc, rc); return -1; } /* if (!tcrdbput(html_db, url_md5_vdom.c_str(), url_md5_vdom.size(), record.vdom.c_str(), record.vdom.size())) { int ecode = tcrdbecode(html_db); // TODO log LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size() << " error " << tcrdberrmsg(ecode); return -1; } */ } return 0; }