int tt_put(char * key,char * zv,int nv,value_t vt)
{
int ecode;
int rt = 0;
int nk = strlen(key);
TCRDB * rdb = tcrdbnew();
int db_port = get_db_port(vt);
printf("db_port -- %d\n",db_port);
if(!tcrdbopen(rdb,LOCALHOST,db_port))
{
ecode = tcrdbecode(rdb);
fprintf(stderr,"tt_put:open %s error -- %s\n",db_port,tcrdberrmsg(ecode));
rt = 1;
}
else
{
printf("db open success!\n");
if(!tcrdbput(rdb,key,nk,zv,nv))
{
ecode = tcrdbecode(rdb);
fprintf(stderr,"(key=%s) tt_put error:%s\n",key,tcrdberrmsg(ecode));
rt = 2;
}
else
{
printf("(key=%s) tt_put success!\n",key);
}
}
tcrdbdel(rdb);
return rt;
}
bool MemCacheServerHandler::UpdatePlatInfo(const string& platID,std::string& dist)
{
if (g_pInst == NULL || !g_pInst->CanUse())
{
return false;
}
time_t time_first = Clock::getUTime();
int dbid = g_pInst->GetServerId(platID);
TCRDB* pConn = g_pInst->GetDB(dbid);
if (pConn == NULL)
{
g_pInst->SetEnbale(false);
LOG4CXX_ERROR(logger,"memcache_put_connect_null_error");
return false;
}
const char* buf = dist.c_str();
int len = dist.length();
if(!tcrdbput(pConn, platID.c_str(), platID.size(), buf, len))
{
int ecode = tcrdbecode(pConn);
g_pInst->SetEnbale(false);
LOG4CXX_ERROR(logger,"memcache_update_put_error"<<tcrdberrmsg(ecode) << platID);
return false;
}
return true;
}
bool QStore::store_thumb(const std::string &thumb_key, const std::string &thumb)
{
assert(need_media_db);
if (!tcrdbput(media_db, thumb_key.c_str(), thumb_key.size(), thumb.c_str(), thumb.size())) {
int ecode = tcrdbecode(media_db);
// TODO log
LOG(ERROR) << "store thumb for thumb key " << thumb_key << " error " << tcrdberrmsg(ecode);
return false;
}
return true;
}
static int
rdb_put(lua_State *L){
TCRDB *rdb = rdb_getrdb(L, 1);
int type = lua_type(L, 2);
if(type==LUA_TSTRING || type==LUA_TNUMBER){
size_t ksiz = 0; size_t vsiz = 0;
const char *key = rdb_getarg(L, 2, &ksiz);
const char *value = rdb_getarg(L, 3, &vsiz);
int width;
// get option & hit db
switch(luaL_optint(L, 4, RDBPUT)){
case RDBPUT:
if(!tcrdbput(rdb, key, ksiz, value, vsiz)) return rdb_error(L, rdb);
break;
case RDBPUTKEEP:
if(!tcrdbputkeep(rdb, key, ksiz, value, vsiz)) return rdb_error(L, rdb);
break;
case RDBPUTCAT:
if(!tcrdbputcat(rdb, key, ksiz, value, vsiz)) return rdb_error(L, rdb);
break;
case RDBPUTSHL:
width = luaL_checknumber(L, 4);
if(!tcrdbputshl(rdb, key, ksiz, value, vsiz, width)) return rdb_error(L, rdb);
break;
case RDBPUTNR:
if(!tcrdbputnr(rdb, key, ksiz, value, vsiz)) return rdb_error(L, rdb);
break;
}
} else if(type==LUA_TTABLE){
TCLIST *list = hash2tclist(L, 2);
int opts = luaL_optint(L, 3, 0);
TCLIST *useless = tcrdbmisc(rdb, "putlist", opts, list);
tclistdel(list);
tclistdel(useless);
} else return luaL_argerror(L, 2, "must be table, string, or number");
lua_pushboolean(L, 1);
return 1;
}
int QStore::process(QContentRecord &record)
{
if (record.want_type == mimetype::image) {
assert(need_media_db);
const std::string &url_md5 = record.url_md5;
std::string media_key = url_md5 + ".i";
if (!tcrdbput(media_db, media_key.c_str(), media_key.size(), record.raw_content.c_str(), record.raw_content.size())) {
int ecode = tcrdbecode(media_db);
// TODO log
LOG(ERROR) << "put media url " << record.url << " error " << tcrdberrmsg(ecode);
return -1;
}
} else { // default text/html
assert(need_html_db && need_record_db);
std::string tmps;
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", record.url.c_str());
tcmapput2(cols, "host", record.host.c_str());
tcmapput2(cols, "url_md5", record.url_md5.c_str());
tcmapput2(cols, "parent_url_md5", record.parent_url_md5.c_str());
strtk::type_to_string(record.crawl_level, tmps);
tcmapput2(cols, "crawl_level", tmps.c_str());
strtk::type_to_string(record.find_time, tmps);
tcmapput2(cols, "find_time", tmps.c_str());
tcmapput2(cols, "anchor_text", record.anchor_text.c_str());
tcmapput2(cols, "crawl_tag", record.crawl_tag.c_str());
// last download_time
strtk::type_to_string(record.download_time, tmps);
tcmapput2(cols, "download_time", tmps.c_str());
strtk::type_to_string(record.http_code, tmps);
tcmapput2(cols, "http_code", tmps.c_str());
if (record.is_list) {
tcmapput2(cols, "is_list", "1");
} else {
tcmapput2(cols, "is_list", "0");
}
if (record.crawled_okay) {
tcmapput2(cols, "crawled_okay", "1");
tcmapput2(cols, "raw_title", record.raw_title.c_str());
tcmapput2(cols, "title", record.title.c_str());
tcmapput2(cols, "keywords", record.keywords.c_str());
tcmapput2(cols, "description", record.description.c_str());
tcmapput2(cols, "content", record.content.c_str());
strtk::type_to_string(record.publish_time, tmps);
tcmapput2(cols, "publish_time", tmps.c_str());
tcmapput2(cols, "images", record.images.c_str());
if (record.is_redirect) {
tcmapput2(cols, "is_redirect", "1");
} else {
tcmapput2(cols, "is_redirect", "0");
}
tcmapput2(cols, "redirect_url", record.redirect_url.c_str());
strtk::type_to_string(record.content_confidence, tmps);
tcmapput2(cols, "content_confidence", tmps.c_str());
strtk::type_to_string(record.list_confidence, tmps);
tcmapput2(cols, "list_confidence", tmps.c_str());
strtk::type_to_string(record.links_size, tmps);
tcmapput2(cols, "links_size", tmps.c_str());
strtk::type_to_string(record.last_modified, tmps);
tcmapput2(cols, "last_modified", tmps.c_str());
strtk::type_to_string(record.loading_time, tmps);
tcmapput2(cols, "loading_time", tmps.c_str());
strtk::type_to_string(record.new_links_size, tmps);
tcmapput2(cols, "new_links_size", tmps.c_str());
} else {
tcmapput2(cols, "crawled_okay", "0");
}
if(!tcrdbtblput(record_db, record.url_md5.c_str(), record.url_md5.size(), cols)){
int ecode = tcrdbecode(record_db);
LOG(ERROR) << "put record error " << tcrdberrmsg(ecode);
tcmapdel(cols);
return -1;
}
tcmapdel(cols);
std::string url_md5_vdom = record.url_md5 + ".v";
memcached_return_t rc = memcached_set(html_memc, url_md5_vdom.c_str(), url_md5_vdom.size(),
record.vdom.c_str(), record.vdom.size(), 0, 0);
if (rc != MEMCACHED_SUCCESS) {
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size()
<< " error: " << memcached_strerror(html_memc, rc);
return -1;
}
/*
if (!tcrdbput(html_db, url_md5_vdom.c_str(), url_md5_vdom.size(), record.vdom.c_str(), record.vdom.size())) {
int ecode = tcrdbecode(html_db);
// TODO log
LOG(ERROR) << "put vdom url " << record.url << " size: " << record.vdom.size() << " error " << tcrdberrmsg(ecode);
return -1;
}
*/
}
return 0;
}
int pack_file(char *file, TCXSTR *root_key, bool resume) {
struct stat st;
int ecode;
TCXSTR *key;
key = tcxstrdup(root_key);
size_t file_path_len;
file_path_len = strlen(file);
char *file_name;
file_name = get_file_name(file, file_path_len);
tcxstrcat2(key, file_name);
if (resume && key_exists(tcxstrptr(key))) {
fprintf(stdout, "already exists");
return;
}
if (-1 == stat(file, &st)) {
return 1;
}
if (!S_ISREG(st.st_mode)) {
fprintf(stdout, "Not regular file: %s", file);
return 2;
}
int fd;
if (-1 == (fd = open(file, O_RDONLY))) {
fprintf(stdout, "***Failed open file %s", file);
return 1;
}
void *fmap;
if (MAP_FAILED == (fmap = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0))) {
fprintf(stdout, "mmaping failed for %s", file);
close(fd);
return -1;
}
/* store records */
if (use_cabinet_lib) {
if (!tchdbput(hdb, tcxstrptr(key), tcxstrsize(key), fmap, st.st_size)) {
ecode = tchdbecode(hdb);
fprintf(stdout, "put error: %s", tchdberrmsg(ecode));
}
} else {
if (!tcrdbput(tdb, tcxstrptr(key), tcxstrsize(key), fmap, st.st_size)) {
ecode = tcrdbecode(tdb);
fprintf(stdout, "put error: %s", tcrdberrmsg(ecode));
}
}
fprintf(stdout, "%d bytes", st.st_size);
munmap(fmap, st.st_size);
close(fd);
tcxstrdel(key);
}
bool db_obj_remote::dbput(const void *kbuf, int ksiz,
const void *vbuf, int vsiz)
{
return tcrdbput(_hdb,kbuf,ksiz,vbuf,vsiz);
}
