static VALUE cDB_put_method(VALUE vself, VALUE vkey, VALUE vstr, int method){
int ecode;
bool res;
TCRDB *db;
Data_Get_Struct(rb_iv_get(vself, RDBVNDATA), TCRDB, db);
vkey = StringValueEx(vkey);
vstr = StringValueEx(vstr);
switch(method){
case TTPUT:
res = tcrdbput2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr));
break;
case TTPUTKEEP:
res = tcrdbputkeep2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr));
break;
case TTPUTCAT:
res = tcrdbputcat2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr));
break;
case TTPUTNR:
res = tcrdbputnr2(db, RSTRING_PTR(vkey), RSTRING_PTR(vstr));
break;
default:
res = false;
break;
}
if(!res){
ecode = tcrdbecode(db);
rb_raise(eTokyoTyrantError, "put error: %s", tcrdberrmsg(ecode));
}
return Qtrue;
}
bool QCrawlerDB::storeRecord(QCrawlerRecord &rec) {
QString url = rec.crawl_url().url();
QString host = rec.crawl_url().host();
QString url_md5 = md5_hash(url);
QString parent_url_md5 = rec.crawl_url().parent_url_md5();
int crawl_level = rec.crawl_url().crawl_level();
QString anchor_text = rec.crawl_url().anchor_text();
QString raw_html = rec.raw_html();
// QString raw_title = rec.raw_title();
// QString raw_content = rec.raw_content();
QString raw_content_md5 = md5_hash(rec.raw_content());
int links_size = rec.raw_sub_links().size();
int download_time = rec.download_time();
int last_modified = rec.last_modified();
int loading_time = rec.loading_time();
TCMAP *cols = tcmapnew();
tcmapput2(cols, "url", url.toUtf8().constData());
tcmapput2(cols, "host", host.toUtf8().constData());
tcmapput2(cols, "url_md5", url_md5.toUtf8().constData());
tcmapput2(cols, "parent_url_md5", parent_url_md5.toUtf8().constData());
tcmapput2(cols, "crawl_level", QByteArray::number(crawl_level).constData());
tcmapput2(cols, "anchor_text", anchor_text.toUtf8().constData());
// tcmapput2(cols, "raw_html", raw_html.toUtf8().constData());
// tcmapput2(cols, "raw_title", raw_title.toUtf8().constData());
// tcmapput2(cols, "raw_content", rec.raw_content().toUtf8().constData());
// tcmapput2(cols, "raw_content_md5", md5_hash(rec.raw_content()).toUtf8().constData());
tcmapput2(cols, "title", rec.title().toUtf8().constData());
tcmapput2(cols, "content", rec.content().toUtf8().constData());
tcmapput2(cols, "links_size", QByteArray::number(links_size).constData());
tcmapput2(cols, "download_time", QByteArray::number(download_time).constData());
tcmapput2(cols, "last_modified", QByteArray::number(last_modified).constData());
tcmapput2(cols, "loading_time", QByteArray::number(loading_time).constData());
bool status = true;
if(!tcrdbtblput(record_db, url_md5.toUtf8().constData(), url_md5.toUtf8().size(), cols)){
int ecode = tcrdbecode(record_db);
fprintf(stderr, "store record put error: %s\n", tcrdberrmsg(ecode));
status = false;
}
tcmapdel(cols);
// raw_html store
if (!tcrdbput2(html_record_db, url_md5.toUtf8().constData(), raw_html.toUtf8().constData())) {
int ecode = tcrdbecode(url_hash_db);
fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode));
return false;
}
return status;
}
bool QCrawlerDB::updateUrlStatus(const QString &url, int status) {
char status_str[40];
sprintf(status_str, "%d", status);
if(!tcrdbput2(url_hash_db, url.toUtf8().constData(), status_str)) {
int ecode = tcrdbecode(url_hash_db);
fprintf(stderr, "update url status put error: %s\n", tcrdberrmsg(ecode));
return false;
}
return true;
}
void TTClient::set(string key, string value){
if(!tcrdbput2(tcrdbPtr_, key.c_str(), value.c_str())){
}
}