int Grab::setSendAmount(const uint32_t amount)
{
redisContext *c;
c=redisConnPool::getInstance()->getConnection();
if(c==NULL){
UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
redisReply *reply;
UB_LOG_DEBUG( "HSET GRAB:%d send_amount %d", _id, amount );
reply = (redisReply*)redisCommand( c, "HSET GRAB:%d send_amount %d", _id, amount );
redisConnPool::getInstance()->releaseConnection(c);
if(reply!=NULL){
if(reply->type==REDIS_REPLY_ERROR){
UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str );
if(reply != NULL)
{
freeReplyObject(reply);
}
return -5;
}
freeReplyObject(reply);
}else{
UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
return 0;
}
int Grab::incrSendAmount()
{
redisContext *c;
c=redisConnPool::getInstance()->getConnection();
if(c==NULL){
UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
redisReply *reply;
UB_LOG_DEBUG( "HINCRBY GRAB:%d send_amount 1", _id );
reply = (redisReply*)redisCommand( c, "HINCRBY GRAB:%d send_amount 1", _id );
redisConnPool::getInstance()->releaseConnection(c);
if(reply!=NULL){
if(reply->type!=REDIS_REPLY_ERROR){
_info.send_amount=reply->integer;
}else{
UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str );
if(reply != NULL)
{
freeReplyObject(reply);
}
return -5;
}
freeReplyObject(reply);
}else{
UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
return 0;
}
int segment_words(const char * content,enum wtype type,int len,bool stepword,vector<funit>& features)
{
if(!content || len <= 0){
UB_LOG_WARNING("segment words,parameter error");
return -1;
}
/*get word segment result buffer*/
thread_data * tsd = (thread_data *)pthread_getspecific(key);
if(!tsd){
UB_LOG_FATAL("thread special data is null");
exit(0);
}
scw_out_t *pout = tsd->pout;
/*word segment*/
if(scw_segment_words(pwdict,pout,content,len,LANGTYPE_SIMP_CHINESE,NULL) == -1){
UB_LOG_WARNING("scw segment words failed");
return -1;
}
/*get result to vectore features*/
int i,count;
token_t tokens[1024];
funit tmp;
/*word type,we just need SCW_OUT_WPCOMP*/
u_int tsco[5] = {SCW_OUT_WPCOMP,SCW_OUT_BASIC,SCW_OUT_SUBPH,
SCW_OUT_HUMANNAME,SCW_OUT_BOOKNAME};
/*just SCW_OUT_WPCOMP mode,so j < 1*/
for(int j = 0;j < 1;j ++)
{
count = scw_get_token_1(pout,tsco[j],tokens,1024);
for(i = 0;i < count;i ++)
{
/*filter space and special punc*/
trim_string(tokens[i].buffer);
if(strlen(tokens[i].buffer) <= 1)
continue;
tmp.feature = tokens[i].buffer;
tmp.weight = 1;
features.push_back(tmp);
}
}
/*get weight*/
feature_weight(features,type);
/*output result for debug*/
for(i = 0;i < (int)features.size();i++)
{
tmp = features.at(i);
UB_LOG_DEBUG("word[%s] weight[%f]",tmp.feature.c_str(),tmp.weight);
}
return 0;
}
/*
* word segment init,dict and memory
**/
int segment_init(const char * scwconfile,const char * wordictpath)
{
pgconf = scw_load_conf(scwconfile);
if(pgconf == NULL){
UB_LOG_FATAL("scw conf load failed");
return -1;
}
UB_LOG_DEBUG("scw load conf success");
if((pwdict=scw_load_worddict(wordictpath))==NULL)
{
UB_LOG_FATAL("error: loading wordict failed: %s",wordictpath);
return -1;
}
return 0;
}
int Grab::getSendAmount()
{
redisContext *c;
uint32_t send_amount(0);
c=redisConnPool::getInstance()->getConnection();
if(c==NULL){
UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
redisReply *reply;
UB_LOG_DEBUG( "HGET GRAB:%d send_amount ", _id );
reply = (redisReply*)redisCommand( c, "HGET GRAB:%d send_amount ", _id );
redisConnPool::getInstance()->releaseConnection(c);
if(reply!=NULL){
if(reply->type!=REDIS_REPLY_ERROR){
if(reply->type!=REDIS_REPLY_NIL){
UB_LOG_NOTICE( "send amount:[%s]", reply->str );
send_amount=atoi(reply->str);
}else{
send_amount=0;
}
}else{
UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str );
if(reply != NULL)
{
freeReplyObject(reply);
}
return -5;
}
freeReplyObject(reply);
}else{
UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
return send_amount;
}
int Grab::getGrabNum(const string& user_id, string& num, int32_t is_pre)
{
extern string IP;
int res;
string query_sql;
//DuokooMysql mysql;
DuokooMysql mysql("grab_mysql");
if(!mysql.startTransaction()){
return -5;
}
//get num from mysql
//comment by zhengxie 2013.10.22
/*query_sql="SELECT num FROM `MCP`.`mcp_content_grab_num` WHERE `is_occupy` = 0 AND `grab_id` = " + toString(_id) + " limit 0, 1";*/
//UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str());
//res=mysql.query(query_sql);
//if(res<0){
//UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ );
//mysql.rollback();
//return -5;
//}else if(res==0){
//UB_LOG_FATAL( "query sql[%s] have no record, [%s:%d]", query_sql.c_str(), __FILE__, __LINE__ );
//mysql.rollback();
//return -3;
//}
/*num=mysql.getResult(0,0);*/
/* 2013.10.22 add by zhengxie get num from mysql => get num from redis start */
redisContext *c;
uint32_t send_amount(0);
c=redisConnPool::getInstance()->getConnection();
if(c==NULL)
{
UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
redisReply *reply;
/* 2013.12.03 add by zhengxie add lock */
string thread_id = DuokooTools::toString(CommonInterface::get_thread_id());
string random = DuokooTools::toString(getRandomNum(10000000));
string key = "grab_and_user_id:" + DuokooTools::toString(_id) + "," + user_id;
string value = IP + "," + thread_id + "," + random;
redisCommand(c, "SETNX %s %s", key.c_str(), value.c_str());
LOGD( "[zx]get_value key:%s ", key.c_str() );
reply = (redisReply*)redisCommand( c, "GET %s", key.c_str() );
redisConnPool::getInstance()->releaseConnection(c);
if(reply!=NULL)
{
if(reply->type!=REDIS_REPLY_ERROR)
{
if(reply->type!=REDIS_REPLY_NIL)
{
string temp_value = reply->str;
freeReplyObject(reply);
reply = NULL;
if(strcmp(temp_value.c_str(), value.c_str()))
{
LOGA( "[zx] shit, bad user[%s] ", user_id.c_str() );
return -3;//同一个用户重复抢号
}
}
else
{
freeReplyObject(reply);
reply = NULL;
return -3;
}
}
}
else
{
LOGA( "[zx] key not set[%s] ", key.c_str() );
return -5;
}
/* 2013.12.03 add by zhengxie add lock */
c=redisConnPool::getInstance()->getConnection();
if(c==NULL)
{
UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ );
return -5;
}
redisCommand(c, "EXPIRE %s %ld", key.c_str(), 10*24*60*60);
//LOGD( "[zx]SPOP grab_id:%d ", _id );
reply = (redisReply*)redisCommand( c, "SPOP grab_id:%d", _id );
redisConnPool::getInstance()->releaseConnection(c);
if(reply!=NULL)
{
if(reply->type!=REDIS_REPLY_ERROR)
{
if(reply->type!=REDIS_REPLY_NIL)
{
LOGA( "grab_id:[%d], user_id:[%s], grab_num:[%s]", _id, user_id.c_str(), reply->str );
num=reply->str;
freeReplyObject(reply);
reply = NULL;
}
else
{
LOGD( "[zx] has no grab_nums grab_id[%d] ", _id );
freeReplyObject(reply);
reply = NULL;
return -3;
}
}
}
else
{
LOGD( "[zx] not exist grab_id[%d] ", _id );
return -5;
}
/* 2013.10.22 add by zhengxie get num from mysql => get num from redis end */
//update num
query_sql="UPDATE `MCP`.`mcp_content_grab_num` SET `is_occupy` = "+ toString(is_pre) +" WHERE num = '" + num + "' ";
UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str());
res=mysql.query(query_sql);
if(res<0){
UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ );
mysql.rollback();
return -5;
}
//insert user_num
query_sql="INSERT INTO `MCP`.`mcp_content_user_grab`( `id`, `grab_id`, `user_id`, `grab_time`, `use_time`, `num` )"
"VALUES( NULL, '"+toString(_id)+"', '"+user_id+"', '"+NOW()+"', '0', '"+num+"' ) ";
UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str());
res=mysql.query(query_sql);
if(res<0){
UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ );
mysql.rollback();
return -5;
}
return mysql.commit()?0:-5;
}
int cmmsearch_query_handle_t::parse_query(const pointer_t query_cmd,
ts_buffer_t &req_detail,
basic_req_info *basic_info,
vector < ts_terminfo_t > &term_list)
{
int opret = 0;
nshead_t *req_nshead = (nshead_t *) query_cmd;
bsl::ResourcePool rp;
req_detail.reset();
mc_pack_t *req_mc = mc_pack_open_w_rp(2, req_detail._buf, req_detail._size, &rp);
if(MC_PACK_PTR_ERR(req_mc)){
UB_LOG_WARNING("[function:parse_query>>mc_pack_open_w_rp][ret:][desc:mc_pack_open_w_rp error][intput:]");
return -2;
}
opret = mc_pack_json2pack((char *)(req_nshead+1), req_mc);
if(opret<0){
UB_LOG_WARNING("[function:parse_query>>mc_pack_json2pack][ret:][desc:mc_pack_json2pack error][intput:]");
return -2;
}
//cmd
snprintf(basic_info->cmd_str, TS_CMDSTR_MAXLEN, "%s", mc_pack_get_str(req_mc, "cmd"));
if(MC_PACK_PTR_ERR(basic_info->cmd_str))
{
return -2;
}
//page_no
opret= mc_pack_get_uint32(req_mc, "page_no", &basic_info->page_no);
if(opret < 0)
{
return -2;
}
//num_per_page
opret= mc_pack_get_uint32(req_mc, "num_per_page", &basic_info->num_per_page );
if(opret < 0)
{
return -2;
}
//term_list
const char *keyword = mc_pack_get_str(req_mc, "query");
if(MC_PACK_PTR_ERR(keyword))
{
return -2;
}
const char delimiters[] = " .,;:!-";
char *running;
char *token;
running = strdup(keyword);
ts_terminfo_t terminfo;
token = strsep(&running, delimiters);
while(NULL != token)
{
creat_sign_fs64(token, strlen(token), &terminfo.term.sign1,
&terminfo.term.sign2);
term_list.push_back(terminfo);
UB_LOG_DEBUG("qterm[%s] [%u:%u]",token, terminfo.term.sign1,
terminfo.term.sign2);
token = strsep(&running, delimiters);
}
if (running)
free(running);
mc_pack_close(req_mc);
return 0;
}
/**
* @brief 查询命令处理
*
* @param [in] cmd_no : int 命令号
* @param [in] req_head : nshead_t*
* @param [in] req_buf : ub_buff_t*
* @param [in/out] res_head : nshead_t*
* @param [in/out] res_buf : ub_buff_t*
**/
STATIC int
process_query(nshead_t * req_head, ub_buff_t * req_buf,
nshead_t * res_head, ub_buff_t * res_buf)
{
int opret = 0;
struct timeval s,e;
basic_req_info *basic_info;
ts_query_handle_t *hd = g_runtime.handle->query_handle;
// parse query
gettimeofday(&s,NULL);
ts_buffer_t req_detail(req_buf->buf, req_buf->size);
vector < ts_terminfo_t > term_list; ///todo: 可能影响性能
opret = hd->parse_query(req_head, req_detail, basic_info, term_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->parse_query failed.ret[%d]", opret);
return opret;
}
if(term_list.size() > TS_QTERM_MAXNUM)
{
UB_LOG_WARNING("term_list[%ld>%d] too large", term_list.size(),
TS_QTERM_MAXNUM);
return -1;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("parse_query(us)","%lu",TIME_US_DIFF(s,e));
// get ri libs
vector < ts_ri_t * > ri_libs;
ri_libs.push_back(&g_runtime.mem_ri);
if(0 == g_runtime.mon_curdir || 1 == g_runtime.mon_curdir)
{
ri_libs.push_back(&g_runtime.mon_ri[g_runtime.mon_curdir]);
}
if(g_runtime.need_merge) //can be not thread-safe
{
ri_libs.push_back(&g_runtime.day_ri[2]);
}
vector < ts_index_t > *merged_list =
((vector < ts_index_t > *)get_data("merged_list"));
vector < ts_index_t > *filted_list =
((vector < ts_index_t > *)get_data("filted_list"));
merged_list->clear();
filted_list->clear();
gettimeofday(&s,NULL);
for(vector < ts_ri_t * >::iterator iter = ri_libs.begin();
iter != ri_libs.end(); ++iter)
{
ts_ind_reader_t ind_reader(*iter);
// merge
opret = hd->merge_ind_list(req_head,term_list,ind_reader,merged_list);
if(opret < 0)
{
UB_LOG_WARNING("merge_ind_list failed.ret[%d]",opret);
return opret;
}
}
gettimeofday(&e,NULL);
ub_log_pushnotice("merge_ind_list(us)","%lu",TIME_US_DIFF(s,e));
// washout del/undel nodes
gettimeofday(&s,NULL);
int k = 0;
for(vector < ts_index_t >::iterator i = merged_list->begin();
i != merged_list->end(); ++i)
{
if (1 == g_runtime.mod_table.get(i->id)) continue;
bool bit = g_runtime.del_table.get(i->id);
if( !strcmp(basic_info->cmd_str, "qsearchall")
||(!strcmp(basic_info->cmd_str, "qsearchdel") && bit)
||(!strcmp(basic_info->cmd_str, "qsearch") && !bit))
{
merged_list->at(k++) = *i;
}
}
merged_list->resize(k);
UB_LOG_DEBUG("merged_list.size[%ld]", merged_list->size());
gettimeofday(&e,NULL);
ub_log_pushnotice("washout_del(us)","%lu",TIME_US_DIFF(s,e));
// filt
gettimeofday(&s,NULL);
filted_list->reserve(merged_list->size());
filted_list->clear();
ts_buffer_t brief(g_cfg.brief_size);
opret = hd->index_filt(req_head, brief, merged_list, filted_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->index_filt failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("index_filt(us)","%lu",TIME_US_DIFF(s,e));
// adjust
gettimeofday(&s,NULL);
opret =
hd->adjust_weight(req_head, term_list,filted_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->adjust_weight failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("adjust_weight(us)","%lu",TIME_US_DIFF(s,e));
// res
gettimeofday(&s,NULL);
ts_buffer_t fulltext(g_cfg.fulltext_maxsize);
ts_buffer_t res(res_buf->buf, res_buf->size);
// --page
opret = hd->fill_basic_res(req_head, basic_info, filted_list, res);
if (opret<0)
{
UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret);
return opret;
}
/*if(0 == ret_num)
{
res_head->body_len = sizeof(ts_head_t);
return 0;
}*/
// --abs
opret = hd->init_abs(req_head,res);
if (opret<0)
{
UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret);
return opret;
}
unsigned int i = 0;
int start_num = basic_info->num_per_page*basic_info->page_no;
for(vector < ts_index_t >::iterator iter =
filted_list->begin() + start_num;
iter != filted_list->end() && i < basic_info->ret_num; ++iter, ++i)
{
fulltext._used = fulltext._size;
opret = g_runtime.di.read(iter->id, fulltext._buf, fulltext._used);
if(opret < 0)
{
UB_LOG_WARNING("di.read failed.ret[%d]", opret);
return opret;
}
g_runtime.mg.read(iter->id, brief._buf);
brief._used = brief._size;
opret = hd->add_abs(req_head, i, fulltext, brief, res);
if(opret < 0)
{
UB_LOG_WARNING("hd->add_abs failed.i[%d]ret[%d]", i, opret);
return opret;
}
fulltext.reset();
brief.reset();
}
opret = hd->fini_abs(req_head,res);
if (opret<0)
{
UB_LOG_WARNING("hd->fini_abs failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("add_abs(us)","%lu",TIME_US_DIFF(s,e));
res_head->body_len = res._used;
return 0;
}
