/**
*
* @brief 处理srch命令的函数
*
* @param[in] cmd_no 命令号,一个函数可能处理多条命令,可以用命令号来区分
* @param[in] req_head 请求数据的nshead_t*
* @param[in] req_buf 请求数据的buffer,本buffer不包含nshead_t
* @param[out] res_head 应答数据的nshead_t*
* @param[out] res_buf 应答数据的buffer,本buffer不包含nshead_t
* @return
* 0 : 成功
* -1 : 失败,socket将直接关闭,不给client返回错误信息
**/
STATIC int
srch_process(int cmd_no, nshead_t * req_head, ub_buff_t * req_buf,
nshead_t * res_head, ub_buff_t * res_buf)
{
/**
*在打NOTICE日志时,请使用 ub_log_pushnotice 加入日志信息
*如果返回值不为0,将导致socket被直接关闭而不给客户返回任何信息
**/
srch_thread_data_t *p_thread_data;
if(NULL == req_head || NULL == req_buf || NULL == res_head || NULL == res_buf) {
UB_LOG_FATAL("parameter error in srch process.");
return -1;
}
srch_reset_res(req_head, res_head, res_buf);
char* req = req_buf->buf;
char* res = res_buf->buf;
res_head->body_len = 0;
int response_buffer_size = ub_server_get_write_size() - sizeof (nshead_t);
p_thread_data = (srch_thread_data_t* )ub_server_get_user_data();
if(NULL == p_thread_data) {
UB_LOG_FATAL("thead_data null");
return -1;
}
int post_count = 0;
int max_post_count = p_thread_data->max_post_count;
u_int64_t* pids = p_thread_data->post_ids;
delpost_record_t* records = p_thread_data->records;
if (!unpack_data(req, req_head->body_len, post_count, pids, max_post_count))
{
UB_LOG_WARNING("unpack mcpack error");
return -1;
}
int ret = query(post_count, pids, records,
g_conf.mask_path, g_conf.index_file, g_conf.mask_file);
if (ret < 0)
{
UB_LOG_WARNING("query[ret=%d] error", ret);
return -1;
}
ret = pack_data(res, response_buffer_size, post_count, records);
if (ret < 0)
{
UB_LOG_WARNING("pack mcpack error");
return -1;
}
res_head->body_len = ret;
return 0;
}
int CamelSuperStrategy::fetchServer(slb_request_t * req) {
CHECK_POOL("BEFORE doFilter ",_rp);
if (req == NULL || copyCall == NULL) {
return -1;
}
if (_isDebug) {
_chain_filter->debug(req->serverNum);
}
const int serverNum = req->serverNum;
server_rank_t serverRank[serverNum];
memset(serverRank, 0, sizeof (serverRank)); /**< disable=false */
request_t request;
memset(&request, 0, sizeof (request));
request.serverRanks = serverRank;
request.mustSelectOneServer= true;
request.needHealthyFilter = true;
// get from req
request.key = req->key;
request.nthRetry = req->nthRetry;
snprintf(request.currentMachineRoom, sizeof(request.currentMachineRoom), "%s", req->currentMachineRoom);
request.serverNum = serverNum;
if (serverNum > static_cast<int>(MAX_SERVER_SIZE)) {
UB_LOG_WARNING("too many server while fetchServer, get %d, suppert %d", serverNum, MAX_SERVER_SIZE);
return -1;
}
for (int i = 0; i < serverNum; i++) {
copyCall(req->svr[i], request.server + i);
}
int ret = _chain_filter->doFilter(&request); /**< 返回找到的serverID */
server_rank_t* serv = NULL;
for (int i = 0; ret >= 0 && i < serverNum; i++) /**< 找到该serverID 的server_rank_t */
{
server_rank_t& sr = serverRank[i];
if (sr.serverId == (uint)ret)
{
serv = &sr;
break;
}
}
if (serv == NULL){
ret = -1;
UB_LOG_WARNING("SuperStrategy<fetchServer: failed ret[%d]", ret);
}
/*else{
UB_LOG_DEBUG("SuperStrategy<fetchServer: server_id[%u] machine_room[%s] cross_room[%d] balance[%lu] \
connect[%g] healthy_score[%g] healthy_select[%g] disabled[%d] use_backup[%d] backup_server[%u] \
retry[%d] key[%d]",
serv->serverId, serv->machineRoom, serv->crossRoom, serv->balanceRank, serv->connectScore,
serv->healthyScore, serv->healthySelectRate, serv->disabled, serv->useBackup,
serv->backupServerId, request.nthRetry, request.key);
}*/
CHECK_POOL("AFTER doFilter ",_rp);
return ret;
}
int segment_words(const char * content,enum wtype type,int len,bool stepword,vector<funit>& features)
{
if(!content || len <= 0){
UB_LOG_WARNING("segment words,parameter error");
return -1;
}
/*get word segment result buffer*/
thread_data * tsd = (thread_data *)pthread_getspecific(key);
if(!tsd){
UB_LOG_FATAL("thread special data is null");
exit(0);
}
scw_out_t *pout = tsd->pout;
/*word segment*/
if(scw_segment_words(pwdict,pout,content,len,LANGTYPE_SIMP_CHINESE,NULL) == -1){
UB_LOG_WARNING("scw segment words failed");
return -1;
}
/*get result to vectore features*/
int i,count;
token_t tokens[1024];
funit tmp;
/*word type,we just need SCW_OUT_WPCOMP*/
u_int tsco[5] = {SCW_OUT_WPCOMP,SCW_OUT_BASIC,SCW_OUT_SUBPH,
SCW_OUT_HUMANNAME,SCW_OUT_BOOKNAME};
/*just SCW_OUT_WPCOMP mode,so j < 1*/
for(int j = 0;j < 1;j ++)
{
count = scw_get_token_1(pout,tsco[j],tokens,1024);
for(i = 0;i < count;i ++)
{
/*filter space and special punc*/
trim_string(tokens[i].buffer);
if(strlen(tokens[i].buffer) <= 1)
continue;
tmp.feature = tokens[i].buffer;
tmp.weight = 1;
features.push_back(tmp);
}
}
/*get weight*/
feature_weight(features,type);
/*output result for debug*/
for(i = 0;i < (int)features.size();i++)
{
tmp = features.at(i);
UB_LOG_DEBUG("word[%s] weight[%f]",tmp.feature.c_str(),tmp.weight);
}
return 0;
}
/**
* @brief 主处理函数
*
* @return int
* @retval
**/
int
op_query()
{
int opret = 0;
nshead_t *req_head;
nshead_t *res_head;
ub_buff_t req_buf;
ub_buff_t res_buf;
in_addr_t req_ip;
req_head = (nshead_t *) ub_server_get_read_buf();
res_head = (nshead_t *) ub_server_get_write_buf();
if(NULL == req_head || NULL == res_head)
{
UB_LOG_FATAL("get req_head[%ld] || res_head[%ld] failed.",
(long)req_head, (long)res_head);
return -1;
}
req_buf.buf = (char *)(req_head + 1);
req_buf.size = ub_server_get_read_size() - sizeof(nshead_t);
res_buf.buf = (char *)(res_head + 1);
res_buf.size = ub_server_get_write_size() - sizeof(nshead_t);
//设置一些log需要的字段
char ip_str[20];
ip_str[0] = 0;
req_ip = ub_server_get_ip();
inet_ntop(AF_INET, &req_ip, ip_str, sizeof(ip_str));
ub_log_setbasic(UB_LOG_REQIP, "%s", ip_str);
ub_log_setbasic(UB_LOG_LOGID, "%u", req_head->log_id);
ub_log_setbasic(UB_LOG_REQSVR, "%s", req_head->provider);
ub_log_setbasic(UB_LOG_SVRNAME, "%s", g_cfg.svr_query.svr_name);
ub_log_pushnotice("req_dlen", "%d", req_head->body_len);
*res_head = *req_head;
strncpy(res_head->provider, req_head->provider,
sizeof(res_head->provider));
res_head->body_len = 0;
res_head->reserved = 0;
//处理查询
struct timeval total_s,total_e;
gettimeofday(&total_s, NULL);
opret = process_query(req_head, &req_buf, res_head, &res_buf);
if(opret != 0){
UB_LOG_WARNING("[function:op_update]>>>process_query failed! errno:%d", opret);
return -1;
}
gettimeofday(&total_e, NULL);
ub_log_setbasic(UB_LOG_PROCTIME, "%luus", TIME_US_DIFF(total_s, total_e));
return opret;
}
int RoundRobinBalance::balanceServer(request_t* request)
{
if (request == NULL || request->serverNum == 0)
{
UB_LOG_WARNING("balanceServer arg error[request=%p server_num=%u]",
request, request ? request->serverNum : 0);
return -1;
}
_lock.lock();
int lastId = _lastId++;
_lock.unlock();
uint serverNum = request->serverNum;
// printf("===============================> %d %u\n", lastId, serverNum);
if (lastId < 0)
lastId = -1;
uint curId = (uint)(lastId + 1) % serverNum;
for (uint i = 0; i < request->serverNum; i++)
{
if (i == curId)
{
request->serverRanks[i].balanceRank = serverNum + 1;
}
else
{
request->serverRanks[i].balanceRank = rand() % serverNum;
}
}
// for (uint i = 0; i < request->serverNum; i++)
// {
//// if (i <= curId)
//// request->serverRanks[i].balanceRank = i + serverNum;
//// else
//// request->serverRanks[i].balanceRank = i;
// uint id = (i + curId) % serverNum;
// request->serverRanks[id].balanceRank = serverNum - i;
// }
return 0;
}
// mc_pack 解包
static bool unpack_data(const char* src_buf, int src_buf_len, int& post_count, u_int64_t* pids, int max_post_count)
{
const int TMP_BUF_SIZE = 1024 * 1024;
char tmp_buffer[TMP_BUF_SIZE];
// mc_pack 处理
mc_pack_t* pack = mc_pack_open_r(src_buf, src_buf_len, tmp_buffer, sizeof (tmp_buffer));
if (pack == NULL || MC_PACK_PTR_ERR(pack) != 0)
{
UB_LOG_WARNING("mc_pack_open_r error.");
return false;
}
int ret = 0;
ret = mc_pack_get_int32(pack, "post_count", &post_count);
if(0 != ret || post_count <= 0)
{
UB_LOG_WARNING("post_count[ret=%d, post_count=%d] get error.", ret, post_count);
return false;
}
if (post_count > max_post_count)
{
UB_LOG_WARNING("query post_count[%d] is too large[max=%d].", post_count, max_post_count);
return false;
}
mc_pack_t* pid_array = mc_pack_get_array(pack, "postids");
if (pid_array == NULL || MC_PACK_PTR_ERR(pid_array) != 0)
{
UB_LOG_WARNING("recv postids pack[pid_array=%p, PTR_ERR=%d] is not valid.", pid_array, MC_PACK_PTR_ERR(pid_array));
return false;
}
int c = mc_pack_get_item_count(pid_array);
if (c != post_count)
{
UB_LOG_WARNING("mcpack_pid_count[ret=%d, post_count=%d] get error.", c, post_count);
return false;
}
for (int i = 0; i < post_count; i++)
{
u_int64_t pid = 0;
ret = mc_pack_get_uint64_arr(pid_array, i, (mc_uint64_t*)&pid);
if(0 != ret)
{
UB_LOG_WARNING("get the %d(th) pid[ret = %d, pid=%lu] error. ", i, ret, pid);
return false;
}
pids[i] = pid;
}
return true;
}
int ConsistencyBalance::balanceServer(request_t* request)
{
if (request == NULL || request->serverNum == 0)
{
UB_LOG_WARNING("balanceServer arg error[request=%p server_num=%u]",
request, request ? request->serverNum : 0);
return -1;
}
for (uint i = 0; i < request->serverNum; i++)
{
slb_server_t* svr = request->serverRanks[i].server;
char buff[1024];
int len = snprintf(buff, sizeof(buff), "%s#%u#%d", svr->ip, svr->port, request->key);
unsigned int md5res[4];
unsigned char *md5 = (unsigned char*)(md5res);
//算签名, 直接MD5就好了
MD5((unsigned char*)buff,len, md5);
request->serverRanks[i].balanceRank = md5res[0] + md5res[1] + md5res[2] + md5res[3];
}
return 0;
}
static int pack_data(char* dest_buf, int dest_buf_len, int post_count, const delpost_record_t* records)
{
const int TMP_BUF_SIZE = 1024 * 1024;
char tmp_buffer[TMP_BUF_SIZE];
mc_pack_t* res_pack = mc_pack_open_w(1, dest_buf, dest_buf_len, tmp_buffer, sizeof (tmp_buffer));
if(MC_PACK_PTR_ERR(res_pack) != 0)
{
UB_LOG_WARNING("mc_pack_create error");
return -1;
}
int ret = 0;
ret = mc_pack_put_int32(res_pack, "post_count", post_count);
if (ret != 0)
{
UB_LOG_WARNING("post_count[ret=%d, post_count=%d] put error.", ret, post_count);
return -1;
}
mc_pack_t* res_array = mc_pack_put_array(res_pack, "del_post_info");
if (res_array == NULL || MC_PACK_PTR_ERR(res_array) != 0)
{
UB_LOG_WARNING("mc_pack_put_array del_post_info[ret=%p, MC_PACK_PTR_ERR=%d] error", res_array, MC_PACK_PTR_ERR(res_array));
return -1;
}
for (int i = 0; i < post_count; i++)
{
mc_pack_t* obj = mc_pack_put_object(res_array, NULL);
if (obj == NULL || MC_PACK_PTR_ERR(obj) != 0)
{
UB_LOG_WARNING("mc_pack_put_object res_array[ret=%p, MC_PACK_PTR_ERR=%d] error", obj, MC_PACK_PTR_ERR(obj));
return -1;
}
int r = 0;
r = mc_pack_put_uint64(obj, "post_id", (mc_uint64_t)records[i].post_id);
if (r != 0)
{
UB_LOG_WARNING("mc_pack_put_uint64 post_id[pid=%lu, ret=%d] error", records[i].post_id, r);
return -1;
}
r = mc_pack_put_nstr(obj, "user_name", records[i].username, strlen(records[i].username) + 1);
if (r != 0)
{
UB_LOG_WARNING("mc_pack_put_uint64 user_name[username=%s, ret=%d] error", records[i].username, r);
return -1;
}
r = mc_pack_put_uint32(obj, "del_time", records[i].del_time);
if (r != 0)
{
UB_LOG_WARNING("mc_pack_put_uint64 del_time[del_time=%u, ret=%d] error", records[i].del_time, r);
return -1;
}
}
mc_pack_close(res_pack);
return mc_pack_get_size(res_pack);
}
int cmmsearch_query_handle_t::parse_query(const pointer_t query_cmd,
ts_buffer_t &req_detail,
basic_req_info *basic_info,
vector < ts_terminfo_t > &term_list)
{
int opret = 0;
nshead_t *req_nshead = (nshead_t *) query_cmd;
bsl::ResourcePool rp;
req_detail.reset();
mc_pack_t *req_mc = mc_pack_open_w_rp(2, req_detail._buf, req_detail._size, &rp);
if(MC_PACK_PTR_ERR(req_mc)){
UB_LOG_WARNING("[function:parse_query>>mc_pack_open_w_rp][ret:][desc:mc_pack_open_w_rp error][intput:]");
return -2;
}
opret = mc_pack_json2pack((char *)(req_nshead+1), req_mc);
if(opret<0){
UB_LOG_WARNING("[function:parse_query>>mc_pack_json2pack][ret:][desc:mc_pack_json2pack error][intput:]");
return -2;
}
//cmd
snprintf(basic_info->cmd_str, TS_CMDSTR_MAXLEN, "%s", mc_pack_get_str(req_mc, "cmd"));
if(MC_PACK_PTR_ERR(basic_info->cmd_str))
{
return -2;
}
//page_no
opret= mc_pack_get_uint32(req_mc, "page_no", &basic_info->page_no);
if(opret < 0)
{
return -2;
}
//num_per_page
opret= mc_pack_get_uint32(req_mc, "num_per_page", &basic_info->num_per_page );
if(opret < 0)
{
return -2;
}
//term_list
const char *keyword = mc_pack_get_str(req_mc, "query");
if(MC_PACK_PTR_ERR(keyword))
{
return -2;
}
const char delimiters[] = " .,;:!-";
char *running;
char *token;
running = strdup(keyword);
ts_terminfo_t terminfo;
token = strsep(&running, delimiters);
while(NULL != token)
{
creat_sign_fs64(token, strlen(token), &terminfo.term.sign1,
&terminfo.term.sign2);
term_list.push_back(terminfo);
UB_LOG_DEBUG("qterm[%s] [%u:%u]",token, terminfo.term.sign1,
terminfo.term.sign2);
token = strsep(&running, delimiters);
}
if (running)
free(running);
mc_pack_close(req_mc);
return 0;
}
/**
* @brief 查询命令处理
*
* @param [in] cmd_no : int 命令号
* @param [in] req_head : nshead_t*
* @param [in] req_buf : ub_buff_t*
* @param [in/out] res_head : nshead_t*
* @param [in/out] res_buf : ub_buff_t*
**/
STATIC int
process_query(nshead_t * req_head, ub_buff_t * req_buf,
nshead_t * res_head, ub_buff_t * res_buf)
{
int opret = 0;
struct timeval s,e;
basic_req_info *basic_info;
ts_query_handle_t *hd = g_runtime.handle->query_handle;
// parse query
gettimeofday(&s,NULL);
ts_buffer_t req_detail(req_buf->buf, req_buf->size);
vector < ts_terminfo_t > term_list; ///todo: 可能影响性能
opret = hd->parse_query(req_head, req_detail, basic_info, term_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->parse_query failed.ret[%d]", opret);
return opret;
}
if(term_list.size() > TS_QTERM_MAXNUM)
{
UB_LOG_WARNING("term_list[%ld>%d] too large", term_list.size(),
TS_QTERM_MAXNUM);
return -1;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("parse_query(us)","%lu",TIME_US_DIFF(s,e));
// get ri libs
vector < ts_ri_t * > ri_libs;
ri_libs.push_back(&g_runtime.mem_ri);
if(0 == g_runtime.mon_curdir || 1 == g_runtime.mon_curdir)
{
ri_libs.push_back(&g_runtime.mon_ri[g_runtime.mon_curdir]);
}
if(g_runtime.need_merge) //can be not thread-safe
{
ri_libs.push_back(&g_runtime.day_ri[2]);
}
vector < ts_index_t > *merged_list =
((vector < ts_index_t > *)get_data("merged_list"));
vector < ts_index_t > *filted_list =
((vector < ts_index_t > *)get_data("filted_list"));
merged_list->clear();
filted_list->clear();
gettimeofday(&s,NULL);
for(vector < ts_ri_t * >::iterator iter = ri_libs.begin();
iter != ri_libs.end(); ++iter)
{
ts_ind_reader_t ind_reader(*iter);
// merge
opret = hd->merge_ind_list(req_head,term_list,ind_reader,merged_list);
if(opret < 0)
{
UB_LOG_WARNING("merge_ind_list failed.ret[%d]",opret);
return opret;
}
}
gettimeofday(&e,NULL);
ub_log_pushnotice("merge_ind_list(us)","%lu",TIME_US_DIFF(s,e));
// washout del/undel nodes
gettimeofday(&s,NULL);
int k = 0;
for(vector < ts_index_t >::iterator i = merged_list->begin();
i != merged_list->end(); ++i)
{
if (1 == g_runtime.mod_table.get(i->id)) continue;
bool bit = g_runtime.del_table.get(i->id);
if( !strcmp(basic_info->cmd_str, "qsearchall")
||(!strcmp(basic_info->cmd_str, "qsearchdel") && bit)
||(!strcmp(basic_info->cmd_str, "qsearch") && !bit))
{
merged_list->at(k++) = *i;
}
}
merged_list->resize(k);
UB_LOG_DEBUG("merged_list.size[%ld]", merged_list->size());
gettimeofday(&e,NULL);
ub_log_pushnotice("washout_del(us)","%lu",TIME_US_DIFF(s,e));
// filt
gettimeofday(&s,NULL);
filted_list->reserve(merged_list->size());
filted_list->clear();
ts_buffer_t brief(g_cfg.brief_size);
opret = hd->index_filt(req_head, brief, merged_list, filted_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->index_filt failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("index_filt(us)","%lu",TIME_US_DIFF(s,e));
// adjust
gettimeofday(&s,NULL);
opret =
hd->adjust_weight(req_head, term_list,filted_list);
if(opret < 0)
{
UB_LOG_WARNING("hd->adjust_weight failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("adjust_weight(us)","%lu",TIME_US_DIFF(s,e));
// res
gettimeofday(&s,NULL);
ts_buffer_t fulltext(g_cfg.fulltext_maxsize);
ts_buffer_t res(res_buf->buf, res_buf->size);
// --page
opret = hd->fill_basic_res(req_head, basic_info, filted_list, res);
if (opret<0)
{
UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret);
return opret;
}
/*if(0 == ret_num)
{
res_head->body_len = sizeof(ts_head_t);
return 0;
}*/
// --abs
opret = hd->init_abs(req_head,res);
if (opret<0)
{
UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret);
return opret;
}
unsigned int i = 0;
int start_num = basic_info->num_per_page*basic_info->page_no;
for(vector < ts_index_t >::iterator iter =
filted_list->begin() + start_num;
iter != filted_list->end() && i < basic_info->ret_num; ++iter, ++i)
{
fulltext._used = fulltext._size;
opret = g_runtime.di.read(iter->id, fulltext._buf, fulltext._used);
if(opret < 0)
{
UB_LOG_WARNING("di.read failed.ret[%d]", opret);
return opret;
}
g_runtime.mg.read(iter->id, brief._buf);
brief._used = brief._size;
opret = hd->add_abs(req_head, i, fulltext, brief, res);
if(opret < 0)
{
UB_LOG_WARNING("hd->add_abs failed.i[%d]ret[%d]", i, opret);
return opret;
}
fulltext.reset();
brief.reset();
}
opret = hd->fini_abs(req_head,res);
if (opret<0)
{
UB_LOG_WARNING("hd->fini_abs failed.ret[%d]", opret);
return opret;
}
gettimeofday(&e,NULL);
ub_log_pushnotice("add_abs(us)","%lu",TIME_US_DIFF(s,e));
res_head->body_len = res._used;
return 0;
}
