int Grab::setSendAmount(const uint32_t amount) { redisContext *c; c=redisConnPool::getInstance()->getConnection(); if(c==NULL){ UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ ); return -5; } redisReply *reply; UB_LOG_DEBUG( "HSET GRAB:%d send_amount %d", _id, amount ); reply = (redisReply*)redisCommand( c, "HSET GRAB:%d send_amount %d", _id, amount ); redisConnPool::getInstance()->releaseConnection(c); if(reply!=NULL){ if(reply->type==REDIS_REPLY_ERROR){ UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str ); if(reply != NULL) { freeReplyObject(reply); } return -5; } freeReplyObject(reply); }else{ UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ ); return -5; } return 0; }
int Grab::incrSendAmount() { redisContext *c; c=redisConnPool::getInstance()->getConnection(); if(c==NULL){ UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ ); return -5; } redisReply *reply; UB_LOG_DEBUG( "HINCRBY GRAB:%d send_amount 1", _id ); reply = (redisReply*)redisCommand( c, "HINCRBY GRAB:%d send_amount 1", _id ); redisConnPool::getInstance()->releaseConnection(c); if(reply!=NULL){ if(reply->type!=REDIS_REPLY_ERROR){ _info.send_amount=reply->integer; }else{ UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str ); if(reply != NULL) { freeReplyObject(reply); } return -5; } freeReplyObject(reply); }else{ UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ ); return -5; } return 0; }
int segment_words(const char * content,enum wtype type,int len,bool stepword,vector<funit>& features) { if(!content || len <= 0){ UB_LOG_WARNING("segment words,parameter error"); return -1; } /*get word segment result buffer*/ thread_data * tsd = (thread_data *)pthread_getspecific(key); if(!tsd){ UB_LOG_FATAL("thread special data is null"); exit(0); } scw_out_t *pout = tsd->pout; /*word segment*/ if(scw_segment_words(pwdict,pout,content,len,LANGTYPE_SIMP_CHINESE,NULL) == -1){ UB_LOG_WARNING("scw segment words failed"); return -1; } /*get result to vectore features*/ int i,count; token_t tokens[1024]; funit tmp; /*word type,we just need SCW_OUT_WPCOMP*/ u_int tsco[5] = {SCW_OUT_WPCOMP,SCW_OUT_BASIC,SCW_OUT_SUBPH, SCW_OUT_HUMANNAME,SCW_OUT_BOOKNAME}; /*just SCW_OUT_WPCOMP mode,so j < 1*/ for(int j = 0;j < 1;j ++) { count = scw_get_token_1(pout,tsco[j],tokens,1024); for(i = 0;i < count;i ++) { /*filter space and special punc*/ trim_string(tokens[i].buffer); if(strlen(tokens[i].buffer) <= 1) continue; tmp.feature = tokens[i].buffer; tmp.weight = 1; features.push_back(tmp); } } /*get weight*/ feature_weight(features,type); /*output result for debug*/ for(i = 0;i < (int)features.size();i++) { tmp = features.at(i); UB_LOG_DEBUG("word[%s] weight[%f]",tmp.feature.c_str(),tmp.weight); } return 0; }
/* * word segment init,dict and memory **/ int segment_init(const char * scwconfile,const char * wordictpath) { pgconf = scw_load_conf(scwconfile); if(pgconf == NULL){ UB_LOG_FATAL("scw conf load failed"); return -1; } UB_LOG_DEBUG("scw load conf success"); if((pwdict=scw_load_worddict(wordictpath))==NULL) { UB_LOG_FATAL("error: loading wordict failed: %s",wordictpath); return -1; } return 0; }
int Grab::getSendAmount() { redisContext *c; uint32_t send_amount(0); c=redisConnPool::getInstance()->getConnection(); if(c==NULL){ UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ ); return -5; } redisReply *reply; UB_LOG_DEBUG( "HGET GRAB:%d send_amount ", _id ); reply = (redisReply*)redisCommand( c, "HGET GRAB:%d send_amount ", _id ); redisConnPool::getInstance()->releaseConnection(c); if(reply!=NULL){ if(reply->type!=REDIS_REPLY_ERROR){ if(reply->type!=REDIS_REPLY_NIL){ UB_LOG_NOTICE( "send amount:[%s]", reply->str ); send_amount=atoi(reply->str); }else{ send_amount=0; } }else{ UB_LOG_FATAL( "REDIS ERROR[%s]", reply->str ); if(reply != NULL) { freeReplyObject(reply); } return -5; } freeReplyObject(reply); }else{ UB_LOG_FATAL( "get a NULL redis connection, [%s:%d]", __FILE__, __LINE__ ); return -5; } return send_amount; }
int Grab::getGrabNum(const string& user_id, string& num, int32_t is_pre) { extern string IP; int res; string query_sql; //DuokooMysql mysql; DuokooMysql mysql("grab_mysql"); if(!mysql.startTransaction()){ return -5; } //get num from mysql //comment by zhengxie 2013.10.22 /*query_sql="SELECT num FROM `MCP`.`mcp_content_grab_num` WHERE `is_occupy` = 0 AND `grab_id` = " + toString(_id) + " limit 0, 1";*/ //UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str()); //res=mysql.query(query_sql); //if(res<0){ //UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ ); //mysql.rollback(); //return -5; //}else if(res==0){ //UB_LOG_FATAL( "query sql[%s] have no record, [%s:%d]", query_sql.c_str(), __FILE__, __LINE__ ); //mysql.rollback(); //return -3; //} /*num=mysql.getResult(0,0);*/ /* 2013.10.22 add by zhengxie get num from mysql => get num from redis start */ redisContext *c; uint32_t send_amount(0); c=redisConnPool::getInstance()->getConnection(); if(c==NULL) { UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ ); return -5; } redisReply *reply; /* 2013.12.03 add by zhengxie add lock */ string thread_id = DuokooTools::toString(CommonInterface::get_thread_id()); string random = DuokooTools::toString(getRandomNum(10000000)); string key = "grab_and_user_id:" + DuokooTools::toString(_id) + "," + user_id; string value = IP + "," + thread_id + "," + random; redisCommand(c, "SETNX %s %s", key.c_str(), value.c_str()); LOGD( "[zx]get_value key:%s ", key.c_str() ); reply = (redisReply*)redisCommand( c, "GET %s", key.c_str() ); redisConnPool::getInstance()->releaseConnection(c); if(reply!=NULL) { if(reply->type!=REDIS_REPLY_ERROR) { if(reply->type!=REDIS_REPLY_NIL) { string temp_value = reply->str; freeReplyObject(reply); reply = NULL; if(strcmp(temp_value.c_str(), value.c_str())) { LOGA( "[zx] shit, bad user[%s] ", user_id.c_str() ); return -3;//同一个用户重复抢号 } } else { freeReplyObject(reply); reply = NULL; return -3; } } } else { LOGA( "[zx] key not set[%s] ", key.c_str() ); return -5; } /* 2013.12.03 add by zhengxie add lock */ c=redisConnPool::getInstance()->getConnection(); if(c==NULL) { UB_LOG_FATAL( "get redis connection failed, [%s:%d]", __FILE__, __LINE__ ); return -5; } redisCommand(c, "EXPIRE %s %ld", key.c_str(), 10*24*60*60); //LOGD( "[zx]SPOP grab_id:%d ", _id ); reply = (redisReply*)redisCommand( c, "SPOP grab_id:%d", _id ); redisConnPool::getInstance()->releaseConnection(c); if(reply!=NULL) { if(reply->type!=REDIS_REPLY_ERROR) { if(reply->type!=REDIS_REPLY_NIL) { LOGA( "grab_id:[%d], user_id:[%s], grab_num:[%s]", _id, user_id.c_str(), reply->str ); num=reply->str; freeReplyObject(reply); reply = NULL; } else { LOGD( "[zx] has no grab_nums grab_id[%d] ", _id ); freeReplyObject(reply); reply = NULL; return -3; } } } else { LOGD( "[zx] not exist grab_id[%d] ", _id ); return -5; } /* 2013.10.22 add by zhengxie get num from mysql => get num from redis end */ //update num query_sql="UPDATE `MCP`.`mcp_content_grab_num` SET `is_occupy` = "+ toString(is_pre) +" WHERE num = '" + num + "' "; UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str()); res=mysql.query(query_sql); if(res<0){ UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ ); mysql.rollback(); return -5; } //insert user_num query_sql="INSERT INTO `MCP`.`mcp_content_user_grab`( `id`, `grab_id`, `user_id`, `grab_time`, `use_time`, `num` )" "VALUES( NULL, '"+toString(_id)+"', '"+user_id+"', '"+NOW()+"', '0', '"+num+"' ) "; UB_LOG_DEBUG("query_sql:[%s]", query_sql.c_str()); res=mysql.query(query_sql); if(res<0){ UB_LOG_FATAL( "query sql[%s] failed, [%s], [%s:%d]", query_sql.c_str(), mysql.getErrDes().c_str(), __FILE__, __LINE__ ); mysql.rollback(); return -5; } return mysql.commit()?0:-5; }
int cmmsearch_query_handle_t::parse_query(const pointer_t query_cmd, ts_buffer_t &req_detail, basic_req_info *basic_info, vector < ts_terminfo_t > &term_list) { int opret = 0; nshead_t *req_nshead = (nshead_t *) query_cmd; bsl::ResourcePool rp; req_detail.reset(); mc_pack_t *req_mc = mc_pack_open_w_rp(2, req_detail._buf, req_detail._size, &rp); if(MC_PACK_PTR_ERR(req_mc)){ UB_LOG_WARNING("[function:parse_query>>mc_pack_open_w_rp][ret:][desc:mc_pack_open_w_rp error][intput:]"); return -2; } opret = mc_pack_json2pack((char *)(req_nshead+1), req_mc); if(opret<0){ UB_LOG_WARNING("[function:parse_query>>mc_pack_json2pack][ret:][desc:mc_pack_json2pack error][intput:]"); return -2; } //cmd snprintf(basic_info->cmd_str, TS_CMDSTR_MAXLEN, "%s", mc_pack_get_str(req_mc, "cmd")); if(MC_PACK_PTR_ERR(basic_info->cmd_str)) { return -2; } //page_no opret= mc_pack_get_uint32(req_mc, "page_no", &basic_info->page_no); if(opret < 0) { return -2; } //num_per_page opret= mc_pack_get_uint32(req_mc, "num_per_page", &basic_info->num_per_page ); if(opret < 0) { return -2; } //term_list const char *keyword = mc_pack_get_str(req_mc, "query"); if(MC_PACK_PTR_ERR(keyword)) { return -2; } const char delimiters[] = " .,;:!-"; char *running; char *token; running = strdup(keyword); ts_terminfo_t terminfo; token = strsep(&running, delimiters); while(NULL != token) { creat_sign_fs64(token, strlen(token), &terminfo.term.sign1, &terminfo.term.sign2); term_list.push_back(terminfo); UB_LOG_DEBUG("qterm[%s] [%u:%u]",token, terminfo.term.sign1, terminfo.term.sign2); token = strsep(&running, delimiters); } if (running) free(running); mc_pack_close(req_mc); return 0; }
/** * @brief 查询命令处理 * * @param [in] cmd_no : int 命令号 * @param [in] req_head : nshead_t* * @param [in] req_buf : ub_buff_t* * @param [in/out] res_head : nshead_t* * @param [in/out] res_buf : ub_buff_t* **/ STATIC int process_query(nshead_t * req_head, ub_buff_t * req_buf, nshead_t * res_head, ub_buff_t * res_buf) { int opret = 0; struct timeval s,e; basic_req_info *basic_info; ts_query_handle_t *hd = g_runtime.handle->query_handle; // parse query gettimeofday(&s,NULL); ts_buffer_t req_detail(req_buf->buf, req_buf->size); vector < ts_terminfo_t > term_list; ///todo: 可能影响性能 opret = hd->parse_query(req_head, req_detail, basic_info, term_list); if(opret < 0) { UB_LOG_WARNING("hd->parse_query failed.ret[%d]", opret); return opret; } if(term_list.size() > TS_QTERM_MAXNUM) { UB_LOG_WARNING("term_list[%ld>%d] too large", term_list.size(), TS_QTERM_MAXNUM); return -1; } gettimeofday(&e,NULL); ub_log_pushnotice("parse_query(us)","%lu",TIME_US_DIFF(s,e)); // get ri libs vector < ts_ri_t * > ri_libs; ri_libs.push_back(&g_runtime.mem_ri); if(0 == g_runtime.mon_curdir || 1 == g_runtime.mon_curdir) { ri_libs.push_back(&g_runtime.mon_ri[g_runtime.mon_curdir]); } if(g_runtime.need_merge) //can be not thread-safe { ri_libs.push_back(&g_runtime.day_ri[2]); } vector < ts_index_t > *merged_list = ((vector < ts_index_t > *)get_data("merged_list")); vector < ts_index_t > *filted_list = ((vector < ts_index_t > *)get_data("filted_list")); merged_list->clear(); filted_list->clear(); gettimeofday(&s,NULL); for(vector < ts_ri_t * >::iterator iter = ri_libs.begin(); iter != ri_libs.end(); ++iter) { ts_ind_reader_t ind_reader(*iter); // merge opret = hd->merge_ind_list(req_head,term_list,ind_reader,merged_list); if(opret < 0) { UB_LOG_WARNING("merge_ind_list failed.ret[%d]",opret); return opret; } } gettimeofday(&e,NULL); ub_log_pushnotice("merge_ind_list(us)","%lu",TIME_US_DIFF(s,e)); // washout del/undel nodes gettimeofday(&s,NULL); int k = 0; for(vector < ts_index_t >::iterator i = merged_list->begin(); i != merged_list->end(); ++i) { if (1 == g_runtime.mod_table.get(i->id)) continue; bool bit = g_runtime.del_table.get(i->id); if( !strcmp(basic_info->cmd_str, "qsearchall") ||(!strcmp(basic_info->cmd_str, "qsearchdel") && bit) ||(!strcmp(basic_info->cmd_str, "qsearch") && !bit)) { merged_list->at(k++) = *i; } } merged_list->resize(k); UB_LOG_DEBUG("merged_list.size[%ld]", merged_list->size()); gettimeofday(&e,NULL); ub_log_pushnotice("washout_del(us)","%lu",TIME_US_DIFF(s,e)); // filt gettimeofday(&s,NULL); filted_list->reserve(merged_list->size()); filted_list->clear(); ts_buffer_t brief(g_cfg.brief_size); opret = hd->index_filt(req_head, brief, merged_list, filted_list); if(opret < 0) { UB_LOG_WARNING("hd->index_filt failed.ret[%d]", opret); return opret; } gettimeofday(&e,NULL); ub_log_pushnotice("index_filt(us)","%lu",TIME_US_DIFF(s,e)); // adjust gettimeofday(&s,NULL); opret = hd->adjust_weight(req_head, term_list,filted_list); if(opret < 0) { UB_LOG_WARNING("hd->adjust_weight failed.ret[%d]", opret); return opret; } gettimeofday(&e,NULL); ub_log_pushnotice("adjust_weight(us)","%lu",TIME_US_DIFF(s,e)); // res gettimeofday(&s,NULL); ts_buffer_t fulltext(g_cfg.fulltext_maxsize); ts_buffer_t res(res_buf->buf, res_buf->size); // --page opret = hd->fill_basic_res(req_head, basic_info, filted_list, res); if (opret<0) { UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret); return opret; } /*if(0 == ret_num) { res_head->body_len = sizeof(ts_head_t); return 0; }*/ // --abs opret = hd->init_abs(req_head,res); if (opret<0) { UB_LOG_WARNING("hd->init_abs failed.ret[%d]", opret); return opret; } unsigned int i = 0; int start_num = basic_info->num_per_page*basic_info->page_no; for(vector < ts_index_t >::iterator iter = filted_list->begin() + start_num; iter != filted_list->end() && i < basic_info->ret_num; ++iter, ++i) { fulltext._used = fulltext._size; opret = g_runtime.di.read(iter->id, fulltext._buf, fulltext._used); if(opret < 0) { UB_LOG_WARNING("di.read failed.ret[%d]", opret); return opret; } g_runtime.mg.read(iter->id, brief._buf); brief._used = brief._size; opret = hd->add_abs(req_head, i, fulltext, brief, res); if(opret < 0) { UB_LOG_WARNING("hd->add_abs failed.i[%d]ret[%d]", i, opret); return opret; } fulltext.reset(); brief.reset(); } opret = hd->fini_abs(req_head,res); if (opret<0) { UB_LOG_WARNING("hd->fini_abs failed.ret[%d]", opret); return opret; } gettimeofday(&e,NULL); ub_log_pushnotice("add_abs(us)","%lu",TIME_US_DIFF(s,e)); res_head->body_len = res._used; return 0; }