void DispatchTask::handle () { DispatchCost cost; TimeStat t1,t2; int stype = feed_->type & 0x0000ffff; int bigtype = stype / 100;// int appid = feed_->newsMerge;//对于特定的type,newsMerge代表appid if(stype == 8190){ //对于81XX的新鲜事,psource为appid appid = GetAppId(); } if (config_->sendConfig) { //使用数据库里,用户的设置 //1 try { //MCE_DEBUG("DispatchTask::handle --> Invoke --> ConfigManagerAdapter::instance().GetSendConfig(), actor: " << feed_->actor << ",stype:"<<bigtype << ",appid:"<<appid); bool p = ConfigManagerAdapter::instance().GetSendConfig(//现在的SendConfig中只需要查询news的设置 feed_->actor, bigtype, appid); if (!p) { config_->sendNewsConfig = p; config_->sendMiniConfig = p;//根据产品的需求,miniconfig和newsconfig相等 } MCE_INFO("DispatchTask::handle. call FeedConfig source:" << feed_->source << " type:" << feed_->type << " actor:" << feed_->actor << " sendNewsConfig = " << config_->sendNewsConfig << ", p:"<<p << " appid:" << appid); } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle.ConfigManagerAdapter::instance().GetSendConfig().uid:<< "<< feed_->actor << " fid:" << feed_->feed << "exception "<<e); } } cost.send_cfg_ = t1.getTime(); set<int> target; if (config_->sendNewsConfig) { //需要发送News // if (1) { //TODO del MCE_DEBUG("DispatchTask::handle --> sendNewsConfig = true"); //2 string oldexpr = config_->expr; // ChangeExpr(feed_->actor,stype,config_->expr); // MCE_INFO("DispatchTask::handle --> ChangeExpr. fid:" << feed_->feed << " actor = " << feed_->actor << " oldexpr:" << oldexpr << " newexpr:" << config_->expr); try { target = ExprCache::instance().Expr2Ids(config_->expr, feed_->actor); MCE_DEBUG("DispatchTask::handle --> After ExprCache.Expr2Ids ,the target : size = " << target.size()); } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle --> ExprCache::instance().Expr2Ids() --> exception :" <<e); } catch (std::exception& e) { MCE_WARN("DispatchTask::handle --> ExprCache::instance().Expr2Ids() --> exception :" << e.what()); } cost.expr_ = t1.getTime(); if (config_->sendMiniConfig) { int stype = feed_->type & 0xFFFF; if (!(stype >= 2901 && stype <= 2904)) { target.insert(feed_->actor); } } //(feed_->actor>600000000&&feed_->actor<700000000) //3 //target id可能有很多,分批发送 const static unsigned int kMaxSize = 5000; vector<int> in, partIn, partOut;//in为全部的target,partIn为分批发送的target,大小最大为kMaxSize, //partOut为部分的返回的不需要发送target in.assign(target.begin(), target.end()); int i = 0;//表示第几批 size_t not_recv_cnt = 0; for (vector<int>::iterator vit = in.begin(); vit < in.end(); vit += kMaxSize, ++i) { //MCE_DEBUG("DispatchTask::handle --> INVOKE --> ConfigManagerAdapter::instance().FilterRecvConfig --> part :" << i); partIn.assign(vit, vit + kMaxSize < in.end() ? vit + kMaxSize : in.end()); try { partOut = ConfigManagerAdapter::instance().FilterRecvConfig(partIn, bigtype, appid); // MCE_DEBUG("PartIn size = " << partIn.size() << ", PartOut ids size = " << partOut.size()<< ", bigtype:"<<bigtype << ",appid:" << appid); not_recv_cnt += partOut.size(); for (vector<int>::iterator vit = partOut.begin(); vit != partOut.end(); ++vit) { target.erase(*vit); } } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle --> Invoke --> ConfigManagerAdapter::instance().uid : "<< feed_->actor << " fid:" << feed_->feed <<" exception "<<e); } } cost.recv_cfg_ = t1.getTime(); MCE_INFO("DispatchTask::handle --> feedId = " << feed_->feed << ",actor = " << feed_->actor << ",expression = " << config_->expr << ", filter ids size = " << not_recv_cnt << ", result size : " << target.size() << " appid:" << appid); } else { MCE_DEBUG("DispatchTask::handle. sendNewsConfig is false"); } // if(target.empty()){ // return; // } //4 if (bigtype == 29 && (!target.empty())) { /*set<int> tmpids; if(stype == 2901){ for(set<int>::iterator it=target.begin(); it!=target.end(); ++it){ if((*it)%10 == 7){ tmpids.insert(*it); } } }*/ //if(stype != 2901){ // SpecialFilter::instance().Filt(target); //} //target.insert(tmpids.begin(), tmpids.end()); } // if(target.empty()){ // return; // } //5 map<int, int> rank; try { if (!target.empty() && target.size() < 3000) { int total_score=0; rank = FriendRankCacheAdapter::instance().GetRevRank(feed_->actor); for (map<int, int>::iterator it = rank.begin(); it != rank.end(); ++it) { // os << " " << it->first << ":" << it->second; total_score += it->second; } MCE_INFO("DispatchTask::handle. rank .fid:"<< feed_->feed << " average:" << float(total_score)/ rank.size()); } } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle Exception: "<<e); } cost.friend_ = t1.getTime(); TimeStat tb; map<int, int> scores = GetBayesScores(feed_->actor, target, stype); for (map<int, int>::iterator it = scores.begin(); it != scores.end(); ++it) { rank[it->first] = it->second; } MCE_INFO("DispatchTask::handle. call bayes. bayescost:" << tb.getTime()); // GetFocusSource(feed_,rank); //6 //if((feed_->actor==32715654||feed_->actor==234159611||feed_->actor==255329305||feed_->actor==255414584||feed_->actor==256110032||feed_->actor==257876975) && FeedDeliver::instance().deliver(feed_, config_, target, rank,cost); FeedItem item; item.feed = feed_->feed; item.type = feed_->type; item.time = feed_->time; item.weight = feed_->baseWeight; item.merge = feed_->newsMerge; item.actor = feed_->actor; string ext; int aid = GetAppId(); if(aid > 0){ ostringstream os; os << "AppId=" << aid; ext = os.str(); } //PersistenceHelper::write(item, ext); cost.total_ = t2.getTime(); InteractionUploader::instance().UploadRange(feed_->feed,target.size()); MCE_INFO("DispatchTask::handle. fid:"<< feed_->feed <<" source:" << feed_->source << " type:" << stype << " actor:" << feed_->actor <<",expr:"<<config_->expr<<",target.size:"<<target.size() << ", friendRank.size:"<< rank.size() << " #cost# " << cost.ToString()) // string e = config_->expr.empty() ? string(" NULL"):string(config_->expr.begin(), config_->expr.begin() +1); // e = "D_" + e; // StatTimer::instance().Update(feed_->actor,e,t1.getTime(),target.size()); }
void DispatchTask::handle () { DispatchCost cost; TimeStat t1,t2; int stype = feed_->type & 0x0000ffff; int bigtype = stype / 100;// int appid = feed_->newsMerge;//对于特定的type,newsMerge代表appid if(stype == 8190){ //对于81XX的新鲜事,psource为appid appid = GetAppId(); } if (config_->sendConfig) { //使用数据库里,用户的设置 try { if (DegradeManager::instance().IsEnable("FeedDispatcher::module_ConfigManager")) { bool p = ConfigManagerAdapter::instance().GetSendConfig(//现在的SendConfig中只需要查询news的设置 feed_->actor, bigtype, appid); if (!p) { config_->sendNewsConfig = p; config_->sendMiniConfig = p;//根据产品的需求,miniconfig和newsconfig相等 } } } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle.ConfigManagerAdapter::instance().GetSendConfig().uid:<< "<< feed_->actor << " fid:" << feed_->feed << "exception "<<e); } } cost.send_cfg_ = t1.getTime(true); set<int> target; if (config_->sendNewsConfig) { //需要发送News // if (1) { //TODO del MCE_DEBUG("DispatchTask::handle --> sendNewsConfig = true"); //2 string oldexpr = config_->expr; // ChangeExpr(feed_->actor,stype,config_->expr); // MCE_INFO("DispatchTask::handle --> ChangeExpr. fid:" << feed_->feed << " actor = " << feed_->actor << " oldexpr:" << oldexpr << " newexpr:" << config_->expr); try { target = ExprCache::instance().Expr2Ids(config_->expr, feed_->actor); MCE_DEBUG("DispatchTask::handle --> After ExprCache.Expr2Ids ,the target : size = " << target.size()); } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle --> ExprCache::instance().Expr2Ids() --> exception :" <<e); } catch (std::exception& e) { MCE_WARN("DispatchTask::handle --> ExprCache::instance().Expr2Ids() --> exception :" << e.what()); } cost.expr_ = t1.getTime(true); if (config_->sendMiniConfig) { int stype = feed_->type & 0xFFFF; if (!(stype >= 2901 && stype <= 2904)) { target.insert(feed_->actor); } } //(feed_->actor>600000000&&feed_->actor<700000000) //3 //target id可能有很多,分批发送 const static unsigned int kMaxSize = 5000; vector<int> in, partIn, partOut;//in为全部的target,partIn为分批发送的target,大小最大为kMaxSize, //partOut为部分的返回的不需要发送target in.assign(target.begin(), target.end()); int i = 0;//表示第几批 size_t not_recv_cnt = 0; /**********************************add by shilong.li*/ //判断待分发的新鲜事 是否是小站通过addMiniSiteFeed或addMiniSiteFeedToGuides 接口向指定用户推送, // 如果是跳过ConfigManager::FilterRevConfig这块的逻辑 bool isMachineDispatch = false; ::MyUtil::Str2StrMap::iterator expt = feed_->extraProps.find("MachineDispatch"); if ( (expt != feed_->extraProps.end()) && expt->second == "addMiniSiteFeed" ) { isMachineDispatch = true; MCE_INFO("DispatcherTask::handle --> addMiniSiteFeed dispach feed actor:"<<feed_->actor<<" feedid:"<<feed_->feed<<" skip ConfigManager::FilterRecvConfig() "); } /*******************************/ bool isEnable =DegradeManager::instance().IsEnable("FeedDispatcher::module_ConfigManager"); for (vector<int>::iterator vit = in.begin(); vit < in.end()&& (!isMachineDispatch) && isEnable; vit += kMaxSize, ++i) { MCE_DEBUG("DispatchTask::handle --> INVOKE --> ConfigManagerAdapter::instance().FilterRecvConfig --> part :" << i); partIn.assign(vit, vit + kMaxSize < in.end() ? vit + kMaxSize : in.end()); try { partOut = ConfigManagerAdapter::instance().FilterRecvConfig(partIn, bigtype, appid); // MCE_DEBUG("PartIn size = " << partIn.size() << ", PartOut ids size = " << partOut.size()<< ", bigtype:"<<bigtype << ",appid:" << appid); not_recv_cnt += partOut.size(); for (vector<int>::iterator vit = partOut.begin(); vit != partOut.end(); ++vit) { target.erase(*vit); } } catch (Ice::Exception& e) { MCE_WARN("DispatchTask::handle --> Invoke --> ConfigManagerAdapter::instance().uid : "<< feed_->actor << " fid:" << feed_->feed <<" exception "<<e); } } cost.recv_cfg_ = t1.getTime(true); MCE_INFO("DispatchTask::handle --> feedId = " << feed_->feed << ",actor = " << feed_->actor << ",expression = " << config_->expr << ", filter ids size = " << not_recv_cnt << ", result size : " << target.size() << " appid:" << appid); } else { MCE_DEBUG("DispatchTask::handle. sendNewsConfig is false"); } int limitThreshold = 100000; map<int,int> scores; if (target.size() < limitThreshold) { if (DegradeManager::instance().IsEnable("FeedDispatcher::module_BayesScores")) { scores = GetBayesScores(feed_->actor, target, stype); } } else { TaskManager::instance().execute(new DispatchBigFansSizeFeedTask(feed_,config_,target)); return; } FeedDeliver::instance().deliver(feed_, config_, target,scores,cost); cost.total_ = t2.getTime(); InteractionUploader::instance().UploadRange(feed_->feed,target.size()); MCE_INFO("DispatchTask::handle fid:"<< feed_->feed <<" source:" << feed_->source << " type:" << stype << " actor:" << feed_->actor<< " expr:"<<config_->expr<<" target size:"<<target.size()<<" #cost# " << cost.ToString()); }