std::map<std::string, std::set<MessageQueue> > RebalanceImpl::buildProcessQueueTableByBrokerName()
{
std::map<std::string, std::set<MessageQueue> > result ;
std::map<MessageQueue, ProcessQueue*>::iterator it = m_processQueueTable.begin();
for ( ; it != m_processQueueTable.end();)
{
MessageQueue mq = it->first;
std::map<std::string, std::set<MessageQueue> >::iterator itm = result.find(mq.getBrokerName());
if (itm == result.end())
{
std::set<MessageQueue> mqs ;
mqs.insert(mq);
result[mq.getBrokerName()] = mqs;
}
else
{
itm->second.insert(mq);
}
}
return result;
}
void MQClientFactory::topicRouteData2TopicSubscribeInfo( std::string& topic, TopicRouteData* route, std::set<MessageQueue*>& mqList)
{
std::list<QueueData*>& qds = route->getQueueDatas();
std::list<QueueData*>::iterator it = qds.begin();
QueueData* qd = NULL;
for(; it != qds.end(); it++) {
qd = (*it);
for(int i = 0; i < qd->getReadQueueNums(); i++) {
MessageQueue* mq = new MessageQueue(topic, qd->getBrokerName(), i);
std::cout << mq->getBrokerName() << std::endl;
mqList.insert(mq);
}
}
}
bool RebalanceImpl::lock(MessageQueue& mq)
{
FindBrokerResult findBrokerResult =
m_pMQClientFactory->findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll::MASTER_ID, true);
if (!findBrokerResult.brokerAddr.empty())
{
LockBatchRequestBody* requestBody = new LockBatchRequestBody();
requestBody->setConsumerGroup(m_consumerGroup);
requestBody->setClientId(m_pMQClientFactory->getClientId());
requestBody->getMqSet().insert(mq);
try
{
std::set<MessageQueue> lockedMq =
m_pMQClientFactory->getMQClientAPIImpl()->lockBatchMQ(
findBrokerResult.brokerAddr, requestBody, 1000);
std::set<MessageQueue>::iterator it = lockedMq.begin();
for (; it != lockedMq.end(); it++)
{
kpr::ScopedLock<kpr::Mutex> lock(m_processQueueTableLock);
MessageQueue mmqq = *it;
std::map<MessageQueue, ProcessQueue*>::iterator itt = m_processQueueTable.find(mmqq);
if (itt != m_processQueueTable.end())
{
itt->second->setLocked(true);
itt->second->setLastLockTimestamp(GetCurrentTimeMillis());
}
}
it = lockedMq.find(mq);
if (it != lockedMq.end())
{
return true;
}
//TODO log.info("the message queue lock {}, {} {}",//
// (lockOK ? "OK" : "Failed"), //
// this.consumerGroup, //
// mq);
return false;
}
catch (...)
{
//TODO log.error("lockBatchMQ exception, " + mq, e);
}
}
return false;
}
void RebalanceImpl::unlock(MessageQueue& mq, bool oneway)
{
FindBrokerResult findBrokerResult =
m_pMQClientFactory->findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll::MASTER_ID, true);
if (!findBrokerResult.brokerAddr.empty())
{
UnlockBatchRequestBody* requestBody = new UnlockBatchRequestBody();
requestBody->setConsumerGroup(m_consumerGroup);
requestBody->setClientId(m_pMQClientFactory->getClientId());
requestBody->getMqSet().insert(mq);
try
{
m_pMQClientFactory->getMQClientAPIImpl()->unlockBatchMQ(findBrokerResult.brokerAddr,
requestBody, 1000, oneway);
}
catch (...)
{
//TODO log.error("unlockBatchMQ exception, " + mq, e);
}
}
}
SendResult_ptr DefaultMQProducerImpl::sendDefaultImpl(Message_ptr msg, std::string& communicationMode, bool sendCallback) {
long beginTimestamp = 0;
long endTimestamp = beginTimestamp;
TopicPublishInfo* topicPublishInfo = this->tryToFindTopicPublishInfo(msg->getTopic());
if (topicPublishInfo->ok()) {
MessageQueue* mq = NULL;
SendResult_var sendResult = new SendResult();
for (int times = 0; times < 3; times++) {
std::string lastBrokerName;
if(mq != NULL) {
lastBrokerName = mq->getBrokerName();
}
MessageQueue* tmpmq = topicPublishInfo->selectOneMessageQueue(lastBrokerName);
if (tmpmq != NULL) {
mq = tmpmq;
sendResult = this->sendKernelImpl(msg, mq, communicationMode, sendCallback);
endTimestamp = 12;
if(communicationMode == CommunicationMode::SYNC) {
if (sendResult->getSendStatus() != SendStatus::SEND_OK) {
/*
if (this->defaultMQProducer->isRetryAnotherBrokerWhenNotStoreOK()) {
continue;
}
*/
}
return SendResult_var::Duplicate(sendResult);
}
}
else {
break;
}
}
return SendResult_var::Duplicate(sendResult);
}
return NULL;
}
bool RebalanceImpl::updateProcessQueueTableInRebalance(const std::string& topic, std::set<MessageQueue>& mqSet)
{
bool changed = false;
// 将多余的队列删除
std::map<MessageQueue, ProcessQueue*>::iterator it = m_processQueueTable.begin();
for ( ; it != m_processQueueTable.end();)
{
MessageQueue mq = it->first;
if (mq.getTopic() == topic)
{
std::set<MessageQueue>::iterator its = mqSet.find(mq);
if (its == mqSet.end())
{
changed = true;
ProcessQueue* pq = it->second;
if (pq != NULL)
{
pq->setDroped(true);
removeUnnecessaryMessageQueue(mq, *pq);
//TODO log.info("doRebalance, {}, remove unnecessary mq, {}",
// consumerGroup, mq);
}
std::map<MessageQueue, ProcessQueue*>::iterator ittmp = it;
it++;
m_processQueueTable.erase(ittmp);
}
else
{
it++;
}
}
else
{
it++;
}
}
// 增加新增的队列
std::list<PullRequest> pullRequestList;
std::set<MessageQueue>::iterator its = mqSet.begin();
for (; its != mqSet.end(); its++)
{
MessageQueue mq = *its;
std::map<MessageQueue, ProcessQueue*>::iterator itm = m_processQueueTable.find(mq);
if (itm == m_processQueueTable.end())
{
PullRequest pullRequest;
pullRequest.setConsumerGroup(m_consumerGroup);
pullRequest.setMessageQueue(new MessageQueue(mq.getTopic(),mq.getBrokerName(),mq.getQueueId()));
pullRequest.setProcessQueue(new ProcessQueue());
// 这个需要根据策略来设置
long long nextOffset = computePullFromWhere(mq);
if (nextOffset >= 0)
{
pullRequest.setNextOffset(nextOffset);
pullRequestList.push_back(pullRequest);
changed = true;
m_processQueueTable[mq] = pullRequest.getProcessQueue();
//TODO log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq);
}
else
{
// 等待此次Rebalance做重试
//TODO log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq);
}
}
}
dispatchPullRequest(pullRequestList);
return changed;
}