bool redis_builder::cluster_meet(acl::redis& redis,
const acl::redis_node& node)
{
acl::string buf(node.get_addr());
const std::vector<acl::string>& tokens = buf.split2(":");
if (tokens.size() != 2)
{
printf("%s: invalid addr: %s\r\n",
__FUNCTION__, node.get_addr());
return false;
}
if (!redis.cluster_meet(tokens[0].c_str(), atoi(tokens[1].c_str())))
{
printf("%s: cluster meet %s %s error: %s\r\n",
__FUNCTION__, tokens[0].c_str(), tokens[1].c_str(),
redis.result_error());
return false;
}
while (true)
{
if (cluster_meeting(redis, node.get_addr()) == true)
break;
acl_doze(meet_wait_);
}
return true;
}
bool redis_builder::add_node(const char* addr,
const char* new_node_addr, bool slave)
{
acl::redis_client client(new_node_addr);
acl::redis redis(&client);
acl::string buf(addr);
const std::vector<acl::string>& tokens = buf.split2(":");
if (tokens.size() != 2)
{
printf("%s: invalid addr: %s\r\n", __FUNCTION__, addr);
return false;
}
// CLUSTER MEET master node
if (!redis.cluster_meet(tokens[0].c_str(), atoi(tokens[1].c_str())))
{
printf("%s: cluster meet %s %s error: %s\r\n", __FUNCTION__,
tokens[0].c_str(), tokens[1].c_str(),
redis.result_error());
return false;
}
// wait for the master recognizing the slave
while (true)
{
if (cluster_meeting(redis, addr) == true)
break;
acl_doze(meet_wait_);
}
if (!slave)
return true;
acl::string node_id;
if (redis_util::get_node_id(addr, node_id) == false)
{
printf("%s: can't get master(%s)'s node_id\r\n",
__FUNCTION__, addr);
return false;
}
if (redis.cluster_replicate(node_id.c_str()) == false)
{
printf("%s: cluster replicate id: %s, error: %s, "
"master_addr: %s, slave_addr: %s\r\n",
__FUNCTION__, node_id.c_str(), redis.result_error(),
addr, new_node_addr);
return false;
}
return true;
}
void read_respond_body(HWND hWnd)
{
acl_assert(hdr_res_);
/* 如果 HTTP 响应没有数据体则仅输出 HTTP 响应头即可 */
if (hdr_res_->hdr.content_length == 0
|| (hdr_res_->hdr.content_length == -1
&& hdr_res_->reply_status > 300
&& hdr_res_->reply_status < 400))
{
data_.i_ptr = NULL;
data_.i_dlen = 0;
// 如果没有数据体，也发送消息通知调用者数据结束
report(hWnd, HTTP_MSG_DAT);
return;
}
res_ = http_res_new(hdr_res_);
#define BUF_LEN 8192
char* buf;
int ret;
while (true)
{
buf = (char*) acl_mymalloc(BUF_LEN);
ret = read_respond_body(buf, BUF_LEN - 1);
if (ret <= 0)
{
acl_myfree(buf);
break;
}
buf[ret] = 0;
data_.i_ptr = buf;
data_.i_dlen = (size_t) ret;
report(hWnd, HTTP_MSG_DAT);
if (respond_over_)
break;
if (nwait_ > 0)
acl_doze(nwait_);
}
data_.i_ptr = NULL;
data_.i_dlen = 0;
report(hWnd, HTTP_MSG_DAT);
}
bool redis_builder::add_slave(const acl::redis_node& master,
const acl::redis_node& slave)
{
acl::redis_client client(slave.get_addr());
acl::redis redis(&client);
const char* master_addr = master.get_addr();
if (master_addr == NULL || *master_addr == 0)
{
printf("%s: master addr null\r\n", __FUNCTION__);
return false;
}
acl::string buf(master_addr);
const std::vector<acl::string>& tokens = buf.split2(":");
if (tokens.size() != 2)
{
printf("%s: invalid master_addr: %s\r\n",
__FUNCTION__, master_addr);
return false;
}
// CLUSTER MEET master node
if (!redis.cluster_meet(tokens[0].c_str(), atoi(tokens[1].c_str())))
{
printf("%s: cluster meet %s %s error: %s\r\n",
__FUNCTION__, tokens[0].c_str(), tokens[1].c_str(),
redis.result_error());
return false;
}
// wait for the master recognizing the slave
while (true)
{
if (cluster_meeting(redis, master_addr) == true)
break;
acl_doze(meet_wait_);
}
if (redis.cluster_replicate(master.get_id()) == false)
{
printf("%s: cluster replicate id: %s, error: %s, addr: %s\r\n",
__FUNCTION__, master.get_id(), redis.result_error(),
slave.get_addr());
return false;
}
return true;
}
void* run(void)
{
acl_doze(500);
for (int i = 0; i < __nloop; i++)
{
if (mutex_.lock() == false)
{
printf("lock error\r\n");
break;
}
__count++;
// acl_doze(100);
if (__show)
printf("thread-%lu locked ok\r\n",
acl::thread::self());
assert(mutex_.unlock());
// acl_doze(10);
}
return NULL;
}
static void event_loop(ACL_EVENT *eventp)
{
const char *myname = "event_loop";
EVENT_POLL *ev = (EVENT_POLL *) eventp;
ACL_EVENT_TIMER *timer;
int nready, i, revents;
acl_int64 delay;
ACL_EVENT_FDTABLE *fdp;
delay = eventp->delay_sec * 1000000 + eventp->delay_usec;
if (delay < DELAY_MIN)
delay = DELAY_MIN;
/* 调整事件引擎的时间截 */
SET_TIME(eventp->present);
/* 根据定时器任务的最近任务计算 poll 的检测超时上限 */
if ((timer = ACL_FIRST_TIMER(&eventp->timer_head)) != 0) {
acl_int64 n = timer->when - eventp->present;
if (n <= 0)
delay = 0;
else if (n < delay)
delay = n;
}
/* 调用 event_prepare 检查有多少个描述字需要通过 poll 进行检测 */
if (event_prepare(eventp) == 0) {
/* 说明无须 poll 检测 */
if (eventp->ready_cnt == 0)
/* 为避免循环过快，休眠一下 */
acl_doze(delay > DELAY_MIN ? (int) delay / 1000 : 1);
goto TAG_DONE;
}
/* 如果已经有描述字准备好则 poll 检测超时时间置 0 */
if (eventp->ready_cnt > 0)
delay = 0;
/* 调用 poll 系统调用检测可用描述字 */
nready = poll(ev->fds, eventp->fdcnt, (int) (delay / 1000));
if (eventp->nested++ > 0) {
acl_msg_error("%s(%d): recursive call", myname, __LINE__);
exit (1);
}
if (nready < 0) {
if (acl_last_error() != ACL_EINTR) {
acl_msg_error("%s(%d), %s: select: %s", __FILE__,
__LINE__, myname, acl_last_serror());
exit (1);
}
goto TAG_DONE;
} else if (nready == 0)
goto TAG_DONE;
/* 检查 poll 的检测结果集合 */
for (i = 0; i < eventp->fdcnt; i++) {
fdp = acl_fdmap_ctx(ev->fdmap, ev->fds[i].fd);
if (fdp == NULL || fdp->stream == NULL)
continue;
/* 如果该描述字对象已经在被设置为异常或超时状态则继续 */
if ((fdp->event_type & (ACL_EVENT_XCPT | ACL_EVENT_RW_TIMEOUT)))
continue;
revents = ev->fds[i].revents;
/* 检查描述字是否出现异常 */
if ((revents & (POLLHUP | POLLERR)) != 0) {
fdp->event_type |= ACL_EVENT_XCPT;
fdp->fdidx_ready = eventp->ready_cnt;
eventp->ready[eventp->ready_cnt++] = fdp;
continue;
}
/* 检查描述字是否可读 */
if ((fdp->flag & EVENT_FDTABLE_FLAG_READ)
&& (revents & POLLIN) )
{
/* 给该描述字对象附加可读属性 */
if ((fdp->event_type & (ACL_EVENT_READ
| ACL_EVENT_WRITE)) == 0)
{
fdp->event_type |= ACL_EVENT_READ;
fdp->fdidx_ready = eventp->ready_cnt;
eventp->ready[eventp->ready_cnt++] = fdp;
}
if (fdp->listener)
fdp->event_type |= ACL_EVENT_ACCEPT;
/* 该描述字可读则设置 ACL_VSTREAM 的系统可读标志从而
* 触发 ACL_VSTREAM 流在读时调用系统的 read 函数
*/
else
fdp->stream->read_ready = 1;
}
/* 检查描述字是否可写 */
if ((fdp->flag & EVENT_FDTABLE_FLAG_WRITE)
&& (revents & POLLOUT))
{
/* 给该描述字对象附加可写属性 */
if ((fdp->event_type & (ACL_EVENT_READ
| ACL_EVENT_WRITE)) == 0)
{
fdp->event_type |= ACL_EVENT_WRITE;
fdp->fdidx_ready = eventp->ready_cnt;
eventp->ready[eventp->ready_cnt++] = fdp;
}
}
}
TAG_DONE:
event_timer_trigger(eventp);
/* 处理准备好的描述字事件 */
if (eventp->ready_cnt > 0)
event_fire(eventp);
eventp->nested--;
}
bool redis_builder::build_cluster()
{
if (masters_.empty())
{
printf("%s: no master available!\r\n", __FUNCTION__);
return false;
}
size_t range = MAX_SLOTS / masters_.size();
size_t begin = 0, end = MAX_SLOTS % masters_.size() + range -1;
// build every master node, and connect all of its slaves.
std::vector<acl::redis_node*>::iterator it;
for (it = masters_.begin(); it != masters_.end(); ++it)
{
if (it != masters_.begin())
printf("----------------------------------------\r\n");
(*it)->add_slot_range(begin, end);
if (build_master(**it) == false)
return false;
begin = end + 1;
end = end + range;
}
it = masters_.begin();
acl::redis_client client((*it)->get_addr());
acl::redis master(&client);
// let one master to connect all other master nodes
printf("===================================================\r\n");
printf("Meeting all masters and slaves ...\r\n");
std::vector<acl::redis_node*> all_slaves;
std::vector<acl::redis_node*>::const_iterator cit;
for (++it; it != masters_.end(); ++it)
{
if (cluster_meet(master, **it) == false)
return false;
const std::vector<acl::redis_node*>* slaves = (*it)->get_slaves();
for (cit = slaves->begin(); cit != slaves->end(); ++cit)
all_slaves.push_back(*cit);
}
while (true)
{
int nwait = 0;
for (cit = all_slaves.begin(); cit != all_slaves.end(); ++cit)
{
if ((*cit)->is_connected())
continue;
if (cluster_meeting(master, (*cit)->get_addr()) == false)
nwait++;
else
(*cit)->set_connected(true);
}
if (nwait == 0)
break;
acl_doze(meet_wait_);
}
/////////////////////////////////////////////////////////////////////
printf("===================================================\r\n");
printf("All nodes of cluster:\r\n");
const std::map<acl::string, acl::redis_node*>* nodes;
if ((nodes = master.cluster_nodes())== NULL)
{
printf("%s: can't get cluster nodes, addr: %s\r\n",
__FUNCTION__, client.get_stream()->get_peer(true));
return false;
}
redis_status::show_nodes(nodes);
return true;
}
const redis_result* redis_command::run(redis_client_cluster* cluster,
size_t nchild, int* timeout /* = NULL */)
{
redis_client* conn = peek_conn(cluster, slot_);
// 如果没有找到可用的连接对象，则直接返回 NULL 表示出错
if (conn == NULL)
{
logger_error("peek_conn NULL, slot_: %d", slot_);
return NULL;
}
set_client_addr(*conn);
conn->set_check_addr(check_addr_);
redis_result_t type;
bool last_moved = false;
int n = 0;
while (n++ < redirect_max_)
{
// 根据请求过程是否采用内存分片方式调用不同的请求过程
if (slice_req_)
result_ = conn->run(dbuf_, *request_obj_, nchild, timeout);
else
result_ = conn->run(dbuf_, *request_buf_, nchild, timeout);
// 如果连接异常断开，则需要进行重试
if (conn->eof())
{
// 删除哈希槽中的地址映射关系以便下次操作时重新获取
cluster->clear_slot(slot_);
// 将连接对象归还给连接池对象
conn->get_pool()->put(conn, false);
// 如果连接断开且请求数据为空时，则无须重试
if (request_obj_->get_size() == 0 && request_buf_->empty())
{
logger_error("not retry when no request!");
return NULL;
}
// 将连接池对象置为不可用状态
conn->get_pool()->set_alive(false);
// 从连接池集群中顺序取得一个连接对象
conn = peek_conn(cluster, slot_);
if (conn == NULL)
{
logger_error("peek_conn NULL");
return result_;
}
last_moved = true;
clear(true);
set_client_addr(*conn);
continue;
}
if (result_ == NULL)
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
logger_error("result NULL");
return NULL;
}
// 取得服务器的响应结果的类型，并进行分别处理
type = result_->get_type();
if (type == REDIS_RESULT_UNKOWN)
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
logger_error("unknown result type: %d", type);
return NULL;
}
if (type != REDIS_RESULT_ERROR)
{
// 如果发生重定向过程，则设置哈希槽对应 redis 服务地址
if (slot_ < 0 || !last_moved)
{
// 将连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
return result_;
}
// XXX: 因为此处还要引用一次 conn 对象，所以将 conn
// 归还给连接池的过程须放在此段代码之后
const char* addr = conn->get_pool()->get_addr();
cluster->set_slot(slot_, addr);
// 将连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
return result_;
}
#define EQ(x, y) !strncasecmp((x), (y), sizeof(y) -1)
// 对于结果类型为错误类型，则需要进一步判断是否是重定向指令
const char* ptr = result_->get_error();
if (ptr == NULL || *ptr == 0)
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
logger_error("result error: null");
return result_;
}
// 如果出错信息为重定向指令，则执行重定向过程
if (EQ(ptr, "MOVED"))
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
const char* addr = get_addr(ptr);
if (addr == NULL)
{
logger_warn("MOVED invalid, ptr: %s", ptr);
return result_;
}
conn = redirect(cluster, addr);
if (conn == NULL)
{
logger_error("redirect NULL, addr: %s", addr);
return result_;
}
ptr = conn->get_pool()->get_addr();
set_client_addr(ptr);
if (n >= 2 && redirect_sleep_ > 0
&& strcmp(ptr, addr) != 0)
{
logger("redirect %d, curr %s, waiting %s ...",
n, ptr, addr);
acl_doze(redirect_sleep_);
}
last_moved = true;
// 需要保存哈希槽值
clear(true);
}
else if (EQ(ptr, "ASK"))
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
const char* addr = get_addr(ptr);
if (addr == NULL)
{
logger_warn("ASK invalid, ptr: %s", ptr);
return result_;
}
conn = redirect(cluster, addr);
if (conn == NULL)
{
logger_error("redirect NULL, addr: %s", addr);
return result_;
}
ptr = conn->get_pool()->get_addr();
set_client_addr(ptr);
if (n >= 2 && redirect_sleep_ > 0
&& strcmp(ptr, addr) != 0)
{
logger("redirect %d, curr %s, waiting %s ...",
n, ptr, addr);
acl_doze(redirect_sleep_);
}
result_ = conn->run(dbuf_, "ASKING\r\n", 0);
if (result_ == NULL)
{
logger_error("ASKING's reply null");
return NULL;
}
const char* status = result_->get_status();
if (status == NULL || strcasecmp(status, "OK") != 0)
{
logger_error("ASKING's reply error: %s",
status ? status : "null");
return NULL;
}
last_moved = false;
clear(true);
}
// 处理一个主结点失效的情形
else if (EQ(ptr, "CLUSTERDOWN"))
{
cluster->clear_slot(slot_);
if (redirect_sleep_ > 0)
{
logger("%s: redirect %d, slot %d, waiting %s ...",
conn->get_pool()->get_addr(),
n, slot_, ptr);
acl_doze(redirect_sleep_);
}
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
conn = peek_conn(cluster, -1);
if (conn == NULL)
{
logger_error("peek_conn NULL");
return result_;
}
clear(true);
set_client_addr(*conn);
}
// 对于其它错误类型，则直接返回本次得到的响应结果对象
else
{
// 将旧连接对象归还给连接池对象
conn->get_pool()->put(conn, true);
logger_error("server error: %s", ptr);
if (!slice_req_)
logger_error("request: %s",
request_buf_->c_str());
return result_;
}
}
if (conn != NULL)
conn->get_pool()->put(conn, true);
logger_warn("too many redirect: %d, max: %d", n, redirect_max_);
return NULL;
}
