/*
* Returns true if an event is processed
*/
static bool local_process_event(void)
{
struct local_event *ev;
enum cluster_join_result res;
ev = shm_queue_peek();
if (!ev)
return false;
switch (ev->type) {
case EVENT_JOIN_REQUEST:
if (!node_eq(&ev->nodes[0], &this_node))
return false;
res = sd_check_join_cb(&ev->sender, ev->buf);
ev->join_result = res;
ev->type = EVENT_JOIN_RESPONSE;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
if (res == CJ_RES_MASTER_TRANSFER) {
eprintf("failed to join sheepdog cluster: "
"please retry when master is up\n");
shm_queue_unlock();
exit(1);
}
return false;
case EVENT_JOIN_RESPONSE:
if (ev->join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev->nr_nodes = 1;
ev->nodes[0] = this_node;
ev->pids[0] = getpid();
shm_queue_set_chksum();
}
sd_join_handler(&ev->sender, ev->nodes, ev->nr_nodes,
ev->join_result, ev->buf);
shm_queue_pop();
break;
case EVENT_LEAVE:
sd_leave_handler(&ev->sender, ev->nodes, ev->nr_nodes);
shm_queue_pop();
break;
case EVENT_BLOCK:
sd_block_handler(&ev->sender);
return false;
case EVENT_NOTIFY:
sd_notify_handler(&ev->sender, ev->buf, ev->buf_len);
shm_queue_pop();
break;
}
return true;
}
static void interpret_msg_pre_join(void)
{
int ret;
struct sph_msg snd, rcv;
struct sph_msg_join_reply *join_reply;
retry:
read_msg(&rcv);
if (rcv.type == SPH_SRV_MSG_JOIN_RETRY) {
sd_info("join request is rejected, retrying");
do_shepherd_join();
goto retry;
} else if (rcv.type == SPH_SRV_MSG_NEW_NODE) {
struct sph_msg_join *join;
int join_len;
join_len = rcv.body_len;
join = xzalloc(join_len);
ret = xread(sph_comm_fd, join, join_len);
if (ret != join_len) {
sd_err("xread() failed: %m");
exit(1);
}
/*
* FIXME: member change events must be ordered with nonblocked
* events
*/
if (!sd_join_handler(&join->new_node, NULL, 0, join->opaque))
panic("sd_accept_handler() failed");
snd.type = SPH_CLI_MSG_ACCEPT;
snd.body_len = join_len;
ret = writev2(sph_comm_fd, &snd, join, join_len);
if (sizeof(snd) + join_len != ret) {
sd_err("writev2() failed: %m");
exit(1);
}
free(join);
read_msg(&rcv);
}
if (rcv.type != SPH_SRV_MSG_JOIN_REPLY) {
sd_err("unexpected message from shepherd, received message: %s",
sph_srv_msg_to_str(rcv.type));
/*
* In this case, the state of this sheep in shepherd must be
* SHEEP_STATE_CONNECTED. Messages other than SPH_MSG_JOIN_REPLY
* mean bugs of shepherd.
*/
exit(1);
}
join_reply = xzalloc(rcv.body_len);
ret = xread(sph_comm_fd, join_reply, rcv.body_len);
if (ret != rcv.body_len) {
sd_err("xread() failed: %m");
exit(1);
}
sd_info("join reply arrived, nr_nodes: %d", join_reply->nr_nodes);
memcpy(nodes, join_reply->nodes,
join_reply->nr_nodes * sizeof(struct sd_node));
nr_nodes = join_reply->nr_nodes;
/* FIXME: member change events must be ordered with nonblocked events */
sd_accept_handler(&this_node, nodes, nr_nodes, join_reply->opaque);
free(join_reply);
sd_info("shepherd_join() succeed");
state = STATE_JOINED;
}
static void interpret_msg_pre_join(void)
{
int ret;
struct sph_msg snd, rcv;
struct sph_msg_join_reply *join_reply;
enum cluster_join_result res;
retry:
read_msg(&rcv);
if (rcv.type == SPH_SRV_MSG_JOIN_RETRY) {
sd_iprintf("join request is rejected, retrying");
do_shepherd_join();
goto retry;
} else if (rcv.type == SPH_SRV_MSG_NEW_NODE) {
struct sph_msg_join *join;
int join_len;
join_len = rcv.body_len;
join = xzalloc(join_len);
ret = xread(sph_comm_fd, join, join_len);
if (ret != join_len) {
sd_eprintf("xread() failed: %m");
exit(1);
}
res = sd_check_join_cb(&join->node, join->opaque);
if (res == CJ_RES_FAIL) {
sd_eprintf("sd_check_join_cb() failed");
exit(1);
}
assert(res == CJ_RES_SUCCESS);
/* FIXME: join->master_elected is needed? */
assert(join->master_elected);
is_master = true;
snd.type = SPH_CLI_MSG_NEW_NODE_REPLY;
snd.body_len = join_len;
ret = writev2(sph_comm_fd, &snd, join, join_len);
if (sizeof(snd) + join_len != ret) {
sd_eprintf("writev2() failed: %m");
exit(1);
}
free(join);
read_msg(&rcv);
}
if (rcv.type != SPH_SRV_MSG_JOIN_REPLY) {
sd_eprintf("unexpected message from shepherd, "
"received message: %s", sph_srv_msg_to_str(rcv.type));
/*
* In this case, the state of this sheep in shepherd must be
* SHEEP_STATE_CONNECTED. Messages other than SPH_MSG_JOIN_REPLY
* mean bugs of shepherd.
*/
exit(1);
}
join_reply = xzalloc(rcv.body_len);
ret = xread(sph_comm_fd, join_reply, rcv.body_len);
if (ret != rcv.body_len) {
sd_eprintf("xread() failed: %m");
exit(1);
}
sd_iprintf("join reply arrived, nr_nodes: %d", join_reply->nr_nodes);
if (join_reply->res == CJ_RES_MASTER_TRANSFER) {
is_master = true;
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
nr_nodes = 1;
nodes[0] = this_node;
} else {
memcpy(nodes, join_reply->nodes,
join_reply->nr_nodes * sizeof(struct sd_node));
nr_nodes = join_reply->nr_nodes;
}
sd_join_handler(&this_node, nodes, nr_nodes,
join_reply->res, join_reply->opaque);
free(join_reply);
sd_iprintf("shepherd_join() succeed");
state = STATE_JOINED;
}
/* Returns true if an event is processed */
static bool local_process_event(void)
{
struct local_event *ev;
enum cluster_join_result res;
int i;
struct sd_node nodes[SD_MAX_NODES];
size_t nr_nodes;
ev = shm_queue_peek();
if (!ev)
return false;
sd_dprintf("type = %d, sender = %s", ev->type,
lnode_to_str(&ev->sender));
sd_dprintf("callbacked = %d, removed = %d", ev->callbacked,
ev->removed);
nr_nodes = 0;
for (i = 0; i < ev->nr_lnodes; i++) {
sd_dprintf("%d: %s", i, lnode_to_str(ev->lnodes + i));
if (!ev->lnodes[i].gateway)
nodes[nr_nodes++] = ev->lnodes[i].node;
}
if (ev->removed)
goto out;
if (ev->callbacked)
return false; /* wait for unblock event */
if (ev->type == EVENT_JOIN_RESPONSE &&
lnode_eq(&this_node, &ev->sender)) {
sd_dprintf("join Sheepdog");
joined = true;
}
if (!joined) {
if (ev->type == EVENT_JOIN_REQUEST &&
lnode_eq(&this_node, &ev->sender)) {
struct local_node lnodes[SD_MAX_NODES];
get_nodes(lnodes);
if (!lnode_eq(&this_node, &lnodes[0])) {
sd_dprintf("wait for another node"
" to accept this node");
return false;
}
} else
goto out;
}
switch (ev->type) {
case EVENT_JOIN_REQUEST:
res = sd_check_join_cb(&ev->sender.node, ev->buf);
ev->join_result = res;
ev->type = EVENT_JOIN_RESPONSE;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
if (res == CJ_RES_MASTER_TRANSFER) {
sd_eprintf("failed to join sheepdog cluster: "
"please retry when master is up");
shm_queue_unlock();
exit(1);
}
return false;
case EVENT_JOIN_RESPONSE:
if (ev->join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev->nr_lnodes = 1;
ev->lnodes[0] = this_node;
nr_nodes = 1;
nodes[0] = this_node.node;
msync(ev, sizeof(*ev), MS_SYNC);
}
sd_join_handler(&ev->sender.node, nodes, nr_nodes,
ev->join_result, ev->buf);
break;
case EVENT_LEAVE:
case EVENT_GATEWAY:
sd_leave_handler(&ev->sender.node, nodes, nr_nodes);
break;
case EVENT_BLOCK:
ev->callbacked = sd_block_handler(&ev->sender.node);
msync(ev, sizeof(*ev), MS_SYNC);
return false;
case EVENT_NOTIFY:
sd_notify_handler(&ev->sender.node, ev->buf, ev->buf_len);
break;
}
out:
shm_queue_remove(ev);
return true;
}
static void acrd_handler(int listen_fd, int events, void *data)
{
int ret;
eventfd_t value;
struct acrd_event ev;
enum cluster_join_result res;
if (events & EPOLLHUP) {
eprintf("accord driver received EPOLLHUP event, exiting.\n");
log_close();
exit(1);
}
dprintf("read event\n");
ret = eventfd_read(efd, &value);
if (ret < 0)
return;
pthread_mutex_lock(&queue_lock);
ret = acrd_queue_pop(ahandle, &ev);
if (ret < 0)
goto out;
switch (ev.type) {
case EVENT_JOIN_REQUEST:
if (!node_eq(&ev.nodes[0], &this_node)) {
acrd_queue_push_back(ahandle, NULL);
break;
}
res = sd_check_join_cb(&ev.sender, ev.buf);
ev.join_result = res;
ev.type = EVENT_JOIN_RESPONSE;
acrd_queue_push_back(ahandle, &ev);
if (res == CJ_RES_MASTER_TRANSFER) {
eprintf("failed to join sheepdog cluster: "
"please retry when master is up\n");
exit(1);
}
break;
case EVENT_JOIN_RESPONSE:
if (ev.join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev.nr_nodes = 1;
ev.nodes[0] = this_node;
ev.ids[0] = this_id;
acrd_queue_push_back(ahandle, &ev);
acrd_queue_pop(ahandle, &ev);
}
sd_join_handler(&ev.sender, ev.nodes, ev.nr_nodes,
ev.join_result, ev.buf);
break;
case EVENT_LEAVE:
sd_leave_handler(&ev.sender, ev.nodes, ev.nr_nodes);
break;
case EVENT_BLOCK:
acrd_queue_push_back(ahandle, NULL);
sd_block_handler(&ev.sender);
break;
case EVENT_NOTIFY:
sd_notify_handler(&ev.sender, ev.buf, ev.buf_len);
break;
}
out:
pthread_mutex_unlock(&queue_lock);
}
/* Returns true if an event is processed */
static bool local_process_event(void)
{
struct local_event *ev;
int i;
struct sd_node nodes[SD_MAX_NODES];
size_t nr_nodes;
ev = shm_queue_peek();
if (!ev)
return false;
sd_debug("type = %d, sender = %s", ev->type, lnode_to_str(&ev->sender));
sd_debug("callbacked = %d, removed = %d", ev->callbacked, ev->removed);
nr_nodes = 0;
for (i = 0; i < ev->nr_lnodes; i++) {
sd_debug("%d: %s", i, lnode_to_str(ev->lnodes + i));
if (!ev->lnodes[i].gateway)
nodes[nr_nodes++] = ev->lnodes[i].node;
}
if (ev->removed)
goto out;
if (ev->callbacked)
return false; /* wait for unblock event */
if (!joined) {
if (!lnode_eq(&this_node, &ev->sender))
goto out;
switch (ev->type) {
case EVENT_JOIN:
break;
case EVENT_ACCEPT:
sd_debug("join Sheepdog");
joined = true;
break;
default:
goto out;
}
}
switch (ev->type) {
case EVENT_JOIN:
/* nodes[nr_nodes - 1] is a sender, so don't include it */
assert(node_eq(&ev->sender.node, &nodes[nr_nodes - 1]));
if (sd_join_handler(&ev->sender.node, nodes, nr_nodes - 1,
ev->buf)) {
ev->type = EVENT_ACCEPT;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
}
return false;
case EVENT_ACCEPT:
sd_accept_handler(&ev->sender.node, nodes, nr_nodes, ev->buf);
break;
case EVENT_LEAVE:
if (ev->sender.gateway) {
sd_debug("gateway %s left sheepdog",
lnode_to_str(&ev->sender));
break;
}
/* fall through */
case EVENT_GATEWAY:
sd_leave_handler(&ev->sender.node, nodes, nr_nodes);
break;
case EVENT_BLOCK:
ev->callbacked = sd_block_handler(&ev->sender.node);
msync(ev, sizeof(*ev), MS_SYNC);
return false;
case EVENT_NOTIFY:
sd_notify_handler(&ev->sender.node, ev->buf, ev->buf_len);
break;
case EVENT_UPDATE_NODE:
if (lnode_eq(&ev->sender, &this_node))
this_node = ev->sender;
sd_update_node_handler(&ev->sender.node);
break;
}
out:
shm_queue_remove(ev);
return true;
}
