static void shm_queue_init(void)
{
int ret;
shmfd = open(shmfile, O_CREAT | O_RDWR, 0644);
if (shmfd < 0)
panic("cannot open shared file, %s\n", shmfile);
shm_queue_lock();
ret = ftruncate(shmfd, sizeof(*shm_queue));
assert(ret == 0);
shm_queue = mmap(0, sizeof(*shm_queue),
PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0);
assert(shm_queue != MAP_FAILED);
if (is_shm_queue_valid())
event_pos = shm_queue->pos;
else {
/* initialize shared memory */
event_pos = 0;
ret = ftruncate(shmfd, 0);
assert(ret == 0);
ret = ftruncate(shmfd, sizeof(*shm_queue));
assert(ret == 0);
shm_queue_set_chksum();
}
shm_queue_unlock();
}
static void check_pids(void *arg)
{
int i;
size_t nr;
struct local_node lnodes[SD_MAX_NODES];
struct local_event *ev;
shm_queue_lock();
nr = get_nodes(lnodes);
for (i = 0; i < nr; i++)
if (!process_exists(lnodes[i].pid)) {
add_event(EVENT_LEAVE, lnodes + i, NULL, 0);
/* unblock blocking event if sender has gone */
ev = shm_queue_peek_block_event();
if (lnode_eq(lnodes + i, &ev->sender)) {
ev->removed = true;
msync(ev, sizeof(*ev), MS_SYNC);
}
}
shm_queue_unlock();
add_timer(arg, PROCESS_CHECK_INTERVAL);
}
static void local_block(void)
{
shm_queue_lock();
add_event(EVENT_BLOCK, &this_node, NULL, 0);
shm_queue_unlock();
}
/*
* Returns true if an event is processed
*/
static bool local_process_event(void)
{
struct local_event *ev;
enum cluster_join_result res;
ev = shm_queue_peek();
if (!ev)
return false;
switch (ev->type) {
case EVENT_JOIN_REQUEST:
if (!node_eq(&ev->nodes[0], &this_node))
return false;
res = sd_check_join_cb(&ev->sender, ev->buf);
ev->join_result = res;
ev->type = EVENT_JOIN_RESPONSE;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
if (res == CJ_RES_MASTER_TRANSFER) {
eprintf("failed to join sheepdog cluster: "
"please retry when master is up\n");
shm_queue_unlock();
exit(1);
}
return false;
case EVENT_JOIN_RESPONSE:
if (ev->join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev->nr_nodes = 1;
ev->nodes[0] = this_node;
ev->pids[0] = getpid();
shm_queue_set_chksum();
}
sd_join_handler(&ev->sender, ev->nodes, ev->nr_nodes,
ev->join_result, ev->buf);
shm_queue_pop();
break;
case EVENT_LEAVE:
sd_leave_handler(&ev->sender, ev->nodes, ev->nr_nodes);
shm_queue_pop();
break;
case EVENT_BLOCK:
sd_block_handler(&ev->sender);
return false;
case EVENT_NOTIFY:
sd_notify_handler(&ev->sender, ev->buf, ev->buf_len);
shm_queue_pop();
break;
}
return true;
}
static int local_leave(void)
{
shm_queue_lock();
add_event(EVENT_LEAVE, &this_node, NULL, 0);
shm_queue_unlock();
return 0;
}
static int add_event_lock(enum local_event_type type, struct local_node *lnode,
void *buf, size_t buf_len)
{
int ret;
shm_queue_lock();
ret = add_event(type, lnode, buf, buf_len);
shm_queue_unlock();
return ret;
}
static int local_notify(void *msg, size_t msg_len)
{
shm_queue_lock();
add_event(EVENT_NOTIFY, &this_node, msg, msg_len);
shm_queue_unlock();
return 0;
}
/* FIXME: we have to call nr of nodes times to update nodes information */
static void local_update_node(struct sd_node *node)
{
struct local_node n = {
.node = *node,
};
shm_queue_lock();
add_event(EVENT_UPDATE_NODE, &n, NULL, 0);
shm_queue_unlock();
}
static int local_join(struct sd_node *myself,
void *opaque, size_t opaque_len)
{
this_node = *myself;
shm_queue_lock();
add_event(EVENT_JOIN_REQUEST, &this_node, opaque, opaque_len);
shm_queue_unlock();
return 0;
}
static void local_unblock(void *msg, size_t msg_len)
{
struct local_event *ev;
shm_queue_lock();
ev = shm_queue_peek_block_event();
ev->removed = true;
msync(ev, sizeof(*ev), MS_SYNC);
add_event(EVENT_NOTIFY, &this_node, msg, msg_len);
shm_queue_unlock();
}
static int local_join(const struct sd_node *myself,
void *opaque, size_t opaque_len)
{
this_node.node = *myself;
this_node.pid = getpid();
this_node.gateway = false;
shm_queue_lock();
add_event(EVENT_JOIN_REQUEST, &this_node, opaque, opaque_len);
shm_queue_unlock();
return 0;
}
static void local_block(struct work *work, int idx)
{
struct local_event *ev;
shm_queue_lock();
ev = shm_queue_peek();
ev->block_cb(ev->buf);
ev->blocked = 0;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
shm_queue_unlock();
}
static int local_join(struct sheepdog_node_list_entry *myself,
enum cluster_join_result (*check_join_cb)(
struct sheepdog_node_list_entry *joining,
void *opaque),
void *opaque, size_t opaque_len)
{
this_node = *myself;
local_check_join_cb = check_join_cb;
shm_queue_lock();
add_event(EVENT_JOIN, &this_node, opaque, opaque_len, NULL);
shm_queue_unlock();
return 0;
}
static void local_unblock(void *msg, size_t msg_len)
{
struct local_event *ev;
shm_queue_lock();
ev = shm_queue_peek();
ev->type = EVENT_NOTIFY;
ev->buf_len = msg_len;
if (msg)
memcpy(ev->buf, msg, msg_len);
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
shm_queue_unlock();
}
static void check_pids(void *arg)
{
int i;
size_t nr;
struct sd_node nodes[SD_MAX_NODES];
pid_t pids[SD_MAX_NODES];
shm_queue_lock();
nr = get_nodes(nodes, pids);
for (i = 0; i < nr; i++)
if (!process_exists(pids[i]))
add_event(EVENT_LEAVE, nodes + i, NULL, 0);
shm_queue_unlock();
add_timer(arg, 1);
}
static void local_handler(int listen_fd, int events, void *data)
{
struct signalfd_siginfo siginfo;
int ret;
if (events & EPOLLHUP) {
eprintf("local driver received EPOLLHUP event, exiting.\n");
log_close();
exit(1);
}
dprintf("read siginfo\n");
ret = read(sigfd, &siginfo, sizeof(siginfo));
assert(ret == sizeof(siginfo));
shm_queue_lock();
while (local_process_event())
;
shm_queue_unlock();
}
static void shm_queue_init(void)
{
int ret;
shmfd = open(shmfile, O_CREAT | O_RDWR, 0644);
if (shmfd < 0)
panic("cannot open shared file, %s", shmfile);
shm_queue_lock();
ret = xftruncate(shmfd, sizeof(*shm_queue));
if (ret != 0)
panic("failed to truncate shmfile, %m");
shm_queue = mmap(NULL, sizeof(*shm_queue),
PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0);
if (shm_queue == MAP_FAILED)
panic("mmap error, %m");
if (is_shm_queue_valid()) {
block_event_pos = shm_queue->block_event_pos;
nonblock_event_pos = shm_queue->nonblock_event_pos;
} else {
/* initialize shared memory */
block_event_pos = 0;
nonblock_event_pos = 0;
ret = xftruncate(shmfd, 0);
if (ret != 0)
panic("failed to truncate shmfile, %m");
ret = xftruncate(shmfd, sizeof(*shm_queue));
if (ret != 0)
panic("failed to truncate shmfile, %m");
}
shm_queue_unlock();
}
static void local_handler(int listen_fd, int events, void *data)
{
struct signalfd_siginfo siginfo;
int ret;
if (events & EPOLLHUP) {
sd_err("local driver received EPOLLHUP event, exiting.");
log_close();
exit(1);
}
sd_debug("read siginfo");
ret = read(sigfd, &siginfo, sizeof(siginfo));
if (ret != sizeof(siginfo))
panic("failed to read from sigfd, %m");
shm_queue_lock();
while (local_process_event())
;
shm_queue_unlock();
}
/* Returns true if an event is processed */
static bool local_process_event(void)
{
struct local_event *ev;
enum cluster_join_result res;
int i;
struct sd_node nodes[SD_MAX_NODES];
size_t nr_nodes;
ev = shm_queue_peek();
if (!ev)
return false;
sd_dprintf("type = %d, sender = %s", ev->type,
lnode_to_str(&ev->sender));
sd_dprintf("callbacked = %d, removed = %d", ev->callbacked,
ev->removed);
nr_nodes = 0;
for (i = 0; i < ev->nr_lnodes; i++) {
sd_dprintf("%d: %s", i, lnode_to_str(ev->lnodes + i));
if (!ev->lnodes[i].gateway)
nodes[nr_nodes++] = ev->lnodes[i].node;
}
if (ev->removed)
goto out;
if (ev->callbacked)
return false; /* wait for unblock event */
if (ev->type == EVENT_JOIN_RESPONSE &&
lnode_eq(&this_node, &ev->sender)) {
sd_dprintf("join Sheepdog");
joined = true;
}
if (!joined) {
if (ev->type == EVENT_JOIN_REQUEST &&
lnode_eq(&this_node, &ev->sender)) {
struct local_node lnodes[SD_MAX_NODES];
get_nodes(lnodes);
if (!lnode_eq(&this_node, &lnodes[0])) {
sd_dprintf("wait for another node"
" to accept this node");
return false;
}
} else
goto out;
}
switch (ev->type) {
case EVENT_JOIN_REQUEST:
res = sd_check_join_cb(&ev->sender.node, ev->buf);
ev->join_result = res;
ev->type = EVENT_JOIN_RESPONSE;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
if (res == CJ_RES_MASTER_TRANSFER) {
sd_eprintf("failed to join sheepdog cluster: "
"please retry when master is up");
shm_queue_unlock();
exit(1);
}
return false;
case EVENT_JOIN_RESPONSE:
if (ev->join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev->nr_lnodes = 1;
ev->lnodes[0] = this_node;
nr_nodes = 1;
nodes[0] = this_node.node;
msync(ev, sizeof(*ev), MS_SYNC);
}
sd_join_handler(&ev->sender.node, nodes, nr_nodes,
ev->join_result, ev->buf);
break;
case EVENT_LEAVE:
case EVENT_GATEWAY:
sd_leave_handler(&ev->sender.node, nodes, nr_nodes);
break;
case EVENT_BLOCK:
ev->callbacked = sd_block_handler(&ev->sender.node);
msync(ev, sizeof(*ev), MS_SYNC);
return false;
case EVENT_NOTIFY:
sd_notify_handler(&ev->sender.node, ev->buf, ev->buf_len);
break;
}
out:
shm_queue_remove(ev);
return true;
}
static int local_dispatch(void)
{
int ret;
struct signalfd_siginfo siginfo;
struct local_event *ev;
enum cluster_join_result res;
static struct work work = {
.fn = local_block,
.done = local_block_done,
};
dprintf("read siginfo\n");
ret = read(sigfd, &siginfo, sizeof(siginfo));
assert(ret == sizeof(siginfo));
shm_queue_lock();
ev = shm_queue_peek();
if (!ev)
goto out;
switch (ev->type) {
case EVENT_JOIN:
if (ev->blocked) {
if (node_cmp(&ev->nodes[0], &this_node) == 0) {
res = local_check_join_cb(&ev->sender, ev->buf);
ev->join_result = res;
ev->blocked = 0;
msync(ev, sizeof(*ev), MS_SYNC);
shm_queue_notify();
if (res == CJ_RES_MASTER_TRANSFER) {
eprintf("failed to join sheepdog cluster: please retry when master is up\n");
shm_queue_unlock();
exit(1);
}
}
goto out;
}
if (ev->join_result == CJ_RES_MASTER_TRANSFER) {
/* FIXME: This code is tricky, but Sheepdog assumes that */
/* nr_nodes = 1 when join_result = MASTER_TRANSFER... */
ev->nr_nodes = 1;
ev->nodes[0] = this_node;
ev->pids[0] = getpid();
shm_queue_set_chksum();
}
lhdlrs.join_handler(&ev->sender, ev->nodes, ev->nr_nodes,
ev->join_result, ev->buf);
break;
case EVENT_LEAVE:
lhdlrs.leave_handler(&ev->sender, ev->nodes, ev->nr_nodes);
break;
case EVENT_NOTIFY:
if (ev->blocked) {
if (node_cmp(&ev->sender, &this_node) == 0) {
if (!ev->callbacked) {
queue_work(local_block_wq, &work);
ev->callbacked = 1;
}
}
goto out;
}
lhdlrs.notify_handler(&ev->sender, ev->buf, ev->buf_len);
break;
}
shm_queue_pop();
out:
shm_queue_unlock();
return 0;
}
