static void shm_queue_init(void) { int ret; shmfd = open(shmfile, O_CREAT | O_RDWR, 0644); if (shmfd < 0) panic("cannot open shared file, %s\n", shmfile); shm_queue_lock(); ret = ftruncate(shmfd, sizeof(*shm_queue)); assert(ret == 0); shm_queue = mmap(0, sizeof(*shm_queue), PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); assert(shm_queue != MAP_FAILED); if (is_shm_queue_valid()) event_pos = shm_queue->pos; else { /* initialize shared memory */ event_pos = 0; ret = ftruncate(shmfd, 0); assert(ret == 0); ret = ftruncate(shmfd, sizeof(*shm_queue)); assert(ret == 0); shm_queue_set_chksum(); } shm_queue_unlock(); }
static void check_pids(void *arg) { int i; size_t nr; struct local_node lnodes[SD_MAX_NODES]; struct local_event *ev; shm_queue_lock(); nr = get_nodes(lnodes); for (i = 0; i < nr; i++) if (!process_exists(lnodes[i].pid)) { add_event(EVENT_LEAVE, lnodes + i, NULL, 0); /* unblock blocking event if sender has gone */ ev = shm_queue_peek_block_event(); if (lnode_eq(lnodes + i, &ev->sender)) { ev->removed = true; msync(ev, sizeof(*ev), MS_SYNC); } } shm_queue_unlock(); add_timer(arg, PROCESS_CHECK_INTERVAL); }
static void local_block(void) { shm_queue_lock(); add_event(EVENT_BLOCK, &this_node, NULL, 0); shm_queue_unlock(); }
/* * Returns true if an event is processed */ static bool local_process_event(void) { struct local_event *ev; enum cluster_join_result res; ev = shm_queue_peek(); if (!ev) return false; switch (ev->type) { case EVENT_JOIN_REQUEST: if (!node_eq(&ev->nodes[0], &this_node)) return false; res = sd_check_join_cb(&ev->sender, ev->buf); ev->join_result = res; ev->type = EVENT_JOIN_RESPONSE; msync(ev, sizeof(*ev), MS_SYNC); shm_queue_notify(); if (res == CJ_RES_MASTER_TRANSFER) { eprintf("failed to join sheepdog cluster: " "please retry when master is up\n"); shm_queue_unlock(); exit(1); } return false; case EVENT_JOIN_RESPONSE: if (ev->join_result == CJ_RES_MASTER_TRANSFER) { /* FIXME: This code is tricky, but Sheepdog assumes that */ /* nr_nodes = 1 when join_result = MASTER_TRANSFER... */ ev->nr_nodes = 1; ev->nodes[0] = this_node; ev->pids[0] = getpid(); shm_queue_set_chksum(); } sd_join_handler(&ev->sender, ev->nodes, ev->nr_nodes, ev->join_result, ev->buf); shm_queue_pop(); break; case EVENT_LEAVE: sd_leave_handler(&ev->sender, ev->nodes, ev->nr_nodes); shm_queue_pop(); break; case EVENT_BLOCK: sd_block_handler(&ev->sender); return false; case EVENT_NOTIFY: sd_notify_handler(&ev->sender, ev->buf, ev->buf_len); shm_queue_pop(); break; } return true; }
static int local_leave(void) { shm_queue_lock(); add_event(EVENT_LEAVE, &this_node, NULL, 0); shm_queue_unlock(); return 0; }
static int add_event_lock(enum local_event_type type, struct local_node *lnode, void *buf, size_t buf_len) { int ret; shm_queue_lock(); ret = add_event(type, lnode, buf, buf_len); shm_queue_unlock(); return ret; }
static int local_notify(void *msg, size_t msg_len) { shm_queue_lock(); add_event(EVENT_NOTIFY, &this_node, msg, msg_len); shm_queue_unlock(); return 0; }
/* FIXME: we have to call nr of nodes times to update nodes information */ static void local_update_node(struct sd_node *node) { struct local_node n = { .node = *node, }; shm_queue_lock(); add_event(EVENT_UPDATE_NODE, &n, NULL, 0); shm_queue_unlock(); }
static int local_join(struct sd_node *myself, void *opaque, size_t opaque_len) { this_node = *myself; shm_queue_lock(); add_event(EVENT_JOIN_REQUEST, &this_node, opaque, opaque_len); shm_queue_unlock(); return 0; }
static void local_unblock(void *msg, size_t msg_len) { struct local_event *ev; shm_queue_lock(); ev = shm_queue_peek_block_event(); ev->removed = true; msync(ev, sizeof(*ev), MS_SYNC); add_event(EVENT_NOTIFY, &this_node, msg, msg_len); shm_queue_unlock(); }
static int local_join(const struct sd_node *myself, void *opaque, size_t opaque_len) { this_node.node = *myself; this_node.pid = getpid(); this_node.gateway = false; shm_queue_lock(); add_event(EVENT_JOIN_REQUEST, &this_node, opaque, opaque_len); shm_queue_unlock(); return 0; }
static void local_block(struct work *work, int idx) { struct local_event *ev; shm_queue_lock(); ev = shm_queue_peek(); ev->block_cb(ev->buf); ev->blocked = 0; msync(ev, sizeof(*ev), MS_SYNC); shm_queue_notify(); shm_queue_unlock(); }
static int local_join(struct sheepdog_node_list_entry *myself, enum cluster_join_result (*check_join_cb)( struct sheepdog_node_list_entry *joining, void *opaque), void *opaque, size_t opaque_len) { this_node = *myself; local_check_join_cb = check_join_cb; shm_queue_lock(); add_event(EVENT_JOIN, &this_node, opaque, opaque_len, NULL); shm_queue_unlock(); return 0; }
static void local_unblock(void *msg, size_t msg_len) { struct local_event *ev; shm_queue_lock(); ev = shm_queue_peek(); ev->type = EVENT_NOTIFY; ev->buf_len = msg_len; if (msg) memcpy(ev->buf, msg, msg_len); msync(ev, sizeof(*ev), MS_SYNC); shm_queue_notify(); shm_queue_unlock(); }
static void check_pids(void *arg) { int i; size_t nr; struct sd_node nodes[SD_MAX_NODES]; pid_t pids[SD_MAX_NODES]; shm_queue_lock(); nr = get_nodes(nodes, pids); for (i = 0; i < nr; i++) if (!process_exists(pids[i])) add_event(EVENT_LEAVE, nodes + i, NULL, 0); shm_queue_unlock(); add_timer(arg, 1); }
static void local_handler(int listen_fd, int events, void *data) { struct signalfd_siginfo siginfo; int ret; if (events & EPOLLHUP) { eprintf("local driver received EPOLLHUP event, exiting.\n"); log_close(); exit(1); } dprintf("read siginfo\n"); ret = read(sigfd, &siginfo, sizeof(siginfo)); assert(ret == sizeof(siginfo)); shm_queue_lock(); while (local_process_event()) ; shm_queue_unlock(); }
static void shm_queue_init(void) { int ret; shmfd = open(shmfile, O_CREAT | O_RDWR, 0644); if (shmfd < 0) panic("cannot open shared file, %s", shmfile); shm_queue_lock(); ret = xftruncate(shmfd, sizeof(*shm_queue)); if (ret != 0) panic("failed to truncate shmfile, %m"); shm_queue = mmap(NULL, sizeof(*shm_queue), PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); if (shm_queue == MAP_FAILED) panic("mmap error, %m"); if (is_shm_queue_valid()) { block_event_pos = shm_queue->block_event_pos; nonblock_event_pos = shm_queue->nonblock_event_pos; } else { /* initialize shared memory */ block_event_pos = 0; nonblock_event_pos = 0; ret = xftruncate(shmfd, 0); if (ret != 0) panic("failed to truncate shmfile, %m"); ret = xftruncate(shmfd, sizeof(*shm_queue)); if (ret != 0) panic("failed to truncate shmfile, %m"); } shm_queue_unlock(); }
static void local_handler(int listen_fd, int events, void *data) { struct signalfd_siginfo siginfo; int ret; if (events & EPOLLHUP) { sd_err("local driver received EPOLLHUP event, exiting."); log_close(); exit(1); } sd_debug("read siginfo"); ret = read(sigfd, &siginfo, sizeof(siginfo)); if (ret != sizeof(siginfo)) panic("failed to read from sigfd, %m"); shm_queue_lock(); while (local_process_event()) ; shm_queue_unlock(); }
/* Returns true if an event is processed */ static bool local_process_event(void) { struct local_event *ev; enum cluster_join_result res; int i; struct sd_node nodes[SD_MAX_NODES]; size_t nr_nodes; ev = shm_queue_peek(); if (!ev) return false; sd_dprintf("type = %d, sender = %s", ev->type, lnode_to_str(&ev->sender)); sd_dprintf("callbacked = %d, removed = %d", ev->callbacked, ev->removed); nr_nodes = 0; for (i = 0; i < ev->nr_lnodes; i++) { sd_dprintf("%d: %s", i, lnode_to_str(ev->lnodes + i)); if (!ev->lnodes[i].gateway) nodes[nr_nodes++] = ev->lnodes[i].node; } if (ev->removed) goto out; if (ev->callbacked) return false; /* wait for unblock event */ if (ev->type == EVENT_JOIN_RESPONSE && lnode_eq(&this_node, &ev->sender)) { sd_dprintf("join Sheepdog"); joined = true; } if (!joined) { if (ev->type == EVENT_JOIN_REQUEST && lnode_eq(&this_node, &ev->sender)) { struct local_node lnodes[SD_MAX_NODES]; get_nodes(lnodes); if (!lnode_eq(&this_node, &lnodes[0])) { sd_dprintf("wait for another node" " to accept this node"); return false; } } else goto out; } switch (ev->type) { case EVENT_JOIN_REQUEST: res = sd_check_join_cb(&ev->sender.node, ev->buf); ev->join_result = res; ev->type = EVENT_JOIN_RESPONSE; msync(ev, sizeof(*ev), MS_SYNC); shm_queue_notify(); if (res == CJ_RES_MASTER_TRANSFER) { sd_eprintf("failed to join sheepdog cluster: " "please retry when master is up"); shm_queue_unlock(); exit(1); } return false; case EVENT_JOIN_RESPONSE: if (ev->join_result == CJ_RES_MASTER_TRANSFER) { /* FIXME: This code is tricky, but Sheepdog assumes that */ /* nr_nodes = 1 when join_result = MASTER_TRANSFER... */ ev->nr_lnodes = 1; ev->lnodes[0] = this_node; nr_nodes = 1; nodes[0] = this_node.node; msync(ev, sizeof(*ev), MS_SYNC); } sd_join_handler(&ev->sender.node, nodes, nr_nodes, ev->join_result, ev->buf); break; case EVENT_LEAVE: case EVENT_GATEWAY: sd_leave_handler(&ev->sender.node, nodes, nr_nodes); break; case EVENT_BLOCK: ev->callbacked = sd_block_handler(&ev->sender.node); msync(ev, sizeof(*ev), MS_SYNC); return false; case EVENT_NOTIFY: sd_notify_handler(&ev->sender.node, ev->buf, ev->buf_len); break; } out: shm_queue_remove(ev); return true; }
static int local_dispatch(void) { int ret; struct signalfd_siginfo siginfo; struct local_event *ev; enum cluster_join_result res; static struct work work = { .fn = local_block, .done = local_block_done, }; dprintf("read siginfo\n"); ret = read(sigfd, &siginfo, sizeof(siginfo)); assert(ret == sizeof(siginfo)); shm_queue_lock(); ev = shm_queue_peek(); if (!ev) goto out; switch (ev->type) { case EVENT_JOIN: if (ev->blocked) { if (node_cmp(&ev->nodes[0], &this_node) == 0) { res = local_check_join_cb(&ev->sender, ev->buf); ev->join_result = res; ev->blocked = 0; msync(ev, sizeof(*ev), MS_SYNC); shm_queue_notify(); if (res == CJ_RES_MASTER_TRANSFER) { eprintf("failed to join sheepdog cluster: please retry when master is up\n"); shm_queue_unlock(); exit(1); } } goto out; } if (ev->join_result == CJ_RES_MASTER_TRANSFER) { /* FIXME: This code is tricky, but Sheepdog assumes that */ /* nr_nodes = 1 when join_result = MASTER_TRANSFER... */ ev->nr_nodes = 1; ev->nodes[0] = this_node; ev->pids[0] = getpid(); shm_queue_set_chksum(); } lhdlrs.join_handler(&ev->sender, ev->nodes, ev->nr_nodes, ev->join_result, ev->buf); break; case EVENT_LEAVE: lhdlrs.leave_handler(&ev->sender, ev->nodes, ev->nr_nodes); break; case EVENT_NOTIFY: if (ev->blocked) { if (node_cmp(&ev->sender, &this_node) == 0) { if (!ev->callbacked) { queue_work(local_block_wq, &work); ev->callbacked = 1; } } goto out; } lhdlrs.notify_handler(&ev->sender, ev->buf, ev->buf_len); break; } shm_queue_pop(); out: shm_queue_unlock(); return 0; }