/*
* Pass on a notification message from the cluster driver.
*
* Must run in the main thread as it accesses unlocked state like
* sys->pending_list.
*/
void sd_notify_handler(struct sd_node *sender, void *data, size_t data_len)
{
struct vdi_op_message *msg = data;
struct sd_op_template *op = get_sd_op(msg->req.opcode);
int ret = msg->rsp.result;
struct request *req = NULL;
dprintf("op %s, size: %zd, from: %s\n",
op_name(op), data_len, node_to_str(sender));
if (is_myself(sender->nid.addr, sender->nid.port)) {
req = list_first_entry(&sys->pending_list, struct request,
pending_list);
list_del(&req->pending_list);
}
/*
* Pass on a notification message from the cluster driver.
*
* Must run in the main thread as it accesses unlocked state like
* sys->pending_list.
*/
void sd_notify_handler(const struct sd_node *sender, void *data,
size_t data_len)
{
struct vdi_op_message *msg = data;
const struct sd_op_template *op = get_sd_op(msg->req.opcode);
int ret = msg->rsp.result;
struct request *req = NULL;
sd_dprintf("op %s, size: %zd, from: %s\n",
op_name(op), data_len, node_to_str(sender));
if (node_is_local(sender)) {
if (has_process_work(op))
req = list_first_entry(&sys->pending_block_list,
struct request, pending_list);
else
req = list_first_entry(&sys->pending_notify_list,
struct request, pending_list);
list_del(&req->pending_list);
}
static void __sd_notify_done(struct event_struct *cevent)
{
struct work_notify *w = container_of(cevent, struct work_notify, cev);
struct vdi_op_message *msg = w->msg;
struct request *req = w->req;
int ret = msg->rsp.result;
struct sd_op_template *op = get_sd_op(msg->req.opcode);
if (ret == SD_RES_SUCCESS && has_process_main(op))
ret = do_process_main(op, (const struct sd_req *)&msg->req,
(struct sd_rsp *)&msg->rsp, msg->data);
if (!req)
return;
msg->rsp.result = ret;
if (has_process_main(req->op))
memcpy(req->data, msg->data, msg->rsp.data_length);
memcpy(&req->rp, &msg->rsp, sizeof(req->rp));
req->done(req);
}
static int gateway_forward_request(struct request *req, bool all_node)
{
int i, err_ret = SD_RES_SUCCESS, ret, local = -1;
unsigned wlen;
uint64_t oid = req->rq.obj.oid;
int nr_to_send;
struct write_info wi;
struct sd_op_template *op;
struct sd_req hdr;
struct sd_node *target_nodes[SD_MAX_COPIES];
dprintf("%"PRIx64"\n", oid);
gateway_init_fwd_hdr(&hdr, &req->rq);
op = get_sd_op(hdr.opcode);
write_info_init(&wi);
wlen = hdr.data_length;
nr_to_send = init_target_nodes(req, all_node, oid, target_nodes);
for (i = 0; i < nr_to_send; i++) {
struct sockfd *sfd;
struct node_id *nid;
if (node_is_local(target_nodes[i])) {
local = i;
continue;
}
nid = &target_nodes[i]->nid;
sfd = sheep_get_sockfd(nid);
if (!sfd) {
err_ret = SD_RES_NETWORK_ERROR;
break;
}
ret = send_req(sfd->fd, &hdr, req->data, &wlen);
if (ret) {
sheep_del_sockfd(nid, sfd);
err_ret = SD_RES_NETWORK_ERROR;
dprintf("fail %d\n", ret);
break;
}
write_info_advance(&wi, nid, sfd);
}
if (local != -1 && err_ret == SD_RES_SUCCESS) {
assert(op);
ret = sheep_do_op_work(op, req);
if (ret != SD_RES_SUCCESS) {
eprintf("fail to write local %"PRIx32"\n", ret);
err_ret = ret;
}
}
dprintf("nr_sent %d, err %x\n", wi.nr_sent, err_ret);
if (wi.nr_sent > 0) {
ret = wait_forward_request(&wi, req);
if (ret != SD_RES_SUCCESS)
err_ret = ret;
}
return err_ret;
}
static int gateway_forward_request(struct request *req)
{
int i, err_ret = SD_RES_SUCCESS, ret, local = -1;
unsigned wlen;
uint64_t oid = req->rq.obj.oid;
int nr_to_send;
struct write_info wi;
const struct sd_op_template *op;
struct sd_req hdr;
const struct sd_node *target_nodes[SD_MAX_NODES];
sd_debug("%"PRIx64, oid);
gateway_init_fwd_hdr(&hdr, &req->rq);
op = get_sd_op(hdr.opcode);
wlen = hdr.data_length;
nr_to_send = init_target_nodes(req, oid, target_nodes);
write_info_init(&wi, nr_to_send);
if (nr_to_send == 0) {
sd_debug("there is no living nodes");
return SD_RES_HALT;
}
for (i = 0; i < nr_to_send; i++) {
struct sockfd *sfd;
const struct node_id *nid;
if (node_is_local(target_nodes[i])) {
local = i;
continue;
}
nid = &target_nodes[i]->nid;
sfd = sockfd_cache_get(nid);
if (!sfd) {
err_ret = SD_RES_NETWORK_ERROR;
break;
}
ret = send_req(sfd->fd, &hdr, req->data, wlen,
sheep_need_retry, req->rq.epoch,
MAX_RETRY_COUNT);
if (ret) {
sockfd_cache_del_node(nid);
err_ret = SD_RES_NETWORK_ERROR;
sd_debug("fail %d", ret);
break;
}
write_info_advance(&wi, nid, sfd);
}
if (local != -1 && err_ret == SD_RES_SUCCESS) {
assert(op);
ret = sheep_do_op_work(op, req);
if (ret != SD_RES_SUCCESS) {
sd_err("fail to write local %"PRIx64", %s", oid,
sd_strerror(ret));
err_ret = ret;
}
}
sd_debug("nr_sent %d, err %x", wi.nr_sent, err_ret);
if (wi.nr_sent > 0) {
ret = wait_forward_request(&wi, req);
if (ret != SD_RES_SUCCESS)
err_ret = ret;
}
return err_ret;
}