slotid_t get_newest_sid(groupid_t gid, int *is_me) {
apr_thread_mutex_lock(mx_newest_);
slotid_t *s = apr_hash_get(ht_newest_, &gid, sizeof(groupid_t));
apr_thread_mutex_unlock(mx_newest_);
slotid_t sid = (s == NULL) ? 0: *s;
instid_t iid;
memset(&iid, 0, sizeof(instid_t));
iid.gid = gid;
iid.sid = sid;
if (sid == 0) {
// never seen value before.
*is_me = 0;
} else {
apr_thread_mutex_lock(mx_value_);
proposal_t *prop = apr_hash_get(ht_value_, &iid, sizeof(instid_t));
apr_thread_mutex_unlock(mx_value_);
SAFE_ASSERT(prop != NULL);
*is_me = (prop->nid == get_local_nid()) ? 1 : 0;
}
LOG_DEBUG("newest sid %lu for gid %u", sid, gid);
return sid;
}
static int string_copy_to(vm_thread_t *thread, obj_t *odest, obj_t *opos, obj_t *osrc)
{
SAFE_ASSERT(IS_STRING(*odest));
SAFE_ASSERT(IS_HEAP_PTR(*odest));
SAFE_ASSERT(IS_STRING(*osrc));
SAFE_ASSERT(IS_FIXNUM(*opos));
string_t *src = PTR(*osrc);
string_t *dst = PTR(*odest);
SAFE_ASSERT(dst->allocated);
int pos = FIXNUM(*opos);
SAFE_ASSERT((pos >= 0)
&& (pos+src->size <= dst->size));
memcpy(&dst->str[pos], src->str, src->size-1);
RETURN_OBJ(cvoid);
}
rpc_state* handle_msg_prepare(const msg_prepare_t *p_msg_prep) {
SAFE_ASSERT(p_msg_prep->h->t == MPAXOS__MSG_HEADER__MSGTYPE_T__PREPARE);
// This is the msg_promise for response
msg_promise_t msg_prom = MPAXOS__MSG_PROMISE__INIT;
msg_header_t msg_header = MPAXOS__MSG_HEADER__INIT;
msg_prom.h = &msg_header;
msg_prom.h->t = MPAXOS__MSG_HEADER__MSGTYPE_T__PROMISE;
msg_prom.h->tid = p_msg_prep->h->tid;
msg_prom.h->nid = get_local_nid();
msg_prom.n_ress = 0;
msg_prom.ress = (response_t **)
calloc(p_msg_prep->n_rids, sizeof(response_t *));
for (int i = 0; i < p_msg_prep->n_rids; i++) {
roundid_t *rid = p_msg_prep->rids[i];
if (!is_in_group(rid->gid)) {
//Check for every requested group in the prepare message.
//Skip for the groups which I don't belong to.
continue;
}
// For those I belong to, add response.
response_t *p_res = (response_t *)
calloc(sizeof(response_t), 1);
msg_prom.ress[msg_prom.n_ress] = p_res;
mpaxos__response_t__init(p_res);
p_res->rid = rid;
accp_info_t* ainfo = get_accp_info(rid->gid, rid->sid);
SAFE_ASSERT(ainfo != NULL);
// lock on this accp info
apr_thread_mutex_lock(ainfo->mx);
// Check if the ballot id is larger.
// must be greater than. or must be prepared before by the same proposer (TODO)
if (rid->bid > ainfo->bid_max) {
p_res->ack = MPAXOS__ACK_ENUM__SUCCESS;
ainfo->bid_max = rid->bid;
LOG_DEBUG("prepare is ok. bid: %"PRIx64
", seen max bid: %"PRIx64, rid->bid, ainfo->bid_max);
} else if (rid->bid < ainfo->bid_max) {
p_res->ack = MPAXOS__ACK_ENUM__ERR_BID;
LOG_DEBUG("prepare is not ok. bid: %"PRIx64
", seen max bid: %"PRIx64, rid->bid, ainfo->bid_max);
} else {
LOG_DEBUG("receive an undefined bid. bid: %"PRIx64
", seen max bid: %"PRIx64, rid->bid, ainfo->bid_max);
SAFE_ASSERT(0);
}
// Check if already accepted any proposals.
// Add them to the response as well.
apr_array_header_t *arr_prop = ainfo->arr_prop;
p_res->n_props = arr_prop->nelts;
LOG_DEBUG("there is %d proposals i have aready got.", arr_prop->nelts);
p_res->props = (proposal_t**)
malloc(p_res->n_props * sizeof(proposal_t *));
for (int i = 0; i < p_res->n_props; i++) {
proposal_t *oldp = ((proposal_t **)arr_prop->elts)[i];
p_res->props[i] = oldp;
}
msg_prom.n_ress++;
apr_thread_mutex_unlock(ainfo->mx);
}
// Send back the promise message
size_t sz_msg = mpaxos__msg_promise__get_packed_size(&msg_prom);
log_message_res("send", "PROMISE", msg_prom.h, msg_prom.ress,
msg_prom.n_ress, sz_msg);
uint8_t *buf = (uint8_t *) malloc(sz_msg);
mpaxos__msg_promise__pack(&msg_prom, buf);
rpc_state *ret_state = (rpc_state*) malloc(sizeof(rpc_state));
ret_state->raw_output = buf;
ret_state->sz_output = sz_msg;
for (int i = 0; i < msg_prom.n_ress; i++) {
free(msg_prom.ress[i]->props);
free(msg_prom.ress[i]);
}
free(msg_prom.ress);
return ret_state;
}
rpc_state* handle_msg_accept(const msg_accept_t *msg_accp) {
SAFE_ASSERT(msg_accp->h->t == MPAXOS__MSG_HEADER__MSGTYPE_T__ACCEPT);
// This is the msg_accepted for response
msg_accepted_t msg_accd = MPAXOS__MSG_ACCEPTED__INIT;
msg_header_t msg_header = MPAXOS__MSG_HEADER__INIT;
msg_accd.h = &msg_header;
msg_accd.h->t = MPAXOS__MSG_HEADER__MSGTYPE_T__ACCEPTED;
msg_accd.h->tid = msg_accp->h->tid;
msg_accd.h->nid = get_local_nid();
msg_accd.n_ress = 0;
msg_accd.ress = (response_t **)
malloc(msg_accp->prop->n_rids * sizeof(response_t *));
for (int i = 0; i < msg_accp->prop->n_rids; i++) {
roundid_t *rid = msg_accp->prop->rids[i];
if (!is_in_group(rid->gid)) {
continue;
}
// For those I belong to, add resoponse.
response_t *response = (response_t *)
malloc(sizeof(response_t));
msg_accd.ress[msg_accd.n_ress] = response;
mpaxos__response_t__init(response);
response->rid = rid;
accp_info_t *ainfo = get_accp_info(rid->gid, rid->sid);
SAFE_ASSERT(ainfo != NULL);
apr_thread_mutex_lock(ainfo->mx);
// Check if the ballot id is larger.
ballotid_t maxbid = ainfo->bid_max;
if (rid->bid >= maxbid) {
response->ack = MPAXOS__ACK_ENUM__SUCCESS;
record_accepted(rid, msg_accp->prop);
proposal_t **p = apr_array_push(ainfo->arr_prop);
// *p = apr_pcalloc(ainfo->mp, sizeof(proposal_t));
// prop_cpy(*p, msg_accp->prop, ainfo->mp);
*p = prop_copy(msg_accp->prop);
} else if (rid->bid < maxbid) {
response->ack = MPAXOS__ACK_ENUM__ERR_BID;
} else {
SAFE_ASSERT(0);
}
//FIXME whether or not to add proposals in the message.
//now not
msg_accd.n_ress++;
apr_thread_mutex_unlock(ainfo->mx);
}
// send back the msg_accepted.
size_t len = mpaxos__msg_accepted__get_packed_size(&msg_accd);
log_message_res("send", "ACCEPTED", msg_accd.h, msg_accd.ress,
msg_accd.n_ress, len);
uint8_t *buf = (uint8_t *) malloc(len);
mpaxos__msg_accepted__pack(&msg_accd, buf);
for (int i = 0; i < msg_accd.n_ress; i++) {
free(msg_accd.ress[i]);
}
free(msg_accd.ress);
rpc_state *ret_state = (rpc_state*) malloc(sizeof(rpc_state));
ret_state->raw_output = buf;
ret_state->sz_output = len;
return ret_state;
}
void server_reg(server_t *svr, msgid_t msgid, void* fun) {
LOG_TRACE("server regisger function, msg type:%x", (int32_t)msgid);
SAFE_ASSERT(fun != NULL);
mpr_hash_set(svr->comm->ht, &msgid, SZ_MSGID,
&fun, sizeof(void*));
}
void handle_sconn_read(void* arg) {
LOG_TRACE("sconn read handle read.");
sconn_t *sconn = (sconn_t *) arg;
buf_t *buf = sconn->buf_recv;
apr_socket_t *sock = sconn->pjob->pfd.desc.s;
apr_status_t status = APR_SUCCESS;
//status = apr_socket_recv(pfd->desc.s, (char *)buf, &n);
status = buf_from_sock(buf, sock);
// invoke msg handling.
size_t sz_c = 0;
while ((sz_c = buf_sz_content(buf)) > SZ_SZMSG) {
uint32_t sz_msg = 0;
SAFE_ASSERT(buf_peek(buf, (uint8_t*)&sz_msg, SZ_SZMSG) == SZ_SZMSG);
SAFE_ASSERT(sz_msg > 0);
if (sz_c >= sz_msg + SZ_SZMSG + SZ_MSGID) {
SAFE_ASSERT(buf_read(buf, (uint8_t*)&sz_msg, SZ_SZMSG) == SZ_SZMSG);
msgid_t msgid = 0;
SAFE_ASSERT(buf_read(buf, (uint8_t*)&msgid, SZ_MSGID) == SZ_MSGID);
LOG_TRACE("next message size: %d, type: %x", (int32_t)sz_msg, (int32_t)msgid);
rpc_state *state = malloc(sizeof(rpc_state));
state->sz_input = sz_msg;
state->raw_input = malloc(sz_msg);
state->raw_output = NULL;
state->sz_output = 0;
state->sconn = sconn;
state->msgid = msgid;
SAFE_ASSERT(buf_read(buf, (uint8_t*)state->raw_input, sz_msg) == sz_msg);
// apr_thread_pool_push(tp_on_read_, (*(ctx->on_recv)), (void*)state, 0, NULL);
// mpr_thread_pool_push(tp_read_, (void*)state);
// apr_atomic_inc32(&n_data_recv_);
//(*(ctx->on_recv))(NULL, state);
// FIXME call
if (msgid & NEW_THREAD_CALL) {
apr_thread_pool_t *tp = sconn->tp;
SAFE_ASSERT(tp != NULL);
apr_thread_pool_push(tp, msg_call, state, 0, NULL);
} else {
msg_call(NULL, state);
}
} else {
break;
}
}
if (status == APR_SUCCESS) {
} else if (status == APR_EOF) {
LOG_INFO("sconn poll on read, received eof, close socket");
poll_mgr_remove_job(sconn->pjob->mgr, sconn->pjob);
} else if (status == APR_ECONNRESET) {
LOG_INFO("on read. connection reset, close socket");
poll_mgr_remove_job(sconn->pjob->mgr, sconn->pjob);
// TODO [improve] you may retry connect
} else if (status == APR_EAGAIN) {
LOG_TRACE("sconn poll on read, socket busy, resource temporarily unavailable.");
// do nothing.
} else {
LOG_ERROR("unkown error on poll reading. %s\n", apr_strerror(status, malloc(100), 100));
SAFE_ASSERT(0);
}
}
void handle_client_read(void *arg) {
client_t *cli = (client_t*) arg;
buf_t *buf = cli->buf_recv;
apr_socket_t *sock = cli->pjob->pfd.desc.s;
apr_status_t status = buf_from_sock(buf, sock);
// invoke msg handling.
size_t sz_c = 0;
while ((sz_c = buf_sz_content(buf)) > SZ_SZMSG) {
uint32_t sz_msg = 0;
buf_peek(buf, (uint8_t*)&sz_msg, sizeof(sz_msg));
if (sz_c >= sz_msg + SZ_SZMSG + SZ_MSGID) {
buf_read(buf, (uint8_t*)&sz_msg, SZ_SZMSG);
msgid_t msgid = 0;
buf_read(buf, (uint8_t*)&msgid, SZ_MSGID);
rpc_state *state = (rpc_state*)malloc(sizeof(rpc_state));
state->sz_input = sz_msg;
state->raw_input = (uint8_t *)malloc(sz_msg);
// state->ctx = ctx;
buf_read(buf, state->raw_input, sz_msg);
// state->ctx = ctx;
/*
apr_thread_pool_push(tp_on_read_, (*(ctx->on_recv)), (void*)state, 0, NULL);
// mpr_thread_pool_push(tp_read_, (void*)state);
*/
// apr_atomic_inc32(&n_data_recv_);
//(*(ctx->on_recv))(NULL, state);
// FIXME call
rpc_state* (**fun)(void*) = NULL;
size_t sz;
mpr_hash_get(cli->comm->ht, &msgid, SZ_MSGID, (void**)&fun, &sz);
SAFE_ASSERT(fun != NULL);
LOG_TRACE("going to call function %x", *fun);
// ctx->n_rpc++;
// ctx->sz_recv += n;
(**fun)(state);
free(state->raw_input);
free(state);
} else {
break;
}
}
if (status == APR_SUCCESS) {
} else if (status == APR_EOF) {
LOG_DEBUG("cli poll on read, received eof, close socket");
poll_mgr_remove_job(cli->pjob->mgr, cli->pjob);
} else if (status == APR_ECONNRESET) {
LOG_ERROR("cli poll on read. connection reset.");
poll_mgr_remove_job(cli->pjob->mgr, cli->pjob);
// TODO [improve] you may retry connect
} else if (status == APR_EAGAIN) {
LOG_DEBUG("cli poll on read. read socket busy, resource temporarily unavailable.");
// do nothing.
} else {
LOG_ERROR("unkown error on poll reading. %s\n",
apr_strerror(status, (char*)malloc(100), 100));
SAFE_ASSERT(0);
}
}
