/**
* acceptor_redirect - Tell a preparer that they are not the proposer and
* thus do not have the right to prepare.
*/
int
acceptor_redirect(struct paxos_peer *source, struct paxos_header *orig_hdr)
{
int r;
struct paxos_header hdr;
struct yakyak yy;
// Initialize a header. Our recipients should use ph_inum rather than the
// ballot ID as the ID of the proposer we are suggesting, since, it may be
// the case that the proposer has assumed the proposership but has not yet
// prepared to us.
header_init(&hdr, OP_REDIRECT, pax->proposer->pa_paxid);
// Pack a payload, which includes the header we were sent which we believe
// to be incorrect.
yakyak_init(&yy, 2);
paxos_header_pack(&yy, &hdr);
paxos_header_pack(&yy, orig_hdr);
// Send the payload.
r = paxos_peer_send(source, yakyak_data(&yy), yakyak_size(&yy));
yakyak_destroy(&yy);
return r;
}
/**
* paxos_retrieve - Ask the originator of request data to send us data which
* we do not have in our cache.
*
* We call this function when and only when we are issued a commit for an
* instance whose associated request is not in our request cache.
*/
int
paxos_retrieve(struct paxos_instance *inst)
{
int r;
struct paxos_header hdr;
struct paxos_acceptor *acc;
struct paxos_yak py;
// Initialize a header. We set ph_inum to the instance number of the
// request.
header_init(&hdr, OP_RETRIEVE, inst->pi_hdr.ph_inum);
// Pack the retrieve.
paxos_payload_init(&py, 2);
paxos_header_pack(&py, &hdr);
paxos_payload_begin_array(&py, 2);
paxos_paxid_pack(&py, pax->self_id);
paxos_value_pack(&py, &inst->pi_val);
// Determine the request originator and send. If we are no longer connected
// to the request originator, broadcast the retrieve instead.
acc = acceptor_find(&pax->alist, inst->pi_val.pv_reqid.id);
if (acc == NULL || acc->pa_peer == NULL) {
r = paxos_broadcast(&py);
} else {
r = paxos_send(acc, &py);
}
paxos_payload_destroy(&py);
return r;
}
void
paxos_instance_pack(struct yakyak *yy, struct paxos_instance *inst)
{
msgpack_pack_array(yy->pk, 3);
paxos_header_pack(yy, &inst->pi_hdr);
inst->pi_committed ? msgpack_pack_true(yy->pk) : msgpack_pack_false(yy->pk);
paxos_value_pack(yy, &inst->pi_val);
}
/**
* proposer_sync - Send a sync command to all acceptors.
*
* For a sync to succeed, all acceptors need to tell us the instance number
* of their last contiguous learn. We take the minimum of these values
* and then command everyone to truncate everything before this minimum.
*/
int
proposer_sync()
{
int r;
struct paxos_header hdr;
struct yakyak yy;
// If we haven't finished preparing as the proposer, don't sync.
if (pax->prep != NULL) {
return 1;
}
// If not everyone is live, we should delay syncing.
if (pax->live_count != LIST_COUNT(&pax->alist)) {
return 1;
}
// If our local last contiguous learn is the same as the previous sync
// point, we don't need to sync.
if (pax->ihole - 1 == pax->sync_prev) {
return 0;
}
// If we're already syncing, increment the skip counter.
if (pax->sync != NULL) {
// Resync if we've waited too long for the sync to finish.
if (++pax->sync->ps_skips < SYNC_SKIP_THRESH) {
return 1;
} else {
g_free(pax->sync);
}
}
// Create a new sync.
pax->sync = g_malloc0(sizeof(*(pax->sync)));
pax->sync->ps_total = LIST_COUNT(&pax->alist);
pax->sync->ps_acks = 1; // Including ourselves.
pax->sync->ps_skips = 0;
pax->sync->ps_last = 0;
// Initialize a header.
header_init(&hdr, OP_SYNC, ++pax->sync_id);
// Pack and broadcast the sync.
yakyak_init(&yy, 1);
paxos_header_pack(&yy, &hdr);
r = paxos_broadcast(&yy);
yakyak_destroy(&yy);
return r;
}
/**
* acceptor_reject - Notify the proposer that we reject their decree.
*/
int
acceptor_reject(struct paxos_header *hdr)
{
int r;
struct yakyak yy;
// Pack a header.
hdr->ph_opcode = OP_REJECT;
yakyak_init(&yy, 1);
paxos_header_pack(&yy, hdr);
// Send the payload.
r = paxos_send_to_proposer(&yy);
yakyak_destroy(&yy);
return r;
}
/**
* acceptor_last - Send the instance number of our last contiguous learn to
* the proposer.
*/
int acceptor_last(struct paxos_header *hdr)
{
int r;
struct yakyak yy;
// Modify the header opcode.
hdr->ph_opcode = OP_LAST;
// Pack and send the response.
yakyak_init(&yy, 2);
paxos_header_pack(&yy, hdr);
paxos_paxid_pack(&yy, pax->ihole - 1);
r = paxos_send_to_proposer(&yy);
yakyak_destroy(&yy);
return r;
}
/**
* paxos_resend - Resend request data that some acceptor didn't have at
* commit time.
*/
int
paxos_resend(struct paxos_acceptor *acc, struct paxos_header *hdr,
struct paxos_request *req)
{
int r;
struct paxos_yak py;
// Modify the header.
hdr->ph_opcode = OP_RESEND;
// Just pack and send the resend.
paxos_payload_init(&py, 2);
paxos_header_pack(&py, hdr);
paxos_request_pack(&py, req);
r = paxos_send(acc, &py);
paxos_payload_destroy(&py);
return r;
}
/**
* proposer_truncate - Command all acceptors to drop the contiguous prefix
* of Paxos instances for which every participant has learned.
*/
int
proposer_truncate(struct paxos_header *hdr)
{
int r;
struct yakyak yy;
// Obtain our own last contiguous learn.
if (pax->ihole - 1 < pax->sync->ps_last) {
pax->sync->ps_last = pax->ihole - 1;
}
// Record the sync point.
pax->sync_prev = pax->sync->ps_last;
// Make this instance our new ibase; this ensures that our list always has
// at least one committed instance.
assert(pax->sync->ps_last >= pax->ibase);
pax->ibase = pax->sync->ps_last;
// Modify the header.
hdr->ph_opcode = OP_TRUNCATE;
// Pack a truncate.
yakyak_init(&yy, 2);
paxos_header_pack(&yy, hdr);
paxos_paxid_pack(&yy, pax->ibase);
// Broadcast it.
r = paxos_broadcast(&yy);
yakyak_destroy(&yy);
if (r) {
return r;
}
// Do the truncate (< pax->ibase).
ilist_truncate_prefix(&pax->ilist, pax->ibase);
// End the sync.
g_free(pax->sync);
pax->sync = NULL;
return 0;
}
/**
* continue_ack_refuse - If we were able to reestablish connection with the
* purported proposer, reset our proposer and reintroduce ourselves.
*/
int
do_continue_ack_refuse(GIOChannel *chan, struct paxos_acceptor *acc,
struct paxos_continuation *k)
{
int r = 0;
struct paxos_header hdr;
struct paxos_request *req;
struct yakyak yy;
// If we are the proposer and have finished preparing, anyone higher-ranked
// than we are is dead to us. However, their parts may not yet have gone
// through, so we make sure to ignore attempts at reconnection.
if (is_proposer() && pax->prep == NULL) {
return 0;
}
// Register the reconnection.
acc->pa_peer = paxos_peer_init(chan);
if (acc->pa_peer != NULL) {
// Account for a new acceptor.
pax->live_count++;
// Free any prep we have. Although we dispatch as an acceptor when we
// acknowledge a refuse, when the acknowledgement continues here, we may
// have become the proposer. Thus, if we are preparing, we should just
// give up. If the acceptor we are reconnecting to fails, we'll find
// out about the drop and then reprepare.
g_free(pax->prep);
pax->prep = NULL;
instance_container_destroy(&pax->idefer);
// Say hello.
ERR_ACCUM(r, paxos_hello(acc));
if (acc->pa_paxid < pax->proposer->pa_paxid) {
// Update the proposer only if we have not reconnected to an even
// higher-ranked acceptor.
pax->proposer = acc;
// Resend our request.
// XXX: What about the problematic case where A is connected to B, B
// thinks it's the proposer and accepts A's request, but in fact B is not
// the proposer and C, the real proposer, gets neither of their requests?
header_init(&hdr, OP_REQUEST, pax->proposer->pa_paxid);
req = request_find(&pax->rcache, k->pk_data.req.pr_val.pv_reqid);
if (req == NULL) {
req = &k->pk_data.req;
}
yakyak_init(&yy, 2);
paxos_header_pack(&yy, &hdr);
paxos_request_pack(&yy, req);
ERR_ACCUM(r, paxos_send_to_proposer(&yy));
yakyak_destroy(&yy);
}
}
return r;
}
/**
* paxos_request - Request that the proposer make a decree for us.
*
* If the request has data attached to it, we broadcast an out-of-band message
* to all acceptors, asking that they cache our message until the proposer
* commits it.
*
* We send the request as a header along with a two-object array consisting
* of a paxos_value (itself an array) and a msgpack raw (i.e., a data
* string).
*/
int
paxos_request(struct paxos_session *session, dkind_t dkind, const void *msg,
size_t len)
{
int r, needs_cached;
struct paxos_header hdr;
struct paxos_request *req;
struct paxos_yak py;
// Set the session. The client should pass us a pointer to the correct
// session object which we returned when the session was created.
pax = session;
// We can't make requests if we're not part of a protocol.
if (pax == NULL) {
return 1;
}
// Do we need to cache this request?
needs_cached = request_needs_cached(dkind);
// Initialize a header. We overload ph_inum to the ID of the acceptor who
// we believe to be the proposer.
header_init(&hdr, OP_REQUEST, pax->proposer->pa_paxid);
// Allocate a request and initialize it.
req = g_malloc0(sizeof(*req));
req->pr_val.pv_dkind = dkind;
req->pr_val.pv_reqid.id = pax->self_id;
req->pr_val.pv_reqid.gen = (++pax->req_id); // Increment our req_id.
req->pr_val.pv_extra = 0; // Always 0 for requests.
req->pr_size = len;
req->pr_data = g_memdup(msg, len);
// Add it to the request cache if needed.
if (needs_cached) {
request_insert(&pax->rcache, req);
}
if (!is_proposer() || needs_cached) {
// We need to send iff either we are not the proposer or the request
// has nontrivial data.
paxos_payload_init(&py, 2);
paxos_header_pack(&py, &hdr);
paxos_request_pack(&py, req);
// Broadcast only if it needs caching.
if (!needs_cached) {
r = paxos_send_to_proposer(&py);
} else {
r = paxos_broadcast(&py);
}
paxos_payload_destroy(&py);
if (r) {
return r;
}
}
// Decree the request if we're the proposer; otherwise just return.
if (is_proposer()) {
return proposer_decree_request(req);
} else {
return 0;
}
}