/**
* paxos_ack_resend - Receive a resend of request data, and re-commit the
* instance to which the request belongs.
*/
int
paxos_ack_resend(struct paxos_header *hdr, msgpack_object *o)
{
struct paxos_instance *inst;
struct paxos_request *req;
// Grab the instance for which we wanted the request. If we find that it
// has already been cached and learned since we began our retrieval, we can
// just return.
//
// Note also that an instance should only be NULL if it was committed and
// learned and then truncated in a sync operation.
inst = instance_find(&pax->ilist, hdr->ph_inum);
if (inst == NULL || inst->pi_cached) {
return 0;
}
// See if we already got the request. It is possible that we received a
// commit message for the instance before the original request broadcast
// reached us. However, since the instance-request mapping is one way, we
// wait until a resend is received before committing.
req = request_find(&pax->rcache, inst->pi_val.pv_reqid);
if (req == NULL) {
// Allocate a request and unpack it.
req = g_malloc0(sizeof(*req));
paxos_request_unpack(req, o);
// Insert it to our request cache.
request_insert(&pax->rcache, req);
}
// Commit again, now that we have the associated request.
return paxos_commit(inst);
}
/**
* paxos_ack_retrieve - Acknowledge a retrieve.
*
* This basically just unpacks and wraps a resend.
*/
int paxos_ack_retrieve(struct paxos_header *hdr, msgpack_object *o)
{
paxid_t paxid;
msgpack_object *p;
struct paxos_value val;
struct paxos_request *req;
struct paxos_acceptor *acc;
// Make sure the payload is well-formed.
assert(o->type == MSGPACK_OBJECT_ARRAY);
assert(o->via.array.size == 2);
p = o->via.array.ptr;
// Unpack the retriever's ID and the value being retrieved.
paxos_paxid_unpack(&paxid, p++);
paxos_value_unpack(&val, p++);
// Retrieve the request.
assert(request_needs_cached(val.pv_dkind));
req = request_find(&pax->rcache, val.pv_reqid);
if (req != NULL) {
// If we have the request, look up the recipient and resend.
acc = acceptor_find(&pax->alist, paxid);
return paxos_resend(acc, hdr, req);
} else {
// If we don't have the request either, just return.
return 0;
}
}
static void recv(struct mwService *srvc, struct mwChannel *chan,
guint16 type, struct mwOpaque *data) {
/* process into results, trigger callbacks */
struct mwGetBuffer *b;
struct mwServiceStorage *srvc_stor;
struct mwStorageReq *req;
guint32 id;
g_return_if_fail(srvc != NULL);
srvc_stor = (struct mwServiceStorage *) srvc;
g_return_if_fail(chan != NULL);
g_return_if_fail(chan == srvc_stor->channel);
g_return_if_fail(data != NULL);
b = mwGetBuffer_wrap(data);
id = guint32_peek(b);
req = request_find(srvc_stor, id);
if(! req) {
g_warning("couldn't find request 0x%x in storage service", id);
mwGetBuffer_free(b);
return;
}
g_return_if_fail(req->action == type);
request_get(b, req);
if(mwGetBuffer_error(b)) {
mw_mailme_opaque(data, "storage request 0x%x, type: 0x%x", id, type);
} else {
request_trigger(srvc_stor, req);
}
mwGetBuffer_free(b);
request_remove(srvc_stor, req);
}
/**
* Truncate an ilist up to (but not including) a given inum.
*
* We also free all associated requests.
*/
static void
ilist_truncate_prefix(instance_container *ilist, paxid_t inum)
{
struct paxos_instance *it;
struct paxos_request *req;
for (it = LIST_FIRST(ilist); it != (void *)ilist; it = LIST_FIRST(ilist)) {
// Break if we've hit the desired stopping point.
if (it->pi_hdr.ph_inum >= inum) {
break;
}
// Free the instance and its associated request.
req = request_find(&pax->rcache, it->pi_val.pv_reqid);
if (req != NULL) {
LIST_REMOVE(&pax->rcache, req, pr_le);
request_destroy(req);
}
LIST_REMOVE(ilist, it, pi_le);
instance_destroy(it);
}
}
/**
* continue_ack_refuse - If we were able to reestablish connection with the
* purported proposer, reset our proposer and reintroduce ourselves.
*/
int
do_continue_ack_refuse(GIOChannel *chan, struct paxos_acceptor *acc,
struct paxos_continuation *k)
{
int r = 0;
struct paxos_header hdr;
struct paxos_request *req;
struct yakyak yy;
// If we are the proposer and have finished preparing, anyone higher-ranked
// than we are is dead to us. However, their parts may not yet have gone
// through, so we make sure to ignore attempts at reconnection.
if (is_proposer() && pax->prep == NULL) {
return 0;
}
// Register the reconnection.
acc->pa_peer = paxos_peer_init(chan);
if (acc->pa_peer != NULL) {
// Account for a new acceptor.
pax->live_count++;
// Free any prep we have. Although we dispatch as an acceptor when we
// acknowledge a refuse, when the acknowledgement continues here, we may
// have become the proposer. Thus, if we are preparing, we should just
// give up. If the acceptor we are reconnecting to fails, we'll find
// out about the drop and then reprepare.
g_free(pax->prep);
pax->prep = NULL;
instance_container_destroy(&pax->idefer);
// Say hello.
ERR_ACCUM(r, paxos_hello(acc));
if (acc->pa_paxid < pax->proposer->pa_paxid) {
// Update the proposer only if we have not reconnected to an even
// higher-ranked acceptor.
pax->proposer = acc;
// Resend our request.
// XXX: What about the problematic case where A is connected to B, B
// thinks it's the proposer and accepts A's request, but in fact B is not
// the proposer and C, the real proposer, gets neither of their requests?
header_init(&hdr, OP_REQUEST, pax->proposer->pa_paxid);
req = request_find(&pax->rcache, k->pk_data.req.pr_val.pv_reqid);
if (req == NULL) {
req = &k->pk_data.req;
}
yakyak_init(&yy, 2);
paxos_header_pack(&yy, &hdr);
paxos_request_pack(&yy, req);
ERR_ACCUM(r, paxos_send_to_proposer(&yy));
yakyak_destroy(&yy);
}
}
return r;
}
/**
* paxos_commit - Commit a value for an instance of the Paxos protocol.
*
* We totally order calls to paxos_learn by instance number in order to make
* the join and greet protocols behave properly. This also gives our chat
* clients an easy mechanism for totally ordering their logs without extra
* work on their part.
*
* It is possible that failed DEC_PART decrees (i.e., decrees in which the
* proposer attempts to disconnect an acceptor who a majority of acceptors
* believe is still alive) could delay the learning of committed chat
* messages. To avoid this, once a proposer receives enough rejections
* of the decree, the part decree is replaced with a null decree. The
* proposer can then issue the part again with a higher instance number
* if desired.
*/
int
paxos_commit(struct paxos_instance *inst)
{
int r;
struct paxos_request *req = NULL;
struct paxos_instance *it;
// Mark the commit.
inst->pi_committed = true;
// Pull the request from the request cache if applicable.
if (request_needs_cached(inst->pi_val.pv_dkind)) {
req = request_find(&pax->rcache, inst->pi_val.pv_reqid);
// If we can't find a request and need one, send out a retrieve to the
// request originator and defer the commit.
if (req == NULL) {
return paxos_retrieve(inst);
}
}
// Mark the cache.
inst->pi_cached = true;
// We should already have committed and learned everything before the hole.
assert(inst->pi_hdr.ph_inum >= pax->ihole);
// Since we want our learns to be totally ordered, if we didn't just fill
// the hole, we cannot learn.
if (inst->pi_hdr.ph_inum != pax->ihole) {
// If we're the proposer, we have to just wait it out.
if (is_proposer()) {
return 0;
}
// If the hole has committed but is just waiting on a retrieve, we'll learn
// when we receive the resend.
if (pax->istart->pi_hdr.ph_inum == pax->ihole && pax->istart->pi_committed) {
assert(!pax->istart->pi_cached);
return 0;
}
// The hole is either missing or uncommitted and we are not the proposer,
// so issue a retry.
return acceptor_retry(pax->ihole);
}
// Set pax->istart to point to the instance numbered pax->ihole.
if (pax->istart->pi_hdr.ph_inum != pax->ihole) {
pax->istart = LIST_NEXT(pax->istart, pi_le);
}
assert(pax->istart->pi_hdr.ph_inum == pax->ihole);
// Now learn as many contiguous commits as we can. This function is the
// only path by which we learn commits, and we always learn in contiguous
// blocks. Therefore, it is an invariant of our system that all the
// instances numbered lower than pax->ihole are learned and committed, and
// none of the instances geq to pax->ihole are learned (although some may
// be committed).
//
// We iterate over the instance list, detecting and breaking if we find a
// hole and learning whenever we don't.
for (it = pax->istart; ; it = LIST_NEXT(it, pi_le), ++pax->ihole) {
// If we reached the end of the list, set pax->istart to the last existing
// instance.
if (it == (void *)&pax->ilist) {
pax->istart = LIST_LAST(&pax->ilist);
break;
}
// If we skipped over an instance number because we were missing an
// instance, set pax->istart to the last instance before the hole.
if (it->pi_hdr.ph_inum != pax->ihole) {
pax->istart = LIST_PREV(it, pi_le);
break;
}
// If we found an uncommitted or uncached instance, set pax->istart to it.
if (!it->pi_committed || !it->pi_cached) {
pax->istart = it;
break;
}
// By our invariant, since we are past our original hole, no instance
// should be learned.
assert(!it->pi_learned);
// Grab its associated request. This is guaranteed to exist because we
// have checked that pi_cached holds.
req = NULL;
if (request_needs_cached(it->pi_val.pv_dkind)) {
req = request_find(&pax->rcache, it->pi_val.pv_reqid);
assert(req != NULL);
}
// Learn the value.
ERR_RET(r, paxos_learn(it, req));
}
return 0;
}
/* XXX This function needs a really serious re-write/audit/etc. */
int peer_readres(struct peer_t *p) {
struct request_t *r;
int ret;
unsigned short int *ul;
int id;
int req;
unsigned short int *l;
int len;
l = (unsigned short int*)p->b;
/* BUG: we’re reading on a TCP socket here, so we could in theory
get a partial response. Using TCP puts the onus on the user
program (i.e. this code) to buffer bytes until we have a
parseable response. This probably won’t happen very often in
practice because even with DF, the path MTU is unlikely to be
smaller than the DNS response. But it could happen. And then
we fall into the `processanswer` code below without having the
whole answer. */
/* This is reading data from Tor over TCP */
while ((ret = read(p->tcp_fd, (p->b + p->bl), (RECV_BUF_SIZE - p->bl))) < 0 && errno == EAGAIN);
if (ret == 0) {
peer_mark_as_dead(p);
return 3;
}
p->bl += ret;
// we have an answer
// get answer from receive buffer
do {
if (p->bl < 2) {
return 2;
}
else {
len = ntohs(*l);
if ((len + 2) > p->bl)
return 2;
}
ul = (unsigned short int*)(p->b + 2);
id = ntohs(*ul);
log_packet(id, TCP, IN, PEER_IP(p), PEER_PORT(p));
if ((req = request_find(id)) == -1) {
memmove(p->b, (p->b + len + 2), (p->bl - len - 2));
p->bl -= len + 2;
return 0;
}
r = &requests[req];
// write back real id
*ul = htons(r->rid);
// Remove the AD flag from the reply if it has one. Because we might be
// answering requests to 127.0.0.1, the client might consider us
// trusted. While trusted, we shouldn't indicate that data is DNSSEC
// valid when we haven't checked it.
// See http://tools.ietf.org/html/rfc2535#section-6.1
if (len >= 6)
p->b[5] &= 0xdf;
/* This is where we send the answer over UDP to the client */
r->a.sin_family = AF_INET;
log_packet(id, UDP, OUT, REQ_IP(r), REQ_PORT(r));
while (sendto(server_fd, (p->b + 2), len, 0, (struct sockaddr*)&r->a, sizeof(struct sockaddr_in)) < 0 && errno == EAGAIN);
memmove(p->b, p->b + len +2, p->bl - len - 2);
p->bl -= len + 2;
// mark as handled/unused
r->id = 0;
} while (p->bl > 0);
return 1;
}
