static void
leader_shutdown() {
LOG(0, ("Proposer %d dropping leadership\n", this_proposer_id));
evtimer_del(&p1_check_event);
evtimer_del(&p2_check_event);
#ifdef LEADER_EVENTS_UPDATE_INTERVAL
evtimer_del(&print_events_event);
#endif
//Iterate over currently open instances
p_inst_info * ii;
unsigned int i;
for(i = current_iid; i <= p1_info.highest_open; i++) {
ii = GET_PRO_INSTANCE(i);
if(ii->status != p2_completed && ii->p2_value != NULL) {
// A value was assigned to this instance, but it did
// not complete. Send back to the pending list for now
vh_push_back_value(ii->p2_value);
ii->p2_value = NULL;
}
//Clear all instances
pro_clear_instance_info(ii);
}
//This will clear all values in the pending list
// and notify the respective clients
vh_shutdown();
}
//Returns 1 if the instance became ready, 0 otherwise
static int
handle_prepare_ack(prepare_ack * pa, short int acceptor_id) {
p_inst_info * ii = GET_PRO_INSTANCE(pa->iid);
// If not p1_pending, drop
if(ii->status != p1_pending) {
LOG(DBG, ("Promise dropped, iid:%ld not pending\n", pa->iid));
return 0;
}
// If not our ballot, drop
if(pa->ballot != ii->my_ballot) {
LOG(DBG, ("Promise dropped, iid:%ld not our ballot\n", pa->iid));
return 0;
}
//Save the acknowledgement from this acceptor
//Takes also care of value that may be there
pro_save_prepare_ack(ii, pa, acceptor_id);
//Not a majority yet for this instance
if(ii->promises_count < QUORUM) {
LOG(DBG, ("Not yet a quorum for iid:%ld\n", pa->iid));
return 0;
}
//Quorum reached!
ii->status = p1_ready;
p1_info.pending_count -= 1;
p1_info.ready_count += 1;
LOG(DBG, ("Quorum for iid:%ld reached\n", pa->iid));
return 1;
}
/*-------------------------------------------------------------------------*/
static void
leader_deliver(char * value, size_t size, iid_t iid, ballot_t ballot, int proposer) {
UNUSED_ARG(ballot);
UNUSED_ARG(proposer);
LOG(DBG, ("Instance %lu delivered to Leader\n", iid));
//Verify that the value is the one found or associated
p_inst_info * ii = GET_PRO_INSTANCE(iid);
//Instance not even initialized, skip
if(ii->iid != iid) {
return;
}
if(ii->status == p1_pending) {
p1_info.pending_count -= 1;
}
if(p2_info.next_unused_iid == iid) {
p2_info.next_unused_iid += 1;
}
int opened_by_me = (ii->status == p1_pending && ii->p2_value != NULL) ||
(ii->status == p1_ready && ii->p2_value != NULL) ||
(ii->status == p2_pending);
if(opened_by_me) {
p2_info.open_count -= 1;
}
int my_val = (ii->p2_value != NULL) &&
(ii->p2_value->value_size == size) &&
(memcmp(value, ii->p2_value->value, size) == 0);
if(my_val) {
//Our value accepted, notify client that submitted it
vh_notify_client(0, ii->p2_value);
} else if(ii->p2_value != NULL) {
//Different value accepted, push back our value
vh_push_back_value(ii->p2_value);
ii->p2_value = NULL;
} else {
//We assigned no value to this instance,
//it comes from somebody else??
}
//Clear current instance
pro_clear_instance_info(ii);
//If enough instances are ready to
// be opened, start phase2 for them
leader_open_instances_p2_new();
}
static void
leader_check_p1_pending() {
iid_t iid_iterator;
p_inst_info * ii;
//Create an empty prepare batch in send buffer
sendbuf_clear(to_acceptors, prepare_reqs, this_proposer_id);
LOG(DBG, ("Checking pending phase 1 from %lu to %lu\n",
current_iid, p1_info.highest_open));
//Get current time for checking expired
struct timeval time_now;
gettimeofday(&time_now, NULL);
for(iid_iterator = current_iid;
iid_iterator <= p1_info.highest_open; iid_iterator++) {
//Get instance from state array
ii = GET_PRO_INSTANCE(iid_iterator);
assert(ii->iid == iid_iterator);
//Still pending -> it's expired
if(ii->status == p1_pending && leader_is_expired(&ii->timeout, &time_now)) {
LOG(DBG, ("Phase 1 of instance %ld expired!\n", ii->iid));
//Reset fields used for previous phase 1
ii->promises_bitvector = 0;
ii->promises_count = 0;
ii->p1_value_ballot = 0;
if(ii->p1_value != NULL) {
PAX_FREE(ii->p1_value);
}
ii->p1_value = NULL;
//Ballot is incremented
ii->my_ballot = NEXT_BALLOT(ii->my_ballot);
//Send prepare to acceptors
sendbuf_add_prepare_req(to_acceptors, ii->iid, ii->my_ballot);
leader_set_expiration(ii, P1_TIMEOUT_INTERVAL);
COUNT_EVENT(p1_timeout);
}
}
//Send if something is still there
sendbuf_flush(to_acceptors);
}
// Scan trough p1_ready that have a value
// assigned or found, execute phase2
static void
leader_open_instances_p2_expired() {
unsigned int count = 0;
//Create a batch of accept requests
sendbuf_clear(to_acceptors, accept_reqs, this_proposer_id);
//Start new phase 2 for all instances found in status p1_ready
// if they are in the range below, phase2 timed-out and
// we went successfully trough phase1 again
iid_t iid;
p_inst_info * ii;
for(iid = current_iid; iid < p2_info.next_unused_iid; iid++) {
ii = GET_PRO_INSTANCE(iid);
assert(ii->iid == iid);
//This instance is in status p1_ready but it's in the range
// of previously opened phase2, it's now time to retry
if(ii->status == p1_ready) {
assert(ii->p2_value != NULL || ii->p2_value != NULL);
leader_execute_p2(ii);
//Count opened
count += 1;
}
}
//Count p1_ready that were consumed
p1_info.ready_count -= count;
//Flush last accept_req batch
sendbuf_flush(to_acceptors);
LOG(DBG, ("Opened %u old (timed-out) instances\n", count));
}
static void
leader_periodic_p2_check(int fd, short event, void *arg) {
UNUSED_ARG(fd);
UNUSED_ARG(event);
UNUSED_ARG(arg);
// create a prepare batch for expired instances
sendbuf_clear(to_acceptors, prepare_reqs, this_proposer_id);
struct timeval now;
gettimeofday(&now, NULL);
// from current to highest open, check deadline
// if instances in status p2_pending
iid_t i;
p_inst_info * ii;
for(i = current_iid; i < p2_info.next_unused_iid; i++) {
ii = GET_PRO_INSTANCE(i);
//Not p2_pending, skip
if(ii->status != p2_pending) {
continue;
}
//Check if it was closed in the meanwhile
// (but not delivered yet)
if(learner_is_closed(i)) {
ii->status = p2_completed;
p2_info.open_count -= 1;
//The rest (i.e. answering client)
// is done when the value is actually delivered
LOG(VRB, ("Instance %lu closed, waiting for deliver\n", i));
continue;
}
//Not expired yet, skip
if(!leader_is_expired(&ii->timeout, &now)) {
continue;
}
//Expired and not closed: must restart from phase 1
ii->status = p1_pending;
p1_info.pending_count += 1;
ii->my_ballot = NEXT_BALLOT(ii->my_ballot);
//Send prepare to acceptors
sendbuf_add_prepare_req(to_acceptors, ii->iid, ii->my_ballot);
leader_set_expiration(ii, P1_TIMEOUT_INTERVAL);
LOG(VRB, ("Instance %lu restarts from phase 1\n", i));
COUNT_EVENT(p2_timeout);
}
//Flush last message if any
sendbuf_flush(to_acceptors);
//Open new instances
leader_open_instances_p2_new();
//Set next invokation of this function
leader_set_next_p2_check();
}
static void
leader_open_instances_p2_new() {
unsigned int count = 0;
p_inst_info * ii;
//For better batching, opening new instances at the end
// is preferred when more than 1 can be opened together
unsigned int treshold = (PROPOSER_P2_CONCURRENCY/3)*2;
if (p2_info.open_count > treshold) {
LOG(DBG, ("Skipping Phase2 open, %u are still active (tresh:%u)\n", p2_info.open_count, treshold));
return;
}
LOG(DBG, ("Could open %u p2 instances\n",
(PROPOSER_P2_CONCURRENCY - p2_info.open_count)));
//Create a batch of accept requests
sendbuf_clear(to_acceptors, accept_reqs, this_proposer_id);
//Start new phase 2 while there is some value from
// client to send and we can open more concurrent instances
while((count + p2_info.open_count) <= PROPOSER_P2_CONCURRENCY) {
ii = GET_PRO_INSTANCE(p2_info.next_unused_iid);
assert(ii->p2_value == NULL);
//No value to send for next unused, stop
if(ii->p1_value == NULL &&
vh_pending_list_size() == 0) {
LOG(DBG, ("No value to use for next instance\n"));
break;
}
int normal_ready = ii->status == p1_ready && ii->iid != p2_info.next_unused_iid;
int inf_ready = ii->status == empty && inf_prepare_info.status == p1_ready;
int ready = normal_ready | inf_ready;
assert(ii->p2_value == NULL || inf_ready);
//Next unused is not ready, stop
if(!ready) {
LOG(DBG, ("Next instance to use for P2 (iid:%lu) is not ready yet\n", p2_info.next_unused_iid));
LOG(DBG, ("%d %d %d %d\n", ii->status, inf_prepare_info.status, ii->iid, p2_info.next_unused_iid))
COUNT_EVENT(p2_waits_p1);
break;
}
if(inf_ready) {
ii->iid = p2_info.next_unused_iid;
}
//Executes phase2, sending an accept request
//Using the found value or getting the next from list
leader_execute_p2(ii);
//Count opened
count += 1;
//Update next to use
p2_info.next_unused_iid += 1;
}
//Count newly opened
p2_info.open_count += count;
//Flush last accept_req batch
sendbuf_flush(to_acceptors);
if(count > 0) {
LOG(DBG, ("Opened %u new instances\n", count));
}
}
