static void leader_shutdown() { LOG(0, ("Proposer %d dropping leadership\n", this_proposer_id)); evtimer_del(&p1_check_event); evtimer_del(&p2_check_event); #ifdef LEADER_EVENTS_UPDATE_INTERVAL evtimer_del(&print_events_event); #endif //Iterate over currently open instances p_inst_info * ii; unsigned int i; for(i = current_iid; i <= p1_info.highest_open; i++) { ii = GET_PRO_INSTANCE(i); if(ii->status != p2_completed && ii->p2_value != NULL) { // A value was assigned to this instance, but it did // not complete. Send back to the pending list for now vh_push_back_value(ii->p2_value); ii->p2_value = NULL; } //Clear all instances pro_clear_instance_info(ii); } //This will clear all values in the pending list // and notify the respective clients vh_shutdown(); }
//Returns 1 if the instance became ready, 0 otherwise static int handle_prepare_ack(prepare_ack * pa, short int acceptor_id) { p_inst_info * ii = GET_PRO_INSTANCE(pa->iid); // If not p1_pending, drop if(ii->status != p1_pending) { LOG(DBG, ("Promise dropped, iid:%ld not pending\n", pa->iid)); return 0; } // If not our ballot, drop if(pa->ballot != ii->my_ballot) { LOG(DBG, ("Promise dropped, iid:%ld not our ballot\n", pa->iid)); return 0; } //Save the acknowledgement from this acceptor //Takes also care of value that may be there pro_save_prepare_ack(ii, pa, acceptor_id); //Not a majority yet for this instance if(ii->promises_count < QUORUM) { LOG(DBG, ("Not yet a quorum for iid:%ld\n", pa->iid)); return 0; } //Quorum reached! ii->status = p1_ready; p1_info.pending_count -= 1; p1_info.ready_count += 1; LOG(DBG, ("Quorum for iid:%ld reached\n", pa->iid)); return 1; }
/*-------------------------------------------------------------------------*/ static void leader_deliver(char * value, size_t size, iid_t iid, ballot_t ballot, int proposer) { UNUSED_ARG(ballot); UNUSED_ARG(proposer); LOG(DBG, ("Instance %lu delivered to Leader\n", iid)); //Verify that the value is the one found or associated p_inst_info * ii = GET_PRO_INSTANCE(iid); //Instance not even initialized, skip if(ii->iid != iid) { return; } if(ii->status == p1_pending) { p1_info.pending_count -= 1; } if(p2_info.next_unused_iid == iid) { p2_info.next_unused_iid += 1; } int opened_by_me = (ii->status == p1_pending && ii->p2_value != NULL) || (ii->status == p1_ready && ii->p2_value != NULL) || (ii->status == p2_pending); if(opened_by_me) { p2_info.open_count -= 1; } int my_val = (ii->p2_value != NULL) && (ii->p2_value->value_size == size) && (memcmp(value, ii->p2_value->value, size) == 0); if(my_val) { //Our value accepted, notify client that submitted it vh_notify_client(0, ii->p2_value); } else if(ii->p2_value != NULL) { //Different value accepted, push back our value vh_push_back_value(ii->p2_value); ii->p2_value = NULL; } else { //We assigned no value to this instance, //it comes from somebody else?? } //Clear current instance pro_clear_instance_info(ii); //If enough instances are ready to // be opened, start phase2 for them leader_open_instances_p2_new(); }
static void leader_check_p1_pending() { iid_t iid_iterator; p_inst_info * ii; //Create an empty prepare batch in send buffer sendbuf_clear(to_acceptors, prepare_reqs, this_proposer_id); LOG(DBG, ("Checking pending phase 1 from %lu to %lu\n", current_iid, p1_info.highest_open)); //Get current time for checking expired struct timeval time_now; gettimeofday(&time_now, NULL); for(iid_iterator = current_iid; iid_iterator <= p1_info.highest_open; iid_iterator++) { //Get instance from state array ii = GET_PRO_INSTANCE(iid_iterator); assert(ii->iid == iid_iterator); //Still pending -> it's expired if(ii->status == p1_pending && leader_is_expired(&ii->timeout, &time_now)) { LOG(DBG, ("Phase 1 of instance %ld expired!\n", ii->iid)); //Reset fields used for previous phase 1 ii->promises_bitvector = 0; ii->promises_count = 0; ii->p1_value_ballot = 0; if(ii->p1_value != NULL) { PAX_FREE(ii->p1_value); } ii->p1_value = NULL; //Ballot is incremented ii->my_ballot = NEXT_BALLOT(ii->my_ballot); //Send prepare to acceptors sendbuf_add_prepare_req(to_acceptors, ii->iid, ii->my_ballot); leader_set_expiration(ii, P1_TIMEOUT_INTERVAL); COUNT_EVENT(p1_timeout); } } //Send if something is still there sendbuf_flush(to_acceptors); }
// Scan trough p1_ready that have a value // assigned or found, execute phase2 static void leader_open_instances_p2_expired() { unsigned int count = 0; //Create a batch of accept requests sendbuf_clear(to_acceptors, accept_reqs, this_proposer_id); //Start new phase 2 for all instances found in status p1_ready // if they are in the range below, phase2 timed-out and // we went successfully trough phase1 again iid_t iid; p_inst_info * ii; for(iid = current_iid; iid < p2_info.next_unused_iid; iid++) { ii = GET_PRO_INSTANCE(iid); assert(ii->iid == iid); //This instance is in status p1_ready but it's in the range // of previously opened phase2, it's now time to retry if(ii->status == p1_ready) { assert(ii->p2_value != NULL || ii->p2_value != NULL); leader_execute_p2(ii); //Count opened count += 1; } } //Count p1_ready that were consumed p1_info.ready_count -= count; //Flush last accept_req batch sendbuf_flush(to_acceptors); LOG(DBG, ("Opened %u old (timed-out) instances\n", count)); }
static void leader_periodic_p2_check(int fd, short event, void *arg) { UNUSED_ARG(fd); UNUSED_ARG(event); UNUSED_ARG(arg); // create a prepare batch for expired instances sendbuf_clear(to_acceptors, prepare_reqs, this_proposer_id); struct timeval now; gettimeofday(&now, NULL); // from current to highest open, check deadline // if instances in status p2_pending iid_t i; p_inst_info * ii; for(i = current_iid; i < p2_info.next_unused_iid; i++) { ii = GET_PRO_INSTANCE(i); //Not p2_pending, skip if(ii->status != p2_pending) { continue; } //Check if it was closed in the meanwhile // (but not delivered yet) if(learner_is_closed(i)) { ii->status = p2_completed; p2_info.open_count -= 1; //The rest (i.e. answering client) // is done when the value is actually delivered LOG(VRB, ("Instance %lu closed, waiting for deliver\n", i)); continue; } //Not expired yet, skip if(!leader_is_expired(&ii->timeout, &now)) { continue; } //Expired and not closed: must restart from phase 1 ii->status = p1_pending; p1_info.pending_count += 1; ii->my_ballot = NEXT_BALLOT(ii->my_ballot); //Send prepare to acceptors sendbuf_add_prepare_req(to_acceptors, ii->iid, ii->my_ballot); leader_set_expiration(ii, P1_TIMEOUT_INTERVAL); LOG(VRB, ("Instance %lu restarts from phase 1\n", i)); COUNT_EVENT(p2_timeout); } //Flush last message if any sendbuf_flush(to_acceptors); //Open new instances leader_open_instances_p2_new(); //Set next invokation of this function leader_set_next_p2_check(); }
static void leader_open_instances_p2_new() { unsigned int count = 0; p_inst_info * ii; //For better batching, opening new instances at the end // is preferred when more than 1 can be opened together unsigned int treshold = (PROPOSER_P2_CONCURRENCY/3)*2; if (p2_info.open_count > treshold) { LOG(DBG, ("Skipping Phase2 open, %u are still active (tresh:%u)\n", p2_info.open_count, treshold)); return; } LOG(DBG, ("Could open %u p2 instances\n", (PROPOSER_P2_CONCURRENCY - p2_info.open_count))); //Create a batch of accept requests sendbuf_clear(to_acceptors, accept_reqs, this_proposer_id); //Start new phase 2 while there is some value from // client to send and we can open more concurrent instances while((count + p2_info.open_count) <= PROPOSER_P2_CONCURRENCY) { ii = GET_PRO_INSTANCE(p2_info.next_unused_iid); assert(ii->p2_value == NULL); //No value to send for next unused, stop if(ii->p1_value == NULL && vh_pending_list_size() == 0) { LOG(DBG, ("No value to use for next instance\n")); break; } int normal_ready = ii->status == p1_ready && ii->iid != p2_info.next_unused_iid; int inf_ready = ii->status == empty && inf_prepare_info.status == p1_ready; int ready = normal_ready | inf_ready; assert(ii->p2_value == NULL || inf_ready); //Next unused is not ready, stop if(!ready) { LOG(DBG, ("Next instance to use for P2 (iid:%lu) is not ready yet\n", p2_info.next_unused_iid)); LOG(DBG, ("%d %d %d %d\n", ii->status, inf_prepare_info.status, ii->iid, p2_info.next_unused_iid)) COUNT_EVENT(p2_waits_p1); break; } if(inf_ready) { ii->iid = p2_info.next_unused_iid; } //Executes phase2, sending an accept request //Using the found value or getting the next from list leader_execute_p2(ii); //Count opened count += 1; //Update next to use p2_info.next_unused_iid += 1; } //Count newly opened p2_info.open_count += count; //Flush last accept_req batch sendbuf_flush(to_acceptors); if(count > 0) { LOG(DBG, ("Opened %u new instances\n", count)); } }