示例#1
0
static void handle_missing_ack(consensus_component* comp,void* data){
    
    missing_ack* msg = data;
    request_record* origin = (request_record*)msg->data;
    SYS_LOG(comp,"Node %d Handle Missing Ack From Node %d.\n",
            comp->node_id,msg->node_id);
    if(view_stamp_comp(comp->highest_committed_vs,&msg->missing_vs)>=0){
        goto handle_missing_ack_exit;
    }else{
       db_key_type record_no = vstol(&msg->missing_vs);
       request_record* record_data = NULL;
       size_t data_size;
       retrieve_record(comp->db_ptr,sizeof(record_no),&record_no,&data_size,(void**)&record_data);

       if(record_data!=NULL){
           goto handle_missing_ack_exit;
       }

       record_data =(request_record*)malloc(REQ_RECORD_SIZE(origin));

       if(record_data==NULL){
           goto handle_missing_ack_exit;
       }

       gettimeofday(&record_data->created_time,NULL);
       record_data->data_size = origin->data_size;
       memcpy(record_data->data,origin->data,origin->data_size);
       store_record(comp->db_ptr,sizeof(record_no),&record_no,REQ_RECORD_SIZE(record_data),record_data);
    }
    try_to_execute(comp);
handle_missing_ack_exit:
    
    return;
};
示例#2
0
static void deliver_msg_data(consensus_component* comp,view_stamp* vs){

    // in order to accelerate deliver process of the program
    // we may just give the record number instead of the real data 
    // to the proxy, and then the proxy will take the overhead of database operation
    
    db_key_type vstokey = vstol(vs);
    if(comp->deliver_mode==1)
    {
        request_record* data = NULL;
        size_t data_size=0;
        retrieve_record(comp->db_ptr,sizeof(db_key_type),&vstokey,&data_size,(void**)&data);
        SYS_LOG(comp,"Node %d Deliver View Stamp %u : %u To The User.\n",
        comp->node_id,vs->view_id,vs->req_id);
        STAT_LOG(comp,"Request:%lu\n",vstokey);
        if(NULL!=data){
            if(comp->ucb!=NULL){
                comp->ucb(data->data_size,data->data,comp->up_para);
            }else{
                SYS_LOG(comp,"No Such Call Back Func.\n");
            }
        }
    }else{
        STAT_LOG(comp,"Request %lu.\n",vstokey);
        if(comp->ucb!=NULL){
            comp->ucb(sizeof(db_key_type),&vstokey,comp->up_para);
        }else{
            SYS_LOG(comp,"No Such Call Back Func.\n");
        }
    }
    return;
}
示例#3
0
static void handle_accept_ack(consensus_component* comp,void* data){
    accept_ack* msg = data;
    // if currently the node is not the leader, then it should ignore all the
    // accept ack, because that can must be the msg from previous view
    SYS_LOG(comp,"Node %d Handle Accept Ack From Node %u.\n",
            comp->node_id,msg->node_id);
    if(comp->my_role!=LEADER){
        goto handle_accept_ack_exit;
    }
    // the request has reached quorum
    if(view_stamp_comp(&msg->msg_vs,comp->highest_committed_vs)<=0){
        goto handle_accept_ack_exit;
    }
    db_key_type record_no = vstol(&msg->msg_vs);
    request_record* record_data = NULL;
    size_t data_size;
    retrieve_record(comp->db_ptr,sizeof(record_no),&record_no,&data_size,(void**)&record_data);
    if(record_data==NULL){
        SYS_LOG(comp,"Received Ack To Non-Exist Record %lu.\n",
                record_no);
        goto handle_accept_ack_exit;
    }
    update_record(record_data,msg->node_id);
    // we do not care about whether the update is successful, otherwise this can
    // be treated as a message loss
    store_record(comp->db_ptr,sizeof(record_no),&record_no,REQ_RECORD_SIZE(record_data),record_data);
handle_accept_ack_exit:
    try_to_execute(comp);
    return;
};
示例#4
0
void mgr_on_accept(int fd, event_manager* ev_mgr)
{
    if (internal_threads(ev_mgr->excluded_threads, pthread_self()))
        return;

    uint32_t leader_id = get_leader_id(ev_mgr->con_node);
    if (ev_mgr->node_id == leader_id)
    {
        leader_tcp_pair* new_conn = malloc(sizeof(leader_tcp_pair));
        memset(new_conn,0,sizeof(leader_tcp_pair));

        new_conn->key = fd;
        HASH_ADD_INT(ev_mgr->leader_tcp_map, key, new_conn);

        rsm_op(ev_mgr->con_node, 0, NULL, P_TCP_CONNECT, &new_conn->vs);
    } else {
        request_record* retrieve_data = NULL;
        size_t data_size;
    
        while (retrieve_data == NULL){
            retrieve_record(ev_mgr->db_ptr, sizeof(db_key_type), &ev_mgr->cur_rec, &data_size, (void**)&retrieve_data);
        }
        replica_tcp_pair* ret = NULL;
        HASH_FIND(hh, ev_mgr->replica_tcp_map, &retrieve_data->clt_id, sizeof(view_stamp), ret);
        ret->s_p = fd;
        ret->accepted = 1;
    }

    return;
}
示例#5
0
// leader has another responsibility to update the highest request that can be executed,
// and if the leader is also synchronous, it can execute the record in this stage
static void leader_try_to_execute(consensus_component* comp){
    SYS_LOG(comp, "highest_seen_req_id %lu.\n", comp->highest_seen_vs->req_id);
    SYS_LOG(comp, "highest_seen_view_id %lu.\n", comp->highest_seen_vs->view_id);
    SYS_LOG(comp, "highest_to_commit_vs_req_id %lu.\n", comp->highest_to_commit_vs->req_id);
    SYS_LOG(comp, "highest_to_commit_vs_view_id %lu.\n", comp->highest_to_commit_vs->view_id);
    db_key_type start;
    db_key_type end = vstol(comp->highest_seen_vs);;

    size_t data_size;
    view_stamp temp_boundary;
    view_boundary* boundary_record = NULL;
    if(comp->highest_seen_vs->view_id != comp->highest_to_commit_vs->view_id){
        // address the boundary
        
        assert(comp->highest_to_commit_vs->view_id + 1 == comp->highest_seen_vs->view_id);
        comp->highest_to_commit_vs->view_id += 1;
        comp->highest_to_commit_vs->req_id = 0;
        comp->highest_committed_vs->view_id = comp->highest_to_commit_vs->view_id;
        comp->highest_committed_vs->req_id = comp->highest_to_commit_vs->req_id;
        start = vstol(comp->highest_to_commit_vs); 
    } else{
        start = vstol(comp->highest_to_commit_vs)+1;
    }

    int exec_flag = (!view_stamp_comp(comp->highest_committed_vs,comp->highest_to_commit_vs));
    request_record* record_data = NULL;
    SYS_LOG(comp,"The Leader Tries To Execute.\n");
    SYS_LOG(comp,"The Start Value Is %lu.\n",start);
    SYS_LOG(comp,"The End Value Is %lu.\n",end);
    for(db_key_type index=start;index<=end;index++){
        retrieve_record(comp->db_ptr,sizeof(index),&index,&data_size,(void**)&record_data);
        assert(record_data!=NULL && "The Record Should Be Inserted By The Node Itself!");
        if(reached_quorum(record_data,comp->group_size)){
            view_stamp temp = ltovs(index);
            SYS_LOG(comp,"Node %d : View Stamp %u : %u Has Reached Quorum.\n",
                    comp->node_id,temp.view_id,temp.req_id);
            
            SYS_LOG(comp,"Before Node %d Inc Execute  %u : %u.\n",
                    comp->node_id,
                    comp->highest_to_commit_vs->view_id,
                    comp->highest_to_commit_vs->req_id);
            view_stamp_inc(comp->highest_to_commit_vs);
            SYS_LOG(comp,"After Node %d Inc Execute  %u : %u.\n",
                    comp->node_id,
                    comp->highest_to_commit_vs->view_id,
                    comp->highest_to_commit_vs->req_id);

            if(exec_flag){
                view_stamp vs = ltovs(index);
                deliver_msg_data(comp,&vs);
                view_stamp_inc(comp->highest_committed_vs);
            }
        }else{
            return;
        }
    }
}
示例#6
0
static void update_state(db_key_type index,void* arg){
    event_manager* ev_mgr = arg;

    request_record* retrieve_data = NULL;
    size_t data_size;

    retrieve_record(ev_mgr->db_ptr, sizeof(index), &index, &data_size, (void**)&retrieve_data);
    ev_mgr->cur_rec = index;

    FILE* output = NULL;
    if(ev_mgr->req_log){
        output = ev_mgr->req_log_file;
    }
    switch(retrieve_data->type){
        case P_TCP_CONNECT:
            if(output!=NULL){
                fprintf(output,"Operation: Connects.\n");
            }
            do_action_tcp_connect(retrieve_data->clt_id,arg);
            break;
        case P_UDP_CONNECT:
            if(output!=NULL){
                fprintf(output,"Operation: Connects.\n");
            }
            do_action_udp_connect(retrieve_data->clt_id,arg);
            break;
        case P_SEND:
            if(output!=NULL){
                fprintf(output,"Operation: Sends data.\n");
            }
            do_action_send(retrieve_data,arg);
            break;
        case P_CLOSE:
            if(output!=NULL){
                fprintf(output,"Operation: Closes.\n");
            }
            do_action_close(retrieve_data->clt_id,arg);
            break;
        case P_NOP:
            if(output!=NULL){
                fprintf(output,"Operation: NOP.\n");
            }
            break; // nop is only for sending the close() consensus result to the replicas
        default:
            break;
    }
    return;
}
示例#7
0
static void* build_missing_ack(consensus_component* comp,view_stamp* vs){
    missing_ack* msg = NULL;
    SYS_LOG(comp,"In Missing Ack, The View Stamp Is %u : %u.\n",
            vs->view_id,vs->req_id);
    db_key_type record_no = vstol(vs);
    request_record* record_data = NULL;
    size_t data_size;
    retrieve_record(comp->db_ptr,sizeof(record_no),&record_no,&data_size,(void**)&record_data);
    if(NULL!=record_data){
        int memsize = MISSING_ACK_SIZE(record_data);
        msg=(missing_ack*)malloc(memsize);
        if(NULL!=msg){
            msg->node_id = comp->node_id;
            msg->data_size = memsize;
            msg->header.msg_type = MISSING_ACK;
            memcpy(msg->data,record_data,memsize);
            msg->missing_vs = *vs;
        }
    }
    return msg;
};
示例#8
0
void consensus_make_progress(struct consensus_component_t* comp){
    if(LEADER!=comp->my_role){
        goto make_progress_exit;
    }
    leader_try_to_execute(comp);
    SYS_LOG(comp,"Let's Make Progress.\n");
    if((view_stamp_comp(comp->highest_committed_vs,comp->highest_seen_vs)<0)&& (comp->highest_seen_vs->view_id==comp->cur_view->view_id)){
        view_stamp temp;
        temp.view_id = comp->cur_view->view_id;
        temp.req_id = 0;
        if(view_stamp_comp(&temp,comp->highest_committed_vs)<0){
            temp = *(comp->highest_committed_vs);
        }
        temp.req_id++;
        record_index_type start =  vstol(&temp);
        record_index_type end = vstol(comp->highest_seen_vs);
        for(record_index_type index = start;index<=end;index++){
            request_record* record_data = NULL;
            size_t data_size=0;
            view_stamp temp_vs = ltovs(index);
            retrieve_record(comp->db_ptr,sizeof(db_key_type),&index,&data_size,(void**)&record_data);
            if(!reached_quorum(record_data,comp->group_size)){
                accept_req* msg = build_accept_req(comp,REQ_RECORD_SIZE(record_data),record_data,&temp_vs);
                if(NULL==msg){
                    continue;
                }else{
                    comp->uc(comp->my_node,ACCEPT_REQ_SIZE(msg),msg,-1);
                    free(msg);
                }
            }
        } 
    }
    force_exec* msg = build_force_exec(comp); 
    if(NULL==msg){goto make_progress_exit;}
    comp->uc(comp->my_node,FORCE_EXEC_SIZE,msg,-1);
    free(msg);
make_progress_exit:
    return;
};
示例#9
0
void *handle_accept_req(void* arg)
{
    consensus_component* comp = arg;

    db_key_type start;
    db_key_type end;
    db_key_type index;

    size_t data_size;
    request_record* retrieve_data = NULL;

    int sock;

    struct timeval start_time;
    struct timeval end_time;
    unsigned long e_usec;
    
    while (1)
    {
        log_entry* new_entry = (log_entry*)((char*)shared_memory.shm[comp->node_id] + shared_memory.log->tail);
        
        if (new_entry->req_canbe_exed.view_id != 0)//TODO atmoic opeartion
        {
            gettimeofday(&start, 0);
            if(new_entry->msg_vs.view_id < comp->cur_view.view_id){
                // TODO
                //goto reloop;
            }
            // if we this message is not from the current leader
            if(new_entry->msg_vs.view_id == comp->cur_view.view_id && new_entry->node_id != comp->cur_view.leader_id){
                // TODO
                //goto reloop;
            }

            // update highest seen request
            if(view_stamp_comp(new_entry->msg_vs, comp->highest_seen_vs) > 0){
                comp->highest_seen_vs = new_entry->msg_vs;
            }

            db_key_type record_no = vstol(new_entry->msg_vs);
            request_record* record_data = (request_record*)malloc(new_entry->data_size + sizeof(request_record));

            gettimeofday(&record_data->created_time, NULL);
            record_data->data_size = new_entry->data_size;
            memcpy(record_data->data, new_entry->data, new_entry->data_size);

            // record the data persistently 
            store_record(comp->db_ptr, sizeof(record_no), &record_no, REQ_RECORD_SIZE(record_data), record_data);
            uint64_t offset = shared_memory.tail + sizeof(accept_ack) * comp->node_id;
            shared_memory.tail = shared_memory.tail + log_entry_len(new_entry);

            accept_ack* reply = (accept_ack*)((char*)new_entry + ACCEPT_ACK_SIZE * comp->node_id);
            reply->node_id = comp->node_id;
            reply->msg_vs.view_id = new_entry->msg_vs.view_id;
            reply->msg_vs.req_id = new_entry->msg_vs.req_id;

            memcpy((void*)((char*)shared_memory.shm[new_entry->node_id] + offset), reply, ACCEPT_ACK_SIZE);

            free(record_data);
            if(view_stamp_comp(new_entry->req_canbe_exed, comp->committed) > 0)
            {
                sock = socket(AF_INET, SOCK_STREAM, 0);
                connect(sock, (struct sockaddr*)&comp->sys_addr.c_addr, comp->sys_addr.c_sock_len); //TODO: why? Broken pipe. Maybe the server closes the socket
                start = vstol(comp->committed)+1;
                end = vstol(new_entry->req_canbe_exed);
                for(index = start; index <= end; index++)
                {
                    retrieve_record(comp->db_ptr, sizeof(index), &index, &data_size, (void**)&retrieve_data);
                    send(sock, retrieve_data->data, retrieve_data->data_size, 0);
                }
                comp->committed = new_entry->req_canbe_exed;
            }
            gettimeofday(&end, 0);
            e_usec = ((end.tv_sec * 1000000) + end.tv_usec) - ((start.tv_sec * 1000000) + start.tv_usec);
            CON_LOG(comp, "%lu\n", e_usec);
        }
    }
};
示例#10
0
static void try_to_execute(consensus_component* comp){
    // there we have assumption, for the currently leader,whose commited request
    // and highest request to execute must be in the same view, otherwise, the
    // leader cannot be the leader 
    
    SYS_LOG(comp,"Node %d Try To Execute.\n",
            comp->node_id);
    if(comp->cur_view->view_id==0){
        SYS_LOG(comp,"Node %d Currently Is A NULL Node\n",
                comp->node_id);
        goto try_to_execute_exit;
    }
    if(LEADER==comp->my_role){
        leader_try_to_execute(comp);
    }
    db_key_type start = vstol(comp->highest_committed_vs)+1;
    db_key_type end;
    view_stamp temp_boundary;
    view_boundary* boundary_record = NULL;
    size_t data_size;
    if(comp->highest_committed_vs->view_id!=comp->highest_to_commit_vs->view_id){
        
        SYS_LOG(comp, "highest_to_commit_vs_req_id %lu.\n", comp->highest_to_commit_vs->req_id);
        SYS_LOG(comp, "highest_to_commit_vs_view_id %lu.\n", comp->highest_to_commit_vs->view_id);
        SYS_LOG(comp, "highest_committed_vs_req_id %lu.\n", comp->highest_committed_vs->req_id);
        SYS_LOG(comp, "highest_committed_vs_view_id %lu.\n", comp->highest_committed_vs->view_id);
        //address the boundary
        view_stamp bound;
        bound.view_id = comp->highest_committed_vs->view_id+1;
        bound.req_id = 0;
        db_key_type bound_record_no = vstol(&bound);
        retrieve_record(comp->db_ptr,sizeof(bound_record_no),&bound_record_no,&data_size,(void**)&boundary_record);
        if(NULL==boundary_record){
           SYS_LOG(comp, "Missing bounday_record.\n");
           send_missing_req(comp,&bound); 
           goto try_to_execute_exit;
        }
        temp_boundary.view_id = boundary_record->view_id;
        temp_boundary.req_id = boundary_record->req_id;
        SYS_LOG(comp, "boundary_record_req_id %lu.\n", boundary_record->req_id);
        SYS_LOG(comp, "boundary_record_view_id %lu.\n", boundary_record->view_id);
        end = vstol(&temp_boundary);
    }else{
        end = vstol(comp->highest_to_commit_vs);
    }
    SYS_LOG(comp,"The End Value Is %lu.\n", end);
    request_record* record_data = NULL;
    // we can only execute thins in sequence
    int exec_flag = 1;
    view_stamp missing_vs;
    for(db_key_type index = start;index<=end;index++){
        missing_vs = ltovs(index);
        if(0!=retrieve_record(comp->db_ptr,sizeof(index),&index,
                    &data_size,(void**)&record_data)){
            exec_flag = 0;
            send_missing_req(comp,&missing_vs);
        }
        if(exec_flag){
            deliver_msg_data(comp,&missing_vs);
//            record_data->is_closed = 1;
//            store_record(comp->db_ptr,sizeof(index),&index,REQ_RECORD_SIZE(record_data),record_data);
            view_stamp_inc(comp->highest_committed_vs);
        }
        record_data = NULL;
    }
    if(NULL!=boundary_record){
        temp_boundary.view_id = boundary_record->view_id;
        temp_boundary.req_id = boundary_record->req_id;
        db_key_type op1 = vstol(comp->highest_committed_vs);
        db_key_type op2 = vstol(&temp_boundary);
        if(op1==op2){
            cross_view(comp->highest_committed_vs);
        }
    }
try_to_execute_exit:
    return;
};