/*检查超时的提案，进行重新提议*/
static void proposer_check_timeouts(evutil_socket_t fd, short event, void* arg)
{
struct evproposer* p = arg;
struct timeout_iterator* iter = proposer_timeout_iterator(p->state);
/*第一个阶段超时提案*/
prepare_req* pr;
while((pr == timeout_iterator_prepare(iter)) != NULL){ /*获取超时的提案(第一阶段)*/
paxos_log_info("Instance %d timed out.", pr->iid);
/*对超时提案重新发起提议过程*/
send_prepares(p, pr);
free(pr);
}
accept_req* ar;
while((ar = timeout_iterator_accept(iter)) != NULL){ /*获得超时提案(第二阶段)*/
paxos_log_info("Instance %d timed out.", ar->iid);
send_accepts(p, ar);
free(ar);
}
/*释放超时管理的迭代器*/
timeout_iterator_free(iter);
/*插入一个定时器*/
event_add(p->timeout_ev, &p->tv);
}
static void
connect_peer(struct peer* p)
{
bufferevent_enable(p->bev, EV_READ|EV_WRITE);
bufferevent_socket_connect(p->bev,
(struct sockaddr*)&p->addr, sizeof(p->addr));
socket_set_nodelay(bufferevent_getfd(p->bev));
paxos_log_info("Connect to %s:%d",
inet_ntoa(p->addr.sin_addr), ntohs(p->addr.sin_port));
}
static void
proposer_check_timeouts(evutil_socket_t fd, short event, void *arg)
{
struct evproposer* p = arg;
struct timeout_iterator* iter = proposer_timeout_iterator(p->state);
prepare_req* pr;
while ((pr = timeout_iterator_prepare(iter)) != NULL) {
paxos_log_info("Instance %d timed out.", pr->iid);
send_prepares(p, pr);
free(pr);
}
accept_req* ar;
while ((ar = timeout_iterator_accept(iter)) != NULL) {
paxos_log_info("Instance %d timed out.", ar->iid);
send_accepts(p, ar);
free(ar);
}
timeout_iterator_free(iter);
event_add(p->timeout_ev, &p->tv); //Thus, if you want to make the event pending again, you can call event_add() on it again from inside the callback function.将超时事件重新加入
}
static void on_accept(struct evconnlistener* l, evutil_socket_t* fd, struct sockaddr* addr, int socklen, void *arg)
{
struct tcp_receiver* r = arg;
struct event_base* b = evconnlistener_get_base(l);
struct bufferevent *bev = bufferevent_socket_new(b, fd, BEV_OPT_CLOSE_ON_FREE);
/*设置读事件函数和错误函数*/
bufferevent_setcb(bev, on_read, NULL, on_error, arg);
/*设置监视的socket事件*/
bufferevent_enable(bev, EV_READ|EV_WRITE);
/*添加到事件管理器中*/
carray_push_back(r->bevs, bev);
paxos_log_info("Accepted connection from %s:%d", inet_ntoa(((struct sockaddr_in*)addr)->sin_addr), ntohs(((struct sockaddr_in*)addr)->sin_port));
}
static int
lmdb_storage_open(void* handle)
{
struct lmdb_storage* s = handle;
char* lmdb_env_path = NULL;
struct stat sb;
int dir_exists, result;
size_t lmdb_env_path_length = strlen(paxos_config.lmdb_env_path) + 16;
lmdb_env_path = malloc(lmdb_env_path_length);
snprintf(lmdb_env_path, lmdb_env_path_length, "%s_%d",
paxos_config.lmdb_env_path, s->acceptor_id);
// Trash bdb files -- testing only
if (paxos_config.trash_files) {
char rm_command[600];
sprintf(rm_command, "rm -r %s", lmdb_env_path);
system(rm_command);
}
dir_exists = (stat(lmdb_env_path, &sb) == 0);
if (!dir_exists && (mkdir(lmdb_env_path, S_IRWXU) != 0)) {
paxos_log_error("Failed to create env dir %s: %s",
lmdb_env_path, strerror(errno));
goto error;
}
if ((result = lmdb_storage_init(s, lmdb_env_path) != 0)) {
paxos_log_error("Failed to open DB handle");
} else {
paxos_log_info("lmdb storage opened successfully");
goto cleanup_exit;
}
error:
if (s) {
lmdb_storage_close(s);
}
return -1;
cleanup_exit:
if (lmdb_env_path) {
free(lmdb_env_path);
}
return 0;
}
static int
lmdb_storage_close(void* handle)
{
struct lmdb_storage* s = handle;
if (s->txn) {
mdb_txn_abort(s->txn);
}
if (s->dbi) {
mdb_close(s->env, s->dbi);
}
if (s->env) {
mdb_env_close(s->env);
}
free(s);
paxos_log_info("lmdb storage closed successfully");
return 0;
}
int
storage_close(struct storage* s)
{
int result = 0;
DB* dbp = s->db;
DB_ENV* dbenv = s->env;
if (dbp->close(dbp, 0) != 0) {
paxos_log_error("DB_ENV close failed");
result = -1;
}
if (dbenv->close(dbenv, 0) != 0) {
paxos_log_error("DB close failed");
result = -1;
}
free(s);
paxos_log_info("Berkeley DB storage closed successfully");
return result;
}
struct tcp_receiver* tcp_receiver_new(struct event_base* b, int port, bufferevent_data_cb cb, void* arg)
{
struct tcp_receiver* r;
struct sockaddr_in sin;
unsigned flags = LEV_OPT_CLOSE_ON_EXEC | LEV_OPT_CLOSE_ON_FREE | LEV_OPT_REUSEABLE;
r = malloc(sizeof(struct tcp_receiver));
set_sockaddr_in(&sin, port);
r->callback = cb;
r->arg = arg;
/*libevent listener和bind端口*/
r->listener = evconnlistener_new_bind(b, on_accept, r, flags, -1, (struct sockaddr*)&sin, sizeof(sin));
assert(r->listener != NULL);
/*设置listen error 回调处理*/
evconnlistener_set_error_cb(r->listener, on_listener_error);
r->bevs = carray_new(10);
paxos_log_info("Listening on port %d", port);
return r;
}
static void
on_accept(struct evconnlistener *l, evutil_socket_t fd,
struct sockaddr* addr, int socklen, void *arg)
{
struct peer* peer;
struct peers* peers = arg;
peers->clients = realloc(peers->clients,
sizeof(struct peer*) * (peers->clients_count+1));
peers->clients[peers->clients_count] =
make_peer(peers, peers->clients_count, (struct sockaddr_in*)addr);
peer = peers->clients[peers->clients_count];
bufferevent_setfd(peer->bev, fd);
bufferevent_setcb(peer->bev, on_read, NULL, on_client_event, peer);
bufferevent_enable(peer->bev, EV_READ|EV_WRITE);
socket_set_nodelay(fd);
paxos_log_info("Accepted connection from %s:%d",
inet_ntoa(((struct sockaddr_in*)addr)->sin_addr),
ntohs(((struct sockaddr_in*)addr)->sin_port));
peers->clients_count++;
}
static void
on_peer_event(struct bufferevent* bev, short ev, void *arg)
{
struct peer* p = (struct peer*)arg;
if (ev & BEV_EVENT_CONNECTED) {
paxos_log_info("Connected to %s:%d",
inet_ntoa(p->addr.sin_addr), ntohs(p->addr.sin_port));
p->status = ev;
} else if (ev & BEV_EVENT_ERROR || ev & BEV_EVENT_EOF) {
struct event_base* base;
int err = EVUTIL_SOCKET_ERROR();
paxos_log_error("%s (%s:%d)", evutil_socket_error_to_string(err),
inet_ntoa(p->addr.sin_addr), ntohs(p->addr.sin_port));
base = bufferevent_get_base(p->bev);
bufferevent_free(p->bev);
p->bev = bufferevent_socket_new(base, -1, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setcb(p->bev, on_read, NULL, on_peer_event, p);
event_add(p->reconnect_ev, &reconnect_timeout);
p->status = ev;
} else {
paxos_log_error("Event %d not handled", ev);
}
}
int
peers_listen(struct peers* p, int port)
{
struct sockaddr_in addr;
unsigned flags = LEV_OPT_CLOSE_ON_EXEC
| LEV_OPT_CLOSE_ON_FREE
| LEV_OPT_REUSEABLE;
/* listen on the given port at address 0.0.0.0 */
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(0);
addr.sin_port = htons(port);
p->listener = evconnlistener_new_bind(p->base, on_accept, p, flags, -1,
(struct sockaddr*)&addr, sizeof(addr));
if (p->listener == NULL) {
paxos_log_error("Failed to bind on port %d", port);
return 0;
}
evconnlistener_set_error_cb(p->listener, on_listener_error);
paxos_log_info("Listening on port %d", port);
return 1;
}
static int
bdb_init_tx_handle(struct storage* s, char* db_env_path)
{
int result;
DB_ENV* dbenv;
//Create environment handle
result = db_env_create(&dbenv, 0);
if (result != 0) {
paxos_log_error("DB_ENV creation failed: %s", db_strerror(result));
return -1;
}
//Durability mode
if (!paxos_config.bdb_sync)
result = dbenv->set_flags(dbenv, DB_TXN_WRITE_NOSYNC, 1);
if (result != 0) {
paxos_log_error("DB_ENV set_flags failed: %s", db_strerror(result));
return -1;
}
//Redirect errors to sdout
dbenv->set_errfile(dbenv, stdout);
//Set the size of the memory cache
result = dbenv->set_cachesize(dbenv, 0, paxos_config.bdb_cachesize, 1);
if (result != 0) {
paxos_log_error("DB_ENV set_cachesize failed: %s",
db_strerror(result));
return -1;
}
//TODO see page size impact
//Set page size for this db
// result = dbp->set_pagesize(dbp, pagesize);
// assert(result == 0);
//FIXME set log size
// Environment open flags
int flags;
flags =
DB_CREATE | /* Create if not existing */
DB_RECOVER | /* Run normal recovery. */
DB_INIT_LOCK | /* Initialize the locking subsystem */
DB_INIT_LOG | /* Initialize the logging subsystem */
DB_INIT_TXN | /* Initialize the transactional subsystem. */
DB_THREAD | /* Cause the environment to be free-threaded */
DB_REGISTER |
DB_INIT_MPOOL; /* Initialize the memory pool (in-memory cache) */
//Open the DB environment
result = dbenv->open(dbenv,
db_env_path, /* Environment directory */
flags, /* Open flags */
0); /* Default file permissions */
if (result != 0) {
paxos_log_error("DB_ENV open failed: %s", db_strerror(result));
return -1;
}
paxos_log_info("Berkeley DB storage opened successfully");
s->env = dbenv;
return 0;
}
