static void
mln_ipc_fd_handler_worker_send(mln_event_t *ev, int fd, void *data)
{
mln_tcp_conn_t *conn = &master_conn;
mln_chain_t *c;
int ret;
while ((c = mln_tcp_conn_get_head(conn, M_C_SEND)) != NULL) {
ret = mln_tcp_conn_send(conn);
if (ret == M_C_FINISH) {
continue;
} else if (ret == M_C_NOTYET) {
mln_chain_pool_release_all(mln_tcp_conn_remove(conn, M_C_SENT));
mln_event_set_fd(ev, \
mln_tcp_conn_get_fd(conn), \
M_EV_SEND|M_EV_APPEND|M_EV_NONBLOCK|M_EV_ONESHOT, \
M_EV_UNLIMITED, \
NULL, \
mln_ipc_fd_handler_worker_send);
return;
} else if (ret == M_C_ERROR) {
mln_log(report, "master process dead!\n");
exit(127);
} else {
mln_log(error, "Shouldn't be here.\n");
abort();
}
}
mln_chain_pool_release_all(mln_tcp_conn_remove(conn, M_C_SENT));
}
void mln_socketpair_close_handler(mln_event_t *ev, mln_fork_t *f, int fd)
{
mln_event_set_fd(ev, fd, M_EV_CLR, M_EV_UNLIMITED, NULL, NULL);
enum proc_exec_type etype = f->etype;
enum proc_state_type stype = f->stype;
mln_s8ptr_t *args = f->args;
mln_u32_t n_args = f->n_args;
if (stype == M_PST_SUP) {
mln_fork_destroy(f, 0);
if (etype == M_PET_DFL) {
int rv = mln_fork_restart(ev);
if (rv < 0) {
mln_log(error, "mln_fork_restart() error.\n");
abort();
} else if (rv == 0) {
mln_event_set_break(ev);
} else {
mln_log(report, "Child process restart.\n");
}
} else {/*M_PET_EXE*/
if (mln_fork_spawn(stype, args, n_args, ev) < 0) {
mln_log(error, "mln_fork_spawn() error.\n");
abort();
}
mln_log(report, "Process %s restart.\n", args[0]);
}
} else {/*M_PST_DFL*/
mln_fork_destroy(f, 1);
}
}
/*
* Message Format:
* [Length(include type length) 4bytes|type 4bytes|content Nbytes]
*/
void mln_ipc_fd_handler_master(mln_event_t *ev, int fd, void *data)
{
int ret;
mln_fork_t *f = (mln_fork_t *)data;
mln_tcp_conn_t *conn = &(f->conn);
while (1) {
ret = mln_tcp_conn_recv(conn, M_C_TYPE_MEMORY);
if (ret == M_C_FINISH) {
continue;
} else if (ret == M_C_NOTYET) {
break;
} else if (ret == M_C_CLOSED) {
mln_log(report, "Child process dead!\n");
mln_socketpair_close_handler(ev, f, fd);
return ;
} else {
mln_log(error, "recv msg error. %s\n", strerror(errno));
mln_socketpair_close_handler(ev, f, fd);
return ;
}
}
mln_ipc_fd_handler_master_process(ev, f);
}
int mln_fork_spawn(enum proc_state_type stype, \
mln_s8ptr_t *args, \
mln_u32_t n_args, \
mln_event_t *master_ev)
{
mln_log(none, "Start up process '%s'\n", args[0]);
int ret = do_fork_core(M_PET_EXE, \
stype, \
args, \
n_args, \
master_ev);
if (ret < 0) {
return -1;
} else if (ret == 0) {
char fd_str[256] = {0};
snprintf(fd_str, sizeof(fd_str)-1, "%d", \
mln_tcp_conn_get_fd(&master_conn));
args[n_args] = fd_str;
if (master_ev != NULL) mln_event_destroy(master_ev);
mln_log_destroy();
if (execv(args[0], args) < 0) {
mln_log(error, "execv error, %s\n", strerror(errno));
exit(1);
}
}
return 0;
}
/*mln_set_worker_ipc_handler*/
void mln_set_worker_ipc_handler(mln_ipc_handler_t *ih)
{
mln_rbtree_node_t *rn = mln_rbtree_search(worker_ipc_tree, \
worker_ipc_tree->root, \
ih);
if (rn->data != NULL) {
mln_log(error, "IPC type '%d' already existed.\n", ih->type);
abort();
}
rn = mln_rbtree_new_node(worker_ipc_tree, ih);
if (rn == NULL) {
mln_log(error, "No memory.\n");
abort();
}
mln_rbtree_insert(worker_ipc_tree, rn);
}
static void
mln_ipc_fd_handler_master_process(mln_event_t *ev, mln_fork_t *f)
{
mln_tcp_conn_t *tc = &(f->conn);
while (1) {
while (f->error_bytes) {
f->error_bytes = mln_ipc_discard_bytes(tc, f->error_bytes);
}
switch (f->state) {
case STATE_IDLE:
{
if (mln_ipc_get_buf_with_len(tc, &(f->msg_len), M_F_LENLEN) < 0) {
return;
}
f->state = STATE_LENGTH;
}
case STATE_LENGTH:
{
if (f->msg_content == NULL) {
f->msg_content = malloc(f->msg_len);
if (f->msg_content == NULL) {
f->error_bytes = f->msg_len;
f->state = STATE_IDLE;
break;
}
}
if (mln_ipc_get_buf_with_len(tc, f->msg_content, f->msg_len) < 0) {
return;
}
memcpy(&(f->msg_type), f->msg_content, M_F_TYPELEN);
f->state = STATE_IDLE;
mln_ipc_handler_t ih;
ih.type = f->msg_type;
mln_rbtree_node_t *rn = mln_rbtree_search(master_ipc_tree, \
master_ipc_tree->root, \
&ih);
if (!mln_rbtree_null(rn, master_ipc_tree)) {
mln_ipc_handler_t *ihp = (mln_ipc_handler_t *)(rn->data);
if (ihp->handler != NULL)
ihp->handler(ev, \
f, \
f->msg_content+M_F_TYPELEN, \
f->msg_len-M_F_TYPELEN, \
&(ihp->data));
}
free(f->msg_content);
f->msg_content = NULL;
break;
}
default:
mln_log(error, "No such state!\n");
abort();
}
}
}
static void
mln_ipc_fd_handler_worker_process(mln_event_t *ev, mln_tcp_conn_t *tc)
{
while (1) {
while (child_error_bytes) {
child_error_bytes = mln_ipc_discard_bytes(tc, child_error_bytes);
}
switch (child_state) {
case STATE_IDLE:
{
if (mln_ipc_get_buf_with_len(tc, &(cur_msg_len), M_F_LENLEN) < 0) {
return;
}
child_state = STATE_LENGTH;
}
case STATE_LENGTH:
{
if (child_msg_content == NULL) {
child_msg_content = (mln_u8ptr_t)malloc(cur_msg_len);
if (child_msg_content == NULL) {
child_error_bytes = cur_msg_len;
child_state = STATE_IDLE;
break;
}
}
if (mln_ipc_get_buf_with_len(tc, child_msg_content, cur_msg_len) < 0) {
return;
}
memcpy(&cur_msg_type, child_msg_content, M_F_TYPELEN);
child_state = STATE_IDLE;
mln_ipc_handler_t ih;
ih.type = cur_msg_type;
mln_rbtree_node_t *rn = mln_rbtree_search(worker_ipc_tree, \
worker_ipc_tree->root, \
&ih);
if (!mln_rbtree_null(rn, worker_ipc_tree)) {
mln_ipc_handler_t *ihp = (mln_ipc_handler_t *)(rn->data);
if (ihp->handler != NULL)
ihp->handler(ev, \
tc, \
child_msg_content+M_F_TYPELEN, \
cur_msg_len-M_F_TYPELEN, \
&(ihp->data));
}
free(child_msg_content);
child_msg_content = NULL;
break;
}
default:
mln_log(error, "No such state!\n");
abort();
}
}
}
void mln_conf_get_cmds(mln_conf_t *cf, char *domain, mln_conf_cmd_t **v)
{
mln_conf_domain_t *cd = cf->search(cf, domain);
if (cd == NULL) return;
struct conf_cmds_scan_s ccs;
ccs.cc = v;
ccs.pos = 0;
if (mln_rbtree_scan_all(cd->cmd, cd->cmd->root, mln_conf_get_cmds_scan, (void *)&ccs) < 0) {
mln_log(error, "Shouldn't be here.\n");
abort();
}
}
void mln_ipc_fd_handler_worker(mln_event_t *ev, int fd, void *data)
{
int ret;
mln_tcp_conn_t *conn = &master_conn;
while (1) {
ret = mln_tcp_conn_recv(conn, M_C_TYPE_MEMORY);
if (ret == M_C_FINISH) {
continue;
} else if (ret == M_C_NOTYET) {
break;
} else if (ret == M_C_CLOSED) {
mln_log(report, "Master process dead!\n");
exit(127);
} else {
mln_log(error, "recv msg error. %s\n", strerror(errno));
exit(127);
}
}
mln_ipc_fd_handler_worker_process(ev, conn);
}
void mln_fork_worker_set_events(mln_event_t *ev)
{
if (mln_event_set_fd(ev, \
mln_tcp_conn_get_fd(&master_conn), \
M_EV_RECV|M_EV_NONBLOCK, \
M_EV_UNLIMITED, \
NULL, \
mln_ipc_fd_handler_worker) < 0)
{
mln_log(error, "mln_event_set_fd() failed.\n");
abort();
}
}
static int
do_fork_worker_process(mln_sauto_t n_worker_proc)
{
mln_sauto_t i;
int ret;
for (i = 0; i < n_worker_proc; ++i) {
mln_log(none, "Start up worker process No.%l\n", i+1);
if ((ret = mln_fork_restart(NULL)) < 0) {
continue;
} else if (ret == 0) {
return 0;
}
}
return 1;
}
/*
* events
*/
void mln_fork_master_set_events(mln_event_t *ev)
{
mln_fork_t *f;
for (f = worker_list_head; f != NULL; f = f->next) {
if (mln_event_set_fd(ev, \
mln_tcp_conn_get_fd(&(f->conn)), \
M_EV_RECV|M_EV_NONBLOCK, \
M_EV_UNLIMITED, \
f, \
mln_ipc_fd_handler_master) < 0)
{
mln_log(error, "mln_event_set_fd() failed.\n");
abort();
}
}
}
/*pre-fork*/
int mln_pre_fork(void)
{
if (signal(SIGCHLD, SIG_IGN) == SIG_ERR) {
mln_log(error, "signal() to ignore SIGCHLD failed, %s\n", strerror(errno));
return -1;
}
if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) {
mln_log(error, "signal() to ignore SIGPIPE failed, %s\n", strerror(errno));
return -1;
}
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = do_fork_sig_handler;
sigemptyset(&(act.sa_mask));
sigfillset(&(act.sa_mask));
if (sigaction(wait_signo, &act, NULL) < 0) {
mln_log(error, "sigaction error, signo:%d. %s\n", \
wait_signo, strerror(errno));
return -1;
}
if (mln_tcp_conn_init(&master_conn, -1) < 0) {
mln_log(error, "No memory.\n");
return -1;
}
child_state = STATE_IDLE;
child_error_bytes = 0;
cur_msg_len = 0;
child_msg_content = NULL;
cur_msg_type = 0;
struct mln_rbtree_attr rbattr;
rbattr.cmp = mln_fork_rbtree_cmp;
rbattr.data_free = NULL;
if ((master_ipc_tree = mln_rbtree_init(&rbattr)) < 0) {
mln_log(error, "No memory.\n");
if (mln_tcp_conn_get_fd(&master_conn) >= 0)
close(mln_tcp_conn_get_fd(&master_conn));
mln_tcp_conn_destroy(&master_conn);
return -1;
}
if ((worker_ipc_tree = mln_rbtree_init(&rbattr)) < 0) {
mln_log(error, "No memory.\n");
mln_rbtree_destroy(master_ipc_tree);
master_ipc_tree = NULL;
if (mln_tcp_conn_get_fd(&master_conn) >= 0)
close(mln_tcp_conn_get_fd(&master_conn));
mln_tcp_conn_destroy(&master_conn);
return -1;
}
mln_set_ipc_handlers();
return 0;
}
static int
do_fork_core(enum proc_exec_type etype, \
enum proc_state_type stype, \
mln_s8ptr_t *args, \
mln_u32_t n_args, \
mln_event_t *master_ev)
{
int fds[2];
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) {
mln_log(error, "socketpair() error. %s\n", strerror(errno));
return -1;
}
pid_t pid = fork();
if (pid > 0) {
close(fds[1]);
/*
* In linux 2.6.32-279, there is a loophole in process restart.
* If you use select() or kqueue(), you wouldn't get this problem.
* If I don't add a signal to make parent process wait for child,
* the event which combined with the fd that
* created by socketpair() and used to receive child process messages
* would be tiggered by epoll_wait() with an empty receive buffer.
* If fd is blocking model, the parent routine would be blocked
* in read(). Because there is a moment that child process not be built yet,
* but parent process have already added the fd into the epoll and
* jump into the mln_event_dispatch().
* And then, child process built, that would make the event
* which is triggered in parent process to be a lie.
*/
while (child_startup == 0)
;
child_startup = 0;
struct mln_fork_attr fattr;
fattr.args = args;
fattr.n_args = n_args;
fattr.fd = fds[0];
fattr.pid = pid;
fattr.etype = etype;
fattr.stype = stype;
mln_fork_t *f = mln_fork_init(&fattr);
if (f == NULL) {
mln_log(error, "No memory.\n");
abort();
}
if (master_ev != NULL) {
if (mln_event_set_fd(master_ev, \
mln_tcp_conn_get_fd(&(f->conn)), \
M_EV_RECV, \
M_EV_UNLIMITED, \
f, \
mln_ipc_fd_handler_master) < 0)
{
mln_log(error, "mln_event_set_fd() failed.\n");
abort();
}
}
return 1;
} else if (pid == 0) {
close(fds[0]);
mln_fork_destroy_all();
mln_rbtree_destroy(master_ipc_tree);
if (rs_clr_handler != NULL)
rs_clr_handler(rs_clr_data);
master_ipc_tree = NULL;
mln_tcp_conn_set_fd(&master_conn, fds[1]);
signal(SIGCHLD, SIG_DFL);
signal(wait_signo, SIG_DFL);
if (kill(getppid(), wait_signo) < 0) {
mln_log(error, "kill() error. %s\n", strerror(errno));
abort();
}
return 0;
}
mln_log(error, "fork() error. %s\n", strerror(errno));
return -1;
}
/*
* fork processes
*/
int do_fork(void)
{
mln_conf_t *cf = mln_get_conf();
if (cf == NULL) {
mln_log(error, "configuration crashed.\n");
abort();
}
mln_conf_domain_t *cd = cf->search(cf, "main");
if (cd == NULL) {
mln_log(error, "Domain 'main' NOT existed.\n");
abort();
}
mln_sauto_t n_worker_proc = 0;
mln_conf_cmd_t *cmd = cd->search(cd, "worker_proc");
if (cmd != NULL) {
if (mln_conf_get_argNum(cmd) > 1) {
mln_log(error, "Too many arguments follow 'worker_proc'.\n");
exit(1);
}
mln_conf_item_t *ci = cmd->search(cmd, 1);
if (ci == NULL) {
mln_log(error, "'worker_proc' need an integer argument.\n");
exit(1);
}
if (ci->type != CONF_INT) {
mln_log(error, "'worker_proc' need an integer argument.\n");
exit(1);
}
n_worker_proc = ci->val.i;
if (n_worker_proc < 0) {
mln_log(error, "Invalid value to 'worker_process'.\n");
exit(1);
}
}
if (!do_fork_worker_process(n_worker_proc)) return 0;
mln_conf_cmd_t **v, **cc;
mln_u32_t i, n_args;
mln_conf_item_t *arg_ci;
mln_s8ptr_t *v_args;
mln_u32_t n = mln_conf_get_cmdNum(cf, "exec_proc");
if (n == 0) return 1;
v = (mln_conf_cmd_t **)calloc(n+1, sizeof(mln_conf_cmd_t *));
if (v == NULL) {
mln_log(error, "No memory.\n");
return -1;
}
mln_conf_get_cmds(cf, "exec_proc", v);
for (cc = v; *cc != NULL; ++cc) {
n_args = mln_conf_get_argNum(*cc);
if (n_args == 0) {
mln_log(error, "Demand arguments in 'exec_proc'.\n");
exit(1);
}
v_args = (mln_s8ptr_t *)calloc(n_args+2, sizeof(mln_s8ptr_t));
if (v_args == NULL) {
mln_log(error, "No memory.\n");
continue;
}
for (i = 0; i < n_args; ++i) {
arg_ci = (*cc)->search(*cc, i+1);
if (arg_ci->type != CONF_STR) {
mln_log(error, "Demand string arguments in 'exec_proc'.\n");
exit(1);
}
v_args[i] = (char *)(arg_ci->val.s->data);
}
if (!mln_string_constStrcmp((*cc)->cmd_name, "keepalive")) {
mln_fork_spawn(M_PST_SUP, v_args, n_args, NULL);
} else if (!mln_string_constStrcmp((*cc)->cmd_name, "default")) {
mln_fork_spawn(M_PST_DFL, v_args, n_args, NULL);
} else {
mln_log(error, "Invalid command '%S' in 'exec_proc'.\n", (*cc)->cmd_name);
exit(1);
}
}
free(v);
return 1;
}
static inline int
mln_Allocator_init(mln_salloc_t **sa)
{
if (*sa != NULL) return 0;
int err = 0;
#ifdef MAP_ANONYMOUS
*sa = (mln_salloc_t *)mmap(NULL, \
sizeof(mln_salloc_t), \
PROT_READ|PROT_WRITE, \
MAP_PRIVATE|MAP_ANONYMOUS, \
-1, 0);
#else
*sa = (mln_salloc_t *)mmap(NULL, \
sizeof(mln_salloc_t), \
PROT_READ|PROT_WRITE, \
MAP_PRIVATE|MAP_ANON, \
-1, 0);
#endif
if (*sa == MAP_FAILED) {
mln_log(error, "Allocator init failed. %s\n", strerror(errno));
return -1;
}
mln_index_table_init((*sa)->index_tbl);
(*sa)->current_size = 0;
(*sa)->threshold = 0;
err = MLN_LOCK_INIT(&((*sa)->index_lock));
if (err != 0) {
mln_log(error, "index_lock init error. %s\n", strerror(err));
goto err1;
}
err = MLN_LOCK_INIT(&((*sa)->stat_lock));
if (err != 0) {
mln_log(error, "stat_lock init error. %s\n", strerror(err));
goto err2;
}
if (sa != &mAllocator) {
MLN_LOCK(&mMiscLock);
if (!mForkSet) {
if ((err = pthread_atfork(mln_sa_atfork_prepare, \
mln_sa_atfork_parent, \
mln_sa_atfork_child)) != 0)
{
mln_log(error, "pthread_atfork error. %s\n", strerror(err));
MLN_UNLOCK(&mMiscLock);
goto err3;
}
mForkSet = 1;
}
if (!mKeyInit) {
if ((err = pthread_key_create(&mKey, mln_move)) != 0) {
mln_log(error, "pthread_key_create error. %s\n", strerror(err));
MLN_UNLOCK(&mMiscLock);
goto err3;
}
mKeyInit = 1;
}
if ((err = pthread_setspecific(mKey, *sa)) != 0) {
mln_log(error, "pthread_setspecific error. %s\n", strerror(err));
if (mRef == 0) {
pthread_key_delete(mKey);
mKeyInit = 0;
}
MLN_UNLOCK(&mMiscLock);
goto err3;
}
++mRef;
sa_q_chain_add(&mAHead, &mATail, *sa);
MLN_UNLOCK(&mMiscLock);
return 0;
}
mPageSize = sysconf(_SC_PAGESIZE);
if (mPageSize < 0) {
mPageSize = M_DFL_PAGESIZE;
}
mLargeTree.nil.data = NULL;
mLargeTree.nil.parent = &(mLargeTree.nil);
mLargeTree.nil.left = &(mLargeTree.nil);
mLargeTree.nil.right = &(mLargeTree.nil);
mLargeTree.nil.color = M_SARB_BLACK;
mLargeTree.root = &(mLargeTree.nil);
mLargeTree.cmp = mln_large_tree_cmp;
err = MLN_LOCK_INIT(&mHeapLock);
if (err != 0) {
mln_log(error, "mHeapLock init error. %s\n", strerror(err));
goto err3;
}
err = MLN_LOCK_INIT(&mLargeTreeLock);
if (err != 0) {
mln_log(error, "mLargeTreeLock init error. %s\n", strerror(err));
goto err4;
}
err = MLN_LOCK_INIT(&mMmapLock);
if (err != 0) {
mln_log(error, "mMmapLock init error. %s\n", strerror(err));
goto err5;
}
err = MLN_LOCK_INIT(&mMiscLock);
if (err != 0) {
mln_log(error, "mMiscLock init error. %s\n", strerror(err));
goto err6;
}
return 0;
err6:
MLN_LOCK_DESTROY(&mMmapLock);
err5:
MLN_LOCK_DESTROY(&mLargeTreeLock);
err4:
MLN_LOCK_DESTROY(&mHeapLock);
err3:
MLN_LOCK_DESTROY(&((*sa)->stat_lock));
err2:
MLN_LOCK_DESTROY(&((*sa)->index_lock));
err1:
munmap((*sa), sizeof(mln_salloc_t));
*sa = NULL;
return -1;
}