// listen, accpet client connection
rstatus_t
tw_master_listen(int family, struct sockaddr* addr, socklen_t addrlen, int backlog)
{
rstatus_t status;
int reuse;
socklen_t len;
int sd;
sd = socket(family, SOCK_STREAM, 0);
if (sd < 0) {
log_error("socket failed: %s", strerror(errno));
return NC_ERROR;
}
// reuse addr
reuse = 1;
len = sizeof(reuse);
status = setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &reuse, len);
if (status < 0) {
log_error("reuse of addr failed: %s",
strerror(errno));
return NC_ERROR;
}
// get rid of thundering herd
status = setsockopt(sd, SOL_SOCKET, SO_REUSEPORT, &reuse, len);
if (status < 0) {
log_error("reuse of port failed: %s",
strerror(errno));
return NC_ERROR;
}
status = bind(sd, addr, addrlen);
if (status < 0) {
log_error("bind on p %d: %s",sd, strerror(errno));
return NC_ERROR;
}
status = listen(sd, backlog);
if (status < 0) {
log_error("listen on p %d failed: %s",sd,
sd, strerror(errno));
return NC_ERROR;
}
status = nc_set_nonblocking(sd);
if (status < 0) {
log_error("set nonblock on p", strerror(errno));
return NC_ERROR;
}
return sd;
}
nc_pid_t
nc_spawn_process( nc_spawn_proc_pt proc, int data,
char *name, int reload)
{
unsigned long on;
nc_pid_t pid;
int s;
/*
if (respawn >= 0) {
//s = respawn;
} else {
for (s = 0; s < nc_last_process; s++) {
if (nc_processes[s].pid == -1) {
break;
}
}
if (s == NC_MAX_PROCESSES) {
log_error("no more than %d processes can be spawned",
NC_MAX_PROCESSES);
return NC_INVALID_PID;
}
}*/
for (s = 0; s < nc_last_process; s++) {
if (nc_processes[s].pid == -1) {
break;
}
}
if (s == NC_MAX_PROCESSES) {
log_error("no more than %d processes can be spawned",
NC_MAX_PROCESSES);
return NC_INVALID_PID;
}
// Master to Worker
if (socketpair(AF_UNIX, SOCK_STREAM, 0, nc_processes[s].channel) == -1)
{
log_error("socketpair() failed while spawning \"%s\"", name);
return NC_INVALID_PID;
}
log_debug(LOG_DEBUG, "channel %d:%d", nc_processes[s].channel[0], nc_processes[s].channel[1]);
if (nc_set_nonblocking(nc_processes[s].channel[0]) < 0) {
log_error("set channel %d nonblock failed while spawning %s: %s",
nc_processes[s].channel[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}
if (nc_set_nonblocking(nc_processes[s].channel[1]) < 0) {
log_error("set channel %d nonblock failed while spawning %s: %s",
nc_processes[s].channel[1] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}
/*
on = 1;
if (ioctl(nc_processes[s].channel[0], FIOASYNC, &on) == -1) {
log_error("set channel %d FIOASYNC failed while spawning %s: %s",
nc_processes[s].channel[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}*/
nc_pid = getpid();
if (fcntl(nc_processes[s].channel[0], F_SETOWN, nc_pid) == -1) {
log_error("set channel %d F_SETOWN failed while spawning %s: %s",
nc_processes[s].channel[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}
if (fcntl(nc_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
log_error("set channel %d FD_CLOEXEC failed while spawning %s: %s",
nc_processes[s].channel[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}
if (fcntl(nc_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
log_error("set channel %d FD_CLOEXEC failed while spawning %s: %s",
nc_processes[s].channel[1] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
return NC_INVALID_PID;
}
// Worker to Master
/*
if (socketpair(AF_UNIX, SOCK_STREAM, 0, nc_processes[s].channel_back) == -1)
{
log_error("socketpair() failed while spawning \"%s\"", name);
return NC_INVALID_PID;
}
log_debug(LOG_DEBUG, "channel %d:%d", nc_processes[s].channel_back[0], nc_processes[s].channel_back[1]);
if (nc_set_nonblocking(nc_processes[s].channel_back[0]) < 0) {
log_error("set channel %d nonblock failed while spawning %s: %s",
nc_processes[s].channel_back[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
}
if (nc_set_nonblocking(nc_processes[s].channel_back[1]) < 0) {
log_error("set channel %d nonblock failed while spawning %s: %s",
nc_processes[s].channel_back[1] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
}
nc_pid = getpid();
if (fcntl(nc_processes[s].channel_back[0], F_SETOWN, nc_pid) == -1) {
log_error("set channel %d F_SETOWN failed while spawning %s: %s",
nc_processes[s].channel_back[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
}
if (fcntl(nc_processes[s].channel_back[0], F_SETFD, FD_CLOEXEC) == -1) {
log_error("set channel %d FD_CLOEXEC failed while spawning %s: %s",
nc_processes[s].channel_back[0] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
}
if (fcntl(nc_processes[s].channel_back[1], F_SETFD, FD_CLOEXEC) == -1) {
log_error("set channel %d FD_CLOEXEC failed while spawning %s: %s",
nc_processes[s].channel_back[1] , name, strerror(errno));
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
}
*/
nc_worker_channel = nc_processes[s].channel[1];
//nc_worker_channel_write = nc_processes[s].channel_back[1];
//global variable
nc_process_slot = s;
pid = fork();
switch (pid) {
case -1:
log_error("fork() failed while spawning %s: %s",
name, strerror(errno));
nc_close_channel(nc_processes[s].channel);
nc_close_channel(nc_processes[s].channel_back);
return NC_INVALID_PID;
case 0:
// children
nc_pid = getpid();
close(nc_processes[s].channel[0]);
/*
if (prctl( PR_SET_NAME, name, NULL, NULL, NULL)!=0) {
log_error("set name error, %s %s", name, strerror(errno));
} else {
log_error("set name success, %s", name);
}*/
proc(data);
break;
default:
break;
}
// master
log_debug(LOG_NOTICE, "start %s %P", name, pid);
close(nc_processes[s].channel[1]);
nc_processes[s].pid = pid;
nc_processes[s].exited = 0;
nc_processes[s].isNew = 1;
nc_processes[s].idxWorker = data;
nc_processes[s].proc = proc;
//nc_processes[s].data = data;
nc_processes[s].name = name;
nc_processes[s].exiting = 0;
if (s == nc_last_process) {
nc_last_process++;
}
return pid;
}
rstatus_t
server_connect(struct context *ctx, struct server *server, struct conn *conn)
{
rstatus_t status;
ASSERT(!conn->client && !conn->proxy);
if (conn->sd > 0) {
/* already connected on server connection */
return NC_OK;
}
log_debug(LOG_VVERB, "connect to server '%.*s'", server->pname.len,
server->pname.data);
conn->sd = socket(conn->family, SOCK_STREAM, 0);
if (conn->sd < 0) {
log_error("socket for server '%.*s' failed: %s", server->pname.len,
server->pname.data, strerror(errno));
status = NC_ERROR;
goto error;
}
status = nc_set_nonblocking(conn->sd);
if (status != NC_OK) {
log_error("set nonblock on s %d for server '%.*s' failed: %s",
conn->sd, server->pname.len, server->pname.data,
strerror(errno));
goto error;
}
if (server->pname.data[0] != '/') {
status = nc_set_tcpnodelay(conn->sd);
if (status != NC_OK) {
log_warn("set tcpnodelay on s %d for server '%.*s' failed, ignored: %s",
conn->sd, server->pname.len, server->pname.data,
strerror(errno));
}
}
status = event_add_conn(ctx->evb, conn);
if (status != NC_OK) {
log_error("event add conn s %d for server '%.*s' failed: %s",
conn->sd, server->pname.len, server->pname.data,
strerror(errno));
goto error;
}
ASSERT(!conn->connecting && !conn->connected);
status = connect(conn->sd, conn->addr, conn->addrlen);
if (status != NC_OK) {
if (errno == EINPROGRESS) {
conn->connecting = 1;
log_debug(LOG_DEBUG, "connecting on s %d to server '%.*s'",
conn->sd, server->pname.len, server->pname.data);
return NC_OK;
}
log_error("connect on s %d to server '%.*s' failed: %s", conn->sd,
server->pname.len, server->pname.data, strerror(errno));
goto error;
}
ASSERT(!conn->connecting);
conn->connected = 1;
log_debug(LOG_INFO, "connected on s %d to server '%.*s'", conn->sd,
server->pname.len, server->pname.data);
return NC_OK;
error:
conn->err = errno;
return status;
}
static rstatus_t
proxy_accept(struct context *ctx, struct conn *p)
{
rstatus_t status;
struct conn *c;
int sd;
ASSERT(p->proxy && !p->client);
ASSERT(p->sd > 0);
ASSERT(p->recv_active && p->recv_ready);
for (;;) {
sd = accept(p->sd, NULL, NULL);
if (sd < 0) {
if (errno == EINTR) {
log_debug(LOG_VERB, "accept on p %d not ready - eintr", p->sd);
continue;
}
if (errno == EAGAIN || errno == EWOULDBLOCK) {
log_debug(LOG_VERB, "accept on p %d not ready - eagain", p->sd);
p->recv_ready = 0;
return NC_OK;
}
/*
* FIXME: On EMFILE or ENFILE mask out IN event on the proxy; mask
* it back in when some existing connection gets closed
*/
log_error("accept on p %d failed: %s", p->sd, strerror(errno));
return NC_ERROR;
}
break;
}
c = conn_get(p->owner, true, p->redis);
if (c == NULL) {
log_error("get conn for c %d from p %d failed: %s", sd, p->sd,
strerror(errno));
status = close(sd);
if (status < 0) {
log_error("close c %d failed, ignored: %s", sd, strerror(errno));
}
return NC_ENOMEM;
}
c->sd = sd;
stats_pool_incr(ctx, c->owner, client_connections);
status = nc_set_nonblocking(c->sd);
if (status < 0) {
log_error("set nonblock on c %d from p %d failed: %s", c->sd, p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
if (p->family == AF_INET || p->family == AF_INET6) {
status = nc_set_tcpnodelay(c->sd);
if (status < 0) {
log_warn("set tcpnodelay on c %d from p %d failed, ignored: %s",
c->sd, p->sd, strerror(errno));
}
}
status = event_add_conn(ctx->center->ep, c);
if (status < 0) {
log_error("event add conn of c %d from p %d failed: %s", c->sd, p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
log_debug(LOG_NOTICE, "accepted c %d on p %d from '%s'", c->sd, p->sd,
nc_unresolve_peer_desc(c->sd));
return NC_OK;
}
static rstatus_t
proxy_listen(struct context *ctx, struct conn *p)
{
rstatus_t status;
struct server_pool *pool = p->owner;
ASSERT(p->proxy);
p->sd = socket(p->family, SOCK_STREAM, 0);
if (p->sd < 0) {
log_error("socket failed: %s", strerror(errno));
return NC_ERROR;
}
status = proxy_reuse(p);
if (status < 0) {
log_error("reuse of addr '%.*s' for listening on p %d failed: %s",
pool->addrstr.len, pool->addrstr.data, p->sd,
strerror(errno));
return NC_ERROR;
}
status = bind(p->sd, p->addr, p->addrlen);
if (status < 0) {
log_error("bind on p %d to addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return NC_ERROR;
}
status = listen(p->sd, pool->backlog);
if (status < 0) {
log_error("listen on p %d on addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return NC_ERROR;
}
status = nc_set_nonblocking(p->sd);
if (status < 0) {
log_error("set nonblock on p %d on addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return NC_ERROR;
}
status = event_add_conn(ctx->center->ep, p);
if (status < 0) {
log_error("event add conn e %d p %d on addr '%.*s' failed: %s",
ctx->center->ep, p->sd, pool->addrstr.len, pool->addrstr.data,
strerror(errno));
return NC_ERROR;
}
status = event_del_out(ctx->center->ep, p);
if (status < 0) {
log_error("event del out e %d p %d on addr '%.*s' failed: %s",
ctx->center->ep, p->sd, pool->addrstr.len, pool->addrstr.data,
strerror(errno));
return NC_ERROR;
}
return NC_OK;
}
//接收到客户端连接后，返回新的fd，为该fd创建新的conn来读取数据
static rstatus_t
proxy_accept(struct context *ctx, struct conn *p) //p对应的是proxy conn 也就是用于监听客户端的conn信息
{
rstatus_t status;
struct conn *c;
int sd;
struct server_pool *pool = p->owner;
ASSERT(p->proxy && !p->client);
ASSERT(p->sd > 0);
ASSERT(p->recv_active && p->recv_ready);
for (;;) {
sd = accept(p->sd, NULL, NULL); //获取到新的客户端连接，产生新的fd
if (sd < 0) {
if (errno == EINTR) {
log_debug(LOG_VERB, "accept on p %d not ready - eintr", p->sd);
continue;
}
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED) {
log_debug(LOG_VERB, "accept on p %d not ready - eagain", p->sd);
p->recv_ready = 0;
return NC_OK;
}
/*
* Workaround of https://github.com/twitter/twemproxy/issues/97
*
* We should never reach here because the check for conn_ncurr_cconn()
* against ctx->max_ncconn should catch this earlier in the cycle.
* If we reach here ignore EMFILE/ENFILE, return NC_OK will enable
* the server continue to run instead of close the server socket
*
* The right solution however, is on EMFILE/ENFILE to mask out IN
* event on the proxy and mask it back in when some existing
* connections gets closed
*/
if (errno == EMFILE || errno == ENFILE) {
log_debug(LOG_CRIT, "accept on p %d with max fds %"PRIu32" "
"used connections %"PRIu32" max client connections %"PRIu32" "
"curr client connections %"PRIu32" failed: %s",
p->sd, ctx->max_nfd, conn_ncurr_conn(),
ctx->max_ncconn, conn_ncurr_cconn(), strerror(errno));
p->recv_ready = 0;
return NC_OK;
}
log_error("accept on p %d failed: %s", p->sd, strerror(errno));
return NC_ERROR;
}
break;
}
if (conn_ncurr_cconn() >= ctx->max_ncconn) {
log_debug(LOG_CRIT, "client connections %"PRIu32" exceed limit %"PRIu32,
conn_ncurr_cconn(), ctx->max_ncconn);
status = close(sd);
if (status < 0) {
log_error("close c %d failed, ignored: %s", sd, strerror(errno));
}
return NC_OK;
}
c = conn_get(p->owner, true, p->redis);
if (c == NULL) {
log_error("get conn for c %d from p %d failed: %s", sd, p->sd,
strerror(errno));
status = close(sd);
if (status < 0) {
log_error("close c %d failed, ignored: %s", sd, strerror(errno));
}
return NC_ENOMEM;
}
c->sd = sd;
stats_pool_incr(ctx, c->owner, client_connections);
status = nc_set_nonblocking(c->sd);
if (status < 0) {
log_error("set nonblock on c %d from p %d failed: %s", c->sd, p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
if (pool->tcpkeepalive) {
status = nc_set_tcpkeepalive(c->sd);
if (status < 0) {
log_warn("set tcpkeepalive on c %d from p %d failed, ignored: %s",
c->sd, p->sd, strerror(errno));
}
}
if (p->family == AF_INET || p->family == AF_INET6) {
status = nc_set_tcpnodelay(c->sd);
if (status < 0) {
log_warn("set tcpnodelay on c %d from p %d failed, ignored: %s",
c->sd, p->sd, strerror(errno));
}
}
status = event_add_conn(ctx->evb, c);
if (status < 0) {
log_error("event add conn from p %d failed: %s", p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
log_debug(LOG_INFO, "accepted c %d on p %d from '%s'", c->sd, p->sd,
nc_unresolve_peer_desc(c->sd));
return NC_OK;
}
static rstatus_t
stats_master_send_rsp(int *psd)
{
int n;
ssize_t len;
int sd;
int i;
nc_channel_msg_t message;
char buf[1000];
memset(buf, 0, sizeof(char)*1000);
char *snd_buf = NULL;
memset(&message, 0, sizeof(nc_channel_msg_t));
message.command = NC_CMD_GET_STATS;
if (*psd) {
sd = accept(*psd, NULL, NULL);
if (sd < 0) {
log_error("accept on m %d failed: %s", sd, strerror(errno));
return NC_ERROR;
}
}
// still in reconfiguration process
if (nc_reload_start) {
sprintf(buf, "%s", "no stats get due to still during reconfiguration period.");
if (*psd) {
len = nc_strlen(buf)+1;
len = nc_sendn(sd, buf, len);
if (len < 0) {
log_error("send stats on sd %d failed: %s", sd, strerror(errno));
}
close(sd);
}
return NC_OK;
}
//broadcast
for (i = 0; i < nc_last_process; i++)
{
if (nc_processes[i].pid == -1 || nc_processes[i].pid == 0)
{
continue;
}
if (nc_write_channel(nc_processes[i].channel[0],
&message, sizeof(nc_channel_msg_t))
== NC_OK)
{
if (nc_set_blocking(nc_processes[i].channel[0]) < 0) {
log_error("set channel %d block failed while core timeout %s",
nc_processes[i].channel[0] , strerror(errno));
continue;
}
n = nc_read_channel(nc_processes[i].channel[0], &env_global.ctrl_msg, sizeof(nc_channel_msg_t));
if (env_global.ctrl_msg.command != NC_CMD_GET_STATS || n < 0) {
log_error("failure: get stats from worker %d receive length, %s",
nc_processes[i].channel[0] , strerror(errno));
} else {
log_error("success: get stats from worker %d receive length, %d",
nc_processes[i].channel[0] , n);
snd_buf = nc_alloc(n + n);
len = nc_recvn(nc_processes[i].channel[0], snd_buf, n);
if (len < 0) {
if (*psd) {
log_error("recv stats on sd %d failed: %s", sd, strerror(errno));
}
nc_free(snd_buf);
snd_buf = NULL;
continue;
}
if (*psd) {
len = nc_sendn(sd, snd_buf, len);
if (len < 0) {
log_error("send stats on sd %d failed: %s", sd, strerror(errno));
nc_free(snd_buf);
snd_buf = NULL;
continue;
}
}
len = nc_write(env_global.stats_fd, snd_buf, len);
if (len < 0) {
log_error("nc_write %d failed: %s", env_global.stats_fd, strerror(errno));
continue;
}
nc_free(snd_buf);
snd_buf = NULL;
}
if (nc_set_nonblocking(nc_processes[i].channel[0]) < 0) {
log_error("set channel %d nonblock failed while core timeout %s",
nc_processes[i].channel[0] , strerror(errno));
}
}
}
if (*psd) {
shutdown(sd, SHUT_RDWR);
close(sd);
}
return NC_OK;
}
static rstatus_t
proxy_accept(struct context *ctx, struct conn *p)
{
rstatus_t status;
struct conn *c;
int sd;
struct sockaddr_storage addr;
socklen_t addr_len;
ASSERT(p->proxy && !p->client);
ASSERT(p->sd > 0);
ASSERT(p->recv_active && p->recv_ready);
for (;;) {
sd = accept(p->sd, NULL, NULL);
if (sd < 0) {
if (errno == EINTR) {
log_debug(LOG_VERB, "accept on p %d not ready - eintr", p->sd);
continue;
}
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED) {
log_debug(LOG_VERB, "accept on p %d not ready - eagain", p->sd);
p->recv_ready = 0;
return NC_OK;
}
/*
* FIXME: On EMFILE or ENFILE mask out IN event on the proxy; mask
* it back in when some existing connection gets closed
*/
/*
* Workaround of https://github.com/twitter/twemproxy/issues/97
* Just ignore EMFILE/ENFILE, return NC_OK will enable the server
* continue to run instead of close the server socket
*/
if (errno == EMFILE || errno == ENFILE) {
log_crit("accept on p %d failed: %s", p->sd,
strerror(errno));
p->recv_ready = 0;
log_crit("connections status: rlimit nofile %d, "
"used connections: %d, max client connections %d, "
"curr client connections %d", ctx->rlimit_nofile,
conn_ncurr(), ctx->max_ncconn, conn_ncurr_cconn());
/* Since we maintain a safe max_ncconn and check
* it after every accept, we should not reach here.
* So we will panic after this log */
log_panic("HIT MAX OPEN FILES, IT SHOULD NOT HAPPEN. ABORT.");
return NC_OK;
}
log_error("accept on p %d failed: %s", p->sd, strerror(errno));
return NC_ERROR;
}
addr_len = sizeof(addr);
if (getsockname(sd, (struct sockaddr *)&addr, &addr_len)) {
log_error("getsockname on p %d failed: %s", p->sd, strerror(errno));
close(sd);
continue;
}
break;
}
if (conn_ncurr_cconn() >= ctx->max_ncconn) {
stats_pool_incr(ctx, p->owner, rejected_connections);
log_crit("client connections %d exceed limit %d",
conn_ncurr_cconn(), ctx->max_ncconn);
status = close(sd);
if (status < 0) {
log_error("close c %d failed, ignored: %s", sd, strerror(errno));
}
return NC_OK;
}
c = conn_get(p->owner, true, p->redis);
if (c == NULL) {
log_error("get conn for c %d from p %d failed: %s", sd, p->sd,
strerror(errno));
status = close(sd);
if (status < 0) {
log_error("close c %d failed, ignored: %s", sd, strerror(errno));
}
return NC_ENOMEM;
}
c->sd = sd;
c->family = addr.ss_family;
c->addrlen = addr_len;
c->ss = addr;
c->addr = (struct sockaddr *)&c->ss;
stats_pool_incr(ctx, c->owner, client_connections);
status = nc_set_nonblocking(c->sd);
if (status < 0) {
log_error("set nonblock on c %d from p %d failed: %s", c->sd, p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
if (p->family == AF_INET || p->family == AF_INET6) {
status = nc_set_tcpnodelay(c->sd);
if (status < 0) {
log_warn("set tcpnodelay on c %d from p %d failed, ignored: %s",
c->sd, p->sd, strerror(errno));
}
}
status = event_add_conn(ctx->evb, c);
if (status < 0) {
log_error("event add conn from p %d failed: %s", p->sd,
strerror(errno));
c->close(ctx, c);
return status;
}
log_notice("accepted c %d on p %d from '%s'", c->sd, p->sd,
nc_unresolve_peer_desc(c->sd));
return NC_OK;
}
