int
event_del_conn(struct event_base *evb, struct conn *c)
{
int i;
ASSERT(evb->kq > 0);
ASSERT(c != NULL);
ASSERT(c->sd > 0);
ASSERT(evb->nchange < evb->nevent);
event_del_out(evb, c);
event_del_in(evb, c);
/*
* Now, eliminate pending events for c->sd (there should be at most one
* other event). This is important because we will close c->sd and free
* c when we return.
*/
for (i = evb->nprocessed + 1; i < evb->nreturned; i++) {
struct kevent *ev = &evb->event[i];
if (ev->ident == (uintptr_t)c->sd) {
ev->flags = 0;
ev->filter = 0;
break;
}
}
return 0;
}
/**
*发送下一个messsage
*/
struct msg *
rsp_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *msg, *pmsg; /* response and it's peer request */
ASSERT(conn->client && !conn->proxy);
pmsg = TAILQ_FIRST(&conn->omsg_q);
if (pmsg == NULL || !req_done(conn, pmsg)) {//pmsg为空 并且请求未完成
/* nothing is outstanding, initiate close? */
if (pmsg == NULL && conn->eof) {//如果消息为假 并且连接eof为真
conn->done = 1;//请求完成
log_debug(LOG_INFO, "c %d is done", conn->sd);
}
//删除当前的请求
status = event_del_out(ctx->evb, conn);
if (status != NC_OK) {//删除失败
conn->err = errno;//设置连接错误码
}
return NULL;
}
//连接发送的信息
msg = conn->smsg;
if (msg != NULL) {
ASSERT(!msg->request && msg->peer != NULL);
ASSERT(req_done(conn, msg->peer));
pmsg = TAILQ_NEXT(msg->peer, c_tqe);
}
if (pmsg == NULL || !req_done(conn, pmsg)) {
conn->smsg = NULL;
return NULL;
}
ASSERT(pmsg->request && !pmsg->swallow);
if (req_error(conn, pmsg)) {//如果有error
msg = rsp_make_error(ctx, conn, pmsg);
if (msg == NULL) {
conn->err = errno;
return NULL;
}
msg->peer = pmsg;
pmsg->peer = msg;
stats_pool_incr(ctx, conn->owner, forward_error);
} else {
msg = pmsg->peer;
}
//获取msg
ASSERT(!msg->request);
conn->smsg = msg;
log_debug(LOG_VVERB, "send next rsp %"PRIu64" on c %d", msg->id, conn->sd);
return msg;
}
struct msg *
rsp_send_next_proxy_adm(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *msg, *pmsg; /* response and it's peer request */
struct proxy_adm *padm = ctx->padm;
ASSERT(padm != NULL);
ASSERT(conn->client && !conn->proxy);
pmsg = TAILQ_FIRST(&conn->omsg_q);
msg_print(pmsg, LOG_DEBUG);
if (pmsg == NULL || !req_done(conn, pmsg)) {
/* nothing is outstanding, initiate close? */
if (pmsg == NULL && conn->eof) {
conn->done = 1;
log_debug(LOG_INFO, "c %d is done", conn->sd);
}
status = event_del_out(padm->evb, conn);
if (status != NC_OK) {
conn->err = errno;
}
return NULL;
}
msg = conn->smsg;
if (msg != NULL) {
ASSERT(!msg->request && msg->peer != NULL);
ASSERT(req_done(conn, msg->peer));
pmsg = TAILQ_NEXT(msg->peer, c_tqe);
}
if (pmsg == NULL || !req_done(conn, pmsg)) {
conn->smsg = NULL;
return NULL;
}
ASSERT(pmsg->request && !pmsg->swallow);
ASSERT(!req_error(conn, pmsg));
msg = pmsg->peer;
ASSERT(!msg->request);
conn->smsg = msg;
log_debug(LOG_VVERB, "send next rsp %"PRIu64" on c %d", msg->id, conn->sd);
return msg;
}
rstatus_t
conn_del_out(struct conn *conn)
{
rstatus_t status;
server_t *srv = conn->owner;
status = event_del_out(srv->evb, conn);
if (status != NC_OK) {
conn->err = errno;
}
return status;
}
struct msg *
req_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *msg, *nmsg; /* current and next message */
ASSERT((!conn->client && !conn->proxy) || (!conn->dnode_client && !conn->dnode_server));
if (conn->connecting) {
if (!conn->dyn_mode && !conn->client) {
server_connected(ctx, conn);
} else if (conn->dyn_mode && !conn->dnode_client) {
dnode_peer_connected(ctx, conn);
}
}
nmsg = TAILQ_FIRST(&conn->imsg_q);
if (nmsg == NULL) {
/* nothing to send as the server inq is empty */
status = event_del_out(ctx->evb, conn);
if (status != DN_OK) {
conn->err = errno;
}
return NULL;
}
msg = conn->smsg;
if (msg != NULL) {
ASSERT(msg->request && !msg->done);
nmsg = TAILQ_NEXT(msg, s_tqe);
}
conn->smsg = nmsg;
if (nmsg == NULL) {
return NULL;
}
ASSERT(nmsg->request && !nmsg->done);
log_debug(LOG_VVERB, "send next req %"PRIu64" len %"PRIu32" type %d on "
"s %d", nmsg->id, nmsg->mlen, nmsg->type, conn->sd);
return nmsg;
}
//接收的客户端msg信息通过req_server_enqueue_imsgq添加到该队列 在req_send_next中发往后端真实服务器
//从imsg_q队列中取出msg然后发往后端真实服务器
struct msg * //发往客户端用rsp_send_next 发往后端服务器用req_send_next msg_send或者msg_send_chain中执行
req_send_next(struct context *ctx, struct conn *conn) //从imsg_q队列中取出需要发往后端的msg
{//从队列imsg_q中取出msg发送 //msg_send中执行 msg_send或者msg_send_chain中执行
rstatus_t status;
struct msg *msg, *nmsg; /* current and next message */
ASSERT(!conn->client && !conn->proxy);
if (conn->connecting) { //走到这里说明在server_connect中建立连接，添加写事件后，触发走到这里，说明连接建立成功了
server_connected(ctx, conn);
}
nmsg = TAILQ_FIRST(&conn->imsg_q);
if (nmsg == NULL) { //说明已经写数据完成，把imsg_q队列上的所有msg信息已经发送完毕
/* nothing to send as the server inq is empty */
status = event_del_out(ctx->evb, conn); //没有数据需要发往后端了，去除写事件
if (status != NC_OK) {
conn->err = errno;
}
return NULL;
}
msg = conn->smsg;
if (msg != NULL) {
//如果上次还有未发送的msg，则取出来返回。也就是如果conn->smsg上面有还有未发送出去的数据，则从该smsg取出一个msg发送，否则从imsg_q队列上面取出来发送
ASSERT(msg->request && !msg->done);
nmsg = TAILQ_NEXT(msg, s_tqe); //指向imsg_q的下一个msg
}
conn->smsg = nmsg; //需要发送往后端真实服务器的真实msg全部保存到conn->smsg中
if (nmsg == NULL) {
return NULL;
}
ASSERT(nmsg->request && !nmsg->done);
log_debug(LOG_VVERB, "send next req %"PRIu64" len %"PRIu32" type %d on "
"s %d", nmsg->id, nmsg->mlen, nmsg->type, conn->sd);
return nmsg;
}
int
event_del_conn(struct evbase *evb, struct conn *c)
{
int i;
struct kevent *event;
int kq = evb->kq;
ASSERT(kq > 0);
ASSERT(c != NULL);
ASSERT(c->sd > 0);
ASSERT(evb->n_changes < evb->nevent);
event = &evb->changes[(evb->n_changes)++];
EV_SET(event, c->sd, EVFILT_READ, EV_DELETE, 0, 0, (void *)c);
event_del_out(evb, c);
c->recv_active = 0;
c->send_active = 0;
/*
* Now, eliminate pending events for c->sd (there should be at most one
* other event). This is important because we will close c->sd and free
* c when we return.
*/
for (i = evb->n_processed + 1; i < evb->n_returned; i++) {
struct kevent *ev = &evb->kevents[i];
if (ev->ident == (uintptr_t)c->sd) {
ev->flags = 0;
ev->filter = 0;
break;
}
}
return 0;
}
static rstatus_t
proxy_listen(struct context *ctx, struct conn *p)
{
rstatus_t status;
struct server_pool *pool = p->owner;
ASSERT(p->proxy);
p->sd = socket(p->family, SOCK_STREAM, 0);
if (p->sd < 0) {
log_error("socket failed: %s", strerror(errno));
return DN_ERROR;
}
status = proxy_reuse(p);
if (status < 0) {
log_error("reuse of addr '%.*s' for listening on p %d failed: %s",
pool->addrstr.len, pool->addrstr.data, p->sd,
strerror(errno));
return DN_ERROR;
}
status = bind(p->sd, p->addr, p->addrlen);
if (status < 0) {
log_error("bind on p %d to addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return DN_ERROR;
}
status = listen(p->sd, pool->backlog);
if (status < 0) {
log_error("listen on p %d on addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return DN_ERROR;
}
status = dn_set_nonblocking(p->sd);
if (status < 0) {
log_error("set nonblock on p %d on addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return DN_ERROR;
}
status = event_add_conn(ctx->evb, p);
if (status < 0) {
log_error("event add conn p %d on addr '%.*s' failed: %s",
p->sd, pool->addrstr.len, pool->addrstr.data,
strerror(errno));
return DN_ERROR;
}
status = event_del_out(ctx->evb, p);
if (status < 0) {
log_error("event del out p %d on addr '%.*s' failed: %s",
p->sd, pool->addrstr.len, pool->addrstr.data,
strerror(errno));
return DN_ERROR;
}
return DN_OK;
}
struct msg *
rsp_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *rsp, *req; /* response and it's peer request */
ASSERT_LOG((conn->type == CONN_DNODE_PEER_CLIENT) ||
(conn->type = CONN_CLIENT), "conn %s", conn_get_type_string(conn));
req = TAILQ_FIRST(&conn->omsg_q);
if (req == NULL || (!req->selected_rsp && !req_done(conn, req))) {
/* nothing is outstanding, initiate close? */
if (req == NULL && conn->eof) {
conn->done = 1;
log_debug(LOG_INFO, "c %d is done", conn->sd);
}
status = event_del_out(ctx->evb, conn);
if (status != DN_OK) {
conn->err = errno;
}
return NULL;
}
rsp = conn->smsg;
if (rsp != NULL) {
ASSERT(!rsp->request);
ASSERT(rsp->peer != NULL);
req = TAILQ_NEXT(rsp->peer, c_tqe);
}
if (req == NULL || !req_done(conn, req)) {
conn->smsg = NULL;
return NULL;
}
ASSERT(req->request && !req->swallow);
if (req_error(conn, req)) {
rsp = rsp_make_error(ctx, conn, req);
if (rsp == NULL) {
conn->err = errno;
return NULL;
}
rsp->peer = req;
req->selected_rsp = rsp;
log_debug(LOG_VERB, "creating new error rsp %p", rsp);
if (conn->dyn_mode) {
stats_pool_incr(ctx, peer_forward_error);
} else {
stats_pool_incr(ctx, forward_error);
}
} else {
rsp = req->selected_rsp;
}
ASSERT(!rsp->request);
conn->smsg = rsp;
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "send next rsp %"PRIu64" on c %d", rsp->id, conn->sd);
}
return rsp;
}
static rstatus_t
dnode_listen(struct context *ctx, struct conn *p)
{
rstatus_t status;
struct server_pool *pool = p->owner;
ASSERT(p->dnode_server);
p->sd = socket(p->family, SOCK_STREAM, 0);
if (p->sd < 0) {
log_error("dyn: socket failed: %s", strerror(errno));
return DN_ERROR;
}
status = dnode_reuse(p);
if (status < 0) {
log_error("dyn: reuse of addr '%.*s' for listening on p %d failed: %s",
pool->d_addrstr.len, pool->d_addrstr.data, p->sd,
strerror(errno));
return DN_ERROR;
}
status = bind(p->sd, p->addr, p->addrlen);
if (status < 0) {
log_error("dyn: bind on p %d to addr '%.*s' failed: %s", p->sd,
pool->addrstr.len, pool->addrstr.data, strerror(errno));
return DN_ERROR;
}
status = listen(p->sd, pool->backlog);
if (status < 0) {
log_error("dyn: listen on p %d on addr '%.*s' failed: %s", p->sd,
pool->d_addrstr.len, pool->d_addrstr.data, strerror(errno));
return DN_ERROR;
}
status = dn_set_nonblocking(p->sd);
if (status < 0) {
log_error("dyn: set nonblock on p %d on addr '%.*s' failed: %s", p->sd,
pool->d_addrstr.len, pool->d_addrstr.data, strerror(errno));
return DN_ERROR;
}
log_debug(LOG_INFO, "dyn: e %d with nevent %d", ctx->evb->ep, ctx->evb->nevent);
status = event_add_conn(ctx->evb, p);
if (status < 0) {
log_error("dyn: event add conn p %d on addr '%.*s' failed: %s",
p->sd, pool->d_addrstr.len, pool->d_addrstr.data,
strerror(errno));
return DN_ERROR;
}
status = event_del_out(ctx->evb, p);
if (status < 0) {
log_error("dyn: event del out p %d on addr '%.*s' failed: %s",
strerror(errno));
return DN_ERROR;
}
return DN_OK;
}
struct msg *
rsp_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *msg, *pmsg; /* response and it's peer request */
ASSERT((conn->client && !conn->proxy) || (conn->dnode_client && !conn->dnode_server));
pmsg = TAILQ_FIRST(&conn->omsg_q);
if (pmsg == NULL || !req_done(conn, pmsg)) {
/* nothing is outstanding, initiate close? */
if (pmsg == NULL && conn->eof) {
conn->done = 1;
log_debug(LOG_INFO, "c %d is done", conn->sd);
}
status = event_del_out(ctx->evb, conn);
if (status != DN_OK) {
conn->err = errno;
}
return NULL;
}
msg = conn->smsg;
if (msg != NULL) {
ASSERT(!msg->request && msg->peer != NULL);
ASSERT(req_done(conn, msg->peer));
pmsg = TAILQ_NEXT(msg->peer, c_tqe);
}
if (pmsg == NULL || !req_done(conn, pmsg)) {
conn->smsg = NULL;
return NULL;
}
ASSERT(pmsg->request && !pmsg->swallow);
if (req_error(conn, pmsg)) {
msg = rsp_make_error(ctx, conn, pmsg);
if (msg == NULL) {
conn->err = errno;
return NULL;
}
msg->peer = pmsg;
pmsg->peer = msg;
if (!conn->dyn_mode) {
stats_pool_incr(ctx, conn->owner, forward_error);
} else { //dyn_mode
stats_pool_incr(ctx, conn->owner, peer_forward_error);
}
} else {
msg = pmsg->peer;
}
ASSERT(!msg->request);
conn->smsg = msg;
log_debug(LOG_VVERB, "send next rsp %"PRIu64" on c %d", msg->id, conn->sd);
return msg;
}
