void
server_close(struct context *ctx, struct conn *conn)
{
rstatus_t status;
struct msg *msg, *nmsg; /* current and next message */
struct conn *c_conn; /* peer client connection */
ASSERT(!conn->client && !conn->proxy);
server_close_stats(ctx, conn->owner, conn->err, conn->eof,
conn->connected);
conn->connected = false;
if (conn->sd < 0) {
server_failure(ctx, conn->owner);
conn->unref(conn);
conn_put(conn);
return;
}
for (msg = TAILQ_FIRST(&conn->imsg_q); msg != NULL; msg = nmsg) {
nmsg = TAILQ_NEXT(msg, s_tqe);
/* dequeue the message (request) from server inq */
conn->dequeue_inq(ctx, conn, msg);
/*
* Don't send any error response, if
* 1. request is tagged as noreply or,
* 2. client has already closed its connection
*/
if (msg->swallow || msg->noreply) {
log_debug(LOG_INFO, "close s %d swallow req %"PRIu64" len %"PRIu32
" type %d", conn->sd, msg->id, msg->mlen, msg->type);
req_put(msg);
} else {
c_conn = msg->owner;
ASSERT(c_conn->client && !c_conn->proxy);
msg->done = 1;
msg->error = 1;
msg->err = conn->err;
if (msg->frag_owner != NULL) {
msg->frag_owner->nfrag_done++;
}
if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) {
event_add_out(ctx->evb, msg->owner);
}
log_debug(LOG_INFO, "close s %d schedule error for req %"PRIu64" "
"len %"PRIu32" type %d from c %d%c %s", conn->sd, msg->id,
msg->mlen, msg->type, c_conn->sd, conn->err ? ':' : ' ',
conn->err ? strerror(conn->err): " ");
}
}
ASSERT(TAILQ_EMPTY(&conn->imsg_q));
for (msg = TAILQ_FIRST(&conn->omsg_q); msg != NULL; msg = nmsg) {
nmsg = TAILQ_NEXT(msg, s_tqe);
/* dequeue the message (request) from server outq */
conn->dequeue_outq(ctx, conn, msg);
if (msg->swallow) {
log_debug(LOG_INFO, "close s %d swallow req %"PRIu64" len %"PRIu32
" type %d", conn->sd, msg->id, msg->mlen, msg->type);
req_put(msg);
} else {
c_conn = msg->owner;
ASSERT(c_conn->client && !c_conn->proxy);
msg->done = 1;
msg->error = 1;
msg->err = conn->err;
if (msg->frag_owner != NULL) {
msg->frag_owner->nfrag_done++;
}
if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) {
event_add_out(ctx->evb, msg->owner);
}
log_debug(LOG_INFO, "close s %d schedule error for req %"PRIu64" "
"len %"PRIu32" type %d from c %d%c %s", conn->sd, msg->id,
msg->mlen, msg->type, c_conn->sd, conn->err ? ':' : ' ',
conn->err ? strerror(conn->err): " ");
}
}
ASSERT(TAILQ_EMPTY(&conn->omsg_q));
msg = conn->rmsg;
if (msg != NULL) {
conn->rmsg = NULL;
ASSERT(!msg->request);
ASSERT(msg->peer == NULL);
rsp_put(msg);
log_debug(LOG_INFO, "close s %d discarding rsp %"PRIu64" len %"PRIu32" "
"in error", conn->sd, msg->id, msg->mlen);
}
ASSERT(conn->smsg == NULL);
server_failure(ctx, conn->owner);
conn->unref(conn);
status = close(conn->sd);
if (status < 0) {
log_error("close s %d failed, ignored: %s", conn->sd, strerror(errno));
}
conn->sd = -1;
conn_put(conn);
}
static void
rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *msg)
{
rstatus_t status;
struct msg *pmsg;
struct conn *c_conn;
ASSERT(!s_conn->client && !s_conn->proxy);
/* response from server implies that server is ok and heartbeating */
server_ok(ctx, s_conn);
/* dequeue peer message (request) from server */
pmsg = TAILQ_FIRST(&s_conn->omsg_q);
ASSERT(pmsg != NULL && pmsg->peer == NULL);
ASSERT(pmsg->request && !pmsg->done);
s_conn->dequeue_outq(ctx, s_conn, pmsg);
pmsg->done = 1;
/* establish msg <-> pmsg (response <-> request) link */
pmsg->peer = msg;
msg->peer = pmsg;
/*
* Readjust responses of fragmented messages by not including the end
* marker for all but the last response
*
* Valid responses for a fragmented requests are MSG_RSP_VALUE or,
* MSG_RSP_END. For an invalid response, we send out SERVER_ERRROR with
* EINVAL errno
*/
if (pmsg->frag_id != 0) {
if (msg->type != MSG_RSP_VALUE && msg->type != MSG_RSP_END) {
pmsg->error = 1;
pmsg->err = EINVAL;
} else if (!pmsg->last_fragment) {
ASSERT(msg->end != NULL);
for (;;) {
struct mbuf *mbuf;
mbuf = STAILQ_LAST(&msg->mhdr, mbuf, next);
ASSERT(mbuf != NULL);
/*
* We cannot assert that end marker points to the last mbuf
* Consider a scenario where end marker points to the
* penultimate mbuf and the last mbuf only contains spaces
* and CRLF: mhdr -> [...END] -> [\r\n]
*/
if (msg->end >= mbuf->pos && msg->end < mbuf->last) {
/* end marker is within this mbuf */
msg->mlen -= (uint32_t)(mbuf->last - msg->end);
mbuf->last = msg->end;
break;
}
/* end marker is not in this mbuf */
msg->mlen -= mbuf_length(mbuf);
mbuf_remove(&msg->mhdr, mbuf);
mbuf_put(mbuf);
}
}
}
c_conn = pmsg->owner;
ASSERT(c_conn->client && !c_conn->proxy);
if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) {
status = event_add_out(ctx->ep, c_conn);
if (status != NC_OK) {
c_conn->err = errno;
}
}
rsp_forward_stats(ctx, msg, s_conn, c_conn);
}
void
req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg,
struct msg *nmsg)
{
rstatus_t status;
struct server_pool *pool;
struct msg_tqh frag_msgq;
struct msg *sub_msg;
struct msg *tmsg; /* tmp next message */
ASSERT(conn->client && !conn->proxy);
ASSERT(msg->request);
ASSERT(msg->owner == conn);
ASSERT(conn->rmsg == msg);
ASSERT(nmsg == NULL || nmsg->request);
/* enqueue next message (request), if any */
conn->rmsg = nmsg;
if (req_filter(ctx, conn, msg)) {
return;
}
if (msg->noforward) {
status = req_make_reply(ctx, conn, msg);
if (status != NC_OK) {
conn->err = errno;
return;
}
status = msg->reply(msg);
if (status != NC_OK) {
conn->err = errno;
return;
}
status = event_add_out(ctx->evb, conn);
if (status != NC_OK) {
conn->err = errno;
}
return;
}
/* do fragment */
pool = conn->owner;
TAILQ_INIT(&frag_msgq);
status = msg->fragment(msg, pool->ncontinuum, &frag_msgq);
if (status != NC_OK) {
if (!msg->noreply) {
conn->enqueue_outq(ctx, conn, msg);
}
req_forward_error(ctx, conn, msg);
}
/* if no fragment happened */
if (TAILQ_EMPTY(&frag_msgq)) {
req_forward(ctx, conn, msg);
return;
}
status = req_make_reply(ctx, conn, msg);
if (status != NC_OK) {
if (!msg->noreply) {
conn->enqueue_outq(ctx, conn, msg);
}
req_forward_error(ctx, conn, msg);
}
for (sub_msg = TAILQ_FIRST(&frag_msgq); sub_msg != NULL; sub_msg = tmsg) {
tmsg = TAILQ_NEXT(sub_msg, m_tqe);
TAILQ_REMOVE(&frag_msgq, sub_msg, m_tqe);
req_forward(ctx, conn, sub_msg);
}
ASSERT(TAILQ_EMPTY(&frag_msgq));
return;
}
/* Forward a client request over to a peer */
void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn,
struct conn *p_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
struct server *server = p_conn->owner;
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd),
rack->name->len, rack->name->data,
server->dc.len, server->dc.data);
struct string *dc = rack->dc;
rstatus_t status;
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue the message (request) into peer inq */
status = event_add_out(ctx->evb, p_conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, p_conn, msg);
p_conn->err = errno;
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
req_put(msg);
return;
}
struct server_pool *pool = c_conn->owner;
dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ;
if (p_conn->dnode_secured) {
//Encrypting and adding header for a request
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN));
}
//write dnode header
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(msg, p_conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(msg);
return;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the msg payload");
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
} else {
//write dnode header
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: ");
msg_dump(msg);
}
conn_enqueue_inq(ctx, p_conn, msg);
dnode_peer_req_forward_stats(ctx, p_conn->owner, msg);
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
/*
* Sending a mbuf of gossip data over the wire to a peer
*/
void
dnode_peer_gossip_forward(struct context *ctx, struct conn *conn, int data_store, struct mbuf *data_buf)
{
rstatus_t status;
struct msg *msg = msg_get(conn, 1, data_store, __FUNCTION__);
if (msg == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a msg");
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a data_buf");
rsp_put(msg);
return;
}
uint64_t msg_id = peer_msg_id++;
if (conn->dnode_secured) {
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "Assemble a secured msg to send");
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
}
if (ENCRYPTION) {
struct mbuf *encrypted_buf = mbuf_get();
if (encrypted_buf == NULL) {
loga("Unable to obtain an data_buf for encryption!");
return; //TODOs: need to clean up
}
status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, conn->aes_key);
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
//write dnode header
dmsg_write(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(encrypted_buf));
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: ");
log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: ");
}
mbuf_remove(&msg->mhdr, data_buf);
mbuf_insert(&msg->mhdr, encrypted_buf);
//free data_buf as no one will need it again
mbuf_put(data_buf); //TODOS: need to remove this from the msg->mhdr as in the other method
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "No encryption");
}
dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf));
mbuf_insert(&msg->mhdr, data_buf);
}
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "Assemble a non-secured msg to send");
}
dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf));
mbuf_insert(&msg->mhdr, data_buf);
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn gossip message header: ");
msg_dump(msg);
}
/* enqueue the message (request) into peer inq */
if (TAILQ_EMPTY(&conn->imsg_q)) {
status = event_add_out(ctx->evb, conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, conn, msg);
conn->err = errno;
return;
}
}
//need to handle a reply
//conn->enqueue_outq(ctx, conn, msg);
msg->noreply = 1;
conn_enqueue_inq(ctx, conn, msg);
}
void
local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
uint8_t *key, uint32_t keylen)
{
rstatus_t status;
struct conn *s_conn;
if (log_loggable(LOG_VVERB)) {
loga("local_req_forward entering ............");
}
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply) {
conn_enqueue_outq(ctx, c_conn, msg);
}
s_conn = get_datastore_conn(ctx, c_conn->owner);
log_debug(LOG_VERB, "c_conn %p got server conn %p", c_conn, s_conn);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
ASSERT(s_conn->type == CONN_SERVER);
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to storage conn '%s'",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(s_conn->sd));
}
if (ctx->dyn_state == NORMAL) {
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
} else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients
log_debug(LOG_INFO, "Node is in STANDBY state. Drop write/read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) {
//no reads from peers/clients but allow writes from peers/clients
log_debug(LOG_INFO, "Node is in WRITES_ONLY state. Drop read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == RESUMING) {
log_debug(LOG_INFO, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes");
if (msg->is_read) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
conn_enqueue_inq(ctx, s_conn, msg);
req_forward_stats(ctx, msg);
if(g_data_store == DATA_REDIS){
req_redis_stats(ctx, msg);
}
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
static void
req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
{
rstatus_t status;
struct conn *s_conn;
struct server_pool *pool;
uint8_t *key;
uint32_t keylen;
ASSERT(c_conn->client && !c_conn->proxy);
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply) {
c_conn->enqueue_outq(ctx, c_conn, msg);
}
pool = c_conn->owner;
key = NULL;
keylen = 0;
/*
* If hash_tag: is configured for this server pool, we use the part of
* the key within the hash tag as an input to the distributor. Otherwise
* we use the full key
*/
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = nc_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = nc_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg);
return;
}
ASSERT(!s_conn->client && !s_conn->proxy);
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
status = event_add_out(ctx->center->ep, s_conn);
if (status != NC_OK) {
req_forward_error(ctx, c_conn, msg);
s_conn->err = errno;
return;
}
}
s_conn->enqueue_inq(ctx, s_conn, msg);
req_forward_stats(ctx, s_conn->owner, msg);
log_debug(LOG_VERB, "forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
void
local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
uint8_t *key, uint32_t keylen)
{
rstatus_t status;
struct conn *s_conn;
if (get_tracking_level() >= LOG_VVERB) {
loga("local_req_forward entering ............");
}
ASSERT((c_conn->client || c_conn->dnode_client) && !c_conn->proxy && !c_conn->dnode_server);
if (c_conn->dyn_mode && !c_conn->same_dc && !msg->is_read) {
msg->noreply = 1;
}
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply) {
c_conn->enqueue_outq(ctx, c_conn, msg);
}
s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg);
return;
}
ASSERT(!s_conn->client && !s_conn->proxy);
if (ctx->dyn_state == NORMAL) {
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg);
s_conn->err = errno;
return;
}
}
} else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients
log_debug(LOG_VERB, "Node is in STANDBY state. Drop write/read requests");
req_forward_error(ctx, c_conn, msg);
return;
} else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) {
//no reads from peers/clients but allow writes from peers/clients
log_debug(LOG_VERB, "Node is in WRITES_ONLY state. Drop read requests");
req_forward_error(ctx, c_conn, msg);
return;
} else if (ctx->dyn_state == RESUMING) {
log_debug(LOG_VERB, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes");
if (msg->is_read) {
req_forward_error(ctx, c_conn, msg);
return;
}
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg);
s_conn->err = errno;
return;
}
}
s_conn->enqueue_inq(ctx, s_conn, msg);
req_forward_stats(ctx, s_conn->owner, msg);
log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
void
req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg,
struct msg *nmsg)
{
rstatus_t status;
struct server_pool *pool;
struct msg_tqh frag_msgq;
struct msg *sub_msg;
struct msg *tmsg; /* tmp next message */
ASSERT(conn->client && !conn->proxy);
ASSERT(msg->request);
ASSERT(msg->owner == conn);
ASSERT(conn->rmsg == msg);
ASSERT(nmsg == NULL || nmsg->request);
//如果读取出来的KV都是完整的，则conn->rmsg = NULL,如果读取内核协议栈缓冲区的数据最好一个KV没有读取完整，则conn->rmsg = nmsg(也就是新的一个msg)
/* enqueue next message (request), if any */
conn->rmsg = nmsg;
if (req_filter(ctx, conn, msg)) {
return; //客户端发送了quit命令过来，则不用再处理KV对了
}
if (msg->noforward) { //不需要转到后端服务器，因为没有认证成功
status = req_make_reply(ctx, conn, msg);
if (status != NC_OK) {
conn->err = errno;
return;
}
status = msg->reply(msg);
if (status != NC_OK) {
conn->err = errno;
return;
}
//通过core_core中的写事件触发写操作
status = event_add_out(ctx->evb, conn);
if (status != NC_OK) {
conn->err = errno;
}
return;
}
/* do fragment */
pool = conn->owner;
TAILQ_INIT(&frag_msgq);
//分片 mget mset等批处理命令中的不同KV可能分布在后端不同服务器上因此需要拆分
status = msg->fragment(msg, pool->ncontinuum, &frag_msgq);//如果需要分发到多个后端服务器，则frag_msgq不为空
if (status != NC_OK) {
if (!msg->noreply) {
conn->enqueue_outq(ctx, conn, msg);
}
req_forward_error(ctx, conn, msg);
}
/* if no fragment happened */
if (TAILQ_EMPTY(&frag_msgq)) {//如果需要分发到多个后端服务器，则frag_msgq不为空
req_forward(ctx, conn, msg); //转到后端服务器
return;
}
status = req_make_reply(ctx, conn, msg);
if (status != NC_OK) {
if (!msg->noreply) {
conn->enqueue_outq(ctx, conn, msg);
}
req_forward_error(ctx, conn, msg);
}
for (sub_msg = TAILQ_FIRST(&frag_msgq); sub_msg != NULL; sub_msg = tmsg) {
tmsg = TAILQ_NEXT(sub_msg, m_tqe);
TAILQ_REMOVE(&frag_msgq, sub_msg, m_tqe);
req_forward(ctx, conn, sub_msg); //
}
ASSERT(TAILQ_EMPTY(&frag_msgq));
return;
}
//转发到后端服务器
static void
req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
{
rstatus_t status;
struct conn *s_conn;
struct server_pool *pool;
uint8_t *key;
uint32_t keylen;
struct keypos *kpos;
ASSERT(c_conn->client && !c_conn->proxy);
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply) {
c_conn->enqueue_outq(ctx, c_conn, msg);
//req_forward把msg入队到客户端连接c_conn->enqueue_outq req_send_done把msg入队到服务端连接s_conn->enqueue_outq
}
pool = c_conn->owner;
ASSERT(array_n(msg->keys) > 0);
kpos = array_get(msg->keys, 0);
key = kpos->start;
keylen = (uint32_t)(kpos->end - kpos->start);
//选举后端服务器并建立连接
s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg);
return;
}
ASSERT(!s_conn->client && !s_conn->proxy);
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
//现在队列上面没有msg,但是下面会往该队列加msg,往后端的队列上面有msg,则添加些事件，通过epoll触发发送出去
status = event_add_out(ctx->evb, s_conn); //该写事件触发在core_core中的写事件把imsg_q中的msg发送出去
if (status != NC_OK) {
req_forward_error(ctx, c_conn, msg);
s_conn->err = errno;
return;
}
}
if (!conn_authenticated(s_conn)) { //现在还没有认证成功，则先进行认证
status = msg->add_auth(ctx, c_conn, s_conn);
if (status != NC_OK) {
req_forward_error(ctx, c_conn, msg);
s_conn->err = errno;
return;
}
}
//req_server_enqueue_imsgq
s_conn->enqueue_inq(ctx, s_conn, msg);//在core_core中的写事件把imsg_q中的msg发送出去
req_forward_stats(ctx, s_conn->owner, msg);
log_debug(LOG_VERB, "forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
