void server_close(struct context *ctx, struct conn *conn) { rstatus_t status; struct msg *msg, *nmsg; /* current and next message */ struct conn *c_conn; /* peer client connection */ ASSERT(!conn->client && !conn->proxy); server_close_stats(ctx, conn->owner, conn->err, conn->eof, conn->connected); conn->connected = false; if (conn->sd < 0) { server_failure(ctx, conn->owner); conn->unref(conn); conn_put(conn); return; } for (msg = TAILQ_FIRST(&conn->imsg_q); msg != NULL; msg = nmsg) { nmsg = TAILQ_NEXT(msg, s_tqe); /* dequeue the message (request) from server inq */ conn->dequeue_inq(ctx, conn, msg); /* * Don't send any error response, if * 1. request is tagged as noreply or, * 2. client has already closed its connection */ if (msg->swallow || msg->noreply) { log_debug(LOG_INFO, "close s %d swallow req %"PRIu64" len %"PRIu32 " type %d", conn->sd, msg->id, msg->mlen, msg->type); req_put(msg); } else { c_conn = msg->owner; ASSERT(c_conn->client && !c_conn->proxy); msg->done = 1; msg->error = 1; msg->err = conn->err; if (msg->frag_owner != NULL) { msg->frag_owner->nfrag_done++; } if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) { event_add_out(ctx->evb, msg->owner); } log_debug(LOG_INFO, "close s %d schedule error for req %"PRIu64" " "len %"PRIu32" type %d from c %d%c %s", conn->sd, msg->id, msg->mlen, msg->type, c_conn->sd, conn->err ? ':' : ' ', conn->err ? strerror(conn->err): " "); } } ASSERT(TAILQ_EMPTY(&conn->imsg_q)); for (msg = TAILQ_FIRST(&conn->omsg_q); msg != NULL; msg = nmsg) { nmsg = TAILQ_NEXT(msg, s_tqe); /* dequeue the message (request) from server outq */ conn->dequeue_outq(ctx, conn, msg); if (msg->swallow) { log_debug(LOG_INFO, "close s %d swallow req %"PRIu64" len %"PRIu32 " type %d", conn->sd, msg->id, msg->mlen, msg->type); req_put(msg); } else { c_conn = msg->owner; ASSERT(c_conn->client && !c_conn->proxy); msg->done = 1; msg->error = 1; msg->err = conn->err; if (msg->frag_owner != NULL) { msg->frag_owner->nfrag_done++; } if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) { event_add_out(ctx->evb, msg->owner); } log_debug(LOG_INFO, "close s %d schedule error for req %"PRIu64" " "len %"PRIu32" type %d from c %d%c %s", conn->sd, msg->id, msg->mlen, msg->type, c_conn->sd, conn->err ? ':' : ' ', conn->err ? strerror(conn->err): " "); } } ASSERT(TAILQ_EMPTY(&conn->omsg_q)); msg = conn->rmsg; if (msg != NULL) { conn->rmsg = NULL; ASSERT(!msg->request); ASSERT(msg->peer == NULL); rsp_put(msg); log_debug(LOG_INFO, "close s %d discarding rsp %"PRIu64" len %"PRIu32" " "in error", conn->sd, msg->id, msg->mlen); } ASSERT(conn->smsg == NULL); server_failure(ctx, conn->owner); conn->unref(conn); status = close(conn->sd); if (status < 0) { log_error("close s %d failed, ignored: %s", conn->sd, strerror(errno)); } conn->sd = -1; conn_put(conn); }
static void rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *msg) { rstatus_t status; struct msg *pmsg; struct conn *c_conn; ASSERT(!s_conn->client && !s_conn->proxy); /* response from server implies that server is ok and heartbeating */ server_ok(ctx, s_conn); /* dequeue peer message (request) from server */ pmsg = TAILQ_FIRST(&s_conn->omsg_q); ASSERT(pmsg != NULL && pmsg->peer == NULL); ASSERT(pmsg->request && !pmsg->done); s_conn->dequeue_outq(ctx, s_conn, pmsg); pmsg->done = 1; /* establish msg <-> pmsg (response <-> request) link */ pmsg->peer = msg; msg->peer = pmsg; /* * Readjust responses of fragmented messages by not including the end * marker for all but the last response * * Valid responses for a fragmented requests are MSG_RSP_VALUE or, * MSG_RSP_END. For an invalid response, we send out SERVER_ERRROR with * EINVAL errno */ if (pmsg->frag_id != 0) { if (msg->type != MSG_RSP_VALUE && msg->type != MSG_RSP_END) { pmsg->error = 1; pmsg->err = EINVAL; } else if (!pmsg->last_fragment) { ASSERT(msg->end != NULL); for (;;) { struct mbuf *mbuf; mbuf = STAILQ_LAST(&msg->mhdr, mbuf, next); ASSERT(mbuf != NULL); /* * We cannot assert that end marker points to the last mbuf * Consider a scenario where end marker points to the * penultimate mbuf and the last mbuf only contains spaces * and CRLF: mhdr -> [...END] -> [\r\n] */ if (msg->end >= mbuf->pos && msg->end < mbuf->last) { /* end marker is within this mbuf */ msg->mlen -= (uint32_t)(mbuf->last - msg->end); mbuf->last = msg->end; break; } /* end marker is not in this mbuf */ msg->mlen -= mbuf_length(mbuf); mbuf_remove(&msg->mhdr, mbuf); mbuf_put(mbuf); } } } c_conn = pmsg->owner; ASSERT(c_conn->client && !c_conn->proxy); if (req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) { status = event_add_out(ctx->ep, c_conn); if (status != NC_OK) { c_conn->err = errno; } } rsp_forward_stats(ctx, msg, s_conn, c_conn); }
void req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg, struct msg *nmsg) { rstatus_t status; struct server_pool *pool; struct msg_tqh frag_msgq; struct msg *sub_msg; struct msg *tmsg; /* tmp next message */ ASSERT(conn->client && !conn->proxy); ASSERT(msg->request); ASSERT(msg->owner == conn); ASSERT(conn->rmsg == msg); ASSERT(nmsg == NULL || nmsg->request); /* enqueue next message (request), if any */ conn->rmsg = nmsg; if (req_filter(ctx, conn, msg)) { return; } if (msg->noforward) { status = req_make_reply(ctx, conn, msg); if (status != NC_OK) { conn->err = errno; return; } status = msg->reply(msg); if (status != NC_OK) { conn->err = errno; return; } status = event_add_out(ctx->evb, conn); if (status != NC_OK) { conn->err = errno; } return; } /* do fragment */ pool = conn->owner; TAILQ_INIT(&frag_msgq); status = msg->fragment(msg, pool->ncontinuum, &frag_msgq); if (status != NC_OK) { if (!msg->noreply) { conn->enqueue_outq(ctx, conn, msg); } req_forward_error(ctx, conn, msg); } /* if no fragment happened */ if (TAILQ_EMPTY(&frag_msgq)) { req_forward(ctx, conn, msg); return; } status = req_make_reply(ctx, conn, msg); if (status != NC_OK) { if (!msg->noreply) { conn->enqueue_outq(ctx, conn, msg); } req_forward_error(ctx, conn, msg); } for (sub_msg = TAILQ_FIRST(&frag_msgq); sub_msg != NULL; sub_msg = tmsg) { tmsg = TAILQ_NEXT(sub_msg, m_tqe); TAILQ_REMOVE(&frag_msgq, sub_msg, m_tqe); req_forward(ctx, conn, sub_msg); } ASSERT(TAILQ_EMPTY(&frag_msgq)); return; }
/* Forward a client request over to a peer */ void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn, struct conn *p_conn, struct msg *msg, struct rack *rack, uint8_t *key, uint32_t keylen) { struct server *server = p_conn->owner; log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ", dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd), rack->name->len, rack->name->data, server->dc.len, server->dc.data); struct string *dc = rack->dc; rstatus_t status; /* enqueue message (request) into client outq, if response is expected */ if (!msg->noreply && !msg->swallow) { conn_enqueue_outq(ctx, c_conn, msg); } ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER); ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); /* enqueue the message (request) into peer inq */ status = event_add_out(ctx->evb, p_conn); if (status != DN_OK) { dnode_req_forward_error(ctx, p_conn, msg); p_conn->err = errno; return; } struct mbuf *header_buf = mbuf_get(); if (header_buf == NULL) { loga("Unable to obtain an mbuf for dnode msg's header!"); req_put(msg); return; } struct server_pool *pool = c_conn->owner; dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ; if (p_conn->dnode_secured) { //Encrypting and adding header for a request if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN)); } //write dnode header if (ENCRYPTION) { status = dyn_aes_encrypt_msg(msg, p_conn->aes_key); if (status == DN_ERROR) { loga("OOM to obtain an mbuf for encryption!"); mbuf_put(header_buf); req_put(msg); return; } if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "#encrypted bytes : %d", status); } dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "no encryption on the msg payload"); } dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } } else { //write dnode header dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } mbuf_insert_head(&msg->mhdr, header_buf); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: "); msg_dump(msg); } conn_enqueue_inq(ctx, p_conn, msg); dnode_peer_req_forward_stats(ctx, p_conn->owner, msg); if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); } }
/* * Sending a mbuf of gossip data over the wire to a peer */ void dnode_peer_gossip_forward(struct context *ctx, struct conn *conn, int data_store, struct mbuf *data_buf) { rstatus_t status; struct msg *msg = msg_get(conn, 1, data_store, __FUNCTION__); if (msg == NULL) { log_debug(LOG_DEBUG, "Unable to obtain a msg"); return; } struct mbuf *header_buf = mbuf_get(); if (header_buf == NULL) { log_debug(LOG_DEBUG, "Unable to obtain a data_buf"); rsp_put(msg); return; } uint64_t msg_id = peer_msg_id++; if (conn->dnode_secured) { if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "Assemble a secured msg to send"); log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN)); } if (ENCRYPTION) { struct mbuf *encrypted_buf = mbuf_get(); if (encrypted_buf == NULL) { loga("Unable to obtain an data_buf for encryption!"); return; //TODOs: need to clean up } status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, conn->aes_key); if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "#encrypted bytes : %d", status); } //write dnode header dmsg_write(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(encrypted_buf)); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: "); log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: "); } mbuf_remove(&msg->mhdr, data_buf); mbuf_insert(&msg->mhdr, encrypted_buf); //free data_buf as no one will need it again mbuf_put(data_buf); //TODOS: need to remove this from the msg->mhdr as in the other method } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "No encryption"); } dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf)); mbuf_insert(&msg->mhdr, data_buf); } } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "Assemble a non-secured msg to send"); } dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf)); mbuf_insert(&msg->mhdr, data_buf); } mbuf_insert_head(&msg->mhdr, header_buf); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn gossip message header: "); msg_dump(msg); } /* enqueue the message (request) into peer inq */ if (TAILQ_EMPTY(&conn->imsg_q)) { status = event_add_out(ctx->evb, conn); if (status != DN_OK) { dnode_req_forward_error(ctx, conn, msg); conn->err = errno; return; } } //need to handle a reply //conn->enqueue_outq(ctx, conn, msg); msg->noreply = 1; conn_enqueue_inq(ctx, conn, msg); }
void local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg, uint8_t *key, uint32_t keylen) { rstatus_t status; struct conn *s_conn; if (log_loggable(LOG_VVERB)) { loga("local_req_forward entering ............"); } ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); /* enqueue message (request) into client outq, if response is expected */ if (msg->expect_datastore_reply) { conn_enqueue_outq(ctx, c_conn, msg); } s_conn = get_datastore_conn(ctx, c_conn->owner); log_debug(LOG_VERB, "c_conn %p got server conn %p", c_conn, s_conn); if (s_conn == NULL) { req_forward_error(ctx, c_conn, msg, errno); return; } ASSERT(s_conn->type == CONN_SERVER); if (log_loggable(LOG_DEBUG)) { log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to storage conn '%s'", dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(s_conn->sd)); } if (ctx->dyn_state == NORMAL) { /* enqueue the message (request) into server inq */ if (TAILQ_EMPTY(&s_conn->imsg_q)) { status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg, errno); s_conn->err = errno; return; } } } else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients log_debug(LOG_INFO, "Node is in STANDBY state. Drop write/read requests"); req_forward_error(ctx, c_conn, msg, errno); return; } else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) { //no reads from peers/clients but allow writes from peers/clients log_debug(LOG_INFO, "Node is in WRITES_ONLY state. Drop read requests"); req_forward_error(ctx, c_conn, msg, errno); return; } else if (ctx->dyn_state == RESUMING) { log_debug(LOG_INFO, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes"); if (msg->is_read) { req_forward_error(ctx, c_conn, msg, errno); return; } status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg, errno); s_conn->err = errno; return; } } conn_enqueue_inq(ctx, s_conn, msg); req_forward_stats(ctx, msg); if(g_data_store == DATA_REDIS){ req_redis_stats(ctx, msg); } if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); } }
static void req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg) { rstatus_t status; struct conn *s_conn; struct server_pool *pool; uint8_t *key; uint32_t keylen; ASSERT(c_conn->client && !c_conn->proxy); /* enqueue message (request) into client outq, if response is expected */ if (!msg->noreply) { c_conn->enqueue_outq(ctx, c_conn, msg); } pool = c_conn->owner; key = NULL; keylen = 0; /* * If hash_tag: is configured for this server pool, we use the part of * the key within the hash tag as an input to the distributor. Otherwise * we use the full key */ if (!string_empty(&pool->hash_tag)) { struct string *tag = &pool->hash_tag; uint8_t *tag_start, *tag_end; tag_start = nc_strchr(msg->key_start, msg->key_end, tag->data[0]); if (tag_start != NULL) { tag_end = nc_strchr(tag_start + 1, msg->key_end, tag->data[1]); if (tag_end != NULL) { key = tag_start + 1; keylen = (uint32_t)(tag_end - key); } } } if (keylen == 0) { key = msg->key_start; keylen = (uint32_t)(msg->key_end - msg->key_start); } s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen); if (s_conn == NULL) { req_forward_error(ctx, c_conn, msg); return; } ASSERT(!s_conn->client && !s_conn->proxy); /* enqueue the message (request) into server inq */ if (TAILQ_EMPTY(&s_conn->imsg_q)) { status = event_add_out(ctx->center->ep, s_conn); if (status != NC_OK) { req_forward_error(ctx, c_conn, msg); s_conn->err = errno; return; } } s_conn->enqueue_inq(ctx, s_conn, msg); req_forward_stats(ctx, s_conn->owner, msg); log_debug(LOG_VERB, "forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); }
void local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg, uint8_t *key, uint32_t keylen) { rstatus_t status; struct conn *s_conn; if (get_tracking_level() >= LOG_VVERB) { loga("local_req_forward entering ............"); } ASSERT((c_conn->client || c_conn->dnode_client) && !c_conn->proxy && !c_conn->dnode_server); if (c_conn->dyn_mode && !c_conn->same_dc && !msg->is_read) { msg->noreply = 1; } /* enqueue message (request) into client outq, if response is expected */ if (!msg->noreply) { c_conn->enqueue_outq(ctx, c_conn, msg); } s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen); if (s_conn == NULL) { req_forward_error(ctx, c_conn, msg); return; } ASSERT(!s_conn->client && !s_conn->proxy); if (ctx->dyn_state == NORMAL) { /* enqueue the message (request) into server inq */ if (TAILQ_EMPTY(&s_conn->imsg_q)) { status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg); s_conn->err = errno; return; } } } else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients log_debug(LOG_VERB, "Node is in STANDBY state. Drop write/read requests"); req_forward_error(ctx, c_conn, msg); return; } else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) { //no reads from peers/clients but allow writes from peers/clients log_debug(LOG_VERB, "Node is in WRITES_ONLY state. Drop read requests"); req_forward_error(ctx, c_conn, msg); return; } else if (ctx->dyn_state == RESUMING) { log_debug(LOG_VERB, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes"); if (msg->is_read) { req_forward_error(ctx, c_conn, msg); return; } status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg); s_conn->err = errno; return; } } s_conn->enqueue_inq(ctx, s_conn, msg); req_forward_stats(ctx, s_conn->owner, msg); log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); }
void req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg, struct msg *nmsg) { rstatus_t status; struct server_pool *pool; struct msg_tqh frag_msgq; struct msg *sub_msg; struct msg *tmsg; /* tmp next message */ ASSERT(conn->client && !conn->proxy); ASSERT(msg->request); ASSERT(msg->owner == conn); ASSERT(conn->rmsg == msg); ASSERT(nmsg == NULL || nmsg->request); //如果读取出来的KV都是完整的,则conn->rmsg = NULL,如果读取内核协议栈缓冲区的数据最好一个KV没有读取完整,则conn->rmsg = nmsg(也就是新的一个msg) /* enqueue next message (request), if any */ conn->rmsg = nmsg; if (req_filter(ctx, conn, msg)) { return; //客户端发送了quit命令过来,则不用再处理KV对了 } if (msg->noforward) { //不需要转到后端服务器,因为没有认证成功 status = req_make_reply(ctx, conn, msg); if (status != NC_OK) { conn->err = errno; return; } status = msg->reply(msg); if (status != NC_OK) { conn->err = errno; return; } //通过core_core中的写事件触发写操作 status = event_add_out(ctx->evb, conn); if (status != NC_OK) { conn->err = errno; } return; } /* do fragment */ pool = conn->owner; TAILQ_INIT(&frag_msgq); //分片 mget mset等批处理命令中的不同KV可能分布在后端不同服务器上因此需要拆分 status = msg->fragment(msg, pool->ncontinuum, &frag_msgq);//如果需要分发到多个后端服务器,则frag_msgq不为空 if (status != NC_OK) { if (!msg->noreply) { conn->enqueue_outq(ctx, conn, msg); } req_forward_error(ctx, conn, msg); } /* if no fragment happened */ if (TAILQ_EMPTY(&frag_msgq)) {//如果需要分发到多个后端服务器,则frag_msgq不为空 req_forward(ctx, conn, msg); //转到后端服务器 return; } status = req_make_reply(ctx, conn, msg); if (status != NC_OK) { if (!msg->noreply) { conn->enqueue_outq(ctx, conn, msg); } req_forward_error(ctx, conn, msg); } for (sub_msg = TAILQ_FIRST(&frag_msgq); sub_msg != NULL; sub_msg = tmsg) { tmsg = TAILQ_NEXT(sub_msg, m_tqe); TAILQ_REMOVE(&frag_msgq, sub_msg, m_tqe); req_forward(ctx, conn, sub_msg); // } ASSERT(TAILQ_EMPTY(&frag_msgq)); return; }
//转发到后端服务器 static void req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg) { rstatus_t status; struct conn *s_conn; struct server_pool *pool; uint8_t *key; uint32_t keylen; struct keypos *kpos; ASSERT(c_conn->client && !c_conn->proxy); /* enqueue message (request) into client outq, if response is expected */ if (!msg->noreply) { c_conn->enqueue_outq(ctx, c_conn, msg); //req_forward把msg入队到客户端连接c_conn->enqueue_outq req_send_done把msg入队到服务端连接s_conn->enqueue_outq } pool = c_conn->owner; ASSERT(array_n(msg->keys) > 0); kpos = array_get(msg->keys, 0); key = kpos->start; keylen = (uint32_t)(kpos->end - kpos->start); //选举后端服务器并建立连接 s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen); if (s_conn == NULL) { req_forward_error(ctx, c_conn, msg); return; } ASSERT(!s_conn->client && !s_conn->proxy); /* enqueue the message (request) into server inq */ if (TAILQ_EMPTY(&s_conn->imsg_q)) { //现在队列上面没有msg,但是下面会往该队列加msg,往后端的队列上面有msg,则添加些事件,通过epoll触发发送出去 status = event_add_out(ctx->evb, s_conn); //该写事件触发在core_core中的写事件把imsg_q中的msg发送出去 if (status != NC_OK) { req_forward_error(ctx, c_conn, msg); s_conn->err = errno; return; } } if (!conn_authenticated(s_conn)) { //现在还没有认证成功,则先进行认证 status = msg->add_auth(ctx, c_conn, s_conn); if (status != NC_OK) { req_forward_error(ctx, c_conn, msg); s_conn->err = errno; return; } } //req_server_enqueue_imsgq s_conn->enqueue_inq(ctx, s_conn, msg);//在core_core中的写事件把imsg_q中的msg发送出去 req_forward_stats(ctx, s_conn->owner, msg); log_debug(LOG_VERB, "forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); }