/* Forward a client request over to a peer */ void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn, struct conn *p_conn, struct msg *msg, struct rack *rack, uint8_t *key, uint32_t keylen) { struct server *server = p_conn->owner; log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ", dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd), rack->name->len, rack->name->data, server->dc.len, server->dc.data); struct string *dc = rack->dc; rstatus_t status; /* enqueue message (request) into client outq, if response is expected */ if (!msg->noreply && !msg->swallow) { conn_enqueue_outq(ctx, c_conn, msg); } ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER); ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); /* enqueue the message (request) into peer inq */ status = event_add_out(ctx->evb, p_conn); if (status != DN_OK) { dnode_req_forward_error(ctx, p_conn, msg); p_conn->err = errno; return; } struct mbuf *header_buf = mbuf_get(); if (header_buf == NULL) { loga("Unable to obtain an mbuf for dnode msg's header!"); req_put(msg); return; } struct server_pool *pool = c_conn->owner; dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ; if (p_conn->dnode_secured) { //Encrypting and adding header for a request if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN)); } //write dnode header if (ENCRYPTION) { status = dyn_aes_encrypt_msg(msg, p_conn->aes_key); if (status == DN_ERROR) { loga("OOM to obtain an mbuf for encryption!"); mbuf_put(header_buf); req_put(msg); return; } if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "#encrypted bytes : %d", status); } dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VERB, "no encryption on the msg payload"); } dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } } else { //write dnode header dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg)); } mbuf_insert_head(&msg->mhdr, header_buf); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: "); msg_dump(msg); } conn_enqueue_inq(ctx, p_conn, msg); dnode_peer_req_forward_stats(ctx, p_conn->owner, msg); if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); } }
/* * Sending a mbuf of gossip data over the wire to a peer */ void dnode_peer_gossip_forward(struct context *ctx, struct conn *conn, int data_store, struct mbuf *data_buf) { rstatus_t status; struct msg *msg = msg_get(conn, 1, data_store, __FUNCTION__); if (msg == NULL) { log_debug(LOG_DEBUG, "Unable to obtain a msg"); return; } struct mbuf *header_buf = mbuf_get(); if (header_buf == NULL) { log_debug(LOG_DEBUG, "Unable to obtain a data_buf"); rsp_put(msg); return; } uint64_t msg_id = peer_msg_id++; if (conn->dnode_secured) { if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "Assemble a secured msg to send"); log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN)); } if (ENCRYPTION) { struct mbuf *encrypted_buf = mbuf_get(); if (encrypted_buf == NULL) { loga("Unable to obtain an data_buf for encryption!"); return; //TODOs: need to clean up } status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, conn->aes_key); if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "#encrypted bytes : %d", status); } //write dnode header dmsg_write(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(encrypted_buf)); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: "); log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: "); } mbuf_remove(&msg->mhdr, data_buf); mbuf_insert(&msg->mhdr, encrypted_buf); //free data_buf as no one will need it again mbuf_put(data_buf); //TODOS: need to remove this from the msg->mhdr as in the other method } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "No encryption"); } dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf)); mbuf_insert(&msg->mhdr, data_buf); } } else { if (log_loggable(LOG_VVERB)) { log_debug(LOG_VVERB, "Assemble a non-secured msg to send"); } dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf)); mbuf_insert(&msg->mhdr, data_buf); } mbuf_insert_head(&msg->mhdr, header_buf); if (log_loggable(LOG_VVERB)) { log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn gossip message header: "); msg_dump(msg); } /* enqueue the message (request) into peer inq */ if (TAILQ_EMPTY(&conn->imsg_q)) { status = event_add_out(ctx->evb, conn); if (status != DN_OK) { dnode_req_forward_error(ctx, conn, msg); conn->err = errno; return; } } //need to handle a reply //conn->enqueue_outq(ctx, conn, msg); msg->noreply = 1; conn_enqueue_inq(ctx, conn, msg); }
void local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg, uint8_t *key, uint32_t keylen) { rstatus_t status; struct conn *s_conn; if (log_loggable(LOG_VVERB)) { loga("local_req_forward entering ............"); } ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); /* enqueue message (request) into client outq, if response is expected */ if (msg->expect_datastore_reply) { conn_enqueue_outq(ctx, c_conn, msg); } s_conn = get_datastore_conn(ctx, c_conn->owner); log_debug(LOG_VERB, "c_conn %p got server conn %p", c_conn, s_conn); if (s_conn == NULL) { req_forward_error(ctx, c_conn, msg, errno); return; } ASSERT(s_conn->type == CONN_SERVER); if (log_loggable(LOG_DEBUG)) { log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to storage conn '%s'", dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(s_conn->sd)); } if (ctx->dyn_state == NORMAL) { /* enqueue the message (request) into server inq */ if (TAILQ_EMPTY(&s_conn->imsg_q)) { status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg, errno); s_conn->err = errno; return; } } } else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients log_debug(LOG_INFO, "Node is in STANDBY state. Drop write/read requests"); req_forward_error(ctx, c_conn, msg, errno); return; } else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) { //no reads from peers/clients but allow writes from peers/clients log_debug(LOG_INFO, "Node is in WRITES_ONLY state. Drop read requests"); req_forward_error(ctx, c_conn, msg, errno); return; } else if (ctx->dyn_state == RESUMING) { log_debug(LOG_INFO, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes"); if (msg->is_read) { req_forward_error(ctx, c_conn, msg, errno); return; } status = event_add_out(ctx->evb, s_conn); if (status != DN_OK) { req_forward_error(ctx, c_conn, msg, errno); s_conn->err = errno; return; } } conn_enqueue_inq(ctx, s_conn, msg); req_forward_stats(ctx, msg); if(g_data_store == DATA_REDIS){ req_redis_stats(ctx, msg); } if (log_loggable(LOG_VERB)) { log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32 " type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id, msg->mlen, msg->type, keylen, key); } }