/* Forward a client request over to a peer */
void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn,
struct conn *p_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
struct server *server = p_conn->owner;
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd),
rack->name->len, rack->name->data,
server->dc.len, server->dc.data);
struct string *dc = rack->dc;
rstatus_t status;
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue the message (request) into peer inq */
status = event_add_out(ctx->evb, p_conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, p_conn, msg);
p_conn->err = errno;
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
req_put(msg);
return;
}
struct server_pool *pool = c_conn->owner;
dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ;
if (p_conn->dnode_secured) {
//Encrypting and adding header for a request
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN));
}
//write dnode header
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(msg, p_conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(msg);
return;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the msg payload");
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
} else {
//write dnode header
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: ");
msg_dump(msg);
}
conn_enqueue_inq(ctx, p_conn, msg);
dnode_peer_req_forward_stats(ctx, p_conn->owner, msg);
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
/*
* Sending a mbuf of gossip data over the wire to a peer
*/
void
dnode_peer_gossip_forward(struct context *ctx, struct conn *conn, int data_store, struct mbuf *data_buf)
{
rstatus_t status;
struct msg *msg = msg_get(conn, 1, data_store, __FUNCTION__);
if (msg == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a msg");
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a data_buf");
rsp_put(msg);
return;
}
uint64_t msg_id = peer_msg_id++;
if (conn->dnode_secured) {
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "Assemble a secured msg to send");
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
}
if (ENCRYPTION) {
struct mbuf *encrypted_buf = mbuf_get();
if (encrypted_buf == NULL) {
loga("Unable to obtain an data_buf for encryption!");
return; //TODOs: need to clean up
}
status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, conn->aes_key);
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
//write dnode header
dmsg_write(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(encrypted_buf));
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: ");
log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: ");
}
mbuf_remove(&msg->mhdr, data_buf);
mbuf_insert(&msg->mhdr, encrypted_buf);
//free data_buf as no one will need it again
mbuf_put(data_buf); //TODOS: need to remove this from the msg->mhdr as in the other method
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "No encryption");
}
dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf));
mbuf_insert(&msg->mhdr, data_buf);
}
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "Assemble a non-secured msg to send");
}
dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf));
mbuf_insert(&msg->mhdr, data_buf);
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn gossip message header: ");
msg_dump(msg);
}
/* enqueue the message (request) into peer inq */
if (TAILQ_EMPTY(&conn->imsg_q)) {
status = event_add_out(ctx->evb, conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, conn, msg);
conn->err = errno;
return;
}
}
//need to handle a reply
//conn->enqueue_outq(ctx, conn, msg);
msg->noreply = 1;
conn_enqueue_inq(ctx, conn, msg);
}
void
local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
uint8_t *key, uint32_t keylen)
{
rstatus_t status;
struct conn *s_conn;
if (log_loggable(LOG_VVERB)) {
loga("local_req_forward entering ............");
}
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply) {
conn_enqueue_outq(ctx, c_conn, msg);
}
s_conn = get_datastore_conn(ctx, c_conn->owner);
log_debug(LOG_VERB, "c_conn %p got server conn %p", c_conn, s_conn);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
ASSERT(s_conn->type == CONN_SERVER);
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to storage conn '%s'",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(s_conn->sd));
}
if (ctx->dyn_state == NORMAL) {
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
} else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients
log_debug(LOG_INFO, "Node is in STANDBY state. Drop write/read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) {
//no reads from peers/clients but allow writes from peers/clients
log_debug(LOG_INFO, "Node is in WRITES_ONLY state. Drop read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == RESUMING) {
log_debug(LOG_INFO, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes");
if (msg->is_read) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
conn_enqueue_inq(ctx, s_conn, msg);
req_forward_stats(ctx, msg);
if(g_data_store == DATA_REDIS){
req_redis_stats(ctx, msg);
}
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
