static void
send_rsp_integer(struct context *ctx, struct conn *c_conn, struct msg *req)
{
//do nothing
struct msg *rsp = msg_get_rsp_integer(c_conn);
if (req->expect_datastore_reply)
conn_enqueue_outq(ctx, c_conn, req);
req->peer = rsp;
rsp->peer = req;
req->selected_rsp = rsp;
req->done = 1;
//req->pre_coalesce(req);
rstatus_t status = event_add_out(ctx->evb, c_conn);
IGNORE_RET_VAL(status);
}
/* Forward a client request over to a peer */
void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn,
struct conn *p_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
struct server *server = p_conn->owner;
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd),
rack->name->len, rack->name->data,
server->dc.len, server->dc.data);
struct string *dc = rack->dc;
rstatus_t status;
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue the message (request) into peer inq */
status = event_add_out(ctx->evb, p_conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, p_conn, msg);
p_conn->err = errno;
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
req_put(msg);
return;
}
struct server_pool *pool = c_conn->owner;
dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ;
if (p_conn->dnode_secured) {
//Encrypting and adding header for a request
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN));
}
//write dnode header
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(msg, p_conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(msg);
return;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the msg payload");
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
} else {
//write dnode header
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: ");
msg_dump(msg);
}
conn_enqueue_inq(ctx, p_conn, msg);
dnode_peer_req_forward_stats(ctx, p_conn->owner, msg);
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
void
remote_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
struct node * peer = dnode_peer_pool_server(ctx, c_conn->owner, rack, key,
keylen, msg->msg_routing);
if (peer->is_local) {
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d is local", c_conn,
msg->id, msg->parent_id);
local_req_forward(ctx, c_conn, msg, key, keylen);
return;
}
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
// now get a peer connection
struct conn *p_conn = dnode_peer_pool_server_conn(ctx, peer);
if ((p_conn == NULL) || (p_conn->connecting)) {
if (p_conn) {
usec_t now = dn_usec_now();
static usec_t next_log = 0; // Log every 1 sec
if (now > next_log) {
log_warn("still connecting to peer '%.*s'......",
peer->endpoint.pname.len, peer->endpoint.pname.data);
next_log = now + 1000 * 1000;
}
}
// No response for DC_ONE & swallow
if ((msg->consistency == DC_ONE) && (msg->swallow)) {
msg_put(msg);
return;
}
// No response for remote dc
struct server_pool *pool = c_conn->owner;
bool same_dc = is_same_dc(pool, peer)? 1 : 0;
if (!same_dc) {
msg_put(msg);
return;
}
// All other cases return a response
struct msg *rsp = msg_get(c_conn, false, __FUNCTION__);
msg->done = 1;
rsp->error = msg->error = 1;
rsp->err = msg->err = (p_conn ? PEER_HOST_NOT_CONNECTED : PEER_HOST_DOWN);
rsp->dyn_error = msg->dyn_error = (p_conn ? PEER_HOST_NOT_CONNECTED:
PEER_HOST_DOWN);
rsp->dmsg = dmsg_get();
rsp->peer = msg;
rsp->dmsg->id = msg->id;
log_info("%lu:%lu <-> %lu:%lu Short circuit....", msg->id, msg->parent_id, rsp->id, rsp->parent_id);
conn_handle_response(c_conn, msg->parent_id ? msg->parent_id : msg->id,
rsp);
if (msg->swallow)
msg_put(msg);
return;
}
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d to p_conn %p", c_conn,
msg->id, msg->parent_id, p_conn);
dnode_peer_req_forward(ctx, c_conn, p_conn, msg, rack, key, keylen);
}
void
local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
uint8_t *key, uint32_t keylen)
{
rstatus_t status;
struct conn *s_conn;
if (log_loggable(LOG_VVERB)) {
loga("local_req_forward entering ............");
}
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply) {
conn_enqueue_outq(ctx, c_conn, msg);
}
s_conn = get_datastore_conn(ctx, c_conn->owner);
log_debug(LOG_VERB, "c_conn %p got server conn %p", c_conn, s_conn);
if (s_conn == NULL) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
ASSERT(s_conn->type == CONN_SERVER);
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to storage conn '%s'",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(s_conn->sd));
}
if (ctx->dyn_state == NORMAL) {
/* enqueue the message (request) into server inq */
if (TAILQ_EMPTY(&s_conn->imsg_q)) {
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
} else if (ctx->dyn_state == STANDBY) { //no reads/writes from peers/clients
log_debug(LOG_INFO, "Node is in STANDBY state. Drop write/read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == WRITES_ONLY && msg->is_read) {
//no reads from peers/clients but allow writes from peers/clients
log_debug(LOG_INFO, "Node is in WRITES_ONLY state. Drop read requests");
req_forward_error(ctx, c_conn, msg, errno);
return;
} else if (ctx->dyn_state == RESUMING) {
log_debug(LOG_INFO, "Node is in RESUMING state. Still drop read requests and flush out all the queued writes");
if (msg->is_read) {
req_forward_error(ctx, c_conn, msg, errno);
return;
}
status = event_add_out(ctx->evb, s_conn);
if (status != DN_OK) {
req_forward_error(ctx, c_conn, msg, errno);
s_conn->err = errno;
return;
}
}
conn_enqueue_inq(ctx, s_conn, msg);
req_forward_stats(ctx, msg);
if(g_data_store == DATA_REDIS){
req_redis_stats(ctx, msg);
}
if (log_loggable(LOG_VERB)) {
log_debug(LOG_VERB, "local forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, s_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}