static void
dnode_req_local_forward(struct context *ctx, struct conn *conn, struct msg *msg)
{
struct server_pool *pool;
uint8_t *key;
uint32_t keylen;
log_debug(LOG_VERB, "dnode_req_local_forward entering ");
ASSERT(conn->dnode_client && !conn->dnode_server);
pool = conn->owner;
key = NULL;
keylen = 0;
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = dn_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = dn_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
local_req_forward(ctx, conn, msg, key, keylen);
}
static void
admin_local_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
struct node *peer = dnode_peer_pool_server(ctx, c_conn->owner, rack, key, keylen, msg->msg_routing);
if (!peer->is_local) {
send_rsp_integer(ctx, c_conn, msg);
return;
}
struct conn *p_conn = dnode_peer_pool_server_conn(ctx, peer);
if (p_conn == NULL) {
c_conn->err = EHOSTDOWN;
req_forward_error(ctx, c_conn, msg, c_conn->err);
return;
}
log_debug(LOG_NOTICE, "Need to delete [%.*s] ", keylen, key);
local_req_forward(ctx, c_conn, msg, key, keylen);
}
void
remote_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
struct conn *p_conn;
ASSERT(c_conn->client || c_conn->dnode_client);
p_conn = dnode_peer_pool_conn(ctx, c_conn->owner, rack, key, keylen, msg->msg_type);
if (p_conn == NULL) {
req_forward_error(ctx, c_conn, msg);
return;
}
//jeb - check if s_conn is _this_ node, and if so, get conn from server_pool_conn instead
struct server *peer = p_conn->owner;
if (peer->is_local) {
local_req_forward(ctx, c_conn, msg, key, keylen);
return;
} else {
dnode_peer_req_forward(ctx, c_conn, p_conn, msg, rack, key, keylen);
}
}
static void
dnode_req_forward(struct context *ctx, struct conn *conn, struct msg *msg)
{
struct server_pool *pool;
uint8_t *key;
uint32_t keylen;
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "dnode_req_forward entering ");
}
log_debug(LOG_DEBUG, "DNODE REQ RECEIVED %s %d dmsg->id %u",
conn_get_type_string(conn), conn->sd, msg->dmsg->id);
ASSERT(conn->type == CONN_DNODE_PEER_CLIENT);
pool = conn->owner;
key = NULL;
keylen = 0;
log_debug(LOG_DEBUG, "conn %p adding message %d:%d", conn, msg->id, msg->parent_id);
dictAdd(conn->outstanding_msgs_dict, &msg->id, msg);
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = dn_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = dn_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
ASSERT(msg->dmsg != NULL);
if (msg->dmsg->type == DMSG_REQ) {
local_req_forward(ctx, conn, msg, key, keylen);
} else if (msg->dmsg->type == DMSG_REQ_FORWARD) {
struct mbuf *orig_mbuf = STAILQ_FIRST(&msg->mhdr);
struct datacenter *dc = server_get_dc(pool, &pool->dc);
uint32_t rack_cnt = array_n(&dc->racks);
uint32_t rack_index;
for(rack_index = 0; rack_index < rack_cnt; rack_index++) {
struct rack *rack = array_get(&dc->racks, rack_index);
//log_debug(LOG_DEBUG, "forwarding to rack '%.*s'",
// rack->name->len, rack->name->data);
struct msg *rack_msg;
if (string_compare(rack->name, &pool->rack) == 0 ) {
rack_msg = msg;
} else {
rack_msg = msg_get(conn, msg->request, __FUNCTION__);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed now, buddy. no inter-rack messages for you!");
continue;
}
if (msg_clone(msg, orig_mbuf, rack_msg) != DN_OK) {
msg_put(rack_msg);
continue;
}
rack_msg->swallow = true;
}
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request from conn '%s' to rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(conn->sd), rack->name->len, rack->name->data, rack->dc->len, rack->dc->data);
}
remote_req_forward(ctx, conn, rack_msg, rack, key, keylen);
}
}
}
static void
req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
{
struct server_pool *pool = c_conn->owner;
uint8_t *key;
uint32_t keylen;
ASSERT(c_conn->type == CONN_CLIENT);
if (msg->is_read) {
if (msg->type != MSG_REQ_REDIS_PING)
stats_pool_incr(ctx, client_read_requests);
} else
stats_pool_incr(ctx, client_write_requests);
key = NULL;
keylen = 0;
// add the message to the dict
log_debug(LOG_DEBUG, "conn %p adding message %d:%d", c_conn, msg->id, msg->parent_id);
dictAdd(c_conn->outstanding_msgs_dict, &msg->id, msg);
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = dn_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = dn_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
// need to capture the initial mbuf location as once we add in the dynomite
// headers (as mbufs to the src msg), that will bork the request sent to
// secondary racks
struct mbuf *orig_mbuf = STAILQ_FIRST(&msg->mhdr);
if (ctx->admin_opt == 1) {
if (msg->type == MSG_REQ_REDIS_DEL || msg->type == MSG_REQ_MC_DELETE) {
struct rack * rack = server_get_rack_by_dc_rack(pool, &pool->rack, &pool->dc);
admin_local_req_forward(ctx, c_conn, msg, rack, key, keylen);
return;
}
}
if (msg->msg_routing == ROUTING_LOCAL_NODE_ONLY) {
// Strictly local host only
msg->consistency = DC_ONE;
msg->rsp_handler = msg_local_one_rsp_handler;
local_req_forward(ctx, c_conn, msg, key, keylen);
return;
}
if (msg->is_read) {
msg->consistency = conn_get_read_consistency(c_conn);
} else {
msg->consistency = conn_get_write_consistency(c_conn);
}
/* forward the request */
uint32_t dc_cnt = array_n(&pool->datacenters);
uint32_t dc_index;
for(dc_index = 0; dc_index < dc_cnt; dc_index++) {
struct datacenter *dc = array_get(&pool->datacenters, dc_index);
if (dc == NULL) {
log_error("Wow, this is very bad, dc is NULL");
return;
}
if (string_compare(dc->name, &pool->dc) == 0)
req_forward_local_dc(ctx, c_conn, msg, orig_mbuf, key, keylen, dc);
else if (request_send_to_all_dcs(msg)) {
req_forward_remote_dc(ctx, c_conn, msg, orig_mbuf, key, keylen, dc);
}
}
}
void
remote_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
struct node * peer = dnode_peer_pool_server(ctx, c_conn->owner, rack, key,
keylen, msg->msg_routing);
if (peer->is_local) {
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d is local", c_conn,
msg->id, msg->parent_id);
local_req_forward(ctx, c_conn, msg, key, keylen);
return;
}
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
// now get a peer connection
struct conn *p_conn = dnode_peer_pool_server_conn(ctx, peer);
if ((p_conn == NULL) || (p_conn->connecting)) {
if (p_conn) {
usec_t now = dn_usec_now();
static usec_t next_log = 0; // Log every 1 sec
if (now > next_log) {
log_warn("still connecting to peer '%.*s'......",
peer->endpoint.pname.len, peer->endpoint.pname.data);
next_log = now + 1000 * 1000;
}
}
// No response for DC_ONE & swallow
if ((msg->consistency == DC_ONE) && (msg->swallow)) {
msg_put(msg);
return;
}
// No response for remote dc
struct server_pool *pool = c_conn->owner;
bool same_dc = is_same_dc(pool, peer)? 1 : 0;
if (!same_dc) {
msg_put(msg);
return;
}
// All other cases return a response
struct msg *rsp = msg_get(c_conn, false, __FUNCTION__);
msg->done = 1;
rsp->error = msg->error = 1;
rsp->err = msg->err = (p_conn ? PEER_HOST_NOT_CONNECTED : PEER_HOST_DOWN);
rsp->dyn_error = msg->dyn_error = (p_conn ? PEER_HOST_NOT_CONNECTED:
PEER_HOST_DOWN);
rsp->dmsg = dmsg_get();
rsp->peer = msg;
rsp->dmsg->id = msg->id;
log_info("%lu:%lu <-> %lu:%lu Short circuit....", msg->id, msg->parent_id, rsp->id, rsp->parent_id);
conn_handle_response(c_conn, msg->parent_id ? msg->parent_id : msg->id,
rsp);
if (msg->swallow)
msg_put(msg);
return;
}
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d to p_conn %p", c_conn,
msg->id, msg->parent_id, p_conn);
dnode_peer_req_forward(ctx, c_conn, p_conn, msg, rack, key, keylen);
}
