struct msg *
dnode_req_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
ASSERT(conn->type == CONN_DNODE_PEER_SERVER);
uint32_t now = time(NULL);
//throttling the sending traffics here
if (!conn->same_dc) {
if (conn->last_sent != 0) {
uint32_t elapsed_time = now - conn->last_sent;
uint32_t earned_tokens = elapsed_time * msgs_per_sec();
conn->avail_tokens = (conn->avail_tokens + earned_tokens) < msgs_per_sec()?
conn->avail_tokens + earned_tokens : msgs_per_sec();
}
conn->last_sent = now;
if (conn->avail_tokens > 0) {
conn->avail_tokens--;
return req_send_next(ctx, conn);
}
//requeue
status = event_add_out(ctx->evb, conn);
IGNORE_RET_VAL(status);
return NULL;
}
conn->last_sent = now;
return req_send_next(ctx, conn);
}
static struct gossip_node *
gossip_add_node_to_rack(struct server_pool *sp, struct string *dc, struct gossip_rack *g_rack,
struct string *address, struct string *ip, struct string *port, struct dyn_token *token)
{
rstatus_t status;
log_debug(LOG_VERB, "gossip_add_node_to_rack : dc[%.*s] rack[%.*s] address[%.*s] ip[%.*s] port[%.*s]",
dc->len, dc->data, g_rack->name, address->len, address->data, ip->len, ip->data, port->len, port->data);
int port_i = dn_atoi(port->data, port->len);
if (port_i == 0) {
return NULL; //bad data
}
struct gossip_node *gnode = (struct gossip_node *) array_push(&g_rack->nodes);
node_init(gnode);
status = string_copy(&gnode->dc, dc->data, dc->len);
status = string_copy(&gnode->rack, g_rack->name.data, g_rack->name.len);
status = string_copy(&gnode->name, ip->data, ip->len);
status = string_copy(&gnode->pname, address->data, address->len); //ignore the port for now
IGNORE_RET_VAL(status);
gnode->port = port_i;
struct dyn_token * gtoken = &gnode->token;
copy_dyn_token(token, gtoken);
g_rack->nnodes++;
//add into dicts
dictAdd(g_rack->dict_name_nodes, &gnode->name, gnode);
dictAdd(g_rack->dict_token_nodes, token_to_string(token), gnode);
return gnode;
}
static void
req_forward_error(struct context *ctx, struct conn *conn, struct msg *msg,
err_t err)
{
if (log_loggable(LOG_INFO)) {
log_debug(LOG_INFO, "forward req %"PRIu64" len %"PRIu32" type %d from "
"c %d failed: %s", msg->id, msg->mlen, msg->type, conn->sd,
strerror(err));
}
if (!msg->expect_datastore_reply) {
req_put(msg);
return;
}
// Create an appropriate response for the request so its propagated up;
// This response gets dropped in rsp_make_error anyways. But since this is
// an error path its ok with the overhead.
struct msg *rsp = msg_get(conn, false, __FUNCTION__);
rsp->peer = msg;
rsp->error = 1;
rsp->err = err;
rstatus_t status = conn_handle_response(conn, msg->id, rsp);
IGNORE_RET_VAL(status);
}
void
server_deinit(struct array *server)
{
uint32_t i, nserver;
for (i = 0, nserver = array_n(server); i < nserver; i++) {
struct server *s = array_pop(server);
IGNORE_RET_VAL(s);
ASSERT(TAILQ_EMPTY(&s->s_conn_q) && s->ns_conn_q == 0);
}
array_deinit(server);
}
static void
server_ack_err(struct context *ctx, struct conn *conn, struct msg *req)
{
// I want to make sure we do not have swallow here.
//ASSERT_LOG(!req->swallow, "req %d:%d has swallow set??", req->id, req->parent_id);
if ((req->swallow && req->noreply) ||
(req->swallow && (req->consistency == DC_ONE)) ||
(req->swallow && (req->consistency == DC_QUORUM)
&& (!conn->same_dc))) {
log_debug(LOG_INFO, "dyn: close s %d swallow req %"PRIu64" len %"PRIu32
" type %d", conn->sd, req->id, req->mlen, req->type);
req_put(req);
return;
}
struct conn *c_conn = req->owner;
// At other connections, these responses would be swallowed.
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT), "c_conn type %s",
conn_get_type_string(c_conn));
// Create an appropriate response for the request so its propagated up;
// This response gets dropped in rsp_make_error anyways. But since this is
// an error path its ok with the overhead.
struct msg *rsp = msg_get(conn, false, conn->data_store);
if (rsp == NULL) {
log_warn("Could not allocate msg.");
return;
}
req->done = 1;
req->peer = rsp;
rsp->peer = req;
rsp->error = req->error = 1;
rsp->err = req->err = conn->err;
rsp->dyn_error = req->dyn_error = STORAGE_CONNECTION_REFUSE;
rsp->dmsg = NULL;
log_warn("%d:%d <-> %d:%d", req->id, req->parent_id, rsp->id, rsp->parent_id);
log_warn("dyn: close s %d schedule error for req %u:%u "
"len %"PRIu32" type %d from c %d%c %s", conn->sd, req->id, req->parent_id,
req->mlen, req->type, c_conn->sd, conn->err ? ':' : ' ',
conn->err ? strerror(conn->err): " ");
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow)
req_put(req);
}
static void
send_rsp_integer(struct context *ctx, struct conn *c_conn, struct msg *req)
{
//do nothing
struct msg *rsp = msg_get_rsp_integer(c_conn);
if (req->expect_datastore_reply)
conn_enqueue_outq(ctx, c_conn, req);
req->peer = rsp;
rsp->peer = req;
req->selected_rsp = rsp;
req->done = 1;
//req->pre_coalesce(req);
rstatus_t status = event_add_out(ctx->evb, c_conn);
IGNORE_RET_VAL(status);
}
static void
server_rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(s_conn->type == CONN_SERVER);
/* response from server implies that server is ok and heartbeating */
server_ok(ctx, s_conn);
/* dequeue peer message (request) from server */
req = TAILQ_FIRST(&s_conn->omsg_q);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
conn_dequeue_outq(ctx, s_conn, req);
req->done = 1;
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
c_conn = req->owner;
log_info("c_conn %p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
server_rsp_forward_stats(ctx, s_conn->owner, rsp);
// this should really be the message's response handler be doing it
if (req_done(c_conn, req)) {
// handler owns the response now
status = conn_handle_response(c_conn, c_conn->type == CONN_CLIENT ?
req->id : req->parent_id, rsp);
IGNORE_RET_VAL(status);
}
}
/* There are chances that the request to the remote peer or its response got dropped.
* Hence we may not always receive a response to the request at the head of the FIFO.
* Hence what we do is we mark that request as errored and move on the next one
* in the outgoing queue. This works since we always have message ids in monotonically
* increasing order.
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
while (true) {
req = TAILQ_FIRST(&peer_conn->omsg_q);
log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp);
c_conn = req->owner;
if (req->id == rsp->dmsg->id) {
dnode_rsp_forward_match(ctx, peer_conn, rsp);
return;
}
// Report a mismatch and try to rectify
log_error("MISMATCH: dnode %c %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id,
rsp->parent_id);
if (c_conn && conn_to_ctx(c_conn))
stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner,
peer_mismatch_requests);
// TODO : should you be worried about message id getting wrapped around to 0?
if (rsp->dmsg->id < req->id) {
// We received a response from the past. This indeed proves out of order
// responses. A blunder to the architecture. Log it and drop the response.
log_error("MISMATCH: received response from the past. Dropping it");
dnode_rsp_put(rsp);
return;
}
if (req->consistency == DC_ONE) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_error("MISMATCHED DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, skipping....",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("skipping req: ");
msg_dump(req);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
// Create an appropriate response for the request so its propagated up;
struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store);
err_rsp->error = req->error = 1;
err_rsp->err = req->err = BAD_FORMAT;
err_rsp->dyn_error = req->dyn_error = BAD_FORMAT;
err_rsp->dmsg = dmsg_get();
err_rsp->dmsg->id = req->id;
log_debug(LOG_VERB, "%p <-> %p", req, err_rsp);
/* establish err_rsp <-> req (response <-> request) link */
req->peer = err_rsp;
err_rsp->peer = req;
log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u",
peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id);
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
err_rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, ",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
conn_dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_info("c_conn:%p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT),
"c_conn type %s", conn_get_type_string(c_conn));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
// c_conn owns respnse now
status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_info("swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
