static void
req_forward_error(struct context *ctx, struct conn *conn, struct msg *msg,
err_t err)
{
if (log_loggable(LOG_INFO)) {
log_debug(LOG_INFO, "forward req %"PRIu64" len %"PRIu32" type %d from "
"c %d failed: %s", msg->id, msg->mlen, msg->type, conn->sd,
strerror(err));
}
if (!msg->expect_datastore_reply) {
req_put(msg);
return;
}
// Create an appropriate response for the request so its propagated up;
// This response gets dropped in rsp_make_error anyways. But since this is
// an error path its ok with the overhead.
struct msg *rsp = msg_get(conn, false, __FUNCTION__);
rsp->peer = msg;
rsp->error = 1;
rsp->err = err;
rstatus_t status = conn_handle_response(conn, msg->id, rsp);
IGNORE_RET_VAL(status);
}
static void
server_ack_err(struct context *ctx, struct conn *conn, struct msg *req)
{
// I want to make sure we do not have swallow here.
//ASSERT_LOG(!req->swallow, "req %d:%d has swallow set??", req->id, req->parent_id);
if ((req->swallow && req->noreply) ||
(req->swallow && (req->consistency == DC_ONE)) ||
(req->swallow && (req->consistency == DC_QUORUM)
&& (!conn->same_dc))) {
log_debug(LOG_INFO, "dyn: close s %d swallow req %"PRIu64" len %"PRIu32
" type %d", conn->sd, req->id, req->mlen, req->type);
req_put(req);
return;
}
struct conn *c_conn = req->owner;
// At other connections, these responses would be swallowed.
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT), "c_conn type %s",
conn_get_type_string(c_conn));
// Create an appropriate response for the request so its propagated up;
// This response gets dropped in rsp_make_error anyways. But since this is
// an error path its ok with the overhead.
struct msg *rsp = msg_get(conn, false, conn->data_store);
if (rsp == NULL) {
log_warn("Could not allocate msg.");
return;
}
req->done = 1;
req->peer = rsp;
rsp->peer = req;
rsp->error = req->error = 1;
rsp->err = req->err = conn->err;
rsp->dyn_error = req->dyn_error = STORAGE_CONNECTION_REFUSE;
rsp->dmsg = NULL;
log_warn("%d:%d <-> %d:%d", req->id, req->parent_id, rsp->id, rsp->parent_id);
log_warn("dyn: close s %d schedule error for req %u:%u "
"len %"PRIu32" type %d from c %d%c %s", conn->sd, req->id, req->parent_id,
req->mlen, req->type, c_conn->sd, conn->err ? ':' : ' ',
conn->err ? strerror(conn->err): " ");
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow)
req_put(req);
}
static void
server_rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(s_conn->type == CONN_SERVER);
/* response from server implies that server is ok and heartbeating */
server_ok(ctx, s_conn);
/* dequeue peer message (request) from server */
req = TAILQ_FIRST(&s_conn->omsg_q);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
conn_dequeue_outq(ctx, s_conn, req);
req->done = 1;
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
c_conn = req->owner;
log_info("c_conn %p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
server_rsp_forward_stats(ctx, s_conn->owner, rsp);
// this should really be the message's response handler be doing it
if (req_done(c_conn, req)) {
// handler owns the response now
status = conn_handle_response(c_conn, c_conn->type == CONN_CLIENT ?
req->id : req->parent_id, rsp);
IGNORE_RET_VAL(status);
}
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, ",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_debug(LOG_VERB, "%p <-> %p", req, rsp);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT((c_conn->client && !c_conn->proxy) || (c_conn->dnode_client && !c_conn->dnode_server));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
if (TAILQ_FIRST(&c_conn->omsg_q) != NULL && dnode_req_done(c_conn, req)) {
log_debug(LOG_INFO, "handle rsp %d:%d for req %d:%d conn %p",
rsp->id, rsp->parent_id, req->id, req->parent_id, c_conn);
// c_conn owns respnse now
rstatus_t status = conn_handle_response(c_conn,
req->parent_id ? req->parent_id : req->id,
rsp);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
/* There are chances that the request to the remote peer or its response got dropped.
* Hence we may not always receive a response to the request at the head of the FIFO.
* Hence what we do is we mark that request as errored and move on the next one
* in the outgoing queue. This works since we always have message ids in monotonically
* increasing order.
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
while (true) {
req = TAILQ_FIRST(&peer_conn->omsg_q);
log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp);
c_conn = req->owner;
if (req->id == rsp->dmsg->id) {
dnode_rsp_forward_match(ctx, peer_conn, rsp);
return;
}
// Report a mismatch and try to rectify
log_error("MISMATCH: dnode %c %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id,
rsp->parent_id);
if (c_conn && conn_to_ctx(c_conn))
stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner,
peer_mismatch_requests);
// TODO : should you be worried about message id getting wrapped around to 0?
if (rsp->dmsg->id < req->id) {
// We received a response from the past. This indeed proves out of order
// responses. A blunder to the architecture. Log it and drop the response.
log_error("MISMATCH: received response from the past. Dropping it");
dnode_rsp_put(rsp);
return;
}
if (req->consistency == DC_ONE) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_error("MISMATCHED DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, skipping....",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("skipping req: ");
msg_dump(req);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
// Create an appropriate response for the request so its propagated up;
struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store);
err_rsp->error = req->error = 1;
err_rsp->err = req->err = BAD_FORMAT;
err_rsp->dyn_error = req->dyn_error = BAD_FORMAT;
err_rsp->dmsg = dmsg_get();
err_rsp->dmsg->id = req->id;
log_debug(LOG_VERB, "%p <-> %p", req, err_rsp);
/* establish err_rsp <-> req (response <-> request) link */
req->peer = err_rsp;
err_rsp->peer = req;
log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u",
peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id);
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
err_rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, ",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
conn_dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_info("c_conn:%p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT),
"c_conn type %s", conn_get_type_string(c_conn));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
// c_conn owns respnse now
status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_info("swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
void
remote_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
struct node * peer = dnode_peer_pool_server(ctx, c_conn->owner, rack, key,
keylen, msg->msg_routing);
if (peer->is_local) {
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d is local", c_conn,
msg->id, msg->parent_id);
local_req_forward(ctx, c_conn, msg, key, keylen);
return;
}
/* enqueue message (request) into client outq, if response is expected */
if (msg->expect_datastore_reply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
// now get a peer connection
struct conn *p_conn = dnode_peer_pool_server_conn(ctx, peer);
if ((p_conn == NULL) || (p_conn->connecting)) {
if (p_conn) {
usec_t now = dn_usec_now();
static usec_t next_log = 0; // Log every 1 sec
if (now > next_log) {
log_warn("still connecting to peer '%.*s'......",
peer->endpoint.pname.len, peer->endpoint.pname.data);
next_log = now + 1000 * 1000;
}
}
// No response for DC_ONE & swallow
if ((msg->consistency == DC_ONE) && (msg->swallow)) {
msg_put(msg);
return;
}
// No response for remote dc
struct server_pool *pool = c_conn->owner;
bool same_dc = is_same_dc(pool, peer)? 1 : 0;
if (!same_dc) {
msg_put(msg);
return;
}
// All other cases return a response
struct msg *rsp = msg_get(c_conn, false, __FUNCTION__);
msg->done = 1;
rsp->error = msg->error = 1;
rsp->err = msg->err = (p_conn ? PEER_HOST_NOT_CONNECTED : PEER_HOST_DOWN);
rsp->dyn_error = msg->dyn_error = (p_conn ? PEER_HOST_NOT_CONNECTED:
PEER_HOST_DOWN);
rsp->dmsg = dmsg_get();
rsp->peer = msg;
rsp->dmsg->id = msg->id;
log_info("%lu:%lu <-> %lu:%lu Short circuit....", msg->id, msg->parent_id, rsp->id, rsp->parent_id);
conn_handle_response(c_conn, msg->parent_id ? msg->parent_id : msg->id,
rsp);
if (msg->swallow)
msg_put(msg);
return;
}
log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d to p_conn %p", c_conn,
msg->id, msg->parent_id, p_conn);
dnode_peer_req_forward(ctx, c_conn, p_conn, msg, rack, key, keylen);
}
