/* There are chances that the request to the remote peer or its response got dropped.
* Hence we may not always receive a response to the request at the head of the FIFO.
* Hence what we do is we mark that request as errored and move on the next one
* in the outgoing queue. This works since we always have message ids in monotonically
* increasing order.
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
while (true) {
req = TAILQ_FIRST(&peer_conn->omsg_q);
log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp);
c_conn = req->owner;
if (req->id == rsp->dmsg->id) {
dnode_rsp_forward_match(ctx, peer_conn, rsp);
return;
}
// Report a mismatch and try to rectify
log_error("MISMATCH: dnode %c %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id,
rsp->parent_id);
if (c_conn && conn_to_ctx(c_conn))
stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner,
peer_mismatch_requests);
// TODO : should you be worried about message id getting wrapped around to 0?
if (rsp->dmsg->id < req->id) {
// We received a response from the past. This indeed proves out of order
// responses. A blunder to the architecture. Log it and drop the response.
log_error("MISMATCH: received response from the past. Dropping it");
dnode_rsp_put(rsp);
return;
}
if (req->consistency == DC_ONE) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_error("MISMATCHED DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, skipping....",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("skipping req: ");
msg_dump(req);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
// Create an appropriate response for the request so its propagated up;
struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store);
err_rsp->error = req->error = 1;
err_rsp->err = req->err = BAD_FORMAT;
err_rsp->dyn_error = req->dyn_error = BAD_FORMAT;
err_rsp->dmsg = dmsg_get();
err_rsp->dmsg->id = req->id;
log_debug(LOG_VERB, "%p <-> %p", req, err_rsp);
/* establish err_rsp <-> req (response <-> request) link */
req->peer = err_rsp;
err_rsp->peer = req;
log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u",
peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id);
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
err_rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, ",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_debug(LOG_VERB, "%p <-> %p", req, rsp);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT((c_conn->client && !c_conn->proxy) || (c_conn->dnode_client && !c_conn->dnode_server));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
if (TAILQ_FIRST(&c_conn->omsg_q) != NULL && dnode_req_done(c_conn, req)) {
log_debug(LOG_INFO, "handle rsp %d:%d for req %d:%d conn %p",
rsp->id, rsp->parent_id, req->id, req->parent_id, c_conn);
// c_conn owns respnse now
rstatus_t status = conn_handle_response(c_conn,
req->parent_id ? req->parent_id : req->id,
rsp);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, ",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
conn_dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_info("c_conn:%p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT),
"c_conn type %s", conn_get_type_string(c_conn));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
// c_conn owns respnse now
status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_info("swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
