void
req_client_dequeue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg)
{
ASSERT(msg->request);
ASSERT(conn->client && !conn->proxy);
uint64_t latency = dn_usec_now() - msg->stime_in_microsec;
histo_add(latency);
TAILQ_REMOVE(&conn->omsg_q, msg, c_tqe);
}
static void
req_client_enqueue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg)
{
ASSERT(msg->request);
ASSERT(conn->type == CONN_CLIENT);
conn->omsg_count++;
histo_add(&ctx->stats->client_out_queue, conn->omsg_count);
TAILQ_INSERT_TAIL(&conn->omsg_q, msg, c_tqe);
log_debug(LOG_VERB, "conn %p enqueue outq %d:%d", conn, msg->id, msg->parent_id);
}
static void
dnode_req_client_dequeue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg)
{
ASSERT(msg->request);
ASSERT(conn->type == CONN_DNODE_PEER_CLIENT);
TAILQ_REMOVE(&conn->omsg_q, msg, c_tqe);
log_debug(LOG_VERB, "conn %p dequeue outq %p", conn, msg);
//use the 1st pool
conn->omsg_count--;
histo_add(&ctx->stats->dnode_client_out_queue, conn->omsg_count);
stats_pool_decr(ctx, dnode_client_out_queue);
stats_pool_decr_by(ctx, dnode_client_out_queue_bytes, msg->mlen);
}
static void
req_client_dequeue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg)
{
ASSERT(msg->request);
ASSERT(conn->type == CONN_CLIENT);
if (msg->stime_in_microsec) {
usec_t latency = dn_usec_now() - msg->stime_in_microsec;
stats_histo_add_latency(ctx, latency);
}
conn->omsg_count--;
histo_add(&ctx->stats->client_out_queue, conn->omsg_count);
TAILQ_REMOVE(&conn->omsg_q, msg, c_tqe);
log_debug(LOG_VERB, "conn %p dequeue outq %p", conn, msg);
}
void stats_histo_add_payloadsize(struct context *ctx, uint64_t val)
{
struct stats *st = ctx->stats;
histo_add(&st->payload_size_histo, val);
}
//should use macro or something else to make this more elegant
void stats_histo_add_latency(struct context *ctx, uint64_t val)
{
struct stats *st = ctx->stats;
histo_add(&st->latency_histo, val);
}
/* There are chances that the request to the remote peer or its response got dropped.
* Hence we may not always receive a response to the request at the head of the FIFO.
* Hence what we do is we mark that request as errored and move on the next one
* in the outgoing queue. This works since we always have message ids in monotonically
* increasing order.
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(peer_conn->type == CONN_DNODE_PEER_SERVER);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
while (true) {
req = TAILQ_FIRST(&peer_conn->omsg_q);
log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp);
c_conn = req->owner;
if (!peer_conn->same_dc && req->remote_region_send_time) {
struct stats *st = ctx->stats;
uint64_t delay = dn_usec_now() - req->remote_region_send_time;
histo_add(&st->cross_region_histo, delay);
}
if (req->id == rsp->dmsg->id) {
dnode_rsp_forward_match(ctx, peer_conn, rsp);
return;
}
// Report a mismatch and try to rectify
log_error("MISMATCH: dnode %s %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id,
rsp->parent_id);
if (c_conn && conn_to_ctx(c_conn))
stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner,
peer_mismatch_requests);
// TODO : should you be worried about message id getting wrapped around to 0?
if (rsp->dmsg->id < req->id) {
// We received a response from the past. This indeed proves out of order
// responses. A blunder to the architecture. Log it and drop the response.
log_error("MISMATCH: received response from the past. Dropping it");
rsp_put(rsp);
return;
}
if (req->consistency == DC_ONE) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_error("MISMATCHED DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, skipping....",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("skipping req: ");
msg_dump(req);
}
conn_dequeue_outq(ctx, peer_conn, req);
req->done = 1;
// Create an appropriate response for the request so its propagated up;
struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store,
__FUNCTION__);
err_rsp->error = req->error = 1;
err_rsp->err = req->err = BAD_FORMAT;
err_rsp->dyn_error = req->dyn_error = BAD_FORMAT;
err_rsp->dmsg = dmsg_get();
err_rsp->dmsg->id = req->id;
log_debug(LOG_VERB, "%p <-> %p", req, err_rsp);
/* establish err_rsp <-> req (response <-> request) link */
req->peer = err_rsp;
err_rsp->peer = req;
log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u",
peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id);
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
err_rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
