static void req_forward_error(struct context *ctx, struct conn *conn, struct msg *msg, err_t err) { if (log_loggable(LOG_INFO)) { log_debug(LOG_INFO, "forward req %"PRIu64" len %"PRIu32" type %d from " "c %d failed: %s", msg->id, msg->mlen, msg->type, conn->sd, strerror(err)); } if (!msg->expect_datastore_reply) { req_put(msg); return; } // Create an appropriate response for the request so its propagated up; // This response gets dropped in rsp_make_error anyways. But since this is // an error path its ok with the overhead. struct msg *rsp = msg_get(conn, false, __FUNCTION__); rsp->peer = msg; rsp->error = 1; rsp->err = err; rstatus_t status = conn_handle_response(conn, msg->id, rsp); IGNORE_RET_VAL(status); }
static void server_ack_err(struct context *ctx, struct conn *conn, struct msg *req) { // I want to make sure we do not have swallow here. //ASSERT_LOG(!req->swallow, "req %d:%d has swallow set??", req->id, req->parent_id); if ((req->swallow && req->noreply) || (req->swallow && (req->consistency == DC_ONE)) || (req->swallow && (req->consistency == DC_QUORUM) && (!conn->same_dc))) { log_debug(LOG_INFO, "dyn: close s %d swallow req %"PRIu64" len %"PRIu32 " type %d", conn->sd, req->id, req->mlen, req->type); req_put(req); return; } struct conn *c_conn = req->owner; // At other connections, these responses would be swallowed. ASSERT_LOG((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT), "c_conn type %s", conn_get_type_string(c_conn)); // Create an appropriate response for the request so its propagated up; // This response gets dropped in rsp_make_error anyways. But since this is // an error path its ok with the overhead. struct msg *rsp = msg_get(conn, false, conn->data_store); if (rsp == NULL) { log_warn("Could not allocate msg."); return; } req->done = 1; req->peer = rsp; rsp->peer = req; rsp->error = req->error = 1; rsp->err = req->err = conn->err; rsp->dyn_error = req->dyn_error = STORAGE_CONNECTION_REFUSE; rsp->dmsg = NULL; log_warn("%d:%d <-> %d:%d", req->id, req->parent_id, rsp->id, rsp->parent_id); log_warn("dyn: close s %d schedule error for req %u:%u " "len %"PRIu32" type %d from c %d%c %s", conn->sd, req->id, req->parent_id, req->mlen, req->type, c_conn->sd, conn->err ? ':' : ' ', conn->err ? strerror(conn->err): " "); rstatus_t status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id, rsp); IGNORE_RET_VAL(status); if (req->swallow) req_put(req); }
static void server_rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *rsp) { rstatus_t status; struct msg *req; struct conn *c_conn; ASSERT(s_conn->type == CONN_SERVER); /* response from server implies that server is ok and heartbeating */ server_ok(ctx, s_conn); /* dequeue peer message (request) from server */ req = TAILQ_FIRST(&s_conn->omsg_q); ASSERT(req != NULL && req->peer == NULL); ASSERT(req->request && !req->done); conn_dequeue_outq(ctx, s_conn, req); req->done = 1; /* establish rsp <-> req (response <-> request) link */ req->peer = rsp; rsp->peer = req; rsp->pre_coalesce(rsp); c_conn = req->owner; log_info("c_conn %p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id, rsp->id, rsp->parent_id); ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); server_rsp_forward_stats(ctx, s_conn->owner, rsp); // this should really be the message's response handler be doing it if (req_done(c_conn, req)) { // handler owns the response now status = conn_handle_response(c_conn, c_conn->type == CONN_CLIENT ? req->id : req->parent_id, rsp); IGNORE_RET_VAL(status); } }
/* Description: link data from a peer connection to a client-facing connection * peer_conn: a peer connection * msg : msg with data from the peer connection after parsing */ static void dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp) { rstatus_t status; struct msg *req; struct conn *c_conn; req = TAILQ_FIRST(&peer_conn->omsg_q); c_conn = req->owner; /* if client consistency is dc_one forward the response from only the local node. Since dyn_dnode_peer is always a remote node, drop the rsp */ if (req->consistency == DC_ONE) { if (req->swallow) { dnode_rsp_swallow(ctx, peer_conn, req, rsp); return; } log_warn("req %d:%d with DC_ONE consistency is not being swallowed"); } /* if client consistency is dc_quorum, forward the response from only the local region/DC. */ if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) { if (req->swallow) { dnode_rsp_swallow(ctx, peer_conn, req, rsp); return; } log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed"); } log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, ", peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'), peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id, rsp->parent_id); ASSERT(req != NULL && req->peer == NULL); ASSERT(req->request && !req->done); if (log_loggable(LOG_VVERB)) { loga("Dumping content for response: "); msg_dump(rsp); loga("rsp id %d", rsp->id); loga("Dumping content for request:"); msg_dump(req); loga("req id %d", req->id); } peer_conn->dequeue_outq(ctx, peer_conn, req); req->done = 1; log_debug(LOG_VERB, "%p <-> %p", req, rsp); /* establish rsp <-> req (response <-> request) link */ req->peer = rsp; rsp->peer = req; rsp->pre_coalesce(rsp); ASSERT((c_conn->client && !c_conn->proxy) || (c_conn->dnode_client && !c_conn->dnode_server)); dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp); if (TAILQ_FIRST(&c_conn->omsg_q) != NULL && dnode_req_done(c_conn, req)) { log_debug(LOG_INFO, "handle rsp %d:%d for req %d:%d conn %p", rsp->id, rsp->parent_id, req->id, req->parent_id, c_conn); // c_conn owns respnse now rstatus_t status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id, rsp); if (req->swallow) { log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id); req_put(req); } } }
/* There are chances that the request to the remote peer or its response got dropped. * Hence we may not always receive a response to the request at the head of the FIFO. * Hence what we do is we mark that request as errored and move on the next one * in the outgoing queue. This works since we always have message ids in monotonically * increasing order. */ static void dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp) { rstatus_t status; struct msg *req; struct conn *c_conn; ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server); /* response from a peer implies that peer is ok and heartbeating */ dnode_peer_ok(ctx, peer_conn); /* dequeue peer message (request) from peer conn */ while (true) { req = TAILQ_FIRST(&peer_conn->omsg_q); log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp); c_conn = req->owner; if (req->id == rsp->dmsg->id) { dnode_rsp_forward_match(ctx, peer_conn, rsp); return; } // Report a mismatch and try to rectify log_error("MISMATCH: dnode %c %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u", peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'), peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id, rsp->parent_id); if (c_conn && conn_to_ctx(c_conn)) stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner, peer_mismatch_requests); // TODO : should you be worried about message id getting wrapped around to 0? if (rsp->dmsg->id < req->id) { // We received a response from the past. This indeed proves out of order // responses. A blunder to the architecture. Log it and drop the response. log_error("MISMATCH: received response from the past. Dropping it"); dnode_rsp_put(rsp); return; } if (req->consistency == DC_ONE) { if (req->swallow) { // swallow the request and move on the next one dnode_rsp_swallow(ctx, peer_conn, req, NULL); continue; } log_warn("req %d:%d with DC_ONE consistency is not being swallowed"); } if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) { if (req->swallow) { // swallow the request and move on the next one dnode_rsp_swallow(ctx, peer_conn, req, NULL); continue; } log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed"); } log_error("MISMATCHED DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, skipping....", peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'), peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id, rsp->parent_id); ASSERT(req != NULL && req->peer == NULL); ASSERT(req->request && !req->done); if (log_loggable(LOG_VVERB)) { loga("skipping req: "); msg_dump(req); } peer_conn->dequeue_outq(ctx, peer_conn, req); req->done = 1; // Create an appropriate response for the request so its propagated up; struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store); err_rsp->error = req->error = 1; err_rsp->err = req->err = BAD_FORMAT; err_rsp->dyn_error = req->dyn_error = BAD_FORMAT; err_rsp->dmsg = dmsg_get(); err_rsp->dmsg->id = req->id; log_debug(LOG_VERB, "%p <-> %p", req, err_rsp); /* establish err_rsp <-> req (response <-> request) link */ req->peer = err_rsp; err_rsp->peer = req; log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u", peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id); rstatus_t status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id, err_rsp); IGNORE_RET_VAL(status); if (req->swallow) { log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id); req_put(req); } } }
/* Description: link data from a peer connection to a client-facing connection * peer_conn: a peer connection * msg : msg with data from the peer connection after parsing */ static void dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp) { rstatus_t status; struct msg *req; struct conn *c_conn; req = TAILQ_FIRST(&peer_conn->omsg_q); c_conn = req->owner; /* if client consistency is dc_one forward the response from only the local node. Since dyn_dnode_peer is always a remote node, drop the rsp */ if (req->consistency == DC_ONE) { if (req->swallow) { dnode_rsp_swallow(ctx, peer_conn, req, rsp); return; } log_warn("req %d:%d with DC_ONE consistency is not being swallowed"); } /* if client consistency is dc_quorum, forward the response from only the local region/DC. */ if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) { if (req->swallow) { dnode_rsp_swallow(ctx, peer_conn, req, rsp); return; } log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed"); } log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, ", conn_get_type_string(peer_conn), peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id, rsp->parent_id); ASSERT(req != NULL && req->peer == NULL); ASSERT(req->request && !req->done); if (log_loggable(LOG_VVERB)) { loga("Dumping content for response: "); msg_dump(rsp); loga("rsp id %d", rsp->id); loga("Dumping content for request:"); msg_dump(req); loga("req id %d", req->id); } conn_dequeue_outq(ctx, peer_conn, req); req->done = 1; log_info("c_conn:%p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id, rsp->id, rsp->parent_id); /* establish rsp <-> req (response <-> request) link */ req->peer = rsp; rsp->peer = req; rsp->pre_coalesce(rsp); ASSERT_LOG((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT), "c_conn type %s", conn_get_type_string(c_conn)); dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp); // c_conn owns respnse now status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id, rsp); IGNORE_RET_VAL(status); if (req->swallow) { log_info("swallow request %d:%d", req->id, req->parent_id); req_put(req); } }
void remote_req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg, struct rack *rack, uint8_t *key, uint32_t keylen) { ASSERT((c_conn->type == CONN_CLIENT) || (c_conn->type == CONN_DNODE_PEER_CLIENT)); struct node * peer = dnode_peer_pool_server(ctx, c_conn->owner, rack, key, keylen, msg->msg_routing); if (peer->is_local) { log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d is local", c_conn, msg->id, msg->parent_id); local_req_forward(ctx, c_conn, msg, key, keylen); return; } /* enqueue message (request) into client outq, if response is expected */ if (msg->expect_datastore_reply && !msg->swallow) { conn_enqueue_outq(ctx, c_conn, msg); } // now get a peer connection struct conn *p_conn = dnode_peer_pool_server_conn(ctx, peer); if ((p_conn == NULL) || (p_conn->connecting)) { if (p_conn) { usec_t now = dn_usec_now(); static usec_t next_log = 0; // Log every 1 sec if (now > next_log) { log_warn("still connecting to peer '%.*s'......", peer->endpoint.pname.len, peer->endpoint.pname.data); next_log = now + 1000 * 1000; } } // No response for DC_ONE & swallow if ((msg->consistency == DC_ONE) && (msg->swallow)) { msg_put(msg); return; } // No response for remote dc struct server_pool *pool = c_conn->owner; bool same_dc = is_same_dc(pool, peer)? 1 : 0; if (!same_dc) { msg_put(msg); return; } // All other cases return a response struct msg *rsp = msg_get(c_conn, false, __FUNCTION__); msg->done = 1; rsp->error = msg->error = 1; rsp->err = msg->err = (p_conn ? PEER_HOST_NOT_CONNECTED : PEER_HOST_DOWN); rsp->dyn_error = msg->dyn_error = (p_conn ? PEER_HOST_NOT_CONNECTED: PEER_HOST_DOWN); rsp->dmsg = dmsg_get(); rsp->peer = msg; rsp->dmsg->id = msg->id; log_info("%lu:%lu <-> %lu:%lu Short circuit....", msg->id, msg->parent_id, rsp->id, rsp->parent_id); conn_handle_response(c_conn, msg->parent_id ? msg->parent_id : msg->id, rsp); if (msg->swallow) msg_put(msg); return; } log_debug(LOG_VERB, "c_conn: %p forwarding %d:%d to p_conn %p", c_conn, msg->id, msg->parent_id, p_conn); dnode_peer_req_forward(ctx, c_conn, p_conn, msg, rack, key, keylen); }