void
dnode_rsp_recv_done(struct context *ctx, struct conn *conn,
struct msg *msg, struct msg *nmsg)
{
log_debug(LOG_VERB, "dnode_rsp_recv_done entering ...");
ASSERT(!conn->dnode_client && !conn->dnode_server);
ASSERT(msg != NULL && conn->rmsg == msg);
ASSERT(!msg->request);
ASSERT(msg->owner == conn);
ASSERT(nmsg == NULL || !nmsg->request);
if (TRACING_LEVEL == LOG_VVERB) {
loga("Dumping content for msg: ");
msg_dump(msg);
if (nmsg != NULL) {
loga("Dumping content for nmsg :");
msg_dump(nmsg);
}
}
/* enqueue next message (response), if any */
conn->rmsg = nmsg;
if (dnode_rsp_filter(ctx, conn, msg)) {
return;
}
dnode_rsp_forward(ctx, conn, msg);
}
void
dnode_rsp_recv_done(struct context *ctx, struct conn *conn,
struct msg *msg, struct msg *nmsg)
{
log_debug(LOG_VERB, "dnode_rsp_recv_done entering ...");
ASSERT(conn->type == CONN_DNODE_PEER_SERVER);
ASSERT(msg != NULL && conn->rmsg == msg);
ASSERT(!msg->request);
ASSERT(msg->owner == conn);
ASSERT(nmsg == NULL || !nmsg->request);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for msg: ");
msg_dump(msg);
if (nmsg != NULL) {
loga("Dumping content for nmsg :");
msg_dump(nmsg);
}
}
/* enqueue next message (response), if any */
conn->rmsg = nmsg;
if (dnode_rsp_filter(ctx, conn, msg)) {
return;
}
dnode_rsp_forward(ctx, conn, msg);
}
static void dump(Msg *msg)
{
switch (verbose) {
case 0:
break;
case 1:
msg_dump(msg, 0);
break;
case 2:
wdp_event_dump(msg);
break;
case 3:
msg_dump(msg, 0);
wdp_event_dump(msg);
break;
case 4:
wtp_event_dump(msg);
break;
case 5:
msg_dump(msg, 0);
wtp_event_dump(msg);
break;
case 6:
wdp_event_dump(msg);
wtp_event_dump(msg);
break;
case 7:
msg_dump(msg, 0);
wdp_event_dump(msg);
wtp_event_dump(msg);
break;
}
}
void dyn_parse_rsp(struct msg *r)
{
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, ":::::::::::::::::::::: In dyn_parse_rsp, start to process response :::::::::::::::::::::::: ");
msg_dump(r);
}
if (dyn_parse_core(r)) {
struct dmsg *dmsg = r->dmsg;
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
if (dmsg->type != DMSG_UNKNOWN && dmsg->type != DMSG_RES) {
log_debug(LOG_DEBUG, "Resp parser: I got a dnode msg of type %d", dmsg->type);
r->state = 0;
r->result = MSG_PARSE_OK;
r->dyn_state = DYN_DONE;
return;
}
//check whether we need to decrypt the payload
if (dmsg->bit_field == 1) {
//dmsg->owner->owner->dnode_secured = 1;
struct mbuf *decrypted_buf = mbuf_get();
if (decrypted_buf == NULL) {
log_debug(LOG_INFO, "Unable to obtain an mbuf for dnode msg's header!");
r->result = MSG_OOM_ERROR;
return;
}
//Dont need to decrypt AES key - pull it out from the conn
dyn_aes_decrypt(dmsg->payload, dmsg->plen, decrypted_buf, r->owner->aes_key);
b->pos = b->pos + dmsg->plen;
r->pos = decrypted_buf->start;
mbuf_copy(decrypted_buf, b->pos, mbuf_length(b));
mbuf_insert(&r->mhdr, decrypted_buf);
mbuf_remove(&r->mhdr, b);
mbuf_put(b);
r->mlen = mbuf_length(decrypted_buf);
}
if (r->redis) {
return redis_parse_rsp(r);
}
return memcache_parse_rsp(r);
}
//bad case
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "Resp: bad message - cannot parse"); //fix me to do something
msg_dump(r);
}
r->result = MSG_PARSE_AGAIN;
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *msg)
{
rstatus_t status;
struct msg *pmsg;
struct conn *c_conn;
log_debug(LOG_VERB, "dnode_rsp_forward entering ...");
ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
pmsg = TAILQ_FIRST(&peer_conn->omsg_q);
ASSERT(pmsg != NULL && pmsg->peer == NULL);
ASSERT(pmsg->request && !pmsg->done);
if (TRACING_LEVEL >= LOG_VVERB) {
loga("Dumping content for msg: ");
msg_dump(msg);
loga("msg id %d", msg->id);
loga("Dumping content for pmsg :");
msg_dump(pmsg);
loga("pmsg id %d", pmsg->id);
}
peer_conn->dequeue_outq(ctx, peer_conn, pmsg);
pmsg->done = 1;
/* establish msg <-> pmsg (response <-> request) link */
pmsg->peer = msg;
msg->peer = pmsg;
msg->pre_coalesce(msg);
c_conn = pmsg->owner;
ASSERT(c_conn->client && !c_conn->proxy);
if (TAILQ_FIRST(&c_conn->omsg_q) != NULL && dnode_req_done(c_conn, TAILQ_FIRST(&c_conn->omsg_q))) {
status = event_add_out(ctx->evb, c_conn);
if (status != DN_OK) {
c_conn->err = errno;
}
}
dnode_rsp_forward_stats(ctx, peer_conn->owner, msg);
}
static int msg_recv_prio(struct conn *conn, int64_t priority)
{
struct kdbus_cmd_recv recv = {
.flags = KDBUS_RECV_USE_PRIORITY,
.priority = priority,
};
struct kdbus_msg *msg;
int ret;
ret = ioctl(conn->fd, KDBUS_CMD_MSG_RECV, &recv);
if (ret < 0) {
ret = -errno;
fprintf(stderr, "error receiving message: %d (%m)\n", ret);
return ret;
}
msg = (struct kdbus_msg *)(conn->buf + recv.offset);
msg_dump(conn, msg);
ret = ioctl(conn->fd, KDBUS_CMD_FREE, &recv.offset);
if (ret < 0) {
ret = -errno;
fprintf(stderr, "error free message: %d (%m)\n", ret);
return ret;
}
return 0;
}
/******************************************************************************
*
* EXTERNAL FUNCTIONS:
*
*/
WAPEvent *wtls_unpack_wdp_datagram(Msg * msg)
{
WAPEvent *unitdataIndEvent;
List *wtlsPayloadList;
/* Dump the Msg */
msg_dump(msg, 0);
/* Then, stuff it into a T_Unitdata_Ind Event */
unitdataIndEvent = wap_event_create(T_Unitdata_Ind);
info(0, "Event created");
/* Firstly, the address */
unitdataIndEvent->u.T_Unitdata_Ind.addr_tuple =
wap_addr_tuple_create(msg->wdp_datagram.source_address,
msg->wdp_datagram.source_port,
msg->wdp_datagram.destination_address,
msg->wdp_datagram.destination_port);
info(0, "Set address and stuff");
/* Attempt to stuff this baby into a list-of-WTLS-PDUs */
wtlsPayloadList = wtls_unpack_payloadlist(msg->wdp_datagram.user_data);
info(0, "Datagram unpacked!");
/* Then, the pdu material */
unitdataIndEvent->u.T_Unitdata_Ind.pdu_list = wtlsPayloadList;
/* And return the event */
return unitdataIndEvent;
}
/*
* Remove msg from sent queue.
* Return 0 if message should be deleted from store and 1 if not (e.g. tmp nack)
*/
static void boxc_sent_pop(Boxc *conn, Msg *m, Msg **orig)
{
Octstr *os;
char id[UUID_STR_LEN + 1];
Msg *msg;
if (conn->is_wap || !conn->sent || !m || (msg_type(m) != ack && msg_type(m) != sms))
return;
if (orig != NULL)
*orig = NULL;
uuid_unparse((msg_type(m) == sms ? m->sms.id : m->ack.id), id);
os = octstr_create(id);
msg = dict_remove(conn->sent, os);
octstr_destroy(os);
if (!msg) {
error(0, "BOXC: Got ack for nonexistend message!");
msg_dump(m, 0);
return;
}
semaphore_up(conn->pending);
if (orig == NULL)
msg_destroy(msg);
else
*orig = msg;
}
static void *transmit_start(void *arg)
{
int fd = (unsigned long)arg;
struct msg *msg;
ssize_t res;
while (1) {
msg = opt_master ? msg_gen(-opt_msglen) : msg_get();
if (opt_verbose)
msg_dump(msg);
res = write(fd, msg->buf, msg->len);
if (res < 0) {
pr_error("Write error %d\n", errno);
exit(-1);
}
if (res < msg->len) {
pr_error("Short write %zd < %u\n", res, msg->len);
exit(-1);
}
tx_bytes += res;
if (opt_master && opt_chain)
msg_add(msg);
else
free(msg);
}
return NULL;
}
static int msg_ntoh(struct nmrp_msg *msg)
{
struct nmrp_opt *opt = msg->opts;
int remaining;
remaining = msg->len - NMRP_HDR_LEN;
// FIXME maximum of two options supported, maximum option
// size is 12
if (remaining < NMRP_MAX_OPT_NUM * NMRP_MAX_OPT_SIZE) {
while (remaining > 0) {
if (remaining < NMRP_OPT_HDR_LEN) {
break;
}
opt->type = ntohs(opt->type);
opt->len = ntohs(opt->len);
if (opt->len > NMRP_MAX_OPT_SIZE) {
break;
}
remaining -= opt->len;
opt = NMRP_OPT_NEXT(opt);
}
if (!remaining) {
return 0;
}
}
fprintf(stderr, "Unexpected message format.\n");
msg_dump(msg, 0);
return 1;
}
int msg_recv(struct conn *conn)
{
struct kdbus_cmd_recv recv = {};
struct kdbus_msg *msg;
int ret;
ret = ioctl(conn->fd, KDBUS_CMD_MSG_RECV, &recv);
if (ret < 0) {
ret = -errno;
fprintf(stderr, "error receiving message: %d (%m)\n", ret);
return ret;
}
msg = (struct kdbus_msg *)(conn->buf + recv.offset);
msg_dump(conn, msg);
ret = ioctl(conn->fd, KDBUS_CMD_FREE, &recv.offset);
if (ret < 0) {
ret = -errno;
fprintf(stderr, "error free message: %d (%m)\n", ret);
return ret;
}
return 0;
}
/* dump msgs */
static void m_dump_msg(char *arg)
{
struct msg *mptr = NULL;
if(arg)
{
if(chars_isdigit(*arg))
mptr = msg_find_id(atoi(arg));
else
mptr = msg_find(arg);
}
msg_dump(mptr);
}
int msg_recv(struct conn *conn)
{
uint64_t off;
struct kdbus_msg *msg;
int ret;
ret = ioctl(conn->fd, KDBUS_CMD_MSG_RECV, &off);
if (ret < 0) {
fprintf(stderr, "error receiving message: %d (%m)\n", ret);
return EXIT_FAILURE;
}
msg = (struct kdbus_msg *)(conn->buf + off);
msg_dump(conn, msg);
ret = ioctl(conn->fd, KDBUS_CMD_FREE, &off);
if (ret < 0) {
fprintf(stderr, "error free message: %d (%m)\n", ret);
return EXIT_FAILURE;
}
return 0;
}
/* dnode sends a response back to a peer */
struct msg *
dnode_rsp_send_next(struct context *ctx, struct conn *conn)
{
if (TRACING_LEVEL == LOG_VVERB) {
log_debug(LOG_VVERB, "dnode_rsp_send_next entering");
}
ASSERT(conn->dnode_client && !conn->dnode_server);
struct msg *msg = rsp_send_next(ctx, conn);
if (msg != NULL && conn->dyn_mode) {
struct msg *pmsg = TAILQ_FIRST(&conn->omsg_q); //peer request's msg
//need to deal with multi-block later
uint64_t msg_id = pmsg->dmsg->id;
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for header!");
return NULL; //need to address error here properly
}
//TODOs: need to set the outcoming conn to be secured too if the incoming conn is secured
if (pmsg->owner->dnode_secured || conn->dnode_secured) {
if (TRACING_LEVEL == LOG_VVERB) {
log_debug(LOG_VVERB, "Encrypting response ...");
loga("AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
}
struct mbuf *data_buf = STAILQ_LAST(&msg->mhdr, mbuf, next);
//if (ENCRYPTION) {
struct mbuf *encrypted_buf = mbuf_get();
if (encrypted_buf == NULL) {
loga("Unable to obtain an mbuf for encryption!");
return NULL; //TODOs: need to clean up
}
rstatus_t status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf),
encrypted_buf, conn->aes_key);
if (TRACING_LEVEL == LOG_VVERB) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg_id, DMSG_RES, conn, mbuf_length(encrypted_buf));
if (TRACING_LEVEL == LOG_VVERB) {
log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "resp dyn message - original payload: ");
log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: ");
}
mbuf_copy(header_buf, encrypted_buf->start, mbuf_length(encrypted_buf));
mbuf_insert(&msg->mhdr, header_buf);
//remove the original dbuf out of the queue and insert encrypted mbuf to replace
mbuf_remove(&msg->mhdr, data_buf);
//mbuf_insert(&msg->mhdr, encrypted_buf);
mbuf_put(data_buf);
mbuf_put(encrypted_buf);
//} else {
// log_debug(LOG_VERB, "no encryption on the response's payload");
// dmsg_write(header_buf, msg_id, DMSG_RES, conn, mbuf_length(data_buf));
//}
} else {
dmsg_write(header_buf, msg_id, DMSG_RES, conn, 0);//Dont care about 0 or the real length as we don't use that value in unencryption mode
mbuf_insert_head(&msg->mhdr, header_buf);
}
if (TRACING_LEVEL == LOG_VVERB) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "resp dyn message - header: ");
msg_dump(msg);
}
}
return msg;
}
/* Forward a client request over to a peer */
void dnode_peer_req_forward(struct context *ctx, struct conn *c_conn,
struct conn *p_conn, struct msg *msg,
struct rack *rack, uint8_t *key, uint32_t keylen)
{
struct server *server = p_conn->owner;
log_debug(LOG_DEBUG, "forwarding request from client conn '%s' to peer conn '%s' on rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(c_conn->sd), dn_unresolve_peer_desc(p_conn->sd),
rack->name->len, rack->name->data,
server->dc.len, server->dc.data);
struct string *dc = rack->dc;
rstatus_t status;
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply && !msg->swallow) {
conn_enqueue_outq(ctx, c_conn, msg);
}
ASSERT(p_conn->type == CONN_DNODE_PEER_SERVER);
ASSERT((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT));
/* enqueue the message (request) into peer inq */
status = event_add_out(ctx->evb, p_conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, p_conn, msg);
p_conn->err = errno;
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
req_put(msg);
return;
}
struct server_pool *pool = c_conn->owner;
dmsg_type_t msg_type = (string_compare(&pool->dc, dc) != 0)? DMSG_REQ_FORWARD : DMSG_REQ;
if (p_conn->dnode_secured) {
//Encrypting and adding header for a request
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN));
}
//write dnode header
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(msg, p_conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(msg);
return;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the msg payload");
}
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
} else {
//write dnode header
dmsg_write(header_buf, msg->id, msg_type, p_conn, msg_length(msg));
}
mbuf_insert_head(&msg->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: ");
msg_dump(msg);
}
conn_enqueue_inq(ctx, p_conn, msg);
dnode_peer_req_forward_stats(ctx, p_conn->owner, msg);
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
struct msg*
rspmgr_get_response(struct response_mgr *rspmgr)
{
// no quorum possible
if (rspmgr->good_responses < rspmgr->quorum_responses) {
ASSERT(rspmgr->err_rsp);
rspmgr_incr_non_quorum_responses_stats(rspmgr);
log_error("req: %lu return non quorum error rsp %p good rsp:%u quorum: %u",
rspmgr->msg->id, rspmgr->err_rsp, rspmgr->good_responses,
rspmgr->quorum_responses);
if (log_loggable(LOG_INFO))
msg_dump(rspmgr->err_rsp);
return rspmgr->err_rsp;
}
ASSERT_LOG(rspmgr->good_responses == 3, "rspmgr req: %lu has %d good responses",
rspmgr->msg->id, rspmgr->good_responses);
uint32_t chk0, chk1, chk2;
chk0 = rspmgr->checksums[0];
chk1 = rspmgr->checksums[1];
if (chk0 == chk1) {
return rspmgr->responses[0];
} else if (rspmgr->good_responses == 3) {
chk2 = rspmgr->checksums[2];
if (chk1 == chk2)
return rspmgr->responses[1];
else if (chk0 == chk2)
return rspmgr->responses[0];
}
rspmgr_incr_non_quorum_responses_stats(rspmgr);
if (log_loggable(LOG_DEBUG)) {
log_error("Request: ");
msg_dump(rspmgr->msg);
}
if (log_loggable(LOG_VVERB)) {
log_error("Respone 0: ");
msg_dump(rspmgr->responses[0]);
log_error("Respone 1: ");
msg_dump(rspmgr->responses[1]);
if (rspmgr->good_responses == 3) {
log_error("Respone 2: ");
msg_dump(rspmgr->responses[2]);
}
}
if (rspmgr->msg->consistency == DC_QUORUM) {
log_info("none of the responses match, returning first");
return rspmgr->responses[0];
} else {
log_info("none of the responses match, returning error");
struct msg *rsp = msg_get(rspmgr->conn, false, rspmgr->conn->data_store,
__FUNCTION__);
rsp->error = 1;
rsp->err = NO_QUORUM_ACHIEVED;
rsp->dyn_error = NO_QUORUM_ACHIEVED;
ASSERT(rspmgr->err_rsp == NULL);
rspmgr->err_rsp = rsp;
rspmgr->error_responses++;
return rsp;
}
}
void dyn_parse_rsp(struct msg *r)
{
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, ":::::::::::::::::::::: In dyn_parse_rsp, start to process response :::::::::::::::::::::::: ");
msg_dump(r);
}
bool done_parsing = false;
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
if (dyn_parse_core(r)) {
struct dmsg *dmsg = r->dmsg;
struct conn *conn = r->owner;
conn->same_dc = dmsg->same_dc;
if (dmsg->type != DMSG_UNKNOWN && dmsg->type != DMSG_RES) {
log_debug(LOG_DEBUG, "Resp parser: I got a dnode msg of type %d", dmsg->type);
r->state = 0;
r->result = MSG_PARSE_OK;
r->dyn_state = DYN_DONE;
return;
}
if (r->dyn_state == DYN_DONE && dmsg->bit_field == 1) {
dmsg->owner->owner->dnode_secured = 1;
r->owner->dnode_crypto_state = 1;
r->dyn_state = DYN_POST_DONE;
r->result = MSG_PARSE_REPAIR;
if (dmsg->mlen > 1) {
//Decrypt AES key
dyn_rsa_decrypt(dmsg->data, aes_decrypted_buf);
strncpy(r->owner->aes_key, aes_decrypted_buf, strlen(aes_decrypted_buf));
}
if (dmsg->plen + b->pos <= b->last) {
struct mbuf *decrypted_buf = mbuf_get();
if (decrypted_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
r->result = MSG_OOM_ERROR;
return;
}
dyn_aes_decrypt(b->pos, dmsg->plen, decrypted_buf, r->owner->aes_key);
b->pos = b->pos + dmsg->plen;
r->pos = decrypted_buf->start;
mbuf_copy(decrypted_buf, b->pos, mbuf_length(b));
mbuf_insert(&r->mhdr, decrypted_buf);
mbuf_remove(&r->mhdr, b);
mbuf_put(b);
r->mlen = mbuf_length(decrypted_buf);
return data_store_parse_rsp(r);
}
//Subtract already received bytes
dmsg->plen -= b->last - b->pos;
return;
} else if (r->dyn_state == DYN_POST_DONE) {
struct mbuf *last_buf = STAILQ_LAST(&r->mhdr, mbuf, next);
if (last_buf->read_flip == 1) {
data_store_parse_rsp(r);
} else {
r->result = MSG_PARSE_AGAIN;
}
return;
}
if (done_parsing)
return;
return data_store_parse_rsp(r);
}
//bad case
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "Resp: bad message - cannot parse"); //fix me to do something
msg_dump(r);
}
r->result = MSG_PARSE_AGAIN;
}
void
dyn_parse_req(struct msg *r)
{
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, ":::::::::::::::::::::: In dyn_parse_req, start to process request :::::::::::::::::::::: ");
msg_dump(r);
}
bool done_parsing = false;
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
if (dyn_parse_core(r)) {
struct dmsg *dmsg = r->dmsg;
struct conn *conn = r->owner;
conn->same_dc = dmsg->same_dc;
if (dmsg->type != DMSG_UNKNOWN && dmsg->type != DMSG_REQ &&
dmsg->type != DMSG_REQ_FORWARD && dmsg->type != GOSSIP_SYN) {
r->state = 0;
r->result = MSG_PARSE_OK;
r->dyn_state = DYN_DONE;
return;
}
if (r->dyn_state == DYN_DONE && dmsg->bit_field == 1) {
dmsg->owner->owner->dnode_secured = 1;
r->owner->dnode_crypto_state = 1;
r->dyn_state = DYN_POST_DONE;
r->result = MSG_PARSE_REPAIR;
if (dmsg->mlen > 1) {
//Decrypt AES key
dyn_rsa_decrypt(dmsg->data, aes_decrypted_buf);
strncpy(r->owner->aes_key, aes_decrypted_buf, strlen(aes_decrypted_buf));
}
if (dmsg->plen + b->pos <= b->last) {
struct mbuf *decrypted_buf = mbuf_get();
if (decrypted_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
r->result = MSG_OOM_ERROR;
return;
}
dyn_aes_decrypt(b->pos, dmsg->plen, decrypted_buf, r->owner->aes_key);
b->pos = b->pos + dmsg->plen;
r->pos = decrypted_buf->start;
mbuf_copy(decrypted_buf, b->pos, mbuf_length(b));
mbuf_insert(&r->mhdr, decrypted_buf);
mbuf_remove(&r->mhdr, b);
mbuf_put(b);
r->mlen = mbuf_length(decrypted_buf);
data_store_parse_req(r);
}
//substract alraedy received bytes
dmsg->plen -= b->last - b->pos;
return;
} else if (r->dyn_state == DYN_POST_DONE) {
struct mbuf *last_buf = STAILQ_LAST(&r->mhdr, mbuf, next);
if (last_buf->read_flip == 1) {
data_store_parse_req(r);
} else {
r->result = MSG_PARSE_AGAIN;
}
return;
}
if (dmsg->type == GOSSIP_SYN) {
//TODOs: need to address multi-buffer msg later
dmsg->payload = b->pos;
b->pos = b->pos + dmsg->plen;
r->pos = b->pos;
done_parsing = true;
}
if (done_parsing)
return;
return data_store_parse_req(r);
}
//bad case
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "Bad or splitted message"); //fix me to do something
msg_dump(r);
}
r->result = MSG_PARSE_AGAIN;
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %s %d dmsg->id %u req %u:%u rsp %u:%u, ",
conn_get_type_string(peer_conn),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
conn_dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_info("c_conn:%p %d:%d <-> %d:%d", c_conn, req->id, req->parent_id,
rsp->id, rsp->parent_id);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT_LOG((c_conn->type == CONN_CLIENT) ||
(c_conn->type == CONN_DNODE_PEER_CLIENT),
"c_conn type %s", conn_get_type_string(c_conn));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
// c_conn owns respnse now
status = conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_info("swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
/* dnode sends a response back to a peer */
struct msg *
dnode_rsp_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
ASSERT(conn->dnode_client && !conn->dnode_server);
struct msg *rsp = rsp_send_next(ctx, conn);
if (rsp != NULL && conn->dyn_mode) {
struct msg *pmsg = rsp->peer;
//need to deal with multi-block later
uint64_t msg_id = pmsg->dmsg->id;
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for header!");
return NULL; //need to address error here properly
}
dmsg_type_t msg_type = DMSG_RES;
//TODOs: need to set the outcoming conn to be secured too if the incoming conn is secured
if (pmsg->owner->dnode_secured || conn->dnode_secured) {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "Encrypting response ...");
loga("AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
}
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(rsp, conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(rsp);
return NULL;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the rsp payload");
}
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
}
} else {
//write dnode header
log_info("sending dnode response with msg_id %u", msg_id);
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
}
mbuf_insert_head(&rsp->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "resp dyn message - header: ");
msg_dump(rsp);
}
}
return rsp;
}
/* Forward a client request over to a peer */
void
dnode_peer_req_forward(struct context *ctx, struct conn *c_conn, struct conn *p_conn,
struct msg *msg, struct rack *rack, uint8_t *key, uint32_t keylen)
{
if (TRACING_LEVEL == LOG_VVERB) {
log_debug(LOG_VVERB, "dnode_peer_req_forward entering");
}
rstatus_t status;
/* enqueue message (request) into client outq, if response is expected */
if (!msg->noreply) {
c_conn->enqueue_outq(ctx, c_conn, msg);
}
ASSERT(!p_conn->dnode_client && !p_conn->dnode_server);
ASSERT(c_conn->client);
/* enqueue the message (request) into peer inq */
if (TAILQ_EMPTY(&p_conn->imsg_q)) {
status = event_add_out(ctx->evb, p_conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, p_conn, msg);
p_conn->err = errno;
return;
}
}
uint64_t msg_id = peer_msg_id++;
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
return;
}
if (p_conn->dnode_secured) {
//Encrypting and adding header for a request
struct mbuf *data_buf = STAILQ_LAST(&msg->mhdr, mbuf, next);
//TODOs: need to deal with multi-block later
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(p_conn->aes_key, AES_KEYLEN));
struct mbuf *encrypted_buf = mbuf_get();
if (encrypted_buf == NULL) {
loga("Unable to obtain an mbuf for encryption!");
return; //TODOs: need to clean up
}
status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, p_conn->aes_key);
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
//write dnode header
dmsg_write(header_buf, msg_id, DMSG_REQ, p_conn, mbuf_length(encrypted_buf));
mbuf_insert_head(&msg->mhdr, header_buf);
log_hexdump(LOG_VERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: ");
log_hexdump(LOG_VERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: ");
//remove the original dbuf out of the queue and insert encrypted mbuf to replace
mbuf_remove(&msg->mhdr, data_buf);
mbuf_insert(&msg->mhdr, encrypted_buf);
//free it as no one will need it again
mbuf_put(data_buf);
} else {
//write dnode header
dmsg_write(header_buf, msg_id, DMSG_REQ, p_conn, 0);
mbuf_insert_head(&msg->mhdr, header_buf);
}
if (TRACING_LEVEL == LOG_VVERB) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn message header: ");
msg_dump(msg);
}
p_conn->enqueue_inq(ctx, p_conn, msg);
dnode_peer_req_forward_stats(ctx, p_conn->owner, msg);
if (TRACING_LEVEL == LOG_VERB) {
log_debug(LOG_VERB, "remote forward from c %d to s %d req %"PRIu64" len %"PRIu32
" type %d with key '%.*s'", c_conn->sd, p_conn->sd, msg->id,
msg->mlen, msg->type, keylen, key);
}
}
static int store_file_load(void(*receive_msg)(Msg*))
{
List *keys;
Octstr *store_file, *key;
Msg *msg;
int retval, msgs;
long end, pos;
if (filename == NULL)
return 0;
mutex_lock(file_mutex);
if (file != NULL) {
fclose(file);
file = NULL;
}
store_file = octstr_read_file(octstr_get_cstr(filename));
if (store_file != NULL)
info(0, "Loading store file `%s'", octstr_get_cstr(filename));
else {
store_file = octstr_read_file(octstr_get_cstr(newfile));
if (store_file != NULL)
info(0, "Loading store file `%s'", octstr_get_cstr(newfile));
else {
store_file = octstr_read_file(octstr_get_cstr(bakfile));
if (store_file != NULL)
info(0, "Loading store file `%s'", octstr_get_cstr(bakfile));
else {
info(0, "Cannot open any store file, starting a new one");
retval = open_file(filename);
goto end;
}
}
}
info(0, "Store-file size %ld, starting to unpack%s", octstr_len(store_file),
octstr_len(store_file) > 10000 ? " (may take awhile)" : "");
pos = 0;
msgs = 0;
end = octstr_len(store_file);
while (pos < end) {
if (read_msg(&msg, store_file, &pos) == -1) {
error(0, "Garbage at store-file, skipped.");
continue;
}
if (msg_type(msg) == sms) {
store_to_dict(msg);
msgs++;
} else if (msg_type(msg) == ack) {
store_to_dict(msg);
} else {
warning(0, "Strange message in store-file, discarded, "
"dump follows:");
msg_dump(msg, 0);
}
msg_destroy(msg);
}
octstr_destroy(store_file);
info(0, "Retrieved %d messages, non-acknowledged messages: %ld",
msgs, dict_key_count(sms_dict));
/* now create a new sms_store out of messages left */
keys = dict_keys(sms_dict);
while ((key = gwlist_extract_first(keys)) != NULL) {
msg = dict_remove(sms_dict, key);
if (store_to_dict(msg) != -1) {
receive_msg(msg);
} else {
error(0, "Found unknown message type in store file.");
msg_dump(msg, 0);
msg_destroy(msg);
}
octstr_destroy(key);
}
gwlist_destroy(keys, octstr_destroy_item);
/* Finally, generate new store file out of left messages */
retval = do_dump();
end:
mutex_unlock(file_mutex);
/* allow using of store */
gwlist_remove_producer(loaded);
/* start dumper thread */
if ((cleanup_thread = gwthread_create(store_dumper, NULL))==-1)
panic(0, "Failed to create a cleanup thread!");
return retval;
}
// Incoming messages start here.
// - Could get a request that we need to service.
// - Could get a response to one of our requests - need to find the request and
// send the real response to the remote end.
int
proxy_msg_fn(cf_node id, msg *m, void *udata)
{
int rv;
if (cf_rc_count((void*)m) == 0) {
cf_debug(AS_PROXY, " proxy_msg_fn was given a refcount 0 message! Someone has been naugty %p", m);
return -1;
}
uint32_t op = 99999;
msg_get_uint32(m, PROXY_FIELD_OP, &op);
uint32_t transaction_id = 0;
msg_get_uint32(m, PROXY_FIELD_TID, &transaction_id);
cf_detail(AS_PROXY, "received proxy message: tid %d type %d from %"PRIx64, transaction_id, op, id);
switch (op) {
case PROXY_OP_REQUEST:
{
cf_atomic_int_incr(&g_config.proxy_action);
#ifdef DEBUG
cf_debug(AS_PROXY, "Proxy_msg: received request");
#ifdef DEBUG_VERBOSE
msg_dump(m, "incoming proxy msg");
#endif
#endif
cf_digest *key;
size_t sz = 0;
if (0 != msg_get_buf(m, PROXY_FIELD_DIGEST, (byte **) &key, &sz, MSG_GET_DIRECT)) {
cf_info(AS_PROXY, "proxy msg function: no digest, problem");
as_fabric_msg_put(m);
return 0;
}
cl_msg *msgp;
size_t as_msg_sz = 0;
if (0 != msg_get_buf(m, PROXY_FIELD_AS_PROTO, (byte **) &msgp, &as_msg_sz, MSG_GET_COPY_MALLOC)) {
cf_info(AS_PROXY, "proxy msg function: no as msg, problem");
as_fabric_msg_put(m);
return 0;
}
uint64_t cluster_key = 0;
if (0 != msg_get_uint64(m, PROXY_FIELD_CLUSTER_KEY, &cluster_key)) {
cf_info(AS_PROXY, "proxy msg function: no cluster key, problem");
as_fabric_msg_put(m);
return 0;
}
// This is allowed to fail - this is a new field, and gets defaulted
// to 0 if it doesn't exist.
uint32_t timeout_ms = 0;
msg_get_uint32(m, PROXY_FIELD_TIMEOUT_MS, &timeout_ms);
// cf_info(AS_PROXY, "proxy msg: received timeout_ms of %d",timeout_ms);
// Put the as_msg on the normal queue for processing.
// INIT_TR
as_transaction tr;
as_transaction_init(&tr, key, msgp);
tr.incoming_cluster_key = cluster_key;
tr.end_time = (timeout_ms != 0) ? ((uint64_t)timeout_ms * 1000000) + tr.start_time : 0;
tr.proxy_node = id;
tr.proxy_msg = m;
// Check here if this is shipped op.
uint32_t info = 0;
msg_get_uint32(m, PROXY_FIELD_INFO, &info);
if (info & PROXY_INFO_SHIPPED_OP) {
tr.flag |= AS_TRANSACTION_FLAG_SHIPPED_OP;
cf_detail_digest(AS_PROXY, &tr.keyd, "SHIPPED_OP WINNER Operation Received");
} else {
cf_detail_digest(AS_PROXY, &tr.keyd, "Received Proxy Request digest tid(%d)", tr.trid);
}
MICROBENCHMARK_RESET();
thr_tsvc_enqueue(&tr);
}
break;
case PROXY_OP_RESPONSE:
{
#ifdef DEBUG
// Got the response from the actual endpoint.
cf_debug(AS_PROXY, " proxy: received response! tid %d node %"PRIx64, transaction_id, id);
#ifdef DEBUG_VERBOSE
msg_dump(m, "incoming proxy response");
#endif
#endif
// Look up the element.
proxy_request pr;
bool free_msg = true;
if (SHASH_OK == shash_get_and_delete(g_proxy_hash, &transaction_id, &pr)) {
// Found the element (sometimes we get two acks so it's OK for
// an ack to not find the transaction).
if (pr.wr) {
as_proxy_shipop_response_hdlr(m, &pr, &free_msg);
} else {
as_proto *proto;
size_t proto_sz;
if (0 != msg_get_buf(m, PROXY_FIELD_AS_PROTO, (byte **) &proto, &proto_sz, MSG_GET_DIRECT)) {
cf_info(AS_PROXY, "msg get buf failed!");
}
#ifdef DEBUG_VERBOSE
cf_debug(AS_PROXY, "proxy: sending proto response: ptr %p sz %"PRIu64" %d", proto, proto_sz, pr.fd);
for (size_t _i = 0; _i < proto_sz; _i++) {
fprintf(stderr, " %x", ((byte *)proto)[_i]);
if (_i % 16 == 15) {
fprintf(stderr, "\n");
}
}
#endif
#ifdef EXTRA_CHECKS
as_proto proto_copy = *proto;
as_proto_swap(&proto_copy);
if (proto_copy.sz + 8 != proto_sz) {
cf_info(AS_PROXY, "BONE BONE BONE!!!");
cf_info(AS_PROXY, "proto sz: %"PRIu64" sz %u", (uint64_t) proto_copy.sz, proto_sz);
}
#endif
// Write to the file descriptor.
cf_detail(AS_PROXY, "direct write fd %d", pr.fd_h->fd);
cf_assert(pr.fd_h->fd, AS_PROXY, CF_WARNING, "attempted write to fd 0");
if (pr.batch_shared) {
cf_digest* digest;
size_t digest_sz = 0;
if (msg_get_buf(pr.fab_msg, PROXY_FIELD_DIGEST, (byte **)&digest, &digest_sz, MSG_GET_DIRECT) == 0) {
as_batch_add_proxy_result(pr.batch_shared, pr.batch_index, digest, (cl_msg*)proto, proto_sz);
as_proxy_set_stat_counters(0);
}
else {
cf_warning(AS_PROXY, "Failed to find batch proxy digest %u", transaction_id);
as_batch_add_error(pr.batch_shared, pr.batch_index, AS_PROTO_RESULT_FAIL_UNKNOWN);
as_proxy_set_stat_counters(-1);
}
cf_hist_track_insert_data_point(g_config.px_hist, pr.start_time);
}
else {
size_t pos = 0;
while (pos < proto_sz) {
rv = send(pr.fd_h->fd, (((uint8_t *)proto) + pos), proto_sz - pos, MSG_NOSIGNAL);
if (rv > 0) {
pos += rv;
}
else if (rv < 0) {
if (errno != EWOULDBLOCK) {
// Common message when a client aborts.
cf_debug(AS_PROTO, "protocol proxy write fail: fd %d sz %d pos %d rv %d errno %d", pr.fd_h->fd, proto_sz, pos, rv, errno);
shutdown(pr.fd_h->fd, SHUT_RDWR);
as_proxy_set_stat_counters(-1);
goto SendFin;
}
usleep(1); // yield
}
else {
cf_info(AS_PROTO, "protocol write fail zero return: fd %d sz %d pos %d ", pr.fd_h->fd, proto_sz, pos);
shutdown(pr.fd_h->fd, SHUT_RDWR);
as_proxy_set_stat_counters(-1);
goto SendFin;
}
}
as_proxy_set_stat_counters(0);
SendFin:
cf_hist_track_insert_data_point(g_config.px_hist, pr.start_time);
// Return the fabric message or the direct file descriptor -
// after write and complete.
pr.fd_h->t_inprogress = false;
AS_RELEASE_FILE_HANDLE(pr.fd_h);
pr.fd_h = 0;
}
as_fabric_msg_put(pr.fab_msg);
pr.fab_msg = 0;
}
}
else {
cf_debug(AS_PROXY, "proxy: received result but no transaction, tid %d", transaction_id);
as_proxy_set_stat_counters(-1);
}
if (free_msg) {
as_fabric_msg_put(m);
}
}
break;
case PROXY_OP_REDIRECT:
{
// Sometimes the destination we proxied a request to isn't able to
// satisfy it (for example, their copy of the partition in question
// might be desync).
cf_node new_dst = 0;
msg_get_uint64(m, PROXY_FIELD_REDIRECT, &new_dst);
cf_detail(AS_PROXY, "proxy redirect message: transaction %d to node %"PRIx64, transaction_id, new_dst);
// Look in the proxy retransmit hash for the tid.
proxy_request *pr;
pthread_mutex_t *pr_lock;
int r = 0;
if (0 != (r = shash_get_vlock(g_proxy_hash, &transaction_id, (void **)&pr, &pr_lock))) {
cf_debug(AS_PROXY, "redirect: could not find transaction %d", transaction_id);
as_fabric_msg_put(m);
return -1;
}
if (g_config.self_node == new_dst) {
// Although we don't know we're the final destination, undo the
// proxy-nature and put back on the main queue. Dangerous, as it
// leaves open the possibility of a looping message.
cf_digest *key;
size_t sz = 0;
if (0 != msg_get_buf(pr->fab_msg, PROXY_FIELD_DIGEST, (byte **) &key, &sz, MSG_GET_DIRECT)) {
cf_warning(AS_PROXY, "op_redirect: proxy msg function: no digest, problem");
pthread_mutex_unlock(pr_lock);
as_fabric_msg_put(m);
return -1;
}
cl_msg *msgp;
sz = 0;
if (0 != msg_get_buf(pr->fab_msg, PROXY_FIELD_AS_PROTO, (byte **) &msgp, &sz, MSG_GET_COPY_MALLOC)) {
cf_warning(AS_PROXY, "op_redirect: proxy msg function: no as proto, problem");
pthread_mutex_unlock(pr_lock);
as_fabric_msg_put(m);
return -1;
}
// Put the as_msg on the normal queue for processing.
// INIT_TR
as_transaction tr;
as_transaction_init(&tr, key, msgp);
tr.start_time = pr->start_time; // start time
tr.end_time = pr->end_time;
tr.proto_fd_h = pr->fd_h;
tr.batch_shared = pr->batch_shared;
tr.batch_index = pr->batch_index;
MICROBENCHMARK_RESET();
thr_tsvc_enqueue(&tr);
as_fabric_msg_put(pr->fab_msg);
shash_delete_lockfree(g_proxy_hash, &transaction_id);
}
else {
// Change the destination, update the retransmit time.
pr->dest = new_dst;
pr->xmit_ms = cf_getms() + 1;
// Send it.
msg_incr_ref(pr->fab_msg);
if (0 != (rv = as_fabric_send(pr->dest, pr->fab_msg, AS_FABRIC_PRIORITY_MEDIUM))) {
cf_debug(AS_PROXY, "redirect: change destination: %"PRIx64" send error %d", pr->dest, rv);
as_fabric_msg_put(pr->fab_msg);
}
}
pthread_mutex_unlock(pr_lock);
}
as_fabric_msg_put(m);
break;
default:
cf_debug(AS_PROXY, "proxy_msg_fn: received unknown, unsupported message %d from remote endpoint", op);
msg_dump(m, "proxy received unknown msg");
as_fabric_msg_put(m);
break;
} // end switch
return 0;
} // end proxy_msg_fn()
void dyn_parse_rsp(struct msg *r)
{
#ifdef DN_DEBUG_LOG
log_debug(LOG_VERB, "In dyn_parse_rsp, start to process response :::::::::::::::::::::::: ");
msg_dump(r);
#endif
if (dyn_parse_core(r)) {
struct dmsg *dmsg = r->dmsg;
if (dmsg->type != DMSG_UNKNOWN && dmsg->type != DMSG_RES) {
log_debug(LOG_DEBUG, "Resp parser: I got a dnode msg of type %d", dmsg->type);
r->state = 0;
r->result = MSG_PARSE_OK;
r->dyn_state = DYN_DONE;
return;
}
//check whether we need to decrypt the payload
if (dmsg->bit_field == 1) {
//dmsg->owner->owner->dnode_secured = 1;
struct mbuf *decrypted_buf = mbuf_get();
if (decrypted_buf == NULL) {
log_debug(LOG_INFO, "Unable to obtain an mbuf for dnode msg's header!");
return;
}
#ifdef DN_DEBUG_LOG
log_debug(LOG_VERB, "encrypted aes key length : %d", dmsg->mlen);
loga("AES encryption key from conn: %s\n", base64_encode(r->owner->aes_key, AES_KEYLEN));
#endif
//Dont need to decrypt AES key - pull it out from the conn
dyn_aes_decrypt(dmsg->payload, dmsg->plen, decrypted_buf, r->owner->aes_key);
#ifdef DN_DEBUG_LOG
log_hexdump(LOG_VERB, decrypted_buf->pos, mbuf_length(decrypted_buf), "dyn message decrypted payload: ");
#endif
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
b->last = b->pos;
r->pos = decrypted_buf->start;
mbuf_insert(&r->mhdr, decrypted_buf);
}
if (r->redis)
return redis_parse_rsp(r);
return memcache_parse_rsp(r);
}
#ifdef DN_DEBUG_LOG
//bad case
log_debug(LOG_DEBUG, "Bad message - cannot parse"); //fix me to do something
msg_dump(r);
#endif
//r->state = 0;
//r->result = MSG_PARSE_OK;
}
/* dnode sends a response back to a peer */
static struct msg *
dnode_rsp_send_next(struct context *ctx, struct conn *conn)
{
rstatus_t status;
// SMB: There is some non trivial thing happening here. And I think it is very
// important to read this before anything is changed in here. There is also a
// bug that exists which I will mention briefly:
// A message is a structure that has a list of mbufs which hold the actual data.
// Each mbuf has start, pos, last as pointers (amongst others) which indicate start of the
// buffer, current read position and end of the buffer respectively.
//
// Every time a message is sent to a peer within dynomite, a DNODE header is
// prepended which is created using dmsg_write. A message remembers this case
// in dnode_header_prepended, so that if the messsage is sent in parts, the
// header is not prepended again for the subsequent parts.
//
// Like I said earlier there is a pos pointer in mbuf. If a message is sent
// partially (or it is parsed partially too I think) the pos reflects that
// case such that things can be resumed where it left off.
//
// dmsg_write has a parameter which reflects the payload length following the
// dnode header calculated by msg_length. msg_length is a summation of all
// mbuf sizes (last - start). Which I think is wrong.
//
// +------------+ +---------------+
// | DC1N1 +---------> | DC2N1 |
// +------------+ +-------+-------+
// |
// |
// |
// |
// +-------v-------+
// | DC2N2 |
// +---------------+
//
// Consider the case where
// a node DC1N1 in region DC1 sends a request to DC2N1 which forwards it to
// to local token owner DC2N2. Now DC2N1 receives a response from DC2N2 which
// has to be relayed back to DC1N1. This response from DC2N2 already has a
// dnode header but for the link between DC2N1 and DC2N2. DC2N1 should strip
// this header and prepend its own header for sending it back to DC1N1. This
// gets handled in encryption case since we overwrite all mbufs in the response
// However if the encryption is off, the message length sent to dmsg_write
// consists of the header from DC2N2 also which is wrong. So this relaying
// of responses will not work for the case where encryption is disabled.
//
// So msg_length should really be from mbuf->pos and not mbuf->start. This
// is a problem only with remote region replication since that is the only
// case where we CAN have 2 hops to send the request/response. This is also
// not a problem if encryption is ON.
ASSERT(conn->type == CONN_DNODE_PEER_CLIENT);
struct msg *rsp = rsp_send_next(ctx, conn);
if (rsp != NULL && conn->dyn_mode) {
struct msg *pmsg = rsp->peer;
//need to deal with multi-block later
uint64_t msg_id = pmsg->dmsg->id;
if (rsp->dnode_header_prepended) {
return rsp;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
loga("Unable to obtain an mbuf for header!");
return NULL; //need to address error here properly
}
dmsg_type_t msg_type = DMSG_RES;
//TODOs: need to set the outcoming conn to be secured too if the incoming conn is secured
if (pmsg->owner->dnode_secured || conn->dnode_secured) {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VVERB, "Encrypting response ...");
loga("AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
}
if (ENCRYPTION) {
status = dyn_aes_encrypt_msg(rsp, conn->aes_key);
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(rsp);
return NULL;
}
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
}
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
} else {
if (log_loggable(LOG_VVERB)) {
log_debug(LOG_VERB, "no encryption on the rsp payload");
}
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
}
} else {
//write dnode header
log_debug(LOG_VERB, "sending dnode response with msg_id %u", msg_id);
dmsg_write(header_buf, msg_id, msg_type, conn, msg_length(rsp));
}
rsp->dnode_header_prepended = 1;
mbuf_insert_head(&rsp->mhdr, header_buf);
if (log_loggable(LOG_VVERB)) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "resp dyn message - header: ");
msg_dump(rsp);
}
}
return rsp;
}
/* Description: link data from a peer connection to a client-facing connection
* peer_conn: a peer connection
* msg : msg with data from the peer connection after parsing
*/
static void
dnode_rsp_forward_match(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
req = TAILQ_FIRST(&peer_conn->omsg_q);
c_conn = req->owner;
/* if client consistency is dc_one forward the response from only the
local node. Since dyn_dnode_peer is always a remote node, drop the rsp */
if (req->consistency == DC_ONE) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
/* if client consistency is dc_quorum, forward the response from only the
local region/DC. */
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
dnode_rsp_swallow(ctx, peer_conn, req, rsp);
return;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_debug(LOG_DEBUG, "DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, ",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("Dumping content for response: ");
msg_dump(rsp);
loga("rsp id %d", rsp->id);
loga("Dumping content for request:");
msg_dump(req);
loga("req id %d", req->id);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
log_debug(LOG_VERB, "%p <-> %p", req, rsp);
/* establish rsp <-> req (response <-> request) link */
req->peer = rsp;
rsp->peer = req;
rsp->pre_coalesce(rsp);
ASSERT((c_conn->client && !c_conn->proxy) || (c_conn->dnode_client && !c_conn->dnode_server));
dnode_rsp_forward_stats(ctx, peer_conn->owner, rsp);
if (TAILQ_FIRST(&c_conn->omsg_q) != NULL && dnode_req_done(c_conn, req)) {
log_debug(LOG_INFO, "handle rsp %d:%d for req %d:%d conn %p",
rsp->id, rsp->parent_id, req->id, req->parent_id, c_conn);
// c_conn owns respnse now
rstatus_t status = conn_handle_response(c_conn,
req->parent_id ? req->parent_id : req->id,
rsp);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
int nmrp_do(struct nmrpd_args *args)
{
struct nmrp_pkt tx, rx;
uint8_t *src, dest[6];
uint16_t len, region;
char *filename;
time_t beg;
int i, status, ulreqs, expect, upload_ok, autoip;
struct ethsock *sock;
uint32_t intf_addr;
void (*sigh_orig)(int);
struct {
struct in_addr addr;
struct in_addr mask;
} PACKED ipconf;
if (args->op != NMRP_UPLOAD_FW) {
fprintf(stderr, "Operation not implemented.\n");
return 1;
}
if (!mac_parse(args->mac, dest)) {
fprintf(stderr, "Invalid MAC address '%s'.\n", args->mac);
return 1;
}
if ((ipconf.mask.s_addr = inet_addr(args->ipmask)) == INADDR_NONE) {
fprintf(stderr, "Invalid subnet mask '%s'.\n", args->ipmask);
return 1;
}
if (!args->ipaddr) {
autoip = true;
/* The MAC of the device that was used to test this utility starts
* with a4:2b:8c, hence 164 (0xa4) and 183 (0x2b + 0x8c)
*/
args->ipaddr = "10.164.183.252";
if (!args->ipaddr_intf) {
args->ipaddr_intf = "10.164.183.253";
}
} else if (args->ipaddr_intf) {
autoip = true;
} else {
autoip = false;
}
if ((ipconf.addr.s_addr = inet_addr(args->ipaddr)) == INADDR_NONE) {
fprintf(stderr, "Invalid IP address '%s'.\n", args->ipaddr);
return 1;
}
if (args->ipaddr_intf && (intf_addr = inet_addr(args->ipaddr_intf)) == INADDR_NONE) {
fprintf(stderr, "Invalid IP address '%s'.\n", args->ipaddr_intf);
return 1;
}
if (args->file_local && strcmp(args->file_local, "-") && access(args->file_local, R_OK) == -1) {
fprintf(stderr, "Error accessing file '%s'.\n", args->file_local);
return 1;
}
if (args->file_remote) {
if (!tftp_is_valid_filename(args->file_remote)) {
fprintf(stderr, "Invalid remote filename '%s'.\n",
args->file_remote);
return 1;
}
}
if (args->region) {
region = htons(to_region_code(args->region));
if (!region) {
fprintf(stderr, "Invalid region code '%s'.\n", args->region);
return 1;
}
} else {
region = 0;
}
status = 1;
sock = ethsock_create(args->intf, ETH_P_NMRP);
if (!sock) {
return 1;
}
gsock = sock;
garp = 0;
sigh_orig = signal(SIGINT, sigh);
if (!autoip) {
status = is_valid_ip(sock, &ipconf.addr, &ipconf.mask);
if (status <= 0) {
if (!status) {
fprintf(stderr, "Address %s/%s cannot be used on interface %s.\n",
args->ipaddr, args->ipmask, args->intf);
}
goto out;
}
} else {
if (verbosity) {
printf("Adding %s to interface %s.\n", args->ipaddr_intf, args->intf);
}
if (ethsock_ip_add(sock, intf_addr, ipconf.mask.s_addr, &gundo) != 0) {
goto out;
}
}
if (ethsock_set_timeout(sock, args->rx_timeout)) {
goto out;
}
src = ethsock_get_hwaddr(sock);
if (!src) {
goto out;
}
memcpy(tx.eh.ether_shost, src, 6);
memcpy(tx.eh.ether_dhost, dest, 6);
tx.eh.ether_type = htons(ETH_P_NMRP);
msg_init(&tx.msg, NMRP_C_ADVERTISE);
msg_opt_add(&tx.msg, NMRP_O_MAGIC_NO, "NTGR", 4);
msg_hton(&tx.msg);
i = 0;
upload_ok = 0;
beg = time(NULL);
while (1) {
printf("\rAdvertising NMRP server on %s ... %c",
args->intf, spinner[i]);
fflush(stdout);
i = (i + 1) & 3;
if (pkt_send(sock, &tx) < 0) {
perror("sendto");
goto out;
}
status = pkt_recv(sock, &rx);
if (status == 0 && memcmp(rx.eh.ether_dhost, src, 6) == 0) {
break;
} else if (status == 1) {
goto out;
} else {
if ((time(NULL) - beg) >= 60) {
printf("\nNo response after 60 seconds. Bailing out.\n");
goto out;
}
}
}
printf("\n");
expect = NMRP_C_CONF_REQ;
ulreqs = 0;
do {
if (expect != NMRP_C_NONE && rx.msg.code != expect) {
fprintf(stderr, "Received %s while waiting for %s!\n",
msg_code_str(rx.msg.code), msg_code_str(expect));
}
msg_init(&tx.msg, NMRP_C_NONE);
status = 1;
switch (rx.msg.code) {
case NMRP_C_ADVERTISE:
printf("Received NMRP advertisement from %s.\n",
mac_to_str(rx.eh.ether_shost));
status = 1;
goto out;
case NMRP_C_CONF_REQ:
tx.msg.code = NMRP_C_CONF_ACK;
msg_opt_add(&tx.msg, NMRP_O_DEV_IP, &ipconf, 8);
msg_opt_add(&tx.msg, NMRP_O_FW_UP, NULL, 0);
#ifdef NMRPFLASH_SET_REGION
if (region) {
msg_opt_add(&tx.msg, NMRP_O_DEV_REGION, ®ion, 2);
}
#endif
expect = NMRP_C_TFTP_UL_REQ;
printf("Received configuration request from %s.\n",
mac_to_str(rx.eh.ether_shost));
memcpy(tx.eh.ether_dhost, rx.eh.ether_shost, 6);
printf("Sending configuration: ip %s, mask %s.\n",
args->ipaddr, args->ipmask);
memcpy(arpmac, rx.eh.ether_shost, 6);
memcpy(&arpip, &ipconf.addr, sizeof(ipconf.addr));
if (ethsock_arp_add(sock, arpmac, &arpip) != 0) {
goto out;
}
garp = 1;
break;
case NMRP_C_TFTP_UL_REQ:
if (!upload_ok) {
if (++ulreqs > 5) {
printf("Bailing out after %d upload requests.\n",
ulreqs);
tx.msg.code = NMRP_C_CLOSE_REQ;
break;
}
} else {
if (verbosity) {
printf("Ignoring extra upload request.\n");
}
ethsock_set_timeout(sock, args->ul_timeout);
tx.msg.code = NMRP_C_KEEP_ALIVE_REQ;
break;
}
len = 0;
filename = msg_opt_data(&rx.msg, NMRP_O_FILE_NAME, &len);
if (filename) {
if (!args->file_remote) {
args->file_remote = filename;
}
printf("Received upload request: filename '%.*s'.\n",
len, filename);
} else if (!args->file_remote) {
args->file_remote = args->file_local;
printf("Received upload request with empty filename.\n");
}
status = 0;
if (args->tftpcmd) {
printf("Executing '%s' ... \n", args->tftpcmd);
setenv("IP", inet_ntoa(ipconf.addr), 1);
setenv("MAC", mac_to_str(rx.eh.ether_shost), 1);
setenv("NETMASK", inet_ntoa(ipconf.mask), 1);
status = system(args->tftpcmd);
}
if (!status && args->file_local) {
if (!autoip) {
status = is_valid_ip(sock, &ipconf.addr, &ipconf.mask);
if (status < 0) {
goto out;
} else if (!status) {
printf("IP address of %s has changed. Please assign a "
"static ip to the interface.\n", args->intf);
tx.msg.code = NMRP_C_CLOSE_REQ;
break;
}
}
if (verbosity) {
printf("Using remote filename '%s'.\n",
args->file_remote);
}
if (!strcmp(args->file_local, "-")) {
printf("Uploading from stdin ... ");
} else {
printf("Uploading %s ... ", leafname(args->file_local));
}
fflush(stdout);
status = tftp_put(args);
}
if (!status) {
printf("OK\nWaiting for remote to respond.\n");
upload_ok = 1;
ethsock_set_timeout(sock, args->ul_timeout);
tx.msg.code = NMRP_C_KEEP_ALIVE_REQ;
expect = NMRP_C_NONE;
} else if (status == -2) {
expect = NMRP_C_TFTP_UL_REQ;
} else {
goto out;
}
break;
case NMRP_C_KEEP_ALIVE_REQ:
tx.msg.code = NMRP_C_KEEP_ALIVE_ACK;
ethsock_set_timeout(sock, args->ul_timeout);
printf("Received keep-alive request.\n");
break;
case NMRP_C_CLOSE_REQ:
tx.msg.code = NMRP_C_CLOSE_ACK;
break;
case NMRP_C_CLOSE_ACK:
status = 0;
goto out;
default:
fprintf(stderr, "Unknown message code 0x%02x!\n",
rx.msg.code);
msg_dump(&rx.msg, 0);
}
if (tx.msg.code != NMRP_C_NONE) {
msg_hton(&tx.msg);
if (pkt_send(sock, &tx) < 0) {
perror("sendto");
goto out;
}
if (tx.msg.code == NMRP_C_CLOSE_REQ) {
goto out;
}
}
if (rx.msg.code == NMRP_C_CLOSE_REQ) {
printf("Remote finished. Closing connection.\n");
break;
}
status = pkt_recv(sock, &rx);
if (status) {
if (status == 2) {
fprintf(stderr, "Timeout while waiting for %s.\n",
msg_code_str(expect));
}
goto out;
}
ethsock_set_timeout(sock, args->rx_timeout);
} while (1);
status = 0;
if (ulreqs) {
printf("Reboot your device now.\n");
} else {
printf("No upload request received.\n");
}
out:
signal(SIGINT, sigh_orig);
gsock = NULL;
ethsock_arp_del(sock, arpmac, &arpip);
ethsock_ip_del(sock, &gundo);
ethsock_close(sock);
return status;
}
/*
* Sending a mbuf of gossip data over the wire to a peer
*/
void
dnode_peer_gossip_forward(struct context *ctx, struct conn *conn, bool redis, struct mbuf *data_buf)
{
rstatus_t status;
struct msg *msg = msg_get(conn, 1, redis);
if (msg == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a msg");
return;
}
struct mbuf *header_buf = mbuf_get();
if (header_buf == NULL) {
log_debug(LOG_DEBUG, "Unable to obtain a data_buf");
msg_put(msg);
return;
}
uint64_t msg_id = peer_msg_id++;
if (conn->dnode_secured) {
log_debug(LOG_VERB, "Assemble a secured msg to send");
log_debug(LOG_VERB, "AES encryption key: %s\n", base64_encode(conn->aes_key, AES_KEYLEN));
struct mbuf *encrypted_buf = mbuf_get();
if (encrypted_buf == NULL) {
loga("Unable to obtain an data_buf for encryption!");
return; //TODOs: need to clean up
}
status = dyn_aes_encrypt(data_buf->pos, mbuf_length(data_buf), encrypted_buf, conn->aes_key);
log_debug(LOG_VERB, "#encrypted bytes : %d", status);
//write dnode header
dmsg_write(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(encrypted_buf));
mbuf_insert_head(&msg->mhdr, header_buf);
if (TRACING_LEVEL == LOG_VVERB) {
log_hexdump(LOG_VVERB, data_buf->pos, mbuf_length(data_buf), "dyn message original payload: ");
log_hexdump(LOG_VVERB, encrypted_buf->pos, mbuf_length(encrypted_buf), "dyn message encrypted payload: ");
}
mbuf_insert(&msg->mhdr, encrypted_buf);
//free data_buf as no one will need it again
mbuf_put(data_buf);
} else {
log_debug(LOG_VERB, "Assemble a non-secured msg to send");
dmsg_write_mbuf(header_buf, msg_id, GOSSIP_SYN, conn, mbuf_length(data_buf));
mbuf_insert_head(&msg->mhdr, header_buf);
mbuf_insert(&msg->mhdr, data_buf);
}
if (TRACING_LEVEL == LOG_VVERB) {
log_hexdump(LOG_VVERB, header_buf->pos, mbuf_length(header_buf), "dyn gossip message header: ");
msg_dump(msg);
}
/* enqueue the message (request) into peer inq */
if (TAILQ_EMPTY(&conn->imsg_q)) {
status = event_add_out(ctx->evb, conn);
if (status != DN_OK) {
dnode_req_forward_error(ctx, conn, msg);
conn->err = errno;
return;
}
}
//need to handle a reply
//conn->enqueue_outq(ctx, conn, msg);
msg->noreply = 1;
conn->enqueue_inq(ctx, conn, msg);
}
void
dyn_parse_req(struct msg *r)
{
#ifdef DN_DEBUG_LOG
log_debug(LOG_VVERB, "In dyn_parse_req, start to process request :::::::::::::::::::::: ");
msg_dump(r);
#endif
bool done_parsing = false;
if (dyn_parse_core(r)) {
struct dmsg *dmsg = r->dmsg;
if (dmsg->type != DMSG_UNKNOWN && dmsg->type != DMSG_REQ && dmsg->type != GOSSIP_SYN) {
log_debug(LOG_DEBUG, "Req parser: I got a dnode msg of type %d", dmsg->type);
r->state = 0;
r->result = MSG_PARSE_OK;
r->dyn_state = DYN_DONE;
return;
}
if (dmsg->type == GOSSIP_SYN) {
#ifdef DN_DEBUG_LOG
log_debug(LOG_DEBUG, "Req parser: I got a GOSSIP_SYN msg");
#endif
//TODOs: need to address multi-buffer msg later
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
dmsg->payload = b->pos;
b->pos = b->pos + dmsg->plen;
r->pos = b->pos;
done_parsing = true;
}
//check whether we need to decrypt the payload
if (dmsg->bit_field == 1) {
dmsg->owner->owner->dnode_secured = 1;
r->owner->dnode_crypto_state = 1;
struct mbuf *decrypted_buf = mbuf_get();
if (decrypted_buf == NULL) {
loga("Unable to obtain an mbuf for dnode msg's header!");
return;
}
#ifdef DN_DEBUG_LOG
log_debug(LOG_DEBUG, "data or encrypted aes key length : %d", dmsg->plen);
#endif
if (dmsg->mlen > 1) {
#ifdef DN_DEBUG_LOG
log_debug(LOG_DEBUG, "dmsg->mlen is something: %d, need to process it", dmsg->plen);
#endif
//Decrypt AES key
dyn_rsa_decrypt(dmsg->data, aes_decrypted_buf);
strncpy(r->owner->aes_key, aes_decrypted_buf, strlen(aes_decrypted_buf));
//Decrypt payload
dyn_aes_decrypt(dmsg->payload, dmsg->plen, decrypted_buf, aes_decrypted_buf);
} else {
#ifdef DN_DEBUG_LOG
log_debug(LOG_DEBUG, "dmsg->mlen is a dummy: %d, NO need to process it", dmsg->plen);
#endif
dyn_aes_decrypt(dmsg->payload, dmsg->plen, decrypted_buf, r->owner->aes_key);
}
#ifdef DN_DEBUG_LOG
loga("AES encryption key: %s\n", base64_encode(aes_decrypted_buf, AES_KEYLEN));
log_hexdump(LOG_VERB, decrypted_buf->pos, mbuf_length(decrypted_buf), "dyn message decrypted payload: ");
#endif
struct mbuf *b = STAILQ_LAST(&r->mhdr, mbuf, next);
b->last = b->pos;
r->pos = decrypted_buf->start;
mbuf_insert(&r->mhdr, decrypted_buf);
//reset these variables
dmsg->payload = decrypted_buf->start;
dmsg->plen = mbuf_length(decrypted_buf);
}
if (done_parsing)
return;
if (r->redis)
return redis_parse_req(r);
return memcache_parse_req(r);
}
//bad case
log_debug(LOG_DEBUG, "Bad message - cannot parse"); //fix me to do something
msg_dump(r);
}
/* There are chances that the request to the remote peer or its response got dropped.
* Hence we may not always receive a response to the request at the head of the FIFO.
* Hence what we do is we mark that request as errored and move on the next one
* in the outgoing queue. This works since we always have message ids in monotonically
* increasing order.
*/
static void
dnode_rsp_forward(struct context *ctx, struct conn *peer_conn, struct msg *rsp)
{
rstatus_t status;
struct msg *req;
struct conn *c_conn;
ASSERT(!peer_conn->dnode_client && !peer_conn->dnode_server);
/* response from a peer implies that peer is ok and heartbeating */
dnode_peer_ok(ctx, peer_conn);
/* dequeue peer message (request) from peer conn */
while (true) {
req = TAILQ_FIRST(&peer_conn->omsg_q);
log_debug(LOG_VERB, "dnode_rsp_forward entering req %p rsp %p...", req, rsp);
c_conn = req->owner;
if (req->id == rsp->dmsg->id) {
dnode_rsp_forward_match(ctx, peer_conn, rsp);
return;
}
// Report a mismatch and try to rectify
log_error("MISMATCH: dnode %c %d rsp_dmsg_id %u req %u:%u dnode rsp %u:%u",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id, req->id, req->parent_id, rsp->id,
rsp->parent_id);
if (c_conn && conn_to_ctx(c_conn))
stats_pool_incr(conn_to_ctx(c_conn), c_conn->owner,
peer_mismatch_requests);
// TODO : should you be worried about message id getting wrapped around to 0?
if (rsp->dmsg->id < req->id) {
// We received a response from the past. This indeed proves out of order
// responses. A blunder to the architecture. Log it and drop the response.
log_error("MISMATCH: received response from the past. Dropping it");
dnode_rsp_put(rsp);
return;
}
if (req->consistency == DC_ONE) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_ONE consistency is not being swallowed");
}
if ((req->consistency == DC_QUORUM) && !peer_conn->same_dc) {
if (req->swallow) {
// swallow the request and move on the next one
dnode_rsp_swallow(ctx, peer_conn, req, NULL);
continue;
}
log_warn("req %d:%d with DC_QUORUM consistency is not being swallowed");
}
log_error("MISMATCHED DNODE RSP RECEIVED %c %d dmsg->id %u req %u:%u rsp %u:%u, skipping....",
peer_conn->dnode_client ? 'c' : (peer_conn->dnode_server ? 's' : 'p'),
peer_conn->sd, rsp->dmsg->id,
req->id, req->parent_id, rsp->id, rsp->parent_id);
ASSERT(req != NULL && req->peer == NULL);
ASSERT(req->request && !req->done);
if (log_loggable(LOG_VVERB)) {
loga("skipping req: ");
msg_dump(req);
}
peer_conn->dequeue_outq(ctx, peer_conn, req);
req->done = 1;
// Create an appropriate response for the request so its propagated up;
struct msg *err_rsp = msg_get(peer_conn, false, peer_conn->data_store);
err_rsp->error = req->error = 1;
err_rsp->err = req->err = BAD_FORMAT;
err_rsp->dyn_error = req->dyn_error = BAD_FORMAT;
err_rsp->dmsg = dmsg_get();
err_rsp->dmsg->id = req->id;
log_debug(LOG_VERB, "%p <-> %p", req, err_rsp);
/* establish err_rsp <-> req (response <-> request) link */
req->peer = err_rsp;
err_rsp->peer = req;
log_error("Peer connection s %d skipping request %u:%u, dummy err_rsp %u:%u",
peer_conn->sd, req->id, req->parent_id, err_rsp->id, err_rsp->parent_id);
rstatus_t status =
conn_handle_response(c_conn, req->parent_id ? req->parent_id : req->id,
err_rsp);
IGNORE_RET_VAL(status);
if (req->swallow) {
log_debug(LOG_INFO, "swallow request %d:%d", req->id, req->parent_id);
req_put(req);
}
}
}
