static void
req_forward_remote_dc(struct context *ctx, struct conn *c_conn, struct msg *msg,
struct mbuf *orig_mbuf, uint8_t *key, uint32_t keylen,
struct datacenter *dc)
{
uint32_t rack_cnt = array_n(&dc->racks);
if (rack_cnt == 0)
return;
struct rack *rack = dc->preselected_rack_for_replication;
if (rack == NULL)
rack = array_get(&dc->racks, 0);
struct msg *rack_msg = msg_get(c_conn, msg->request, __FUNCTION__);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed now, buddy. no inter-rack messages for you!");
msg_put(rack_msg);
return;
}
msg_clone(msg, orig_mbuf, rack_msg);
log_info("msg (%d:%d) clone to remote rack msg (%d:%d)",
msg->id, msg->parent_id, rack_msg->id, rack_msg->parent_id);
rack_msg->swallow = true;
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request to conn '%s' on rack '%.*s'",
dn_unresolve_peer_desc(c_conn->sd), rack->name->len, rack->name->data);
}
remote_req_forward(ctx, c_conn, rack_msg, rack, key, keylen);
}
/**Forward a request.
*
* @deprecated
* Use nta_outgoing_mcreate() instead.
*/
nta_outgoing_t *nta_outgoing_tclone(nta_agent_t *agent,
nta_response_f *callback,
nta_outgoing_magic_t *magic,
url_string_t const *route_url,
msg_t *parent,
tag_type_t tag, tag_value_t value, ...)
{
ta_list ta;
msg_t *msg;
nta_outgoing_t *orq = NULL;
if (parent == NULL)
return NULL;
if ((msg = nta_msg_create(agent, 0)) == NULL)
return NULL;
ta_start(ta, tag, value);
msg_clone(msg, parent);
if (parent && sip_copy_all(msg, sip_object(msg), sip_object(parent)) < 0)
;
else if (sip_add_tl(msg, sip_object(msg), ta_tags(ta)) < 0)
;
else
orq = nta_outgoing_mcreate(agent, callback, magic, route_url, msg);
ta_end(ta);
if (!orq)
msg_destroy(msg);
return orq;
}
/** Fork an outgoing request (stdarg version of nta_outgoing_fork()).
*
* @deprecated
* Use nta_outgoing_mcreate() instead.
*/
nta_outgoing_t *nta_outgoing_vfork(nta_outgoing_t *old_orq,
nta_response_f *callback,
nta_outgoing_magic_t *magic,
url_string_t const *route_url,
url_string_t const *request_uri,
void const *extra,
va_list headers)
{
nta_outgoing_t * orq;
msg_t *msg, *imsg;
sip_t *sip, *isip;
nta_agent_t *agent;
su_home_t *home;
if (!old_orq || !old_orq->orq_request || !request_uri)
return NULL;
agent = old_orq->orq_agent;
imsg = old_orq->orq_request;
if (!(msg = nta_msg_create(agent, 0)))
return NULL;
msg_clone(msg, imsg);
sip = sip_object(msg); isip = sip_object(imsg);
home = msg_home(msg);
/* Copy the SIP headers from the imsg message */
if (sip_copy_all(msg, sip, isip) < 0)
orq = NULL;
else if (sip_via_remove(msg, sip) == NULL)
orq = NULL;
else if (sip_add_dup(msg, sip_object(msg),
(sip_header_t const *)
sip_request_create(home,
sip->sip_request->rq_method,
sip->sip_request->rq_method_name,
request_uri,
NULL)) < 0)
orq = NULL;
else if (sip_add_headers(msg, sip, extra, headers) < 0)
orq = NULL;
else
orq = nta_outgoing_mcreate(agent, callback, magic, route_url, msg);
if (!orq)
msg_destroy(msg);
return orq;
}
/**Get request message.
*
* The function nta_outgoing_getrequest() retrieves the request message sent
* to the network. The request message is copied; the original copy is kept
* by the transaction.
*
* @param orq outgoing transaction handle
*
* @retval
* A pointer to the copy of the request message is returned, or NULL if an
* error occurred.
*/
msg_t *nta_outgoing_getrequest(nta_outgoing_t *orq)
{
if (orq && orq->orq_request) {
msg_t *msg = nta_msg_create(orq->orq_agent, 0);
sip_t *sip = sip_object(msg);
msg_clone(msg, orq->orq_request);
/* Copy the SIP headers from the old message */
if (sip_copy_all(msg, sip, sip_object(orq->orq_request)) >= 0)
return msg;
msg_destroy(msg);
}
return NULL;
}
/**Get response message.
*
* The function nta_incoming_getresponse() retrieves a copy of the latest
* outgoing response message. The response message is copied; the original
* copy is kept by the transaction.
*
* @param irq incoming (server) transaction handle
*
* @retval
* A pointer to the copy of the response message is returned, or NULL if an
* error occurred.
*/
msg_t *nta_incoming_getresponse(nta_incoming_t *irq)
{
if (irq && irq->irq_response) {
msg_t *msg = nta_msg_create(irq->irq_agent, 0);
sip_t *sip = sip_object(msg);
msg_clone(msg, irq->irq_response);
/* Copy the SIP headers from the old message */
if (msg_copy_all(msg, sip, sip_object(irq->irq_response)) >= 0)
return msg;
msg_destroy(msg);
}
return NULL;
}
static void
req_forward_all_local_racks(struct context *ctx, struct conn *c_conn,
struct msg *msg, struct mbuf *orig_mbuf,
uint8_t *key, uint32_t keylen, struct datacenter *dc)
{
//log_debug(LOG_DEBUG, "dc name '%.*s'",
// dc->name->len, dc->name->data);
uint8_t rack_cnt = (uint8_t)array_n(&dc->racks);
uint8_t rack_index;
msg->rsp_handler = msg_get_rsp_handler(msg);
init_response_mgr(&msg->rspmgr, msg, msg->is_read, rack_cnt, c_conn);
log_info("msg %d:%d same DC racks:%d expect replies %d",
msg->id, msg->parent_id, rack_cnt, msg->rspmgr.max_responses);
for(rack_index = 0; rack_index < rack_cnt; rack_index++) {
struct rack *rack = array_get(&dc->racks, rack_index);
//log_debug(LOG_DEBUG, "rack name '%.*s'",
// rack->name->len, rack->name->data);
struct msg *rack_msg;
// clone message even for local node
struct server_pool *pool = c_conn->owner;
if (string_compare(rack->name, &pool->rack) == 0 ) {
rack_msg = msg;
} else {
rack_msg = msg_get(c_conn, msg->request, __FUNCTION__);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed "
"now, buddy. no inter-rack messages for "
"you!");
continue;
}
msg_clone(msg, orig_mbuf, rack_msg);
log_info("msg (%d:%d) clone to rack msg (%d:%d)",
msg->id, msg->parent_id, rack_msg->id, rack_msg->parent_id);
rack_msg->swallow = true;
}
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request to conn '%s' on rack '%.*s'",
dn_unresolve_peer_desc(c_conn->sd), rack->name->len, rack->name->data);
}
log_debug(LOG_VERB, "c_conn: %p forwarding (%d:%d)",
c_conn, rack_msg->id, rack_msg->parent_id);
remote_req_forward(ctx, c_conn, rack_msg, rack, key, keylen);
}
}
/**Forward a request belonging to the leg
* (stdarg version of nta_outgoing_forward()).
*
* @deprecated
* Use nta_outgoing_mcreate() instead.
*/
nta_outgoing_t *nta_outgoing_vforward(nta_leg_t *leg,
nta_response_f *callback,
nta_outgoing_magic_t *magic,
url_string_t const *route_url,
url_string_t const *request_uri,
nta_incoming_t const *ireq,
sip_t const *isip,
void const *extra,
va_list headers)
{
nta_agent_t *agent = leg->leg_agent;
nta_outgoing_t *orq = NULL;
msg_t *msg, *imsg;
sip_t *sip;
su_home_t *home;
assert(leg); assert(ireq);
if (isip == NULL)
imsg = ireq->irq_request, isip = sip_object(ireq->irq_request);
else if (isip == sip_object(ireq->irq_request))
imsg = ireq->irq_request;
else if (isip == sip_object(ireq->irq_request2))
imsg = ireq->irq_request2;
else {
SU_DEBUG_3(("nta_outgoing_forward: invalid arguments\n"));
return NULL;
}
assert(isip); assert(isip->sip_request);
if (!route_url)
route_url = (url_string_t *)agent->sa_default_proxy;
if (!(msg = nta_msg_create(agent, 0)))
return NULL;
msg_clone(msg, imsg);
sip = sip_object(msg);
home = msg_home(msg);
/* Copy the SIP headers from the @c imsg message */
do {
if (sip_copy_all(msg, sip, isip) < 0)
break;
if (sip_add_headers(msg, sip, extra, headers) < 0)
break;
if (!route_url && sip->sip_route) {
request_uri = (url_string_t *)sip->sip_route->r_url;
if (!sip_route_remove(msg, sip))
break;
}
if (request_uri) {
sip_request_t *rq;
rq = sip_request_create(home,
sip->sip_request->rq_method,
sip->sip_request->rq_method_name,
request_uri,
NULL);
if (!rq || sip_header_insert(msg, sip, (sip_header_t *)rq) < 0)
break;
}
if ((orq = nta_outgoing_mcreate(agent, callback, magic, route_url, msg)))
return orq;
} while (0);
msg_destroy(msg);
return NULL;
}
static void
dnode_req_forward(struct context *ctx, struct conn *conn, struct msg *msg)
{
struct server_pool *pool;
uint8_t *key;
uint32_t keylen;
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "dnode_req_forward entering ");
}
log_debug(LOG_DEBUG, "DNODE REQ RECEIVED %s %d dmsg->id %u",
conn_get_type_string(conn), conn->sd, msg->dmsg->id);
ASSERT(conn->type == CONN_DNODE_PEER_CLIENT);
pool = conn->owner;
key = NULL;
keylen = 0;
log_debug(LOG_DEBUG, "conn %p adding message %d:%d", conn, msg->id, msg->parent_id);
dictAdd(conn->outstanding_msgs_dict, &msg->id, msg);
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = dn_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = dn_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
ASSERT(msg->dmsg != NULL);
if (msg->dmsg->type == DMSG_REQ) {
local_req_forward(ctx, conn, msg, key, keylen);
} else if (msg->dmsg->type == DMSG_REQ_FORWARD) {
struct mbuf *orig_mbuf = STAILQ_FIRST(&msg->mhdr);
struct datacenter *dc = server_get_dc(pool, &pool->dc);
uint32_t rack_cnt = array_n(&dc->racks);
uint32_t rack_index;
for(rack_index = 0; rack_index < rack_cnt; rack_index++) {
struct rack *rack = array_get(&dc->racks, rack_index);
//log_debug(LOG_DEBUG, "forwarding to rack '%.*s'",
// rack->name->len, rack->name->data);
struct msg *rack_msg;
if (string_compare(rack->name, &pool->rack) == 0 ) {
rack_msg = msg;
} else {
rack_msg = msg_get(conn, msg->request, __FUNCTION__);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed now, buddy. no inter-rack messages for you!");
continue;
}
if (msg_clone(msg, orig_mbuf, rack_msg) != DN_OK) {
msg_put(rack_msg);
continue;
}
rack_msg->swallow = true;
}
if (log_loggable(LOG_DEBUG)) {
log_debug(LOG_DEBUG, "forwarding request from conn '%s' to rack '%.*s' dc '%.*s' ",
dn_unresolve_peer_desc(conn->sd), rack->name->len, rack->name->data, rack->dc->len, rack->dc->data);
}
remote_req_forward(ctx, conn, rack_msg, rack, key, keylen);
}
}
}
static void
req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
{
struct server_pool *pool = c_conn->owner;
uint8_t *key;
uint32_t keylen;
ASSERT(c_conn->client && !c_conn->proxy);
if (msg->is_read)
stats_pool_incr(ctx, pool, client_read_requests);
else
stats_pool_incr(ctx, pool, client_write_requests);
key = NULL;
keylen = 0;
if (!string_empty(&pool->hash_tag)) {
struct string *tag = &pool->hash_tag;
uint8_t *tag_start, *tag_end;
tag_start = dn_strchr(msg->key_start, msg->key_end, tag->data[0]);
if (tag_start != NULL) {
tag_end = dn_strchr(tag_start + 1, msg->key_end, tag->data[1]);
if (tag_end != NULL) {
key = tag_start + 1;
keylen = (uint32_t)(tag_end - key);
}
}
}
if (keylen == 0) {
key = msg->key_start;
keylen = (uint32_t)(msg->key_end - msg->key_start);
}
// need to capture the initial mbuf location as once we add in the dynomite headers (as mbufs to the src msg),
// that will bork the request sent to secondary racks
struct mbuf *orig_mbuf = STAILQ_FIRST(&msg->mhdr);
if (request_send_to_all_racks(msg)) {
uint32_t dc_cnt = array_n(&pool->datacenters);
uint32_t dc_index;
for(dc_index = 0; dc_index < dc_cnt; dc_index++) {
struct datacenter *dc = array_get(&pool->datacenters, dc_index);
if (dc == NULL) {
log_error("Wow, this is very bad, dc is NULL");
return;
}
if (string_compare(dc->name, &pool->dc) == 0) { //send to all local racks
//log_debug(LOG_DEBUG, "dc name '%.*s'", dc->name->len, dc->name->data);
uint32_t rack_cnt = array_n(&dc->racks);
uint32_t rack_index;
for(rack_index = 0; rack_index < rack_cnt; rack_index++) {
struct rack *rack = array_get(&dc->racks, rack_index);
//log_debug(LOG_DEBUG, "rack name '%.*s'", rack->name->len, rack->name->data);
struct msg *rack_msg;
if (string_compare(rack->name, &pool->rack) == 0 ) {
rack_msg = msg;
} else {
rack_msg = msg_get(c_conn, msg->request, msg->redis);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed now, buddy. no inter-rack messages for you!");
continue;
}
msg_clone(msg, orig_mbuf, rack_msg);
rack_msg->noreply = true;
}
log_debug(LOG_DEBUG, "forwarding request to conn '%s' on rack '%.*s'",
dn_unresolve_peer_desc(c_conn->sd), rack->name->len, rack->name->data);
remote_req_forward(ctx, c_conn, rack_msg, rack, key, keylen);
}
} else {
uint32_t rack_cnt = array_n(&dc->racks);
if (rack_cnt == 0)
continue;
uint32_t ran_index = rand() % rack_cnt;
struct rack *rack = array_get(&dc->racks, ran_index);
struct msg *rack_msg = msg_get(c_conn, msg->request, msg->redis);
if (rack_msg == NULL) {
log_debug(LOG_VERB, "whelp, looks like yer screwed now, buddy. no inter-rack messages for you!");
continue;
}
msg_clone(msg, orig_mbuf, rack_msg);
rack_msg->noreply = true;
log_debug(LOG_DEBUG, "forwarding request to conn '%s' on rack '%.*s'",
dn_unresolve_peer_desc(c_conn->sd), rack->name->len, rack->name->data);
remote_req_forward(ctx, c_conn, rack_msg, rack, key, keylen);
}
}
} else { //for read only requests
struct rack * rack = server_get_rack_by_dc_rack(pool, &pool->rack, &pool->dc);
remote_req_forward(ctx, c_conn, msg, rack, key, keylen);
}
}
