void uwsgi_corerouter_manage_subscription(struct uwsgi_corerouter *ucr, int id, struct uwsgi_gateway_socket *ugs) {
int i;
struct uwsgi_subscribe_req usr;
char bbuf[4096];
ssize_t len = recv(ugs->fd, bbuf, 4096, 0);
#ifdef UWSGI_EVENT_USE_PORT
event_queue_add_fd_read(ucr->queue, ugs->fd);
#endif
if (len > 0) {
memset(&usr, 0, sizeof(struct uwsgi_subscribe_req));
uwsgi_hooked_parse(bbuf + 4, len - 4, corerouter_manage_subscription, &usr);
// subscribe request ?
if (bbuf[3] == 0) {
if (uwsgi_add_subscribe_node(&ucr->subscriptions, &usr, ucr->subscription_regexp) && ucr->i_am_cheap) {
struct uwsgi_gateway_socket *ugs = uwsgi.gateway_sockets;
while (ugs) {
if (!strcmp(ugs->owner, ucr->name) && !ugs->subscription) {
event_queue_add_fd_read(ucr->queue, ugs->fd);
}
ugs = ugs->next;
}
ucr->i_am_cheap = 0;
uwsgi_log("[%s pid %d] leaving cheap mode...\n", ucr->name, (int) uwsgi.mypid);
}
}
//unsubscribe
else {
struct uwsgi_subscribe_node *node = uwsgi_get_subscribe_node_by_name(&ucr->subscriptions, usr.key, usr.keylen, usr.address, usr.address_len, ucr->subscription_regexp);
if (node && node->len) {
if (node->death_mark == 0)
uwsgi_log("[%s pid %d] %.*s => marking %.*s as failed\n", ucr->name, (int) uwsgi.mypid, (int) usr.keylen, usr.key, (int) usr.address_len, usr.address);
node->failcnt++;
node->death_mark = 1;
// check if i can remove the node
if (node->reference == 0) {
uwsgi_remove_subscribe_node(&ucr->subscriptions, node);
}
if (ucr->subscriptions == NULL && ucr->cheap && !ucr->i_am_cheap) {
uwsgi_gateway_go_cheap(ucr->name, ucr->queue, &ucr->i_am_cheap);
}
}
}
// propagate the subscription to other nodes
for (i = 0; i < ushared->gateways_cnt; i++) {
if (i == id)
continue;
if (!strcmp(ushared->gateways[i].name, ucr->name)) {
if (send(ushared->gateways[i].internal_subscription_pipe[0], bbuf, len, 0) != len) {
uwsgi_error("send()");
}
}
}
}
}
static void close_session(struct fastrouter_session **fr_table, struct fastrouter_session *fr_session) {
// check timeout expired
if (fr_session == NULL) {
if (ufr.subscription_server) {
//time_t current_time = time(NULL);
uwsgi_log("checking for node health\n");
struct uwsgi_subscribe_slot *slot = ufr.subscriptions;
while(slot) {
struct uwsgi_subscribe_node *node = slot->nodes;
while(node) {
uwsgi_log("%.*s (hits: %llu) %.*s\n", slot->keylen, slot->key, slot->hits, node->len, node->name);
node = node->next;
}
slot = slot->next;
}
del_check_timeout(ufr.subscriptions_check);
ufr.subscriptions_check = add_check_timeout(10);
}
return;
}
close(fr_session->fd);
fr_table[fr_session->fd] = NULL;
if (fr_session->instance_fd != -1) {
if (ufr.subscriptions && (fr_session->instance_failed || fr_session->status == FASTROUTER_STATUS_CONNECTING)) {
if (fr_session->un && fr_session->un->len > 0) {
uwsgi_log("[uwsgi-fastrouter] %.*s => marking %.*s as failed\n", (int) fr_session->hostname_len, fr_session->hostname, (int) fr_session->instance_address_len,fr_session->instance_address);
uwsgi_remove_subscribe_node(&ufr.subscriptions, fr_session->un);
if (ufr.subscriptions == NULL && ufr.cheap && !ufr.i_am_cheap) {
uwsgi_log("[uwsgi-fastrouter] no more nodes available. Going cheap...\n");
struct uwsgi_fastrouter_socket *ufr_sock = ufr.sockets;
while(ufr_sock) {
event_queue_del_fd(ufr.queue, ufr_sock->fd, event_queue_read());
ufr_sock = ufr_sock->next;
}
ufr.i_am_cheap = 1;
}
}
}
close(fr_session->instance_fd);
fr_table[fr_session->instance_fd] = NULL;
}
del_timeout(fr_session);
free(fr_session);
}
void uwsgi_corerouter_manage_internal_subscription(struct uwsgi_corerouter *ucr, int fd) {
struct uwsgi_subscribe_req usr;
char bbuf[4096];
ssize_t len = recv(fd, bbuf, 4096, 0);
if (len > 0) {
memset(&usr, 0, sizeof(struct uwsgi_subscribe_req));
uwsgi_hooked_parse(bbuf + 4, len - 4, corerouter_manage_subscription, &usr);
// subscribe request ?
if (bbuf[3] == 0) {
if (uwsgi_add_subscribe_node(ucr->subscriptions, &usr) && ucr->i_am_cheap) {
struct uwsgi_gateway_socket *ugs = uwsgi.gateway_sockets;
while (ugs) {
if (!strcmp(ugs->owner, ucr->name) && !ugs->subscription) {
event_queue_add_fd_read(ucr->queue, ugs->fd);
}
ugs = ugs->next;
}
ucr->i_am_cheap = 0;
uwsgi_log("[%s pid %d] leaving cheap mode...\n", ucr->name, (int) uwsgi.mypid);
}
}
//unsubscribe
else {
struct uwsgi_subscribe_node *node = uwsgi_get_subscribe_node_by_name(ucr->subscriptions, usr.key, usr.keylen, usr.address, usr.address_len);
if (node && node->len) {
if (node->death_mark == 0)
uwsgi_log("[%s pid %d] %.*s => marking %.*s as failed\n", ucr->name, (int) uwsgi.mypid, (int) usr.keylen, usr.key, (int) usr.address_len, usr.address);
node->failcnt++;
node->death_mark = 1;
// check if i can remove the node
if (node->reference == 0) {
uwsgi_remove_subscribe_node(ucr->subscriptions, node);
}
if (ucr->cheap && !ucr->i_am_cheap && uwsgi_no_subscriptions(ucr->subscriptions)) {
uwsgi_gateway_go_cheap(ucr->name, ucr->queue, &ucr->i_am_cheap);
}
}
}
}
}
struct uwsgi_subscribe_node *uwsgi_get_subscribe_node(struct uwsgi_subscribe_slot **slot, char *key, uint16_t keylen) {
if (keylen > 0xff)
return NULL;
struct uwsgi_subscribe_slot *current_slot = uwsgi_get_subscribe_slot(slot, key, keylen);
if (!current_slot)
return NULL;
// slot found, move up in the list increasing hits
current_slot->hits++;
time_t now = uwsgi_now();
struct uwsgi_subscribe_node *node = current_slot->nodes;
while (node) {
// is the node alive ?
if (now - node->last_check > uwsgi.subscription_tolerance) {
if (node->death_mark == 0)
uwsgi_log("[uwsgi-subscription for pid %d] %.*s => marking %.*s as failed (no announce received in %d seconds)\n", (int) uwsgi.mypid, (int) keylen, key, (int) node->len, node->name, uwsgi.subscription_tolerance);
node->failcnt++;
node->death_mark = 1;
}
// do i need to remove the node ?
if (node->death_mark && node->reference == 0) {
// remove the node and move to next
struct uwsgi_subscribe_node *dead_node = node;
node = node->next;
// if the slot has been removed, return NULL;
if (uwsgi_remove_subscribe_node(slot, dead_node) == 1) {
return NULL;
}
continue;
}
struct uwsgi_subscribe_node *choosen_node = uwsgi.subscription_algo(current_slot, node);
if (choosen_node)
return choosen_node;
node = node->next;
}
return uwsgi.subscription_algo(current_slot, node);
}
void corerouter_close_peer(struct uwsgi_corerouter *ucr, struct corerouter_peer *peer) {
struct corerouter_session *cs = peer->session;
// manage subscription reference count
if (ucr->subscriptions && peer->un && peer->un->len > 0) {
// decrease reference count
#ifdef UWSGI_DEBUG
uwsgi_log("[1] node %.*s refcnt: %llu\n", peer->un->len, peer->un->name, peer->un->reference);
#endif
peer->un->reference--;
#ifdef UWSGI_DEBUG
uwsgi_log("[2] node %.*s refcnt: %llu\n", peer->un->len, peer->un->name, peer->un->reference);
#endif
}
if (peer->failed) {
if (peer->soopt) {
if (!ucr->quiet)
uwsgi_log("[uwsgi-%s] unable to connect() to node \"%.*s\" (%d retries): %s\n", ucr->short_name, (int) peer->instance_address_len, peer->instance_address, peer->retries, strerror(peer->soopt));
}
else if (peer->timed_out) {
if (peer->instance_address_len > 0) {
if (peer->connecting) {
if (!ucr->quiet)
uwsgi_log("[uwsgi-%s] unable to connect() to node \"%.*s\" (%d retries): timeout\n", ucr->short_name, (int) peer->instance_address_len, peer->instance_address, peer->retries);
}
}
}
// now check for dead nodes
if (ucr->subscriptions && peer->un && peer->un->len > 0) {
if (peer->un->death_mark == 0)
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) peer->key_len, peer->key, (int) peer->instance_address_len, peer->instance_address);
peer->un->failcnt++;
peer->un->death_mark = 1;
// check if i can remove the node
if (peer->un->reference == 0) {
uwsgi_remove_subscribe_node(ucr->subscriptions, peer->un);
}
if (ucr->cheap && !ucr->i_am_cheap && !ucr->fallback && uwsgi_no_subscriptions(ucr->subscriptions)) {
uwsgi_gateway_go_cheap(ucr->name, ucr->queue, &ucr->i_am_cheap);
}
}
else if (peer->static_node) {
peer->static_node->custom = uwsgi_now();
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) peer->key_len, peer->key, (int) peer->instance_address_len, peer->instance_address);
}
// check if the router supports the retry hook
if (!peer->can_retry) goto end;
if (peer->retries >= (size_t) ucr->max_retries) goto end;
peer->retries++;
// reset the peer
uwsgi_cr_peer_reset(peer);
// set new timeout
peer->timeout = cr_add_timeout(ucr, peer);
if (ucr->fallback) {
// ok let's try with the fallback nodes
if (!cs->fallback) {
cs->fallback = ucr->fallback;
}
else {
cs->fallback = cs->fallback->next;
if (!cs->fallback) goto end;
}
peer->instance_address = cs->fallback->value;
peer->instance_address_len = cs->fallback->len;
if (cs->retry(peer)) {
if (!peer->failed) goto end;
}
return;
}
peer->instance_address = NULL;
peer->instance_address_len = 0;
if (cs->retry(peer)) {
if (!peer->failed) goto end;
}
return;
}
end:
uwsgi_cr_peer_del(peer);
if (peer == cs->main_peer) {
cs->main_peer = NULL;
corerouter_close_session(ucr, cs);
}
else {
if (cs->can_keepalive == 0 && cs->wait_full_write == 0) {
corerouter_close_session(ucr, cs);
}
}
}
void corerouter_close_session(struct uwsgi_corerouter *ucr, struct corerouter_session *cr_session) {
if (cr_session->instance_fd != -1) {
close(cr_session->instance_fd);
ucr->cr_table[cr_session->instance_fd] = NULL;
}
if (ucr->subscriptions && cr_session->un && cr_session->un->len > 0) {
// decrease reference count
#ifdef UWSGI_DEBUG
uwsgi_log("[1] node %.*s refcnt: %llu\n", cr_session->un->len, cr_session->un->name, cr_session->un->reference);
#endif
cr_session->un->reference--;
#ifdef UWSGI_DEBUG
uwsgi_log("[2] node %.*s refcnt: %llu\n", cr_session->un->len, cr_session->un->name, cr_session->un->reference);
#endif
}
if (cr_session->instance_failed) {
if (cr_session->soopt) {
if (!ucr->quiet)
uwsgi_log("unable to connect() to uwsgi instance \"%.*s\": %s\n", (int) cr_session->instance_address_len, cr_session->instance_address, strerror(cr_session->soopt));
}
else if (cr_session->timed_out) {
if (cr_session->instance_address_len > 0) {
if (cr_session->status == COREROUTER_STATUS_CONNECTING) {
if (!ucr->quiet)
uwsgi_log("unable to connect() to uwsgi instance \"%.*s\": timeout\n", (int) cr_session->instance_address_len, cr_session->instance_address);
}
else if (cr_session->status == COREROUTER_STATUS_RESPONSE) {
uwsgi_log("timeout waiting for instance \"%.*s\"\n", (int) cr_session->instance_address_len, cr_session->instance_address);
}
}
}
// now check for dead nodes
if (ucr->subscriptions && cr_session->un && cr_session->un->len > 0) {
if (cr_session->un->death_mark == 0)
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) cr_session->hostname_len, cr_session->hostname, (int) cr_session->instance_address_len, cr_session->instance_address);
cr_session->un->failcnt++;
cr_session->un->death_mark = 1;
// check if i can remove the node
if (cr_session->un->reference == 0) {
uwsgi_remove_subscribe_node(ucr->subscriptions, cr_session->un);
}
if (ucr->cheap && !ucr->i_am_cheap && !ucr->fallback && uwsgi_no_subscriptions(ucr->subscriptions)) {
uwsgi_gateway_go_cheap(ucr->name, ucr->queue, &ucr->i_am_cheap);
}
}
else if (cr_session->static_node) {
cr_session->static_node->custom = uwsgi_now();
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) cr_session->hostname_len, cr_session->hostname, (int) cr_session->instance_address_len, cr_session->instance_address);
}
if (cr_session->tmp_socket_name) {
free(cr_session->tmp_socket_name);
cr_session->tmp_socket_name = NULL;
}
if (ucr->fallback) {
// ok let's try with the fallback nodes
if (!cr_session->fallback) {
cr_session->fallback = ucr->fallback;
}
else {
cr_session->fallback = cr_session->fallback->next;
if (!cr_session->fallback) goto end;
}
cr_session->instance_address = cr_session->fallback->value;
cr_session->instance_address_len = cr_session->fallback->len;
// reset error and timeout
cr_session->timeout = corerouter_reset_timeout(ucr, cr_session);
cr_session->timed_out = 0;
cr_session->soopt = 0;
// reset nodes
cr_session->un = NULL;
cr_session->static_node = NULL;
cr_session->pass_fd = is_unix(cr_session->instance_address, cr_session->instance_address_len);
cr_session->instance_fd = uwsgi_connectn(cr_session->instance_address, cr_session->instance_address_len, 0, 1);
if (cr_session->instance_fd < 0) {
cr_session->instance_failed = 1;
cr_session->soopt = errno;
corerouter_close_session(ucr, cr_session);
return;
}
ucr->cr_table[cr_session->instance_fd] = cr_session;
cr_session->status = COREROUTER_STATUS_CONNECTING;
ucr->cr_table[cr_session->instance_fd] = cr_session;
event_queue_add_fd_write(ucr->queue, cr_session->instance_fd);
return;
}
}
end:
if (cr_session->tmp_socket_name) {
free(cr_session->tmp_socket_name);
}
if (cr_session->buf_file)
fclose(cr_session->buf_file);
if (cr_session->buf_file_name) {
if (unlink(cr_session->buf_file_name)) {
uwsgi_error("unlink()");
}
free(cr_session->buf_file_name);
}
// could be used to free additional resources
if (cr_session->close)
cr_session->close(ucr, cr_session);
if (cr_session->keepalive) {
cr_session->keepalive = 0;
return;
}
close(cr_session->fd);
ucr->cr_table[cr_session->fd] = NULL;
cr_del_timeout(ucr, cr_session);
free(cr_session);
}
