static void corerouter_expire_timeouts(struct uwsgi_corerouter *ucr) {
time_t current = time(NULL);
struct uwsgi_rb_timer *urbt;
struct corerouter_session *cr_session;
for (;;) {
urbt = uwsgi_min_rb_timer(ucr->timeouts);
if (urbt == NULL)
return;
if (urbt->key <= current) {
cr_session = (struct corerouter_session *) urbt->data;
cr_session->timed_out = 1;
if (cr_session->retry) {
cr_session->retry = 0;
ucr->switch_events(ucr, cr_session, -1);
if (cr_session->retry) {
cr_del_timeout(ucr, cr_session);
cr_session->timeout = cr_add_fake_timeout(ucr, cr_session);
}
else {
cr_session->timeout = corerouter_reset_timeout(ucr, cr_session);
}
}
else {
corerouter_close_session(ucr, cr_session);
}
continue;
}
break;
}
}
void uwsgi_corerouter_loop(int id, void *data) {
int i;
struct uwsgi_corerouter *ucr = (struct uwsgi_corerouter *) data;
ucr->cr_stats_server = -1;
ucr->cr_table = uwsgi_malloc(sizeof(struct corerouter_session *) * uwsgi.max_fd);
for (i = 0; i < (int) uwsgi.max_fd; i++) {
ucr->cr_table[i] = NULL;
}
ucr->i_am_cheap = ucr->cheap;
void *events = uwsgi_corerouter_setup_event_queue(ucr, id);
if (ucr->has_subscription_sockets)
event_queue_add_fd_read(ucr->queue, ushared->gateways[id].internal_subscription_pipe[1]);
if (!ucr->socket_timeout)
ucr->socket_timeout = 30;
if (!ucr->static_node_gracetime)
ucr->static_node_gracetime = 30;
int i_am_the_first = 1;
for(i=0;i<id;i++) {
if (!strcmp(ushared->gateways[i].name, ucr->name)) {
i_am_the_first = 0;
break;
}
}
if (ucr->stats_server && i_am_the_first) {
char *tcp_port = strchr(ucr->stats_server, ':');
if (tcp_port) {
// disable deferred accept for this socket
int current_defer_accept = uwsgi.no_defer_accept;
uwsgi.no_defer_accept = 1;
ucr->cr_stats_server = bind_to_tcp(ucr->stats_server, uwsgi.listen_queue, tcp_port);
uwsgi.no_defer_accept = current_defer_accept;
}
else {
ucr->cr_stats_server = bind_to_unix(ucr->stats_server, uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
}
event_queue_add_fd_read(ucr->queue, ucr->cr_stats_server);
uwsgi_log("*** %s stats server enabled on %s fd: %d ***\n", ucr->short_name, ucr->stats_server, ucr->cr_stats_server);
}
if (ucr->use_socket) {
ucr->to_socket = uwsgi_get_socket_by_num(ucr->socket_num);
if (ucr->to_socket) {
// fix socket name_len
if (ucr->to_socket->name_len == 0 && ucr->to_socket->name) {
ucr->to_socket->name_len = strlen(ucr->to_socket->name);
}
}
}
if (!ucr->pb_base_dir) {
ucr->pb_base_dir = getenv("TMPDIR");
if (!ucr->pb_base_dir)
ucr->pb_base_dir = "/tmp";
}
int nevents;
time_t delta;
struct uwsgi_rb_timer *min_timeout;
int interesting_fd;
int new_connection;
if (ucr->pattern) {
init_magic_table(ucr->magic_table);
}
union uwsgi_sockaddr cr_addr;
socklen_t cr_addr_len = sizeof(struct sockaddr_un);
struct corerouter_session *cr_session;
ucr->mapper = uwsgi_cr_map_use_void;
if (ucr->use_cache) {
ucr->mapper = uwsgi_cr_map_use_cache;
}
else if (ucr->pattern) {
ucr->mapper = uwsgi_cr_map_use_pattern;
}
else if (ucr->has_subscription_sockets) {
ucr->mapper = uwsgi_cr_map_use_subscription;
}
else if (ucr->base) {
ucr->mapper = uwsgi_cr_map_use_base;
}
else if (ucr->code_string_code && ucr->code_string_function) {
ucr->mapper = uwsgi_cr_map_use_cs;
}
else if (ucr->to_socket) {
ucr->mapper = uwsgi_cr_map_use_to;
}
else if (ucr->static_nodes) {
ucr->mapper = uwsgi_cr_map_use_static_nodes;
}
else if (ucr->use_cluster) {
ucr->mapper = uwsgi_cr_map_use_cluster;
}
ucr->timeouts = uwsgi_init_rb_timer();
for (;;) {
min_timeout = uwsgi_min_rb_timer(ucr->timeouts);
if (min_timeout == NULL) {
delta = -1;
}
else {
delta = min_timeout->key - time(NULL);
if (delta <= 0) {
corerouter_expire_timeouts(ucr);
delta = 0;
}
}
if (uwsgi.master_process && ucr->harakiri > 0) {
ushared->gateways_harakiri[id] = 0;
}
nevents = event_queue_wait_multi(ucr->queue, delta, events, ucr->nevents);
if (uwsgi.master_process && ucr->harakiri > 0) {
ushared->gateways_harakiri[id] = time(NULL) + ucr->harakiri;
}
if (nevents == 0) {
corerouter_expire_timeouts(ucr);
}
for (i = 0; i < nevents; i++) {
interesting_fd = event_queue_interesting_fd(events, i);
struct uwsgi_gateway_socket *ugs = uwsgi.gateway_sockets;
int taken = 0;
while (ugs) {
if (ugs->gateway == &ushared->gateways[id] && interesting_fd == ugs->fd) {
if (!ugs->subscription) {
new_connection = accept(interesting_fd, (struct sockaddr *) &cr_addr, &cr_addr_len);
#ifdef UWSGI_EVENT_USE_PORT
event_queue_add_fd_read(ucr->queue, interesting_fd);
#endif
if (new_connection < 0) {
taken = 1;
break;
}
// set socket blocking mode, on non-linux platforms, clients get the server mode
#ifndef __linux__
if (!ugs->nb) {
uwsgi_socket_b(new_connection);
}
#else
if (ugs->nb) {
uwsgi_socket_nb(new_connection);
}
#endif
corerouter_alloc_session(ucr, ugs, new_connection, (struct sockaddr *) &cr_addr, cr_addr_len);
}
else if (ugs->subscription) {
uwsgi_corerouter_manage_subscription(ucr, id, ugs);
}
taken = 1;
break;
}
ugs = ugs->next;
}
if (taken) {
continue;
}
if (interesting_fd == ushared->gateways[id].internal_subscription_pipe[1]) {
uwsgi_corerouter_manage_internal_subscription(ucr, interesting_fd);
}
else if (interesting_fd == ucr->cr_stats_server) {
corerouter_send_stats(ucr);
}
else {
cr_session = ucr->cr_table[interesting_fd];
// something is going wrong...
if (cr_session == NULL)
continue;
if (event_queue_interesting_fd_has_error(events, i)) {
corerouter_close_session(ucr, cr_session);
continue;
}
cr_session->timeout = corerouter_reset_timeout(ucr, cr_session);
// mplementation specific cycle;
ucr->switch_events(ucr, cr_session, interesting_fd);
}
}
}
}
void corerouter_close_session(struct uwsgi_corerouter *ucr, struct corerouter_session *cr_session) {
if (cr_session->instance_fd != -1) {
close(cr_session->instance_fd);
ucr->cr_table[cr_session->instance_fd] = NULL;
}
if (ucr->subscriptions && cr_session->un && cr_session->un->len > 0) {
// decrease reference count
#ifdef UWSGI_DEBUG
uwsgi_log("[1] node %.*s refcnt: %llu\n", cr_session->un->len, cr_session->un->name, cr_session->un->reference);
#endif
cr_session->un->reference--;
#ifdef UWSGI_DEBUG
uwsgi_log("[2] node %.*s refcnt: %llu\n", cr_session->un->len, cr_session->un->name, cr_session->un->reference);
#endif
}
if (cr_session->instance_failed) {
if (cr_session->soopt) {
if (!ucr->quiet)
uwsgi_log("unable to connect() to uwsgi instance \"%.*s\": %s\n", (int) cr_session->instance_address_len, cr_session->instance_address, strerror(cr_session->soopt));
}
else if (cr_session->timed_out) {
if (cr_session->instance_address_len > 0) {
if (cr_session->status == COREROUTER_STATUS_CONNECTING) {
if (!ucr->quiet)
uwsgi_log("unable to connect() to uwsgi instance \"%.*s\": timeout\n", (int) cr_session->instance_address_len, cr_session->instance_address);
}
else if (cr_session->status == COREROUTER_STATUS_RESPONSE) {
uwsgi_log("timeout waiting for instance \"%.*s\"\n", (int) cr_session->instance_address_len, cr_session->instance_address);
}
}
}
// now check for dead nodes
if (ucr->subscriptions && cr_session->un && cr_session->un->len > 0) {
if (cr_session->un->death_mark == 0)
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) cr_session->hostname_len, cr_session->hostname, (int) cr_session->instance_address_len, cr_session->instance_address);
cr_session->un->failcnt++;
cr_session->un->death_mark = 1;
// check if i can remove the node
if (cr_session->un->reference == 0) {
uwsgi_remove_subscribe_node(ucr->subscriptions, cr_session->un);
}
if (ucr->cheap && !ucr->i_am_cheap && !ucr->fallback && uwsgi_no_subscriptions(ucr->subscriptions)) {
uwsgi_gateway_go_cheap(ucr->name, ucr->queue, &ucr->i_am_cheap);
}
}
else if (cr_session->static_node) {
cr_session->static_node->custom = uwsgi_now();
uwsgi_log("[uwsgi-%s] %.*s => marking %.*s as failed\n", ucr->short_name, (int) cr_session->hostname_len, cr_session->hostname, (int) cr_session->instance_address_len, cr_session->instance_address);
}
if (cr_session->tmp_socket_name) {
free(cr_session->tmp_socket_name);
cr_session->tmp_socket_name = NULL;
}
if (ucr->fallback) {
// ok let's try with the fallback nodes
if (!cr_session->fallback) {
cr_session->fallback = ucr->fallback;
}
else {
cr_session->fallback = cr_session->fallback->next;
if (!cr_session->fallback) goto end;
}
cr_session->instance_address = cr_session->fallback->value;
cr_session->instance_address_len = cr_session->fallback->len;
// reset error and timeout
cr_session->timeout = corerouter_reset_timeout(ucr, cr_session);
cr_session->timed_out = 0;
cr_session->soopt = 0;
// reset nodes
cr_session->un = NULL;
cr_session->static_node = NULL;
cr_session->pass_fd = is_unix(cr_session->instance_address, cr_session->instance_address_len);
cr_session->instance_fd = uwsgi_connectn(cr_session->instance_address, cr_session->instance_address_len, 0, 1);
if (cr_session->instance_fd < 0) {
cr_session->instance_failed = 1;
cr_session->soopt = errno;
corerouter_close_session(ucr, cr_session);
return;
}
ucr->cr_table[cr_session->instance_fd] = cr_session;
cr_session->status = COREROUTER_STATUS_CONNECTING;
ucr->cr_table[cr_session->instance_fd] = cr_session;
event_queue_add_fd_write(ucr->queue, cr_session->instance_fd);
return;
}
}
end:
if (cr_session->tmp_socket_name) {
free(cr_session->tmp_socket_name);
}
if (cr_session->buf_file)
fclose(cr_session->buf_file);
if (cr_session->buf_file_name) {
if (unlink(cr_session->buf_file_name)) {
uwsgi_error("unlink()");
}
free(cr_session->buf_file_name);
}
// could be used to free additional resources
if (cr_session->close)
cr_session->close(ucr, cr_session);
if (cr_session->keepalive) {
cr_session->keepalive = 0;
return;
}
close(cr_session->fd);
ucr->cr_table[cr_session->fd] = NULL;
cr_del_timeout(ucr, cr_session);
free(cr_session);
}
void uwsgi_corerouter_loop(int id, void *data) {
int i;
struct uwsgi_corerouter *ucr = (struct uwsgi_corerouter *) data;
ucr->cr_stats_server = -1;
ucr->cr_table = uwsgi_malloc(sizeof(struct corerouter_session *) * uwsgi.max_fd);
for (i = 0; i < (int) uwsgi.max_fd; i++) {
ucr->cr_table[i] = NULL;
}
ucr->i_am_cheap = ucr->cheap;
void *events = uwsgi_corerouter_setup_event_queue(ucr, id);
if (ucr->has_subscription_sockets)
event_queue_add_fd_read(ucr->queue, ushared->gateways[id].internal_subscription_pipe[1]);
if (!ucr->socket_timeout)
ucr->socket_timeout = 30;
if (!ucr->static_node_gracetime)
ucr->static_node_gracetime = 30;
int i_am_the_first = 1;
for(i=0;i<id;i++) {
if (!strcmp(ushared->gateways[i].name, ucr->name)) {
i_am_the_first = 0;
break;
}
}
if (ucr->stats_server && i_am_the_first) {
char *tcp_port = strchr(ucr->stats_server, ':');
if (tcp_port) {
// disable deferred accept for this socket
int current_defer_accept = uwsgi.no_defer_accept;
uwsgi.no_defer_accept = 1;
ucr->cr_stats_server = bind_to_tcp(ucr->stats_server, uwsgi.listen_queue, tcp_port);
uwsgi.no_defer_accept = current_defer_accept;
}
else {
ucr->cr_stats_server = bind_to_unix(ucr->stats_server, uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
}
event_queue_add_fd_read(ucr->queue, ucr->cr_stats_server);
uwsgi_log("*** %s stats server enabled on %s fd: %d ***\n", ucr->short_name, ucr->stats_server, ucr->cr_stats_server);
}
if (ucr->use_socket) {
ucr->to_socket = uwsgi_get_socket_by_num(ucr->socket_num);
if (ucr->to_socket) {
// fix socket name_len
if (ucr->to_socket->name_len == 0 && ucr->to_socket->name) {
ucr->to_socket->name_len = strlen(ucr->to_socket->name);
}
}
}
if (!ucr->pb_base_dir) {
ucr->pb_base_dir = getenv("TMPDIR");
if (!ucr->pb_base_dir)
ucr->pb_base_dir = "/tmp";
}
int nevents;
time_t delta;
struct uwsgi_rb_timer *min_timeout;
int interesting_fd;
int new_connection;
if (ucr->pattern) {
init_magic_table(ucr->magic_table);
}
union uwsgi_sockaddr cr_addr;
socklen_t cr_addr_len = sizeof(struct sockaddr_un);
struct corerouter_session *cr_session;
ucr->mapper = uwsgi_cr_map_use_void;
if (ucr->use_cache) {
ucr->mapper = uwsgi_cr_map_use_cache;
}
else if (ucr->pattern) {
ucr->mapper = uwsgi_cr_map_use_pattern;
}
else if (ucr->has_subscription_sockets) {
ucr->mapper = uwsgi_cr_map_use_subscription;
if (uwsgi.subscription_dotsplit) {
ucr->mapper = uwsgi_cr_map_use_subscription_dotsplit;
}
}
else if (ucr->base) {
ucr->mapper = uwsgi_cr_map_use_base;
}
else if (ucr->code_string_code && ucr->code_string_function) {
ucr->mapper = uwsgi_cr_map_use_cs;
}
else if (ucr->to_socket) {
ucr->mapper = uwsgi_cr_map_use_to;
}
else if (ucr->static_nodes) {
ucr->mapper = uwsgi_cr_map_use_static_nodes;
}
else if (ucr->use_cluster) {
ucr->mapper = uwsgi_cr_map_use_cluster;
}
ucr->timeouts = uwsgi_init_rb_timer();
for (;;) {
// set timeouts and harakiri
min_timeout = uwsgi_min_rb_timer(ucr->timeouts);
if (min_timeout == NULL) {
delta = -1;
}
else {
delta = min_timeout->key - uwsgi_now();
if (delta <= 0) {
corerouter_expire_timeouts(ucr);
delta = 0;
}
}
if (uwsgi.master_process && ucr->harakiri > 0) {
ushared->gateways_harakiri[id] = 0;
}
// wait for events
nevents = event_queue_wait_multi(ucr->queue, delta, events, ucr->nevents);
if (uwsgi.master_process && ucr->harakiri > 0) {
ushared->gateways_harakiri[id] = uwsgi_now() + ucr->harakiri;
}
if (nevents == 0) {
corerouter_expire_timeouts(ucr);
}
for (i = 0; i < nevents; i++) {
// get the interesting fd
interesting_fd = event_queue_interesting_fd(events, i);
// something bad happened
if (interesting_fd < 0) continue;
// check if the interesting_fd matches a gateway socket
struct uwsgi_gateway_socket *ugs = uwsgi.gateway_sockets;
int taken = 0;
while (ugs) {
if (ugs->gateway == &ushared->gateways[id] && interesting_fd == ugs->fd) {
if (!ugs->subscription) {
#if defined(__linux__) && defined(SOCK_NONBLOCK) && !defined(OBSOLETE_LINUX_KERNEL)
new_connection = accept4(interesting_fd, (struct sockaddr *) &cr_addr, &cr_addr_len, SOCK_NONBLOCK);
if (new_connection < 0) {
taken = 1;
break;
}
#else
new_connection = accept(interesting_fd, (struct sockaddr *) &cr_addr, &cr_addr_len);
if (new_connection < 0) {
taken = 1;
break;
}
// set socket in non-blocking mode, on non-linux platforms, clients get the server mode
#ifdef __linux__
uwsgi_socket_nb(new_connection);
#endif
#endif
corerouter_alloc_session(ucr, ugs, new_connection, (struct sockaddr *) &cr_addr, cr_addr_len);
}
else if (ugs->subscription) {
uwsgi_corerouter_manage_subscription(ucr, id, ugs);
}
taken = 1;
break;
}
ugs = ugs->next;
}
if (taken) {
continue;
}
// manage internal subscription
if (interesting_fd == ushared->gateways[id].internal_subscription_pipe[1]) {
uwsgi_corerouter_manage_internal_subscription(ucr, interesting_fd);
}
// manage a stats request
else if (interesting_fd == ucr->cr_stats_server) {
corerouter_send_stats(ucr);
}
else {
cr_session = ucr->cr_table[interesting_fd];
// something is going wrong...
if (cr_session == NULL)
continue;
// on error, destroy the session
if (event_queue_interesting_fd_has_error(events, i)) {
if (interesting_fd == cr_session->instance_fd) {
cr_session->instance_failed = 1;
}
corerouter_close_session(ucr, cr_session);
continue;
}
// set timeout
cr_session->timeout = corerouter_reset_timeout(ucr, cr_session);
// call event hook
ssize_t (*hook)(struct corerouter_session *) = NULL;
if (interesting_fd == cr_session->fd) {
if (event_queue_interesting_fd_is_read(events, i)) {
hook = cr_session->event_hook_read;
}
else if (event_queue_interesting_fd_is_write(events, i)) {
hook = cr_session->event_hook_write;
}
}
else if (interesting_fd == cr_session->instance_fd) {
if (event_queue_interesting_fd_is_read(events, i)) {
hook = cr_session->event_hook_instance_read;
}
else if (event_queue_interesting_fd_is_write(events, i)) {
hook = cr_session->event_hook_instance_write;
}
}
if (!hook) {
uwsgi_log("[uwsgi-corerouter] BUG, unexpected event received !!!\n");
corerouter_close_session(ucr, cr_session);
continue;
}
ssize_t ret = hook(cr_session);
// connection closed
if (ret == 0) {
corerouter_close_session(ucr, cr_session);
continue;
}
else if (ret < 0) {
if (errno == EINPROGRESS) continue;
corerouter_close_session(ucr, cr_session);
continue;
}
}
}
}
}
