void uwsgi_master_check_idle() {
static time_t last_request_timecheck = 0;
static uint64_t last_request_count = 0;
int i;
int waitpid_status;
if (!uwsgi.idle || uwsgi.status.is_cheap)
return;
uwsgi.current_time = uwsgi_now();
if (!last_request_timecheck)
last_request_timecheck = uwsgi.current_time;
// security check, stop the check if there are busy workers
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
if (uwsgi_worker_is_busy(i)) {
return;
}
}
}
if (last_request_count != uwsgi.workers[0].requests) {
last_request_timecheck = uwsgi.current_time;
last_request_count = uwsgi.workers[0].requests;
}
// a bit of over-engeneering to avoid clock skews
else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle)) {
uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck);
uwsgi.status.is_cheap = 1;
if (uwsgi.die_on_idle) {
if (uwsgi.has_emperor) {
char byte = 22;
if (write(uwsgi.emperor_fd, &byte, 1) != 1) {
uwsgi_error("write()");
kill_them_all(0);
}
}
else {
kill_them_all(0);
}
return;
}
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].cheaped = 1;
if (uwsgi.workers[i].pid == 0)
continue;
kill(uwsgi.workers[i].pid, SIGKILL);
if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) {
if (errno != ECHILD)
uwsgi_error("uwsgi_master_check_idle()/waitpid()");
}
}
uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1);
uwsgi_log("cheap mode enabled: waiting for socket connection...\n");
last_request_timecheck = 0;
}
}
struct uwsgi_stats *uwsgi_master_generate_stats() {
int i;
struct uwsgi_stats *us = uwsgi_stats_new(8192);
if (uwsgi_stats_keyval_comma(us, "version", UWSGI_VERSION))
goto end;
#ifdef __linux__
if (uwsgi_stats_keylong_comma(us, "listen_queue", (unsigned long long) uwsgi.shared->options[UWSGI_OPTION_BACKLOG_STATUS]))
goto end;
if (uwsgi_stats_keylong_comma(us, "listen_queue_errors", (unsigned long long) uwsgi.shared->options[UWSGI_OPTION_BACKLOG_ERRORS]))
goto end;
#endif
int signal_queue = 0;
if (ioctl(uwsgi.shared->worker_signal_pipe[1], FIONREAD, &signal_queue)) {
uwsgi_error("uwsgi_master_generate_stats() -> ioctl()\n");
}
if (uwsgi_stats_keylong_comma(us, "signal_queue", (unsigned long long) signal_queue))
goto end;
if (uwsgi_stats_keylong_comma(us, "load", (unsigned long long) uwsgi.shared->load))
goto end;
if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) getpid()))
goto end;
if (uwsgi_stats_keylong_comma(us, "uid", (unsigned long long) getuid()))
goto end;
if (uwsgi_stats_keylong_comma(us, "gid", (unsigned long long) getgid()))
goto end;
char *cwd = uwsgi_get_cwd();
if (uwsgi_stats_keyval_comma(us, "cwd", cwd)) {
free(cwd);
goto end;
}
free(cwd);
if (uwsgi.daemons) {
if (uwsgi_stats_key(us, "daemons"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
struct uwsgi_daemon *ud = uwsgi.daemons;
while (ud) {
if (uwsgi_stats_object_open(us))
goto end;
// allocate 2x the size of original command
// in case we need to escape all chars
char *cmd = uwsgi_malloc(strlen(ud->command)*2);
escape_json(ud->command, strlen(ud->command), cmd);
if (uwsgi_stats_keyval_comma(us, "cmd", cmd)) {
free(cmd);
goto end;
}
free(cmd);
if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) (ud->pid < 0) ? 0 : ud->pid))
goto end;
if (uwsgi_stats_keylong(us, "respawns", (unsigned long long) ud->respawns ? 0 : ud->respawns))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
if (ud->next) {
if (uwsgi_stats_comma(us))
goto end;
}
ud = ud->next;
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
}
if (uwsgi_stats_key(us, "locks"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
struct uwsgi_lock_item *uli = uwsgi.registered_locks;
while (uli) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keylong(us, uli->id, (unsigned long long) uli->pid))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
if (uli->next) {
if (uwsgi_stats_comma(us))
goto end;
}
uli = uli->next;
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
if (uwsgi.caches) {
if (uwsgi_stats_key(us, "caches"))
goto end;
if (uwsgi_stats_list_open(us)) goto end;
struct uwsgi_cache *uc = uwsgi.caches;
while(uc) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keyval_comma(us, "name", uc->name ? uc->name : "default"))
goto end;
if (uwsgi_stats_keyval_comma(us, "hash", uc->hash->name))
goto end;
if (uwsgi_stats_keylong_comma(us, "hashsize", (unsigned long long) uc->hashsize))
goto end;
if (uwsgi_stats_keylong_comma(us, "keysize", (unsigned long long) uc->keysize))
goto end;
if (uwsgi_stats_keylong_comma(us, "max_items", (unsigned long long) uc->max_items))
goto end;
if (uwsgi_stats_keylong_comma(us, "blocks", (unsigned long long) uc->blocks))
goto end;
if (uwsgi_stats_keylong_comma(us, "blocksize", (unsigned long long) uc->blocksize))
goto end;
if (uwsgi_stats_keylong_comma(us, "items", (unsigned long long) uc->n_items))
goto end;
if (uwsgi_stats_keylong_comma(us, "hits", (unsigned long long) uc->hits))
goto end;
if (uwsgi_stats_keylong_comma(us, "miss", (unsigned long long) uc->miss))
goto end;
if (uwsgi_stats_keylong_comma(us, "full", (unsigned long long) uc->full))
goto end;
if (uwsgi_stats_keylong(us, "last_modified_at", (unsigned long long) uc->last_modified_at))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
if (uc->next) {
if (uwsgi_stats_comma(us))
goto end;
}
uc = uc->next;
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
}
if (uwsgi_stats_key(us, "sockets"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
struct uwsgi_socket *uwsgi_sock = uwsgi.sockets;
while (uwsgi_sock) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keyval_comma(us, "name", uwsgi_sock->name))
goto end;
if (uwsgi_stats_keyval_comma(us, "proto", uwsgi_sock->proto_name ? uwsgi_sock->proto_name : "uwsgi"))
goto end;
if (uwsgi_stats_keylong_comma(us, "queue", (unsigned long long) uwsgi_sock->queue))
goto end;
if (uwsgi_stats_keylong_comma(us, "shared", (unsigned long long) uwsgi_sock->shared))
goto end;
if (uwsgi_stats_keylong(us, "can_offload", (unsigned long long) uwsgi_sock->can_offload))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
uwsgi_sock = uwsgi_sock->next;
if (uwsgi_sock) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
if (uwsgi_stats_key(us, "workers"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
for (i = 0; i < uwsgi.numproc; i++) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keylong_comma(us, "id", (unsigned long long) uwsgi.workers[i + 1].id))
goto end;
if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) uwsgi.workers[i + 1].pid))
goto end;
if (uwsgi_stats_keylong_comma(us, "requests", (unsigned long long) uwsgi.workers[i + 1].requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "delta_requests", (unsigned long long) uwsgi.workers[i + 1].delta_requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "exceptions", (unsigned long long) uwsgi_worker_exceptions(i + 1)))
goto end;
if (uwsgi_stats_keylong_comma(us, "harakiri_count", (unsigned long long) uwsgi.workers[i + 1].harakiri_count))
goto end;
if (uwsgi_stats_keylong_comma(us, "signals", (unsigned long long) uwsgi.workers[i + 1].signals))
goto end;
if (ioctl(uwsgi.workers[i + 1].signal_pipe[1], FIONREAD, &signal_queue)) {
uwsgi_error("uwsgi_master_generate_stats() -> ioctl()\n");
}
if (uwsgi_stats_keylong_comma(us, "signal_queue", (unsigned long long) signal_queue))
goto end;
if (uwsgi.workers[i + 1].cheaped) {
if (uwsgi_stats_keyval_comma(us, "status", "cheap"))
goto end;
}
else if (uwsgi.workers[i + 1].suspended && !uwsgi_worker_is_busy(i+1)) {
if (uwsgi_stats_keyval_comma(us, "status", "pause"))
goto end;
}
else {
if (uwsgi.workers[i + 1].sig) {
if (uwsgi_stats_keyvalnum_comma(us, "status", "sig", (unsigned long long) uwsgi.workers[i + 1].signum))
goto end;
}
else if (uwsgi_worker_is_busy(i+1)) {
if (uwsgi_stats_keyval_comma(us, "status", "busy"))
goto end;
}
else {
if (uwsgi_stats_keyval_comma(us, "status", "idle"))
goto end;
}
}
if (uwsgi_stats_keylong_comma(us, "rss", (unsigned long long) uwsgi.workers[i + 1].rss_size))
goto end;
if (uwsgi_stats_keylong_comma(us, "vsz", (unsigned long long) uwsgi.workers[i + 1].vsz_size))
goto end;
if (uwsgi_stats_keylong_comma(us, "running_time", (unsigned long long) uwsgi.workers[i + 1].running_time))
goto end;
if (uwsgi_stats_keylong_comma(us, "last_spawn", (unsigned long long) uwsgi.workers[i + 1].last_spawn))
goto end;
if (uwsgi_stats_keylong_comma(us, "respawn_count", (unsigned long long) uwsgi.workers[i + 1].respawn_count))
goto end;
if (uwsgi_stats_keylong_comma(us, "tx", (unsigned long long) uwsgi.workers[i + 1].tx))
goto end;
if (uwsgi_stats_keylong_comma(us, "avg_rt", (unsigned long long) uwsgi.workers[i + 1].avg_response_time))
goto end;
// applications list
if (uwsgi_stats_key(us, "apps"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
int j;
for (j = 0; j < uwsgi.workers[i + 1].apps_cnt; j++) {
struct uwsgi_app *ua = &uwsgi.workers[i + 1].apps[j];
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keylong_comma(us, "id", (unsigned long long) j))
goto end;
if (uwsgi_stats_keylong_comma(us, "modifier1", (unsigned long long) ua->modifier1))
goto end;
if (uwsgi_stats_keyvaln_comma(us, "mountpoint", ua->mountpoint, ua->mountpoint_len))
goto end;
if (uwsgi_stats_keylong_comma(us, "startup_time", ua->startup_time))
goto end;
if (uwsgi_stats_keylong_comma(us, "requests", ua->requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "exceptions", ua->exceptions))
goto end;
if (ua->chdir) {
if (uwsgi_stats_keyval(us, "chdir", ua->chdir))
goto end;
}
else {
if (uwsgi_stats_keyval(us, "chdir", ""))
goto end;
}
if (uwsgi_stats_object_close(us))
goto end;
if (j < uwsgi.workers[i + 1].apps_cnt - 1) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
// cores list
if (uwsgi_stats_key(us, "cores"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
for (j = 0; j < uwsgi.cores; j++) {
struct uwsgi_core *uc = &uwsgi.workers[i + 1].cores[j];
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keylong_comma(us, "id", (unsigned long long) j))
goto end;
if (uwsgi_stats_keylong_comma(us, "requests", (unsigned long long) uc->requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "static_requests", (unsigned long long) uc->static_requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "routed_requests", (unsigned long long) uc->routed_requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "offloaded_requests", (unsigned long long) uc->offloaded_requests))
goto end;
if (uwsgi_stats_keylong_comma(us, "write_errors", (unsigned long long) uc->write_errors))
goto end;
if (uwsgi_stats_keylong_comma(us, "in_request", (unsigned long long) uc->in_request))
goto end;
if (uwsgi_stats_key(us, "vars"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
if (uwsgi_stats_dump_vars(us, uc)) goto end;
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
if (j < uwsgi.cores - 1) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
if (i < uwsgi.numproc - 1) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
struct uwsgi_spooler *uspool = uwsgi.spoolers;
if (uspool) {
if (uwsgi_stats_comma(us))
goto end;
if (uwsgi_stats_key(us, "spoolers"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
while (uspool) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keyval_comma(us, "dir", uspool->dir))
goto end;
if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) uspool->pid))
goto end;
if (uwsgi_stats_keylong_comma(us, "tasks", (unsigned long long) uspool->tasks))
goto end;
if (uwsgi_stats_keylong_comma(us, "respawns", (unsigned long long) uspool->respawned))
goto end;
if (uwsgi_stats_keylong(us, "running", (unsigned long long) uspool->running))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
uspool = uspool->next;
if (uspool) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
}
struct uwsgi_cron *ucron = uwsgi.crons;
if (ucron) {
if (uwsgi_stats_comma(us))
goto end;
if (uwsgi_stats_key(us, "crons"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
while (ucron) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keyslong_comma(us, "minute", (long long) ucron->minute))
goto end;
if (uwsgi_stats_keyslong_comma(us, "hour", (long long) ucron->hour))
goto end;
if (uwsgi_stats_keyslong_comma(us, "day", (long long) ucron->day))
goto end;
if (uwsgi_stats_keyslong_comma(us, "month", (long long) ucron->month))
goto end;
if (uwsgi_stats_keyslong_comma(us, "week", (long long) ucron->week))
goto end;
char *cmd = uwsgi_malloc(strlen(ucron->command)*2);
escape_json(ucron->command, strlen(ucron->command), cmd);
if (uwsgi_stats_keyval_comma(us, "command", cmd)) {
free(cmd);
goto end;
}
free(cmd);
if (uwsgi_stats_keylong_comma(us, "unique", (unsigned long long) ucron->unique))
goto end;
#ifdef UWSGI_SSL
if (uwsgi_stats_keyval_comma(us, "legion", ucron->legion ? ucron->legion : ""))
goto end;
#endif
if (uwsgi_stats_keyslong_comma(us, "pid", (long long) ucron->pid))
goto end;
if (uwsgi_stats_keylong(us, "started_at", (unsigned long long) ucron->started_at))
goto end;
if (uwsgi_stats_object_close(us))
goto end;
ucron = ucron->next;
if (ucron) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
}
#ifdef UWSGI_SSL
struct uwsgi_legion *legion = NULL;
if (uwsgi.legions) {
if (uwsgi_stats_comma(us))
goto end;
if (uwsgi_stats_key(us, "legions"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
legion = uwsgi.legions;
while (legion) {
if (uwsgi_stats_object_open(us))
goto end;
if (uwsgi_stats_keyval_comma(us, "legion", legion->legion))
goto end;
if (uwsgi_stats_keyval_comma(us, "addr", legion->addr))
goto end;
if (uwsgi_stats_keyval_comma(us, "uuid", legion->uuid))
goto end;
if (uwsgi_stats_keylong_comma(us, "valor", (unsigned long long) legion->valor))
goto end;
if (uwsgi_stats_keylong_comma(us, "checksum", (unsigned long long) legion->checksum))
goto end;
if (uwsgi_stats_keylong_comma(us, "quorum", (unsigned long long) legion->quorum))
goto end;
if (uwsgi_stats_keylong_comma(us, "i_am_the_lord", (unsigned long long) legion->i_am_the_lord))
goto end;
if (uwsgi_stats_keylong_comma(us, "lord_valor", (unsigned long long) legion->lord_valor))
goto end;
if (uwsgi_stats_keyvaln_comma(us, "lord_uuid", legion->lord_uuid, 36))
goto end;
// legion nodes start
if (uwsgi_stats_key(us, "nodes"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
struct uwsgi_string_list *nodes = legion->nodes;
while (nodes) {
if (uwsgi_stats_str(us, nodes->value))
goto end;
nodes = nodes->next;
if (nodes) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
if (uwsgi_stats_comma(us))
goto end;
// legion members start
if (uwsgi_stats_key(us, "members"))
goto end;
if (uwsgi_stats_list_open(us))
goto end;
uwsgi_rlock(legion->lock);
struct uwsgi_legion_node *node = legion->nodes_head;
while (node) {
if (uwsgi_stats_object_open(us))
goto unlock_legion_mutex;
if (uwsgi_stats_keyvaln_comma(us, "name", node->name, node->name_len))
goto unlock_legion_mutex;
if (uwsgi_stats_keyval_comma(us, "uuid", node->uuid))
goto unlock_legion_mutex;
if (uwsgi_stats_keylong_comma(us, "valor", (unsigned long long) node->valor))
goto unlock_legion_mutex;
if (uwsgi_stats_keylong_comma(us, "checksum", (unsigned long long) node->checksum))
goto unlock_legion_mutex;
if (uwsgi_stats_keylong(us, "last_seen", (unsigned long long) node->last_seen))
goto unlock_legion_mutex;
if (uwsgi_stats_object_close(us))
goto unlock_legion_mutex;
node = node->next;
if (node) {
if (uwsgi_stats_comma(us))
goto unlock_legion_mutex;
}
}
uwsgi_rwunlock(legion->lock);
if (uwsgi_stats_list_close(us))
goto end;
// legion nodes end
if (uwsgi_stats_object_close(us))
goto end;
legion = legion->next;
if (legion) {
if (uwsgi_stats_comma(us))
goto end;
}
}
if (uwsgi_stats_list_close(us))
goto end;
}
#endif
if (uwsgi_stats_object_close(us))
goto end;
return us;
#ifdef UWSGI_SSL
unlock_legion_mutex:
if (legion)
uwsgi_rwunlock(legion->lock);
#endif
end:
free(us->base);
free(us);
return NULL;
}
int uwsgi_cheaper_algo_spare(void) {
int i;
static uint64_t overload_count = 0;
static uint64_t idle_count = 0;
// step 1 -> count the number of busy workers
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
// if a non-busy worker is found, the overload_count is decremented and stop the cycle
if (uwsgi_worker_is_busy(i) == 0) {
if (overload_count > 0)
overload_count--;
goto healthy;
}
}
}
overload_count++;
idle_count = 0;
healthy:
// are we overloaded ?
if (overload_count > uwsgi.cheaper_overload) {
#ifdef UWSGI_DEBUG
uwsgi_log("overloaded !!!\n");
#endif
// activate the first available worker (taking step into account)
int decheaped = 0;
// search for cheaped workers
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 1 && uwsgi.workers[i].pid == 0) {
decheaped++;
if (decheaped >= uwsgi.cheaper_step)
break;
}
}
// reset overload
overload_count = 0;
// return the maximum number of workers to spawn
return decheaped;
}
// we are no more overloaded
else if (overload_count == 0) {
// how many active workers ?
int active_workers = 0;
int busy_workers = 0;
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
active_workers++;
if (uwsgi_worker_is_busy(i) == 1)
busy_workers++;
}
}
#ifdef UWSGI_DEBUG
uwsgi_log("active workers %d busy_workers %d\n", active_workers, busy_workers);
#endif
// special condition: uwsgi.cheaper running workers and 1 free
if (active_workers > busy_workers && active_workers - busy_workers == 1) {
#ifdef UWSGI_DEBUG
uwsgi_log("stable status: 1 spare worker\n");
#endif
return 0;
}
idle_count++;
if (active_workers > uwsgi.cheaper_count && idle_count % uwsgi.cheaper_overload == 0) {
// we are in "cheap them all"
return -1;
}
}
return 0;
}
int cheaper_busyness_algo(int can_spawn) {
int i;
// we use microseconds
uint64_t t = uwsgi.cheaper_overload*1000000;
int active_workers = 0;
uint64_t total_busyness = 0;
uint64_t avg_busyness = 0;
for (i = 0; i < uwsgi.numproc; i++) {
if (uwsgi.workers[i+1].cheaped == 0 && uwsgi.workers[i+1].pid > 0) {
active_workers++;
uwsgi_cheaper_busyness_global.was_busy[i] += uwsgi_worker_is_busy(i+1);
} else {
uwsgi_cheaper_busyness_global.was_busy[i] = 0;
}
}
#ifdef __linux__
int backlog = uwsgi.shared->backlog;
#endif
uint64_t now = uwsgi_micros();
if (now - uwsgi_cheaper_busyness_global.tcheck >= t) {
uwsgi_cheaper_busyness_global.tcheck = now;
for (i = 0; i < uwsgi.numproc; i++) {
if (uwsgi.workers[i+1].cheaped == 0 && uwsgi.workers[i+1].pid > 0) {
uint64_t percent = (( (uwsgi.workers[i+1].running_time-uwsgi_cheaper_busyness_global.last_values[i])*100)/t);
if (percent > 100) {
percent = 100;
}
else if (uwsgi.workers[i+1].running_time-uwsgi_cheaper_busyness_global.last_values[i] == 0 && percent == 0 && uwsgi_cheaper_busyness_global.was_busy[i] > 0) {
// running_time did not change but workers were busy
// this means that workers had response times > busyness check interval
if (uwsgi_cheaper_busyness_global.verbose) {
uwsgi_log("[busyness] worker %d was busy %d time(s) in last cycle but no request was completed during this time, marking as 100%% busy\n",
i+1, uwsgi_cheaper_busyness_global.was_busy[i]);
}
percent = 100;
}
uwsgi_cheaper_busyness_global.was_busy[i] = 0;
total_busyness += percent;
if (uwsgi_cheaper_busyness_global.verbose && active_workers > 1)
uwsgi_log("[busyness] worker nr %d %llus average busyness is at %llu%%\n",
i+1, uwsgi.cheaper_overload, percent);
if (uwsgi.has_metrics) {
// update metrics
uwsgi_wlock(uwsgi.metrics_lock);
uwsgi_cheaper_busyness_global.current_busyness[i] = percent;
uwsgi_rwunlock(uwsgi.metrics_lock);
}
}
uwsgi_cheaper_busyness_global.last_values[i] = uwsgi.workers[i+1].running_time;
}
avg_busyness = (active_workers ? total_busyness / active_workers : 0);
if (uwsgi.has_metrics) {
uwsgi_wlock(uwsgi.metrics_lock);
uwsgi_cheaper_busyness_global.total_avg_busyness = avg_busyness;
uwsgi_rwunlock(uwsgi.metrics_lock);
}
if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] %ds average busyness of %d worker(s) is at %d%%\n",
(int) uwsgi.cheaper_overload, (int) active_workers, (int) avg_busyness);
if (avg_busyness > uwsgi_cheaper_busyness_global.busyness_max) {
// we need to reset this to 0 since this is not idle cycle
uwsgi_cheaper_busyness_global.tolerance_counter = 0;
int decheaped = 0;
if (can_spawn) {
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 1 && uwsgi.workers[i].pid == 0) {
decheaped++;
if (decheaped >= uwsgi.cheaper_step) break;
}
}
}
if (decheaped > 0) {
// store information that we just spawned new workers
uwsgi_cheaper_busyness_global.last_action = 1;
// calculate number of seconds since last worker was cheaped
if ((now - uwsgi_cheaper_busyness_global.last_cheaped)/uwsgi.cheaper_overload/1000000 <= uwsgi_cheaper_busyness_global.cheap_multi) {
// worker was cheaped and then spawned back in less than current multiplier*cheaper_overload seconds
// we will increase the multiplier so that next time worker will need to wait longer before being cheaped
uwsgi_cheaper_busyness_global.cheap_multi += uwsgi_cheaper_busyness_global.penalty;
uwsgi_log("[busyness] worker(s) respawned to fast, increasing cheaper multiplier to %llu (+%llu)\n",
uwsgi_cheaper_busyness_global.cheap_multi, uwsgi_cheaper_busyness_global.penalty);
} else {
decrease_multi();
}
set_next_cheap_time();
uwsgi_log("[busyness] %llus average busyness is at %llu%%, will spawn %d new worker(s)\n",
uwsgi.cheaper_overload, avg_busyness, decheaped);
} else {
uwsgi_log("[busyness] %llus average busyness is at %llu%% but we already started maximum number of workers available with current limits (%d)\n",
uwsgi.cheaper_overload, avg_busyness, active_workers);
}
// return the maximum number of workers to spawn
return decheaped;
#ifdef __linux__
} else if (can_spawn && backlog > uwsgi_cheaper_busyness_global.backlog_alert && active_workers < uwsgi.numproc) {
return spawn_emergency_worker(backlog);
#endif
} else if (avg_busyness < uwsgi_cheaper_busyness_global.busyness_min) {
// with only 1 worker running there is no point in doing all that magic
if (active_workers == 1) return 0;
// we need to reset this to 0 since this is not idle cycle
uwsgi_cheaper_busyness_global.tolerance_counter = 0;
if (active_workers > uwsgi.cheaper_count) {
// cheap a worker if too much are running
if (now >= uwsgi_cheaper_busyness_global.next_cheap) {
// lower cheaper multiplier if this is subsequent cheap
if (uwsgi_cheaper_busyness_global.last_action == 2) decrease_multi();
set_next_cheap_time();
uwsgi_log("[busyness] %llus average busyness is at %llu%%, cheap one of %d running workers\n",
uwsgi.cheaper_overload, avg_busyness, (int) active_workers);
// store timestamp
uwsgi_cheaper_busyness_global.last_cheaped = uwsgi_micros();
// store information that last action performed was cheaping worker
uwsgi_cheaper_busyness_global.last_action = 2;
if (uwsgi_cheaper_busyness_global.emergency_workers > 0)
uwsgi_cheaper_busyness_global.emergency_workers--;
return -1;
} else if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] need to wait %llu more second(s) to cheap worker\n", (uwsgi_cheaper_busyness_global.next_cheap - now)/1000000);
}
} else {
// with only 1 worker running there is no point in doing all that magic
if (active_workers == 1) return 0;
if (uwsgi_cheaper_busyness_global.emergency_workers > 0)
// we had emergency workers running and we went down to the busyness
// level that is high enough to slow down cheaping workers at extra speed
uwsgi_cheaper_busyness_global.emergency_workers--;
// we have min <= busyness <= max we need to check what happened before
uwsgi_cheaper_busyness_global.tolerance_counter++;
if (uwsgi_cheaper_busyness_global.tolerance_counter >= 3) {
// we had three or more cycles when min <= busyness <= max, lets reset the cheaper timer
// this is to prevent workers from being cheaped if we had idle cycles for almost all
// time needed to cheap them, than a lot min<busy<max when we do not reset timer
// and then another idle cycle than would trigger cheaping
if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] %llus average busyness is at %llu%%, %llu non-idle cycle(s), resetting cheaper timer\n",
uwsgi.cheaper_overload, avg_busyness, uwsgi_cheaper_busyness_global.tolerance_counter);
set_next_cheap_time();
} else {
// we had < 3 idle cycles in a row so we won't reset idle timer yet since this might be just short load spike
// but we need to add cheaper-overload seconds to the cheaper timer so this cycle isn't counted as idle
if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] %llus average busyness is at %llu%%, %llu non-idle cycle(s), adjusting cheaper timer\n",
uwsgi.cheaper_overload, avg_busyness, uwsgi_cheaper_busyness_global.tolerance_counter);
uwsgi_cheaper_busyness_global.next_cheap += uwsgi.cheaper_overload*1000000;
}
}
}
#ifdef __linux__
else if (can_spawn && backlog > uwsgi_cheaper_busyness_global.backlog_alert && active_workers < uwsgi.numproc) {
// we check for backlog overload every cycle
return spawn_emergency_worker(backlog);
}
else if (backlog > 0) {
if (uwsgi_cheaper_busyness_global.backlog_is_nonzero) {
// backlog was > 0 last time, check timestamp and spawn workers if needed
if (can_spawn && (now - uwsgi_cheaper_busyness_global.backlog_nonzero_since)/1000000 >= uwsgi_cheaper_busyness_global.backlog_nonzero_alert) {
uwsgi_log("[busyness] backlog was non-zero for %llu second(s), spawning new worker(s)\n", (now - uwsgi_cheaper_busyness_global.backlog_nonzero_since)/1000000);
uwsgi_cheaper_busyness_global.backlog_nonzero_since = now;
return spawn_emergency_worker(backlog);
}
}
else {
// this is first > 0 pass, setup timer
if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] backlog is starting to fill (%d)\n", backlog);
uwsgi_cheaper_busyness_global.backlog_is_nonzero = 1;
uwsgi_cheaper_busyness_global.backlog_nonzero_since = now;
}
}
else if (uwsgi_cheaper_busyness_global.backlog_is_nonzero) {
if (uwsgi_cheaper_busyness_global.verbose)
uwsgi_log("[busyness] backlog is now empty\n");
uwsgi_cheaper_busyness_global.backlog_is_nonzero = 0;
}
#endif
return 0;
}
static void carbon_push_stats(int retry_cycle, time_t now) {
struct carbon_server_list *usl = u_carbon.servers_data;
if (!u_carbon.servers_data) return;
int i;
int fd;
int wok;
for (i = 0; i < uwsgi.numproc; i++) {
u_carbon.current_busyness_values[i] = uwsgi.workers[i+1].running_time - u_carbon.last_busyness_values[i];
u_carbon.last_busyness_values[i] = uwsgi.workers[i+1].running_time;
u_carbon.was_busy[i-1] += uwsgi_worker_is_busy(i+1);
}
u_carbon.need_retry = 0;
while(usl) {
if (retry_cycle && usl->healthy)
// skip healthy servers during retry cycle
goto nxt;
if (retry_cycle && usl->healthy == 0)
uwsgi_log("[carbon] Retrying failed server at %s (%d)\n", usl->value, usl->errors);
if (!retry_cycle) {
usl->healthy = 1;
usl->errors = 0;
}
fd = uwsgi_connect(usl->value, u_carbon.timeout, 0);
if (fd < 0) {
uwsgi_log("[carbon] Could not connect to carbon server at %s\n", usl->value);
if (usl->errors < u_carbon.max_retries) {
u_carbon.need_retry = 1;
u_carbon.next_retry = uwsgi_now() + u_carbon.retry_delay;
} else {
uwsgi_log("[carbon] Maximum number of retries for %s (%d)\n",
usl->value, u_carbon.max_retries);
usl->healthy = 0;
usl->errors = 0;
}
usl->healthy = 0;
usl->errors++;
goto nxt;
}
// put the socket in non-blocking mode
uwsgi_socket_nb(fd);
unsigned long long total_rss = 0;
unsigned long long total_vsz = 0;
unsigned long long total_tx = 0;
unsigned long long total_avg_rt = 0; // total avg_rt
unsigned long long avg_rt = 0; // per worker avg_rt reported to carbon
unsigned long long active_workers = 0; // number of workers used to calculate total avg_rt
unsigned long long total_busyness = 0;
unsigned long long total_avg_busyness = 0;
unsigned long long worker_busyness = 0;
unsigned long long total_harakiri = 0;
wok = carbon_write(fd, "%s%s.%s.requests %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) uwsgi.workers[0].requests, (unsigned long long) now);
if (!wok) goto clear;
for(i=1;i<=uwsgi.numproc;i++) {
total_tx += uwsgi.workers[i].tx;
total_harakiri += uwsgi.workers[i].harakiri_count;
if (uwsgi.workers[i].cheaped) {
// also if worker is cheaped than we report its average response time as zero, sending last value might be confusing
avg_rt = 0;
worker_busyness = 0;
}
else {
// global average response time is calculated from active/idle workers, cheaped workers are excluded, otherwise it is not accurate
avg_rt = uwsgi.workers[i].avg_response_time;
active_workers++;
total_avg_rt += uwsgi.workers[i].avg_response_time;
// calculate worker busyness
if (u_carbon.current_busyness_values[i-1] == 0 && u_carbon.was_busy[i-1]) {
worker_busyness = 100;
}
else {
worker_busyness = ((u_carbon.current_busyness_values[i-1]*100) / (u_carbon.freq*1000000));
if (worker_busyness > 100) worker_busyness = 100;
}
total_busyness += worker_busyness;
u_carbon.was_busy[i-1] = 0;
if (uwsgi.shared->options[UWSGI_OPTION_MEMORY_DEBUG] == 1 || uwsgi.force_get_memusage) {
// only running workers are counted in total memory stats and if memory-report option is enabled
total_rss += uwsgi.workers[i].rss_size;
total_vsz += uwsgi.workers[i].vsz_size;
}
}
//skip per worker metrics when disabled
if (u_carbon.no_workers) continue;
wok = carbon_write(fd, "%s%s.%s.worker%d.requests %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) uwsgi.workers[i].requests, (unsigned long long) now);
if (!wok) goto clear;
if (uwsgi.shared->options[UWSGI_OPTION_MEMORY_DEBUG] == 1 || uwsgi.force_get_memusage) {
wok = carbon_write(fd, "%s%s.%s.worker%d.rss_size %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) uwsgi.workers[i].rss_size, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.worker%d.vsz_size %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) uwsgi.workers[i].vsz_size, (unsigned long long) now);
if (!wok) goto clear;
}
wok = carbon_write(fd, "%s%s.%s.worker%d.avg_rt %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) avg_rt, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.worker%d.tx %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) uwsgi.workers[i].tx, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.worker%d.busyness %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) worker_busyness, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.worker%d.harakiri %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, i, (unsigned long long) uwsgi.workers[i].harakiri_count, (unsigned long long) now);
if (!wok) goto clear;
}
if (uwsgi.shared->options[UWSGI_OPTION_MEMORY_DEBUG] == 1 || uwsgi.force_get_memusage) {
wok = carbon_write(fd, "%s%s.%s.rss_size %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) total_rss, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.vsz_size %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) total_vsz, (unsigned long long) now);
if (!wok) goto clear;
}
wok = carbon_write(fd, "%s%s.%s.avg_rt %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) (active_workers ? total_avg_rt / active_workers : 0), (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.tx %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) total_tx, (unsigned long long) now);
if (!wok) goto clear;
if (active_workers > 0) {
total_avg_busyness = total_busyness / active_workers;
if (total_avg_busyness > 100) total_avg_busyness = 100;
} else {
total_avg_busyness = 0;
}
wok = carbon_write(fd, "%s%s.%s.busyness %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) total_avg_busyness, (unsigned long long) now);
if (!wok) goto clear;
wok = carbon_write(fd, "%s%s.%s.active_workers %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) active_workers, (unsigned long long) now);
if (!wok) goto clear;
if (uwsgi.cheaper) {
wok = carbon_write(fd, "%s%s.%s.cheaped_workers %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) uwsgi.numproc - active_workers, (unsigned long long) now);
if (!wok) goto clear;
}
wok = carbon_write(fd, "%s%s.%s.harakiri %llu %llu\n", u_carbon.root_node, u_carbon.hostname, u_carbon.id, (unsigned long long) total_harakiri, (unsigned long long) now);
if (!wok) goto clear;
usl->healthy = 1;
usl->errors = 0;
clear:
close(fd);
nxt:
usl = usl->next;
}
if (!retry_cycle) u_carbon.last_update = uwsgi_now();
if (u_carbon.need_retry)
// timeouts and retries might cause additional lags in carbon cycles, we will adjust timer to fix that
u_carbon.last_update -= u_carbon.timeout;
}