void suspend_resume_them_all(int signum) {
int i;
int suspend = 0;
if (uwsgi.workers[0].suspended == 1) {
uwsgi_log_verbose("*** (SIGTSTP received) resuming workers ***\n");
uwsgi.workers[0].suspended = 0;
}
else {
uwsgi_log_verbose("*** PAUSE (press start to resume, if you do not have a joypad send SIGTSTP) ***\n");
suspend = 1;
uwsgi.workers[0].suspended = 1;
}
// subscribe/unsubscribe if needed
uwsgi_subscribe_all(suspend, 1);
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].suspended = suspend;
if (uwsgi.workers[i].pid > 0) {
if (kill(uwsgi.workers[i].pid, SIGTSTP)) {
uwsgi_error("kill()");
}
}
}
}
PyObject *py_uwsgi_gevent_graceful(PyObject *self, PyObject *args) {
uwsgi_log("Gracefully killing worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
uwsgi_log_verbose("stopping gevent signals watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
PyObject_CallMethod(ugevent.my_signal_watcher, "stop", NULL);
PyObject_CallMethod(ugevent.signal_watcher, "stop", NULL);
uwsgi_log_verbose("stopping gevent sockets watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int i,count = uwsgi_count_sockets(uwsgi.sockets);
for(i=0; i<count; i++) {
PyObject_CallMethod(ugevent.watchers[i], "stop", NULL);
}
uwsgi_log_verbose("main gevent watchers stopped for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int running_cores = 0;
for(i=0; i<uwsgi.async; i++) {
if (uwsgi.workers[uwsgi.mywid].cores[i].in_request) {
struct wsgi_request *wsgi_req = &uwsgi.workers[uwsgi.mywid].cores[i].req;
uwsgi_log_verbose("worker %d (pid: %d) core %d is managing \"%.*s %.*s\" for %.*s\n", uwsgi.mywid, uwsgi.mypid, i,
wsgi_req->method_len, wsgi_req->method, wsgi_req->uri_len, wsgi_req->uri,
wsgi_req->remote_addr_len, wsgi_req->remote_addr);
running_cores++;
}
}
if (running_cores > 0) {
uwsgi_log_verbose("waiting for %d running requests on worker %d (pid: %d)...\n", running_cores, uwsgi.mywid, uwsgi.mypid);
}
Py_INCREF(Py_None);
return Py_None;
}
void uwsgi_mule_handler() {
ssize_t len;
uint8_t uwsgi_signal;
int rlen;
int interesting_fd;
// this must be configurable
char message[65536];
int mule_queue = event_queue_init();
event_queue_add_fd_read(mule_queue, uwsgi.signal_socket);
event_queue_add_fd_read(mule_queue, uwsgi.my_signal_socket);
event_queue_add_fd_read(mule_queue, uwsgi.mules[uwsgi.muleid - 1].queue_pipe[1]);
event_queue_add_fd_read(mule_queue, uwsgi.shared->mule_queue_pipe[1]);
uwsgi_mule_add_farm_to_queue(mule_queue);
for (;;) {
rlen = event_queue_wait(mule_queue, -1, &interesting_fd);
if (rlen <= 0) {
continue;
}
if (interesting_fd == uwsgi.signal_socket || interesting_fd == uwsgi.my_signal_socket || farm_has_signaled(interesting_fd)) {
len = read(interesting_fd, &uwsgi_signal, 1);
if (len <= 0) {
uwsgi_log_verbose("uWSGI mule %d braying: my master died, i will follow him...\n", uwsgi.muleid);
end_me(0);
}
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("master sent signal %d to mule %d\n", uwsgi_signal, uwsgi.muleid);
#endif
if (uwsgi_signal_handler(uwsgi_signal)) {
uwsgi_log_verbose("error managing signal %d on mule %d\n", uwsgi_signal, uwsgi.muleid);
}
}
else if (interesting_fd == uwsgi.mules[uwsgi.muleid - 1].queue_pipe[1] || interesting_fd == uwsgi.shared->mule_queue_pipe[1] || farm_has_msg(interesting_fd)) {
len = read(interesting_fd, message, 65536);
if (len < 0) {
uwsgi_error("read()");
}
else {
int i, found = 0;
for (i = 0; i < 256; i++) {
if (uwsgi.p[i]->mule_msg) {
if (uwsgi.p[i]->mule_msg(message, len)) {
found = 1;
break;
}
}
}
if (!found)
uwsgi_log("*** mule %d received a %ld bytes message ***\n", uwsgi.muleid, (long) len);
}
}
}
}
PyObject *py_uwsgi_gevent_signal_handler(PyObject * self, PyObject * args) {
uint8_t uwsgi_signal;
int signal_socket;
if (!PyArg_ParseTuple(args, "i:uwsgi_gevent_signal_handler", &signal_socket)) {
return NULL;
}
if (read(signal_socket, &uwsgi_signal, 1) <= 0) {
if (uwsgi.no_orphans) {
uwsgi_log_verbose("uWSGI worker %d screams: UAAAAAAH my master died, i will follow him...\n", uwsgi.mywid);
end_me(0);
}
// close the socket to end the mess...from now on the worker is alone (no master)
else close(signal_socket);
}
else {
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("master sent signal %d to worker %d\n", uwsgi_signal, uwsgi.mywid);
#endif
if (uwsgi_signal_handler(uwsgi_signal)) {
uwsgi_log_verbose("error managing signal %d on worker %d\n", uwsgi_signal, uwsgi.mywid);
}
}
Py_INCREF(Py_None);
return Py_None;
}
void suspend_resume_them_all(int signum) {
int i;
int suspend = 0;
if (uwsgi.workers[0].suspended == 1) {
uwsgi_log_verbose("*** (SIGTSTP received) resuming workers ***\n");
uwsgi.workers[0].suspended = 0;
}
else {
uwsgi_log_verbose("*** PAUSE (press start to resume, if you do not have a joypad send SIGTSTP) ***\n");
suspend = 1;
uwsgi.workers[0].suspended = 1;
}
// subscribe/unsubscribe if needed
struct uwsgi_string_list *subscriptions = uwsgi.subscriptions;
while (subscriptions) {
uwsgi_log("%s %s\n", suspend ? "unsubscribing from" : "subscribing to", subscriptions->value);
uwsgi_subscribe(subscriptions->value, suspend);
subscriptions = subscriptions->next;
}
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].suspended = suspend;
if (uwsgi.workers[i].pid > 0) {
if (kill(uwsgi.workers[i].pid, SIGTSTP)) {
uwsgi_error("kill()");
}
}
}
}
void uwsgi_receive_signal(int fd, char *name, int id) {
uint8_t uwsgi_signal;
ssize_t ret = read(fd, &uwsgi_signal, 1);
if (ret == 0) {
goto destroy;
}
else if (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK) {
uwsgi_error("[uwsgi-signal] read()");
goto destroy;
}
else if (ret > 0) {
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("master sent signal %d to %s %d\n", uwsgi_signal, name, id);
#endif
if (uwsgi_signal_handler(uwsgi_signal)) {
uwsgi_log_verbose("error managing signal %d on %s %d\n", uwsgi_signal, name, id);
}
}
return;
destroy:
// better to kill the whole worker...
uwsgi_log_verbose("uWSGI %s %d screams: UAAAAAAH my master disconnected: i will kill myself !!!\n", name, id);
end_me(0);
}
void uwsgi_chain_reload() {
if (!uwsgi.status.chain_reloading) {
uwsgi_log_verbose("chain reload starting...\n");
uwsgi.status.chain_reloading = 1;
}
else {
uwsgi_log_verbose("chain reload already running...\n");
}
}
PyObject *py_uwsgi_gevent_graceful(PyObject *self, PyObject *args) {
int running_cores = 0;
int rounds = 0;
uwsgi_log("Gracefully killing worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
uwsgi_log_verbose("stopping gevent signals watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
PyObject_CallMethod(ugevent.my_signal_watcher, "stop", NULL);
PyObject_CallMethod(ugevent.signal_watcher, "stop", NULL);
uwsgi_log_verbose("stopping gevent sockets watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int i,count = uwsgi_count_sockets(uwsgi.sockets);
for(i=0;i<count;i++) {
PyObject_CallMethod(ugevent.watchers[i], "stop", NULL);
}
uwsgi_log_verbose("main gevent watchers stopped for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
retry:
running_cores = 0;
for(i=0;i<uwsgi.async;i++) {
if (uwsgi.workers[uwsgi.mywid].cores[i].in_request) {
struct wsgi_request *wsgi_req = &uwsgi.workers[uwsgi.mywid].cores[i].req;
if (!rounds) {
uwsgi_log_verbose("worker %d (pid: %d) core %d is managing \"%.*s %.*s\" for %.*s\n", uwsgi.mywid, uwsgi.mypid, i,
wsgi_req->method_len, wsgi_req->method, wsgi_req->uri_len, wsgi_req->uri,
wsgi_req->remote_addr_len, wsgi_req->remote_addr);
}
running_cores++;
}
}
if (running_cores > 0) {
uwsgi_log_verbose("waiting for %d running requests on worker %d (pid: %d)...\n", running_cores, uwsgi.mywid, uwsgi.mypid);
PyObject *gevent_sleep_args = PyTuple_New(1);
PyTuple_SetItem(gevent_sleep_args, 0, PyInt_FromLong(1));
PyObject *gswitch = python_call(ugevent.greenlet_switch, gevent_sleep_args, 0, NULL);
Py_DECREF(gswitch);
Py_DECREF(gevent_sleep_args);
rounds++;
goto retry;
}
if (!ugevent.wait_for_hub) {
PyObject_CallMethod(ugevent.ctrl_gl, "kill", NULL);
}
Py_INCREF(Py_None);
return Py_None;
}
static void greenlet_schedule_to_req() {
int id = uwsgi.wsgi_req->async_id;
uint8_t modifier1 = uwsgi.wsgi_req->uh->modifier1;
// ensure gil
UWSGI_GET_GIL
if (!uwsgi.wsgi_req->suspended) {
ugl.gl[id] = PyGreenlet_New(ugl.callable, NULL);
PyObject_SetAttrString((PyObject *)ugl.gl[id], "uwsgi_wsgi_req", PyLong_FromLong((long) uwsgi.wsgi_req));
uwsgi.wsgi_req->suspended = 1;
}
// call it in the main core
if (is_not_python(modifier1) && uwsgi.p[modifier1]->suspend) {
uwsgi.p[modifier1]->suspend(NULL);
}
PyObject *ret = PyGreenlet_Switch(ugl.gl[id], NULL, NULL);
if (!ret) {
PyErr_Print();
uwsgi_log_verbose("[BUG] unable to switch greenlet !!!\n");
exit(1);
}
Py_DECREF(ret);
if (is_not_python(modifier1) && uwsgi.p[modifier1]->resume) {
uwsgi.p[modifier1]->resume(NULL);
}
}
PyObject *py_uwsgi_tornado_request(PyObject *self, PyObject *args) {
int fd = -1;
PyObject *events = NULL;
if (!PyArg_ParseTuple(args, "iO:uwsgi_tornado_request", &fd, &events)) {
uwsgi_log_verbose("[BUG] invalid arguments for tornado callback !!!\n");
exit(1);
}
struct wsgi_request *wsgi_req = find_wsgi_req_proto_by_fd(fd);
uwsgi.wsgi_req = wsgi_req;
int status = wsgi_req->socket->proto(wsgi_req);
if (status > 0) goto again;
if (PyObject_CallMethod(utornado.ioloop, "remove_handler", "i", fd) == NULL) {
PyErr_Print();
goto end;
}
if (status == 0) {
// we call this two time... overengineering :(
uwsgi.async_proto_fd_table[wsgi_req->fd] = NULL;
uwsgi.schedule_to_req();
goto again;
}
end:
uwsgi.async_proto_fd_table[wsgi_req->fd] = NULL;
uwsgi_close_request(uwsgi.wsgi_req);
free_req_queue;
again:
Py_INCREF(Py_None);
return Py_None;
}
void master_check_cluster_nodes() {
int i;
for (i = 0; i < MAX_CLUSTER_NODES; i++) {
struct uwsgi_cluster_node *ucn = &uwsgi.shared->nodes[i];
if (ucn->name[0] != 0 && ucn->type == CLUSTER_NODE_STATIC && ucn->status == UWSGI_NODE_FAILED) {
// should i retry ?
if (uwsgi.master_cycles % ucn->errors == 0) {
if (!uwsgi_ping_node(i, uwsgi.wsgi_req)) {
ucn->status = UWSGI_NODE_OK;
uwsgi_log("re-enabled cluster node %d/%s\n", i, ucn->name);
}
else {
ucn->errors++;
}
}
}
else if (ucn->name[0] != 0 && ucn->type == CLUSTER_NODE_DYNAMIC) {
// if the last_seen attr is higher than 30 secs ago, mark the node as dead
if ((uwsgi.current_time - ucn->last_seen) > 30) {
uwsgi_log_verbose("no presence announce in the last 30 seconds by node %s, i assume it is dead.\n", ucn->name);
ucn->name[0] = 0;
}
}
}
}
void get_linux_tcp_info(int fd) {
socklen_t tis = sizeof(struct tcp_info);
if (!getsockopt(fd, IPPROTO_TCP, TCP_INFO, &uwsgi.shared->ti, &tis)) {
// a check for older linux kernels
if (!uwsgi.shared->ti.tcpi_sacked) {
return;
}
uwsgi.shared->load = uwsgi.shared->ti.tcpi_unacked;
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_STATUS] = uwsgi.shared->ti.tcpi_unacked;
if (uwsgi.vassal_sos_backlog > 0 && uwsgi.has_emperor) {
if ((int) uwsgi.shared->ti.tcpi_unacked >= uwsgi.vassal_sos_backlog) {
// ask emperor for help
char byte = 30;
if (write(uwsgi.emperor_fd, &byte, 1) != 1) {
uwsgi_error("write()");
}
else {
uwsgi_log("asking emperor for reinforcements (backlog: %d)...\n", (int) uwsgi.shared->ti.tcpi_unacked);
}
}
}
if (uwsgi.shared->ti.tcpi_unacked >= uwsgi.shared->ti.tcpi_sacked) {
uwsgi_log_verbose("*** uWSGI listen queue of socket %d full !!! (%d/%d) ***\n", fd, uwsgi.shared->ti.tcpi_unacked, uwsgi.shared->ti.tcpi_sacked);
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_ERRORS]++;
}
}
}
void uwsgi_master_manage_emperor() {
char byte;
ssize_t rlen = read(uwsgi.emperor_fd, &byte, 1);
if (rlen > 0) {
uwsgi_log_verbose("received message %d from emperor\n", byte);
// remove me
if (byte == 0) {
close(uwsgi.emperor_fd);
if (!uwsgi.to_hell)
kill_them_all(0);
}
// reload me
else if (byte == 1) {
// un-lazy the stack to trigger a real reload
uwsgi.lazy = 0;
uwsgi_block_signal(SIGHUP);
grace_them_all(0);
uwsgi_unblock_signal(SIGHUP);
}
}
else {
uwsgi_log("lost connection with my emperor !!!\n");
close(uwsgi.emperor_fd);
if (!uwsgi.to_hell)
kill_them_all(0);
sleep(2);
exit(1);
}
}
int uwsgi_master_check_emperor_death(int diedpid) {
if (uwsgi.emperor_pid >= 0 && diedpid == uwsgi.emperor_pid) {
uwsgi_log_verbose("!!! Emperor died !!!\n");
uwsgi_emperor_start();
return -1;
}
return 0;
}
void uwsgi_brutally_reload_workers() {
int i;
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].pid > 0) {
uwsgi_log_verbose("killing worker %d (pid: %d)\n", i, (int) uwsgi.workers[i].pid);
uwsgi_curse(i, SIGINT);
}
}
}
PyObject *py_uwsgi_gevent_graceful(PyObject *self, PyObject *args) {
uwsgi_log("Gracefully killing worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
uwsgi_log_verbose("stopping gevent signals watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
PyObject_CallMethod(ugevent.my_signal_watcher, "stop", NULL);
PyObject_CallMethod(ugevent.signal_watcher, "stop", NULL);
uwsgi_log_verbose("stopping gevent sockets watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int i,count = uwsgi_count_sockets(uwsgi.sockets);
for(i=0;i<count;i++) {
PyObject_CallMethod(ugevent.watchers[i], "stop", NULL);
}
uwsgi_log_verbose("main gevent watchers stopped for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int running_cores = 0;
for(i=0;i<uwsgi.async;i++) {
if (uwsgi.workers[uwsgi.mywid].cores[i].in_request) {
struct wsgi_request *wsgi_req = &uwsgi.workers[uwsgi.mywid].cores[i].req;
uwsgi_log_verbose("worker %d (pid: %d) core %d is managing \"%.*s %.*s\" for %.*s\n", uwsgi.mywid, uwsgi.mypid, i,
wsgi_req->method_len, wsgi_req->method, wsgi_req->uri_len, wsgi_req->uri,
wsgi_req->remote_addr_len, wsgi_req->remote_addr);
running_cores++;
}
}
if (running_cores > 0) {
uwsgi_log_verbose("waiting for %d running requests on worker %d (pid: %d)...\n", running_cores, uwsgi.mywid, uwsgi.mypid);
} else {
// no need to worry about freeing memory
PyObject *uwsgi_dict = get_uwsgi_pydict("uwsgi");
if (uwsgi_dict) {
PyObject *ae = PyDict_GetItemString(uwsgi_dict, "atexit");
if (ae) {
python_call(ae, PyTuple_New(0), 0, NULL);
}
}
}
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *py_uwsgi_asyncio_request(PyObject *self, PyObject *args) {
long wsgi_req_ptr = 0;
int timed_out = 0;
if (!PyArg_ParseTuple(args, "l|i:uwsgi_asyncio_request", &wsgi_req_ptr, &timed_out)) {
uwsgi_log_verbose("[BUG] invalid arguments for asyncio callback !!!\n");
exit(1);
}
struct wsgi_request *wsgi_req = (struct wsgi_request *) wsgi_req_ptr;
uwsgi.wsgi_req = wsgi_req;
PyObject *ob_timeout = (PyObject *) wsgi_req->async_timeout;
if (PyObject_CallMethod(ob_timeout, "cancel", NULL) == NULL) PyErr_Print();
Py_DECREF(ob_timeout);
// avoid mess when closing the request
wsgi_req->async_timeout = NULL;
if (timed_out > 0) {
if (PyObject_CallMethod(uasyncio.loop, "remove_reader", "i", wsgi_req->fd) == NULL) PyErr_Print();
goto end;
}
int status = wsgi_req->socket->proto(wsgi_req);
if (status > 0) {
ob_timeout = PyObject_CallMethod(uasyncio.loop, "call_later", "iOli", uwsgi.socket_timeout, uasyncio.request, wsgi_req_ptr, 1);
if (!ob_timeout) {
if (PyObject_CallMethod(uasyncio.loop, "remove_reader", "i", wsgi_req->fd) == NULL) PyErr_Print();
goto end;
}
// trick for reference counting
wsgi_req->async_timeout = (struct uwsgi_rb_timer *) ob_timeout;
goto again;
}
if (PyObject_CallMethod(uasyncio.loop, "remove_reader", "i", wsgi_req->fd) == NULL) {
PyErr_Print();
goto end;
}
if (status == 0) {
// we call this two time... overengineering :(
uwsgi.async_proto_fd_table[wsgi_req->fd] = NULL;
uwsgi.schedule_to_req();
goto again;
}
end:
uwsgi.async_proto_fd_table[wsgi_req->fd] = NULL;
uwsgi_close_request(uwsgi.wsgi_req);
free_req_queue;
again:
Py_INCREF(Py_None);
return Py_None;
}
void get_linux_unbit_SIOBKLGQ(int fd) {
int queue = 0;
if (ioctl(fd, SIOBKLGQ, &queue) >= 0) {
uwsgi.shared->load = queue;
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_STATUS] = queue;
if (queue >= uwsgi.listen_queue) {
uwsgi_log_verbose("*** uWSGI listen queue of socket %d full !!! (%d/%d) ***\n", fd, queue, uwsgi.listen_queue);
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_ERRORS]++;
}
}
}
void async_add_timeout(struct wsgi_request *wsgi_req, int timeout) {
if (uwsgi.async < 1 || !uwsgi.rb_async_timeouts) {
uwsgi_log_verbose("ASYNC call without async mode !!!\n");
return;
}
wsgi_req->async_ready_fd = 0;
if (timeout > 0 && wsgi_req->async_timeout == NULL) {
wsgi_req->async_timeout = uwsgi_add_rb_timer(uwsgi.rb_async_timeouts, uwsgi_now() + timeout, wsgi_req);
}
}
void uwsgi_master_check_mercy() {
int i;
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cursed_at) {
if (uwsgi_now() > uwsgi.workers[i].no_mercy_at) {
uwsgi_log_verbose("worker %d (pid: %d) is taking too much time to die...NO MERCY !!!\n", i, uwsgi.workers[i].pid);
// yes that look strangem but we avoid callign it again if we skip waitpid() call below
uwsgi_curse(i, SIGKILL);
}
}
}
}
PyObject *py_uwsgi_gevent_int(PyObject *self, PyObject *args) {
uwsgi_log("Brutally killing worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
uwsgi_log_verbose("stopping gevent signals watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
PyObject_CallMethod(ugevent.my_signal_watcher, "stop", NULL);
PyObject_CallMethod(ugevent.signal_watcher, "stop", NULL);
uwsgi_log_verbose("stopping gevent sockets watchers for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
int i,count = uwsgi_count_sockets(uwsgi.sockets);
for(i=0;i<count;i++) {
PyObject_CallMethod(ugevent.watchers[i], "stop", NULL);
}
uwsgi_log_verbose("main gevent watchers stopped for worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid);
if (!ugevent.wait_for_hub) {
PyObject_CallMethod(ugevent.ctrl_gl, "kill", NULL);
}
Py_INCREF(Py_None);
return Py_None;
}
// check for chain reload
void uwsgi_master_check_chain() {
if (!uwsgi.status.chain_reloading) return;
if (uwsgi.status.chain_reloading > uwsgi.numproc) {
uwsgi.status.chain_reloading = 0;
uwsgi_log_verbose("chain reloading complete\n");
}
int i;
uwsgi_block_signal(SIGHUP);
for(i=1;i<=uwsgi.numproc;i++) {
if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].cursed_at == 0 && i == uwsgi.status.chain_reloading) {
uwsgi_curse(i, SIGHUP);
break;
}
}
uwsgi_unblock_signal(SIGHUP);
}
// callback used to asynchronously signal the completion
static void uwsgi_rados_read_async_cb(rados_completion_t comp, void *data) {
struct uwsgi_rados_cb *urcb = (struct uwsgi_rados_cb *) data;
struct uwsgi_rados_io *urio = urcb->urio;
pthread_mutex_lock(&urio->mutex);
if (urcb->rid != urio->rid) {
uwsgi_log_verbose("[uwsgi-rados] callback %llu woke up too late\n", (unsigned long long) urcb->rid);
}
else {
// signal the core
if (write(urio->fds[1], "\1", 1) <= 0) {
uwsgi_error("uwsgi_rados_read_async_cb()/write()");
}
}
pthread_mutex_unlock(&urio->mutex);
free(urcb);
}
int async_add_fd_write(struct wsgi_request *wsgi_req, int fd, int timeout) {
if (uwsgi.async < 1 || !uwsgi.async_waiting_fd_table) {
uwsgi_log_verbose("ASYNC call without async mode !!!\n");
return -1;
}
struct uwsgi_async_fd *last_uad = NULL, *uad = wsgi_req->waiting_fds;
if (fd < 0)
return -1;
// find last slot
while (uad) {
last_uad = uad;
uad = uad->next;
}
uad = uwsgi_malloc(sizeof(struct uwsgi_async_fd));
uad->fd = fd;
uad->event = event_queue_write();
uad->prev = last_uad;
uad->next = NULL;
if (last_uad) {
last_uad->next = uad;
}
else {
wsgi_req->waiting_fds = uad;
}
if (timeout > 0) {
async_add_timeout(wsgi_req, timeout);
}
uwsgi.async_waiting_fd_table[fd] = wsgi_req;
wsgi_req->async_force_again = 1;
return event_queue_add_fd_write(uwsgi.async_queue, fd);
}
void uwsgi_manage_command_cron(time_t now) {
struct tm *uwsgi_cron_delta;
struct uwsgi_cron *current_cron = uwsgi.crons;
int uc_minute, uc_hour, uc_day, uc_month, uc_week;
uwsgi_cron_delta = localtime(&now);
if (!uwsgi_cron_delta) {
uwsgi_error("localtime()");
return;
}
// fix month
uwsgi_cron_delta->tm_mon++;
while (current_cron) {
uc_minute = current_cron->minute;
uc_hour = current_cron->hour;
uc_day = current_cron->day;
uc_month = current_cron->month;
uc_week = current_cron->week;
// negative values as interval -1 = * , -5 = */5
if (current_cron->minute < 0) {
if ((uwsgi_cron_delta->tm_min % abs(current_cron->minute)) == 0) {
uc_minute = uwsgi_cron_delta->tm_min;
}
}
if (current_cron->hour < 0) {
if ((uwsgi_cron_delta->tm_hour % abs(current_cron->hour)) == 0) {
uc_hour = uwsgi_cron_delta->tm_hour;
}
}
if (current_cron->month < 0) {
if ((uwsgi_cron_delta->tm_hour % abs(current_cron->month)) == 0) {
uc_month = uwsgi_cron_delta->tm_mon;
}
}
if (current_cron->day < 0) {
if ((uwsgi_cron_delta->tm_mday % abs(current_cron->day)) == 0) {
uc_day = uwsgi_cron_delta->tm_mday;
}
}
if (current_cron->week < 0) {
if ((uwsgi_cron_delta->tm_wday % abs(current_cron->week)) == 0) {
uc_week = uwsgi_cron_delta->tm_wday;
}
}
int run_task = 0;
// mday and wday are ORed
if (current_cron->day >= 0 && current_cron->week >= 0) {
if (uwsgi_cron_delta->tm_min == uc_minute && uwsgi_cron_delta->tm_hour == uc_hour && uwsgi_cron_delta->tm_mon == uc_month && (uwsgi_cron_delta->tm_mday == uc_day || uwsgi_cron_delta->tm_wday == uc_week)) {
run_task = 1;
}
}
else {
if (uwsgi_cron_delta->tm_min == uc_minute && uwsgi_cron_delta->tm_hour == uc_hour && uwsgi_cron_delta->tm_mon == uc_month && uwsgi_cron_delta->tm_mday == uc_day && uwsgi_cron_delta->tm_wday == uc_week) {
run_task = 1;
}
}
if (run_task == 1) {
// date match, run command ?
if (now - current_cron->last_job > 60) {
//call command
if (current_cron->command) {
if (uwsgi_run_command(current_cron->command, NULL, -1) >= 0) {
uwsgi_log_verbose("[uWSGI-cron] running %s\n", current_cron->command);
}
}
current_cron->last_job = now;
}
}
current_cron = current_cron->next;
}
}
int uwsgi_get_tcp_info(int fd) {
#if defined(__linux__) || defined(__FreeBSD__)
socklen_t tis = sizeof(struct tcp_info);
if (!getsockopt(fd, IPPROTO_TCP, TCP_INFO, &uwsgi.shared->ti, &tis)) {
// checks for older kernels
#if defined(__linux__)
if (!uwsgi.shared->ti.tcpi_sacked) {
#elif defined(__FreeBSD__)
if (!uwsgi.shared->ti.__tcpi_sacked) {
#endif
return 0;
}
#if defined(__linux__)
uwsgi.shared->load = (uint64_t) uwsgi.shared->ti.tcpi_unacked;
uwsgi.shared->max_load = (uint64_t) uwsgi.shared->ti.tcpi_sacked;
#elif defined(__FreeBSD__)
uwsgi.shared->load = (uint64_t) uwsgi.shared->ti.__tcpi_unacked;
uwsgi.shared->max_load = (uint64_t) uwsgi.shared->ti.__tcpi_sacked;
#endif
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_STATUS] = uwsgi.shared->load;
if (uwsgi.vassal_sos_backlog > 0 && uwsgi.has_emperor) {
if (uwsgi.shared->load >= (uint64_t) uwsgi.vassal_sos_backlog) {
// ask emperor for help
char byte = 30;
if (write(uwsgi.emperor_fd, &byte, 1) != 1) {
uwsgi_error("write()");
}
else {
uwsgi_log("asking emperor for reinforcements (backlog: %llu)...\n", (unsigned long long) uwsgi.shared->load);
}
}
}
if (uwsgi.shared->load >= uwsgi.shared->max_load) {
uwsgi_log_verbose("*** uWSGI listen queue of socket %d full !!! (%llu/%llu) ***\n", fd, (unsigned long long) uwsgi.shared->load, (unsigned long long) uwsgi.shared->max_load);
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_ERRORS]++;
}
return uwsgi.shared->load;
}
#endif
return 0;
}
#ifdef __linux__
#include <linux/sockios.h>
#ifdef UNBIT
#define SIOBKLGQ 0x8908
#endif
#ifdef SIOBKLGQ
int get_linux_unbit_SIOBKLGQ(int fd) {
int queue = 0;
if (ioctl(fd, SIOBKLGQ, &queue) >= 0) {
uwsgi.shared->load = queue;
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_STATUS] = queue;
if (queue >= uwsgi.listen_queue) {
uwsgi_log_verbose("*** uWSGI listen queue of socket %d full !!! (%d/%d) ***\n", fd, queue, uwsgi.listen_queue);
uwsgi.shared->options[UWSGI_OPTION_BACKLOG_ERRORS]++;
}
return queue;
}
return 0;
}
int master_loop(char **argv, char **environ) {
struct timeval last_respawn;
int last_respawn_rate = 0;
pid_t diedpid;
int waitpid_status;
time_t now = 0;
int i = 0;
int rlen;
int check_interval = 1;
struct uwsgi_rb_timer *min_timeout;
struct uwsgi_rbtree *rb_timers = uwsgi_init_rb_timer();
if (uwsgi.procname_master) {
uwsgi_set_processname(uwsgi.procname_master);
}
else if (uwsgi.procname) {
uwsgi_set_processname(uwsgi.procname);
}
else if (uwsgi.auto_procname) {
uwsgi_set_processname("uWSGI master");
}
uwsgi.current_time = uwsgi_now();
uwsgi_unix_signal(SIGTSTP, suspend_resume_them_all);
uwsgi_unix_signal(SIGHUP, grace_them_all);
if (uwsgi.die_on_term) {
uwsgi_unix_signal(SIGTERM, kill_them_all);
uwsgi_unix_signal(SIGQUIT, reap_them_all);
}
else {
uwsgi_unix_signal(SIGTERM, reap_them_all);
uwsgi_unix_signal(SIGQUIT, kill_them_all);
}
uwsgi_unix_signal(SIGINT, kill_them_all);
uwsgi_unix_signal(SIGUSR1, stats);
if (uwsgi.auto_snapshot) {
uwsgi_unix_signal(SIGURG, uwsgi_restore_auto_snapshot);
}
atexit(uwsgi_master_cleanup_hooks);
uwsgi.master_queue = event_queue_init();
/* route signals to workers... */
#ifdef UWSGI_DEBUG
uwsgi_log("adding %d to signal poll\n", uwsgi.shared->worker_signal_pipe[0]);
#endif
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_signal_pipe[0]);
if (uwsgi.master_fifo) {
uwsgi.master_fifo_fd = uwsgi_master_fifo();
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.master_fifo_fd);
}
if (uwsgi.spoolers) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->spooler_signal_pipe[0]);
}
if (uwsgi.mules_cnt > 0) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->mule_signal_pipe[0]);
}
if (uwsgi.log_master) {
uwsgi.log_master_buf = uwsgi_malloc(uwsgi.log_master_bufsize);
if (!uwsgi.threaded_logger) {
#ifdef UWSGI_DEBUG
uwsgi_log("adding %d to master logging\n", uwsgi.shared->worker_log_pipe[0]);
#endif
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]);
if (uwsgi.req_log_master) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]);
}
}
else {
uwsgi_threaded_logger_spawn();
}
}
#ifdef UWSGI_SSL
uwsgi_start_legions();
#endif
uwsgi_metrics_start_collector();
uwsgi_add_reload_fds();
uwsgi_cache_start_sweepers();
uwsgi_cache_start_sync_servers();
uwsgi.wsgi_req->buffer = uwsgi.workers[0].cores[0].buffer;
if (uwsgi.has_emperor) {
if (uwsgi.emperor_proxy) {
uwsgi.emperor_fd_proxy = bind_to_unix(uwsgi.emperor_proxy, uwsgi.listen_queue, 0, 0);
if (uwsgi.emperor_fd_proxy < 0) exit(1);
if (chmod(uwsgi.emperor_proxy, S_IRUSR|S_IWUSR)) {
uwsgi_error("[emperor-proxy] chmod()");
exit(1);
}
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.emperor_fd_proxy);
}
else {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.emperor_fd);
}
}
if (uwsgi.zerg_server) {
uwsgi.zerg_server_fd = bind_to_unix(uwsgi.zerg_server, uwsgi.listen_queue, 0, 0);
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.zerg_server_fd);
uwsgi_log("*** Zerg server enabled on %s ***\n", uwsgi.zerg_server);
}
if (uwsgi.stats) {
char *tcp_port = strrchr(uwsgi.stats, ':');
if (tcp_port) {
// disable deferred accept for this socket
int current_defer_accept = uwsgi.no_defer_accept;
uwsgi.no_defer_accept = 1;
uwsgi.stats_fd = bind_to_tcp(uwsgi.stats, uwsgi.listen_queue, tcp_port);
uwsgi.no_defer_accept = current_defer_accept;
}
else {
uwsgi.stats_fd = bind_to_unix(uwsgi.stats, uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
}
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.stats_fd);
uwsgi_log("*** Stats server enabled on %s fd: %d ***\n", uwsgi.stats, uwsgi.stats_fd);
}
if (uwsgi.stats_pusher_instances) {
if (!uwsgi_thread_new(uwsgi_stats_pusher_loop)) {
uwsgi_log("!!! unable to spawn stats pusher thread !!!\n");
exit(1);
}
}
if (uwsgi.udp_socket) {
uwsgi.udp_fd = bind_to_udp(uwsgi.udp_socket, 0, 0);
if (uwsgi.udp_fd < 0) {
uwsgi_log("unable to bind to udp socket. SNMP services will be disabled.\n");
}
else {
uwsgi_log("UDP server enabled.\n");
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.udp_fd);
}
}
uwsgi.snmp_fd = uwsgi_setup_snmp();
if (uwsgi.status.is_cheap) {
uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1);
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].cheaped = 1;
}
uwsgi_log("cheap mode enabled: waiting for socket connection...\n");
}
// spawn mules
for (i = 0; i < uwsgi.mules_cnt; i++) {
size_t mule_patch_size = 0;
uwsgi.mules[i].patch = uwsgi_string_get_list(&uwsgi.mules_patches, i, &mule_patch_size);
uwsgi_mule(i + 1);
}
// spawn gateways
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways[i].pid == 0) {
gateway_respawn(i);
}
}
// spawn daemons
uwsgi_daemons_spawn_all();
// first subscription
uwsgi_subscribe_all(0, 1);
// sync the cache store if needed
uwsgi_cache_sync_all();
if (uwsgi.queue_store && uwsgi.queue_filesize) {
if (msync(uwsgi.queue_header, uwsgi.queue_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
// update touches timestamps
uwsgi_check_touches(uwsgi.touch_reload);
uwsgi_check_touches(uwsgi.touch_logrotate);
uwsgi_check_touches(uwsgi.touch_logreopen);
uwsgi_check_touches(uwsgi.touch_chain_reload);
uwsgi_check_touches(uwsgi.touch_workers_reload);
uwsgi_check_touches(uwsgi.touch_gracefully_stop);
// update exec touches
struct uwsgi_string_list *usl = uwsgi.touch_exec;
while (usl) {
char *space = strchr(usl->value, ' ');
if (space) {
*space = 0;
usl->len = strlen(usl->value);
usl->custom_ptr = space + 1;
}
usl = usl->next;
}
uwsgi_check_touches(uwsgi.touch_exec);
// update signal touches
usl = uwsgi.touch_signal;
while (usl) {
char *space = strchr(usl->value, ' ');
if (space) {
*space = 0;
usl->len = strlen(usl->value);
usl->custom_ptr = space + 1;
}
usl = usl->next;
}
uwsgi_check_touches(uwsgi.touch_signal);
// fsmon
uwsgi_fsmon_setup();
// setup cheaper algos (can be stacked)
uwsgi.cheaper_algo = uwsgi_cheaper_algo_spare;
if (uwsgi.requested_cheaper_algo) {
uwsgi.cheaper_algo = NULL;
struct uwsgi_cheaper_algo *uca = uwsgi.cheaper_algos;
while (uca) {
if (!strcmp(uca->name, uwsgi.requested_cheaper_algo)) {
uwsgi.cheaper_algo = uca->func;
break;
}
uca = uca->next;
}
if (!uwsgi.cheaper_algo) {
uwsgi_log("unable to find requested cheaper algorithm, falling back to spare\n");
uwsgi.cheaper_algo = uwsgi_cheaper_algo_spare;
}
}
// here really starts the master loop
for (;;) {
//uwsgi_log("uwsgi.ready_to_reload %d %d\n", uwsgi.ready_to_reload, uwsgi.numproc);
// run master_cycle hook for every plugin
for (i = 0; i < uwsgi.gp_cnt; i++) {
if (uwsgi.gp[i]->master_cycle) {
uwsgi.gp[i]->master_cycle();
}
}
for (i = 0; i < 256; i++) {
if (uwsgi.p[i]->master_cycle) {
uwsgi.p[i]->master_cycle();
}
}
// check for death (before reload !!!)
uwsgi_master_check_death();
// check for realod
if (uwsgi_master_check_reload(argv)) {
return -1;
}
// check chain reload
uwsgi_master_check_chain();
// check if some worker is taking too much to die...
uwsgi_master_check_mercy();
// check for daemons (smart and dumb)
uwsgi_daemons_smart_check();
if (uwsgi.respawn_snapshots) {
for (i = 1; i <= uwsgi.respawn_snapshots; i++) {
if (uwsgi_respawn_worker(i))
return 0;
}
uwsgi.respawn_snapshots = 0;
}
if (uwsgi.restore_snapshot) {
uwsgi_master_restore_snapshot();
continue;
}
// cheaper management
if (uwsgi.cheaper && !uwsgi.status.is_cheap && !uwsgi_instance_is_reloading && !uwsgi_instance_is_dying && !uwsgi.workers[0].suspended) {
if (!uwsgi_calc_cheaper())
return 0;
}
// check if someone is dead
diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG);
if (diedpid == -1) {
if (errno == ECHILD) {
// something did not work as expected, just assume all has been cleared
uwsgi_master_commit_status();
diedpid = 0;
}
else {
uwsgi_error("waitpid()");
/* here is better to reload all the uWSGI stack */
uwsgi_log("something horrible happened...\n");
reap_them_all(0);
exit(1);
}
}
// no one died just run all of the standard master tasks
if (diedpid == 0) {
/* all processes ok, doing status scan after N seconds */
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
if (!check_interval)
check_interval = 1;
// add unregistered file monitors
// locking is not needed as monitors can only increase
for (i = 0; i < ushared->files_monitored_cnt; i++) {
if (!ushared->files_monitored[i].registered) {
ushared->files_monitored[i].fd = event_queue_add_file_monitor(uwsgi.master_queue, ushared->files_monitored[i].filename, &ushared->files_monitored[i].id);
ushared->files_monitored[i].registered = 1;
}
}
// add unregistered timers
// locking is not needed as timers can only increase
for (i = 0; i < ushared->timers_cnt; i++) {
if (!ushared->timers[i].registered) {
ushared->timers[i].fd = event_queue_add_timer(uwsgi.master_queue, &ushared->timers[i].id, ushared->timers[i].value);
ushared->timers[i].registered = 1;
}
}
// add unregistered rb_timers
// locking is not needed as rb_timers can only increase
for (i = 0; i < ushared->rb_timers_cnt; i++) {
if (!ushared->rb_timers[i].registered) {
ushared->rb_timers[i].uwsgi_rb_timer = uwsgi_add_rb_timer(rb_timers, uwsgi_now() + ushared->rb_timers[i].value, &ushared->rb_timers[i]);
ushared->rb_timers[i].registered = 1;
}
}
int interesting_fd = -1;
if (ushared->rb_timers_cnt > 0) {
min_timeout = uwsgi_min_rb_timer(rb_timers, NULL);
if (min_timeout == NULL) {
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
}
else {
check_interval = min_timeout->value - uwsgi_now();
if (check_interval <= 0) {
expire_rb_timeouts(rb_timers);
check_interval = 0;
}
}
}
// wait for event
rlen = event_queue_wait(uwsgi.master_queue, check_interval, &interesting_fd);
if (rlen == 0) {
if (ushared->rb_timers_cnt > 0) {
expire_rb_timeouts(rb_timers);
}
}
// check uwsgi-cron table
if (ushared->cron_cnt) {
uwsgi_manage_signal_cron(uwsgi_now());
}
if (uwsgi.crons) {
uwsgi_manage_command_cron(uwsgi_now());
}
// some event returned
if (rlen > 0) {
// if the following function returns -1, a new worker has just spawned
if (uwsgi_master_manage_events(interesting_fd)) {
return 0;
}
}
now = uwsgi_now();
if (now - uwsgi.current_time < 1) {
continue;
}
uwsgi.current_time = now;
// checking logsize
if (uwsgi.logfile) {
uwsgi_check_logrotate();
}
// this will be incremented at (more or less) regular intervals
uwsgi.master_cycles++;
// recalculate requests counter on race conditions risky configurations
// a bit of inaccuracy is better than locking;)
uwsgi_master_fix_request_counters();
// check for idle
uwsgi_master_check_idle();
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
if (!check_interval)
check_interval = 1;
// get listen_queue status
struct uwsgi_socket *uwsgi_sock = uwsgi.sockets;
int tmp_queue = 0;
while (uwsgi_sock) {
if (uwsgi_sock->family == AF_INET) {
uwsgi_sock->queue = uwsgi_get_tcp_info(uwsgi_sock->fd);
}
#ifdef __linux__
#ifdef SIOBKLGQ
else if (uwsgi_sock->family == AF_UNIX) {
uwsgi_sock->queue = get_linux_unbit_SIOBKLGQ(uwsgi_sock->fd);
}
#endif
#endif
if (uwsgi_sock->queue > tmp_queue) {
tmp_queue = uwsgi_sock->queue;
}
uwsgi_sock = uwsgi_sock->next;
}
// fix queue size on multiple sockets
uwsgi.shared->load = tmp_queue;
// check if some worker has to die (harakiri, evil checks...)
uwsgi_master_check_workers_deadline();
uwsgi_master_check_gateways_deadline();
uwsgi_master_check_mules_deadline();
uwsgi_master_check_spoolers_deadline();
uwsgi_master_check_crons_deadline();
uwsgi_master_check_mountpoints();
#ifdef __linux__
#ifdef MADV_MERGEABLE
if (uwsgi.linux_ksm > 0 && (uwsgi.master_cycles % uwsgi.linux_ksm) == 0) {
uwsgi_linux_ksm_map();
}
#endif
#endif
// resubscribe every 10 cycles by default
if (((uwsgi.subscriptions || uwsgi.subscriptions2) && ((uwsgi.master_cycles % uwsgi.subscribe_freq) == 0 || uwsgi.master_cycles == 1)) && !uwsgi_instance_is_reloading && !uwsgi_instance_is_dying && !uwsgi.workers[0].suspended) {
uwsgi_subscribe_all(0, 0);
}
uwsgi_cache_sync_all();
if (uwsgi.queue_store && uwsgi.queue_filesize && uwsgi.queue_store_sync && ((uwsgi.master_cycles % uwsgi.queue_store_sync) == 0)) {
if (msync(uwsgi.queue_header, uwsgi.queue_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
// check touch_reload
if (!uwsgi_instance_is_reloading && !uwsgi_instance_is_dying) {
char *touched = uwsgi_check_touches(uwsgi.touch_reload);
if (touched) {
uwsgi_log_verbose("*** %s has been touched... grace them all !!! ***\n", touched);
uwsgi_block_signal(SIGHUP);
grace_them_all(0);
uwsgi_unblock_signal(SIGHUP);
continue;
}
touched = uwsgi_check_touches(uwsgi.touch_workers_reload);
if (touched) {
uwsgi_log_verbose("*** %s has been touched... workers reload !!! ***\n", touched);
uwsgi_reload_workers();
continue;
}
touched = uwsgi_check_touches(uwsgi.touch_chain_reload);
if (touched) {
if (uwsgi.status.chain_reloading == 0) {
uwsgi_log_verbose("*** %s has been touched... chain reload !!! ***\n", touched);
uwsgi.status.chain_reloading = 1;
}
else {
uwsgi_log_verbose("*** %s has been touched... but chain reload is already running ***\n", touched);
}
}
// be sure to run it as the last touch check
touched = uwsgi_check_touches(uwsgi.touch_exec);
if (touched) {
if (uwsgi_run_command(touched, NULL, -1) >= 0) {
uwsgi_log_verbose("[uwsgi-touch-exec] running %s\n", touched);
}
}
touched = uwsgi_check_touches(uwsgi.touch_signal);
if (touched) {
uint8_t signum = atoi(touched);
uwsgi_route_signal(signum);
uwsgi_log_verbose("[uwsgi-touch-signal] raising %u\n", signum);
}
}
continue;
}
// no one died
if (diedpid <= 0)
continue;
// check for deadlocks first
uwsgi_deadlock_check(diedpid);
// reload gateways and daemons only on normal workflow (+outworld status)
if (!uwsgi_instance_is_reloading && !uwsgi_instance_is_dying) {
if (uwsgi_master_check_emperor_death(diedpid))
continue;
if (uwsgi_master_check_spoolers_death(diedpid))
continue;
if (uwsgi_master_check_mules_death(diedpid))
continue;
if (uwsgi_master_check_gateways_death(diedpid))
continue;
if (uwsgi_master_check_daemons_death(diedpid))
continue;
if (uwsgi_master_check_cron_death(diedpid))
continue;
}
/* What happens here ?
case 1) the diedpid is not a worker, report it and continue
case 2) the diedpid is a worker and we are not in a reload procedure -> reload it
case 3) the diedpid is a worker and we are in graceful reload -> uwsgi.ready_to_reload++ and continue
case 3) the diedpid is a worker and we are in brutal reload -> uwsgi.ready_to_die++ and continue
*/
int thewid = find_worker_id(diedpid);
if (thewid <= 0) {
// check spooler, mules, gateways and daemons
struct uwsgi_spooler *uspool = uwsgi.spoolers;
while (uspool) {
if (uspool->pid > 0 && diedpid == uspool->pid) {
uwsgi_log("spooler (pid: %d) annihilated\n", (int) diedpid);
goto next;
}
uspool = uspool->next;
}
for (i = 0; i < uwsgi.mules_cnt; i++) {
if (uwsgi.mules[i].pid == diedpid) {
uwsgi_log("mule %d (pid: %d) annihilated\n", i + 1, (int) diedpid);
goto next;
}
}
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways[i].pid == diedpid) {
uwsgi_log("gateway %d (%s, pid: %d) annihilated\n", i + 1, ushared->gateways[i].fullname, (int) diedpid);
goto next;
}
}
if (uwsgi_daemon_check_pid_death(diedpid))
goto next;
if (WIFEXITED(waitpid_status)) {
uwsgi_log("subprocess %d exited with code %d\n", (int) diedpid, WEXITSTATUS(waitpid_status));
}
else if (WIFSIGNALED(waitpid_status)) {
uwsgi_log("subprocess %d exited by signal %d\n", (int) diedpid, WTERMSIG(waitpid_status));
}
else if (WIFSTOPPED(waitpid_status)) {
uwsgi_log("subprocess %d stopped\n", (int) diedpid);
}
next:
continue;
}
// ok a worker died...
uwsgi.workers[thewid].pid = 0;
// only to be safe :P
uwsgi.workers[thewid].harakiri = 0;
// ok, if we are reloading or dying, just continue the master loop
// as soon as all of the workers have pid == 0, the action (exit, or reload) is triggered
if (uwsgi_instance_is_reloading || uwsgi_instance_is_dying) {
if (!uwsgi.workers[thewid].cursed_at)
uwsgi.workers[thewid].cursed_at = uwsgi_now();
uwsgi_log("worker %d buried after %d seconds\n", thewid, (int) (uwsgi_now() - uwsgi.workers[thewid].cursed_at));
uwsgi.workers[thewid].cursed_at = 0;
continue;
}
// if we are stopping workers, just end here
if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_FAILED_APP_CODE) {
uwsgi_log("OOPS ! failed loading app in worker %d (pid %d) :( trying again...\n", thewid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_DE_HIJACKED_CODE) {
uwsgi_log("...restoring worker %d (pid: %d)...\n", thewid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_EXCEPTION_CODE) {
uwsgi_log("... monitored exception detected, respawning worker %d (pid: %d)...\n", thewid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_QUIET_CODE) {
// noop
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_BRUTAL_RELOAD_CODE) {
uwsgi_log("!!! inconsistent state reported by worker %d (pid: %d) !!!\n", thewid, (int) diedpid);
reap_them_all(0);
continue;
}
else if (uwsgi.workers[thewid].manage_next_request) {
if (WIFSIGNALED(waitpid_status)) {
uwsgi_log("DAMN ! worker %d (pid: %d) died, killed by signal %d :( trying respawn ...\n", thewid, (int) diedpid, (int) WTERMSIG(waitpid_status));
}
else {
uwsgi_log("DAMN ! worker %d (pid: %d) died :( trying respawn ...\n", thewid, (int) diedpid);
}
}
else if (uwsgi.workers[thewid].cursed_at > 0) {
uwsgi_log("worker %d killed successfully (pid: %d)\n", thewid, (int) diedpid);
}
// manage_next_request is zero, but killed by signal...
else if (WIFSIGNALED(waitpid_status)) {
uwsgi_log("DAMN ! worker %d (pid: %d) MISTERIOUSLY killed by signal %d :( trying respawn ...\n", thewid, (int) diedpid, (int) WTERMSIG(waitpid_status));
}
if (uwsgi.workers[thewid].cheaped == 1) {
uwsgi_log("uWSGI worker %d cheaped.\n", thewid);
continue;
}
// avoid fork bombing
gettimeofday(&last_respawn, NULL);
if (last_respawn.tv_sec <= uwsgi.respawn_delta + check_interval) {
last_respawn_rate++;
if (last_respawn_rate > uwsgi.numproc) {
if (uwsgi.forkbomb_delay > 0) {
uwsgi_log("worker respawning too fast !!! i have to sleep a bit (%d seconds)...\n", uwsgi.forkbomb_delay);
/* use --forkbomb-delay 0 to disable sleeping */
sleep(uwsgi.forkbomb_delay);
}
last_respawn_rate = 0;
}
}
else {
last_respawn_rate = 0;
}
gettimeofday(&last_respawn, NULL);
uwsgi.respawn_delta = last_respawn.tv_sec;
// are we chain reloading it ?
if (uwsgi.status.chain_reloading == thewid) {
uwsgi.status.chain_reloading++;
}
// respawn the worker (if needed)
if (uwsgi_respawn_worker(thewid))
return 0;
// end of the loop
}
// never here
}
// this is called whenever a new connection is ready, but there are no cores to handle it
void uwsgi_async_queue_is_full(time_t now) {
if (now > uwsgi.async_queue_is_full) {
uwsgi_log_verbose("[DANGER] async queue is full !!!\n");
uwsgi.async_queue_is_full = now;
}
}
ssize_t uwsgi_mule_get_msg(int manage_signals, int manage_farms, char *message, size_t buffer_size, int timeout) {
ssize_t len = 0;
struct pollfd *mulepoll;
int count = 4;
int farms_count = 0;
uint8_t uwsgi_signal;
int i;
if (uwsgi.muleid == 0)
return -1;
if (manage_signals)
count = 2;
if (!manage_farms)
goto next;
for (i = 0; i < uwsgi.farms_cnt; i++) {
if (uwsgi_farm_has_mule(&uwsgi.farms[i], uwsgi.muleid))
farms_count++;
}
next:
if (timeout > -1)
timeout = timeout * 1000;
mulepoll = uwsgi_malloc(sizeof(struct pollfd) * (count + farms_count));
mulepoll[0].fd = uwsgi.mules[uwsgi.muleid - 1].queue_pipe[1];
mulepoll[0].events = POLLIN;
mulepoll[1].fd = uwsgi.shared->mule_queue_pipe[1];
mulepoll[1].events = POLLIN;
if (count > 2) {
mulepoll[2].fd = uwsgi.signal_socket;
mulepoll[2].events = POLLIN;
mulepoll[3].fd = uwsgi.my_signal_socket;
mulepoll[3].events = POLLIN;
}
if (farms_count > 0) {
int tmp_cnt = 0;
for (i = 0; i < uwsgi.farms_cnt; i++) {
if (uwsgi_farm_has_mule(&uwsgi.farms[i], uwsgi.muleid)) {
mulepoll[count + tmp_cnt].fd = uwsgi.farms[i].queue_pipe[1];
mulepoll[count + tmp_cnt].events = POLLIN;
tmp_cnt++;
}
}
}
int ret = poll(mulepoll, count + farms_count, timeout);
if (ret <= 0) {
uwsgi_error("poll");
}
else {
if (mulepoll[0].revents & POLLIN) {
len = read(uwsgi.mules[uwsgi.muleid - 1].queue_pipe[1], message, buffer_size);
}
else if (mulepoll[1].revents & POLLIN) {
len = read(uwsgi.shared->mule_queue_pipe[1], message, buffer_size);
}
else {
if (count > 2) {
int interesting_fd = -1;
if (mulepoll[2].revents & POLLIN) {
interesting_fd = mulepoll[2].fd;
}
else if (mulepoll[3].revents & POLLIN) {
interesting_fd = mulepoll[3].fd;
}
if (interesting_fd > -1) {
len = read(interesting_fd, &uwsgi_signal, 1);
if (len <= 0) {
uwsgi_log_verbose("uWSGI mule %d braying: my master died, i will follow him...\n", uwsgi.muleid);
end_me(0);
}
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("master sent signal %d to mule %d\n", uwsgi_signal, uwsgi.muleid);
#endif
if (uwsgi_signal_handler(uwsgi_signal)) {
uwsgi_log_verbose("error managing signal %d on mule %d\n", uwsgi_signal, uwsgi.mywid);
}
// set the error condition
len = -1;
goto clear;
}
}
// read messages in the farm
for (i = 0; i < farms_count; i++) {
if (mulepoll[count + i].revents & POLLIN) {
len = read(mulepoll[count + i].fd, message, buffer_size);
break;
}
}
}
}
if (len < 0) {
uwsgi_error("read()");
goto clear;
}
clear:
free(mulepoll);
return len;
}
int master_loop(char **argv, char **environ) {
uint64_t tmp_counter;
struct timeval last_respawn;
int last_respawn_rate = 0;
int pid_found = 0;
pid_t diedpid;
int waitpid_status;
uint8_t uwsgi_signal;
time_t last_request_timecheck = 0, now = 0;
uint64_t last_request_count = 0;
pthread_t logger_thread;
pthread_t cache_sweeper;
#ifdef UWSGI_UDP
int udp_fd = -1;
#ifdef UWSGI_MULTICAST
char *cluster_opt_buf = NULL;
size_t cluster_opt_size = 4;
#endif
#endif
#ifdef UWSGI_SNMP
int snmp_fd = -1;
#endif
int i = 0;
int rlen;
int check_interval = 1;
struct uwsgi_rb_timer *min_timeout;
struct rb_root *rb_timers = uwsgi_init_rb_timer();
if (uwsgi.procname_master) {
uwsgi_set_processname(uwsgi.procname_master);
}
else if (uwsgi.procname) {
uwsgi_set_processname(uwsgi.procname);
}
else if (uwsgi.auto_procname) {
uwsgi_set_processname("uWSGI master");
}
uwsgi.current_time = uwsgi_now();
uwsgi_unix_signal(SIGTSTP, suspend_resume_them_all);
uwsgi_unix_signal(SIGHUP, grace_them_all);
if (uwsgi.die_on_term) {
uwsgi_unix_signal(SIGTERM, kill_them_all);
uwsgi_unix_signal(SIGQUIT, reap_them_all);
}
else {
uwsgi_unix_signal(SIGTERM, reap_them_all);
uwsgi_unix_signal(SIGQUIT, kill_them_all);
}
uwsgi_unix_signal(SIGINT, kill_them_all);
uwsgi_unix_signal(SIGUSR1, stats);
if (uwsgi.auto_snapshot) {
uwsgi_unix_signal(SIGURG, uwsgi_restore_auto_snapshot);
}
atexit(uwsgi_master_cleanup_hooks);
uwsgi.master_queue = event_queue_init();
/* route signals to workers... */
#ifdef UWSGI_DEBUG
uwsgi_log("adding %d to signal poll\n", uwsgi.shared->worker_signal_pipe[0]);
#endif
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_signal_pipe[0]);
#ifdef UWSGI_SPOOLER
if (uwsgi.spoolers) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->spooler_signal_pipe[0]);
}
#endif
if (uwsgi.mules_cnt > 0) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->mule_signal_pipe[0]);
}
if (uwsgi.log_master) {
uwsgi.log_master_buf = uwsgi_malloc(uwsgi.log_master_bufsize);
if (!uwsgi.threaded_logger) {
#ifdef UWSGI_DEBUG
uwsgi_log("adding %d to master logging\n", uwsgi.shared->worker_log_pipe[0]);
#endif
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]);
}
else {
if (pthread_create(&logger_thread, NULL, logger_thread_loop, NULL)) {
uwsgi_error("pthread_create()");
uwsgi_log("falling back to non-threaded logger...\n");
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]);
uwsgi.threaded_logger = 0;
}
}
#ifdef UWSGI_ALARM
// initialize the alarm subsystem
uwsgi_alarms_init();
#endif
}
if (uwsgi.cache_max_items > 0 && !uwsgi.cache_no_expire) {
if (pthread_create(&cache_sweeper, NULL, cache_sweeper_loop, NULL)) {
uwsgi_error("pthread_create()");
uwsgi_log("unable to run the cache sweeper !!!\n");
}
else {
uwsgi_log("cache sweeper thread enabled\n");
}
}
uwsgi.wsgi_req->buffer = uwsgi.workers[0].cores[0].buffer;
if (uwsgi.has_emperor) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.emperor_fd);
}
if (uwsgi.zerg_server) {
uwsgi.zerg_server_fd = bind_to_unix(uwsgi.zerg_server, uwsgi.listen_queue, 0, 0);
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.zerg_server_fd);
uwsgi_log("*** Zerg server enabled on %s ***\n", uwsgi.zerg_server);
}
if (uwsgi.stats) {
char *tcp_port = strchr(uwsgi.stats, ':');
if (tcp_port) {
// disable deferred accept for this socket
int current_defer_accept = uwsgi.no_defer_accept;
uwsgi.no_defer_accept = 1;
uwsgi.stats_fd = bind_to_tcp(uwsgi.stats, uwsgi.listen_queue, tcp_port);
uwsgi.no_defer_accept = current_defer_accept;
}
else {
uwsgi.stats_fd = bind_to_unix(uwsgi.stats, uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
}
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.stats_fd);
uwsgi_log("*** Stats server enabled on %s fd: %d ***\n", uwsgi.stats, uwsgi.stats_fd);
}
#ifdef UWSGI_UDP
if (uwsgi.udp_socket) {
udp_fd = bind_to_udp(uwsgi.udp_socket, 0, 0);
if (udp_fd < 0) {
uwsgi_log("unable to bind to udp socket. SNMP and cluster management services will be disabled.\n");
}
else {
uwsgi_log("UDP server enabled.\n");
event_queue_add_fd_read(uwsgi.master_queue, udp_fd);
}
}
#ifdef UWSGI_MULTICAST
if (uwsgi.cluster) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.cluster_fd);
cluster_opt_buf = uwsgi_setup_clusterbuf(&cluster_opt_size);
}
#endif
#endif
#ifdef UWSGI_SNMP
snmp_fd = uwsgi_setup_snmp();
#endif
if (uwsgi.cheap) {
uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1);
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].cheaped = 1;
}
uwsgi_log("cheap mode enabled: waiting for socket connection...\n");
}
// spawn mules
for (i = 0; i < uwsgi.mules_cnt; i++) {
size_t mule_patch_size = 0;
uwsgi.mules[i].patch = uwsgi_string_get_list(&uwsgi.mules_patches, i, &mule_patch_size);
uwsgi_mule(i + 1);
}
// spawn gateways
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways[i].pid == 0) {
gateway_respawn(i);
}
}
// spawn daemons
uwsgi_daemons_spawn_all();
// first subscription
struct uwsgi_string_list *subscriptions = uwsgi.subscriptions;
while (subscriptions) {
uwsgi_subscribe(subscriptions->value, 0);
subscriptions = subscriptions->next;
}
// sync the cache store if needed
if (uwsgi.cache_store && uwsgi.cache_filesize) {
if (msync(uwsgi.cache_items, uwsgi.cache_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
if (uwsgi.queue_store && uwsgi.queue_filesize) {
if (msync(uwsgi.queue_header, uwsgi.queue_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
// update touches timestamps
uwsgi_check_touches(uwsgi.touch_reload);
uwsgi_check_touches(uwsgi.touch_logrotate);
uwsgi_check_touches(uwsgi.touch_logreopen);
// setup cheaper algos
uwsgi.cheaper_algo = uwsgi_cheaper_algo_spare;
if (uwsgi.requested_cheaper_algo) {
uwsgi.cheaper_algo = NULL;
struct uwsgi_cheaper_algo *uca = uwsgi.cheaper_algos;
while (uca) {
if (!strcmp(uca->name, uwsgi.requested_cheaper_algo)) {
uwsgi.cheaper_algo = uca->func;
break;
}
uca = uca->next;
}
if (!uwsgi.cheaper_algo) {
uwsgi_log("unable to find requested cheaper algorithm, falling back to spare\n");
uwsgi.cheaper_algo = uwsgi_cheaper_algo_spare;
}
}
// here really starts the master loop
for (;;) {
//uwsgi_log("uwsgi.ready_to_reload %d %d\n", uwsgi.ready_to_reload, uwsgi.numproc);
// run master_cycle hook for every plugin
for (i = 0; i < uwsgi.gp_cnt; i++) {
if (uwsgi.gp[i]->master_cycle) {
uwsgi.gp[i]->master_cycle();
}
}
for (i = 0; i < 256; i++) {
if (uwsgi.p[i]->master_cycle) {
uwsgi.p[i]->master_cycle();
}
}
uwsgi_daemons_smart_check();
// count the number of active workers
int active_workers = 0;
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
active_workers++;
}
}
if (uwsgi.to_outworld) {
//uwsgi_log("%d/%d\n", uwsgi.lazy_respawned, uwsgi.numproc);
if (uwsgi.lazy_respawned >= active_workers) {
uwsgi.to_outworld = 0;
uwsgi.master_mercy = 0;
uwsgi.lazy_respawned = 0;
}
}
if (uwsgi_master_check_mercy())
return 0;
if (uwsgi.respawn_workers) {
for (i = 1; i <= uwsgi.respawn_workers; i++) {
if (uwsgi_respawn_worker(i))
return 0;
}
uwsgi.respawn_workers = 0;
}
if (uwsgi.restore_snapshot) {
uwsgi_master_restore_snapshot();
continue;
}
// cheaper management
if (uwsgi.cheaper && !uwsgi.cheap && !uwsgi.to_heaven && !uwsgi.to_hell && !uwsgi.to_outworld && !uwsgi.workers[0].suspended) {
if (!uwsgi_calc_cheaper())
return 0;
}
if ((uwsgi.cheap || uwsgi.ready_to_die >= active_workers) && uwsgi.to_hell) {
// call a series of waitpid to ensure all processes (gateways, mules and daemons) are dead
for (i = 0; i < (ushared->gateways_cnt + uwsgi.daemons_cnt + uwsgi.mules_cnt); i++) {
diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG);
}
uwsgi_log("goodbye to uWSGI.\n");
exit(0);
}
if ((uwsgi.cheap || uwsgi.ready_to_reload >= active_workers) && uwsgi.to_heaven) {
uwsgi_reload(argv);
// never here (unless in shared library mode)
return -1;
}
diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG);
if (diedpid == -1) {
if (errno == ECHILD) {
// something did not work as expected, just assume all has been cleared
if (uwsgi.to_heaven) {
uwsgi.ready_to_reload = uwsgi.numproc;
continue;
}
else if (uwsgi.to_hell) {
uwsgi.ready_to_die = uwsgi.numproc;
continue;
}
else if (uwsgi.to_outworld) {
uwsgi.lazy_respawned = uwsgi.numproc;
uwsgi_log("*** no workers to reload found ***\n");
continue;
}
diedpid = 0;
}
else {
uwsgi_error("waitpid()");
/* here is better to reload all the uWSGI stack */
uwsgi_log("something horrible happened...\n");
reap_them_all(0);
exit(1);
}
}
if (diedpid == 0) {
/* all processes ok, doing status scan after N seconds */
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
if (!check_interval)
check_interval = 1;
// add unregistered file monitors
// locking is not needed as monitors can only increase
for (i = 0; i < ushared->files_monitored_cnt; i++) {
if (!ushared->files_monitored[i].registered) {
ushared->files_monitored[i].fd = event_queue_add_file_monitor(uwsgi.master_queue, ushared->files_monitored[i].filename, &ushared->files_monitored[i].id);
ushared->files_monitored[i].registered = 1;
}
}
// add unregistered timers
// locking is not needed as timers can only increase
for (i = 0; i < ushared->timers_cnt; i++) {
if (!ushared->timers[i].registered) {
ushared->timers[i].fd = event_queue_add_timer(uwsgi.master_queue, &ushared->timers[i].id, ushared->timers[i].value);
ushared->timers[i].registered = 1;
}
}
// add unregistered rb_timers
// locking is not needed as rb_timers can only increase
for (i = 0; i < ushared->rb_timers_cnt; i++) {
if (!ushared->rb_timers[i].registered) {
ushared->rb_timers[i].uwsgi_rb_timer = uwsgi_add_rb_timer(rb_timers, uwsgi_now() + ushared->rb_timers[i].value, &ushared->rb_timers[i]);
ushared->rb_timers[i].registered = 1;
}
}
int interesting_fd = -1;
if (ushared->rb_timers_cnt > 0) {
min_timeout = uwsgi_min_rb_timer(rb_timers);
if (min_timeout == NULL) {
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
}
else {
check_interval = min_timeout->key - uwsgi_now();
if (check_interval <= 0) {
expire_rb_timeouts(rb_timers);
check_interval = 0;
}
}
}
// wait for event
rlen = event_queue_wait(uwsgi.master_queue, check_interval, &interesting_fd);
if (rlen == 0) {
if (ushared->rb_timers_cnt > 0) {
expire_rb_timeouts(rb_timers);
}
}
// check uwsgi-cron table
if (ushared->cron_cnt) {
uwsgi_manage_signal_cron(uwsgi_now());
}
if (uwsgi.crons) {
uwsgi_manage_command_cron(uwsgi_now());
}
// check for probes
if (ushared->probes_cnt > 0) {
uwsgi_lock(uwsgi.probe_table_lock);
for (i = 0; i < ushared->probes_cnt; i++) {
if (interesting_fd == -1) {
// increment cycles
ushared->probes[i].cycles++;
}
if (ushared->probes[i].func(interesting_fd, &ushared->probes[i])) {
uwsgi_route_signal(ushared->probes[i].sig);
}
}
uwsgi_unlock(uwsgi.probe_table_lock);
}
if (rlen > 0) {
if (uwsgi.log_master && !uwsgi.threaded_logger) {
if (interesting_fd == uwsgi.shared->worker_log_pipe[0]) {
uwsgi_master_log();
goto health_cycle;
}
}
if (uwsgi.stats && uwsgi.stats_fd > -1) {
if (interesting_fd == uwsgi.stats_fd) {
uwsgi_send_stats(uwsgi.stats_fd);
goto health_cycle;
}
}
if (uwsgi.zerg_server) {
if (interesting_fd == uwsgi.zerg_server_fd) {
uwsgi_manage_zerg(uwsgi.zerg_server_fd, 0, NULL);
goto health_cycle;
}
}
if (uwsgi.has_emperor) {
if (interesting_fd == uwsgi.emperor_fd) {
uwsgi_master_manage_emperor();
goto health_cycle;
}
}
if (uwsgi.cheap) {
int found = 0;
struct uwsgi_socket *uwsgi_sock = uwsgi.sockets;
while (uwsgi_sock) {
if (interesting_fd == uwsgi_sock->fd) {
found = 1;
uwsgi.cheap = 0;
uwsgi_del_sockets_from_queue(uwsgi.master_queue);
int needed = uwsgi.numproc;
if (uwsgi.cheaper) {
needed = uwsgi.cheaper_count;
}
for (i = 1; i <= needed; i++) {
if (uwsgi_respawn_worker(i))
return 0;
}
break;
}
uwsgi_sock = uwsgi_sock->next;
}
// here is better to continue instead going to health_cycle
if (found)
continue;
}
#ifdef UWSGI_SNMP
if (uwsgi.snmp_addr && interesting_fd == snmp_fd) {
uwsgi_master_manage_snmp(snmp_fd);
goto health_cycle;
}
#endif
#ifdef UWSGI_UDP
if (uwsgi.udp_socket && interesting_fd == udp_fd) {
uwsgi_master_manage_udp(udp_fd);
goto health_cycle;
}
#ifdef UWSGI_MULTICAST
if (interesting_fd == uwsgi.cluster_fd) {
if (uwsgi_get_dgram(uwsgi.cluster_fd, &uwsgi.workers[0].cores[0].req)) {
goto health_cycle;
}
manage_cluster_message(cluster_opt_buf, cluster_opt_size);
goto health_cycle;
}
#endif
#endif
int next_iteration = 0;
uwsgi_lock(uwsgi.fmon_table_lock);
for (i = 0; i < ushared->files_monitored_cnt; i++) {
if (ushared->files_monitored[i].registered) {
if (interesting_fd == ushared->files_monitored[i].fd) {
struct uwsgi_fmon *uf = event_queue_ack_file_monitor(uwsgi.master_queue, interesting_fd);
// now call the file_monitor handler
if (uf)
uwsgi_route_signal(uf->sig);
break;
}
}
}
uwsgi_unlock(uwsgi.fmon_table_lock);
if (next_iteration)
goto health_cycle;;
next_iteration = 0;
uwsgi_lock(uwsgi.timer_table_lock);
for (i = 0; i < ushared->timers_cnt; i++) {
if (ushared->timers[i].registered) {
if (interesting_fd == ushared->timers[i].fd) {
struct uwsgi_timer *ut = event_queue_ack_timer(interesting_fd);
// now call the file_monitor handler
if (ut)
uwsgi_route_signal(ut->sig);
break;
}
}
}
uwsgi_unlock(uwsgi.timer_table_lock);
if (next_iteration)
goto health_cycle;;
// check for worker signal
if (interesting_fd == uwsgi.shared->worker_signal_pipe[0]) {
rlen = read(interesting_fd, &uwsgi_signal, 1);
if (rlen < 0) {
uwsgi_error("read()");
}
else if (rlen > 0) {
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("received uwsgi signal %d from a worker\n", uwsgi_signal);
#endif
uwsgi_route_signal(uwsgi_signal);
}
else {
uwsgi_log_verbose("lost connection with worker %d\n", i);
close(interesting_fd);
}
goto health_cycle;
}
#ifdef UWSGI_SPOOLER
// check for spooler signal
if (uwsgi.spoolers) {
if (interesting_fd == uwsgi.shared->spooler_signal_pipe[0]) {
rlen = read(interesting_fd, &uwsgi_signal, 1);
if (rlen < 0) {
uwsgi_error("read()");
}
else if (rlen > 0) {
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("received uwsgi signal %d from a spooler\n", uwsgi_signal);
#endif
uwsgi_route_signal(uwsgi_signal);
}
else {
uwsgi_log_verbose("lost connection with the spooler\n");
close(interesting_fd);
}
goto health_cycle;
}
}
#endif
// check for mules signal
if (uwsgi.mules_cnt > 0) {
if (interesting_fd == uwsgi.shared->mule_signal_pipe[0]) {
rlen = read(interesting_fd, &uwsgi_signal, 1);
if (rlen < 0) {
uwsgi_error("read()");
}
else if (rlen > 0) {
#ifdef UWSGI_DEBUG
uwsgi_log_verbose("received uwsgi signal %d from a mule\n", uwsgi_signal);
#endif
uwsgi_route_signal(uwsgi_signal);
}
else {
uwsgi_log_verbose("lost connection with a mule\n");
close(interesting_fd);
}
goto health_cycle;
}
}
}
health_cycle:
now = uwsgi_now();
if (now - uwsgi.current_time < 1) {
continue;
}
uwsgi.current_time = now;
// checking logsize
if (uwsgi.logfile) {
uwsgi_check_logrotate();
}
// this will be incremented at (more or less) regular intervals
uwsgi.master_cycles++;
// recalculate requests counter on race conditions risky configurations
// a bit of inaccuracy is better than locking;)
if (uwsgi.numproc > 1) {
tmp_counter = 0;
for (i = 1; i < uwsgi.numproc + 1; i++)
tmp_counter += uwsgi.workers[i].requests;
uwsgi.workers[0].requests = tmp_counter;
}
if (uwsgi.idle > 0 && !uwsgi.cheap) {
uwsgi.current_time = uwsgi_now();
if (!last_request_timecheck)
last_request_timecheck = uwsgi.current_time;
int busy_workers = 0;
for (i = 1; i <= uwsgi.numproc; i++) {
if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
if (uwsgi.workers[i].busy == 1) {
busy_workers = 1;
break;
}
}
}
if (last_request_count != uwsgi.workers[0].requests) {
last_request_timecheck = uwsgi.current_time;
last_request_count = uwsgi.workers[0].requests;
}
// a bit of over-engeneering to avoid clock skews
else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle) && !busy_workers) {
uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck);
uwsgi.cheap = 1;
if (uwsgi.die_on_idle) {
if (uwsgi.has_emperor) {
char byte = 22;
if (write(uwsgi.emperor_fd, &byte, 1) != 1) {
uwsgi_error("write()");
kill_them_all(0);
}
}
else {
kill_them_all(0);
}
continue;
}
for (i = 1; i <= uwsgi.numproc; i++) {
uwsgi.workers[i].cheaped = 1;
if (uwsgi.workers[i].pid == 0)
continue;
kill(uwsgi.workers[i].pid, SIGKILL);
if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) {
if (errno != ECHILD)
uwsgi_error("waitpid()");
}
}
uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1);
uwsgi_log("cheap mode enabled: waiting for socket connection...\n");
last_request_timecheck = 0;
continue;
}
}
check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL];
if (!check_interval)
check_interval = 1;
#ifdef __linux__
// get listen_queue status
struct uwsgi_socket *uwsgi_sock = uwsgi.sockets;
while (uwsgi_sock) {
if (uwsgi_sock->family == AF_INET) {
get_linux_tcp_info(uwsgi_sock->fd);
}
#ifdef SIOBKLGQ
else if (uwsgi_sock->family == AF_UNIX) {
get_linux_unbit_SIOBKLGQ(uwsgi_sock->fd);
}
#endif
uwsgi_sock = uwsgi_sock->next;
}
#endif
for (i = 1; i <= uwsgi.numproc; i++) {
/* first check for harakiri */
if (uwsgi.workers[i].harakiri > 0) {
if (uwsgi.workers[i].harakiri < (time_t) uwsgi.current_time) {
trigger_harakiri(i);
}
}
/* then user-defined harakiri */
if (uwsgi.workers[i].user_harakiri > 0) {
if (uwsgi.workers[i].user_harakiri < (time_t) uwsgi.current_time) {
trigger_harakiri(i);
}
}
// then for evil memory checkers
if (uwsgi.evil_reload_on_as) {
if ((rlim_t) uwsgi.workers[i].vsz_size >= uwsgi.evil_reload_on_as) {
uwsgi_log("*** EVIL RELOAD ON WORKER %d ADDRESS SPACE: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].vsz_size, uwsgi.workers[i].pid);
kill(uwsgi.workers[i].pid, SIGKILL);
uwsgi.workers[i].vsz_size = 0;
}
}
if (uwsgi.evil_reload_on_rss) {
if ((rlim_t) uwsgi.workers[i].rss_size >= uwsgi.evil_reload_on_rss) {
uwsgi_log("*** EVIL RELOAD ON WORKER %d RSS: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].rss_size, uwsgi.workers[i].pid);
kill(uwsgi.workers[i].pid, SIGKILL);
uwsgi.workers[i].rss_size = 0;
}
}
// need to find a better way
//uwsgi.workers[i].last_running_time = uwsgi.workers[i].running_time;
}
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways_harakiri[i] > 0) {
if (ushared->gateways_harakiri[i] < (time_t) uwsgi.current_time) {
if (ushared->gateways[i].pid > 0) {
kill(ushared->gateways[i].pid, SIGKILL);
}
ushared->gateways_harakiri[i] = 0;
}
}
}
for (i = 0; i < uwsgi.mules_cnt; i++) {
if (uwsgi.mules[i].harakiri > 0) {
if (uwsgi.mules[i].harakiri < (time_t) uwsgi.current_time) {
uwsgi_log("*** HARAKIRI ON MULE %d HANDLING SIGNAL %d (pid: %d) ***\n", i + 1, uwsgi.mules[i].signum, uwsgi.mules[i].pid);
kill(uwsgi.mules[i].pid, SIGKILL);
uwsgi.mules[i].harakiri = 0;
}
}
}
#ifdef UWSGI_SPOOLER
struct uwsgi_spooler *uspool = uwsgi.spoolers;
while (uspool) {
if (uspool->harakiri > 0 && uspool->harakiri < (time_t) uwsgi.current_time) {
uwsgi_log("*** HARAKIRI ON THE SPOOLER (pid: %d) ***\n", uspool->pid);
kill(uspool->pid, SIGKILL);
uspool->harakiri = 0;
}
uspool = uspool->next;
}
#endif
#ifdef __linux__
#ifdef MADV_MERGEABLE
if (uwsgi.linux_ksm > 0 && (uwsgi.master_cycles % uwsgi.linux_ksm) == 0) {
uwsgi_linux_ksm_map();
}
#endif
#endif
#ifdef UWSGI_UDP
// check for cluster nodes
master_check_cluster_nodes();
// reannounce myself every 10 cycles
if (uwsgi.cluster && uwsgi.cluster_fd >= 0 && !uwsgi.cluster_nodes && (uwsgi.master_cycles % 10) == 0) {
uwsgi_cluster_add_me();
}
// resubscribe every 10 cycles by default
if ((uwsgi.subscriptions && ((uwsgi.master_cycles % uwsgi.subscribe_freq) == 0 || uwsgi.master_cycles == 1)) && !uwsgi.to_heaven && !uwsgi.to_hell && !uwsgi.workers[0].suspended) {
struct uwsgi_string_list *subscriptions = uwsgi.subscriptions;
while (subscriptions) {
uwsgi_subscribe(subscriptions->value, 0);
subscriptions = subscriptions->next;
}
}
#endif
if (uwsgi.cache_store && uwsgi.cache_filesize && uwsgi.cache_store_sync && ((uwsgi.master_cycles % uwsgi.cache_store_sync) == 0)) {
if (msync(uwsgi.cache_items, uwsgi.cache_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
if (uwsgi.queue_store && uwsgi.queue_filesize && uwsgi.queue_store_sync && ((uwsgi.master_cycles % uwsgi.queue_store_sync) == 0)) {
if (msync(uwsgi.queue_header, uwsgi.queue_filesize, MS_ASYNC)) {
uwsgi_error("msync()");
}
}
// check touch_reload
if (!uwsgi.to_heaven && !uwsgi.to_hell) {
char *touched = uwsgi_check_touches(uwsgi.touch_reload);
if (touched) {
uwsgi_log("*** %s has been touched... grace them all !!! ***\n", touched);
uwsgi_block_signal(SIGHUP);
grace_them_all(0);
uwsgi_unblock_signal(SIGHUP);
}
}
continue;
}
// no one died
if (diedpid <= 0)
continue;
// check for deadlocks first
uwsgi_deadlock_check(diedpid);
// reload gateways and daemons only on normal workflow (+outworld status)
if (!uwsgi.to_heaven && !uwsgi.to_hell) {
#ifdef UWSGI_SPOOLER
/* reload the spooler */
struct uwsgi_spooler *uspool = uwsgi.spoolers;
pid_found = 0;
while (uspool) {
if (uspool->pid > 0 && diedpid == uspool->pid) {
uwsgi_log("OOOPS the spooler is no more...trying respawn...\n");
uspool->respawned++;
uspool->pid = spooler_start(uspool);
pid_found = 1;
break;
}
uspool = uspool->next;
}
if (pid_found)
continue;
#endif
pid_found = 0;
for (i = 0; i < uwsgi.mules_cnt; i++) {
if (uwsgi.mules[i].pid == diedpid) {
uwsgi_log("OOOPS mule %d (pid: %d) crippled...trying respawn...\n", i + 1, uwsgi.mules[i].pid);
uwsgi_mule(i + 1);
pid_found = 1;
break;
}
}
if (pid_found)
continue;
/* reload the gateways */
pid_found = 0;
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways[i].pid == diedpid) {
gateway_respawn(i);
pid_found = 1;
break;
}
}
if (pid_found)
continue;
/* reload the daemons */
pid_found = uwsgi_daemon_check_pid_reload(diedpid);
if (pid_found)
continue;
}
/* What happens here ?
case 1) the diedpid is not a worker, report it and continue
case 2) the diedpid is a worker and we are not in a reload procedure -> reload it
case 3) the diedpid is a worker and we are in graceful reload -> uwsgi.ready_to_reload++ and continue
case 3) the diedpid is a worker and we are in brutal reload -> uwsgi.ready_to_die++ and continue
*/
uwsgi.mywid = find_worker_id(diedpid);
if (uwsgi.mywid <= 0) {
// check spooler, mules, gateways and daemons
#ifdef UWSGI_SPOOLER
struct uwsgi_spooler *uspool = uwsgi.spoolers;
while (uspool) {
if (uspool->pid > 0 && diedpid == uspool->pid) {
uwsgi_log("spooler (pid: %d) annihilated\n", (int) diedpid);
goto next;
}
uspool = uspool->next;
}
#endif
for (i = 0; i < uwsgi.mules_cnt; i++) {
if (uwsgi.mules[i].pid == diedpid) {
uwsgi_log("mule %d (pid: %d) annihilated\n", i + 1, (int) diedpid);
goto next;
}
}
for (i = 0; i < ushared->gateways_cnt; i++) {
if (ushared->gateways[i].pid == diedpid) {
uwsgi_log("gateway %d (%s, pid: %d) annihilated\n", i + 1, ushared->gateways[i].fullname, (int) diedpid);
goto next;
}
}
if (uwsgi_daemon_check_pid_death(diedpid)) goto next;
if (WIFEXITED(waitpid_status)) {
uwsgi_log("subprocess %d exited with code %d\n", (int) diedpid, WEXITSTATUS(waitpid_status));
}
else if (WIFSIGNALED(waitpid_status)) {
uwsgi_log("subprocess %d exited by signal %d\n", (int) diedpid, WTERMSIG(waitpid_status));
}
else if (WIFSTOPPED(waitpid_status)) {
uwsgi_log("subprocess %d stopped\n", (int) diedpid);
}
next:
continue;
}
// ok a worker died...
if (uwsgi.to_heaven) {
uwsgi.ready_to_reload++;
uwsgi.workers[uwsgi.mywid].pid = 0;
// only to be safe :P
uwsgi.workers[uwsgi.mywid].harakiri = 0;
continue;
}
else if (uwsgi.to_hell) {
uwsgi.ready_to_die++;
uwsgi.workers[uwsgi.mywid].pid = 0;
// only to be safe :P
uwsgi.workers[uwsgi.mywid].harakiri = 0;
continue;
}
else if (uwsgi.to_outworld) {
uwsgi.lazy_respawned++;
uwsgi.workers[uwsgi.mywid].destroy = 0;
uwsgi.workers[uwsgi.mywid].pid = 0;
// only to be safe :P
uwsgi.workers[uwsgi.mywid].harakiri = 0;
}
if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_FAILED_APP_CODE) {
uwsgi_log("OOPS ! failed loading app in worker %d (pid %d) :( trying again...\n", uwsgi.mywid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_DE_HIJACKED_CODE) {
uwsgi_log("...restoring worker %d (pid: %d)...\n", uwsgi.mywid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_EXCEPTION_CODE) {
uwsgi_log("... monitored exception detected, respawning worker %d (pid: %d)...\n", uwsgi.mywid, (int) diedpid);
}
else if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_QUIET_CODE) {
// noop
}
else if (uwsgi.workers[uwsgi.mywid].manage_next_request) {
if (WIFSIGNALED(waitpid_status)) {
uwsgi_log("DAMN ! worker %d (pid: %d) died, killed by signal %d :( trying respawn ...\n", uwsgi.mywid, (int) diedpid, (int) WTERMSIG(waitpid_status));
}
else {
uwsgi_log("DAMN ! worker %d (pid: %d) died :( trying respawn ...\n", uwsgi.mywid, (int) diedpid);
}
}
else {
uwsgi_log("DAMN ! worker %d (pid: %d) MISTERIOUSLY died :( trying respawn ...\n", uwsgi.mywid, (int) diedpid);
}
if (uwsgi.workers[uwsgi.mywid].cheaped == 1) {
uwsgi.workers[uwsgi.mywid].pid = 0;
uwsgi_log("uWSGI worker %d cheaped.\n", uwsgi.mywid);
uwsgi.workers[uwsgi.mywid].harakiri = 0;
continue;
}
gettimeofday(&last_respawn, NULL);
if (last_respawn.tv_sec <= uwsgi.respawn_delta + check_interval) {
last_respawn_rate++;
if (last_respawn_rate > uwsgi.numproc) {
if (uwsgi.forkbomb_delay > 0) {
uwsgi_log("worker respawning too fast !!! i have to sleep a bit (%d seconds)...\n", uwsgi.forkbomb_delay);
/* use --forkbomb-delay 0 to disable sleeping */
sleep(uwsgi.forkbomb_delay);
}
last_respawn_rate = 0;
}
}
else {
last_respawn_rate = 0;
}
gettimeofday(&last_respawn, NULL);
uwsgi.respawn_delta = last_respawn.tv_sec;
if (uwsgi_respawn_worker(uwsgi.mywid))
return 0;
// end of the loop
}
// never here
}
