static ngx_int_t spool_nextmsg(subscriber_pool_t *spool, nchan_msg_id_t *new_last_id) {
subscriber_pool_t *newspool;
channel_spooler_t *spl = spool->spooler;
DBG("spool nextmsg %p (%i:%i) newid %i:%i", spool, spool->id.time, spool->id.tag, new_last_id->time, new_last_id->tag);
assert(spool->id.time != new_last_id->time || spool->id.tag != new_last_id->tag);
if((newspool = find_spool(spl, new_last_id)) != NULL) {
assert(spool != newspool);
spool_transfer_subscribers(spool, newspool, 0);
destroy_spool(spool);
}
else {
ngx_rbtree_node_t *node;
node = rbtree_node_from_data(spool);
rbtree_remove_node(&spl->spoolseed, node);
spool->id = *new_last_id;
rbtree_insert_node(&spl->spoolseed, node);
spool->msg_status = MSG_INVALID;
spool->msg = NULL;
newspool = spool;
/*
newspool = get_spool(spl, new_last_id);
assert(spool != newspool);
spool_transfer_subscribers(spool, newspool, 0);
destroy_spool(spool);
*/
}
if(newspool->sub_count > 0 && newspool->msg_status == MSG_INVALID) {
spool_fetch_msg(newspool);
}
return NGX_OK;
}
ngx_int_t stop_spooler(channel_spooler_t *spl, uint8_t dequeue_subscribers) {
ngx_rbtree_node_t *cur, *sentinel;
spooler_event_ll_t *ecur, *ecur_next;
subscriber_pool_t *spool;
rbtree_seed_t *seed = &spl->spoolseed;
ngx_rbtree_t *tree = &seed->tree;
ngx_int_t n=0;
sentinel = tree->sentinel;
fetchmsg_data_t *dcur;
#if NCHAN_RBTREE_DBG
ngx_int_t active_before = seed->active_nodes, allocd_before = seed->active_nodes;
#endif
if(spl->running) {
for(ecur = spl->spooler_dependent_events; ecur != NULL; ecur = ecur_next) {
ecur_next = ecur->next;
if(ecur->cancel) {
ecur->cancel(ecur->ev.data);
}
ngx_event_del_timer(&ecur->ev);
ngx_free(ecur);
}
for(cur = tree->root; cur != NULL && cur != sentinel; cur = tree->root) {
spool = (subscriber_pool_t *)rbtree_data_from_node(cur);
if(dequeue_subscribers) {
destroy_spool(spool);
}
else {
remove_spool(spool);
rbtree_destroy_node(seed, cur);
}
n++;
}
for(dcur = spl->fetchmsg_cb_data_list; dcur != NULL; dcur = dcur->next) {
dcur->spooler = NULL;
}
DBG("stopped %i spools in SPOOLER %p", n, *spl);
}
else {
DBG("SPOOLER %p not running", *spl);
}
#if NCHAN_RBTREE_DBG
assert(active_before - n == 0);
assert(allocd_before - n == 0);
assert(seed->active_nodes == 0);
assert(seed->allocd_nodes == 0);
#endif
nchan_free_msg_id(&spl->prev_msg_id);
spl->running = 0;
return NGX_OK;
}
static ngx_int_t spooler_respond_status(channel_spooler_t *self, nchan_msg_id_t *id, ngx_int_t status_code, ngx_str_t *status_line) {
subscriber_pool_t *spool = find_spool(self, id);
//validate_spooler(self, "before respond_status");
if(spool) {
if(status_code == NGX_HTTP_NO_CONTENT) {
spool->msg_status = MSG_EXPECTED;
}
spool_respond_general(spool, NULL, status_code, status_line, 0);
destroy_spool(spool);
}
//validate_spooler(self, "after respond_status");
return NGX_OK;
}
static ngx_int_t spool_fetch_msg_callback(nchan_msg_status_t findmsg_status, nchan_msg_t *msg, fetchmsg_data_t *data) {
nchan_msg_id_t anymsg = {0,0};
subscriber_pool_t *spool, *nuspool;
if((spool = find_spool(data->spooler, &data->msgid)) == NULL) {
ERR("spool for msgid %i:%i not found. discarding getmsg callback response.", data->msgid.time, data->msgid.tag);
ngx_free(data);
return NGX_ERROR;
}
ngx_free(data);
spool->msg_status = findmsg_status;
switch(findmsg_status) {
case MSG_FOUND:
DBG("fetchmsg callback for spool %p msg FOUND %p %i:%i", spool, msg, msg->id.time, msg->id.tag);
assert(msg != NULL);
spool->msg = msg;
spool_respond_general(spool, spool->msg, 0, NULL);
spool_nextmsg(spool, &msg->id);
break;
case MSG_EXPECTED:
// ♫ It's gonna be the future soon ♫
DBG("fetchmsg callback for spool %p msg EXPECTED", spool);
assert(msg == NULL);
spool->msg = NULL;
break;
case MSG_NOTFOUND:
case MSG_EXPIRED:
//is this right?
//TODO: maybe message-expired notification
//spool_respond_general(spool, NULL, NGX_HTTP_NO_CONTENT, NULL);
nuspool = get_spool(spool->spooler, &anymsg);
assert(spool != nuspool);
spool_transfer_subscribers(spool, nuspool, 1);
destroy_spool(spool);
break;
default:
assert(0);
break;
}
return NGX_OK;
}
ngx_int_t stop_spooler(channel_spooler_t *spl, uint8_t dequeue_subscribers) {
ngx_rbtree_node_t *cur, *sentinel;
subscriber_pool_t *spool;
rbtree_seed_t *seed = &spl->spoolseed;
ngx_rbtree_t *tree = &seed->tree;
ngx_int_t n=0;
sentinel = tree->sentinel;
#if NCHAN_RBTREE_DBG
ngx_int_t active_before = seed->active_nodes, allocd_before = seed->active_nodes;
#endif
if(spl->running) {
for(cur = tree->root; cur != NULL && cur != sentinel; cur = tree->root) {
spool = (subscriber_pool_t *)rbtree_data_from_node(cur);
if(dequeue_subscribers) {
destroy_spool(spool);
}
else {
remove_spool(spool);
rbtree_destroy_node(seed, cur);
}
n++;
}
DBG("stopped %i spools in SPOOLER %p", n, *spl);
}
else {
DBG("SPOOLER %p not running", *spl);
}
#if NCHAN_RBTREE_DBG
assert(active_before - n == 0);
assert(allocd_before - n == 0);
assert(seed->active_nodes == 0);
assert(seed->allocd_nodes == 0);
#endif
spl->running = 0;
return NGX_OK;
}
void spooler_manage_task(struct uwsgi_spooler *uspool, char *dir, char *task) {
int i, ret;
char spool_buf[0xffff];
struct uwsgi_header uh;
char *body = NULL;
size_t body_len = 0;
int spool_fd;
if (!dir) dir = uspool->dir;
if (!strncmp("uwsgi_spoolfile_on_", task, 19) || (uwsgi.spooler_ordered && is_a_number(task))) {
struct stat sf_lstat;
if (lstat(task, &sf_lstat)) {
return;
}
// a spool request for the future
if (sf_lstat.st_mtime > uwsgi_now()) {
return;
}
#ifdef __linux__
if (S_ISDIR(sf_lstat.st_mode) && uwsgi.spooler_ordered) {
if (chdir(task)) {
uwsgi_error("chdir()");
return;
}
char *prio_path = realpath(".", NULL);
spooler_scandir(uspool, prio_path);
free(prio_path);
if (chdir(dir)) {
uwsgi_error("chdir()");
}
return;
}
#endif
if (!S_ISREG(sf_lstat.st_mode)) {
return;
}
if (!access(task, R_OK | W_OK)) {
spool_fd = open(task, O_RDWR);
if (spool_fd < 0) {
if (errno != ENOENT)
uwsgi_error_open(task);
return;
}
// check if the file is locked by another process
if (uwsgi_fcntl_is_locked(spool_fd)) {
uwsgi_protected_close(spool_fd);
return;
}
// unlink() can destroy the lock !!!
if (access(task, R_OK | W_OK)) {
uwsgi_protected_close(spool_fd);
return;
}
ssize_t rlen = uwsgi_protected_read(spool_fd, &uh, 4);
if (rlen != 4) {
// it could be here for broken file or just opened one
if (rlen < 0)
uwsgi_error("read()");
uwsgi_protected_close(spool_fd);
return;
}
#ifdef __BIG_ENDIAN__
uh.pktsize = uwsgi_swap16(uh.pktsize);
#endif
if (uwsgi_protected_read(spool_fd, spool_buf, uh.pktsize) != uh.pktsize) {
uwsgi_error("read()");
destroy_spool(dir, task);
uwsgi_protected_close(spool_fd);
return;
}
// body available ?
if (sf_lstat.st_size > (uh.pktsize+4)) {
body_len = sf_lstat.st_size - (uh.pktsize+4);
body = uwsgi_malloc(body_len);
if ((size_t)uwsgi_protected_read(spool_fd, body, body_len) != body_len) {
uwsgi_error("read()");
destroy_spool(dir, task);
uwsgi_protected_close(spool_fd);
free(body);
return;
}
}
// now the task is running and should not be waken up
uspool->running = 1;
if (!uwsgi.spooler_quiet)
uwsgi_log("[spooler %s pid: %d] managing request %s ...\n", uspool->dir, (int) uwsgi.mypid, task);
// chdir before running the task (if requested)
if (uwsgi.spooler_chdir) {
if (chdir(uwsgi.spooler_chdir)) {
uwsgi_error("chdir()");
}
}
int callable_found = 0;
for(i=0; i<256; i++) {
if (uwsgi.p[i]->spooler) {
time_t now = uwsgi_now();
if(uwsgi.shared->options[UWSGI_OPTION_SPOOLER_HARAKIRI] > 0) {
set_spooler_harakiri(uwsgi.shared->options[UWSGI_OPTION_SPOOLER_HARAKIRI]);
}
ret = uwsgi.p[i]->spooler(task, spool_buf, uh.pktsize, body, body_len);
if(uwsgi.shared->options[UWSGI_OPTION_SPOOLER_HARAKIRI] > 0) {
set_spooler_harakiri(0);
}
if (ret == 0) continue;
callable_found = 1;
// increase task counter
uspool->tasks++;
if (ret == -2) {
if (!uwsgi.spooler_quiet)
uwsgi_log("[spooler %s pid: %d] done with task %s after %d seconds\n", uspool->dir, (int) uwsgi.mypid, task, uwsgi_now()-now);
destroy_spool(dir, task);
}
// re-spool it
break;
}
}
if (body)
free(body);
// here we free and unlock the task
uwsgi_protected_close(spool_fd);
uspool->running = 0;
if (chdir(dir)) {
uwsgi_error("chdir()");
uwsgi_log("[spooler] something horrible happened to the spooler. Better to kill it.\n");
exit(1);
}
if (!callable_found) {
uwsgi_log("unable to find the spooler function, have you loaded it into the spooler process ?\n");
}
}
}
}
static ngx_int_t spool_fetch_msg_callback(nchan_msg_status_t findmsg_status, nchan_msg_t *msg, fetchmsg_data_t *data) {
nchan_msg_status_t prev_status;
subscriber_pool_t *spool, *nuspool;
channel_spooler_t *spl = data->spooler;
int free_msg_id = 1;
if(spl && data == spl->fetchmsg_cb_data_list) {
spl->fetchmsg_cb_data_list = data->next;
}
if(data->next) {
data->next->prev = data->prev;
}
if(data->prev) {
data->prev->next = data->next;
}
if(spl == NULL) { //channel already deleted
nchan_free_msg_id(&data->msgid);
ngx_free(data);
return NGX_OK;
}
if(spl->handlers->get_message_finish) {
spl->handlers->get_message_finish(spl, spl->handlers_privdata);
}
if((spool = find_spool(spl, &data->msgid)) == NULL) {
DBG("spool for msgid %V not found. discarding getmsg callback response.", msgid_to_str(&data->msgid));
nchan_free_msg_id(&data->msgid);
ngx_free(data);
return NGX_ERROR;
}
prev_status = spool->msg_status;
switch(findmsg_status) {
case MSG_FOUND:
spool->msg_status = findmsg_status;
DBG("fetchmsg callback for spool %p msg FOUND %p %V", spool, msg, msgid_to_str(&msg->id));
assert(msg != NULL);
spool->msg = msg;
spool_respond_general(spool, spool->msg, 0, NULL, 0);
spool_nextmsg(spool, &msg->id);
break;
case MSG_EXPECTED:
// ♫ It's gonna be the future soon ♫
if(spool->id.time == NCHAN_NTH_MSGID_TIME) {
//wait for message in the NEWEST_ID spool
nchan_msg_id_t newest_id = NCHAN_NEWEST_MSGID;
spool_nextmsg(spool, &newest_id);
}
else {
spool->msg_status = findmsg_status;
DBG("fetchmsg callback for spool %p msg EXPECTED", spool);
spool_respond_general(spool, NULL, NGX_HTTP_NO_CONTENT, NULL, 0);
assert(msg == NULL);
spool->msg = NULL;
}
break;
case MSG_NORESPONSE:
if(prev_status == MSG_PENDING) {
spool->msg_status = MSG_INVALID;
if(spool->sub_count > 0) {
nomsg_retry_data_t *retry_data = ngx_alloc(sizeof(*retry_data), ngx_cycle->log);
retry_data->spooler = spl;
free_msg_id = 0;
retry_data->msg_id = data->msgid;
spooler_add_timer(spl, NCHAN_MSG_NORESPONSE_RETRY_TIME, spool_fetch_msg_noresponse_retry_callback, spool_fetch_msg_noresponse_retry_cancel, retry_data);
}
}
break;
case MSG_NOTFOUND:
if(spl->fetching_strategy == FETCH_IGNORE_MSG_NOTFOUND) {
spool->msg_status = prev_status;
break;
}
case MSG_EXPIRED:
//is this right?
//TODO: maybe message-expired notification
spool->msg_status = findmsg_status;
spool_respond_general(spool, NULL, NGX_HTTP_NO_CONTENT, NULL, 0);
nuspool = get_spool(spool->spooler, &oldest_msg_id);
if(spool != nuspool) {
spool_transfer_subscribers(spool, nuspool, 1);
destroy_spool(spool);
}
else {
ERR("Unexpected spool == nuspool during spool fetch_msg_callback. This is weird, please report this to the developers. findmsg_status: %i", findmsg_status);
assert(0);
}
break;
case MSG_PENDING:
ERR("spool %p set status to MSG_PENDING", spool);
break;
default:
assert(0);
break;
}
if(free_msg_id) {
nchan_free_msg_id(&data->msgid);
}
ngx_free(data);
return NGX_OK;
}
static ngx_int_t spool_nextmsg(subscriber_pool_t *spool, nchan_msg_id_t *new_last_id) {
subscriber_pool_t *newspool;
channel_spooler_t *spl = spool->spooler;
ngx_int_t immortal_spool = spool->id.time == NCHAN_NEWEST_MSGID_TIME;
int16_t largetags[NCHAN_MULTITAG_MAX];
nchan_msg_id_t new_id = NCHAN_ZERO_MSGID;
nchan_copy_msg_id(&new_id, &spool->id, largetags);
nchan_update_multi_msgid(&new_id, new_last_id, largetags);
//ERR("spool %p nextmsg (%V) --", spool, msgid_to_str(&spool->id));
//ERR(" -- update with (%V) --", msgid_to_str(new_last_id));
//ERR(" -- newid %V", msgid_to_str(&new_id));
if(msg_ids_equal(&spool->id, &new_id)) {
ERR("nextmsg id same as curmsg (%V)", msgid_to_str(&spool->id));
assert(0);
}
else {
newspool = !immortal_spool ? find_spool(spl, &new_id) : get_spool(spl, &new_id);
if(newspool != NULL) {
assert(spool != newspool);
spool_transfer_subscribers(spool, newspool, 0);
if(!immortal_spool) destroy_spool(spool);
}
else {
ngx_rbtree_node_t *node;
assert(!immortal_spool);
node = rbtree_node_from_data(spool);
rbtree_remove_node(&spl->spoolseed, node);
nchan_copy_msg_id(&spool->id, &new_id, NULL);
rbtree_insert_node(&spl->spoolseed, node);
spool->msg_status = MSG_INVALID;
spool->msg = NULL;
newspool = spool;
/*
newspool = get_spool(spl, &new_id);
assert(spool != newspool);
spool_transfer_subscribers(spool, newspool, 0);
destroy_spool(spool);
*/
}
if(newspool->non_internal_sub_count > 0 && spl->handlers->bulk_post_subscribe != NULL) {
spl->handlers->bulk_post_subscribe(spl, newspool->non_internal_sub_count, spl->handlers_privdata);
}
if(newspool->sub_count > 0) {
switch(newspool->msg_status) {
case MSG_CHANNEL_NOTREADY:
newspool->msg_status = MSG_INVALID;
case MSG_INVALID:
spool_fetch_msg(newspool);
break;
case MSG_EXPECTED:
spool_respond_general(newspool, NULL, NGX_HTTP_NO_CONTENT, NULL, 0);
break;
default:
break;
}
}
}
return NGX_OK;
}
void spooler_manage_task(struct uwsgi_spooler *uspool, char *dir, char *task) {
int i, ret;
char spool_buf[0xffff];
struct uwsgi_header uh;
char *body = NULL;
size_t body_len = 0;
int spool_fd;
if (!dir)
dir = uspool->dir;
if (!strncmp("uwsgi_spoolfile_on_", task, 19) || (uwsgi.spooler_ordered && is_a_number(task))) {
struct stat sf_lstat;
if (lstat(task, &sf_lstat)) {
return;
}
// a spool request for the future
if (sf_lstat.st_mtime > uwsgi_now()) {
return;
}
if (S_ISDIR(sf_lstat.st_mode) && uwsgi.spooler_ordered) {
if (chdir(task)) {
uwsgi_error("spooler_manage_task()/chdir()");
return;
}
#ifdef __UCLIBC__
char *prio_path = uwsgi_malloc(PATH_MAX);
realpath(".", prio_path);
#else
char *prio_path = realpath(".", NULL);
#endif
spooler_scandir(uspool, prio_path);
free(prio_path);
if (chdir(dir)) {
uwsgi_error("spooler_manage_task()/chdir()");
}
return;
}
if (!S_ISREG(sf_lstat.st_mode)) {
return;
}
if (!access(task, R_OK | W_OK)) {
spool_fd = open(task, O_RDWR);
if (spool_fd < 0) {
if (errno != ENOENT)
uwsgi_error_open(task);
return;
}
if (uwsgi_spooler_read_header(task, spool_fd, &uh))
return;
// access lstat second time after getting a lock
// first-time lstat could be dirty (for example between writes in master)
if (lstat(task, &sf_lstat)) {
return;
}
if (uwsgi_spooler_read_content(spool_fd, spool_buf, &body, &body_len, &uh, &sf_lstat)) {
destroy_spool(dir, task);
return;
}
// now the task is running and should not be woken up
uspool->running = 1;
// this is used in cheap mode for making decision about who must die
uspool->last_task_managed = uwsgi_now();
if (!uwsgi.spooler_quiet)
uwsgi_log("[spooler %s pid: %d] managing request %s ...\n", uspool->dir, (int) uwsgi.mypid, task);
// chdir before running the task (if requested)
if (uwsgi.spooler_chdir) {
if (chdir(uwsgi.spooler_chdir)) {
uwsgi_error("spooler_manage_task()/chdir()");
}
}
int callable_found = 0;
for (i = 0; i < 256; i++) {
if (uwsgi.p[i]->spooler) {
time_t now = uwsgi_now();
if (uwsgi.harakiri_options.spoolers > 0) {
set_spooler_harakiri(uwsgi.harakiri_options.spoolers);
}
ret = uwsgi.p[i]->spooler(task, spool_buf, uh._pktsize, body, body_len);
if (uwsgi.harakiri_options.spoolers > 0) {
set_spooler_harakiri(0);
}
if (ret == 0)
continue;
callable_found = 1;
// increase task counter
uspool->tasks++;
if (ret == -2) {
if (!uwsgi.spooler_quiet)
uwsgi_log("[spooler %s pid: %d] done with task %s after %lld seconds\n", uspool->dir, (int) uwsgi.mypid, task, (long long) uwsgi_now() - now);
destroy_spool(dir, task);
}
// re-spool it
break;
}
}
if (body)
free(body);
// here we free and unlock the task
uwsgi_protected_close(spool_fd);
uspool->running = 0;
// need to recycle ?
if (uwsgi.spooler_max_tasks > 0 && uspool->tasks >= (uint64_t) uwsgi.spooler_max_tasks) {
uwsgi_log("[spooler %s pid: %d] maximum number of tasks reached (%d) recycling ...\n", uspool->dir, (int) uwsgi.mypid, uwsgi.spooler_max_tasks);
end_me(0);
}
if (chdir(dir)) {
uwsgi_error("chdir()");
uwsgi_log("[spooler] something horrible happened to the spooler. Better to kill it.\n");
exit(1);
}
if (!callable_found) {
uwsgi_log("unable to find the spooler function, have you loaded it into the spooler process ?\n");
}
}
}
}
