static void ipc_record_alert_receive_delay(ngx_uint_t delay) {
delayed_received_alerts_count ++;
delayed_received_alerts_delay += delay;
nchan_update_stub_status(ipc_total_receive_delay, delay);
if(!receive_alert_delay_log_timer.timer_set && !ngx_exiting && !ngx_quit) {
ngx_add_timer(&receive_alert_delay_log_timer, 1000);
}
}
static void ipc_read_handler(ngx_event_t *ev) {
DBG("IPC channel handler");
//copypasta from os/unix/ngx_process_cycle.c (ngx_channel_handler)
ngx_int_t n;
ipc_alert_t alert;
ngx_connection_t *c;
if (ev->timedout) {
ev->timedout = 0;
return;
}
c = ev->data;
while(1) {
n = ipc_read_socket(c->fd, &alert, ev->log);
if (n == NGX_ERROR) {
ERR("IPC_READ_SOCKET failed: bad connection. This should never have happened, yet here we are...");
assert(0);
return;
}
if (n == NGX_AGAIN) {
return;
}
//ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, "nchan: channel command: %d", ch.command);
assert(n == sizeof(alert));
if(alert.worker_generation < memstore_worker_generation) {
ERR("Got IPC alert for previous generation's worker. discarding.");
}
else {
#if DEBUG_DELAY_IPC_RECEIVE_ALERT_MSEC
delayed_alert_glob_t *glob = ngx_alloc(sizeof(*glob), ngx_cycle->log);
if (NULL == glob) {
ERR("Couldn't allocate memory for alert glob data.");
return;
}
ngx_memzero(&glob->timer, sizeof(glob->timer));
nchan_init_timer(&glob->timer, fake_ipc_alert_delay_handler, glob);
glob->alert = alert;
glob->ipc = (ipc_t *)c->data;
ngx_add_timer(&glob->timer, DEBUG_DELAY_IPC_RECEIVE_ALERT_MSEC);
#else
if(ngx_time() - alert.time_sent >= 2) {
ipc_record_alert_receive_delay(ngx_time() - alert.time_sent);
}
nchan_update_stub_status(ipc_total_alerts_received, 1);
((ipc_t *)c->data)->handler(alert.src_slot, alert.code, alert.data);
#endif
}
}
}
static void nchan_publisher_post_request(ngx_http_request_t *r, ngx_str_t *content_type, size_t content_length, ngx_chain_t *request_body_chain, ngx_str_t *channel_id, nchan_loc_conf_t *cf) {
ngx_buf_t *buf;
nchan_msg_t *msg;
ngx_str_t *eventsource_event;
safe_request_ptr_t *pd;
#if FAKESHARD
memstore_pub_debug_start();
#endif
if((msg = ngx_pcalloc(r->pool, sizeof(*msg))) == NULL) {
nchan_log_request_error(r, "can't allocate msg in request pool");
nchan_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
msg->storage = NCHAN_MSG_POOL;
if(cf->eventsource_event.len > 0) {
msg->eventsource_event = &cf->eventsource_event;
}
else if((eventsource_event = nchan_get_header_value(r, NCHAN_HEADER_EVENTSOURCE_EVENT)) != NULL) {
msg->eventsource_event = eventsource_event;
}
//content type
if(content_type) {
msg->content_type = content_type;
}
if(content_length == 0) {
buf = ngx_create_temp_buf(r->pool, 0);
}
else if(request_body_chain!=NULL) {
buf = nchan_chain_to_single_buffer(r->pool, request_body_chain, content_length);
}
else {
nchan_log_request_error(r, "unexpected publisher message request body buffer location. please report this to the nchan developers.");
nchan_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
msg->id.time = 0;
msg->id.tag.fixed[0] = 0;
msg->id.tagactive = 0;
msg->id.tagcount = 1;
msg->buf = *buf;
#if NCHAN_MSG_LEAK_DEBUG
msg->lbl = r->uri;
#endif
#if NCHAN_BENCHMARK
nchan_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_nchan_module);
msg->start_tv = ctx->start_tv;
#endif
nchan_deflate_message_if_needed(msg, cf, r, r->pool);
if((pd = nchan_set_safe_request_ptr(r)) == NULL) {
return;
}
cf->storage_engine->publish(channel_id, msg, cf, (callback_pt) &publish_callback, pd);
nchan_update_stub_status(total_published_messages, 1);
#if FAKESHARD
memstore_pub_debug_end();
#endif
}
ngx_int_t ipc_alert(ipc_t *ipc, ngx_int_t slot, ngx_uint_t code, void *data, size_t data_size) {
DBG("IPC send alert code %i to slot %i", code, slot);
if(data_size > IPC_DATA_SIZE) {
ERR("IPC_DATA_SIZE too small. wanted %i, have %i", data_size, IPC_DATA_SIZE);
assert(0);
}
nchan_update_stub_status(ipc_total_alerts_sent, 1);
#if (FAKESHARD)
ipc_alert_t alert = {0};
alert.src_slot = memstore_slot();
alert.time_sent = ngx_time();
alert.worker_generation = memstore_worker_generation;
alert.code = code;
ngx_memcpy(alert.data, data, data_size);
//switch to destination
memstore_fakeprocess_push(slot);
ipc->handler(alert.src_slot, alert.code, alert.data);
memstore_fakeprocess_pop();
//switch back
#else
ipc_process_t *proc = &ipc->process[slot];
ipc_writebuf_t *wb = &proc->wbuf;
ipc_alert_t *alert;
assert(proc->active);
nchan_update_stub_status(ipc_queue_size, 1);
if(wb->n < IPC_WRITEBUF_SIZE) {
alert = &wb->alerts[(wb->first + wb->n++) % IPC_WRITEBUF_SIZE];
}
else { //overflow
ipc_writebuf_overflow_t *overflow;
DBG("writebuf overflow, allocating memory");
if((overflow = ngx_alloc(sizeof(*overflow), ngx_cycle->log)) == NULL) {
ERR("can't allocate memory for IPC write buffer overflow");
return NGX_ERROR;
}
overflow->next = NULL;
alert= &overflow->alert;
if(wb->overflow_first == NULL) {
wb->overflow_first = overflow;
}
if(wb->overflow_last) {
wb->overflow_last->next = overflow;
}
wb->overflow_last = overflow;
wb->overflow_n++;
}
alert->src_slot = ngx_process_slot;
alert->time_sent = ngx_time();
alert->code = code;
alert->worker_generation = memstore_worker_generation;
ngx_memcpy(&alert->data, data, data_size);
ipc_write_handler(proc->c->write);
//ngx_handle_write_event(ipc->c[slot]->write, 0);
//ngx_add_event(ipc->c[slot]->write, NGX_WRITE_EVENT, NGX_CLEAR_EVENT);
#endif
return NGX_OK;
}
static void ipc_write_handler(ngx_event_t *ev) {
ngx_connection_t *c = ev->data;
ngx_socket_t fd = c->fd;
ipc_process_t *proc = (ipc_process_t *) c->data;
ipc_alert_t *alerts = proc->wbuf.alerts;
int n = proc->wbuf.n;
int i, first = proc->wbuf.first, last = first + n;
uint8_t write_aborted = 0;
//DBG("%i alerts to write, with %i in overflow", proc->wbuf.n, proc->wbuf.overflow_n);
if(!memstore_ready()) {
#if nginx_version >= 1008000
ev->cancelable = 1;
#endif
ngx_add_timer(ev, 1000);
return;
}
#if nginx_version >= 1008000
else {
ev->cancelable = 0;
}
#endif
for(i = first; i < last; i++) {
//ERR("send alert at %i", i % IPC_WRITEBUF_SIZE );
if(ipc_write_alert_fd(fd, &alerts[i % IPC_WRITEBUF_SIZE]) != NGX_OK) {
write_aborted = 1;
//DBG("write aborted at %i iter. first: %i, n: %i", i - first, first, proc->wbuf.n);
break;
}
/*
else {
DBG("wrote alert at %i", i % IPC_WRITEBUF_SIZE);
}
*/
}
if(i==last) { //sent all outstanding alerts
//DBG("finished writing %i alerts.", proc->wbuf.n);
proc->wbuf.first = 0; // for debugging and stuff
proc->wbuf.n = 0;
}
else {
proc->wbuf.first = i;
proc->wbuf.n -= (i - first);
//DBG("first now at %i, %i alerts remain", i, proc->wbuf.n);
}
nchan_update_stub_status(ipc_queue_size, proc->wbuf.n - n);
if(proc->wbuf.overflow_n > 0 && i - first > 0) {
ipc_writebuf_overflow_t *of;
first = proc->wbuf.first + proc->wbuf.n;
last = first + (IPC_WRITEBUF_SIZE - proc->wbuf.n);
//DBG("try to squeeze in overflow between %i and %i", first, last);
for(i = first; i < last; i++) {
of = proc->wbuf.overflow_first;
///DBG("looking at overflow %p next %p", of, of->next);
alerts[i % IPC_WRITEBUF_SIZE] = of->alert;
proc->wbuf.overflow_n--;
proc->wbuf.n++;
assert(proc->wbuf.overflow_n >= 0);
proc->wbuf.overflow_first = of->next;
ngx_free(of);
//DBG("squeezed in overflow at %i, %i overflow remaining", i, proc->wbuf.overflow_n);
if(proc->wbuf.overflow_first == NULL) {
proc->wbuf.overflow_last = NULL;
break;
}
}
if(!write_aborted) {
//retry
//DBG("retry write after squeezing in overflow");
ipc_write_handler(ev);
return;
}
}
if(write_aborted) {
//DBG("re-add event because the write failed");
ngx_handle_write_event(c->write, 0);
}
}
