static ssize_t
ngx_zeromq_send_part(void *zmq, ngx_event_t *wev, u_char *buf, size_t size,
int flags)
{
zmq_msg_t zmq_msg;
if (zmq_msg_init_size(&zmq_msg, size) == -1) {
ngx_log_error(NGX_LOG_ALERT, wev->log, 0,
"zmq_msg_init_size(%uz) failed (%d: %s)",
size, ngx_errno, zmq_strerror(ngx_errno));
goto failed_zmq;
}
ngx_memcpy(zmq_msg_data(&zmq_msg), buf, size);
for (;;) {
if (zmq_sendmsg(zmq, &zmq_msg, ZMQ_DONTWAIT|flags) == -1) {
if (ngx_errno == NGX_EINTR) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, wev->log, 0,
"zmq_send: interrupted");
wev->ready = 0;
continue;
}
ngx_zeromq_log_error(wev->log, "zmq_sendmsg()");
goto failed;
}
break;
}
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, wev->log, 0,
"zmq_send: %uz eom:%d", size, flags != ZMQ_SNDMORE);
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(wev->log, "zmq_msg_close()");
goto failed_zmq;
}
return size;
failed:
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(wev->log, "zmq_msg_close()");
}
failed_zmq:
wev->error = 1;
return NGX_ERROR;
}
static void
ngx_zeromq_process_exit(ngx_cycle_t *cycle)
{
if (zmq_context != NULL && zmq_term(zmq_context) == -1) {
ngx_zeromq_log_error(cycle->log, "zmq_term()");
}
}
static ssize_t
ngx_zeromq_send_part(void *zmq, ngx_event_t *wev, u_char *buf, size_t size,
int flags)
{
zmq_msg_t zmq_msg;
ssize_t n;
if (zmq_msg_init_size(&zmq_msg, size) == -1) {
ngx_log_error(NGX_LOG_ALERT, wev->log, 0,
"zmq_msg_init_size(%uz) failed (%d: %s)",
size, ngx_errno, zmq_strerror(ngx_errno));
return NGX_ERROR;
}
ngx_memcpy(zmq_msg_data(&zmq_msg), buf, size);
n = ngx_zeromq_sendmsg(zmq, wev, &zmq_msg, flags);
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(wev->log, "zmq_msg_close()");
return NGX_ERROR;
}
return n;
}
static ssize_t
ngx_zeromq_recvmsg(void *zmq, ngx_event_t *ev, zmq_msg_t *msg)
{
int more;
for (;;) {
if (zmq_msg_recv(msg, zmq, ZMQ_DONTWAIT) == -1) {
if (ngx_errno == NGX_EAGAIN) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_recv: not ready");
ev->ready = 0;
return NGX_AGAIN;
}
if (ngx_errno == NGX_EINTR) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_recv: interrupted");
ev->ready = 0;
continue;
}
ngx_zeromq_log_error(ev->log, "zmq_msg_recv()");
ev->error = 1;
return NGX_ERROR;
}
break;
}
more = zmq_msg_get(msg, ZMQ_MORE);
if (more == -1) {
ngx_zeromq_log_error(ev->log, "zmq_msg_more()");
ev->error = 1;
return NGX_ERROR;
}
ev->eof = more ? 0 : 1;
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_recv: %uz eom:%d", zmq_msg_size(msg), ev->eof);
return zmq_msg_size(msg);
}
void
ngx_zeromq_close(ngx_connection_t *c)
{
void *zmq;
if (c->fd == -1) {
return;
}
zmq = c->data;
ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
"zmq_close: zmq:%p fd:%d #%d", zmq, c->fd, c->number);
if (c->read->timer_set) {
ngx_del_timer(c->read);
}
if (c->write->timer_set) {
ngx_del_timer(c->write);
}
if (ngx_del_conn) {
ngx_del_conn(c, NGX_CLOSE_EVENT);
} else {
if (c->read->active || c->read->disabled) {
ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
}
if (c->write->active || c->write->disabled) {
ngx_del_event(c->write, NGX_WRITE_EVENT, NGX_CLOSE_EVENT);
}
}
if (c->read->prev) {
ngx_delete_posted_event(c->read);
}
if (c->write->prev) {
ngx_delete_posted_event(c->write);
}
c->read->closed = 1;
c->write->closed = 1;
ngx_reusable_connection(c, 0);
ngx_free_connection(c);
c->fd = (ngx_socket_t) -1;
if (zmq_close(zmq) == -1) {
ngx_zeromq_log_error(ngx_cycle->log, "zmq_close()");
}
}
static void
ngx_zeromq_process_exit(ngx_cycle_t *cycle)
{
if (ngx_zeromq_ctx) {
if (zmq_ctx_destroy(ngx_zeromq_ctx) == -1) {
ngx_zeromq_log_error(cycle->log, "zmq_ctx_destroy()");
}
ngx_zeromq_ctx = NULL;
}
}
static ngx_int_t
ngx_zeromq_process_init(ngx_cycle_t *cycle)
{
ngx_zeromq_conf_t *zcf;
zcf = (ngx_zeromq_conf_t *) ngx_get_conf(cycle->conf_ctx,
ngx_zeromq_module);
if (ngx_zeromq_used) {
ngx_zeromq_ctx = zmq_ctx_new();
if (ngx_zeromq_ctx == NULL) {
ngx_zeromq_log_error(cycle->log, "zmq_ctx_new()");
return NGX_ERROR;
}
if (zmq_ctx_set(ngx_zeromq_ctx, ZMQ_IO_THREADS, zcf->threads) == -1) {
ngx_zeromq_log_error(cycle->log, "zmq_ctx_set(ZMQ_IO_THREADS)");
}
}
return NGX_OK;
}
static ssize_t
ngx_zeromq_recv_part(void *zmq, ngx_event_t *rev, u_char *buf, size_t size)
{
zmq_msg_t zmq_msg;
ssize_t n;
if (zmq_msg_init(&zmq_msg) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_msg_init()");
return NGX_ERROR;
}
n = ngx_zeromq_recvmsg(zmq, rev, &zmq_msg);
if (n < 0) {
goto done;
}
if ((size_t) n > size) {
ngx_log_error(NGX_LOG_ALERT, rev->log, 0,
"zmq_recv: ZeroMQ message part too big (%uz) to fit"
" into buffer (%uz)", n, size);
n = NGX_ERROR;
goto done;
}
ngx_memcpy(buf, zmq_msg_data(&zmq_msg), n);
done:
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_msg_close()");
return NGX_ERROR;
}
return n;
}
static ngx_int_t
ngx_zeromq_process_init(ngx_cycle_t *cycle)
{
ngx_zeromq_conf_t *zcf;
zcf = (ngx_zeromq_conf_t *) ngx_get_conf(cycle->conf_ctx,
ngx_zeromq_module);
if (zmq_used) {
zmq_context = zmq_init(zcf->threads);
if (zmq_context == NULL) {
ngx_zeromq_log_error(cycle->log, "zmq_init()");
return NGX_ERROR;
}
}
return NGX_OK;
}
static ngx_int_t
ngx_zeromq_ready(void *zmq, ngx_event_t *ev, const char *what, uint32_t want)
{
uint32_t flags;
size_t fsize;
fsize = sizeof(uint32_t);
if (zmq_getsockopt(zmq, ZMQ_EVENTS, &flags, &fsize) == -1) {
ngx_zeromq_log_error(ev->log, "zmq_getsockopt(ZMQ_EVENTS)");
ev->error = 1;
return NGX_ERROR;
}
if (!(flags & want)) {
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "%s: not ready", what);
ev->ready = 0;
return NGX_AGAIN;
}
return NGX_OK;
}
static ssize_t
ngx_zeromq_recv_part(void *zmq, ngx_event_t *rev, u_char *buf, size_t size)
{
zmq_msg_t zmq_msg;
int64_t more;
size_t msize;
if (zmq_msg_init(&zmq_msg) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_msg_init()");
goto failed_zmq;
}
for (;;) {
if (zmq_recvmsg(zmq, &zmq_msg, ZMQ_DONTWAIT) == -1) {
if (ngx_errno == NGX_EINTR) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, rev->log, 0,
"zmq_recv: interrupted");
rev->ready = 0;
continue;
}
ngx_zeromq_log_error(rev->log, "zmq_recvmsg()");
goto failed;
}
break;
}
if (zmq_msg_size(&zmq_msg) > size) {
ngx_log_error(NGX_LOG_ALERT, rev->log, 0,
"zmq_recv: ZeroMQ message part too big (%uz) to fit"
" into buffer (%uz)", zmq_msg_size(&zmq_msg), size);
goto failed;
}
msize = sizeof(int64_t);
if (zmq_getsockopt(zmq, ZMQ_RCVMORE, &more, &msize) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_getsockopt(ZMQ_RCVMORE)");
goto failed;
}
rev->eof = more ? 0 : 1;
size = zmq_msg_size(&zmq_msg);
ngx_memcpy(buf, zmq_msg_data(&zmq_msg), size);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, rev->log, 0,
"zmq_recv: %uz eom:%d", size, rev->eof);
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_msg_close()");
goto failed_zmq;
}
return size;
failed:
if (zmq_msg_close(&zmq_msg) == -1) {
ngx_zeromq_log_error(rev->log, "zmq_msg_close()");
}
failed_zmq:
rev->error = 1;
return NGX_ERROR;
}
ngx_int_t
ngx_zeromq_connect(ngx_peer_connection_t *pc)
{
ngx_connection_t *c;
ngx_event_t *rev, *wev;
void *zmq;
int fd, zero;
size_t fdsize;
zmq = zmq_socket(zmq_context, ZMQ_REQ);
if (zmq == NULL) {
ngx_zeromq_log_error(pc->log, "zmq_socket(ZMQ_REQ)");
return NGX_ERROR;
}
fdsize = sizeof(int);
if (zmq_getsockopt(zmq, ZMQ_FD, &fd, &fdsize) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_getsockopt(ZMQ_FD)");
goto failed_zmq;
}
zero = 0;
if (zmq_setsockopt(zmq, ZMQ_LINGER, &zero, sizeof(int)) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_setsockopt(ZMQ_LINGER)");
goto failed_zmq;
}
c = ngx_get_connection(fd, pc->log);
if (c == NULL) {
goto failed_zmq;
}
c->data = zmq;
c->recv = ngx_zeromq_recv;
c->send = ngx_zeromq_send;
c->recv_chain = ngx_zeromq_recv_chain;
c->send_chain = ngx_zeromq_send_chain;
/* This won't fly with ZeroMQ */
c->sendfile = 0;
c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
c->log_error = pc->log_error;
rev = c->read;
wev = c->write;
rev->log = pc->log;
wev->log = pc->log;
pc->connection = c;
c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
if (pc->local) {
ngx_log_error(NGX_LOG_WARN, pc->log, 0,
"zmq_connect: binding to local address is not supported");
}
if (zmq_connect(zmq, (const char *) pc->data) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_connect()");
goto failed;
}
ngx_log_debug4(NGX_LOG_DEBUG_EVENT, pc->log, 0,
"zmq_connect: lazily connected to tcp://%V,"
" zmq:%p fd:%d #%d", pc->name, zmq, fd, c->number);
if (ngx_add_conn) {
/* rtsig */
if (ngx_add_conn(c) == NGX_ERROR) {
goto failed;
}
} else {
if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {
/* kqueue, epoll */
if (ngx_add_event(rev, NGX_READ_EVENT, NGX_CLEAR_EVENT) != NGX_OK) {
goto failed;
}
} else {
/* select, poll, /dev/poll */
if (ngx_add_event(rev, NGX_READ_EVENT, NGX_LEVEL_EVENT) != NGX_OK) {
goto failed;
}
}
}
/*
* ZeroMQ assumes that new socket is read-ready (but it really isn't)
* and it won't notify us about any new events if we don't fail to read
* from it first. Sigh.
*/
rev->ready = 1;
wev->ready = 1;
return NGX_OK;
failed:
ngx_free_connection(c);
c->fd = (ngx_socket_t) -1;
failed_zmq:
if (zmq_close(zmq) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_close()");
}
return NGX_ERROR;
}
void
ngx_zeromq_close(ngx_zeromq_connection_t *zc)
{
ngx_connection_t *c;
c = &zc->connection;
if (c->fd == -1) {
return;
}
ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
"zmq_close: fd:%d #%d zc:%p zmq:%p",
c->fd, c->number, zc, zc->socket);
if (c->read->timer_set) {
ngx_del_timer(c->read);
}
if (c->write->timer_set) {
ngx_del_timer(c->write);
}
if (ngx_del_conn) {
ngx_del_conn(c, NGX_CLOSE_EVENT);
} else {
if (c->read->active || c->read->disabled) {
ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
}
if (c->write->active || c->write->disabled) {
ngx_del_event(c->write, NGX_WRITE_EVENT, NGX_CLOSE_EVENT);
}
}
#if (nginx_version >= 1007005)
if (c->read->posted) {
#else
if (c->read->prev) {
#endif
ngx_delete_posted_event(c->read);
}
#if (nginx_version >= 1007005)
if (c->write->posted) {
#else
if (c->write->prev) {
#endif
ngx_delete_posted_event(c->write);
}
c->read->closed = 1;
c->write->closed = 1;
ngx_reusable_connection(zc->connection_ptr, 0);
ngx_free_connection(zc->connection_ptr);
c->fd = (ngx_socket_t) -1;
zc->connection_ptr->fd = (ngx_socket_t) -1;
if (zmq_close(zc->socket) == -1) {
ngx_zeromq_log_error(ngx_cycle->log, "zmq_close()");
}
zc->socket = NULL;
}
static void
ngx_zeromq_event_handler(ngx_event_t *ev)
{
ngx_zeromq_connection_t *zc;
ngx_connection_t *c;
void *zmq;
int events;
size_t esize;
/*
* ZeroMQ notifies us about new events in edge-triggered fashion
* by changing state of the notification socket to read-ready.
*
* Write-readiness doesn't indicate anything and can be ignored.
*/
if (ev->write) {
return;
}
zc = ev->data;
zc = zc->send;
esize = sizeof(int);
#if (NGX_DEBUG)
if (zc->recv != zc->send) {
zmq = zc->request_sent ? zc->socket : zc->recv->socket;
if (zmq_getsockopt(zmq, ZMQ_EVENTS, &events, &esize) == -1) {
ngx_zeromq_log_error(ev->log, "zmq_getsockopt(ZMQ_EVENTS)");
ev->error = 1;
return;
}
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_event: %s:%d (ignored)",
zc->request_sent ? "send" : "recv", events);
}
#endif
zmq = zc->request_sent ? zc->recv->socket : zc->socket;
if (zmq_getsockopt(zmq, ZMQ_EVENTS, &events, &esize) == -1) {
ngx_zeromq_log_error(ev->log, "zmq_getsockopt(ZMQ_EVENTS)");
ev->error = 1;
return;
}
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_event: %s:%d",
zc->request_sent ? "recv" : "send", events);
c = &zc->connection;
if (zc->request_sent) {
c->read->ready = events & ZMQ_POLLIN ? 1 : 0;
if (c->read->ready) {
zc->handler(c->read);
}
} else {
c->write->ready = events & ZMQ_POLLOUT ? 1 : 0;
if (c->write->ready) {
zc->handler(c->write);
}
}
}
static ssize_t
ngx_zeromq_sendmsg(void *zmq, ngx_event_t *ev, zmq_msg_t *msg, int flags)
{
size_t size;
size = zmq_msg_size(msg);
for (;;) {
if (zmq_msg_send(msg, zmq, ZMQ_DONTWAIT|flags) == -1) {
if (ngx_errno == NGX_EAGAIN) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_send: not ready");
ev->ready = 0;
return NGX_AGAIN;
}
if (ngx_errno == NGX_EINTR) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_send: interrupted");
ev->ready = 0;
continue;
}
ngx_zeromq_log_error(ev->log, "zmq_msg_send()");
ev->error = 1;
return NGX_ERROR;
}
break;
}
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"zmq_send: %uz eom:%d", size, flags != ZMQ_SNDMORE);
return size;
}
ngx_int_t
ngx_zeromq_connect(ngx_peer_connection_t *pc)
{
ngx_zeromq_connection_t *zc = pc->data;
ngx_zeromq_endpoint_t *zep;
ngx_connection_t *c;
ngx_event_t *rev, *wev;
void *zmq;
int fd, zero;
size_t fdsize;
ngx_uint_t i;
zep = zc->endpoint;
zmq = zmq_socket(ngx_zeromq_ctx, zep->type->value);
if (zmq == NULL) {
ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
"zmq_socket(%V) failed (%d: %s)",
&zep->type->name, ngx_errno, zmq_strerror(ngx_errno));
return NGX_ERROR;
}
fdsize = sizeof(int);
if (zmq_getsockopt(zmq, ZMQ_FD, &fd, &fdsize) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_getsockopt(ZMQ_FD)");
goto failed_zmq;
}
zero = 0;
if (zmq_setsockopt(zmq, ZMQ_LINGER, &zero, sizeof(int)) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_setsockopt(ZMQ_LINGER)");
goto failed_zmq;
}
c = ngx_get_connection(fd, pc->log);
if (c == NULL) {
goto failed_zmq;
}
c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
c->recv = ngx_zeromq_recv;
c->send = NULL;
c->recv_chain = ngx_zeromq_recv_chain;
c->send_chain = ngx_zeromq_send_chain;
/* This won't fly with ZeroMQ */
c->sendfile = 0;
c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
c->log_error = pc->log_error;
rev = c->read;
wev = c->write;
rev->data = zc;
wev->data = zc;
rev->handler = ngx_zeromq_event_handler;
wev->handler = ngx_zeromq_event_handler;
rev->log = pc->log;
wev->log = pc->log;
pc->connection = &zc->connection;
zc->connection_ptr = c;
memcpy(&zc->connection, c, sizeof(ngx_connection_t));
zc->socket = zmq;
if (zep->type->can_send) {
zc->send = zc;
}
if (zep->type->can_recv) {
zc->recv = zc;
}
if (pc->local) {
ngx_log_error(NGX_LOG_WARN, pc->log, 0,
"zmq_connect: binding to local address is not supported");
}
if (zep->bind) {
if (zep->rand) {
for (i = 0; ; i++) {
ngx_zeromq_randomized_endpoint_regen(&zep->addr);
if (zmq_bind(zmq, (const char *) zep->addr.data) == -1) {
if (ngx_errno == NGX_EADDRINUSE && i < 65535) {
continue;
}
ngx_zeromq_log_error(pc->log, "zmq_bind()");
goto failed;
}
break;
}
} else {
if (zmq_bind(zmq, (const char *) zep->addr.data) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_bind()");
goto failed;
}
}
} else {
if (zmq_connect(zmq, (const char *) zep->addr.data) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_connect()");
goto failed;
}
}
ngx_log_debug7(NGX_LOG_DEBUG_EVENT, pc->log, 0,
"zmq_connect: %s to %V (%V), fd:%d #%d zc:%p zmq:%p",
zep->bind ? "bound" : "lazily connected",
&zep->addr, &zep->type->name, fd, c->number, zc, zmq);
if (ngx_add_conn) {
/* rtsig */
if (ngx_add_conn(c) == NGX_ERROR) {
goto failed;
}
} else {
if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {
/* kqueue, epoll */
if (ngx_add_event(rev, NGX_READ_EVENT, NGX_CLEAR_EVENT) != NGX_OK) {
goto failed;
}
} else {
/* select, poll, /dev/poll */
if (ngx_add_event(rev, NGX_READ_EVENT, NGX_LEVEL_EVENT) != NGX_OK) {
goto failed;
}
}
}
/*
* ZeroMQ assumes that new socket is read-ready (but it really isn't)
* and it won't notify us about any new events if we don't fail to read
* from it first. Sigh.
*/
rev->ready = 1;
wev->ready = zep->type->can_send;
return NGX_OK;
failed:
ngx_free_connection(c);
c->fd = (ngx_socket_t) -1;
pc->connection = NULL;
zc->socket = NULL;
failed_zmq:
if (zmq_close(zmq) == -1) {
ngx_zeromq_log_error(pc->log, "zmq_close()");
}
return NGX_ERROR;
}
