void nn_usock_connect (struct nn_usock *self, const struct sockaddr *addr,
size_t addrlen)
{
int rc;
/* Notify the state machine that we've started connecting. */
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_CONNECT);
/* Do the connect itself. */
rc = connect(self->s,addr,(socklen_t)addrlen);
//printf("usock.%d <- connect (%llx) rc.%d errno.%d %s\n",self->s,*(long long *)addr,rc,errno,nn_strerror(errno));
/* Immediate success. */
if ( nn_fast(rc == 0) )
{
nn_fsm_action(&self->fsm,NN_USOCK_ACTION_DONE);
return;
}
/* Immediate error. */
if ( nn_slow(errno != EINPROGRESS) )
{
self->errnum = errno;
PNACL_message("error.%d not EINPROGRESS\n",errno);
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_ERROR);
return;
}
/* Start asynchronous connect. */
nn_worker_execute (self->worker, &self->task_connecting);
}
void nn_usock_send (struct nn_usock *self, const struct nn_iovec *iov,
int iovcnt)
{
int rc;
int i;
int out;
/* Make sure that the socket is actually alive. */
nn_assert_state (self, NN_USOCK_STATE_ACTIVE);
/* Copy the iovecs to the socket. */
nn_assert (iovcnt <= NN_USOCK_MAX_IOVCNT);
self->out.hdr.msg_iov = self->out.iov;
out = 0;
for (i = 0; i != iovcnt; ++i) {
if (iov [i].iov_len == 0)
continue;
self->out.iov [out].iov_base = iov [i].iov_base;
self->out.iov [out].iov_len = iov [i].iov_len;
out++;
//PNACL_message("{%d} ",(int)iov [i].iov_len);
}
//PNACL_message("iov[%d]\n",out);
self->out.hdr.msg_iovlen = out;
/* Try to send the data immediately. */
rc = nn_usock_send_raw (self, &self->out.hdr);
/* Success. */
if (nn_fast (rc == 0)) {
nn_fsm_raise (&self->fsm, &self->event_sent, NN_USOCK_SENT);
return;
}
/* Errors. */
if (nn_slow (rc != -EAGAIN)) {
errnum_assert (rc == -ECONNRESET, -rc);
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_ERROR);
return;
}
/* Ask the worker thread to send the remaining data. */
nn_worker_execute (self->worker, &self->task_send);
}
void nn_usock_recv (struct nn_usock *self, void *buf, size_t len, int *fd)
{
int rc;
size_t nbytes;
/* Make sure that the socket is actually alive. */
nn_assert_state (self, NN_USOCK_STATE_ACTIVE);
/* Try to receive the data immediately. */
nbytes = len;
self->in.pfd = fd;
rc = nn_usock_recv_raw (self, buf, &nbytes);
if (nn_slow (rc < 0)) {
errnum_assert (rc == -ECONNRESET, -rc);
//PNACL_message("rc.%d vs ECONNRESET\n",rc,ECONNRESET);
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_ERROR);
return;
}
//int i;
//for (i=0; i<16&&i<nbytes; i++)
// PNACL_message("%02x ",((uint8_t *)buf)[i]);
//PNACL_message("nn_usock_recv nbytes.%d\n",(int)nbytes);
/* Success. */
if (nn_fast (nbytes == len)) {
//PNACL_message("raise NN_USOCK_RECEIVED\n");
nn_fsm_raise (&self->fsm, &self->event_received, NN_USOCK_RECEIVED);
return;
}
/* There are still data to receive in the background. */
self->in.buf = ((uint8_t*) buf) + nbytes;
self->in.len = len - nbytes;
/* Ask the worker thread to receive the remaining data. */
nn_worker_execute (self->worker, &self->task_recv);
}
static void nn_timer_shutdown (struct nn_fsm *self, int src, int type,
void *srcptr)
{
struct nn_timer *timer;
timer = nn_cont (self, struct nn_timer, fsm);
if (nn_slow (src == NN_FSM_ACTION && type == NN_FSM_STOP)) {
nn_worker_execute (timer->worker, &timer->stop_task);
timer->state = NN_TIMER_STATE_STOPPING;
return;
}
if (nn_slow (timer->state == NN_TIMER_STATE_STOPPING)) {
if (src != NN_TIMER_SRC_STOP_TASK)
return;
nn_assert (type == NN_WORKER_TASK_EXECUTE);
nn_worker_rm_timer (timer->worker, &timer->wtimer);
timer->state = NN_TIMER_STATE_IDLE;
nn_fsm_stopped (&timer->fsm, NN_TIMER_STOPPED);
return;
}
nn_fsm_bad_state(timer->state, src, type);
}
static void nn_usock_handler (struct nn_fsm *self, int src, int type,
NN_UNUSED void *srcptr)
{
int rc;
struct nn_usock *usock;
int s;
size_t sz;
int sockerr;
usock = nn_cont (self, struct nn_usock, fsm);
if(nn_internal_tasks(usock, src, type))
return;
switch (usock->state) {
/******************************************************************************/
/* IDLE state. */
/* nn_usock object is initialised, but underlying OS socket is not yet */
/* created. */
/******************************************************************************/
case NN_USOCK_STATE_IDLE:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_FSM_START:
usock->state = NN_USOCK_STATE_STARTING;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* STARTING state. */
/* Underlying OS socket is created, but it's not yet passed to the worker */
/* thread. In this state we can set socket options, local and remote */
/* address etc. */
/******************************************************************************/
case NN_USOCK_STATE_STARTING:
/* Events from the owner of the usock. */
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_LISTEN:
usock->state = NN_USOCK_STATE_LISTENING;
return;
case NN_USOCK_ACTION_CONNECT:
usock->state = NN_USOCK_STATE_CONNECTING;
return;
case NN_USOCK_ACTION_BEING_ACCEPTED:
usock->state = NN_USOCK_STATE_BEING_ACCEPTED;
return;
case NN_USOCK_ACTION_STARTED:
nn_worker_add_fd (usock->worker, usock->s, &usock->wfd);
usock->state = NN_USOCK_STATE_ACTIVE;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* BEING_ACCEPTED state. */
/* accept() was called on the usock. Now the socket is waiting for a new */
/* connection to arrive. */
/******************************************************************************/
case NN_USOCK_STATE_BEING_ACCEPTED:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_DONE:
usock->state = NN_USOCK_STATE_ACCEPTED;
nn_fsm_raise (&usock->fsm, &usock->event_established,
NN_USOCK_ACCEPTED);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* ACCEPTED state. */
/* Connection was accepted, now it can be tuned. Afterwards, it'll move to */
/* the active state. */
/******************************************************************************/
case NN_USOCK_STATE_ACCEPTED:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_ACTIVATE:
nn_worker_add_fd (usock->worker, usock->s, &usock->wfd);
usock->state = NN_USOCK_STATE_ACTIVE;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* CONNECTING state. */
/* Asynchronous connecting is going on. */
/******************************************************************************/
case NN_USOCK_STATE_CONNECTING:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_DONE:
usock->state = NN_USOCK_STATE_ACTIVE;
nn_worker_execute (usock->worker, &usock->task_connected);
nn_fsm_raise (&usock->fsm, &usock->event_established,
NN_USOCK_CONNECTED);
return;
case NN_USOCK_ACTION_ERROR:
nn_closefd (usock->s);
usock->s = -1;
usock->state = NN_USOCK_STATE_DONE;
nn_fsm_raise (&usock->fsm, &usock->event_error,
NN_USOCK_ERROR);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
case NN_USOCK_SRC_FD:
switch (type) {
case NN_WORKER_FD_OUT:
nn_worker_reset_out (usock->worker, &usock->wfd);
usock->state = NN_USOCK_STATE_ACTIVE;
sockerr = nn_usock_geterr(usock);
if (sockerr == 0) {
nn_fsm_raise (&usock->fsm, &usock->event_established,
NN_USOCK_CONNECTED);
} else {
usock->errnum = sockerr;
nn_worker_rm_fd (usock->worker, &usock->wfd);
rc = close (usock->s);
errno_assert (rc == 0);
usock->s = -1;
usock->state = NN_USOCK_STATE_DONE;
nn_fsm_raise (&usock->fsm,
&usock->event_error, NN_USOCK_ERROR);
}
return;
case NN_WORKER_FD_ERR:
nn_worker_rm_fd (usock->worker, &usock->wfd);
nn_closefd (usock->s);
usock->s = -1;
usock->state = NN_USOCK_STATE_DONE;
nn_fsm_raise (&usock->fsm, &usock->event_error, NN_USOCK_ERROR);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* ACTIVE state. */
/* Socket is connected. It can be used for sending and receiving data. */
/******************************************************************************/
case NN_USOCK_STATE_ACTIVE:
switch (src) {
case NN_USOCK_SRC_FD:
switch (type) {
case NN_WORKER_FD_IN:
sz = usock->in.len;
rc = nn_usock_recv_raw (usock, usock->in.buf, &sz);
if (nn_fast (rc == 0)) {
usock->in.len -= sz;
usock->in.buf += sz;
if (!usock->in.len) {
nn_worker_reset_in (usock->worker, &usock->wfd);
nn_fsm_raise (&usock->fsm, &usock->event_received,
NN_USOCK_RECEIVED);
}
return;
}
errnum_assert (rc == -ECONNRESET, -rc);
goto error;
case NN_WORKER_FD_OUT:
rc = nn_usock_send_raw (usock, &usock->out.hdr);
if (nn_fast (rc == 0)) {
nn_worker_reset_out (usock->worker, &usock->wfd);
nn_fsm_raise (&usock->fsm, &usock->event_sent,
NN_USOCK_SENT);
return;
}
if (nn_fast (rc == -EAGAIN))
return;
errnum_assert (rc == -ECONNRESET, -rc);
goto error;
case NN_WORKER_FD_ERR:
error:
nn_worker_rm_fd (usock->worker, &usock->wfd);
nn_closefd (usock->s);
usock->s = -1;
usock->state = NN_USOCK_STATE_DONE;
nn_fsm_raise (&usock->fsm, &usock->event_error, NN_USOCK_ERROR);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_ERROR:
usock->state = NN_USOCK_STATE_REMOVING_FD;
nn_usock_async_stop (usock);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source(usock->state, src, type);
}
/******************************************************************************/
/* REMOVING_FD state. */
/******************************************************************************/
case NN_USOCK_STATE_REMOVING_FD:
switch (src) {
case NN_USOCK_SRC_TASK_STOP:
switch (type) {
case NN_WORKER_TASK_EXECUTE:
nn_worker_rm_fd (usock->worker, &usock->wfd);
nn_closefd (usock->s);
usock->s = -1;
usock->state = NN_USOCK_STATE_DONE;
nn_fsm_raise (&usock->fsm, &usock->event_error, NN_USOCK_ERROR);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
/* Events from the file descriptor are ignored while it is being
removed. */
case NN_USOCK_SRC_FD:
return;
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* DONE state. */
/* Socket is closed. The only thing that can be done in this state is */
/* stopping the usock. */
/******************************************************************************/
case NN_USOCK_STATE_DONE:
nn_fsm_bad_source (usock->state, src, type);
/******************************************************************************/
/* LISTENING state. */
/* Socket is listening for new incoming connections, however, user is not */
/* accepting a new connection. */
/******************************************************************************/
case NN_USOCK_STATE_LISTENING:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_ACCEPT:
usock->state = NN_USOCK_STATE_ACCEPTING;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* ACCEPTING state. */
/* User is waiting asynchronouslyfor a new inbound connection */
/* to be accepted. */
/******************************************************************************/
case NN_USOCK_STATE_ACCEPTING:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_DONE:
usock->state = NN_USOCK_STATE_LISTENING;
return;
case NN_USOCK_ACTION_CANCEL:
usock->state = NN_USOCK_STATE_CANCELLING;
nn_worker_execute (usock->worker, &usock->task_stop);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
case NN_USOCK_SRC_FD:
switch (type) {
case NN_WORKER_FD_IN:
/* New connection arrived in asynchronous manner. */
#if NN_HAVE_ACCEPT4
s = accept4 (usock->s, NULL, NULL, SOCK_CLOEXEC);
#else
s = accept (usock->s, NULL, NULL);
#endif
/* ECONNABORTED is an valid error. New connection was closed
by the peer before we were able to accept it. If it happens
do nothing and wait for next incoming connection. */
if (nn_slow (s < 0 && errno == ECONNABORTED))
return;
/* Resource allocation errors. It's not clear from POSIX
specification whether the new connection is closed in this
case or whether it remains in the backlog. In the latter
case it would be wise to wait here for a while to prevent
busy looping. */
if (nn_slow (s < 0 && (errno == ENFILE || errno == EMFILE ||
errno == ENOBUFS || errno == ENOMEM))) {
usock->errnum = errno;
usock->state = NN_USOCK_STATE_ACCEPTING_ERROR;
/* Wait till the user starts accepting once again. */
nn_worker_rm_fd (usock->worker, &usock->wfd);
nn_fsm_raise (&usock->fsm,
&usock->event_error, NN_USOCK_ACCEPT_ERROR);
return;
}
/* Any other error is unexpected. */
errno_assert (s >= 0);
/* Initialise the new usock object. */
nn_usock_init_from_fd (usock->asock, s);
usock->asock->state = NN_USOCK_STATE_ACCEPTED;
/* Notify the user that connection was accepted. */
nn_fsm_raise (&usock->asock->fsm,
&usock->asock->event_established, NN_USOCK_ACCEPTED);
/* Disassociate the listener socket from the accepted
socket. */
usock->asock->asock = NULL;
usock->asock = NULL;
/* Wait till the user starts accepting once again. */
nn_worker_rm_fd (usock->worker, &usock->wfd);
usock->state = NN_USOCK_STATE_LISTENING;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* ACCEPTING_ERROR state. */
/* Waiting the socket to accept the error and restart */
/******************************************************************************/
case NN_USOCK_STATE_ACCEPTING_ERROR:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_USOCK_ACTION_ACCEPT:
usock->state = NN_USOCK_STATE_ACCEPTING;
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* CANCELLING state. */
/******************************************************************************/
case NN_USOCK_STATE_CANCELLING:
switch (src) {
case NN_USOCK_SRC_TASK_STOP:
switch (type) {
case NN_WORKER_TASK_EXECUTE:
nn_worker_rm_fd (usock->worker, &usock->wfd);
usock->state = NN_USOCK_STATE_LISTENING;
/* Notify the accepted socket that it was stopped. */
nn_fsm_action (&usock->asock->fsm, NN_USOCK_ACTION_DONE);
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
case NN_USOCK_SRC_FD:
switch (type) {
case NN_WORKER_FD_IN:
return;
default:
nn_fsm_bad_action (usock->state, src, type);
}
default:
nn_fsm_bad_source (usock->state, src, type);
}
/******************************************************************************/
/* Invalid state */
/******************************************************************************/
default:
nn_fsm_bad_state (usock->state, src, type);
}
}
void nn_usock_accept (struct nn_usock *self, struct nn_usock *listener)
{
int s;
/* Start the actual accepting. */
if (nn_fsm_isidle(&self->fsm)) {
nn_fsm_start (&self->fsm);
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_BEING_ACCEPTED);
}
nn_fsm_action (&listener->fsm, NN_USOCK_ACTION_ACCEPT);
/* Try to accept new connection in synchronous manner. */
#if NN_HAVE_ACCEPT4
s = accept4 (listener->s, NULL, NULL, SOCK_CLOEXEC);
#else
s = accept (listener->s, NULL, NULL);
#endif
//printf("usock.%d -> accept errno.%d s.%d %s\n",self->s,errno,s,nn_strerror(errno));
/* Immediate success. */
if (nn_fast (s >= 0)) {
/* Disassociate the listener socket from the accepted
socket. Is useful if we restart accepting on ACCEPT_ERROR */
listener->asock = NULL;
self->asock = NULL;
nn_usock_init_from_fd (self, s);
nn_fsm_action (&listener->fsm, NN_USOCK_ACTION_DONE);
nn_fsm_action (&self->fsm, NN_USOCK_ACTION_DONE);
return;
}
/* Detect a failure. Note that in ECONNABORTED case we simply ignore
the error and wait for next connection in asynchronous manner. */
errno_assert (errno == EAGAIN || errno == EWOULDBLOCK ||
errno == ECONNABORTED || errno == ENFILE || errno == EMFILE ||
errno == ENOBUFS || errno == ENOMEM);
/* Pair the two sockets. They are already paired in case
previous attempt failed on ACCEPT_ERROR */
nn_assert (!self->asock || self->asock == listener);
self->asock = listener;
nn_assert (!listener->asock || listener->asock == self);
listener->asock = self;
/* Some errors are just ok to ignore for now. We also stop repeating
any errors until next IN_FD event so that we are not in a tight loop
and allow processing other events in the meantime */
if (nn_slow (errno != EAGAIN && errno != EWOULDBLOCK && errno != ECONNABORTED && errno != listener->errnum))
{
PNACL_message("listen errno.%d\n",errno);
listener->errnum = errno;
listener->state = NN_USOCK_STATE_ACCEPTING_ERROR;
nn_fsm_raise (&listener->fsm,
&listener->event_error, NN_USOCK_ACCEPT_ERROR);
return;
}
/* Ask the worker thread to wait for the new connection. */
nn_worker_execute (listener->worker, &listener->task_accept);
}
void nn_usock_async_stop (struct nn_usock *self)
{
nn_worker_execute (self->worker, &self->task_stop);
nn_fsm_raise (&self->fsm, &self->event_error, NN_USOCK_SHUTDOWN);
}
static void nn_timer_handler (struct nn_fsm *self, int src, int type,
void *srcptr)
{
struct nn_timer *timer;
timer = nn_cont (self, struct nn_timer, fsm);
switch (timer->state) {
/******************************************************************************/
/* IDLE state. */
/******************************************************************************/
case NN_TIMER_STATE_IDLE:
switch (src) {
case NN_FSM_ACTION:
switch (type) {
case NN_FSM_START:
/* Send start event to the worker thread. */
nn_worker_execute (timer->worker, &timer->start_task);
timer->state = NN_TIMER_STATE_ACTIVE;
return;
default:
nn_fsm_bad_action (timer->state, src, type);
}
default:
nn_fsm_bad_source (timer->state, src, type);
}
/******************************************************************************/
/* ACTIVE state. */
/******************************************************************************/
case NN_TIMER_STATE_ACTIVE:
if (src == NN_TIMER_SRC_START_TASK) {
nn_assert (type == NN_WORKER_TASK_EXECUTE);
nn_assert (timer->timeout >= 0);
nn_worker_add_timer (timer->worker, timer->timeout,
&timer->wtimer);
timer->timeout = -1;
return;
}
if (srcptr == &timer->wtimer) {
switch (type) {
case NN_WORKER_TIMER_TIMEOUT:
/* Notify the user about the timeout. */
nn_assert (timer->timeout == -1);
nn_fsm_raise (&timer->fsm, &timer->done, NN_TIMER_TIMEOUT);
return;
default:
nn_fsm_bad_action (timer->state, src, type);
}
}
nn_fsm_bad_source (timer->state, src, type);
/******************************************************************************/
/* Invalid state. */
/******************************************************************************/
default:
nn_fsm_bad_state (timer->state, src, type);
}
}
static void nn_timer_handler (struct nn_fsm *self, int src, int type,
void *srcptr)
{
struct nn_timer *timer;
timer = nn_cont (self, struct nn_timer, fsm);
/******************************************************************************/
/* STOP procedure. */
/******************************************************************************/
if (nn_slow (srcptr == NULL && type == NN_FSM_STOP)) {
nn_worker_execute (timer->worker, &timer->stop_task);
timer->state = NN_TIMER_STATE_STOPPING;
return;
}
if (nn_slow (timer->state == NN_TIMER_STATE_STOPPING)) {
if (srcptr != &timer->stop_task)
return;
nn_assert (type == NN_WORKER_TASK_EXECUTE);
nn_worker_rm_timer (timer->worker, &timer->wtimer);
timer->state = NN_TIMER_STATE_IDLE;
nn_fsm_stopped (&timer->fsm, NN_TIMER_STOPPED);
return;
}
switch (timer->state) {
/******************************************************************************/
/* IDLE state. */
/******************************************************************************/
case NN_TIMER_STATE_IDLE:
if (srcptr == NULL) {
switch (type) {
case NN_FSM_START:
/* Send start event to the worker thread. */
nn_worker_execute (timer->worker, &timer->start_task);
timer->state = NN_TIMER_STATE_ACTIVE;
return;
default:
nn_assert (0);
}
}
nn_assert (0);
/******************************************************************************/
/* ACTIVE state. */
/******************************************************************************/
case NN_TIMER_STATE_ACTIVE:
if (srcptr == &timer->start_task) {
nn_assert (type == NN_WORKER_TASK_EXECUTE);
nn_assert (timer->timeout >= 0);
nn_worker_add_timer (timer->worker, timer->timeout,
&timer->wtimer);
timer->timeout = -1;
return;
}
if (srcptr == &timer->wtimer) {
switch (type) {
case NN_WORKER_TIMER_TIMEOUT:
/* Notify the user about the timeout. */
nn_assert (timer->timeout == -1);
nn_fsm_raise (&timer->fsm, &timer->done, NN_TIMER_TIMEOUT);
return;
default:
nn_assert (0);
}
}
nn_assert (0);
/******************************************************************************/
/* Invalid state. */
/******************************************************************************/
default:
nn_assert (0);
}
}
