/*! Run signal handler immediatedly, just now.
** \param signum Signal number
** \param info Saved info for sa_sigaction handler
** \param context Saved context for sa_sigaction handler
** \param our_info Our signal info
*/
ci_inline void citp_signal_run_now(int signum, siginfo_t *info,
void *context,
citp_signal_info *our_info)
{
int need_restart;
LOG_SIG(log("%s: SIGNAL %d - run immediately", __FUNCTION__, signum));
/* Try to keep order: old signals first, and need_restart is from the
* last one */
if (our_info && (our_info->aflags & OO_SIGNAL_FLAG_HAVE_PENDING))
citp_signal_run_pending(our_info);
need_restart = citp_signal_run_app_handler(signum, info, context);
/* Set need_restart flag in accordance with sa_restart.
* The last signal wins, so we set need_restart to 1 if necessary.
*/
if (our_info) {
LOG_SIG(log("%s: SIGNAL %d - set need restart flag to %d", __FUNCTION__,
signum, need_restart));
if( need_restart )
ci_atomic32_or(&our_info->aflags, OO_SIGNAL_FLAG_NEED_RESTART);
else
ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_NEED_RESTART);
}
}
void citp_waitable_obj_free_to_cache(ci_netif* ni, citp_waitable* w)
{
#if defined (__KERNEL__) && !defined(NDEBUG)
/* There should be no non-atomic work queued for endpoints going to cache -
* they don't get their filters removed.
*/
tcp_helper_endpoint_t* ep = ci_netif_get_valid_ep(ni, w->bufid);
ci_assert(!(ep->ep_aflags & OO_THR_EP_AFLAG_NON_ATOMIC));
#endif
ci_assert(!(w->sb_aflags & CI_SB_AFLAG_ORPHAN));
ci_assert(w->sb_aflags & CI_SB_AFLAG_NOT_READY);
ci_assert(w->sb_aflags & CI_SB_AFLAG_IN_CACHE);
ci_assert(w->state == CI_TCP_CLOSED);
ci_assert(ci_ni_dllist_is_self_linked(ni, &w->post_poll_link));
ci_assert(OO_SP_IS_NULL(w->wt_next));
/* This resets a subset of the state done by __citp_waitable_obj_free.
* We do not set the orphan flag, as cached endpoints remain attached.
* We do not alter the state, as that too remains accurate.
*
* We preserve cache related aflags. If the endpoint is freed before being
* accepted from the cache then these will be cleared when
* __citp_waitable_obj_free is called, otherwise they'll be checked for
* correctness, and updated if necessary when the socket is accepted.
*/
w->wake_request = 0;
w->sb_flags = 0;
ci_atomic32_and(&w->sb_aflags, CI_SB_AFLAG_NOT_READY |
CI_SB_AFLAG_CACHE_PRESERVE);
w->lock.wl_val = 0;
w->ready_list_id = 0;
CI_USER_PTR_SET(w->eitem, NULL);
}
/*! Run any pending signal handlers
** \param our_info Thread-specific context for current thread
*/
void citp_signal_run_pending(citp_signal_info *our_info)
{
/* preserve errno across calls to this function, as it's often
called at error time as a result of EXIT_LIB */
int old_errno = errno;
int i;
LOG_SIG(log("%s: start", __FUNCTION__));
ci_wmb();
ci_assert_equal(our_info->inside_lib, 0);
ci_assert(our_info->aflags & OO_SIGNAL_FLAG_HAVE_PENDING);
ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_HAVE_PENDING);
for( i = 0; i < OO_SIGNAL_MAX_PENDING; i++ ) {
siginfo_t saved_info;
void *saved_context;
int signum;
if (our_info->signals[i].signum == 0)
break;
saved_context = our_info->signals[i].saved_context;
if( our_info->signals[i].saved_context )
memcpy(&saved_info, &our_info->signals[i].saved_info,
sizeof(saved_info));
signum = our_info->signals[i].signum;
if( ci_cas32_fail(&our_info->signals[i].signum, signum, 0) )
break;
if( citp_signal_run_app_handler(
signum,
saved_context == NULL ? NULL : &saved_info,
saved_context) )
ci_atomic32_or(&our_info->aflags, OO_SIGNAL_FLAG_NEED_RESTART);
else
ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_NEED_RESTART);
}
LOG_SIG(log("%s: end", __FUNCTION__));
errno = old_errno;
}
int ci_tcp_shutdown(citp_socket* ep, int how, ci_fd_t fd)
{
ci_sock_cmn* s = ep->s;
int rc;
if( s->b.state == CI_TCP_LISTEN )
return ci_tcp_shutdown_listen(ep, how, fd);
if( SOCK_TO_TCP(s)->snd_delegated ) {
/* We do not know which seq number to use. Call
* onload_delegated_send_cancel(). */
CI_SET_ERROR(rc, EBUSY);
return rc;
}
if( ! ci_netif_trylock(ep->netif) ) {
/* Can't get lock, so try to defer shutdown to the lock holder. */
unsigned flags = 0;
switch( s->b.state ) {
case CI_TCP_CLOSED:
case CI_TCP_TIME_WAIT:
CI_SET_ERROR(rc, ENOTCONN);
return rc;
}
if( how == SHUT_RD || how == SHUT_RDWR )
flags |= CI_SOCK_AFLAG_NEED_SHUT_RD;
if( how == SHUT_WR || how == SHUT_RDWR )
flags |= CI_SOCK_AFLAG_NEED_SHUT_WR;
ci_atomic32_or(&s->s_aflags, flags);
if( ! ci_netif_lock_or_defer_work(ep->netif, &s->b) )
return 0;
ci_atomic32_and(&s->s_aflags, ~flags);
}
if( 0 ) {
/* Poll to get up-to-date. This is slightly spurious but done to ensure
* ordered response to all packets preceding this FIN (e.g. ANVL tcp_core
* 9.18)
*
* DJR: I've disabled this because it can hurt performance for
* high-connection-rate apps. May consider adding back (as option?) if
* needed.
*/
ci_netif_poll(ep->netif);
}
rc = __ci_tcp_shutdown(ep->netif, SOCK_TO_TCP(s), how);
if( rc < 0 )
CI_SET_ERROR(rc, -rc);
ci_netif_unlock(ep->netif);
return rc;
}
static int
efab_tcp_helper_move_state(ci_private_t* priv, void *arg)
{
oo_tcp_move_state_t *op = arg;
tcp_helper_endpoint_t *new_ep;
tcp_helper_resource_t * new_trs = NULL;
ci_netif* ni, *new_ni;
ci_tcp_state * ts, *new_ts;
tcp_helper_endpoint_t* ep;
int rc = efab_ioctl_get_ep(priv, op->ep_id, &ep);
if (rc != 0)
return rc;
OO_DEBUG_TCPH(ci_log("%s: (trs=%p (%u), priv=%p, ep_id=%u, new_trs_id=%u, "
"new_ep_id=%u", __FUNCTION__, priv->thr, priv->thr->id,
priv, OO_SP_FMT(op->ep_id), op->new_trs_id,
OO_SP_FMT(op->new_ep_id)));
do {
/* check that the existing id is valid */
ni = &priv->thr->netif;
ts = SP_TO_TCP(ni, ep->id);
/* TODO: check this endpoint belongs to the tcp helper resource of priv and not
* somewhere else */
/* this function does not change fd_type or fd ops, so it is not able
* to cope with changing the socket type. We think this only makes sense
* for TCP, so assert we are taking a TCP endpoint.
*/
ci_assert_equal(ts->s.pkt.ip.ip_protocol, IPPROTO_TCP);
ci_assert_equal(priv->fd_type, CI_PRIV_TYPE_TCP_EP);
/* get pointer to resource from handle - increments ref count */
rc = efab_thr_table_lookup(NULL, op->new_trs_id,
EFAB_THR_TABLE_LOOKUP_CHECK_USER, &new_trs);
if (rc < 0) {
OO_DEBUG_ERR( ci_log("%s: invalid new resource handle", __FUNCTION__) );
break;
}
ci_assert(new_trs != NULL);
/* check valid endpoint in new netif */
new_ni = &new_trs->netif;
new_ep = ci_netif_get_valid_ep(new_ni, op->new_ep_id);
new_ts = SP_TO_TCP(new_ni, new_ep->id);
/* check the two endpoint states look valid */
if( (ts->s.pkt.ip.ip_protocol != new_ts->s.pkt.ip.ip_protocol) ||
(ts->s.b.state != CI_TCP_CLOSED) ||
(ep->oofilter.sf_local_port != NULL) ) {
efab_thr_release(new_trs);
rc = -EINVAL;
OO_DEBUG_ERR(ci_log("%s: invalid endpoint states", __FUNCTION__));
break;
}
/* should be fine to complete */
ci_assert(new_trs);
{
tcp_helper_resource_t *old_trs;
again:
old_trs = priv->thr;
if (ci_cas_uintptr_fail((ci_uintptr_t *)&priv->thr,
(ci_uintptr_t)old_trs, (ci_uintptr_t)new_trs))
goto again;
efab_thr_release(old_trs);
}
/* move file to hold details of new resource, new endpoint */
ci_assert(OO_SP_EQ(priv->sock_id, op->ep_id));
priv->sock_id = new_ep->id;
OO_DEBUG_TCPH(ci_log("%s: set epid %u", __FUNCTION__,
OO_SP_FMT(priv->sock_id)));
/* copy across any necessary state */
ci_assert_equal(new_ep->os_socket, NULL);
new_ep->os_socket = ep->os_socket;
ep->os_socket = NULL;
/* set ORPHAN flag in current as not attached to an FD */
ci_bit_set(&ts->s.b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
/* remove ORPHAN flag in new TCP state */
ci_atomic32_and(&new_ts->s.b.sb_aflags,
~(CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_TCP_IN_ACCEPTQ));
return 0;
} while (0);
return rc;
}
static int
onload_alloc_file(tcp_helper_resource_t *thr, oo_sp ep_id, int flags,
int fd_type)
{
struct qstr name = { .name = "" };
#ifdef EFX_HAVE_STRUCT_PATH
struct path path;
#define my_dentry path.dentry
#else
struct dentry *dentry;
#define my_dentry dentry
#endif
struct file *file;
int fd;
struct inode *inode;
ci_private_t *priv;
struct file_operations *fops;
fops = oo_fops_by_type(fd_type);
if( fops == NULL )
return -EINVAL;
ci_assert_equal(fops->owner, THIS_MODULE);
inode = new_inode(onload_mnt->mnt_sb);
if( inode == NULL )
return -ENOMEM;
#ifdef EFX_FSTYPE_HAS_MOUNT
inode->i_ino = get_next_ino();
#endif
if( fd_type == CI_PRIV_TYPE_NETIF )
inode->i_mode = S_IRWXUGO;
if( fd_type == CI_PRIV_TYPE_TCP_EP || fd_type == CI_PRIV_TYPE_UDP_EP )
inode->i_mode =
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21)
/* in 2.6.18 this flag makes us "socket" and sendmsg crashes;
* see sock_from_file() */
S_IFSOCK |
#endif
S_IRWXUGO;
else
inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
priv = &container_of(inode, struct onload_inode, vfs_inode)->priv;
priv->thr = thr;
priv->sock_id = ep_id;
priv->fd_type = fd_type;
fd = get_unused_fd();
if( fd < 0 ) {
iput(inode);
return fd;
}
/*ci_log("[%d]%s(%d:%d) return %d priv=%p", current->pid, __func__,
thr->id, ep_id, fd, priv);*/
#ifdef EFX_FSTYPE_HAS_MOUNT
#if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,37)
path.dentry = d_alloc(onload_mnt->mnt_sb->s_root, &name);
if( path.dentry != NULL )
path.dentry->d_op = &onloadfs_dentry_operations;
#else
path.dentry = d_alloc_pseudo(onload_mnt->mnt_sb, &name);
#endif
#else /* EFX_FSTYPE_HAS_MOUNT */
#ifdef EFX_HAVE_D_DNAME
my_dentry = d_alloc(onload_mnt->mnt_sb->s_root, &name);
#else
{
char str[32];
name.len = onloadfs_name(&container_of(inode, struct onload_inode,
vfs_inode)->priv,
str, sizeof(str));
name.name = str;
name.hash = inode->i_ino;
my_dentry = d_alloc(onload_mnt->mnt_sb->s_root, &name);
}
#endif
#endif /* EFX_FSTYPE_HAS_MOUNT */
if( my_dentry == NULL ) {
put_unused_fd(fd);
iput(inode);
return -ENOMEM;
}
#if !defined(EFX_FSTYPE_HAS_MOUNT) || defined(EFX_OLD_MOUNT_PSEUDO)
my_dentry->d_op = &onloadfs_dentry_operations;
#if !defined(EFX_HAVE_STRUCT_PATH) && defined(EFX_HAVE_D_DNAME)
my_dentry->d_flags &= ~DCACHE_UNHASHED;
#endif
#endif
d_instantiate(my_dentry, inode);
#ifndef EFX_HAVE_D_DNAME
d_rehash(my_dentry);
#endif
inode->i_fop = fops;
#ifdef EFX_HAVE_STRUCT_PATH
path.mnt = mntget(onload_mnt);
file = alloc_file(&path, FMODE_READ | FMODE_WRITE, fops);
#else
file = alloc_file(onload_mnt, dentry, FMODE_READ | FMODE_WRITE, fops);
#endif
if( file == NULL) {
#ifdef EFX_HAVE_STRUCT_PATH
path_put(&path);
#else
dput(dentry);
iput(inode);
#endif
put_unused_fd(fd);
return -ENFILE;
}
priv->_filp = file;
file->f_flags = O_RDWR | (flags & O_NONBLOCK);
file->f_pos = 0;
file->private_data = priv;
if( flags & O_CLOEXEC ) {
struct files_struct *files = current->files;
struct fdtable *fdt;
spin_lock(&files->file_lock);
fdt = files_fdtable(files);
rcu_assign_pointer(fdt->fd[fd], file);
efx_set_close_on_exec(fd, fdt);
spin_unlock(&files->file_lock);
} else
fd_install(fd, file);
try_module_get(THIS_MODULE);
ci_assert_equal(file->f_op, fops);
return fd;
}
void onload_priv_free(ci_private_t *priv)
{
if( priv->_filp->f_vfsmnt != onload_mnt)
ci_free(priv);
/* inode will free the priv automatically */
}
int
oo_create_fd(tcp_helper_endpoint_t* ep, int flags, int fd_type)
{
int fd;
tcp_helper_resource_t *trs = ep->thr;
citp_waitable_obj *wo = SP_TO_WAITABLE_OBJ(&trs->netif, ep->id);
efab_thr_ref(trs);
fd = onload_alloc_file(trs, ep->id, flags, fd_type);
if( fd < 0 ) {
efab_thr_release(trs);
OO_DEBUG_ERR(ci_log("%s: onload_alloc_file failed (%d)", __FUNCTION__, fd));
return fd;
}
ci_atomic32_and(&wo-> waitable.sb_aflags,
~(CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_TCP_IN_ACCEPTQ));
return fd;
}
/* we don't register protocol impl */
int citp_pipe_create(int fds[2], int flags)
{
citp_pipe_fdi* epi_read;
citp_pipe_fdi* epi_write;
struct oo_pipe* p = NULL; /* make compiler happy */
ci_netif* ni;
int rc = -1;
ef_driver_handle fd = -1;
Log_V(log(LPF "pipe()"));
/* citp_netif_exists() does not need citp_ul_lock here */
if( CITP_OPTS.ul_pipe == CI_UNIX_PIPE_ACCELERATE_IF_NETIF &&
! citp_netif_exists() ) {
return CITP_NOT_HANDLED;
}
rc = citp_netif_alloc_and_init(&fd, &ni);
if( rc != 0 ) {
if( rc == CI_SOCKET_HANDOVER ) {
/* This implies EF_DONT_ACCELERATE is set, so we handover
* regardless of CITP_OPTS.no_fail */
return CITP_NOT_HANDLED;
}
/* may be lib mismatch - errno will be ELIBACC */
goto fail1;
}
rc = -1;
CI_MAGIC_CHECK(ni, NETIF_MAGIC);
/* add another reference as we have 2 fdis */
citp_netif_add_ref(ni);
epi_read = citp_pipe_epi_alloc(ni, O_RDONLY);
if( epi_read == NULL )
goto fail2;
epi_write = citp_pipe_epi_alloc(ni, O_WRONLY);
if( epi_write == NULL )
goto fail3;
/* oo_pipe init code */
if( fdtable_strict() ) CITP_FDTABLE_LOCK();
rc = oo_pipe_ctor(ni, &p, fds, flags);
if( rc < 0 )
goto fail4;
citp_fdtable_new_fd_set(fds[0], fdip_busy, fdtable_strict());
citp_fdtable_new_fd_set(fds[1], fdip_busy, fdtable_strict());
if( fdtable_strict() ) CITP_FDTABLE_UNLOCK();
LOG_PIPE("%s: pipe=%p id=%d", __FUNCTION__, p, p->b.bufid);
/* as pipe is created it should be attached to the end-points */
epi_read->pipe = p;
epi_write->pipe = p;
/* We're ready. Unleash us onto the world! */
ci_assert(epi_read->pipe->b.sb_aflags & CI_SB_AFLAG_NOT_READY);
ci_assert(epi_write->pipe->b.sb_aflags & CI_SB_AFLAG_NOT_READY);
ci_atomic32_and(&epi_read->pipe->b.sb_aflags, ~CI_SB_AFLAG_NOT_READY);
ci_atomic32_and(&epi_read->pipe->b.sb_aflags, ~CI_SB_AFLAG_NOT_READY);
citp_fdtable_insert(&epi_read->fdinfo, fds[0], 0);
citp_fdtable_insert(&epi_write->fdinfo, fds[1], 0);
CI_MAGIC_CHECK(ni, NETIF_MAGIC);
return 0;
fail4:
if( fdtable_strict() ) CITP_FDTABLE_UNLOCK();
fail3:
CI_FREE_OBJ(epi_write);
fail2:
CI_FREE_OBJ(epi_read);
citp_netif_release_ref(ni, 0);
citp_netif_release_ref(ni, 0);
fail1:
if( CITP_OPTS.no_fail && errno != ELIBACC ) {
Log_U(ci_log("%s: failed (errno:%d) - PASSING TO OS", __FUNCTION__, errno));
return CITP_NOT_HANDLED;
}
return rc;
}
static int citp_udp_socket(int domain, int type, int protocol)
{
citp_fdinfo* fdi;
citp_sock_fdi* epi;
ef_driver_handle fd;
int rc;
ci_netif* ni;
Log_V(log(LPF "socket(%d, %d, %d)", domain, type, protocol));
epi = CI_ALLOC_OBJ(citp_sock_fdi);
if( ! epi ) {
Log_U(ci_log(LPF "socket: failed to allocate epi"));
errno = ENOMEM;
goto fail1;
}
fdi = &epi->fdinfo;
citp_fdinfo_init(fdi, &citp_udp_protocol_impl);
rc = citp_netif_alloc_and_init(&fd, &ni);
if( rc != 0 ) {
if( rc == CI_SOCKET_HANDOVER ) {
/* This implies EF_DONT_ACCELERATE is set, so we handover
* regardless of CITP_OPTS.no_fail */
CI_FREE_OBJ(epi);
return rc;
}
goto fail2;
}
/* Protect the fdtable entry until we're done initialising. */
if( fdtable_strict() ) CITP_FDTABLE_LOCK();
if((fd = ci_udp_ep_ctor(&epi->sock, ni, domain, type)) < 0) {
/*! ?? \TODO unpick the ci_udp_ep_ctor according to how failed */
Log_U(ci_log(LPF "socket: udp_ep_ctor failed"));
errno = -fd;
goto fail3;
}
citp_fdtable_new_fd_set(fd, fdip_busy, fdtable_strict());
if( fdtable_strict() ) CITP_FDTABLE_UNLOCK();
CI_DEBUG(epi->sock.s->pid = getpid());
/* We're ready. Unleash us onto the world! */
ci_assert(epi->sock.s->b.sb_aflags & CI_SB_AFLAG_NOT_READY);
ci_atomic32_and(&epi->sock.s->b.sb_aflags, ~CI_SB_AFLAG_NOT_READY);
citp_fdtable_insert(fdi, fd, 0);
Log_VSS(log(LPF "socket(%d, %d, %d) = "EF_FMT, domain, type, protocol,
EF_PRI_ARGS(epi,fd)));
return fd;
fail3:
if( CITP_OPTS.no_fail && errno != ELIBACC )
CITP_STATS_NETIF(++ni->state->stats.udp_handover_socket);
citp_netif_release_ref(ni, 0);
fail2:
CI_FREE_OBJ(epi);
fail1:
/* BUG1408: Graceful failure. We'll only fail outright if there's a
* driver/library mismatch */
if( CITP_OPTS.no_fail && errno != ELIBACC ) {
Log_U(ci_log("%s: failed (errno:%d) - PASSING TO OS", __FUNCTION__, errno));
return CI_SOCKET_HANDOVER;
}
return -1;
}
