void __citp_fdinfo_ref_count_zero(citp_fdinfo* fdi, int fdt_locked)
{
Log_V(log("%s: fd=%d on_rcz=%d", __FUNCTION__, fdi->fd,
fdi->on_ref_count_zero));
citp_fdinfo_assert_valid(fdi);
ci_assert(oo_atomic_read(&fdi->ref_count) == 0);
ci_assert_ge(fdi->fd, 0);
ci_assert_lt(fdi->fd, citp_fdtable.inited_count);
ci_assert_nequal(fdi_to_fdip(fdi), citp_fdtable.table[fdi->fd].fdip);
switch( fdi->on_ref_count_zero ) {
case FDI_ON_RCZ_CLOSE:
#if CI_CFG_FD_CACHING
if( citp_fdinfo_get_ops(fdi)->cache(fdi) == 1 ) {
if( ! fdt_locked && fdtable_strict() ) CITP_FDTABLE_LOCK();
fdtable_swap(fdi->fd, fdip_closing, fdip_unknown,
fdt_locked | fdtable_strict());
citp_fdinfo_get_ops(fdi)->dtor(fdi, fdt_locked | fdtable_strict());
if( ! fdt_locked && fdtable_strict() ) CITP_FDTABLE_UNLOCK();
citp_fdinfo_free(fdi);
break;
}
else
#endif
{
if( ! fdt_locked && fdtable_strict() ) CITP_FDTABLE_LOCK();
ci_tcp_helper_close_no_trampoline(fdi->fd);
/* The swap must occur after the close, otherwise another thread could
* cause a probe of the old endpoint info, which is about be freed.
*/
fdtable_swap(fdi->fd, fdip_closing, fdip_unknown,
fdt_locked | fdtable_strict());
citp_fdinfo_get_ops(fdi)->dtor(fdi, fdt_locked | fdtable_strict());
if( ! fdt_locked && fdtable_strict() ) CITP_FDTABLE_UNLOCK();
citp_fdinfo_free(fdi);
break;
}
case FDI_ON_RCZ_DUP2:
dup2_complete(fdi, fdi_to_fdip(fdi), fdt_locked);
break;
case FDI_ON_RCZ_HANDOVER:
citp_fdinfo_do_handover(fdi, fdt_locked);
break;
case FDI_ON_RCZ_MOVED:
citp_fdinfo_get_ops(fdi)->dtor(fdi, fdt_locked);
citp_fdinfo_free(fdi);
break;
default:
CI_DEBUG(ci_log("%s: fd=%d on_ref_count_zero=%d", __FUNCTION__,
fdi->fd, fdi->on_ref_count_zero));
ci_assert(0);
}
}
void citp_log_fn_ul(const char* msg)
{
struct iovec v[2];
int tmp_fd = 0;
if( citp.log_fd < 0 ) {
if( citp.init_level >= CITP_INIT_SYSCALLS ) {
citp.log_fd = oo_fcntl_dupfd_cloexec(STDERR_FILENO, 3);
if( citp.log_fd >= 0 && citp_fdtable.table != NULL )
citp_fdtable.table[citp.log_fd].fdip =
fdi_to_fdip(&citp_the_reserved_fd);
}
if( citp.log_fd < 0 ) {
citp.log_fd = STDERR_FILENO;
tmp_fd = 1;
}
}
v[0].iov_base = (void*) msg;
v[0].iov_len = strlen(v[0].iov_base);
v[1].iov_base = "\n";
v[1].iov_len = strlen(v[1].iov_base);
my_syscall3(writev, citp.log_fd, (long) v, 2);
if( tmp_fd )
citp.log_fd = -1;
}
void citp_fdinfo_handover(citp_fdinfo* fdi, int nonb_switch)
{
/* Please see comments in internal.h. */
volatile citp_fdinfo_p* p_fdip;
citp_fdinfo_p fdip;
unsigned fd = fdi->fd;
/* We're about to free some user-level state, so we need to interlock
** against select and poll.
*/
CITP_FDTABLE_LOCK();
p_fdip = &citp_fdtable.table[fd].fdip;
again:
fdip = *p_fdip;
if( fdip_is_busy(fdip) ) fdip = citp_fdtable_busy_wait(fd, 1);
if( fdip == fdi_to_fdip(fdi) ) {
if( fdip_cas_fail(p_fdip, fdip, fdip_busy) )
goto again;
}
else {
/* [fd] must have changed meaning under our feet. It must be closing,
** so do nothing except drop the ref passed in.
*/
ci_assert(fdip_is_closing(fdip));
ci_assert_nequal(fdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
}
if( fdip == fdi_to_fdip(fdi) ) {
ci_assert_equal(fdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
fdi->on_ref_count_zero = FDI_ON_RCZ_HANDOVER;
fdi->on_rcz.handover_nonb_switch = nonb_switch;
/* Drop the fdtable ref. When the ref count goes to zero, the handover
** will be done. We return without waiting, because the caller
** shouldn't do anything more with this socket anyway.
*/
citp_fdinfo_release_ref(fdi, 1);
}
/* Drop the ref passed in. */
citp_fdinfo_release_ref(fdi, 1);
CITP_FDTABLE_UNLOCK();
}
void citp_fdtable_insert(citp_fdinfo* fdi, unsigned fd, int fdt_locked)
{
ci_assert(fdi);
ci_assert(fdi->protocol);
ci_assert(citp_fdtable.inited_count > fd);
ci_assert_ge(oo_atomic_read(&fdi->ref_count), 1);
fdi->fd = fd;
CI_DEBUG(fdi->on_ref_count_zero = FDI_ON_RCZ_NONE);
fdi->is_special = 0;
citp_fdtable_busy_clear(fd, fdi_to_fdip(fdi), fdt_locked);
}
void citp_log_fn_drv(const char* msg)
{
if( citp.log_fd < 0 ) {
if( ef_onload_driver_open(&citp.log_fd, OO_STACK_DEV, 1) ) return;
if( citp_fdtable.table )
citp_fdtable.table[citp.log_fd].fdip=fdi_to_fdip(&citp_the_reserved_fd);
/* just to be sure: */
ci_sys_fcntl(citp.log_fd, F_SETFD, FD_CLOEXEC);
}
my_syscall3(ioctl, citp.log_fd, OO_IOC_PRINTK, (long) msg);
}
static citp_fdinfo_p
citp_fdtable_probe_restore(int fd, ci_ep_info_t * info, int print_banner)
{
citp_protocol_impl* proto = 0;
citp_fdinfo* fdi = 0;
ci_netif* ni;
int rc;
int c_sock_fdi = 1;
/* Must be holding the FD table writer lock */
CITP_FDTABLE_ASSERT_LOCKED(1);
ci_assert_nequal(info->resource_id, CI_ID_POOL_ID_NONE);
/* Will need to review this function if the following assert fires */
switch( info->fd_type ) {
case CI_PRIV_TYPE_TCP_EP: proto = &citp_tcp_protocol_impl; break;
case CI_PRIV_TYPE_UDP_EP: proto = &citp_udp_protocol_impl; break;
case CI_PRIV_TYPE_PASSTHROUGH_EP:
proto = &citp_passthrough_protocol_impl;
c_sock_fdi = 0;
break;
case CI_PRIV_TYPE_ALIEN_EP:
proto = NULL;
c_sock_fdi = 0;
break;
#if CI_CFG_USERSPACE_PIPE
case CI_PRIV_TYPE_PIPE_READER:
proto = &citp_pipe_read_protocol_impl;
c_sock_fdi = 0;
break;
case CI_PRIV_TYPE_PIPE_WRITER:
proto = &citp_pipe_write_protocol_impl;
c_sock_fdi = 0;
break;
#endif
default: ci_assert(0);
}
/* Attempt to find the user-level netif for this endpoint */
ni = citp_find_ul_netif(info->resource_id, 1);
if( ! ni ) {
ef_driver_handle netif_fd;
/* Not found, rebuild/restore the netif for this endpoint */
rc = citp_netif_recreate_probed(fd, &netif_fd, &ni);
if ( rc < 0 ) {
Log_E(log("%s: citp_netif_recreate_probed failed! (%d)",
__FUNCTION__, rc));
goto fail;
}
if( print_banner ) {
ci_log("Importing "ONLOAD_PRODUCT" "ONLOAD_VERSION" "ONLOAD_COPYRIGHT
" [%s]", ni->state->pretty_name);
}
}
else
citp_netif_add_ref(ni);
/* There is a race condition where the fd can have been created, but it has
* not yet been initialised, as we can't put a busy marker in the right place
* in the fdtable until we know what the fd is. In this case we don't want
* to probe this new info, so return the closed fd.
*/
if( SP_TO_WAITABLE(ni, info->sock_id)->sb_aflags & CI_SB_AFLAG_NOT_READY ) {
citp_fdtable_busy_clear(fd, fdip_unknown, 1);
fdi = &citp_the_closed_fd;
citp_fdinfo_ref(fdi);
return fdi_to_fdip(fdi);
}
if (c_sock_fdi) {
citp_sock_fdi* sock_fdi;
sock_fdi = CI_ALLOC_OBJ(citp_sock_fdi);
if( ! sock_fdi ) {
Log_E(log("%s: out of memory (sock_fdi)", __FUNCTION__));
goto fail;
}
fdi = &sock_fdi->fdinfo;
sock_fdi->sock.s = SP_TO_SOCK_CMN(ni, info->sock_id);
sock_fdi->sock.netif = ni;
}
else if( info->fd_type == CI_PRIV_TYPE_PASSTHROUGH_EP ) {
citp_waitable* w = SP_TO_WAITABLE(ni, info->sock_id);
citp_alien_fdi* alien_fdi;
if( ~w->sb_aflags & CI_SB_AFLAG_MOVED_AWAY_IN_EPOLL &&
fdtable_fd_move(fd, OO_IOC_FILE_MOVED) == 0 ) {
citp_netif_release_ref(ni, 1);
return fdip_passthru;
}
alien_fdi = CI_ALLOC_OBJ(citp_alien_fdi);
if( ! alien_fdi ) {
Log_E(log("%s: out of memory (alien_fdi)", __FUNCTION__));
goto fail;
}
fdi = &alien_fdi->fdinfo;
alien_fdi->netif = ni;
alien_fdi->ep = SP_TO_WAITABLE(ni, info->sock_id);
citp_passthrough_init(alien_fdi);
}
else if( info->fd_type == CI_PRIV_TYPE_ALIEN_EP ) {
citp_waitable* w = SP_TO_WAITABLE(ni, info->sock_id);
citp_sock_fdi* sock_fdi;
ci_netif* alien_ni;
sock_fdi = CI_ALLOC_OBJ(citp_sock_fdi);
if( ! sock_fdi ) {
Log_E(log("%s: out of memory (alien sock_fdi)", __FUNCTION__));
goto fail;
}
fdi = &sock_fdi->fdinfo;
rc = citp_netif_by_id(w->moved_to_stack_id, &alien_ni, 1);
if( rc != 0 ) {
goto fail;
}
sock_fdi->sock.s = SP_TO_SOCK_CMN(alien_ni, w->moved_to_sock_id);
sock_fdi->sock.netif = alien_ni;
citp_netif_release_ref(ni, 1);
/* Replace the file under this fd if possible */
if( ~w->sb_aflags & CI_SB_AFLAG_MOVED_AWAY_IN_EPOLL )
fdtable_fd_move(fd, OO_IOC_FILE_MOVED);
if( sock_fdi->sock.s->b.state & CI_TCP_STATE_TCP )
proto = &citp_tcp_protocol_impl;
else if( sock_fdi->sock.s->b.state == CI_TCP_STATE_UDP )
proto = &citp_udp_protocol_impl;
else {
CI_TEST(0);
}
}
#if CI_CFG_USERSPACE_PIPE
else {
citp_pipe_fdi* pipe_fdi;
pipe_fdi = CI_ALLOC_OBJ(citp_pipe_fdi);
if( ! pipe_fdi ) {
Log_E(log("%s: out of memory (pipe_fdi)", __FUNCTION__));
goto fail;
}
fdi = &pipe_fdi->fdinfo;
pipe_fdi->pipe = SP_TO_PIPE(ni, info->sock_id);
pipe_fdi->ni = ni;
}
#endif
citp_fdinfo_init(fdi, proto);
/* We're returning a reference to the caller. */
citp_fdinfo_ref(fdi);
citp_fdtable_insert(fdi, fd, 1);
return fdi_to_fdip(fdi);
fail:
if( ni ) citp_netif_release_ref(ni, 1);
return fdip_unknown;
}
