void
cicp_ip_cache_update_from(ci_netif* ni, ci_ip_cached_hdrs* ipcache,
const ci_ip_cached_hdrs* from_ipcache)
{
/* We can't check the inputs that come from oo_sock_cplane, but this at
* least gives us a little checking...
*/
ci_assert_equal(ipcache->ip.ip_daddr_be32, from_ipcache->ip.ip_daddr_be32);
ci_assert_equal(ipcache->dport_be16, from_ipcache->dport_be16);
ipcache->mac_integrity = from_ipcache->mac_integrity;
ipcache->ip_saddr = from_ipcache->ip_saddr;
ipcache->ip.ip_ttl = from_ipcache->ip.ip_ttl;
ipcache->status = from_ipcache->status;
ipcache->flags = from_ipcache->flags;
ipcache->nexthop = from_ipcache->nexthop;
/* ipcache->pmtus = something; */
ipcache->mtu = from_ipcache->mtu;
ipcache->ifindex = from_ipcache->ifindex;
ipcache->encap = from_ipcache->encap;
ipcache->intf_i = from_ipcache->intf_i;
ipcache->hwport = from_ipcache->hwport;
ipcache->ether_offset = from_ipcache->ether_offset;
memcpy(ipcache->ether_header, from_ipcache->ether_header,
sizeof(ipcache->ether_header));
}
int oo_hw_filter_set_thc(struct oo_hw_filter* oofilter,
tcp_helper_cluster_t* thc, int protocol,
unsigned daddr, int dport,
unsigned hwport_mask)
{
struct efx_filter_spec spec;
int hwport, base_vi_id, rc;
ci_assert_equal(oofilter->trs, NULL);
oofilter->thc = thc;
for( hwport = 0; hwport < CI_CFG_MAX_REGISTER_INTERFACES; ++hwport )
if( hwport_mask & (1 << hwport) && thc->thc_vi_set[hwport] != NULL ) {
base_vi_id = efrm_vi_set_get_base(thc->thc_vi_set[hwport]);
efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED,
EFX_FILTER_FLAG_RX_SCATTER | EFX_FILTER_FLAG_RX_RSS,
base_vi_id);
spec.rss_context = efrm_vi_set_get_rss_context(thc->thc_vi_set[hwport]);
#if EFX_DRIVERLINK_API_VERSION >= 15
{
int stack_id = tcp_helper_cluster_vi_hw_stack_id(thc, hwport);
ci_assert( stack_id >= 0 );
efx_filter_set_stack_id(&spec, stack_id);
}
#endif
rc = efx_filter_set_ipv4_local(&spec, protocol, daddr, dport);
ci_assert_equal(rc, 0);
rc = efrm_filter_insert(get_client(hwport), &spec, false);
if( rc < 0 ) {
oo_hw_filter_clear(oofilter);
return rc;
}
oofilter->filter_id[hwport] = rc;
}
return 0;
}
static int
efab_tcp_drop_from_acceptq(ci_private_t *priv, void *arg)
{
struct oo_op_tcp_drop_from_acceptq *carg = arg;
tcp_helper_resource_t *thr;
tcp_helper_endpoint_t *ep;
citp_waitable *w;
ci_tcp_state *ts;
int rc = -EINVAL;
/* find stack */
rc = efab_thr_table_lookup(NULL, carg->stack_id,
EFAB_THR_TABLE_LOOKUP_CHECK_USER |
EFAB_THR_TABLE_LOOKUP_NO_UL,
&thr);
if( rc < 0 )
return rc;
ci_assert( thr->k_ref_count & TCP_HELPER_K_RC_NO_USERLAND );
/* find endpoint and drop OS socket */
ep = ci_trs_get_valid_ep(thr, carg->sock_id);
if( ep == NULL )
goto fail1;
w = SP_TO_WAITABLE(&thr->netif, carg->sock_id);
if( !(w->state & CI_TCP_STATE_TCP) || w->state == CI_TCP_LISTEN )
goto fail2;
ts = SP_TO_TCP(&thr->netif, carg->sock_id);
ci_assert(ep->os_port_keeper);
ci_assert_equal(ep->os_socket, NULL);
LOG_TV(ci_log("%s: send reset to non-accepted connection", __FUNCTION__));
/* copy from ci_tcp_listen_shutdown_queues() */
ci_assert(ts->s.b.sb_aflags & CI_SB_AFLAG_TCP_IN_ACCEPTQ);
rc = ci_netif_lock(&thr->netif);
if( rc != 0 ) {
ci_assert_equal(rc, -EINTR);
rc = -ERESTARTSYS;
goto fail2;
}
ci_bit_clear(&ts->s.b.sb_aflags, CI_SB_AFLAG_TCP_IN_ACCEPTQ_BIT);
/* We have no way to close this connection from the other side:
* there was no RST from peer. */
ci_assert_nequal(ts->s.b.state, CI_TCP_CLOSED);
ci_assert_nequal(ts->s.b.state, CI_TCP_TIME_WAIT);
ci_tcp_send_rst(&thr->netif, ts);
ci_tcp_drop(&thr->netif, ts, ECONNRESET);
ci_assert_equal(ep->os_port_keeper, NULL);
ci_netif_unlock(&thr->netif);
efab_tcp_helper_k_ref_count_dec(thr, 1);
return 0;
fail1:
efab_thr_release(thr);
fail2:
ci_log("%s: inconsistent ep %d:%d", __func__, carg->stack_id, carg->sock_id);
return rc;
}
citp_waitable_obj* citp_waitable_obj_alloc(ci_netif* netif)
{
citp_waitable_obj* wo;
ci_assert(netif);
ci_assert(ci_netif_is_locked(netif));
if( netif->state->deferred_free_eps_head != CI_ILL_END ) {
ci_uint32 link;
do
link = netif->state->deferred_free_eps_head;
while( ci_cas32_fail(&netif->state->deferred_free_eps_head,
link, CI_ILL_END));
while( link != CI_ILL_END ) {
citp_waitable* w = ID_TO_WAITABLE(netif, link);
link = w->next_id;
CI_DEBUG(w->next_id = CI_ILL_END);
ci_assert_equal(w->state, CI_TCP_STATE_FREE);
ci_assert(OO_SP_IS_NULL(w->wt_next));
w->wt_next = netif->state->free_eps_head;
netif->state->free_eps_head = W_SP(w);
}
}
if( OO_SP_IS_NULL(netif->state->free_eps_head) ) {
ci_tcp_helper_more_socks(netif);
if( OO_SP_IS_NULL(netif->state->free_eps_head) )
ci_netif_timeout_reap(netif);
}
if( OO_SP_IS_NULL(netif->state->free_eps_head) )
return NULL;
LOG_TV(ci_log("%s: allocating %d", __FUNCTION__,
OO_SP_FMT(netif->state->free_eps_head)));
ci_assert(IS_VALID_SOCK_P(netif, netif->state->free_eps_head));
#if !defined(__KERNEL__) && !defined (CI_HAVE_OS_NOPAGE)
ci_netif_mmap_shmbuf(netif,
(netif->state->free_eps_head >> EP_BUF_BLOCKSHIFT) + 1);
#endif
wo = SP_TO_WAITABLE_OBJ(netif, netif->state->free_eps_head);
ci_assert(OO_SP_EQ(W_SP(&wo->waitable), netif->state->free_eps_head));
ci_assert_equal(wo->waitable.state, CI_TCP_STATE_FREE);
ci_assert_equal(wo->waitable.sb_aflags, (CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_NOT_READY));
ci_assert_equal(wo->waitable.lock.wl_val, 0);
netif->state->free_eps_head = wo->waitable.wt_next;
CI_DEBUG(wo->waitable.wt_next = OO_SP_NULL);
ci_assert_equal(wo->waitable.state, CI_TCP_STATE_FREE);
return wo;
}
static int
efab_vi_rm_mmap_io(struct efrm_vi *virs,
unsigned long *bytes, void *opaque,
int *map_num, unsigned long *offset)
{
int rc;
int len;
int instance;
int base;
unsigned vi_stride;
struct efhw_nic *nic;
nic = efrm_client_get_nic(virs->rs.rs_client);
instance = virs->rs.rs_instance;
len = CI_MIN(*bytes, CI_PAGE_SIZE);
*bytes -=len;
/* Make sure we can get away with a single page here. */
switch (nic->devtype.arch) {
case EFHW_ARCH_FALCON:
ci_assert_lt(falcon_tx_dma_page_offset(instance), CI_PAGE_SIZE);
ci_assert_lt(falcon_rx_dma_page_offset(instance), CI_PAGE_SIZE);
ci_assert_equal(falcon_tx_dma_page_base(instance),
falcon_rx_dma_page_base(instance));
base = falcon_tx_dma_page_base(instance);
break;
case EFHW_ARCH_EF10:
vi_stride = nic->vi_stride;
ci_assert_lt(ef10_tx_dma_page_offset(vi_stride, instance), CI_PAGE_SIZE);
ci_assert_lt(ef10_rx_dma_page_offset(vi_stride, instance), CI_PAGE_SIZE);
ci_assert_equal(ef10_tx_dma_page_base(vi_stride, instance),
ef10_rx_dma_page_base(vi_stride, instance));
base = ef10_tx_dma_page_base(vi_stride, instance);
break;
default:
EFCH_ERR("%s: ERROR: unknown nic type (%d)", __FUNCTION__,
nic->devtype.arch);
base = 0; /* To quiet the compiler */
BUG();
}
rc = ci_mmap_bar(nic, base, len, opaque, map_num, offset, 0);
if (rc < 0 ) {
EFCH_ERR("%s: ERROR: ci_mmap_bar failed rc=%d", __FUNCTION__, rc);
return rc;
}
return 0;
}
static int oo_hw_filter_set_hwport(struct oo_hw_filter* oofilter, int hwport,
int protocol,
unsigned saddr, int sport,
unsigned daddr, int dport,
ci_uint16 vlan_id, unsigned src_flags)
{
struct efx_filter_spec spec;
int rc = 0, vi_id;
ci_assert_equal(oofilter->thc, NULL);
ci_assert(oofilter->filter_id[hwport] < 0);
if( (vi_id = tcp_helper_rx_vi_id(oofilter->trs, hwport)) >= 0 ) {
int flags = EFX_FILTER_FLAG_RX_SCATTER;
int hw_rx_loopback_supported =
tcp_helper_vi_hw_rx_loopback_supported(oofilter->trs, hwport);
ci_assert( hw_rx_loopback_supported >= 0 );
if( hw_rx_loopback_supported && (src_flags & OO_HW_SRC_FLAG_LOOPBACK) ) {
flags |= EFX_FILTER_FLAG_TX;
}
efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED, flags, vi_id);
#if EFX_DRIVERLINK_API_VERSION >= 15
{
unsigned stack_id = tcp_helper_vi_hw_stack_id(oofilter->trs, hwport);
ci_assert( stack_id >= 0 );
efx_filter_set_stack_id(&spec, stack_id);
}
#endif
if( saddr != 0 )
rc = efx_filter_set_ipv4_full(&spec, protocol, daddr, dport,
saddr, sport);
else
rc = efx_filter_set_ipv4_local(&spec, protocol, daddr, dport);
ci_assert_equal(rc, 0);
/* note: bug 42561 affecting loopback on VLAN 0 with fw <= v4_0_6_6688 */
if( vlan_id != OO_HW_VLAN_UNSPEC ) {
rc = efx_filter_set_eth_local(&spec, vlan_id, NULL);
ci_assert_equal(rc, 0);
}
rc = efrm_filter_insert(get_client(hwport), &spec, false);
if( rc >= 0 ) {
oofilter->filter_id[hwport] = rc;
rc = 0;
}
}
return rc;
}
int efab_vi_resource_mmap(struct efrm_vi *virs, unsigned long *bytes,
void *opaque, int *map_num, unsigned long *offset,
int index)
{
int rc = -EINVAL;
EFRM_RESOURCE_ASSERT_VALID(&virs->rs, 0);
ci_assert_equal((*bytes &~ CI_PAGE_MASK), 0);
switch( index ) {
case EFCH_VI_MMAP_IO:
rc = efab_vi_rm_mmap_io(virs, bytes, opaque, map_num, offset);
break;
case EFCH_VI_MMAP_MEM:
rc = efab_vi_rm_mmap_mem(virs, bytes, opaque, map_num, offset);
break;
case EFCH_VI_MMAP_PIO:
rc = efab_vi_rm_mmap_pio(virs, bytes, opaque, map_num, offset);
break;
case EFCH_VI_MMAP_CTPIO:
rc = efab_vi_rm_mmap_ctpio(virs, bytes, opaque, map_num, offset);
break;
default:
ci_assert(0);
}
return rc;
}
/* Free a thc.
*
* You must hold the thc_mutex before calling this function.
*/
static void thc_cluster_free(tcp_helper_cluster_t* thc)
{
int i;
tcp_helper_cluster_t *thc_walk, *thc_prev;
thc_uninstall_tproxy(thc);
/* Free up resources within the thc */
for( i = 0; i < CI_CFG_MAX_REGISTER_INTERFACES; ++i )
if( thc->thc_vi_set[i] != NULL )
efrm_vi_set_release(thc->thc_vi_set[i]);
ci_assert(ci_dllist_is_empty(&thc->thc_tlos));
/* Remove from the thc_head list */
thc_walk = thc_head;
thc_prev = NULL;
while( thc_walk != NULL ) {
if( thc_walk == thc ) {
if( thc_walk == thc_head ) {
ci_assert_equal(thc_prev, NULL);
thc_head = thc_walk->thc_next;
}
else {
thc_prev->thc_next = thc_walk->thc_next;
}
kfree(thc_walk);
return;
}
thc_prev = thc_walk;
thc_walk = thc_walk->thc_next;
}
ci_assert(0);
}
static int
efab_eplock_unlock_and_wake_rsop(ci_private_t *priv, void *unused)
{
if (priv->thr == NULL)
return -EINVAL;
ci_assert_equal(priv->thr->netif.flags & CI_NETIF_FLAG_IN_DL_CONTEXT, 0);
return efab_eplock_unlock_and_wake(&priv->thr->netif, 0);
}
static void citp_fdinfo_do_handover(citp_fdinfo* fdi, int fdt_locked)
{
int rc;
citp_fdinfo* epoll_fdi = NULL;
int os_fd = fdi->fd;
#ifndef NDEBUG
/* Yuk: does for UDP too. */
volatile citp_fdinfo_p* p_fdip;
p_fdip = &citp_fdtable.table[fdi->fd].fdip;
ci_assert(fdip_is_busy(*p_fdip));
#endif
Log_V(ci_log("%s: fd=%d nonb_switch=%d", __FUNCTION__, fdi->fd,
fdi->on_rcz.handover_nonb_switch));
if( fdi->epoll_fd >= 0 ) {
epoll_fdi = citp_epoll_fdi_from_member(fdi, fdt_locked);
if( epoll_fdi->protocol->type == CITP_EPOLLB_FD )
citp_epollb_on_handover(epoll_fdi, fdi);
}
rc = fdtable_fd_move(fdi->fd, OO_IOC_TCP_HANDOVER);
if( rc == -EBUSY && fdi->epoll_fd >= 0 ) {
ci_assert(fdi_to_sock_fdi(fdi)->sock.s->b.sb_aflags &
CI_SB_AFLAG_MOVED_AWAY);
/* If this is our epoll, we can do full handover: we manually add os
* fd into the epoll set.
* Fixme: ensure we are not in _other_ epoll sets */
ci_bit_clear(&fdi_to_sock_fdi(fdi)->sock.s->b.sb_aflags,
CI_SB_AFLAG_MOVED_AWAY_IN_EPOLL_BIT);
rc = fdtable_fd_move(fdi->fd, OO_IOC_FILE_MOVED);
}
if( rc != 0 ) {
citp_fdinfo* new_fdi;
if( ! fdt_locked ) CITP_FDTABLE_LOCK();
new_fdi = citp_fdtable_probe_locked(fdi->fd, CI_TRUE, CI_TRUE);
citp_fdinfo_release_ref(new_fdi, 1);
if( ! fdt_locked ) CITP_FDTABLE_UNLOCK();
ci_assert_equal(citp_fdinfo_get_type(new_fdi), CITP_PASSTHROUGH_FD);
os_fd = fdi_to_alien_fdi(new_fdi)->os_socket;
}
if( fdi->on_rcz.handover_nonb_switch >= 0 ) {
int on_off = !! fdi->on_rcz.handover_nonb_switch;
int rc = ci_sys_ioctl(os_fd, FIONBIO, &on_off);
if( rc < 0 )
Log_E(ci_log("%s: ioctl failed on_off=%d", __FUNCTION__, on_off));
}
if( rc != 0 )
goto exit;
citp_fdtable_busy_clear(fdi->fd, fdip_passthru, fdt_locked);
exit:
citp_fdinfo_get_ops(fdi)->dtor(fdi, fdt_locked);
if( epoll_fdi != NULL && epoll_fdi->protocol->type == CITP_EPOLL_FD )
citp_epoll_on_handover(epoll_fdi, fdi, fdt_locked);
if( epoll_fdi != NULL )
citp_fdinfo_release_ref(epoll_fdi, fdt_locked);
citp_fdinfo_free(fdi);
}
void oo_hw_filter_clear_hwports(struct oo_hw_filter* oofilter,
unsigned hwport_mask)
{
int hwport;
ci_assert_equal(oofilter->thc, NULL);
if( oofilter->trs != NULL )
for( hwport = 0; hwport < CI_CFG_MAX_REGISTER_INTERFACES; ++hwport )
if( hwport_mask & (1 << hwport) )
oo_hw_filter_clear_hwport(oofilter, hwport);
}
static int citp_pipe_send(citp_fdinfo* fdinfo,
const struct msghdr* msg, int flags)
{
citp_pipe_fdi* epi = fdi_to_pipe_fdi(fdinfo);
ci_assert_equal(flags, 0);
ci_assert(msg);
ci_assert(msg->msg_iov);
return ci_pipe_write(epi->ni, epi->pipe, msg->msg_iov, msg->msg_iovlen);
}
void oo_hw_filter_transfer(struct oo_hw_filter* oofilter_old,
struct oo_hw_filter* oofilter_new,
unsigned hwport_mask)
{
int hwport;
ci_assert_equal(oofilter_new->thc, NULL);
ci_assert_equal(oofilter_old->thc, NULL);
if( oofilter_old->trs == NULL )
return;
ci_assert_equal(oofilter_old->trs, oofilter_new->trs);
for( hwport = 0; hwport < CI_CFG_MAX_REGISTER_INTERFACES; ++hwport )
if( (hwport_mask & (1u << hwport)) &&
oofilter_old->filter_id[hwport] >= 0 ) {
ci_assert(oofilter_new->filter_id[hwport] < 0);
oofilter_new->filter_id[hwport] = oofilter_old->filter_id[hwport];
oofilter_old->filter_id[hwport] = -1;
}
}
int
citp_passthrough_accept(citp_fdinfo* fdi,
struct sockaddr* sa, socklen_t* p_sa_len, int flags,
citp_lib_context_t* lib_context)
{
#if CI_LIBC_HAS_accept4
return ci_sys_accept4(fdi_to_alien_fdi(fdi)->os_socket,
sa, p_sa_len, flags);
#else
ci_assert_equal(flags, 0);
return ci_sys_accept(fdi_to_alien_fdi(fdi)->os_socket,
sa, p_sa_len);
#endif
}
void citp_fdinfo_handover(citp_fdinfo* fdi, int nonb_switch)
{
/* Please see comments in internal.h. */
volatile citp_fdinfo_p* p_fdip;
citp_fdinfo_p fdip;
unsigned fd = fdi->fd;
/* We're about to free some user-level state, so we need to interlock
** against select and poll.
*/
CITP_FDTABLE_LOCK();
p_fdip = &citp_fdtable.table[fd].fdip;
again:
fdip = *p_fdip;
if( fdip_is_busy(fdip) ) fdip = citp_fdtable_busy_wait(fd, 1);
if( fdip == fdi_to_fdip(fdi) ) {
if( fdip_cas_fail(p_fdip, fdip, fdip_busy) )
goto again;
}
else {
/* [fd] must have changed meaning under our feet. It must be closing,
** so do nothing except drop the ref passed in.
*/
ci_assert(fdip_is_closing(fdip));
ci_assert_nequal(fdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
}
if( fdip == fdi_to_fdip(fdi) ) {
ci_assert_equal(fdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
fdi->on_ref_count_zero = FDI_ON_RCZ_HANDOVER;
fdi->on_rcz.handover_nonb_switch = nonb_switch;
/* Drop the fdtable ref. When the ref count goes to zero, the handover
** will be done. We return without waiting, because the caller
** shouldn't do anything more with this socket anyway.
*/
citp_fdinfo_release_ref(fdi, 1);
}
/* Drop the ref passed in. */
citp_fdinfo_release_ref(fdi, 1);
CITP_FDTABLE_UNLOCK();
}
void ci_sock_cmn_init(ci_netif* ni, ci_sock_cmn* s)
{
oo_p sp;
/* Poison. */
CI_DEBUG(memset(&s->b + 1, 0xf0, (char*) (s + 1) - (char*) (&s->b + 1)));
citp_waitable_reinit(ni, &s->b);
oo_sock_cplane_init(&s->cp);
s->local_peer = OO_SP_NULL;
s->s_flags = CI_SOCK_FLAG_CONNECT_MUST_BIND | CI_SOCK_FLAG_PMTU_DO;
s->s_aflags = 0u;
ci_assert_equal( 0, CI_IP_DFLT_TOS );
s->so_priority = 0;
/* SO_SNDBUF & SO_RCVBUF. See also ci_tcp_set_established_state() which
* may modify these values.
*/
memset(&s->so, 0, sizeof(s->so));
s->so.sndbuf = NI_OPTS(ni).tcp_sndbuf_def;
s->so.rcvbuf = NI_OPTS(ni).tcp_rcvbuf_def;
s->rx_bind2dev_ifindex = CI_IFID_BAD;
/* These don't really need to be initialised, as only significant when
* rx_bind2dev_ifindex != CI_IFID_BAD. But makes stackdump output
* cleaner this way...
*/
s->rx_bind2dev_base_ifindex = 0;
s->rx_bind2dev_vlan = 0;
s->cmsg_flags = 0u;
s->timestamping_flags = 0u;
s->os_sock_status = OO_OS_STATUS_TX;
ci_ip_queue_init(&s->timestamp_q);
s->timestamp_q_extract = OO_PP_NULL;
ci_sock_cmn_reinit(ni, s);
sp = oo_sockp_to_statep(ni, SC_SP(s));
OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_sock_cmn, reap_link));
ci_ni_dllist_link_init(ni, &s->reap_link, sp, "reap");
ci_ni_dllist_self_link(ni, &s->reap_link);
}
static int
oo_copy_pkt_to_iovec_no_adv(ci_netif* ni, const ci_ip_pkt_fmt* pkt,
ci_iovec_ptr* piov, int bytes_to_copy)
{
/* Copy data from [pkt] to [piov], following [pkt->frag_next] as
* necessary. Does not modify [pkt]. May or may not advance [piov].
* The packet must contain at least [bytes_to_copy] of data in the
* [pkt->buf]. [piov] may contain an arbitrary amount of space.
*
* Returns number of bytes copied on success, or -EFAULT otherwise.
*/
int n, pkt_left, pkt_off = 0;
int bytes_copied = 0;
while( 1 ) {
pkt_left = oo_offbuf_left(&pkt->buf) - pkt_off;
n = CI_MIN(pkt_left, CI_IOVEC_LEN(&piov->io));
n = CI_MIN(n, bytes_to_copy);
if(CI_UNLIKELY( do_copy(CI_IOVEC_BASE(&piov->io),
oo_offbuf_ptr(&pkt->buf) + pkt_off, n) != 0 ))
return -EFAULT;
bytes_copied += n;
pkt_off += n;
if( n == bytes_to_copy )
return bytes_copied;
bytes_to_copy -= n;
if( n == pkt_left ) {
/* Caller guarantees that packet contains at least [bytes_to_copy]. */
ci_assert(OO_PP_NOT_NULL(pkt->frag_next));
ci_iovec_ptr_advance(piov, n);
pkt = PKT_CHK_NNL(ni, pkt->frag_next);
pkt_off = 0;
/* We're unlikely to hit end-of-pkt-buf and end-of-iovec at the same
* time, and if we do, just go round the loop again.
*/
continue;
}
ci_assert_equal(n, CI_IOVEC_LEN(&piov->io));
if( piov->iovlen == 0 )
return bytes_copied;
piov->io = *piov->iov++;
--piov->iovlen;
}
}
void ci_sock_cmn_timestamp_q_drop(ci_netif* netif, ci_sock_cmn* s)
{
ci_ip_pkt_queue* qu = &s->timestamp_q;
ci_ip_pkt_fmt* p;
CI_DEBUG(int i = qu->num);
ci_assert(netif);
ci_assert(qu);
while( OO_PP_NOT_NULL(qu->head) CI_DEBUG( && i-- > 0) ) {
p = PKT_CHK(netif, qu->head);
qu->head = p->tsq_next;
ci_netif_pkt_release(netif, p);
}
ci_assert_equal(i, 0);
ci_assert(OO_PP_IS_NULL(qu->head));
qu->num = 0;
}
int oo_hw_filter_set(struct oo_hw_filter* oofilter,
tcp_helper_resource_t* trs, int protocol,
unsigned saddr, int sport,
unsigned daddr, int dport,
ci_uint16 vlan_id, unsigned set_vlan_mask,
unsigned hwport_mask, unsigned src_flags)
{
int rc;
ci_assert_equal(oofilter->thc, NULL);
oo_hw_filter_clear(oofilter);
oofilter->trs = trs;
rc = oo_hw_filter_add_hwports(oofilter, protocol, saddr, sport, daddr, dport,
vlan_id, set_vlan_mask, hwport_mask, src_flags);
if( rc < 0 )
oo_hw_filter_clear(oofilter);
return rc;
}
static int citp_udp_bind(citp_fdinfo* fdinfo, const struct sockaddr* sa,
socklen_t sa_len)
{
citp_sock_fdi *epi = fdi_to_sock_fdi(fdinfo);
citp_socket* ep = &epi->sock;
ci_sock_cmn* s = ep->s;
int rc;
Log_V(log(LPF "bind(%d, sa, %d)", fdinfo->fd, sa_len));
ci_udp_handle_force_reuseport(fdinfo->fd, ep, sa, sa_len);
if( (s->s_flags & CI_SOCK_FLAG_REUSEPORT) != 0 ) {
if( (rc = ci_udp_reuseport_bind(ep, fdinfo->fd, sa, sa_len)) == 0 ) {
/* The socket has moved so need to reprobe the fd. This will also
* map the the new stack into user space of the executing process.
*/
fdinfo = citp_fdtable_lookup(fdinfo->fd);
fdinfo = citp_reprobe_moved(fdinfo, CI_FALSE);
epi = fdi_to_sock_fdi(fdinfo);
ep = &epi->sock;
ci_netif_cluster_prefault(ep->netif);
}
else {
goto done;
}
}
ci_netif_lock_fdi(epi);
rc = ci_udp_bind(ep, fdinfo->fd, sa, sa_len);
ci_netif_unlock_fdi(epi);
done:
if( rc == CI_SOCKET_HANDOVER ) {
ci_assert_equal(s->s_flags & CI_SOCK_FLAG_REUSEPORT_LEGACY, 0);
CITP_STATS_NETIF(++epi->sock.netif->state->stats.udp_handover_bind);
citp_fdinfo_handover(fdinfo, -1);
return 0;
}
citp_fdinfo_release_ref( fdinfo, 0 );
return rc;
}
int oo_hw_filter_add_hwports(struct oo_hw_filter* oofilter,
int protocol,
unsigned saddr, int sport,
unsigned daddr, int dport,
ci_uint16 vlan_id, unsigned set_vlan_mask,
unsigned hwport_mask, unsigned src_flags)
{
int rc1, rc = 0, ok_seen = 0, hwport;
uint16_t set_vlan_id;
ci_assert(oofilter->trs != NULL);
ci_assert_equal(oofilter->thc, NULL);
for( hwport = 0; hwport < CI_CFG_MAX_REGISTER_INTERFACES; ++hwport )
if( (hwport_mask & (1u << hwport)) && oofilter->filter_id[hwport] < 0 ) {
/* If we've been told to set the vlan when installing the filter on this
* port then use provided vlan_id, otherwise use OO_HW_VLAN_UNSPEC.
*/
set_vlan_id = (set_vlan_mask & (1u << hwport)) ?
vlan_id : OO_HW_VLAN_UNSPEC;
rc1 = oo_hw_filter_set_hwport(oofilter, hwport, protocol,
saddr, sport, daddr, dport, set_vlan_id,
src_flags);
/* Need to know if any interfaces are ok */
if( ! rc1 )
ok_seen = 1;
/* Preserve the most severe error seen - other errors are more severe
* then firewall denial, and it is more severe than no error.
*/
if( rc1 && ( !rc || rc == -EACCES ||
(rc == -EBUSY && !oof_all_ports_required) ) )
rc = rc1;
}
if( ok_seen &&
( ( rc == -EACCES ) ||
( !oof_all_ports_required && ( rc == -EBUSY ) ) ) ) {
/* If some interfaces, but not ALL interfaces, have blocked the filter
* then consider the filter added.
*/
rc = 0;
}
return rc;
}
/* Kill an orphan stack in the thc
*
* You must hold the thc_mutex before calling this function.
*
* You cannot hold the THR_TABLE.lock when calling this function.
*/
static void thc_kill_an_orphan(tcp_helper_cluster_t* thc)
{
tcp_helper_resource_t* thr = NULL;
int rc;
rc = thc_get_an_orphan(thc, &thr);
ci_assert_equal(rc, 0);
/* This is generally called when the stack is being freed. But as
* we are holding the thc_mutex, we will deadlock if we took that
* path. So we remove thr from the thc now. */
thc_remove_thr(thc, thr);
LOG_U(ci_log("Clustering: Killing orphan stack %d", thr->id));
rc = tcp_helper_kill_stack_by_id(thr->id);
#ifndef NDEBUG
if( rc != 0 && rc != -EBUSY )
LOG_U(ci_log("%s: tcp_helper_kill_stack_by_id(%d): failed %d", __FUNCTION__,
thr->id, rc));
#endif
}
/*! Run any pending signal handlers
** \param our_info Thread-specific context for current thread
*/
void citp_signal_run_pending(citp_signal_info *our_info)
{
/* preserve errno across calls to this function, as it's often
called at error time as a result of EXIT_LIB */
int old_errno = errno;
int i;
LOG_SIG(log("%s: start", __FUNCTION__));
ci_wmb();
ci_assert_equal(our_info->inside_lib, 0);
ci_assert(our_info->aflags & OO_SIGNAL_FLAG_HAVE_PENDING);
ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_HAVE_PENDING);
for( i = 0; i < OO_SIGNAL_MAX_PENDING; i++ ) {
siginfo_t saved_info;
void *saved_context;
int signum;
if (our_info->signals[i].signum == 0)
break;
saved_context = our_info->signals[i].saved_context;
if( our_info->signals[i].saved_context )
memcpy(&saved_info, &our_info->signals[i].saved_info,
sizeof(saved_info));
signum = our_info->signals[i].signum;
if( ci_cas32_fail(&our_info->signals[i].signum, signum, 0) )
break;
if( citp_signal_run_app_handler(
signum,
saved_context == NULL ? NULL : &saved_info,
saved_context) )
ci_atomic32_or(&our_info->aflags, OO_SIGNAL_FLAG_NEED_RESTART);
else
ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_NEED_RESTART);
}
LOG_SIG(log("%s: end", __FUNCTION__));
errno = old_errno;
}
/* Remove the thr from the list of stacks tracked by the thc.
*
* You must hold the thc_mutex before calling this function.
*/
static void thc_remove_thr(tcp_helper_cluster_t* thc,
tcp_helper_resource_t* thr)
{
tcp_helper_resource_t* thr_walk = thc->thc_thr_head;
tcp_helper_resource_t* thr_prev = NULL;
while( thr_walk != NULL ) {
if( thr_walk == thr ) {
if( thr_prev == NULL ) {
ci_assert_equal(thr_walk, thc->thc_thr_head);
thc->thc_thr_head = thr_walk->thc_thr_next;
}
else {
thr_prev->thc_thr_next = thr_walk->thc_thr_next;
}
thr->thc = NULL;
return;
}
thr_prev = thr_walk;
thr_walk = thr_walk->thc_thr_next;
}
ci_assert(0);
}
int oo_os_sock_sendmsg(ci_netif* ni, oo_sp sock_p,
const struct msghdr* msg, int flags)
{
oo_os_sock_sendmsg_t op;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg_iov, msg->msg_iov);
op.msg_iovlen = msg->msg_iovlen;
CI_USER_PTR_SET(op.msg_name, msg->msg_name);
op.msg_namelen = msg->msg_namelen;
#ifdef __i386__
/* compat cmsg is not handled in this function */
ci_assert_equal(msg->msg_controllen, 0);
op.msg_controllen = 0;
CI_USER_PTR_SET(op.msg_control, NULL);
#else
CI_USER_PTR_SET(op.msg_control, msg->msg_control);
op.msg_controllen = msg->msg_controllen;
#endif
return oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_SENDMSG, &op);
}
int citp_ep_close(unsigned fd)
{
volatile citp_fdinfo_p* p_fdip;
citp_fdinfo_p fdip;
int rc, got_lock;
citp_fdinfo* fdi;
/* Do not touch shared fdtable when in vfork child. */
if( oo_per_thread_get()->in_vfork_child )
return ci_tcp_helper_close_no_trampoline(fd);
/* Interlock against other closes, against the fdtable being extended,
** and against select and poll.
*/
CITP_FDTABLE_LOCK();
got_lock = 1;
__citp_fdtable_extend(fd);
if( fd >= citp_fdtable.inited_count ) {
rc = ci_sys_close(fd);
goto done;
}
p_fdip = &citp_fdtable.table[fd].fdip;
again:
fdip = *p_fdip;
if( fdip_is_busy(fdip) ) fdip = citp_fdtable_busy_wait(fd, 1);
if( fdip_is_closing(fdip) | fdip_is_reserved(fdip) ) {
/* Concurrent close or attempt to close reserved. */
Log_V(ci_log("%s: fd=%d closing=%d reserved=%d", __FUNCTION__, fd,
fdip_is_closing(fdip), fdip_is_reserved(fdip)));
errno = EBADF;
rc = -1;
goto done;
}
#if CI_CFG_FD_CACHING
/* Need to check in case this sucker's cached */
if( fdip_is_unknown(fdip) ) {
fdi = citp_fdtable_probe_locked(fd, CI_FALSE, CI_FALSE);
if( fdi == &citp_the_closed_fd ) {
citp_fdinfo_release_ref(fdi, CI_TRUE);
errno = EBADF;
rc = -1;
goto done;
}
if( fdi )
citp_fdinfo_release_ref(fdi, CI_TRUE);
}
#endif
ci_assert(fdip_is_normal(fdip) | fdip_is_passthru(fdip) |
fdip_is_unknown(fdip));
/* Swap in the "closed" pseudo-fdinfo. This lets any other thread know
** that we're in the middle of closing this fd.
*/
if( fdip_cas_fail(p_fdip, fdip, fdip_closing) )
goto again;
if( fdip_is_normal(fdip) ) {
fdi = fdip_to_fdi(fdip);
CITP_FDTABLE_UNLOCK();
got_lock = 0;
if( fdi->is_special ) {
Log_V(ci_log("%s: fd=%d is_special, returning EBADF", __FUNCTION__, fd));
errno = EBADF;
rc = -1;
fdtable_swap(fd, fdip_closing, fdip, 0);
goto done;
}
Log_V(ci_log("%s: fd=%d u/l socket", __FUNCTION__, fd));
ci_assert_equal(fdi->fd, fd);
ci_assert_equal(fdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
fdi->on_ref_count_zero = FDI_ON_RCZ_CLOSE;
if( fdi->epoll_fd >= 0 ) {
citp_fdinfo* epoll_fdi = citp_epoll_fdi_from_member(fdi, 0);
if( epoll_fdi ) {
if( epoll_fdi->protocol->type == CITP_EPOLL_FD )
citp_epoll_on_close(epoll_fdi, fdi, 0);
citp_fdinfo_release_ref(epoll_fdi, 0);
}
}
citp_fdinfo_release_ref(fdi, 0);
rc = 0;
}
else {
ci_assert(fdip_is_passthru(fdip) ||
fdip_is_unknown(fdip));
if( ! fdtable_strict() ) {
CITP_FDTABLE_UNLOCK();
got_lock = 0;
}
Log_V(ci_log("%s: fd=%d passthru=%d unknown=%d", __FUNCTION__, fd,
fdip_is_passthru(fdip), fdip_is_unknown(fdip)));
fdtable_swap(fd, fdip_closing, fdip_unknown, fdtable_strict());
rc = ci_tcp_helper_close_no_trampoline(fd);
}
done:
if( got_lock ) CITP_FDTABLE_UNLOCK();
FDTABLE_ASSERT_VALID();
return rc;
}
int citp_ep_dup3(unsigned fromfd, unsigned tofd, int flags)
{
volatile citp_fdinfo_p* p_tofdip;
citp_fdinfo_p tofdip;
unsigned max;
Log_V(log("%s(%d, %d)", __FUNCTION__, fromfd, tofd));
/* Must be checked by callers. */
ci_assert(fromfd != tofd);
/* Hack: if [tofd] is the fd we're using for logging, we'd better choose
** a different one!
*/
if( tofd == citp.log_fd ) citp_log_change_fd();
ci_assert(citp.init_level >= CITP_INIT_FDTABLE);
max = CI_MAX(fromfd, tofd);
if( max >= citp_fdtable.inited_count ) {
ci_assert(max < citp_fdtable.size);
CITP_FDTABLE_LOCK();
__citp_fdtable_extend(max);
CITP_FDTABLE_UNLOCK();
}
/* Bug1151: Concurrent threads doing dup2(x,y) and dup2(y,x) can deadlock
** against one another. So we take out a fat lock to prevent concurrent
** dup2()s.
*/
/* Lock tofd. We need to interlock against select and poll etc, so we
** also grab the exclusive lock. Also grab the bug1151 lock.
*/
pthread_mutex_lock(&citp_dup_lock);
CITP_FDTABLE_LOCK();
p_tofdip = &citp_fdtable.table[tofd].fdip;
lock_tofdip_again:
tofdip = *p_tofdip;
if( fdip_is_busy(tofdip) )
tofdip = citp_fdtable_busy_wait(tofd, 1);
if( fdip_is_closing(tofdip) )
tofdip = citp_fdtable_closing_wait(tofd, 1);
if( fdip_is_reserved(tofdip) ) {
/* ?? FIXME: we can't cope with this at the moment */
CITP_FDTABLE_UNLOCK();
Log_U(log("%s(%d, %d): target is reserved", __FUNCTION__, fromfd, tofd));
errno = EBUSY;
tofd = -1;
goto out;
}
if( fdip_cas_fail(p_tofdip, tofdip, fdip_busy) )
goto lock_tofdip_again;
CITP_FDTABLE_UNLOCK();
ci_assert(fdip_is_normal(tofdip) | fdip_is_passthru(tofdip) |
fdip_is_unknown(tofdip));
if( fdip_is_normal(tofdip) ) {
/* We're duping onto a user-level socket. */
citp_fdinfo* tofdi = fdip_to_fdi(tofdip);
if( tofdi->epoll_fd >= 0 ) {
citp_fdinfo* epoll_fdi = citp_epoll_fdi_from_member(tofdi, 0);
if( epoll_fdi ) {
if( epoll_fdi->protocol->type == CITP_EPOLL_FD )
citp_epoll_on_close(epoll_fdi, tofdi, 0);
citp_fdinfo_release_ref(epoll_fdi, 0);
}
}
ci_assert_equal(tofdi->on_ref_count_zero, FDI_ON_RCZ_NONE);
tofdi->on_ref_count_zero = FDI_ON_RCZ_DUP2;
tofdi->on_rcz.dup3_args.fd = fromfd;
tofdi->on_rcz.dup3_args.flags = flags;
citp_fdinfo_release_ref(tofdi, 0);
{
int i = 0;
/* We need to free this fdi. If someone is using it right now,
* we are in trouble. So, we spin for a while and interrupt the
* user. See bug 28123. */
while( tofdi->on_ref_count_zero != FDI_ON_RCZ_DONE ) {
if( ci_is_multithreaded() && i % 10000 == 9999 ) {
pthread_t pth = tofdi->thread_id;
if( pth != pthread_self() && pth != PTHREAD_NULL ) {
pthread_kill(pth, SIGONLOAD);
sleep(1);
}
}
ci_spinloop_pause();
i++;
}
ci_rmb();
}
if( tofdi->on_rcz.dup2_result < 0 ) {
errno = -tofdi->on_rcz.dup2_result;
/* Need to re-insert [tofdi] into the table. */
ci_assert_equal(oo_atomic_read(&tofdi->ref_count), 0);
oo_atomic_set(&tofdi->ref_count, 1);
CI_DEBUG(tofdi->on_ref_count_zero = FDI_ON_RCZ_NONE);
citp_fdtable_busy_clear(tofd, tofdip, 0);
tofd = -1;
}
else {
ci_assert(tofdi->on_rcz.dup2_result == tofd);
citp_fdinfo_get_ops(tofdi)->dtor(tofdi, 0);
citp_fdinfo_free(tofdi);
}
goto out;
}
ci_assert(fdip_is_passthru(tofdip) | fdip_is_unknown(tofdip));
{ /* We're dupping onto an O/S descriptor, or it may be closed. Create a
** dummy [citp_fdinfo], just so we can share code with the case above.
*/
citp_fdinfo fdi;
fdi.fd = tofd;
fdi.on_rcz.dup3_args.fd = fromfd;
fdi.on_rcz.dup3_args.flags = flags;
dup2_complete(&fdi, tofdip, 0);
if( fdi.on_rcz.dup2_result < 0 ) {
errno = -fdi.on_rcz.dup2_result;
citp_fdtable_busy_clear(tofd, tofdip, 0);
tofd = -1;
}
else
ci_assert(fdi.on_rcz.dup2_result == tofd);
}
out:
pthread_mutex_unlock(&citp_dup_lock);
return tofd;
}
static void dup2_complete(citp_fdinfo* prev_tofdi,
citp_fdinfo_p prev_tofdip, int fdt_locked)
{
volatile citp_fdinfo_p *p_fromfdip;
unsigned fromfd = prev_tofdi->on_rcz.dup3_args.fd;
unsigned tofd = prev_tofdi->fd;
citp_fdinfo_p fromfdip;
int rc;
#if CI_LIBC_HAS_dup3 || !defined(NDEBUG)
int flags = prev_tofdi->on_rcz.dup3_args.flags;
#endif
#ifndef NDEBUG
volatile citp_fdinfo_p* p_tofdip;
p_tofdip = &citp_fdtable.table[tofd].fdip;
ci_assert(fdip_is_busy(*p_tofdip));
#endif
citp_fdinfo* fromfdi;
p_fromfdip = &citp_fdtable.table[fromfd].fdip;
lock_fromfdip_again:
fromfdip = *p_fromfdip;
if( fdip_is_busy(fromfdip) )
fromfdip = citp_fdtable_busy_wait(fromfd, fdt_locked);
if( fdip_is_closing(fromfdip) | fdip_is_reserved(fromfdip) ) {
prev_tofdi->on_rcz.dup2_result = -EBADF;
ci_wmb();
prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE;
return;
}
#if CI_CFG_FD_CACHING
/* Need to check in case this sucker's cached */
if( fdip_is_unknown(fromfdip) ) {
if( !fdt_locked ) CITP_FDTABLE_LOCK();
fromfdi = citp_fdtable_probe_locked(fromfd, CI_FALSE, CI_FALSE);
if( !fdt_locked ) CITP_FDTABLE_UNLOCK();
if( fromfdi == &citp_the_closed_fd ) {
prev_tofdi->on_rcz.dup2_result = -EBADF;
ci_wmb();
prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE;
citp_fdinfo_release_ref(fromfdi, CI_TRUE);
return;
}
if( fromfdi )
citp_fdinfo_release_ref(fromfdi, CI_TRUE);
}
#endif
if( fdip_cas_fail(p_fromfdip, fromfdip, fdip_busy) )
goto lock_fromfdip_again;
oo_rwlock_lock_write(&citp_dup2_lock);
#if CI_LIBC_HAS_dup3
rc = ci_sys_dup3(fromfd, tofd, flags);
#else
ci_assert_equal(flags, 0);
rc = ci_sys_dup2(fromfd, tofd);
#endif
oo_rwlock_unlock_write(&citp_dup2_lock);
if( rc < 0 ) {
citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked);
prev_tofdi->on_rcz.dup2_result = -errno;
ci_wmb();
prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE;
return;
}
ci_assert(fdip_is_normal(fromfdip) | fdip_is_passthru(fromfdip) |
fdip_is_unknown(fromfdip));
if( fdip_is_normal(fromfdip) &&
(((fromfdi = fdip_to_fdi(fromfdip))->protocol->type) == CITP_EPOLL_FD) ) {
citp_fdinfo* newfdi = citp_fdinfo_get_ops(fromfdi)->dup(fromfdi);
if( newfdi ) {
citp_fdinfo_init(newfdi, fdip_to_fdi(fromfdip)->protocol);
citp_fdtable_insert(newfdi, tofd, fdt_locked);
}
else {
/* Out of memory. Can't probe epoll1 fd later on, so fail. */
citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked);
prev_tofdi->on_rcz.dup2_result = -ENOMEM;
ci_wmb();
prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE;
return;
}
}
else {
/* Mark newfd as unknown. When used, it'll get probed.
*
* We are not just being lazy here: Setting to unknown rather than
* installing a proper fdi (when oldfd is accelerated) is essential to
* vfork()+dup2()+exec() working properly. Reason is that child and
* parent share address space, so child is modifying the parent's
* fdtable. Setting an entry to unknown is safe.
*/
citp_fdtable_busy_clear(tofd, fdip_unknown, fdt_locked);
#if CI_CFG_FD_CACHING
/* Multiple refs to this now, don't allow it to be cached. */
if( fdip_is_normal(fromfdip) )
fdip_to_fdi(fromfdip)->can_cache = 0;
#endif
}
citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked);
prev_tofdi->on_rcz.dup2_result = tofd;
ci_wmb();
prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE;
}
/* run any pending timers */
void ci_ip_timer_poll(ci_netif *netif) {
ci_ip_timer_state* ipts = IPTIMER_STATE(netif);
ci_iptime_t* stime = &ipts->sched_ticks;
ci_ip_timer* ts;
ci_iptime_t rtime;
ci_ni_dllist_link* link;
int changed = 0;
/* The caller is expected to ensure that the current time is sufficiently
** up-to-date.
*/
rtime = ci_ip_time_now(netif);
/* check for sanity i.e. time always goes forwards */
ci_assert( TIME_GE(rtime, *stime) );
/* bug chasing Bug 2855 - check the temp list used is OK before we start */
ci_assert( ci_ni_dllist_is_valid(netif, &ipts->fire_list.l) );
ci_assert( ci_ni_dllist_is_empty(netif, &ipts->fire_list));
while( TIME_LT(*stime, rtime) ) {
DETAILED_CHECK_TIMERS(netif);
/* advance the schedulers view of time */
(*stime)++;
/* cascade through wheels if reached end of current wheel */
if(BUCKETNO(0, *stime) == 0) {
if(BUCKETNO(1, *stime) == 0) {
if(BUCKETNO(2, *stime) == 0) {
ci_ip_timer_cascadewheel(netif, 3, *stime);
}
ci_ip_timer_cascadewheel(netif, 2, *stime);
}
changed = ci_ip_timer_cascadewheel(netif, 1, *stime);
}
/* Bug 1828: We need to be creaful here ... because:
- ci_ip_timer_docallback can set/clear timers
- the timers being set/cleared may not necessarily be the ones firing
- however, they could be in this bucket
In summary, need to ensure the ni_dllist stays valid at all times so
safe to call. Slightly complicated by the case that its not possible to
hold indirected linked lists on the stack */
ci_assert( ci_ni_dllist_is_valid(netif, &ipts->fire_list.l));
ci_assert( ci_ni_dllist_is_empty(netif, &ipts->fire_list));
/* run timers in the current bucket */
ci_ni_dllist_rehome( netif,
&ipts->fire_list,
&ipts->warray[BUCKETNO(0, *stime)] );
DETAILED_CHECK_TIMERS(netif);
while( (link = ci_ni_dllist_try_pop(netif, &ipts->fire_list)) ) {
ts = LINK2TIMER(link);
ci_assert_equal(ts->time, *stime);
/* ensure time marked as NOT pending */
ci_ni_dllist_self_link(netif, &ts->link);
/* callback safe to set/clear this or other timers */
ci_ip_timer_docallback(netif, ts);
}
ci_assert( ci_ni_dllist_is_valid(netif, &ipts->fire_list.l) );
ci_assert( ci_ni_dllist_is_empty(netif, &ipts->fire_list));
DETAILED_CHECK_TIMERS(netif);
}
ci_assert( ci_ni_dllist_is_valid(netif, &ipts->fire_list.l) );
ci_assert( ci_ni_dllist_is_empty(netif, &ipts->fire_list));
/* What is our next timer?
* Let's update if our previous "closest" timer have already been
* handled, or if the previous estimation was "infinity". */
if( TIME_GE(ipts->sched_ticks, ipts->closest_timer) ||
(changed &&
ipts->closest_timer - ipts->sched_ticks > IPTIME_INFINITY_LOW) ) {
/* we peek into the first wheel only */
ci_iptime_t base = ipts->sched_ticks & WHEEL0_MASK;
ci_iptime_t b = ipts->sched_ticks - base;
for( b++ ; b < CI_IPTIME_BUCKETS; b++ ) {
if( !ci_ni_dllist_is_empty(netif, &ipts->warray[b]) ) {
ipts->closest_timer = base + b;
return;
}
}
/* We do not know the next timer. Set it to a sort of infinity. */
ipts->closest_timer = ipts->sched_ticks + IPTIME_INFINITY;
}
}
asmlinkage int efab_linux_sys_epoll_create1(int flags)
{
asmlinkage int (*sys_epoll_create_fn)(int);
int rc;
#ifdef __NR_epoll_create1
if (state.no_replace_epoll_create1)
{
sys_epoll_create_fn = (int (*)(int))THUNKPTR(state.no_replace_epoll_create1->original_entry64);
TRAMP_DEBUG("epoll_create1 via %p .. ", sys_epoll_create_fn);
rc = sys_epoll_create_fn(flags);
if (rc != -ENOSYS)
goto out;
}
#endif
if (!state.no_replace_epoll_create)
{
ci_log("Unexpected epoll_ctl() request before full init");
return -EFAULT;
}
sys_epoll_create_fn = (int (*)(int))THUNKPTR(state.no_replace_epoll_create->original_entry64);
TRAMP_DEBUG("epoll_create via %p .. ", sys_epoll_create_fn);
rc = sys_epoll_create_fn(1);
ci_assert_equal(flags & ~EPOLL_CLOEXEC, 0);
if (rc >= 0 && (flags & EPOLL_CLOEXEC))
{
struct files_struct *files = current->files;
struct fdtable *fdt;
spin_lock(&files->file_lock);
fdt = files_fdtable(files);
efx_set_close_on_exec(rc, fdt);
spin_unlock(&files->file_lock);
}
goto out;
out:
TRAMP_DEBUG(" ... = %d ", rc);
return rc;
}