void citp_waitable_wake_not_in_poll(ci_netif* ni, citp_waitable* sb, unsigned what) { ci_assert(what); ci_assert((what & ~(CI_SB_FLAG_WAKE_RX|CI_SB_FLAG_WAKE_TX)) == 0u); ci_assert(!ni->state->in_poll); ci_wmb(); if( what & CI_SB_FLAG_WAKE_RX ) ++sb->sleep_seq.rw.rx; if( what & CI_SB_FLAG_WAKE_TX ) ++sb->sleep_seq.rw.tx; ci_mb(); #ifdef __KERNEL__ /* Normally we put an object on a ready list in ci_netif_put_on_post_poll, * but in this case we don't go via there, so have to explicitly queue on * the ready list here. */ ci_ni_dllist_remove(ni, &sb->ready_link); ci_ni_dllist_put(ni, &ni->state->ready_lists[sb->ready_list_id], &sb->ready_link); if( what & sb->wake_request ) { sb->sb_flags |= what; citp_waitable_wakeup(ni, sb); } /* Wake the ready list too, if that's requested it. */ if( ni->state->ready_list_flags[sb->ready_list_id] & CI_NI_READY_LIST_FLAG_WAKE ) efab_tcp_helper_ready_list_wakeup(netif2tcp_helper_resource(ni), sb->ready_list_id); #else if( what & sb->wake_request ) { sb->sb_flags |= what; ci_netif_put_on_post_poll(ni, sb); ef_eplock_holder_set_flag(&ni->state->lock, CI_EPLOCK_NETIF_NEED_WAKE); } else { /* Normally we put an object on a ready list in ci_netif_put_on_post_poll, * but in this case we don't go via there, so have to explicitly queue on * the ready list here. */ ci_ni_dllist_remove(ni, &sb->ready_link); ci_ni_dllist_put(ni, &ni->state->ready_lists[sb->ready_list_id], &sb->ready_link); if( ni->state->ready_list_flags[sb->ready_list_id] & CI_NI_READY_LIST_FLAG_WAKE ) ef_eplock_holder_set_flag(&ni->state->lock, CI_EPLOCK_NETIF_NEED_WAKE); } #endif }
/*! Run any pending signal handlers ** \param our_info Thread-specific context for current thread */ void citp_signal_run_pending(citp_signal_info *our_info) { /* preserve errno across calls to this function, as it's often called at error time as a result of EXIT_LIB */ int old_errno = errno; int i; LOG_SIG(log("%s: start", __FUNCTION__)); ci_wmb(); ci_assert_equal(our_info->inside_lib, 0); ci_assert(our_info->aflags & OO_SIGNAL_FLAG_HAVE_PENDING); ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_HAVE_PENDING); for( i = 0; i < OO_SIGNAL_MAX_PENDING; i++ ) { siginfo_t saved_info; void *saved_context; int signum; if (our_info->signals[i].signum == 0) break; saved_context = our_info->signals[i].saved_context; if( our_info->signals[i].saved_context ) memcpy(&saved_info, &our_info->signals[i].saved_info, sizeof(saved_info)); signum = our_info->signals[i].signum; if( ci_cas32_fail(&our_info->signals[i].signum, signum, 0) ) break; if( citp_signal_run_app_handler( signum, saved_context == NULL ? NULL : &saved_info, saved_context) ) ci_atomic32_or(&our_info->aflags, OO_SIGNAL_FLAG_NEED_RESTART); else ci_atomic32_and(&our_info->aflags, ~OO_SIGNAL_FLAG_NEED_RESTART); } LOG_SIG(log("%s: end", __FUNCTION__)); errno = old_errno; }
static void dup2_complete(citp_fdinfo* prev_tofdi, citp_fdinfo_p prev_tofdip, int fdt_locked) { volatile citp_fdinfo_p *p_fromfdip; unsigned fromfd = prev_tofdi->on_rcz.dup3_args.fd; unsigned tofd = prev_tofdi->fd; citp_fdinfo_p fromfdip; int rc; #if CI_LIBC_HAS_dup3 || !defined(NDEBUG) int flags = prev_tofdi->on_rcz.dup3_args.flags; #endif #ifndef NDEBUG volatile citp_fdinfo_p* p_tofdip; p_tofdip = &citp_fdtable.table[tofd].fdip; ci_assert(fdip_is_busy(*p_tofdip)); #endif citp_fdinfo* fromfdi; p_fromfdip = &citp_fdtable.table[fromfd].fdip; lock_fromfdip_again: fromfdip = *p_fromfdip; if( fdip_is_busy(fromfdip) ) fromfdip = citp_fdtable_busy_wait(fromfd, fdt_locked); if( fdip_is_closing(fromfdip) | fdip_is_reserved(fromfdip) ) { prev_tofdi->on_rcz.dup2_result = -EBADF; ci_wmb(); prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE; return; } #if CI_CFG_FD_CACHING /* Need to check in case this sucker's cached */ if( fdip_is_unknown(fromfdip) ) { if( !fdt_locked ) CITP_FDTABLE_LOCK(); fromfdi = citp_fdtable_probe_locked(fromfd, CI_FALSE, CI_FALSE); if( !fdt_locked ) CITP_FDTABLE_UNLOCK(); if( fromfdi == &citp_the_closed_fd ) { prev_tofdi->on_rcz.dup2_result = -EBADF; ci_wmb(); prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE; citp_fdinfo_release_ref(fromfdi, CI_TRUE); return; } if( fromfdi ) citp_fdinfo_release_ref(fromfdi, CI_TRUE); } #endif if( fdip_cas_fail(p_fromfdip, fromfdip, fdip_busy) ) goto lock_fromfdip_again; oo_rwlock_lock_write(&citp_dup2_lock); #if CI_LIBC_HAS_dup3 rc = ci_sys_dup3(fromfd, tofd, flags); #else ci_assert_equal(flags, 0); rc = ci_sys_dup2(fromfd, tofd); #endif oo_rwlock_unlock_write(&citp_dup2_lock); if( rc < 0 ) { citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked); prev_tofdi->on_rcz.dup2_result = -errno; ci_wmb(); prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE; return; } ci_assert(fdip_is_normal(fromfdip) | fdip_is_passthru(fromfdip) | fdip_is_unknown(fromfdip)); if( fdip_is_normal(fromfdip) && (((fromfdi = fdip_to_fdi(fromfdip))->protocol->type) == CITP_EPOLL_FD) ) { citp_fdinfo* newfdi = citp_fdinfo_get_ops(fromfdi)->dup(fromfdi); if( newfdi ) { citp_fdinfo_init(newfdi, fdip_to_fdi(fromfdip)->protocol); citp_fdtable_insert(newfdi, tofd, fdt_locked); } else { /* Out of memory. Can't probe epoll1 fd later on, so fail. */ citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked); prev_tofdi->on_rcz.dup2_result = -ENOMEM; ci_wmb(); prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE; return; } } else { /* Mark newfd as unknown. When used, it'll get probed. * * We are not just being lazy here: Setting to unknown rather than * installing a proper fdi (when oldfd is accelerated) is essential to * vfork()+dup2()+exec() working properly. Reason is that child and * parent share address space, so child is modifying the parent's * fdtable. Setting an entry to unknown is safe. */ citp_fdtable_busy_clear(tofd, fdip_unknown, fdt_locked); #if CI_CFG_FD_CACHING /* Multiple refs to this now, don't allow it to be cached. */ if( fdip_is_normal(fromfdip) ) fdip_to_fdi(fromfdip)->can_cache = 0; #endif } citp_fdtable_busy_clear(fromfd, fromfdip, fdt_locked); prev_tofdi->on_rcz.dup2_result = tofd; ci_wmb(); prev_tofdi->on_ref_count_zero = FDI_ON_RCZ_DONE; }
/* Move priv file to the alien_ni stack. * Should be called with the locked priv stack and socket; * the function returns with this stack being unlocked. * If rc=0, it returns with alien_ni stack locked; * otherwise, both stacks are unlocked. * Socket is always unlocked on return. */ int efab_file_move_to_alien_stack(ci_private_t *priv, ci_netif *alien_ni) { tcp_helper_resource_t *old_thr = priv->thr; tcp_helper_resource_t *new_thr = netif2tcp_helper_resource(alien_ni); ci_sock_cmn *old_s = SP_TO_SOCK(&old_thr->netif, priv->sock_id); ci_sock_cmn *new_s; ci_sock_cmn *mid_s; tcp_helper_endpoint_t *old_ep, *new_ep; int rc, i; int pollwait_register = 0; #if CI_CFG_FD_CACHING oo_p sp; #endif OO_DEBUG_TCPH(ci_log("%s: move %d:%d to %d", __func__, old_thr->id, priv->sock_id, new_thr->id)); /* Poll the old stack - deliver all data to our socket */ ci_netif_poll(&old_thr->netif); /* Endpoints in epoll list should not be moved, because waitq is already * in the epoll internal structures (bug 41152). */ if( !list_empty(&priv->_filp->f_ep_links) ) { rc = -EBUSY; goto fail1; } if( !efab_file_move_supported(&old_thr->netif, old_s) ) { rc = -EINVAL; goto fail1; } /* Lock the second stack */ i = 0; while( ! ci_netif_trylock(alien_ni) ) { ci_netif_unlock(&old_thr->netif); if( i++ >= 1000 ) { rc = -EBUSY; goto fail1_ni_unlocked; } rc = ci_netif_lock(&old_thr->netif); if( rc != 0 ) goto fail1_ni_unlocked; } /* Allocate a new socket in the alien_ni stack */ rc = -ENOMEM; if( old_s->b.state == CI_TCP_STATE_UDP ) { ci_udp_state *new_us = ci_udp_get_state_buf(alien_ni); if( new_us == NULL ) goto fail2; new_s = &new_us->s; } else { ci_tcp_state *new_ts = ci_tcp_get_state_buf(alien_ni); if( new_ts == NULL ) goto fail2; new_s = &new_ts->s; } /* Allocate an intermediate "socket" outside of everything */ mid_s = ci_alloc(CI_MAX(sizeof(ci_tcp_state), sizeof(ci_udp_state))); if( mid_s == NULL ) goto fail3; OO_DEBUG_TCPH(ci_log("%s: move %d:%d to %d:%d", __func__, old_thr->id, priv->sock_id, new_thr->id, new_s->b.bufid)); /* Copy TCP/UDP state */ memcpy(mid_s, old_s, CI_MAX(sizeof(ci_tcp_state), sizeof(ci_udp_state))); /* do not copy old_s->b.bufid * and other fields in stack adress space */ mid_s->b.sb_aflags |= CI_SB_AFLAG_ORPHAN; mid_s->b.bufid = new_s->b.bufid; mid_s->b.post_poll_link = new_s->b.post_poll_link; mid_s->b.ready_link = new_s->b.ready_link; mid_s->reap_link = new_s->reap_link; if( old_s->b.state & CI_TCP_STATE_TCP ) { ci_tcp_state *new_ts = SOCK_TO_TCP(new_s); ci_tcp_state *mid_ts = SOCK_TO_TCP(mid_s); mid_ts->timeout_q_link = new_ts->timeout_q_link; mid_ts->tx_ready_link = new_ts->tx_ready_link; mid_ts->rto_tid = new_ts->rto_tid; mid_ts->delack_tid = new_ts->delack_tid; mid_ts->zwin_tid = new_ts->zwin_tid; mid_ts->kalive_tid = new_ts->kalive_tid; mid_ts->cork_tid = new_ts->cork_tid; ci_ip_queue_init(&mid_ts->recv1); ci_ip_queue_init(&mid_ts->recv2); ci_ip_queue_init(&mid_ts->send); ci_ip_queue_init(&mid_ts->retrans); mid_ts->send_prequeue = OO_PP_ID_NULL; new_ts->retrans_ptr = OO_PP_NULL; mid_ts->tmpl_head = OO_PP_NULL; oo_atomic_set(&mid_ts->send_prequeue_in, 0); *new_ts = *mid_ts; ci_pmtu_state_init(alien_ni, &new_ts->s, &new_ts->pmtus, CI_IP_TIMER_PMTU_DISCOVER); #if CI_CFG_FD_CACHING sp = TS_OFF(alien_ni, new_ts); OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_tcp_state, epcache_link)); ci_ni_dllist_link_init(alien_ni, &new_ts->epcache_link, sp, "epch"); ci_ni_dllist_self_link(alien_ni, &new_ts->epcache_link); sp = TS_OFF(alien_ni, new_ts); OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_tcp_state, epcache_fd_link)); ci_ni_dllist_link_init(alien_ni, &new_ts->epcache_fd_link, sp, "ecfd"); ci_ni_dllist_self_link(alien_ni, &new_ts->epcache_fd_link); #endif /* free temporary mid_ts storage */ CI_FREE_OBJ(mid_ts); } else { ci_udp_state *mid_us = SOCK_TO_UDP(mid_s); *SOCK_TO_UDP(new_s) = *mid_us; CI_FREE_OBJ(mid_us); } /* Move the filter */ old_ep = ci_trs_ep_get(old_thr, priv->sock_id); new_ep = ci_trs_ep_get(new_thr, new_s->b.bufid); rc = tcp_helper_endpoint_move_filters_pre(old_ep, new_ep); if( rc != 0 ) { rc = -EINVAL; goto fail3; } /* Allocate a new file for the new endpoint */ rc = onload_alloc_file(new_thr, new_s->b.bufid, priv->_filp->f_flags, priv->fd_type, &old_ep->alien_ref); if( rc != 0 ) goto fail4; ci_assert(old_ep->alien_ref); /* Copy F_SETOWN_EX, F_SETSIG to the new file */ #ifdef F_SETOWN_EX rcu_read_lock(); __f_setown(old_ep->alien_ref->_filp, priv->_filp->f_owner.pid, priv->_filp->f_owner.pid_type, 1); rcu_read_unlock(); #endif old_ep->alien_ref->_filp->f_owner.signum = priv->_filp->f_owner.signum; old_ep->alien_ref->_filp->f_flags |= priv->_filp->f_flags & O_NONBLOCK; /* Move os_socket from one ep to another */ if( tcp_helper_endpoint_set_aflags(new_ep, OO_THR_EP_AFLAG_ATTACHED) & OO_THR_EP_AFLAG_ATTACHED ) { fput(old_ep->alien_ref->_filp); rc = -EBUSY; goto fail2; /* state & filters are cleared by fput() */ } /********* Point of no return **********/ ci_wmb(); priv->fd_type = CI_PRIV_TYPE_ALIEN_EP; priv->_filp->f_op = &linux_tcp_helper_fops_alien; ci_wmb(); oo_file_moved(priv); /* Read all already-arrived packets after the filters move but before * copying of the receive queue. */ ci_netif_poll(&old_thr->netif); tcp_helper_endpoint_move_filters_post(old_ep, new_ep); ci_assert( efab_file_move_supported(&old_thr->netif, old_s)); /* There's a gap between un-registering the old ep, and registering the * the new. However, the notifications shouldn't be in use for sockets * that are in a state that can be moved, so this shouldn't be a problem. */ if( old_ep->os_sock_pt.whead ) { pollwait_register = 1; efab_tcp_helper_os_pollwait_unregister(old_ep); } ci_assert_equal(new_ep->os_socket, NULL); new_ep->os_socket = oo_file_ref_xchg(&old_ep->os_socket, NULL); ci_assert_equal(old_ep->os_socket, NULL); if( pollwait_register ) efab_tcp_helper_os_pollwait_register(new_ep); ci_bit_clear(&new_s->b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT); if( new_s->b.state == CI_TCP_ESTABLISHED ) CI_TCP_STATS_INC_CURR_ESTAB(alien_ni); /* Copy recv queue */ if( new_s->b.state & CI_TCP_STATE_TCP ) { ci_tcp_state *new_ts = SOCK_TO_TCP(new_s); ci_tcp_state *old_ts = SOCK_TO_TCP(old_s); int i; /* Stop timers */ ci_ip_timer_clear(&old_thr->netif, &old_ts->kalive_tid); ci_ip_timer_clear(&old_thr->netif, &old_ts->delack_tid); efab_ip_queue_copy(alien_ni, &new_ts->recv1, &old_thr->netif, &old_ts->recv1); efab_ip_queue_copy(alien_ni, &new_ts->recv2, &old_thr->netif, &old_ts->recv2); new_ts->recv1_extract = new_ts->recv1.head; /* Drop reorder buffer */ ci_ip_queue_init(&new_ts->rob); new_ts->dsack_block = OO_PP_INVALID; new_ts->dsack_start = new_ts->dsack_end = 0; for( i = 0; i <= CI_TCP_SACK_MAX_BLOCKS; i++ ) new_ts->last_sack[i] = OO_PP_NULL; } else { /* There should not be any recv q, but drop it to be sure */ ci_udp_recv_q_init(&SOCK_TO_UDP(new_s)->recv_q); } /* Old stack can be unlocked */ old_s->b.sb_flags |= CI_SB_FLAG_MOVED; ci_netif_unlock(&old_thr->netif); ci_assert( efab_file_move_supported(alien_ni, new_s) ); /* Move done: poll for any new data. */ ci_netif_poll(alien_ni); if( new_s->b.state & CI_TCP_STATE_TCP ) { ci_tcp_state *new_ts = SOCK_TO_TCP(new_s); /* Timers setup: delack, keepalive */ if( (new_ts->acks_pending & CI_TCP_ACKS_PENDING_MASK) > 0) ci_tcp_timeout_delack(alien_ni, new_ts); ci_tcp_kalive_reset(alien_ni, new_ts); } /* Old ep: we are done. */ ci_bit_set(&old_s->b.sb_aflags, CI_SB_AFLAG_MOVED_AWAY_BIT); old_s->b.moved_to_stack_id = alien_ni->state->stack_id; old_s->b.moved_to_sock_id = new_s->b.bufid; if( ! list_empty(&priv->_filp->f_ep_links) ) ci_bit_set(&old_s->b.sb_aflags, CI_SB_AFLAG_MOVED_AWAY_IN_EPOLL_BIT); ci_sock_unlock(&old_thr->netif, &old_s->b); ci_sock_unlock(alien_ni, &new_s->b); ci_assert(ci_netif_is_locked(alien_ni)); OO_DEBUG_TCPH(ci_log("%s: -> [%d:%d] %s", __func__, new_thr->id, new_s->b.bufid, ci_tcp_state_str(new_s->b.state))); return 0; fail4: /* We clear the filters from the new ep. * For now, we do not need to re-insert old filters because hw filters * are alredy here (in case of accepted socket) or not needed. * We have not removed old sw filters yet. */ tcp_helper_endpoint_move_filters_undo(old_ep, new_ep); fail3: if( new_s->b.state & CI_TCP_STATE_TCP ) ci_tcp_state_free(alien_ni, SOCK_TO_TCP(new_s)); else ci_udp_state_free(alien_ni, SOCK_TO_UDP(new_s)); fail2: ci_netif_unlock(alien_ni); fail1: ci_netif_unlock(&old_thr->netif); fail1_ni_unlocked: ci_sock_unlock(&old_thr->netif, &old_s->b); OO_DEBUG_TCPH(ci_log("%s: rc=%d", __func__, rc)); return rc; }