static int citp_udp_sendmmsg(citp_fdinfo* fdinfo, struct mmsghdr* mmsg,
unsigned vlen, int flags)
{
citp_sock_fdi* epi = fdi_to_sock_fdi(fdinfo);
ci_udp_iomsg_args a;
int i, rc;
Log_V(log(LPF "sendmmsg(%d, msg, %u, %#x)", fdinfo->fd, vlen,
(unsigned) flags));
if( vlen == 0 )
return 0;
a.ep = &epi->sock;
a.fd = fdinfo->fd;
a.ni = epi->sock.netif;
a.us = SOCK_TO_UDP(epi->sock.s);
i = 0;
do {
rc = ci_udp_sendmsg(&a, &mmsg[i].msg_hdr, flags);
if(CI_LIKELY( rc >= 0 ) )
mmsg[i].msg_len = rc;
++i;
} while( rc >= 0 && i < vlen );
return (rc>=0) ? i : rc;
}
int ci_udp_shutdown(citp_socket* ep, ci_fd_t fd, int how)
{
ci_fd_t os_sock;
int rc;
CHECK_UEP(ep);
LOG_UV(log(LPF "shutdown("SF_FMT", %d)", SF_PRI_ARGS(ep,fd), how));
os_sock = ci_get_os_sock_fd(fd);
if( CI_IS_VALID_SOCKET( os_sock ) ) {
rc = ci_sys_shutdown(os_sock, how);
ci_rel_os_sock_fd( os_sock );
if( rc < 0 )
return CI_SOCKET_ERROR;
}
rc = __ci_udp_shutdown(ep->netif, SOCK_TO_UDP(ep->s), how);
if( rc < 0 ) {
CI_SET_ERROR(rc, -rc);
return rc;
}
return 0;
}
static void ci_udp_clr_filters(citp_socket* ep)
{
ci_udp_state* us = SOCK_TO_UDP(ep->s);
if( UDP_GET_FLAG(us, CI_UDPF_FILTERED) ) {
ci_tcp_ep_clear_filters(ep->netif, S_SP(us), 0);
UDP_CLR_FLAG(us, CI_UDPF_FILTERED);
}
}
int ci_udp_ioctl(citp_socket *ep, ci_fd_t fd, int request, void* arg)
{
ci_netif* ni = ep->netif;
ci_udp_state* us = SOCK_TO_UDP(ep->s);
int rc;
ci_netif_lock(ni);
rc = ci_udp_ioctl_locked(ni, us, fd, request, arg);
ci_netif_unlock(ni);
return rc;
}
void ci_udp_set_laddr(citp_socket* ep, unsigned laddr_be32, int lport_be16)
{
ci_udp_state* us = SOCK_TO_UDP(ep->s);
udp_laddr_be32(us) = laddr_be32;
udp_lport_be16(us) = (ci_uint16) lport_be16;
if( CI_IP_IS_MULTICAST(laddr_be32) )
us->s.cp.ip_laddr_be32 = 0;
else
us->s.cp.ip_laddr_be32 = laddr_be32;
us->s.cp.lport_be16 = lport_be16;
}
static int citp_udp_recv(citp_fdinfo* fdinfo, struct msghdr* msg, int flags)
{
citp_sock_fdi* epi = fdi_to_sock_fdi(fdinfo);
ci_udp_iomsg_args a;
Log_V(log(LPF "recv(%d, msg, %#x)", fdinfo->fd, (unsigned) flags));
a.fd = fdinfo->fd;
a.ep = &epi->sock;
a.ni = epi->sock.netif;
a.us = SOCK_TO_UDP(epi->sock.s);
return ci_udp_recvmsg( &a, msg, flags);
}
/* Conclude the EP's binding. This function is abstracted from the
* main bind code to allow implicit binds that occur when sendto() is
* called on an OS socket. [lport] and CI_SIN(addr)->sin_port do not
* have to be the same value. */
static int ci_udp_bind_conclude(citp_socket* ep, const struct sockaddr* addr,
ci_uint16 lport )
{
ci_udp_state* us;
ci_uint32 addr_be32;
int rc;
CHECK_UEP(ep);
ci_assert(addr != NULL);
if( ci_udp_should_handover(ep, addr, lport) )
goto handover;
addr_be32 = ci_get_ip4_addr(ep->s->domain, addr);
ci_udp_set_laddr(ep, addr_be32, lport);
us = SOCK_TO_UDP(ep->s);
if( addr_be32 != 0 )
us->s.cp.sock_cp_flags |= OO_SCP_LADDR_BOUND;
/* reset any rx/tx that have taken place already */
UDP_CLR_FLAG(us, CI_UDPF_EF_SEND);
#ifdef ONLOAD_OFE
if( ep->netif->ofe != NULL )
us->s.ofe_code_start = ofe_socktbl_find(
ep->netif->ofe, OFE_SOCKTYPE_UDP,
udp_laddr_be32(us), udp_raddr_be32(us),
udp_lport_be16(us), udp_rport_be16(us));
#endif
/* OS source addrs have already been handed-over, so this must be one of
* our src addresses.
*/
rc = ci_udp_set_filters( ep, us);
ci_assert( !UDP_GET_FLAG(us, CI_UDPF_EF_BIND) );
/*! \todo FIXME isn't the port the thing to be testing here? */
if( udp_laddr_be32(us) != INADDR_ANY_BE32 )
UDP_SET_FLAG(us, CI_UDPF_EF_BIND);
CI_UDPSTATE_SHOW_EP( ep );
if( rc == CI_SOCKET_ERROR && CITP_OPTS.no_fail) {
CITP_STATS_NETIF(++ep->netif->state->stats.udp_bind_no_filter);
goto handover;
}
return rc;
handover:
LOG_UV(log("%s: "SK_FMT" HANDOVER", __FUNCTION__, SK_PRI_ARGS(ep)));
return CI_SOCKET_HANDOVER;
}
void ci_udp_ep_assert_valid(citp_socket* ep, const char* file, int line)
{
ci_netif* netif;
ci_udp_state* us;
ci_assert(ep);
netif = ep->netif;
ci_netif_assert_valid(netif, file, line);
_ci_assert_equal(ep->s->b.state, CI_TCP_STATE_UDP, file, line);
us = SOCK_TO_UDP(ep->s);
ci_udp_state_assert_valid(netif, us, file, line);
}
void ci_udp_set_laddr(citp_socket* ep, ci_addr_t addr, int lport_be16)
{
ci_udp_state* us = SOCK_TO_UDP(ep->s);
ci_sock_set_laddr(&us->s, addr, lport_be16);
#if CI_CFG_IPV6
if( ipcache_is_ipv6(us->s.pkt) )
return;
#endif
if( CI_IP_IS_MULTICAST(addr.ip4) )
us->s.cp.ip_laddr_be32 = 0;
else
us->s.cp.ip_laddr_be32 = addr.ip4;
us->s.cp.lport_be16 = lport_be16;
}
static int citp_udp_recvmmsg(citp_fdinfo* fdinfo, struct mmsghdr* msg,
unsigned vlen, int flags,
const struct timespec *timeout)
{
citp_sock_fdi* epi = fdi_to_sock_fdi(fdinfo);
ci_udp_iomsg_args a;
Log_V(log(LPF "recvmmsg(%d, msg, %u, %#x)", fdinfo->fd, vlen,
(unsigned) flags));
a.fd = fdinfo->fd;
a.ep = &epi->sock;
a.ni = epi->sock.netif;
a.us = SOCK_TO_UDP(epi->sock.s);
return ci_udp_recvmmsg(&a, msg, vlen, flags, timeout);
}
/* Returns true if move of this endpoint is supported */
static int efab_file_move_supported(ci_netif *ni, ci_sock_cmn *s)
{
/* We do not copy TX timestamping queue yet. */
if( s->timestamping_flags != 0 )
return false;
/* UDP: */
if( s->b.state == CI_TCP_STATE_UDP )
return efab_file_move_supported_udp(ni, SOCK_TO_UDP(s));
/* TCP or UDP only */
if( ! (s->b.state & CI_TCP_STATE_TCP ) )
return false;
/* No listening sockets */
if( s->b.state == CI_TCP_LISTEN )
return false;
return efab_file_move_supported_tcp(ni, SOCK_TO_TCP(s));
}
/*! \todo we can simplify this a lot by letting the kernel have it! */
int ci_udp_getpeername(citp_socket*ep, struct sockaddr* name, socklen_t* namelen)
{
ci_udp_state* us;
CHECK_UEP(ep);
us = SOCK_TO_UDP(ep->s);
/*
* At first, it's necessary to check whether socket is connected or
* not, since we can return ENOTCONN even if name and/or namelen are
* not valid.
*/
if( udp_raddr_be32(us) == 0 ) {
RET_WITH_ERRNO(ENOTCONN);
} else if( name == NULL || namelen == NULL ) {
RET_WITH_ERRNO(EFAULT);
} else {
ci_addr_to_user(name, namelen, ep->s->domain,
udp_rport_be16(us), udp_raddr_be32(us));
return 0;
}
}
static int citp_udp_send(citp_fdinfo* fdinfo, const struct msghdr * msg,
int flags)
{
citp_sock_fdi *epi = fdi_to_sock_fdi(fdinfo);
ci_udp_iomsg_args a;
int rc;
ci_assert(msg != NULL);
a.ep = &epi->sock;
a.fd = fdinfo->fd;
a.ni = epi->sock.netif;
a.us = SOCK_TO_UDP(epi->sock.s);
/* NB. msg_name[len] validated in ci_udp_sendmsg(). */
if(CI_LIKELY( msg->msg_iov != NULL || msg->msg_iovlen == 0 )) {
rc = ci_udp_sendmsg( &a, msg, flags);
}
else {
rc = -1;
errno = EFAULT;
}
return rc;
}
/* Complete a UDP U/L connect. The sys connect() call must have been made
* (and succeeded) before calling this function. So if anything goes wrong
* in here, then it can be consider an internal error or failing of onload.
*/
int ci_udp_connect_conclude(citp_socket* ep, ci_fd_t fd,
const struct sockaddr* serv_addr,
socklen_t addrlen, ci_fd_t os_sock)
{
const struct sockaddr_in* serv_sin = (const struct sockaddr_in*) serv_addr;
ci_uint32 dst_be32;
ci_udp_state* us = SOCK_TO_UDP(ep->s);
int onloadable;
int rc = 0;
CHECK_UEP(ep);
UDP_CLR_FLAG(us, CI_UDPF_EF_SEND);
us->s.rx_errno = 0;
us->s.tx_errno = 0;
if( IS_DISCONNECTING(serv_sin) ) {
rc = ci_udp_disconnect(ep, us, os_sock);
goto out;
}
#if CI_CFG_FAKE_IPV6
if( us->s.domain == PF_INET6 && !ci_tcp_ipv6_is_ipv4(serv_addr) ) {
LOG_UC(log(FNT_FMT "HANDOVER not IPv4", FNT_PRI_ARGS(ep->netif, us)));
goto handover;
}
#endif
dst_be32 = ci_get_ip4_addr(serv_sin->sin_family, serv_addr);
if( (rc = ci_udp_sys_getsockname(os_sock, ep)) != 0 ) {
LOG_E(log(FNT_FMT "ERROR: (%s:%d) sys_getsockname failed (%d)",
FNT_PRI_ARGS(ep->netif, us), ip_addr_str(dst_be32),
CI_BSWAP_BE16(serv_sin->sin_port), errno));
goto out;
}
us->s.cp.sock_cp_flags |= OO_SCP_CONNECTED;
ci_udp_set_raddr(us, dst_be32, serv_sin->sin_port);
cicp_user_retrieve(ep->netif, &us->s.pkt, &us->s.cp);
switch( us->s.pkt.status ) {
case retrrc_success:
case retrrc_nomac:
onloadable = 1;
break;
default:
onloadable = 0;
if( NI_OPTS(ep->netif).udp_connect_handover ) {
LOG_UC(log(FNT_FMT "HANDOVER %s:%d", FNT_PRI_ARGS(ep->netif, us),
ip_addr_str(dst_be32), CI_BSWAP_BE16(serv_sin->sin_port)));
goto handover;
}
break;
}
if( dst_be32 == INADDR_ANY_BE32 || serv_sin->sin_port == 0 ) {
LOG_UC(log(FNT_FMT "%s:%d - route via OS socket",
FNT_PRI_ARGS(ep->netif, us), ip_addr_str(dst_be32),
CI_BSWAP_BE16(serv_sin->sin_port)));
ci_udp_clr_filters(ep);
return 0;
}
if( CI_IP_IS_LOOPBACK(dst_be32) ) {
/* After connecting via loopback it is not possible to connect anywhere
* else.
*/
LOG_UC(log(FNT_FMT "HANDOVER %s:%d", FNT_PRI_ARGS(ep->netif, us),
ip_addr_str(dst_be32), CI_BSWAP_BE16(serv_sin->sin_port)));
goto handover;
}
if( onloadable ) {
#ifdef ONLOAD_OFE
if( ep->netif->ofe != NULL )
us->s.ofe_code_start = ofe_socktbl_find(
ep->netif->ofe, OFE_SOCKTYPE_UDP,
udp_laddr_be32(us), udp_raddr_be32(us),
udp_lport_be16(us), udp_rport_be16(us));
#endif
if( (rc = ci_udp_set_filters(ep, us)) != 0 ) {
/* Failed to set filters. Most likely we've run out of h/w filters.
* Handover to O/S to avoid breaking the app.
*
* TODO: Actually we probably won't break the app if we don't
* handover, as packets will still get delivered via the kernel
* stack. Might be worth having a runtime option to choose whether
* or not to handover in such cases.
*/
LOG_U(log(FNT_FMT "ERROR: (%s:%d) ci_udp_set_filters failed (%d)",
FNT_PRI_ARGS(ep->netif, us), ip_addr_str(dst_be32),
CI_BSWAP_BE16(serv_sin->sin_port), rc));
CITP_STATS_NETIF(++ep->netif->state->stats.udp_connect_no_filter);
goto out;
}
}
else {
ci_udp_clr_filters(ep);
}
LOG_UC(log(LPF "connect: "SF_FMT" %sCONNECTED L:%s:%u R:%s:%u (err:%d)",
SF_PRI_ARGS(ep,fd), udp_raddr_be32(us) ? "" : "DIS",
ip_addr_str(udp_laddr_be32(us)),
(unsigned) CI_BSWAP_BE16(udp_lport_be16(us)),
ip_addr_str(udp_raddr_be32(us)),
(unsigned) CI_BSWAP_BE16(udp_rport_be16(us)), errno));
return 0;
out:
if( rc < 0 && CITP_OPTS.no_fail )
goto handover;
return rc;
handover:
ci_udp_clr_filters(ep);
return CI_SOCKET_HANDOVER;
}
/* Move priv file to the alien_ni stack.
* Should be called with the locked priv stack and socket;
* the function returns with this stack being unlocked.
* If rc=0, it returns with alien_ni stack locked;
* otherwise, both stacks are unlocked.
* Socket is always unlocked on return. */
int efab_file_move_to_alien_stack(ci_private_t *priv, ci_netif *alien_ni)
{
tcp_helper_resource_t *old_thr = priv->thr;
tcp_helper_resource_t *new_thr = netif2tcp_helper_resource(alien_ni);
ci_sock_cmn *old_s = SP_TO_SOCK(&old_thr->netif, priv->sock_id);
ci_sock_cmn *new_s;
ci_sock_cmn *mid_s;
tcp_helper_endpoint_t *old_ep, *new_ep;
int rc, i;
int pollwait_register = 0;
#if CI_CFG_FD_CACHING
oo_p sp;
#endif
OO_DEBUG_TCPH(ci_log("%s: move %d:%d to %d", __func__,
old_thr->id, priv->sock_id, new_thr->id));
/* Poll the old stack - deliver all data to our socket */
ci_netif_poll(&old_thr->netif);
/* Endpoints in epoll list should not be moved, because waitq is already
* in the epoll internal structures (bug 41152). */
if( !list_empty(&priv->_filp->f_ep_links) ) {
rc = -EBUSY;
goto fail1;
}
if( !efab_file_move_supported(&old_thr->netif, old_s) ) {
rc = -EINVAL;
goto fail1;
}
/* Lock the second stack */
i = 0;
while( ! ci_netif_trylock(alien_ni) ) {
ci_netif_unlock(&old_thr->netif);
if( i++ >= 1000 ) {
rc = -EBUSY;
goto fail1_ni_unlocked;
}
rc = ci_netif_lock(&old_thr->netif);
if( rc != 0 )
goto fail1_ni_unlocked;
}
/* Allocate a new socket in the alien_ni stack */
rc = -ENOMEM;
if( old_s->b.state == CI_TCP_STATE_UDP ) {
ci_udp_state *new_us = ci_udp_get_state_buf(alien_ni);
if( new_us == NULL )
goto fail2;
new_s = &new_us->s;
}
else {
ci_tcp_state *new_ts = ci_tcp_get_state_buf(alien_ni);
if( new_ts == NULL )
goto fail2;
new_s = &new_ts->s;
}
/* Allocate an intermediate "socket" outside of everything */
mid_s = ci_alloc(CI_MAX(sizeof(ci_tcp_state), sizeof(ci_udp_state)));
if( mid_s == NULL )
goto fail3;
OO_DEBUG_TCPH(ci_log("%s: move %d:%d to %d:%d", __func__,
old_thr->id, priv->sock_id,
new_thr->id, new_s->b.bufid));
/* Copy TCP/UDP state */
memcpy(mid_s, old_s, CI_MAX(sizeof(ci_tcp_state), sizeof(ci_udp_state)));
/* do not copy old_s->b.bufid
* and other fields in stack adress space */
mid_s->b.sb_aflags |= CI_SB_AFLAG_ORPHAN;
mid_s->b.bufid = new_s->b.bufid;
mid_s->b.post_poll_link = new_s->b.post_poll_link;
mid_s->b.ready_link = new_s->b.ready_link;
mid_s->reap_link = new_s->reap_link;
if( old_s->b.state & CI_TCP_STATE_TCP ) {
ci_tcp_state *new_ts = SOCK_TO_TCP(new_s);
ci_tcp_state *mid_ts = SOCK_TO_TCP(mid_s);
mid_ts->timeout_q_link = new_ts->timeout_q_link;
mid_ts->tx_ready_link = new_ts->tx_ready_link;
mid_ts->rto_tid = new_ts->rto_tid;
mid_ts->delack_tid = new_ts->delack_tid;
mid_ts->zwin_tid = new_ts->zwin_tid;
mid_ts->kalive_tid = new_ts->kalive_tid;
mid_ts->cork_tid = new_ts->cork_tid;
ci_ip_queue_init(&mid_ts->recv1);
ci_ip_queue_init(&mid_ts->recv2);
ci_ip_queue_init(&mid_ts->send);
ci_ip_queue_init(&mid_ts->retrans);
mid_ts->send_prequeue = OO_PP_ID_NULL;
new_ts->retrans_ptr = OO_PP_NULL;
mid_ts->tmpl_head = OO_PP_NULL;
oo_atomic_set(&mid_ts->send_prequeue_in, 0);
*new_ts = *mid_ts;
ci_pmtu_state_init(alien_ni, &new_ts->s, &new_ts->pmtus,
CI_IP_TIMER_PMTU_DISCOVER);
#if CI_CFG_FD_CACHING
sp = TS_OFF(alien_ni, new_ts);
OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_tcp_state, epcache_link));
ci_ni_dllist_link_init(alien_ni, &new_ts->epcache_link, sp, "epch");
ci_ni_dllist_self_link(alien_ni, &new_ts->epcache_link);
sp = TS_OFF(alien_ni, new_ts);
OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_tcp_state, epcache_fd_link));
ci_ni_dllist_link_init(alien_ni, &new_ts->epcache_fd_link, sp, "ecfd");
ci_ni_dllist_self_link(alien_ni, &new_ts->epcache_fd_link);
#endif
/* free temporary mid_ts storage */
CI_FREE_OBJ(mid_ts);
}
else {
ci_udp_state *mid_us = SOCK_TO_UDP(mid_s);
*SOCK_TO_UDP(new_s) = *mid_us;
CI_FREE_OBJ(mid_us);
}
/* Move the filter */
old_ep = ci_trs_ep_get(old_thr, priv->sock_id);
new_ep = ci_trs_ep_get(new_thr, new_s->b.bufid);
rc = tcp_helper_endpoint_move_filters_pre(old_ep, new_ep);
if( rc != 0 ) {
rc = -EINVAL;
goto fail3;
}
/* Allocate a new file for the new endpoint */
rc = onload_alloc_file(new_thr, new_s->b.bufid, priv->_filp->f_flags,
priv->fd_type, &old_ep->alien_ref);
if( rc != 0 )
goto fail4;
ci_assert(old_ep->alien_ref);
/* Copy F_SETOWN_EX, F_SETSIG to the new file */
#ifdef F_SETOWN_EX
rcu_read_lock();
__f_setown(old_ep->alien_ref->_filp, priv->_filp->f_owner.pid,
priv->_filp->f_owner.pid_type, 1);
rcu_read_unlock();
#endif
old_ep->alien_ref->_filp->f_owner.signum = priv->_filp->f_owner.signum;
old_ep->alien_ref->_filp->f_flags |= priv->_filp->f_flags & O_NONBLOCK;
/* Move os_socket from one ep to another */
if( tcp_helper_endpoint_set_aflags(new_ep, OO_THR_EP_AFLAG_ATTACHED) &
OO_THR_EP_AFLAG_ATTACHED ) {
fput(old_ep->alien_ref->_filp);
rc = -EBUSY;
goto fail2; /* state & filters are cleared by fput() */
}
/********* Point of no return **********/
ci_wmb();
priv->fd_type = CI_PRIV_TYPE_ALIEN_EP;
priv->_filp->f_op = &linux_tcp_helper_fops_alien;
ci_wmb();
oo_file_moved(priv);
/* Read all already-arrived packets after the filters move but before
* copying of the receive queue. */
ci_netif_poll(&old_thr->netif);
tcp_helper_endpoint_move_filters_post(old_ep, new_ep);
ci_assert( efab_file_move_supported(&old_thr->netif, old_s));
/* There's a gap between un-registering the old ep, and registering the
* the new. However, the notifications shouldn't be in use for sockets
* that are in a state that can be moved, so this shouldn't be a problem.
*/
if( old_ep->os_sock_pt.whead ) {
pollwait_register = 1;
efab_tcp_helper_os_pollwait_unregister(old_ep);
}
ci_assert_equal(new_ep->os_socket, NULL);
new_ep->os_socket = oo_file_ref_xchg(&old_ep->os_socket, NULL);
ci_assert_equal(old_ep->os_socket, NULL);
if( pollwait_register )
efab_tcp_helper_os_pollwait_register(new_ep);
ci_bit_clear(&new_s->b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
if( new_s->b.state == CI_TCP_ESTABLISHED )
CI_TCP_STATS_INC_CURR_ESTAB(alien_ni);
/* Copy recv queue */
if( new_s->b.state & CI_TCP_STATE_TCP ) {
ci_tcp_state *new_ts = SOCK_TO_TCP(new_s);
ci_tcp_state *old_ts = SOCK_TO_TCP(old_s);
int i;
/* Stop timers */
ci_ip_timer_clear(&old_thr->netif, &old_ts->kalive_tid);
ci_ip_timer_clear(&old_thr->netif, &old_ts->delack_tid);
efab_ip_queue_copy(alien_ni, &new_ts->recv1,
&old_thr->netif, &old_ts->recv1);
efab_ip_queue_copy(alien_ni, &new_ts->recv2,
&old_thr->netif, &old_ts->recv2);
new_ts->recv1_extract = new_ts->recv1.head;
/* Drop reorder buffer */
ci_ip_queue_init(&new_ts->rob);
new_ts->dsack_block = OO_PP_INVALID;
new_ts->dsack_start = new_ts->dsack_end = 0;
for( i = 0; i <= CI_TCP_SACK_MAX_BLOCKS; i++ )
new_ts->last_sack[i] = OO_PP_NULL;
}
else {
/* There should not be any recv q, but drop it to be sure */
ci_udp_recv_q_init(&SOCK_TO_UDP(new_s)->recv_q);
}
/* Old stack can be unlocked */
old_s->b.sb_flags |= CI_SB_FLAG_MOVED;
ci_netif_unlock(&old_thr->netif);
ci_assert( efab_file_move_supported(alien_ni, new_s) );
/* Move done: poll for any new data. */
ci_netif_poll(alien_ni);
if( new_s->b.state & CI_TCP_STATE_TCP ) {
ci_tcp_state *new_ts = SOCK_TO_TCP(new_s);
/* Timers setup: delack, keepalive */
if( (new_ts->acks_pending & CI_TCP_ACKS_PENDING_MASK) > 0)
ci_tcp_timeout_delack(alien_ni, new_ts);
ci_tcp_kalive_reset(alien_ni, new_ts);
}
/* Old ep: we are done. */
ci_bit_set(&old_s->b.sb_aflags, CI_SB_AFLAG_MOVED_AWAY_BIT);
old_s->b.moved_to_stack_id = alien_ni->state->stack_id;
old_s->b.moved_to_sock_id = new_s->b.bufid;
if( ! list_empty(&priv->_filp->f_ep_links) )
ci_bit_set(&old_s->b.sb_aflags, CI_SB_AFLAG_MOVED_AWAY_IN_EPOLL_BIT);
ci_sock_unlock(&old_thr->netif, &old_s->b);
ci_sock_unlock(alien_ni, &new_s->b);
ci_assert(ci_netif_is_locked(alien_ni));
OO_DEBUG_TCPH(ci_log("%s: -> [%d:%d] %s", __func__,
new_thr->id, new_s->b.bufid,
ci_tcp_state_str(new_s->b.state)));
return 0;
fail4:
/* We clear the filters from the new ep.
* For now, we do not need to re-insert old filters because hw filters
* are alredy here (in case of accepted socket) or not needed.
* We have not removed old sw filters yet. */
tcp_helper_endpoint_move_filters_undo(old_ep, new_ep);
fail3:
if( new_s->b.state & CI_TCP_STATE_TCP )
ci_tcp_state_free(alien_ni, SOCK_TO_TCP(new_s));
else
ci_udp_state_free(alien_ni, SOCK_TO_UDP(new_s));
fail2:
ci_netif_unlock(alien_ni);
fail1:
ci_netif_unlock(&old_thr->netif);
fail1_ni_unlocked:
ci_sock_unlock(&old_thr->netif, &old_s->b);
OO_DEBUG_TCPH(ci_log("%s: rc=%d", __func__, rc));
return rc;
}
return rc;
}
#if CI_CFG_USERSPACE_SELECT
static int citp_udp_select(citp_fdinfo* fdi, int* n, int rd, int wr, int ex,
struct oo_ul_select_state*__restrict__ ss)
{
citp_sock_fdi* epi;
ci_udp_state* us;
unsigned mask;
ci_netif* ni;
epi = fdi_to_sock_fdi(fdi);
us = SOCK_TO_UDP(epi->sock.s);
ni = epi->sock.netif;
citp_poll_if_needed(ni, ss->now_frc, ss->ul_select_spin);
mask = ci_udp_poll_events(ni, us);
if( rd && (mask & SELECT_RD_SET) ) {
FD_SET(fdi->fd, ss->rdu);
++*n;
}
if( wr && (mask & SELECT_WR_SET) ) {
FD_SET(fdi->fd, ss->wru);
++*n;
}
return 1;