citp_waitable_obj* citp_waitable_obj_alloc(ci_netif* netif)
{
citp_waitable_obj* wo;
ci_assert(netif);
ci_assert(ci_netif_is_locked(netif));
if( netif->state->deferred_free_eps_head != CI_ILL_END ) {
ci_uint32 link;
do
link = netif->state->deferred_free_eps_head;
while( ci_cas32_fail(&netif->state->deferred_free_eps_head,
link, CI_ILL_END));
while( link != CI_ILL_END ) {
citp_waitable* w = ID_TO_WAITABLE(netif, link);
link = w->next_id;
CI_DEBUG(w->next_id = CI_ILL_END);
ci_assert_equal(w->state, CI_TCP_STATE_FREE);
ci_assert(OO_SP_IS_NULL(w->wt_next));
w->wt_next = netif->state->free_eps_head;
netif->state->free_eps_head = W_SP(w);
}
}
if( OO_SP_IS_NULL(netif->state->free_eps_head) ) {
ci_tcp_helper_more_socks(netif);
if( OO_SP_IS_NULL(netif->state->free_eps_head) )
ci_netif_timeout_reap(netif);
}
if( OO_SP_IS_NULL(netif->state->free_eps_head) )
return NULL;
LOG_TV(ci_log("%s: allocating %d", __FUNCTION__,
OO_SP_FMT(netif->state->free_eps_head)));
ci_assert(IS_VALID_SOCK_P(netif, netif->state->free_eps_head));
#if !defined(__KERNEL__) && !defined (CI_HAVE_OS_NOPAGE)
ci_netif_mmap_shmbuf(netif,
(netif->state->free_eps_head >> EP_BUF_BLOCKSHIFT) + 1);
#endif
wo = SP_TO_WAITABLE_OBJ(netif, netif->state->free_eps_head);
ci_assert(OO_SP_EQ(W_SP(&wo->waitable), netif->state->free_eps_head));
ci_assert_equal(wo->waitable.state, CI_TCP_STATE_FREE);
ci_assert_equal(wo->waitable.sb_aflags, (CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_NOT_READY));
ci_assert_equal(wo->waitable.lock.wl_val, 0);
netif->state->free_eps_head = wo->waitable.wt_next;
CI_DEBUG(wo->waitable.wt_next = OO_SP_NULL);
ci_assert_equal(wo->waitable.state, CI_TCP_STATE_FREE);
return wo;
}
void citp_waitable_obj_free_to_cache(ci_netif* ni, citp_waitable* w)
{
#if defined (__KERNEL__) && !defined(NDEBUG)
/* There should be no non-atomic work queued for endpoints going to cache -
* they don't get their filters removed.
*/
tcp_helper_endpoint_t* ep = ci_netif_get_valid_ep(ni, w->bufid);
ci_assert(!(ep->ep_aflags & OO_THR_EP_AFLAG_NON_ATOMIC));
#endif
ci_assert(!(w->sb_aflags & CI_SB_AFLAG_ORPHAN));
ci_assert(w->sb_aflags & CI_SB_AFLAG_NOT_READY);
ci_assert(w->sb_aflags & CI_SB_AFLAG_IN_CACHE);
ci_assert(w->state == CI_TCP_CLOSED);
ci_assert(ci_ni_dllist_is_self_linked(ni, &w->post_poll_link));
ci_assert(OO_SP_IS_NULL(w->wt_next));
/* This resets a subset of the state done by __citp_waitable_obj_free.
* We do not set the orphan flag, as cached endpoints remain attached.
* We do not alter the state, as that too remains accurate.
*
* We preserve cache related aflags. If the endpoint is freed before being
* accepted from the cache then these will be cleared when
* __citp_waitable_obj_free is called, otherwise they'll be checked for
* correctness, and updated if necessary when the socket is accepted.
*/
w->wake_request = 0;
w->sb_flags = 0;
ci_atomic32_and(&w->sb_aflags, CI_SB_AFLAG_NOT_READY |
CI_SB_AFLAG_CACHE_PRESERVE);
w->lock.wl_val = 0;
w->ready_list_id = 0;
CI_USER_PTR_SET(w->eitem, NULL);
}
static int
ci_tcp_listenq_bucket_drop(ci_netif* ni, ci_tcp_listen_bucket* bucket)
{
ci_ni_aux_mem* aux;
int idx;
oo_p tsr_p;
ci_tcp_state_synrecv* tsr;
int ret = 0;
for( idx = 0; idx < CI_TCP_LISTEN_BUCKET_SIZE; idx++ ) {
if( OO_P_IS_NULL(bucket->bucket[idx]) )
continue;
aux = ci_ni_aux_p2aux(ni, bucket->bucket[idx]);
if( aux->type == CI_TCP_AUX_TYPE_BUCKET )
ret += ci_tcp_listenq_bucket_drop(ni, &aux->u.bucket);
else {
tsr_p = bucket->bucket[idx];
do {
tsr = &ci_ni_aux_p2aux(ni, tsr_p)->u.synrecv;
tsr_p = tsr->bucket_link;
if( OO_SP_IS_NULL(tsr->local_peer) )
ci_ni_dllist_remove(ni, ci_tcp_synrecv2link(tsr));
/* RFC 793 tells us to send FIN and move to FIN-WAIT1 state.
* However, Linux (and probably everybody else) does not do it. */
ci_tcp_synrecv_free(ni, tsr);
ret++;
} while( OO_P_NOT_NULL(tsr_p) );
}
}
ci_ni_aux_free(ni, CI_CONTAINER(ci_ni_aux_mem, u.bucket, bucket));
return ret;
}
static void __citp_waitable_obj_free(ci_netif* ni, citp_waitable* w)
{
ci_assert(w->sb_aflags & CI_SB_AFLAG_ORPHAN);
ci_assert(w->state != CI_TCP_STATE_FREE);
ci_assert(ci_ni_dllist_is_self_linked(ni, &w->post_poll_link));
ci_assert(OO_SP_IS_NULL(w->wt_next));
w->wake_request = 0;
w->sb_flags = 0;
w->sb_aflags = CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_NOT_READY;
w->state = CI_TCP_STATE_FREE;
w->lock.wl_val = 0;
w->ready_list_id = 0;
CI_USER_PTR_SET(w->eitem, NULL);
}
void ci_tcp_listenq_remove(ci_netif* ni, ci_tcp_socket_listen* tls,
ci_tcp_state_synrecv* tsr)
{
ci_assert(ni);
ci_assert(tsr);
ci_assert(tls);
ci_tcp_listenq_bucket_remove(ni, tls,
ci_ni_aux_p2bucket(ni, tls->bucket),
tsr, 0);
if( OO_SP_IS_NULL(tsr->local_peer) ) {
ci_ni_dllist_remove(ni, ci_tcp_synrecv2link(tsr));
if( (tsr->retries & CI_FLAG_TSR_RETRIES_MASK) == 0 )
--tls->n_listenq_new;
}
/* cancel timer if no more synrecv on queue */
if( --tls->n_listenq == 0 &&
(~tls->s.s_flags & CI_SOCK_FLAG_BOUND_ALIEN) )
ci_ip_timer_clear(ni, &tls->listenq_tid);
}
/* Returns:
* 0 on success
*
* CI_SOCKET_ERROR (and errno set)
* this is a normal error that is returned to the
* the application
*
* CI_SOCKET_HANDOVER we tell the upper layers to handover, no need
* to set errno since it isn't a real error
*/
int ci_tcp_connect(citp_socket* ep, const struct sockaddr* serv_addr,
socklen_t addrlen, ci_fd_t fd, int *p_moved)
{
/* Address family is validated earlier. */
struct sockaddr_in* inaddr = (struct sockaddr_in*) serv_addr;
ci_sock_cmn* s = ep->s;
ci_tcp_state* ts = &SOCK_TO_WAITABLE_OBJ(s)->tcp;
int rc = 0, crc;
ci_uint32 dst_be32;
if( NI_OPTS(ep->netif).tcp_connect_handover )
return CI_SOCKET_HANDOVER;
/* Make sure we're up-to-date. */
ci_netif_lock(ep->netif);
CHECK_TEP(ep);
ci_netif_poll(ep->netif);
/*
* 1. Check if state of the socket is OK for connect operation.
*/
start_again:
if( (rc = ci_tcp_connect_handle_so_error(s)) != 0) {
CI_SET_ERROR(rc, rc);
goto unlock_out;
}
if( s->b.state != CI_TCP_CLOSED ) {
/* see if progress can be made on this socket before
** determining status (e.g. non-blocking connect and connect poll)*/
if( s->b.state & CI_TCP_STATE_SYNCHRONISED ) {
if( ts->tcpflags & CI_TCPT_FLAG_NONBLOCK_CONNECT ) {
ts->tcpflags &= ~CI_TCPT_FLAG_NONBLOCK_CONNECT;
rc = 0;
goto unlock_out;
}
if( serv_addr->sa_family == AF_UNSPEC )
LOG_E(ci_log("Onload does not support TCP disconnect via "
"connect(addr->sa_family==AF_UNSPEC)"));
CI_SET_ERROR(rc, EISCONN);
}
else if( s->b.state == CI_TCP_LISTEN ) {
#if CI_CFG_POSIX_CONNECT_AFTER_LISTEN
CI_SET_ERROR(rc, EOPNOTSUPP);
#else
if( ci_tcp_validate_sa(s->domain, serv_addr, addrlen) ) {
/* Request should be forwarded to OS */
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
if( serv_addr->sa_family == AF_UNSPEC ) {
/* Linux does listen shutdown on disconnect (AF_UNSPEC) */
ci_netif_unlock(ep->netif);
rc = ci_tcp_shutdown(ep, SHUT_RD, fd);
goto out;
} else {
/* Linux has curious error reporting in this case */
CI_SET_ERROR(rc, EISCONN);
}
#endif
}
else {
/* Socket is in SYN-SENT state. Let's block for receiving SYN-ACK */
ci_assert_equal(s->b.state, CI_TCP_SYN_SENT);
if( s->b.sb_aflags & (CI_SB_AFLAG_O_NONBLOCK | CI_SB_AFLAG_O_NDELAY) )
CI_SET_ERROR(rc, EALREADY);
else
goto syn_sent;
}
goto unlock_out;
}
/* Check if we've ever been connected. */
if( ts->tcpflags & CI_TCPT_FLAG_WAS_ESTAB ) {
CI_SET_ERROR(rc, EISCONN);
goto unlock_out;
}
/*
* 2. Check address parameter, if it's inappropriate for handover
* decision or handover should be done, try to to call OS and
* do handover on success.
*/
if (
/* Af first, check that address family and length is OK. */
ci_tcp_validate_sa(s->domain, serv_addr, addrlen)
/* rfc793 p54 if the foreign socket is unspecified return */
/* "error: foreign socket unspecified" (EINVAL), but keep it to OS */
|| (dst_be32 = ci_get_ip4_addr(inaddr->sin_family, serv_addr)) == 0
/* Zero destination port is tricky as well, keep it to OS */
|| inaddr->sin_port == 0 )
{
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
/* is this a socket that we can handle? */
rc = ci_tcp_connect_check_dest(ep, dst_be32, inaddr->sin_port);
if( rc ) goto unlock_out;
if( (ts->s.pkt.flags & CI_IP_CACHE_IS_LOCALROUTE) &&
OO_SP_IS_NULL(ts->local_peer) ) {
/* Try to connect to another stack; handover if can't */
struct oo_op_loopback_connect op;
op.dst_port = inaddr->sin_port;
op.dst_addr = dst_be32;
/* this operation unlocks netif */
rc = oo_resource_op(fd, OO_IOC_TCP_LOOPBACK_CONNECT, &op);
if( rc < 0)
return CI_SOCKET_HANDOVER;
if( op.out_moved )
*p_moved = 1;
if( op.out_rc == -EINPROGRESS )
RET_WITH_ERRNO( EINPROGRESS );
else if( op.out_rc == -EAGAIN )
return -EAGAIN;
else if( op.out_rc != 0 )
return CI_SOCKET_HANDOVER;
return 0;
}
/* filters can't handle alien source address */
if( (s->s_flags & CI_SOCK_FLAG_BOUND_ALIEN) &&
! (ts->s.pkt.flags & CI_IP_CACHE_IS_LOCALROUTE) ) {
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
crc = ci_tcp_connect_ul_start(ep->netif, ts, dst_be32, inaddr->sin_port, &rc);
if( crc != CI_CONNECT_UL_OK ) {
switch( crc ) {
case CI_CONNECT_UL_FAIL:
goto unlock_out;
case CI_CONNECT_UL_LOCK_DROPPED:
goto out;
case CI_CONNECT_UL_START_AGAIN:
goto start_again;
}
}
CI_TCP_STATS_INC_ACTIVE_OPENS( ep->netif );
syn_sent:
rc = ci_tcp_connect_ul_syn_sent(ep->netif, ts);
unlock_out:
ci_netif_unlock(ep->netif);
out:
return rc;
}
