int ci_tcp_shutdown(citp_socket* ep, int how, ci_fd_t fd)
{
ci_sock_cmn* s = ep->s;
int rc;
if( s->b.state == CI_TCP_LISTEN )
return ci_tcp_shutdown_listen(ep, how, fd);
if( SOCK_TO_TCP(s)->snd_delegated ) {
/* We do not know which seq number to use. Call
* onload_delegated_send_cancel(). */
CI_SET_ERROR(rc, EBUSY);
return rc;
}
if( ! ci_netif_trylock(ep->netif) ) {
/* Can't get lock, so try to defer shutdown to the lock holder. */
unsigned flags = 0;
switch( s->b.state ) {
case CI_TCP_CLOSED:
case CI_TCP_TIME_WAIT:
CI_SET_ERROR(rc, ENOTCONN);
return rc;
}
if( how == SHUT_RD || how == SHUT_RDWR )
flags |= CI_SOCK_AFLAG_NEED_SHUT_RD;
if( how == SHUT_WR || how == SHUT_RDWR )
flags |= CI_SOCK_AFLAG_NEED_SHUT_WR;
ci_atomic32_or(&s->s_aflags, flags);
if( ! ci_netif_lock_or_defer_work(ep->netif, &s->b) )
return 0;
ci_atomic32_and(&s->s_aflags, ~flags);
}
if( 0 ) {
/* Poll to get up-to-date. This is slightly spurious but done to ensure
* ordered response to all packets preceding this FIN (e.g. ANVL tcp_core
* 9.18)
*
* DJR: I've disabled this because it can hurt performance for
* high-connection-rate apps. May consider adding back (as option?) if
* needed.
*/
ci_netif_poll(ep->netif);
}
rc = __ci_tcp_shutdown(ep->netif, SOCK_TO_TCP(s), how);
if( rc < 0 )
CI_SET_ERROR(rc, -rc);
ci_netif_unlock(ep->netif);
return rc;
}
int ci_udp_shutdown(citp_socket* ep, ci_fd_t fd, int how)
{
ci_fd_t os_sock;
int rc;
CHECK_UEP(ep);
LOG_UV(log(LPF "shutdown("SF_FMT", %d)", SF_PRI_ARGS(ep,fd), how));
os_sock = ci_get_os_sock_fd(fd);
if( CI_IS_VALID_SOCKET( os_sock ) ) {
rc = ci_sys_shutdown(os_sock, how);
ci_rel_os_sock_fd( os_sock );
if( rc < 0 )
return CI_SOCKET_ERROR;
}
rc = __ci_udp_shutdown(ep->netif, SOCK_TO_UDP(ep->s), how);
if( rc < 0 ) {
CI_SET_ERROR(rc, -rc);
return rc;
}
return 0;
}
static int ci_udp_set_filters(citp_socket* ep, ci_udp_state* us)
{
int rc;
ci_assert(ep);
ci_assert(us);
if( udp_lport_be16(us) == 0 )
return 0;
rc = ci_tcp_ep_set_filters(ep->netif, S_SP(us), us->s.cp.so_bindtodevice,
OO_SP_NULL);
if( rc == -EFILTERSSOME ) {
if( CITP_OPTS.no_fail )
rc = 0;
else {
ci_tcp_ep_clear_filters(ep->netif, S_SP(us), 0);
rc = -ENOBUFS;
}
}
if( rc < 0 ) {
LOG_UC(log(FNS_FMT "ci_tcp_ep_set_filters failed (%d)",
FNS_PRI_ARGS(ep->netif, ep->s), -rc));
CI_SET_ERROR(rc, -rc);
return rc;
}
UDP_SET_FLAG(us, CI_UDPF_FILTERED);
return 0;
}
int ci_tcp_getpeername(citp_socket* ep, struct sockaddr* name,
socklen_t* namelen)
{
ci_sock_cmn* s = ep->s;
int rc;
CHECK_TEP_NNL(ep);
/* If we're not connected... */
if( ! (s->b.state & CI_TCP_STATE_SYNCHRONISED) ||
s->b.state == CI_TCP_TIME_WAIT )
CI_SET_ERROR(rc, ENOTCONN);
else if( name == NULL || namelen == NULL )
CI_SET_ERROR(rc, EFAULT);
else {
ci_addr_to_user(name, namelen, s->domain,
S_TCP_HDR(s)->tcp_dest_be16, s->pkt.ip.ip_daddr_be32);
rc = 0;
}
return rc;
}
static int citp_udp_ioctl(citp_fdinfo* fdinfo, int request, void* arg)
{
citp_sock_fdi *epi = fdi_to_sock_fdi(fdinfo);
int rc;
Log_V(log("%s("EF_FMT", %d, 0x%lx)", __FUNCTION__,
EF_PRI_ARGS(epi, fdinfo->fd), request, (long) arg));
rc = ci_udp_ioctl(&epi->sock, fdinfo->fd, request, arg);
Log_V(log(LPF "ioctl()=%d", rc));
if( rc < 0 )
CI_SET_ERROR(rc, -rc);
return rc;
}
/* NOTE: in the kernel version [fd] is unused and, if it's a ptr, [arg] will
* be in user-space and may need to be fetched into kernel memory. */
static int ci_tcp_ioctl_lk(citp_socket* ep, ci_fd_t fd, int request,
void* arg)
{
ci_netif* netif = ep->netif;
ci_sock_cmn* s = ep->s;
ci_tcp_state* ts = NULL;
int rc = 0;
int os_socket_exists = s->b.sb_aflags & CI_SB_AFLAG_OS_BACKED;
if( s->b.state != CI_TCP_LISTEN )
ts = SOCK_TO_TCP(s);
/* Keep the os socket in sync. If this is a "get" request then the
* return will be based on our support, not the os's (except for EFAULT
* handling which we get for free).
* Exceptions:
* - FIONBIO is applied just in time on handover if needed (listening
* sockets always have a non-blocking OS socket)
* - FIONREAD, TIOCOUTQ, SIOCOUTQNSD and SIOCATMARK are useless on OS
* socket, let's avoid syscall.
*/
if( os_socket_exists && request != FIONREAD && request != SIOCATMARK &&
request != FIOASYNC && request != TIOCOUTQ && request != SIOCOUTQNSD &&
request != (int) FIONBIO ) {
rc = oo_os_sock_ioctl(netif, s->b.bufid, request, arg, NULL);
if( rc < 0 )
return rc;
}
/* ioctl defines are listed in `man ioctl_list` and the CI equivalent
* CI defines are in include/ci/net/ioctls.h */
LOG_TV( ci_log("%s: request = %d, arg = %ld", __FUNCTION__, request,
(long)arg));
switch( request ) {
case FIONBIO:
if( CI_IOCTL_ARG_OK(int, arg) ) {
CI_CMN_IOCTL_FIONBIO(ep->s, arg);
rc = 0;
break;
}
goto fail_fault;
case FIONREAD: /* synonym of SIOCINQ */
if( !CI_IOCTL_ARG_OK(int, arg) )
goto fail_fault;
if( s->b.state == CI_TCP_LISTEN )
goto fail_inval;
if( s->b.state == CI_TCP_SYN_SENT ) {
CI_IOCTL_SETARG((int*)arg, 0);
} else {
/* In inline mode, return the total number of bytes in the receive queue.
If SO_OOBINLINE isn't set then return the number of bytes up to the
mark but without counting the mark */
int bytes_in_rxq = tcp_rcv_usr(ts);
if (bytes_in_rxq && ! (ts->s.s_flags & CI_SOCK_FLAG_OOBINLINE)) {
if (tcp_urg_data(ts) & CI_TCP_URG_PTR_VALID) {
/*! \TODO: what if FIN has been received? */
unsigned int readnxt = tcp_rcv_nxt(ts) - bytes_in_rxq;
if (SEQ_LT(readnxt, tcp_rcv_up(ts))) {
bytes_in_rxq = tcp_rcv_up(ts) - readnxt;
} else if (SEQ_EQ(readnxt, tcp_rcv_up(ts))) {
bytes_in_rxq--;
}
}
}
CI_IOCTL_SETARG((int*)arg, bytes_in_rxq);
}
break;
case TIOCOUTQ: /* synonym of SIOCOUTQ */
case SIOCOUTQNSD:
{
CI_BUILD_ASSERT(TIOCOUTQ == SIOCOUTQ);
int outq_bytes = 0;
if( !CI_IOCTL_ARG_OK(int, arg) )
goto fail_fault;
if( s->b.state == CI_TCP_LISTEN )
goto fail_inval;
if( s->b.state != CI_TCP_SYN_SENT ) {
/* TIOCOUTQ counts all unacknowledged data, so includes retrans queue. */
if( request == TIOCOUTQ )
outq_bytes = SEQ_SUB(tcp_enq_nxt(ts), tcp_snd_una(ts));
else
outq_bytes = SEQ_SUB(tcp_enq_nxt(ts), tcp_snd_nxt(ts));
}
CI_IOCTL_SETARG((int*)arg, outq_bytes);
}
break;
case SIOCATMARK:
{
if( !CI_IOCTL_ARG_OK(int, arg) )
goto fail_fault;
/* return true, if we are at the out-of-band byte */
CI_IOCTL_SETARG((int*)arg, 0);
if( s->b.state != CI_TCP_LISTEN ) {
int readnxt;
readnxt = SEQ_SUB(tcp_rcv_nxt(ts), tcp_rcv_usr(ts));
if( ~ts->s.b.state & CI_TCP_STATE_ACCEPT_DATA )
readnxt = SEQ_SUB(readnxt, 1);
if( tcp_urg_data(ts) & CI_TCP_URG_PTR_VALID )
CI_IOCTL_SETARG((int*)arg, readnxt == tcp_rcv_up(ts));
LOG_URG(log(NTS_FMT "SIOCATMARK atmark=%d readnxt=%u rcv_up=%u%s",
NTS_PRI_ARGS(ep->netif, ts), readnxt == tcp_rcv_up(ts),
readnxt, tcp_rcv_up(SOCK_TO_TCP(ep->s)),
(tcp_urg_data(ts)&CI_TCP_URG_PTR_VALID)?"":" (invalid)"));
}
break;
}
#ifndef __KERNEL__
case FIOASYNC:
/* Need to apply this to [fd] so that our fasync file-op will be
* invoked.
*/
rc = ci_sys_ioctl(fd, request, arg);
break;
case SIOCSPGRP:
if( !CI_IOCTL_ARG_OK(int, arg) )
goto fail_fault;
/* Need to apply this to [fd] to get signal delivery to work. However,
* SIOCSPGRP is only supported on sockets, so we need to convert to
* fcntl().
*/
rc = ci_sys_fcntl(fd, F_SETOWN, CI_IOCTL_GETARG(int, arg));
if( rc == 0 ) {
rc = ci_cmn_ioctl(netif, ep->s, request, arg, rc, os_socket_exists);
}
else {
CI_SET_ERROR(rc, -rc);
}
break;
#endif
default:
return ci_cmn_ioctl(netif, ep->s, request, arg, rc, os_socket_exists);
}
static int ci_udp_recvmsg_socklocked_slowpath(ci_udp_iomsg_args* a,
ci_msghdr* msg,
ci_iovec_ptr *piov, int flags)
{
int rc = 0;
ci_netif* ni = a->ni;
ci_udp_state* us = a->us;
if(CI_UNLIKELY( ni->state->rxq_low ))
ci_netif_rxq_low_on_recv(ni, &us->s,
1 /* assume at least one pkt freed */);
/* In the kernel recv() with flags is not called.
* only read(). So flags may only contain MSG_DONTWAIT */
#ifdef __KERNEL__
ci_assert_equal(flags, 0);
#endif
#ifndef __KERNEL__
if( flags & MSG_ERRQUEUE_CHK ) {
if( OO_PP_NOT_NULL(us->timestamp_q.extract) ) {
ci_ip_pkt_fmt* pkt;
struct timespec ts[3];
struct cmsg_state cmsg_state;
ci_udp_hdr* udp;
int paylen;
/* TODO is this necessary? - mirroring ci_udp_recvmsg_get() */
ci_rmb();
pkt = PKT_CHK_NNL(ni, us->timestamp_q.extract);
if( pkt->tx_hw_stamp.tv_sec == CI_PKT_TX_HW_STAMP_CONSUMED ) {
if( OO_PP_IS_NULL(pkt->tsq_next) )
goto errqueue_empty;
us->timestamp_q.extract = pkt->tsq_next;
pkt = PKT_CHK_NNL(ni, us->timestamp_q.extract);
ci_assert(pkt->tx_hw_stamp.tv_sec != CI_PKT_TX_HW_STAMP_CONSUMED);
}
udp = oo_ip_data(pkt);
paylen = CI_BSWAP_BE16(oo_ip_hdr(pkt)->ip_tot_len_be16) -
sizeof(ci_ip4_hdr) - sizeof(udp);
msg->msg_flags = 0;
cmsg_state.msg = msg;
cmsg_state.cm = msg->msg_control;
cmsg_state.cmsg_bytes_used = 0;
ci_iovec_ptr_init_nz(piov, msg->msg_iov, msg->msg_iovlen);
memset(ts, 0, sizeof(ts));
if( us->s.timestamping_flags & ONLOAD_SOF_TIMESTAMPING_RAW_HARDWARE ) {
ts[2].tv_sec = pkt->tx_hw_stamp.tv_sec;
ts[2].tv_nsec = pkt->tx_hw_stamp.tv_nsec;
}
if( (us->s.timestamping_flags & ONLOAD_SOF_TIMESTAMPING_SYS_HARDWARE) &&
(pkt->tx_hw_stamp.tv_nsec & CI_IP_PKT_HW_STAMP_FLAG_IN_SYNC) ) {
ts[1].tv_sec = pkt->tx_hw_stamp.tv_sec;
ts[1].tv_nsec = pkt->tx_hw_stamp.tv_nsec;
}
ci_put_cmsg(&cmsg_state, SOL_SOCKET, ONLOAD_SCM_TIMESTAMPING,
sizeof(ts), &ts);
oo_offbuf_set_start(&pkt->buf, udp + 1);
oo_offbuf_set_len(&pkt->buf, paylen);
rc = oo_copy_pkt_to_iovec_no_adv(ni, pkt, piov, paylen);
/* Mark this packet/timestamp as consumed */
pkt->tx_hw_stamp.tv_sec = CI_PKT_TX_HW_STAMP_CONSUMED;
ci_ip_cmsg_finish(&cmsg_state);
msg->msg_flags |= MSG_ERRQUEUE_CHK;
return rc;
}
errqueue_empty:
/* ICMP is handled via OS, so get OS error */
rc = oo_os_sock_recvmsg(ni, SC_SP(&us->s), msg, flags);
if( rc < 0 ) {
ci_assert(-rc == errno);
return -1;
}
else
return rc;
}
#endif
if( (rc = ci_get_so_error(&us->s)) != 0 ) {
CI_SET_ERROR(rc, rc);
return rc;
}
if( msg->msg_iovlen > 0 && msg->msg_iov == NULL ) {
CI_SET_ERROR(rc, EFAULT);
return rc;
}
#if MSG_OOB_CHK
if( flags & MSG_OOB_CHK ) {
CI_SET_ERROR(rc, EOPNOTSUPP);
return rc;
}
#endif
#if CI_CFG_POSIX_RECV
if( ! udp_lport_be16(us)) {
LOG_UV(log("%s: -1 (ENOTCONN)", __FUNCTION__));
CI_SET_ERROR(rc, ENOTCONN);
return rc;
}
#endif
if( msg->msg_iovlen == 0 ) {
/* We have a difference in behaviour from the Linux stack here. When
** msg_iovlen is 0 Linux 2.4.21-15.EL does not set MSG_TRUNC when a
** datagram has non-zero length. We do. */
CI_IOVEC_LEN(&piov->io) = piov->iovlen = 0;
return IOVLEN_WORKAROUND_RC_VALUE;
}
return 0;
}
/* Returns:
* 0 on success
*
* CI_SOCKET_ERROR (and errno set)
* this is a normal error that is returned to the
* the application
*
* CI_SOCKET_HANDOVER we tell the upper layers to handover, no need
* to set errno since it isn't a real error
*/
int ci_tcp_connect(citp_socket* ep, const struct sockaddr* serv_addr,
socklen_t addrlen, ci_fd_t fd, int *p_moved)
{
/* Address family is validated earlier. */
struct sockaddr_in* inaddr = (struct sockaddr_in*) serv_addr;
ci_sock_cmn* s = ep->s;
ci_tcp_state* ts = &SOCK_TO_WAITABLE_OBJ(s)->tcp;
int rc = 0, crc;
ci_uint32 dst_be32;
if( NI_OPTS(ep->netif).tcp_connect_handover )
return CI_SOCKET_HANDOVER;
/* Make sure we're up-to-date. */
ci_netif_lock(ep->netif);
CHECK_TEP(ep);
ci_netif_poll(ep->netif);
/*
* 1. Check if state of the socket is OK for connect operation.
*/
start_again:
if( (rc = ci_tcp_connect_handle_so_error(s)) != 0) {
CI_SET_ERROR(rc, rc);
goto unlock_out;
}
if( s->b.state != CI_TCP_CLOSED ) {
/* see if progress can be made on this socket before
** determining status (e.g. non-blocking connect and connect poll)*/
if( s->b.state & CI_TCP_STATE_SYNCHRONISED ) {
if( ts->tcpflags & CI_TCPT_FLAG_NONBLOCK_CONNECT ) {
ts->tcpflags &= ~CI_TCPT_FLAG_NONBLOCK_CONNECT;
rc = 0;
goto unlock_out;
}
if( serv_addr->sa_family == AF_UNSPEC )
LOG_E(ci_log("Onload does not support TCP disconnect via "
"connect(addr->sa_family==AF_UNSPEC)"));
CI_SET_ERROR(rc, EISCONN);
}
else if( s->b.state == CI_TCP_LISTEN ) {
#if CI_CFG_POSIX_CONNECT_AFTER_LISTEN
CI_SET_ERROR(rc, EOPNOTSUPP);
#else
if( ci_tcp_validate_sa(s->domain, serv_addr, addrlen) ) {
/* Request should be forwarded to OS */
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
if( serv_addr->sa_family == AF_UNSPEC ) {
/* Linux does listen shutdown on disconnect (AF_UNSPEC) */
ci_netif_unlock(ep->netif);
rc = ci_tcp_shutdown(ep, SHUT_RD, fd);
goto out;
} else {
/* Linux has curious error reporting in this case */
CI_SET_ERROR(rc, EISCONN);
}
#endif
}
else {
/* Socket is in SYN-SENT state. Let's block for receiving SYN-ACK */
ci_assert_equal(s->b.state, CI_TCP_SYN_SENT);
if( s->b.sb_aflags & (CI_SB_AFLAG_O_NONBLOCK | CI_SB_AFLAG_O_NDELAY) )
CI_SET_ERROR(rc, EALREADY);
else
goto syn_sent;
}
goto unlock_out;
}
/* Check if we've ever been connected. */
if( ts->tcpflags & CI_TCPT_FLAG_WAS_ESTAB ) {
CI_SET_ERROR(rc, EISCONN);
goto unlock_out;
}
/*
* 2. Check address parameter, if it's inappropriate for handover
* decision or handover should be done, try to to call OS and
* do handover on success.
*/
if (
/* Af first, check that address family and length is OK. */
ci_tcp_validate_sa(s->domain, serv_addr, addrlen)
/* rfc793 p54 if the foreign socket is unspecified return */
/* "error: foreign socket unspecified" (EINVAL), but keep it to OS */
|| (dst_be32 = ci_get_ip4_addr(inaddr->sin_family, serv_addr)) == 0
/* Zero destination port is tricky as well, keep it to OS */
|| inaddr->sin_port == 0 )
{
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
/* is this a socket that we can handle? */
rc = ci_tcp_connect_check_dest(ep, dst_be32, inaddr->sin_port);
if( rc ) goto unlock_out;
if( (ts->s.pkt.flags & CI_IP_CACHE_IS_LOCALROUTE) &&
OO_SP_IS_NULL(ts->local_peer) ) {
/* Try to connect to another stack; handover if can't */
struct oo_op_loopback_connect op;
op.dst_port = inaddr->sin_port;
op.dst_addr = dst_be32;
/* this operation unlocks netif */
rc = oo_resource_op(fd, OO_IOC_TCP_LOOPBACK_CONNECT, &op);
if( rc < 0)
return CI_SOCKET_HANDOVER;
if( op.out_moved )
*p_moved = 1;
if( op.out_rc == -EINPROGRESS )
RET_WITH_ERRNO( EINPROGRESS );
else if( op.out_rc == -EAGAIN )
return -EAGAIN;
else if( op.out_rc != 0 )
return CI_SOCKET_HANDOVER;
return 0;
}
/* filters can't handle alien source address */
if( (s->s_flags & CI_SOCK_FLAG_BOUND_ALIEN) &&
! (ts->s.pkt.flags & CI_IP_CACHE_IS_LOCALROUTE) ) {
rc = CI_SOCKET_HANDOVER;
goto unlock_out;
}
crc = ci_tcp_connect_ul_start(ep->netif, ts, dst_be32, inaddr->sin_port, &rc);
if( crc != CI_CONNECT_UL_OK ) {
switch( crc ) {
case CI_CONNECT_UL_FAIL:
goto unlock_out;
case CI_CONNECT_UL_LOCK_DROPPED:
goto out;
case CI_CONNECT_UL_START_AGAIN:
goto start_again;
}
}
CI_TCP_STATS_INC_ACTIVE_OPENS( ep->netif );
syn_sent:
rc = ci_tcp_connect_ul_syn_sent(ep->netif, ts);
unlock_out:
ci_netif_unlock(ep->netif);
out:
return rc;
}
static int ci_tcp_connect_ul_syn_sent(ci_netif *ni, ci_tcp_state *ts)
{
int rc = 0;
if( ts->s.b.state == CI_TCP_SYN_SENT ) {
ci_netif_poll(ni);
if( OO_SP_NOT_NULL(ts->local_peer) ) {
/* No reason to sleep. Obviously, listener have dropped our syn
* because of some reason. Go away! */
ci_tcp_drop(ni, ts, EBUSY);
RET_WITH_ERRNO(EBUSY);
}
CI_TCP_SLEEP_WHILE(ni, ts, CI_SB_FLAG_WAKE_RX,
ts->s.so.sndtimeo_msec,
ts->s.b.state == CI_TCP_SYN_SENT, &rc);
}
if( rc == -EAGAIN ) {
LOG_TC(log( LNT_FMT "timeout on sleep: %d",
LNT_PRI_ARGS(ni, ts), -rc));
if( ! (ts->tcpflags & CI_TCPT_FLAG_NONBLOCK_CONNECT) ) {
ts->tcpflags |= CI_TCPT_FLAG_NONBLOCK_CONNECT;
CI_SET_ERROR(rc, EINPROGRESS);
}
else
CI_SET_ERROR(rc, EALREADY);
return rc;
}
else if( rc == -EINTR ) {
LOG_TC(log(LNT_FMT "connect() was interrupted by a signal",
LNT_PRI_ARGS(ni, ts)));
ts->tcpflags |= CI_TCPT_FLAG_NONBLOCK_CONNECT;
CI_SET_ERROR(rc, EINTR);
return rc;
}
/*! \TODO propagate the correct error code: CONNREFUSED, NOROUTE, etc. */
if( ts->s.b.state == CI_TCP_CLOSED ) {
/* Bug 3558:
* Set OS socket state to allow/disallow next bind().
* It is Linux hack. */
#ifdef __ci_driver__
CI_TRY(efab_tcp_helper_set_tcp_close_os_sock(netif2tcp_helper_resource(ni),
S_SP(ts)));
#else
CI_TRY(ci_tcp_helper_set_tcp_close_os_sock(ni, S_SP(ts)));
#endif
/* We should re-bind socket on the next use if the port was determined by
* OS.
*/
if( ! (ts->s.s_flags & CI_SOCK_FLAG_PORT_BOUND) )
ts->s.s_flags |= CI_SOCK_FLAG_CONNECT_MUST_BIND;
/* - if SO_ERROR is set, handle it and return this value;
* - else if rx_errno is set, return it;
* - else (TCP_RX_ERRNO==0, socket is CI_SHUT_RD) return ECONNABORTED */
if( (rc = ci_tcp_connect_handle_so_error(&ts->s)) == 0)
rc = TCP_RX_ERRNO(ts) ? TCP_RX_ERRNO(ts) : ECONNABORTED;
CI_SET_ERROR(rc, rc);
if( ! (ts->s.s_flags & CI_SOCK_FLAG_ADDR_BOUND) ) {
ts->s.pkt.ip.ip_saddr_be32 = 0;
ts->s.cp.ip_laddr_be32 = 0;
}
return rc;
}
return 0;
}
static int ci_tcp_connect_ul_start(ci_netif *ni, ci_tcp_state* ts,
ci_uint32 dst_be32, unsigned dport_be16,
int* fail_rc)
{
ci_ip_pkt_fmt* pkt;
int rc = 0;
ci_assert(ts->s.pkt.mtu);
/* Now that we know the outgoing route, set the MTU related values.
* Note, even these values are speculative since the real MTU
* could change between now and passing the packet to the lower layers
*/
ts->amss = ts->s.pkt.mtu - sizeof(ci_tcp_hdr) - sizeof(ci_ip4_hdr);
#if CI_CFG_LIMIT_AMSS
ts->amss = ci_tcp_limit_mss(ts->amss, ni, __FUNCTION__);
#endif
/* Default smss until discovered by MSS option in SYN - RFC1122 4.2.2.6 */
ts->smss = CI_CFG_TCP_DEFAULT_MSS;
/* set pmtu, eff_mss, snd_buf and adjust windows */
ci_pmtu_set(ni, &ts->pmtus, ts->s.pkt.mtu);
ci_tcp_set_eff_mss(ni, ts);
ci_tcp_set_initialcwnd(ni, ts);
/* Send buffer adjusted by ci_tcp_set_eff_mss(), but we want it to stay
* zero until the connection is established.
*/
ts->so_sndbuf_pkts = 0;
/*
* 3. State and address are OK. It's address routed through our NIC.
* Do connect().
*/
ci_assert_nequal(ts->s.pkt.ip.ip_saddr_be32, INADDR_ANY);
if( ts->s.s_flags & CI_SOCK_FLAG_CONNECT_MUST_BIND ) {
ci_sock_cmn* s = &ts->s;
ci_uint16 source_be16 = 0;
if( s->s_flags & CI_SOCK_FLAG_ADDR_BOUND )
rc = __ci_bind(ni, &ts->s, ts->s.pkt.ip.ip_saddr_be32, &source_be16);
else
rc = __ci_bind(ni, &ts->s, INADDR_ANY, &source_be16);
if(CI_LIKELY( rc == 0 )) {
TS_TCP(ts)->tcp_source_be16 = source_be16;
ts->s.cp.lport_be16 = source_be16;
LOG_TC(log(LNT_FMT "connect: our bind returned %s:%u",
LNT_PRI_ARGS(ni, ts),
ip_addr_str(INADDR_ANY),
(unsigned) CI_BSWAP_BE16(TS_TCP(ts)->tcp_source_be16)));
}
else {
LOG_U(ci_log("__ci_bind returned %d at %s:%d", CI_GET_ERROR(rc),
__FILE__, __LINE__));
*fail_rc = rc;
return CI_CONNECT_UL_FAIL;
}
if(CI_UNLIKELY( ts->s.pkt.ip.ip_saddr_be32 == 0 )) {
CI_SET_ERROR(*fail_rc, EINVAL);
return CI_CONNECT_UL_FAIL;
}
}
ci_tcp_set_peer(ts, dst_be32, dport_be16);
/* Make sure we can get a buffer before we change state. */
pkt = ci_netif_pkt_tx_tcp_alloc(ni);
if( CI_UNLIKELY(! pkt) ) {
/* NB. We've already done a poll above. */
rc = ci_netif_pkt_wait(ni, &ts->s, CI_SLEEP_NETIF_LOCKED|CI_SLEEP_NETIF_RQ);
if( ci_netif_pkt_wait_was_interrupted(rc) ) {
CI_SET_ERROR(*fail_rc, -rc);
return CI_CONNECT_UL_LOCK_DROPPED;
}
/* OK, there are (probably) packets available - go try again. Note we
* jump back to the top of the function because someone may have
* connected this socket in the mean-time, so we need to check the
* state once more.
*/
return CI_CONNECT_UL_START_AGAIN;
}
#ifdef ONLOAD_OFE
if( ni->ofe != NULL )
ts->s.ofe_code_start = ofe_socktbl_find(
ni->ofe, OFE_SOCKTYPE_TCP_ACTIVE,
tcp_laddr_be32(ts), tcp_raddr_be32(ts),
tcp_lport_be16(ts), tcp_rport_be16(ts));
#endif
rc = ci_tcp_ep_set_filters(ni, S_SP(ts), ts->s.cp.so_bindtodevice,
OO_SP_NULL);
if( rc < 0 ) {
/* Perhaps we've run out of filters? See if we can push a socket out
* of timewait and steal its filter.
*/
ci_assert_nequal(rc, -EFILTERSSOME);
if( rc != -EBUSY || ! ci_netif_timewait_try_to_free_filter(ni) ||
(rc = ci_tcp_ep_set_filters(ni, S_SP(ts),
ts->s.cp.so_bindtodevice,
OO_SP_NULL)) < 0 ) {
ci_assert_nequal(rc, -EFILTERSSOME);
/* Either a different error, or our efforts to free a filter did not
* work.
*/
if( ! (ts->s.s_flags & CI_SOCK_FLAG_ADDR_BOUND) ) {
ts->s.pkt.ip.ip_saddr_be32 = 0;
ts->s.cp.ip_laddr_be32 = 0;
}
ci_netif_pkt_release(ni, pkt);
CI_SET_ERROR(*fail_rc, -rc);
return CI_CONNECT_UL_FAIL;
}
}
LOG_TC(log(LNT_FMT "CONNECT %s:%u->%s:%u", LNT_PRI_ARGS(ni, ts),
ip_addr_str(ts->s.pkt.ip.ip_saddr_be32),
(unsigned) CI_BSWAP_BE16(TS_TCP(ts)->tcp_source_be16),
ip_addr_str(ts->s.pkt.ip.ip_daddr_be32),
(unsigned) CI_BSWAP_BE16(TS_TCP(ts)->tcp_dest_be16)));
/* We are going to send the SYN - set states appropriately */
tcp_snd_una(ts) = tcp_snd_nxt(ts) = tcp_enq_nxt(ts) = tcp_snd_up(ts) =
ci_tcp_initial_seqno(ni);
ts->snd_max = tcp_snd_nxt(ts) + 1;
/* Must be after initialising snd_una. */
ci_tcp_clear_rtt_timing(ts);
ci_tcp_set_flags(ts, CI_TCP_FLAG_SYN);
ts->tcpflags &=~ CI_TCPT_FLAG_OPT_MASK;
ts->tcpflags |= NI_OPTS(ni).syn_opts;
if( (ts->tcpflags & CI_TCPT_FLAG_WSCL) ) {
ts->rcv_wscl = ci_tcp_wscl_by_buff(ni, ci_tcp_rcvbuf_established(ni, &ts->s));
CI_IP_SOCK_STATS_VAL_RXWSCL(ts, ts->rcv_wscl);
}
else {
ts->rcv_wscl = 0;
CI_IP_SOCK_STATS_VAL_RXWSCL(ts, 0);
}
ci_tcp_set_rcvbuf(ni, ts);
ci_tcp_init_rcv_wnd(ts, "CONNECT");
/* outgoing_hdrs_len is initialised to include timestamp option. */
if( ! (ts->tcpflags & CI_TCPT_FLAG_TSO) )
ts->outgoing_hdrs_len = sizeof(ci_ip4_hdr)+sizeof(ci_tcp_hdr);
if( ci_tcp_can_stripe(ni, ts->s.pkt.ip.ip_saddr_be32,
ts->s.pkt.ip.ip_daddr_be32) )
ts->tcpflags |= CI_TCPT_FLAG_STRIPE;
ci_tcp_set_slow_state(ni, ts, CI_TCP_SYN_SENT);
/* If the app trys to send data on a socket in SYN_SENT state
** then the data is queued for send until the SYN gets ACKed.
** (rfc793 p56)
**
** Receive calls on the socket should block until data arrives
** (rfc793 p58)
**
** Clearing tx_errno and rx_errno acheive this. The transmit window
** is set to 1 byte which ensures that only the SYN packet gets
** sent until the ACK is received with more window.
*/
ci_assert(ts->snd_max == tcp_snd_nxt(ts) + 1);
ts->s.rx_errno = 0;
ts->s.tx_errno = 0;
ci_tcp_enqueue_no_data(ts, ni, pkt);
ci_tcp_set_flags(ts, CI_TCP_FLAG_ACK);
if( ts->s.b.sb_aflags & (CI_SB_AFLAG_O_NONBLOCK | CI_SB_AFLAG_O_NDELAY) ) {
ts->tcpflags |= CI_TCPT_FLAG_NONBLOCK_CONNECT;
LOG_TC(log( LNT_FMT "Non-blocking connect - return EINPROGRESS",
LNT_PRI_ARGS(ni, ts)));
CI_SET_ERROR(*fail_rc, EINPROGRESS);
return CI_CONNECT_UL_FAIL;
}
return CI_CONNECT_UL_OK;
}
int ci_tcp_listen(citp_socket* ep, ci_fd_t fd, int backlog)
{
/*
** ?? error handling on possible fails not handled robustly...
** ?? Need to check port number is valid TODO
*/
/*! \todo If not bound then we have to be listening on all interfaces.
* It's likely that we won't be coming through here as we have to
* listen on the OS socket too! */
ci_tcp_state* ts;
ci_tcp_socket_listen* tls;
ci_netif* netif = ep->netif;
ci_sock_cmn* s = ep->s;
unsigned ul_backlog = backlog;
int rc;
oo_p sp;
LOG_TC(log("%s "SK_FMT" listen backlog=%d", __FUNCTION__, SK_PRI_ARGS(ep),
backlog));
CHECK_TEP(ep);
if( NI_OPTS(netif).tcp_listen_handover )
return CI_SOCKET_HANDOVER;
if( !NI_OPTS(netif).tcp_server_loopback) {
/* We should handover if the socket is bound to alien address. */
if( s->s_flags & CI_SOCK_FLAG_BOUND_ALIEN )
return CI_SOCKET_HANDOVER;
}
if( ul_backlog < 0 )
ul_backlog = NI_OPTS(netif).max_ep_bufs;
else if( ul_backlog < NI_OPTS(netif).acceptq_min_backlog )
ul_backlog = NI_OPTS(netif).acceptq_min_backlog;
if( s->b.state == CI_TCP_LISTEN ) {
tls = SOCK_TO_TCP_LISTEN(s);
tls->acceptq_max = ul_backlog;
ci_tcp_helper_listen_os_sock(fd, ul_backlog);
return 0;
}
if( s->b.state != CI_TCP_CLOSED ) {
CI_SET_ERROR(rc, EINVAL);
return rc;
}
ts = SOCK_TO_TCP(s);
/* Bug 3376: if socket used for a previous, failed, connect then the error
* numbers will not be as expected. Only seen when not using listening
* netifs (as moving the EP to the new netif resets them).
*/
ts->s.tx_errno = EPIPE;
ts->s.rx_errno = ENOTCONN;
/* fill in address/ports and all TCP state */
if( !(ts->s.s_flags & CI_SOCK_FLAG_BOUND) ) {
ci_uint16 source_be16;
/* They haven't previously done a bind, so we need to choose
* a port. As we haven't been given a hint we let the OS choose. */
source_be16 = 0;
rc = __ci_bind(ep->netif, ep->s, ts->s.pkt.ip.ip_saddr_be32, &source_be16);
if (CI_LIKELY( rc==0 )) {
TS_TCP(ts)->tcp_source_be16 = source_be16;
ts->s.cp.lport_be16 = source_be16;
LOG_TC(log(LNT_FMT "listen: our bind returned %s:%u",
LNT_PRI_ARGS(ep->netif, ts),
ip_addr_str(ts->s.pkt.ip.ip_saddr_be32),
(unsigned) CI_BSWAP_BE16(TS_TCP(ts)->tcp_source_be16)));
} else {
LOG_U(ci_log("__ci_bind returned %d at %s:%d", CI_GET_ERROR(rc),
__FILE__, __LINE__));
return rc;
}
}
ci_sock_lock(netif, &ts->s.b);
ci_tcp_set_slow_state(netif, ts, CI_TCP_LISTEN);
tls = SOCK_TO_TCP_LISTEN(&ts->s);
tcp_raddr_be32(tls) = 0u;
tcp_rport_be16(tls) = 0u;
ci_assert_equal(tls->s.tx_errno, EPIPE);
ci_assert_equal(tls->s.rx_errno, ENOTCONN);
/* setup listen timer - do it before the first return statement,
* because __ci_tcp_listen_to_normal() will be called on error path. */
if( ~tls->s.s_flags & CI_SOCK_FLAG_BOUND_ALIEN ) {
sp = TS_OFF(netif, tls);
OO_P_ADD(sp, CI_MEMBER_OFFSET(ci_tcp_socket_listen, listenq_tid));
ci_ip_timer_init(netif, &tls->listenq_tid, sp, "lstq");
tls->listenq_tid.param1 = S_SP(tls);
tls->listenq_tid.fn = CI_IP_TIMER_TCP_LISTEN;
}
rc = ci_tcp_listen_init(netif, tls);
ci_sock_unlock(netif, &ts->s.b);
if( rc != 0 ) {
CI_SET_ERROR(rc, -rc);
goto listen_fail;
}
tls->acceptq_max = ul_backlog;
CITP_STATS_TCP_LISTEN(CI_ZERO(&tls->stats));
/* install all the filters needed for this connection
* - tcp_laddr_be32(ts) = 0 for IPADDR_ANY
*
* TODO: handle BINDTODEVICE by setting phys_port paramter to correct
* physical L5 port index
* TODO: handle REUSEADDR by setting last paramter to TRUE
*/
if( ~s->s_flags & CI_SOCK_FLAG_BOUND_ALIEN ) {
#ifdef ONLOAD_OFE
if( netif->ofe != NULL ) {
tls->s.ofe_code_start = ofe_socktbl_find(
netif->ofe, OFE_SOCKTYPE_TCP_LISTEN,
tcp_laddr_be32(tls), INADDR_ANY,
tcp_lport_be16(ts), 0);
tls->ofe_promote = ofe_socktbl_find(
netif->ofe, OFE_SOCKTYPE_TCP_PASSIVE,
tcp_laddr_be32(tls), INADDR_ANY,
tcp_lport_be16(ts), 0);
}
#endif
rc = ci_tcp_ep_set_filters(netif, S_SP(tls), tls->s.cp.so_bindtodevice,
OO_SP_NULL);
if( rc == -EFILTERSSOME ) {
if( CITP_OPTS.no_fail )
rc = 0;
else {
ci_tcp_ep_clear_filters(netif, S_SP(tls), 0);
rc = -ENOBUFS;
}
}
ci_assert_nequal(rc, -EFILTERSSOME);
VERB(ci_log("%s: set_filters returned %d", __FUNCTION__, rc));
if (rc < 0) {
CI_SET_ERROR(rc, -rc);
goto post_listen_fail;
}
}
/*
* Call of system listen() is required for listen any, local host
* communications server and multi-homed server (to accept connections
* to L5 assigned address(es), but incoming from other interfaces).
*/
#ifdef __ci_driver__
{
rc = efab_tcp_helper_listen_os_sock( netif2tcp_helper_resource(netif),
S_SP(tls), backlog);
}
#else
rc = ci_tcp_helper_listen_os_sock(fd, backlog);
#endif
if ( rc < 0 ) {
/* clear the filter we've just set */
ci_tcp_ep_clear_filters(netif, S_SP(tls), 0);
goto post_listen_fail;
}
return 0;
post_listen_fail:
ci_tcp_listenq_drop_all(netif, tls);
listen_fail:
/* revert TCP state to a non-listening socket format */
__ci_tcp_listen_to_normal(netif, tls);
/* Above function sets orphan flag but we are attached to an FD. */
ci_bit_clear(&tls->s.b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
#ifdef __ci_driver__
return rc;
#else
return CI_SOCKET_ERROR;
#endif
}
