static int
efab_tcp_drop_from_acceptq(ci_private_t *priv, void *arg)
{
struct oo_op_tcp_drop_from_acceptq *carg = arg;
tcp_helper_resource_t *thr;
tcp_helper_endpoint_t *ep;
citp_waitable *w;
ci_tcp_state *ts;
int rc = -EINVAL;
/* find stack */
rc = efab_thr_table_lookup(NULL, carg->stack_id,
EFAB_THR_TABLE_LOOKUP_CHECK_USER |
EFAB_THR_TABLE_LOOKUP_NO_UL,
&thr);
if( rc < 0 )
return rc;
ci_assert( thr->k_ref_count & TCP_HELPER_K_RC_NO_USERLAND );
/* find endpoint and drop OS socket */
ep = ci_trs_get_valid_ep(thr, carg->sock_id);
if( ep == NULL )
goto fail1;
w = SP_TO_WAITABLE(&thr->netif, carg->sock_id);
if( !(w->state & CI_TCP_STATE_TCP) || w->state == CI_TCP_LISTEN )
goto fail2;
ts = SP_TO_TCP(&thr->netif, carg->sock_id);
ci_assert(ep->os_port_keeper);
ci_assert_equal(ep->os_socket, NULL);
LOG_TV(ci_log("%s: send reset to non-accepted connection", __FUNCTION__));
/* copy from ci_tcp_listen_shutdown_queues() */
ci_assert(ts->s.b.sb_aflags & CI_SB_AFLAG_TCP_IN_ACCEPTQ);
rc = ci_netif_lock(&thr->netif);
if( rc != 0 ) {
ci_assert_equal(rc, -EINTR);
rc = -ERESTARTSYS;
goto fail2;
}
ci_bit_clear(&ts->s.b.sb_aflags, CI_SB_AFLAG_TCP_IN_ACCEPTQ_BIT);
/* We have no way to close this connection from the other side:
* there was no RST from peer. */
ci_assert_nequal(ts->s.b.state, CI_TCP_CLOSED);
ci_assert_nequal(ts->s.b.state, CI_TCP_TIME_WAIT);
ci_tcp_send_rst(&thr->netif, ts);
ci_tcp_drop(&thr->netif, ts, ECONNRESET);
ci_assert_equal(ep->os_port_keeper, NULL);
ci_netif_unlock(&thr->netif);
efab_tcp_helper_k_ref_count_dec(thr, 1);
return 0;
fail1:
efab_thr_release(thr);
fail2:
ci_log("%s: inconsistent ep %d:%d", __func__, carg->stack_id, carg->sock_id);
return rc;
}
/* unpick the ci_ip_timer structure to actually do the callback */
static void ci_ip_timer_docallback(ci_netif *netif, ci_ip_timer* ts)
{
ci_assert( TIME_LE(ts->time, ci_ip_time_now(netif)) );
ci_assert( ts->time == IPTIMER_STATE(netif)->sched_ticks );
switch(ts->fn){
case CI_IP_TIMER_TCP_RTO:
CHECK_TS(netif, SP_TO_TCP(netif, ts->param1));
ci_tcp_timeout_rto(netif, SP_TO_TCP(netif, ts->param1));
break;
case CI_IP_TIMER_TCP_DELACK:
CHECK_TS(netif, SP_TO_TCP(netif, ts->param1));
ci_tcp_timeout_delack(netif, SP_TO_TCP(netif, ts->param1));
break;
case CI_IP_TIMER_TCP_ZWIN:
CHECK_TS(netif, SP_TO_TCP(netif, ts->param1));
ci_tcp_timeout_zwin(netif, SP_TO_TCP(netif, ts->param1));
break;
case CI_IP_TIMER_TCP_KALIVE:
CHECK_TS(netif, SP_TO_TCP(netif, ts->param1));
ci_tcp_timeout_kalive(netif, SP_TO_TCP(netif, ts->param1));
break;
case CI_IP_TIMER_TCP_LISTEN:
ci_tcp_timeout_listen(netif, SP_TO_TCP_LISTEN(netif, ts->param1));
break;
case CI_IP_TIMER_TCP_CORK:
ci_tcp_timeout_cork(netif, SP_TO_TCP(netif, ts->param1));
break;
case CI_IP_TIMER_NETIF_TIMEOUT:
ci_netif_timeout_state(netif);
break;
case CI_IP_TIMER_PMTU_DISCOVER:
ci_pmtu_timeout_pmtu(netif, SP_TO_TCP(netif, ts->param1));
break;
#if CI_CFG_TCP_SOCK_STATS
case CI_IP_TIMER_TCP_STATS:
ci_tcp_stats_action(netif, SP_TO_TCP(netif, ts->param1),
CI_IP_STATS_FLUSH,
CI_IP_STATS_OUTPUT_NONE, NULL, NULL );
break;
#endif
#if CI_CFG_SUPPORT_STATS_COLLECTION
case CI_IP_TIMER_NETIF_STATS:
ci_netif_stats_action(netif, CI_IP_STATS_FLUSH,
CI_IP_STATS_OUTPUT_NONE, NULL, NULL );
break;
#endif
#if CI_CFG_IP_TIMER_DEBUG
case CI_IP_TIMER_DEBUG_HOOK:
ci_ip_timer_debug_fn(netif, ts->link.addr, ts->param1);
break;
#endif
default:
LOG_U(log( LPF "unknown timer callback code:%x param1:%d",
ts->fn, OO_SP_FMT(ts->param1)));
CI_DEBUG(ci_fail_stop_fn());
}
}
/* Locking policy:
* Enterance: priv->thr->netif is assumed to be locked.
* Exit: all stacks (the client stack and the listener's stack) are
* unlocked.
*/
int efab_tcp_loopback_connect(ci_private_t *priv, void *arg)
{
struct oo_op_loopback_connect *carg = arg;
ci_netif *alien_ni = NULL;
oo_sp tls_id;
ci_assert(ci_netif_is_locked(&priv->thr->netif));
carg->out_moved = 0;
if( !CI_PRIV_TYPE_IS_ENDPOINT(priv->fd_type) )
return -EINVAL;
if( NI_OPTS(&priv->thr->netif).tcp_client_loopback !=
CITP_TCP_LOOPBACK_TO_CONNSTACK &&
NI_OPTS(&priv->thr->netif).tcp_client_loopback !=
CITP_TCP_LOOPBACK_TO_LISTSTACK &&
NI_OPTS(&priv->thr->netif).tcp_client_loopback !=
CITP_TCP_LOOPBACK_TO_NEWSTACK) {
ci_netif_unlock(&priv->thr->netif);
return -EINVAL;
}
while( iterate_netifs_unlocked(&alien_ni) == 0 ) {
if( !efab_thr_can_access_stack(netif2tcp_helper_resource(alien_ni),
EFAB_THR_TABLE_LOOKUP_CHECK_USER) )
continue; /* no permission to look in here */
if( NI_OPTS(alien_ni).tcp_server_loopback == CITP_TCP_LOOPBACK_OFF )
continue; /* server does not accept loopback connections */
if( NI_OPTS(&priv->thr->netif).tcp_client_loopback !=
CITP_TCP_LOOPBACK_TO_LISTSTACK &&
NI_OPTS(alien_ni).tcp_server_loopback !=
CITP_TCP_LOOPBACK_ALLOW_ALIEN_IN_ACCEPTQ )
continue; /* options of the stacks to not match */
if( NI_OPTS(&priv->thr->netif).tcp_client_loopback !=
CITP_TCP_LOOPBACK_TO_LISTSTACK &&
!efab_thr_user_can_access_stack(alien_ni->uid, alien_ni->euid,
&priv->thr->netif) )
continue; /* server can't accept our socket */
tls_id = ci_tcp_connect_find_local_peer(alien_ni, carg->dst_addr,
carg->dst_port);
if( OO_SP_NOT_NULL(tls_id) ) {
int rc;
/* We are going to exit in this or other way: get ref and
* drop kref of alien_ni */
efab_thr_ref(netif2tcp_helper_resource(alien_ni));
iterate_netifs_unlocked_dropref(alien_ni);
switch( NI_OPTS(&priv->thr->netif).tcp_client_loopback ) {
case CITP_TCP_LOOPBACK_TO_CONNSTACK:
/* connect_lo_toconn unlocks priv->thr->netif */
carg->out_rc =
ci_tcp_connect_lo_toconn(&priv->thr->netif, priv->sock_id,
carg->dst_addr, alien_ni, tls_id);
efab_thr_release(netif2tcp_helper_resource(alien_ni));
return 0;
case CITP_TCP_LOOPBACK_TO_LISTSTACK:
/* Nobody should be using this socket, so trylock should succeed.
* Overwise we hand over the socket and do not accelerate this
* loopback connection. */
rc = ci_sock_trylock(&priv->thr->netif,
SP_TO_WAITABLE(&priv->thr->netif,
priv->sock_id));
if( rc == 0 ) {
ci_netif_unlock(&priv->thr->netif);
efab_thr_release(netif2tcp_helper_resource(alien_ni));
return -ECONNREFUSED;
}
/* move_to_alien changes locks - see comments near it */
rc = efab_file_move_to_alien_stack(priv, alien_ni);
if( rc != 0 ) {
/* error - everything is already unlocked */
efab_thr_release(netif2tcp_helper_resource(alien_ni));
/* if we return error, UL will hand the socket over. */
return rc;
}
/* now alien_ni is locked */
/* Connect again, using new endpoint */
carg->out_rc =
ci_tcp_connect_lo_samestack(
alien_ni,
SP_TO_TCP(alien_ni,
SP_TO_WAITABLE(&priv->thr->netif,
priv->sock_id)
->moved_to_sock_id),
tls_id);
ci_netif_unlock(alien_ni);
carg->out_moved = 1;
return 0;
case CITP_TCP_LOOPBACK_TO_NEWSTACK:
{
tcp_helper_resource_t *new_thr;
ci_resource_onload_alloc_t alloc;
/* create new stack
* todo: no hardware interfaces are necessary */
strcpy(alloc.in_version, ONLOAD_VERSION);
strcpy(alloc.in_uk_intf_ver, oo_uk_intf_ver);
alloc.in_name[0] = '\0';
alloc.in_flags = 0;
rc = tcp_helper_alloc_kernel(&alloc, &NI_OPTS(&priv->thr->netif), 0,
&new_thr);
if( rc != 0 ) {
ci_netif_unlock(&priv->thr->netif);
efab_thr_release(netif2tcp_helper_resource(alien_ni));
return -ECONNREFUSED;
}
rc = ci_sock_trylock(&priv->thr->netif,
SP_TO_WAITABLE(&priv->thr->netif,
priv->sock_id));
if( rc == 0 ) {
ci_netif_unlock(&priv->thr->netif);
efab_thr_release(netif2tcp_helper_resource(alien_ni));
efab_thr_release(new_thr);
return -ECONNREFUSED;
}
/* move connecting socket to the new stack */
rc = efab_file_move_to_alien_stack(priv, &new_thr->netif);
if( rc != 0 ) {
/* error - everything is already unlocked */
efab_thr_release(netif2tcp_helper_resource(alien_ni));
efab_thr_release(new_thr);
return -ECONNREFUSED;
}
/* now new_thr->netif is locked */
carg->out_moved = 1;
carg->out_rc = -ECONNREFUSED;
/* now connect via CITP_TCP_LOOPBACK_TO_CONNSTACK */
/* connect_lo_toconn unlocks new_thr->netif */
carg->out_rc =
ci_tcp_connect_lo_toconn(
&new_thr->netif,
SP_TO_WAITABLE(&priv->thr->netif,
priv->sock_id)->moved_to_sock_id,
carg->dst_addr, alien_ni, tls_id);
efab_thr_release(netif2tcp_helper_resource(alien_ni));
return 0;
}
}
}
else if( tls_id == OO_SP_INVALID )
break;
}
ci_netif_unlock(&priv->thr->netif);
return -ENOENT;
}
static int
efab_tcp_helper_move_state(ci_private_t* priv, void *arg)
{
oo_tcp_move_state_t *op = arg;
tcp_helper_endpoint_t *new_ep;
tcp_helper_resource_t * new_trs = NULL;
ci_netif* ni, *new_ni;
ci_tcp_state * ts, *new_ts;
tcp_helper_endpoint_t* ep;
int rc = efab_ioctl_get_ep(priv, op->ep_id, &ep);
if (rc != 0)
return rc;
OO_DEBUG_TCPH(ci_log("%s: (trs=%p (%u), priv=%p, ep_id=%u, new_trs_id=%u, "
"new_ep_id=%u", __FUNCTION__, priv->thr, priv->thr->id,
priv, OO_SP_FMT(op->ep_id), op->new_trs_id,
OO_SP_FMT(op->new_ep_id)));
do {
/* check that the existing id is valid */
ni = &priv->thr->netif;
ts = SP_TO_TCP(ni, ep->id);
/* TODO: check this endpoint belongs to the tcp helper resource of priv and not
* somewhere else */
/* this function does not change fd_type or fd ops, so it is not able
* to cope with changing the socket type. We think this only makes sense
* for TCP, so assert we are taking a TCP endpoint.
*/
ci_assert_equal(ts->s.pkt.ip.ip_protocol, IPPROTO_TCP);
ci_assert_equal(priv->fd_type, CI_PRIV_TYPE_TCP_EP);
/* get pointer to resource from handle - increments ref count */
rc = efab_thr_table_lookup(NULL, op->new_trs_id,
EFAB_THR_TABLE_LOOKUP_CHECK_USER, &new_trs);
if (rc < 0) {
OO_DEBUG_ERR( ci_log("%s: invalid new resource handle", __FUNCTION__) );
break;
}
ci_assert(new_trs != NULL);
/* check valid endpoint in new netif */
new_ni = &new_trs->netif;
new_ep = ci_netif_get_valid_ep(new_ni, op->new_ep_id);
new_ts = SP_TO_TCP(new_ni, new_ep->id);
/* check the two endpoint states look valid */
if( (ts->s.pkt.ip.ip_protocol != new_ts->s.pkt.ip.ip_protocol) ||
(ts->s.b.state != CI_TCP_CLOSED) ||
(ep->oofilter.sf_local_port != NULL) ) {
efab_thr_release(new_trs);
rc = -EINVAL;
OO_DEBUG_ERR(ci_log("%s: invalid endpoint states", __FUNCTION__));
break;
}
/* should be fine to complete */
ci_assert(new_trs);
{
tcp_helper_resource_t *old_trs;
again:
old_trs = priv->thr;
if (ci_cas_uintptr_fail((ci_uintptr_t *)&priv->thr,
(ci_uintptr_t)old_trs, (ci_uintptr_t)new_trs))
goto again;
efab_thr_release(old_trs);
}
/* move file to hold details of new resource, new endpoint */
ci_assert(OO_SP_EQ(priv->sock_id, op->ep_id));
priv->sock_id = new_ep->id;
OO_DEBUG_TCPH(ci_log("%s: set epid %u", __FUNCTION__,
OO_SP_FMT(priv->sock_id)));
/* copy across any necessary state */
ci_assert_equal(new_ep->os_socket, NULL);
new_ep->os_socket = ep->os_socket;
ep->os_socket = NULL;
/* set ORPHAN flag in current as not attached to an FD */
ci_bit_set(&ts->s.b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
/* remove ORPHAN flag in new TCP state */
ci_atomic32_and(&new_ts->s.b.sb_aflags,
~(CI_SB_AFLAG_ORPHAN | CI_SB_AFLAG_TCP_IN_ACCEPTQ));
return 0;
} while (0);
return rc;
}
/* c_ni is assumed to be locked on enterance and is always unlocked on
* exit. */
int ci_tcp_connect_lo_toconn(ci_netif *c_ni, oo_sp c_id, ci_uint32 dst,
ci_netif *l_ni, oo_sp l_id)
{
ci_tcp_state *ts;
ci_tcp_socket_listen *tls, *alien_tls;
citp_waitable_obj *wo;
citp_waitable *w;
int rc;
ci_assert(ci_netif_is_locked(c_ni));
ci_assert(OO_SP_NOT_NULL(c_id));
ci_assert(OO_SP_NOT_NULL(l_id));
LOG_TC(log("%s: connect %d:%d to %d:%d", __FUNCTION__,
c_ni->state->stack_id, OO_SP_TO_INT(c_id),
l_ni->state->stack_id, OO_SP_TO_INT(l_id)));
alien_tls = SP_TO_TCP_LISTEN(l_ni, l_id);
if( (int)ci_tcp_acceptq_n(alien_tls) >= alien_tls->acceptq_max ) {
ci_netif_unlock(c_ni);
return -EBUSY;
}
/* In c_ni, create shadow listening socket tls (copy l_id) */
ts = ci_tcp_get_state_buf(c_ni);
if( ts == NULL ) {
ci_netif_unlock(c_ni);
LOG_E(ci_log("%s: [%d] out of socket buffers", __FUNCTION__, NI_ID(c_ni)));
return -ENOMEM;
}
/* init common tcp fields */
ts->s.so = alien_tls->s.so;
ts->s.cp.ip_ttl = alien_tls->s.cp.ip_ttl;
S_TCP_HDR(&ts->s)->tcp_source_be16 =
S_TCP_HDR(&alien_tls->s)->tcp_source_be16;
ts->s.domain = alien_tls->s.domain;
ts->c = alien_tls->c;
ts->c.tcp_defer_accept = OO_TCP_DEFER_ACCEPT_OFF;
/* make sure nobody will ever connect to our "shadow" socket
* except us */
ci_bit_set(&ts->s.b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
ci_tcp_set_slow_state(c_ni, ts, CI_TCP_LISTEN);
tls = SOCK_TO_TCP_LISTEN(&ts->s);
/* no timer: */
tls->s.s_flags = alien_tls->s.s_flags | CI_SOCK_FLAG_BOUND_ALIEN;
tls->acceptq_max = 1;
rc = ci_tcp_listen_init(c_ni, tls);
if( rc != 0 ) {
citp_waitable_obj_free(c_ni, &tls->s.b);
return rc;
}
/* Connect c_id to tls */
ts = SP_TO_TCP(c_ni, c_id);
rc = ci_tcp_connect_lo_samestack(c_ni, ts, tls->s.b.bufid);
/* Accept as from tls */
if( !ci_tcp_acceptq_not_empty(tls) ) {
/* it is possible, for example, if ci_tcp_listenq_try_promote() failed
* because there are no endpoints */
ci_tcp_listenq_drop_all(c_ni, tls);
citp_waitable_obj_free(c_ni, &tls->s.b);
ci_netif_unlock(c_ni);
return -EBUSY;
}
w = ci_tcp_acceptq_get(c_ni, tls);
ci_assert(w);
LOG_TV(ci_log("%s: %d:%d to %d:%d shadow %d:%d accepted %d:%d",
__FUNCTION__,
c_ni->state->stack_id, OO_SP_TO_INT(c_id),
l_ni->state->stack_id, OO_SP_TO_INT(l_id),
c_ni->state->stack_id, tls->s.b.bufid,
c_ni->state->stack_id, w->bufid));
ci_assert(w->state & CI_TCP_STATE_TCP);
ci_assert(w->state != CI_TCP_LISTEN);
/* Destroy tls.
* NB: nobody could possibly connect to it, so no need to do proper
* shutdown.
*/
ci_assert_equal(ci_tcp_acceptq_n(tls), 0);
ci_tcp_listenq_drop_all(c_ni, tls);
citp_waitable_obj_free(c_ni, &tls->s.b);
ci_netif_unlock(c_ni);
/* Keep a port reference */
{
tcp_helper_endpoint_t *l_ep, *a_ep;
struct oo_file_ref* os_sock_ref;
ci_irqlock_state_t lock_flags;
l_ep = ci_trs_ep_get(netif2tcp_helper_resource(l_ni), l_id);
a_ep = ci_trs_ep_get(netif2tcp_helper_resource(c_ni), W_SP(w));
ci_irqlock_lock(&l_ep->thr->lock, &lock_flags);
os_sock_ref = l_ep->os_socket;
ci_assert_equal(a_ep->os_port_keeper, NULL);
if( os_sock_ref != NULL ) {
os_sock_ref = oo_file_ref_add(os_sock_ref);
os_sock_ref = oo_file_ref_xchg(&a_ep->os_port_keeper, os_sock_ref);
ci_irqlock_unlock(&l_ep->thr->lock, &lock_flags);
if( os_sock_ref != NULL )
oo_file_ref_drop(os_sock_ref);
}
else {
ci_irqlock_unlock(&l_ep->thr->lock, &lock_flags);
goto cleanup;
}
}
/* lock l_ni: Check that l_id is the same socket it used to be */
/* create ref-sock in l_ni, put it into acc q */
if( ci_netif_lock(l_ni) != 0 )
goto cleanup;
if( alien_tls->s.b.state != CI_TCP_LISTEN ||
(alien_tls->s.b.sb_aflags & CI_SB_AFLAG_ORPHAN) ||
S_TCP_HDR(&alien_tls->s)->tcp_source_be16 != TS_TCP(ts)->tcp_dest_be16 ||
(alien_tls->s.pkt.ip.ip_saddr_be32 != INADDR_ANY &&
alien_tls->s.pkt.ip.ip_saddr_be32 != ts->s.pkt.ip.ip_daddr_be32) ) {
ci_netif_unlock(l_ni);
goto cleanup;
}
ci_bit_mask_set(&w->sb_aflags,
CI_SB_AFLAG_TCP_IN_ACCEPTQ | CI_SB_AFLAG_ORPHAN);
wo = citp_waitable_obj_alloc(l_ni);
if( wo == NULL ) {
ci_netif_unlock(l_ni);
goto cleanup;
}
wo->waitable.state = CI_TCP_CLOSED;
wo->waitable.sb_aflags |= CI_SB_AFLAG_MOVED_AWAY;
wo->waitable.moved_to_stack_id = c_ni->state->stack_id;
wo->waitable.moved_to_sock_id = W_SP(w);
LOG_TC(log("%s: put to acceptq %d:%d referencing %d:%d", __func__,
l_ni->state->stack_id, OO_SP_TO_INT(W_SP(&wo->waitable)),
c_ni->state->stack_id, OO_SP_TO_INT(W_SP(w))));
ci_tcp_acceptq_put(l_ni, alien_tls, &wo->waitable);
citp_waitable_wake_not_in_poll(l_ni, &alien_tls->s.b, CI_SB_FLAG_WAKE_RX);
ci_netif_unlock(l_ni);
return rc;
cleanup:
ci_assert(w->sb_aflags & CI_SB_AFLAG_ORPHAN);
ci_bit_mask_clear(&w->sb_aflags,
CI_SB_AFLAG_TCP_IN_ACCEPTQ | CI_SB_AFLAG_ORPHAN);
efab_tcp_helper_close_endpoint(netif2tcp_helper_resource(c_ni), w->bufid);
/* we can not guarantee c_ni lock, so we can' call
* ci_tcp_drop(c_ni, ts). So, we return error; UL will handover
* and close ts endpoint. */
return -EBUSY;
}
