void
ci_netif_filter_remove(ci_netif* netif, oo_sp sock_p,
unsigned laddr, unsigned lport,
unsigned raddr, unsigned rport, unsigned protocol)
{
ci_netif_filter_table_entry* entry;
unsigned hash1, hash2, tbl_i;
ci_netif_filter_table* tbl;
int hops = 0;
unsigned first;
ci_assert(ci_netif_is_locked(netif)
#ifdef __KERNEL__
/* release_ep_tbl might be called without the stack lock.
* Do not complain about this. */
|| (netif2tcp_helper_resource(netif)->k_ref_count &
TCP_HELPER_K_RC_DEAD)
#endif
);
tbl = netif->filter_table;
hash1 = tcp_hash1(tbl, laddr, lport, raddr, rport, protocol);
hash2 = tcp_hash2(tbl, laddr, lport, raddr, rport, protocol);
first = hash1;
LOG_TC(ci_log("%s: [%d:%d] REMOVE %s %s:%u->%s:%u hash=%u:%u",
__FUNCTION__, NI_ID(netif), OO_SP_FMT(sock_p),
CI_IP_PROTOCOL_STR(protocol),
ip_addr_str(laddr), (unsigned) CI_BSWAP_BE16(lport),
ip_addr_str(raddr), (unsigned) CI_BSWAP_BE16(rport),
hash1, hash2));
tbl_i = hash1;
while( 1 ) {
entry = &tbl->table[tbl_i];
if( entry->id == OO_SP_TO_INT(sock_p) ) {
if( laddr == entry->laddr )
break;
}
else if( entry->id == EMPTY ) {
/* We allow multiple removes of the same filter -- helps avoid some
* complexity in the filter module.
*/
return;
}
tbl_i = (tbl_i + hash2) & tbl->table_size_mask;
++hops;
if( tbl_i == first ) {
LOG_E(ci_log(FN_FMT "ERROR: LOOP [%d] %s %s:%u->%s:%u",
FN_PRI_ARGS(netif), OO_SP_FMT(sock_p),
CI_IP_PROTOCOL_STR(protocol),
ip_addr_str(laddr), (unsigned) CI_BSWAP_BE16(lport),
ip_addr_str(raddr), (unsigned) CI_BSWAP_BE16(rport)));
return;
}
}
__ci_netif_filter_remove(netif, hash1, hash2, hops, tbl_i);
}
int oo_os_sock_recvmsg(ci_netif* ni, oo_sp sock_p,
struct msghdr* msg, int flags)
{
oo_os_sock_recvmsg_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg_iov, msg->msg_iov);
op.msg_iovlen = msg->msg_iovlen;
CI_USER_PTR_SET(op.msg_name, msg->msg_name);
op.msg_namelen = msg->msg_namelen;
CI_USER_PTR_SET(op.msg_control, msg->msg_control);
op.msg_controllen = msg->msg_controllen;
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_RECVMSG, &op);
ci_assert(op.rc >= 0 || rc < 0);
if( rc == 0 ) {
msg->msg_flags = op.flags;
msg->msg_namelen = op.msg_namelen;
if( msg->msg_controllen )
msg->msg_controllen = op.msg_controllen;
return op.rc;
}
return rc;
}
void citp_waitable_obj_free_nnl(ci_netif* ni, citp_waitable* w)
{
/* Stack lock is probably not held (but not guaranteed). */
__citp_waitable_obj_free(ni, w);
do
w->next_id = ni->state->deferred_free_eps_head;
while( ci_cas32_fail(&ni->state->deferred_free_eps_head,
w->next_id, OO_SP_TO_INT(W_SP(w))) );
/* Must be last, as may result in stack going away. */
ci_drop_orphan(ni);
}
int oo_os_sock_get(ci_netif* ni, oo_sp sock_p, oo_os_file* os_sock_out)
{
oo_os_sock_fd_get_t op;
int rc;
oo_rwlock_lock_read(&citp_dup2_lock);
op.sock_id = OO_SP_TO_INT(sock_p);
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_FD_GET, &op);
if( rc == 0 )
*os_sock_out = op.fd_out;
else
oo_rwlock_unlock_read (&citp_dup2_lock);
return rc;
}
int oo_os_sock_accept(ci_netif* ni, oo_sp sock_p, struct sockaddr *addr,
socklen_t *addrlen, int flags)
{
oo_os_sock_accept_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
CI_USER_PTR_SET(op.addr, addr);
CI_USER_PTR_SET(op.addrlen, addrlen);
op.flags = flags;
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_ACCEPT, &op);
return rc == 0 ? op.rc : rc;
}
int oo_os_sock_get(ci_netif* ni, oo_sp sock_p, oo_os_file* os_sock_out)
{
int sock_id = OO_SP_TO_INT(sock_p);
tcp_helper_endpoint_t* ep;
if( sock_id != TRUSTED_SOCK_ID(ni, sock_id) ) {
LOG_E(ci_log("%s: ERROR: %d:%d bad sock_id",
__FUNCTION__, NI_ID(ni), sock_id));
return -EINVAL;
}
ep = ci_netif_ep_get(ni, sock_p);
if( oo_os_sock_get_from_ep(ep, os_sock_out) == 0 )
return 0;
LOG_E(ci_log("%s: ERROR: %d:%d has no O/S socket",
__FUNCTION__, NI_ID(ni), sock_id));
return -ENOENT;
}
int oo_os_sock_sendmsg_raw(ci_netif* ni, oo_sp sock_p,
const struct msghdr* msg, int flags)
{
unsigned long socketcall_args[8];
oo_os_sock_sendmsg_raw_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg, msg);
CI_USER_PTR_SET(op.socketcall_args, socketcall_args);
oo_rwlock_lock_read(&citp_dup2_lock);
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_SENDMSG_RAW, &op);
oo_rwlock_unlock_read (&citp_dup2_lock);
return rc;
}
int oo_os_sock_sendmsg(ci_netif* ni, oo_sp sock_p,
const struct msghdr* msg, int flags)
{
oo_os_sock_sendmsg_t op;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg_iov, msg->msg_iov);
op.msg_iovlen = msg->msg_iovlen;
CI_USER_PTR_SET(op.msg_name, msg->msg_name);
op.msg_namelen = msg->msg_namelen;
#ifdef __i386__
/* compat cmsg is not handled in this function */
ci_assert_equal(msg->msg_controllen, 0);
op.msg_controllen = 0;
CI_USER_PTR_SET(op.msg_control, NULL);
#else
CI_USER_PTR_SET(op.msg_control, msg->msg_control);
op.msg_controllen = msg->msg_controllen;
#endif
return oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_SENDMSG, &op);
}
/* Insert for either TCP or UDP */
int ci_netif_filter_insert(ci_netif* netif, oo_sp tcp_id,
unsigned laddr, unsigned lport,
unsigned raddr, unsigned rport, unsigned protocol)
{
ci_netif_filter_table_entry* entry;
unsigned hash1, hash2;
ci_netif_filter_table* tbl;
#if !defined(NDEBUG) || CI_CFG_STATS_NETIF
unsigned hops = 1;
#endif
unsigned first;
ci_assert(netif);
ci_assert(ci_netif_is_locked(netif));
ci_assert(netif->filter_table);
tbl = netif->filter_table;
hash1 = tcp_hash1(tbl, laddr, lport, raddr, rport, protocol);
hash2 = tcp_hash2(tbl, laddr, lport, raddr, rport, protocol);
first = hash1;
/* Find a free slot. */
while( 1 ) {
entry = &tbl->table[hash1];
if( entry->id < 0 ) break;
++entry->route_count;
#if !defined(NDEBUG) || CI_CFG_STATS_NETIF
++hops;
#endif
/* A socket can only have multiple entries in the filter table if each
* entry has a different [laddr].
*/
ci_assert(
!((entry->id == OO_SP_TO_INT(tcp_id)) && (laddr == entry->laddr)) );
hash1 = (hash1 + hash2) & tbl->table_size_mask;
if( hash1 == first ) {
ci_sock_cmn *s = SP_TO_SOCK_CMN(netif, tcp_id);
if( ! (s->s_flags & CI_SOCK_FLAG_SW_FILTER_FULL) ) {
LOG_E(ci_log(FN_FMT "%d FULL %s %s:%u->%s:%u hops=%u",
FN_PRI_ARGS(netif),
OO_SP_FMT(tcp_id), CI_IP_PROTOCOL_STR(protocol),
ip_addr_str(laddr), (unsigned) CI_BSWAP_BE16(lport),
ip_addr_str(raddr), (unsigned) CI_BSWAP_BE16(rport),
hops));
s->s_flags |= CI_SOCK_FLAG_SW_FILTER_FULL;
}
CITP_STATS_NETIF_INC(netif, sw_filter_insert_table_full);
return -ENOBUFS;
}
}
/* Now insert the new entry. */
LOG_TC(ci_log(FN_FMT "%d INSERT %s %s:%u->%s:%u hash=%u:%u at=%u "
"over=%d hops=%u", FN_PRI_ARGS(netif), OO_SP_FMT(tcp_id),
CI_IP_PROTOCOL_STR(protocol),
ip_addr_str(laddr), (unsigned) CI_BSWAP_BE16(lport),
ip_addr_str(raddr), (unsigned) CI_BSWAP_BE16(rport),
first, hash2, hash1, entry->id, hops));
#if CI_CFG_STATS_NETIF
if( hops > netif->state->stats.table_max_hops )
netif->state->stats.table_max_hops = hops;
/* Keep a rolling average of the number of hops per entry. */
if( netif->state->stats.table_mean_hops == 0 )
netif->state->stats.table_mean_hops = 1;
netif->state->stats.table_mean_hops =
(netif->state->stats.table_mean_hops * 9 + hops) / 10;
if( entry->id == EMPTY )
++netif->state->stats.table_n_slots;
++netif->state->stats.table_n_entries;
#endif
entry->id = OO_SP_TO_INT(tcp_id);
entry->laddr = laddr;
return 0;
}
/* c_ni is assumed to be locked on enterance and is always unlocked on
* exit. */
int ci_tcp_connect_lo_toconn(ci_netif *c_ni, oo_sp c_id, ci_uint32 dst,
ci_netif *l_ni, oo_sp l_id)
{
ci_tcp_state *ts;
ci_tcp_socket_listen *tls, *alien_tls;
citp_waitable_obj *wo;
citp_waitable *w;
int rc;
ci_assert(ci_netif_is_locked(c_ni));
ci_assert(OO_SP_NOT_NULL(c_id));
ci_assert(OO_SP_NOT_NULL(l_id));
LOG_TC(log("%s: connect %d:%d to %d:%d", __FUNCTION__,
c_ni->state->stack_id, OO_SP_TO_INT(c_id),
l_ni->state->stack_id, OO_SP_TO_INT(l_id)));
alien_tls = SP_TO_TCP_LISTEN(l_ni, l_id);
if( (int)ci_tcp_acceptq_n(alien_tls) >= alien_tls->acceptq_max ) {
ci_netif_unlock(c_ni);
return -EBUSY;
}
/* In c_ni, create shadow listening socket tls (copy l_id) */
ts = ci_tcp_get_state_buf(c_ni);
if( ts == NULL ) {
ci_netif_unlock(c_ni);
LOG_E(ci_log("%s: [%d] out of socket buffers", __FUNCTION__, NI_ID(c_ni)));
return -ENOMEM;
}
/* init common tcp fields */
ts->s.so = alien_tls->s.so;
ts->s.cp.ip_ttl = alien_tls->s.cp.ip_ttl;
S_TCP_HDR(&ts->s)->tcp_source_be16 =
S_TCP_HDR(&alien_tls->s)->tcp_source_be16;
ts->s.domain = alien_tls->s.domain;
ts->c = alien_tls->c;
ts->c.tcp_defer_accept = OO_TCP_DEFER_ACCEPT_OFF;
/* make sure nobody will ever connect to our "shadow" socket
* except us */
ci_bit_set(&ts->s.b.sb_aflags, CI_SB_AFLAG_ORPHAN_BIT);
ci_tcp_set_slow_state(c_ni, ts, CI_TCP_LISTEN);
tls = SOCK_TO_TCP_LISTEN(&ts->s);
/* no timer: */
tls->s.s_flags = alien_tls->s.s_flags | CI_SOCK_FLAG_BOUND_ALIEN;
tls->acceptq_max = 1;
rc = ci_tcp_listen_init(c_ni, tls);
if( rc != 0 ) {
citp_waitable_obj_free(c_ni, &tls->s.b);
return rc;
}
/* Connect c_id to tls */
ts = SP_TO_TCP(c_ni, c_id);
rc = ci_tcp_connect_lo_samestack(c_ni, ts, tls->s.b.bufid);
/* Accept as from tls */
if( !ci_tcp_acceptq_not_empty(tls) ) {
/* it is possible, for example, if ci_tcp_listenq_try_promote() failed
* because there are no endpoints */
ci_tcp_listenq_drop_all(c_ni, tls);
citp_waitable_obj_free(c_ni, &tls->s.b);
ci_netif_unlock(c_ni);
return -EBUSY;
}
w = ci_tcp_acceptq_get(c_ni, tls);
ci_assert(w);
LOG_TV(ci_log("%s: %d:%d to %d:%d shadow %d:%d accepted %d:%d",
__FUNCTION__,
c_ni->state->stack_id, OO_SP_TO_INT(c_id),
l_ni->state->stack_id, OO_SP_TO_INT(l_id),
c_ni->state->stack_id, tls->s.b.bufid,
c_ni->state->stack_id, w->bufid));
ci_assert(w->state & CI_TCP_STATE_TCP);
ci_assert(w->state != CI_TCP_LISTEN);
/* Destroy tls.
* NB: nobody could possibly connect to it, so no need to do proper
* shutdown.
*/
ci_assert_equal(ci_tcp_acceptq_n(tls), 0);
ci_tcp_listenq_drop_all(c_ni, tls);
citp_waitable_obj_free(c_ni, &tls->s.b);
ci_netif_unlock(c_ni);
/* Keep a port reference */
{
tcp_helper_endpoint_t *l_ep, *a_ep;
struct oo_file_ref* os_sock_ref;
ci_irqlock_state_t lock_flags;
l_ep = ci_trs_ep_get(netif2tcp_helper_resource(l_ni), l_id);
a_ep = ci_trs_ep_get(netif2tcp_helper_resource(c_ni), W_SP(w));
ci_irqlock_lock(&l_ep->thr->lock, &lock_flags);
os_sock_ref = l_ep->os_socket;
ci_assert_equal(a_ep->os_port_keeper, NULL);
if( os_sock_ref != NULL ) {
os_sock_ref = oo_file_ref_add(os_sock_ref);
os_sock_ref = oo_file_ref_xchg(&a_ep->os_port_keeper, os_sock_ref);
ci_irqlock_unlock(&l_ep->thr->lock, &lock_flags);
if( os_sock_ref != NULL )
oo_file_ref_drop(os_sock_ref);
}
else {
ci_irqlock_unlock(&l_ep->thr->lock, &lock_flags);
goto cleanup;
}
}
/* lock l_ni: Check that l_id is the same socket it used to be */
/* create ref-sock in l_ni, put it into acc q */
if( ci_netif_lock(l_ni) != 0 )
goto cleanup;
if( alien_tls->s.b.state != CI_TCP_LISTEN ||
(alien_tls->s.b.sb_aflags & CI_SB_AFLAG_ORPHAN) ||
S_TCP_HDR(&alien_tls->s)->tcp_source_be16 != TS_TCP(ts)->tcp_dest_be16 ||
(alien_tls->s.pkt.ip.ip_saddr_be32 != INADDR_ANY &&
alien_tls->s.pkt.ip.ip_saddr_be32 != ts->s.pkt.ip.ip_daddr_be32) ) {
ci_netif_unlock(l_ni);
goto cleanup;
}
ci_bit_mask_set(&w->sb_aflags,
CI_SB_AFLAG_TCP_IN_ACCEPTQ | CI_SB_AFLAG_ORPHAN);
wo = citp_waitable_obj_alloc(l_ni);
if( wo == NULL ) {
ci_netif_unlock(l_ni);
goto cleanup;
}
wo->waitable.state = CI_TCP_CLOSED;
wo->waitable.sb_aflags |= CI_SB_AFLAG_MOVED_AWAY;
wo->waitable.moved_to_stack_id = c_ni->state->stack_id;
wo->waitable.moved_to_sock_id = W_SP(w);
LOG_TC(log("%s: put to acceptq %d:%d referencing %d:%d", __func__,
l_ni->state->stack_id, OO_SP_TO_INT(W_SP(&wo->waitable)),
c_ni->state->stack_id, OO_SP_TO_INT(W_SP(w))));
ci_tcp_acceptq_put(l_ni, alien_tls, &wo->waitable);
citp_waitable_wake_not_in_poll(l_ni, &alien_tls->s.b, CI_SB_FLAG_WAKE_RX);
ci_netif_unlock(l_ni);
return rc;
cleanup:
ci_assert(w->sb_aflags & CI_SB_AFLAG_ORPHAN);
ci_bit_mask_clear(&w->sb_aflags,
CI_SB_AFLAG_TCP_IN_ACCEPTQ | CI_SB_AFLAG_ORPHAN);
efab_tcp_helper_close_endpoint(netif2tcp_helper_resource(c_ni), w->bufid);
/* we can not guarantee c_ni lock, so we can' call
* ci_tcp_drop(c_ni, ts). So, we return error; UL will handover
* and close ts endpoint. */
return -EBUSY;
}