/* This function assumes an option of the same form and types as
* EF_TCP_FORCE_REUSEPORT
*/
static void get_env_opt_port_list(ci_uint64* opt, const char* name)
{
char *s;
unsigned v;
if( (s = getenv(name)) ) {
/* The memory used for this list is never freed, as we need it
* persist until the process terminates
*/
*opt = (ci_uint64)(ci_uintptr_t)malloc(sizeof(ci_dllist));
if( ! *opt )
log("Could not allocate memory for %s list", name);
else {
struct ci_port_list *curr;
ci_dllist *opt_list = (ci_dllist*)(ci_uintptr_t)*opt;
ci_dllist_init(opt_list);
while( sscanf(s, "%u", &v) == 1 ) {
curr = malloc(sizeof(struct ci_port_list));
if( ! curr ) {
log("Could not allocate memory for %s list entry", name);
break;
}
curr->port = v;
if( curr->port != v ) {
log("ERROR: %s contains value that is too large: %u", name, v);
free(curr);
}
else {
curr->port = htons(curr->port);
ci_dllist_push(opt_list, &curr->link);
}
s = strchr(s, ',');
if( s == NULL )
break;
s++;
}
}
}
}
static int ci_bonding_get_lacp_active_slave(char* bond_name, char* slave_name)
{
int len, rc;
char* filename;
struct ci_read_proc_net_bonding_state state;
ci_dllist active_slaves;
state.agg_id = -1;
state.current_slave[0] = '\0';
ci_dllist_init(&active_slaves);
len = strlen(bond_name) + PROC_NET_BONDING_STRLEN + 1;
filename = kmalloc(len, GFP_KERNEL);
if ( filename == NULL )
return -ENOMEM;
sprintf(filename, PROC_NET_BONDING_BASE, bond_name);
rc = ci_bonding_get_list(filename, &active_slaves,
ci_bonding_get_lacp_active_slaves_entry_fn, &state);
kfree(filename);
if( rc != 0 )
return 0;
rc = 0;
while( ci_dllist_not_empty(&active_slaves) ) {
ci_dllink *link = ci_dllist_pop(&active_slaves);
struct ci_bonding_ifname* ifname =
CI_CONTAINER(struct ci_bonding_ifname, list_link, link);
if( strcmp(slave_name, ifname->ifname) == 0 )
rc = 1;
kfree(ifname);
}
return rc;
}
int ci_buddy_ctor2(ci_buddy_allocator* b, unsigned order,
void* (*alloc_fn)(size_t), void (*free_fn)(void*))
{
unsigned o;
ci_assert(b);
b->order = order;
b->free_lists = (ci_dllist*) alloc_fn((order+1) * sizeof(ci_dllist));
if( b->free_lists == 0 ) goto fail1;
b->links = (ci_dllink*) alloc_fn(ci_pow2(order) * sizeof(ci_dllink));
if( b->links == 0 ) goto fail2;
b->orders = (ci_uint8*) alloc_fn(ci_pow2(order));
if( b->orders == 0 ) goto fail3;
CI_DEBUG(CI_ZERO_ARRAY(b->links, ci_pow2(order)));
for( o = 0; o <= b->order; ++o )
ci_dllist_init(b->free_lists + o);
ci_dllist_push(FL(b, b->order), ADDR_TO_LINK(b, 0));
ci_assert(b->order < 255);
b->orders[0] = (ci_uint8)b->order;
ci_assert(!IS_BUSY(b, LINK_TO_ADDR(b, ci_dllist_head(FL(b, b->order)))));
return 0;
fail3:
free_fn(b->links);
fail2:
free_fn(b->free_lists);
fail1:
return -ENOMEM;
}
int citp_epoll_create(int size, int flags)
{
citp_fdinfo *fdi;
citp_epoll_fdi *epi;
struct citp_epoll_fd* ep;
int fd;
if( (epi = CI_ALLOC_OBJ(citp_epoll_fdi)) == NULL )
goto fail0;
if( (ep = CI_ALLOC_OBJ(struct citp_epoll_fd)) == NULL )
goto fail1;
fdi = &epi->fdinfo;
citp_fdinfo_init(fdi, &citp_epoll_protocol_impl);
/* Create the epoll fd. */
CITP_FDTABLE_LOCK();
if( (fd = ci_sys_epoll_create_compat(size, flags, 0)) < 0 )
goto fail2;
citp_fdtable_new_fd_set(fd, fdip_busy, TRUE);
/* Init epfd_os */
#ifdef O_CLOEXEC
ep->epfd_os = ci_sys_open(OO_EPOLL_DEV, O_RDWR | O_CLOEXEC);
#else
ep->epfd_os = ci_sys_open(OO_EPOLL_DEV, O_RDWR);
if( ep->epfd_os >= 0 )
ci_sys_fcntl(ep->epfd_os, F_SETFD, FD_CLOEXEC);
#endif
if( ep->epfd_os < 0 ) {
Log_E(ci_log("%s: ERROR: failed to open(%s) errno=%d",
__FUNCTION__, OO_EPOLL_DEV, errno));
goto fail3;
}
__citp_fdtable_reserve(ep->epfd_os, 1);
ep->shared = mmap(NULL, sizeof(*ep->shared), PROT_READ, MAP_SHARED,
ep->epfd_os, 0);
if( ep->shared == MAP_FAILED ) {
Log_E(ci_log("%s: ERROR: failed to mmap shared segment errno=%d",
__FUNCTION__, errno));
goto fail4;
}
__citp_fdtable_reserve(ep->shared->epfd, 1);
CITP_FDTABLE_UNLOCK();
epi->epoll = ep;
ep->size = size;
oo_wqlock_init(&ep->lock);
ep->not_mt_safe = ! CITP_OPTS.ul_epoll_mt_safe;
ci_dllist_init(&ep->oo_sockets);
ep->oo_sockets_n = 0;
ci_dllist_init(&ep->dead_sockets);
oo_atomic_set(&ep->refcount, 1);
ep->epfd_syncs_needed = 0;
ep->blocking = 0;
citp_fdtable_insert(fdi, fd, 0);
Log_POLL(ci_log("%s: fd=%d driver_fd=%d epfd=%d", __FUNCTION__,
fd, ep->epfd_os, (int) ep->shared->epfd));
return fd;
fail4:
__citp_fdtable_reserve(ep->epfd_os, 0);
ci_sys_close(ep->epfd_os);
fail3:
ci_sys_close(fd);
citp_fdtable_busy_clear(fd, fdip_unknown, 1);
fail2:
CITP_FDTABLE_UNLOCK();
CI_FREE_OBJ(ep);
fail1:
CI_FREE_OBJ(epi);
fail0:
return -1;
}
/* Allocate a new cluster.
*
* You need to hold the thc_mutex before calling this.
*/
static int thc_alloc(const char* cluster_name, int protocol, int port_be16,
uid_t euid, int cluster_size,
unsigned flags,
tcp_helper_cluster_t** thc_out)
{
int rc, i;
int rss_flags;
struct efrm_pd* pd;
int packet_buffer_mode = flags & THC_FLAG_PACKET_BUFFER_MODE;
int tproxy = flags & THC_FLAG_TPROXY;
int hw_loopback_enable = flags & THC_FLAG_HW_LOOPBACK_ENABLE;
tcp_helper_cluster_t* thc = kmalloc(sizeof(*thc), GFP_KERNEL);
if( thc == NULL )
return -ENOMEM;
memset(thc, 0, sizeof(*thc));
ci_dllist_init(&thc->thc_tlos);
strcpy(thc->thc_name, cluster_name);
thc->thc_cluster_size = cluster_size;
thc->thc_euid = euid;
thc->thc_flags = flags;
/* Needed to protect against oo_nics changes */
rtnl_lock();
for( i = 0; i < CI_CFG_MAX_REGISTER_INTERFACES; ++i ) {
if( oo_nics[i].efrm_client == NULL ||
! oo_check_nic_suitable_for_onload(&(oo_nics[i])) )
continue;
if( (rc = efrm_pd_alloc(&pd, oo_nics[i].efrm_client, NULL,
(packet_buffer_mode ? EFRM_PD_ALLOC_FLAG_PHYS_ADDR_MODE : 0) |
(hw_loopback_enable ? EFRM_PD_ALLOC_FLAG_HW_LOOPBACK : 0))) )
goto fail;
/*
* Currently we move on if we fail to get special tproxy RSS_MODE on
* interface(s) (expect Siena, Huntington old fw, run out of rss contexts).
*/
rss_flags = tproxy ? EFRM_RSS_MODE_DST | EFRM_RSS_MODE_SRC :
EFRM_RSS_MODE_DEFAULT;
redo:
rc = efrm_vi_set_alloc(pd, thc->thc_cluster_size, 0,
rss_flags, &thc->thc_vi_set[i]);
if( rc != 0 && rss_flags ) {
LOG_E(ci_log("Installing special RSS mode filter failed on hwport %d, "
"falling back to default mode. Transparent proxy will not "
"work with this interface.", i));
rss_flags = 0;
goto redo;
}
efrm_pd_release(pd);
if( rc != 0 )
goto fail;
}
rtnl_unlock();
thc->thc_next = thc_head;
thc_head = thc;
*thc_out = thc;
return 0;
fail:
rtnl_unlock();
for( i = 0; i < CI_CFG_MAX_REGISTER_INTERFACES; ++i )
if( thc->thc_vi_set[i] != NULL )
efrm_vi_set_release(thc->thc_vi_set[i]);
kfree(thc);
return rc;
}
