void citp_waitable_wakeup(ci_netif* ni, citp_waitable* w)
{
oo_waitable_wake_t op;
op.sock_id = w->bufid;
oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_WAITABLE_WAKE, &op);
}
int oo_os_sock_recvmsg(ci_netif* ni, oo_sp sock_p,
struct msghdr* msg, int flags)
{
oo_os_sock_recvmsg_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg_iov, msg->msg_iov);
op.msg_iovlen = msg->msg_iovlen;
CI_USER_PTR_SET(op.msg_name, msg->msg_name);
op.msg_namelen = msg->msg_namelen;
CI_USER_PTR_SET(op.msg_control, msg->msg_control);
op.msg_controllen = msg->msg_controllen;
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_RECVMSG, &op);
ci_assert(op.rc >= 0 || rc < 0);
if( rc == 0 ) {
msg->msg_flags = op.flags;
msg->msg_namelen = op.msg_namelen;
if( msg->msg_controllen )
msg->msg_controllen = op.msg_controllen;
return op.rc;
}
return rc;
}
ci_fd_t ci_udp_ep_ctor(citp_socket* ep, ci_netif* netif, int domain, int type)
{
ci_udp_state* us;
ci_fd_t fd;
VERB( log(LPFIN "ctor( )" ) );
ci_assert(ep);
ci_assert(netif);
ci_netif_lock(netif);
us = ci_udp_get_state_buf(netif);
if (!us) {
ci_netif_unlock(netif);
LOG_E(ci_log("%s: [%d] out of socket buffers", __FUNCTION__,NI_ID(netif)));
return -ENOMEM;
}
/* It's required to set protocol before ci_tcp_helper_sock_attach()
* since it's used to determine if TCP or UDP file operations should be
* attached to the file descriptor in kernel. */
sock_protocol(&us->s) = IPPROTO_UDP;
/* NB: this attach will close the os_sock_fd */
fd = ci_tcp_helper_sock_attach(ci_netif_get_driver_handle(netif),
SC_SP(&us->s), domain, type);
if( fd < 0 ) {
if( fd == -EAFNOSUPPORT )
LOG_U(ci_log("%s: ci_tcp_helper_sock_attach (domain=%d, type=%d) "
"failed %d", __FUNCTION__, domain, type, fd));
else
LOG_E(ci_log("%s: ci_tcp_helper_sock_attach (domain=%d, type=%d) "
"failed %d", __FUNCTION__, domain, type, fd));
ci_netif_unlock(netif);
return fd;
}
ci_assert(~us->s.b.sb_aflags & CI_SB_AFLAG_ORPHAN);
us->s.rx_errno = 0;
us->s.tx_errno = 0;
us->s.so_error = 0;
us->s.cp.sock_cp_flags |= OO_SCP_UDP_WILD;
ep->s = &us->s;
ep->netif = netif;
CHECK_UEP(ep);
ci_netif_unlock(netif);
return fd;
}
int oo_os_sock_get(ci_netif* ni, oo_sp sock_p, oo_os_file* os_sock_out)
{
oo_os_sock_fd_get_t op;
int rc;
oo_rwlock_lock_read(&citp_dup2_lock);
op.sock_id = OO_SP_TO_INT(sock_p);
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_FD_GET, &op);
if( rc == 0 )
*os_sock_out = op.fd_out;
else
oo_rwlock_unlock_read (&citp_dup2_lock);
return rc;
}
static void do_ofe_command(ci_netif *ni)
{
char command[200];
char *str = command;
int rc;
oo_ofe_config_t op;
int i;
if( ni->ofe == NULL )
return;
for( i = 0; i < cfg_argc; i++ )
str += snprintf(str, sizeof(command) - (str - command), "%s ", cfg_argv[i]);
op.len = str - command;
CI_USER_PTR_SET(op.str, command);
rc = oo_resource_op(ci_netif_get_driver_handle(ni), OO_IOC_OFE_CONFIG, &op);
oo_resource_op(ci_netif_get_driver_handle(ni), OO_IOC_OFE_CONFIG_DONE, NULL);
if( rc == 0 ) {
ci_log("[%s] %s: OK", ni->state->pretty_name, command);
return;
}
ci_log("[%s] Onload Filter Engine fails to process command\n\t%s",
ni->state->pretty_name, command);
if( ofe_engine_get_last_error(ni->ofe) != NULL )
ci_log("OFE ERROR: %s", ofe_engine_get_last_error(ni->ofe));
else {
char err[CI_LOG_MAX_LINE];
oo_resource_op(ci_netif_get_driver_handle(ni), OO_IOC_OFE_GET_LAST_ERROR, err);
err[CI_LOG_MAX_LINE-1] = '\0';
ci_log("OFE ERROR: %s", err);
}
}
int oo_os_sock_accept(ci_netif* ni, oo_sp sock_p, struct sockaddr *addr,
socklen_t *addrlen, int flags)
{
oo_os_sock_accept_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
CI_USER_PTR_SET(op.addr, addr);
CI_USER_PTR_SET(op.addrlen, addrlen);
op.flags = flags;
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_ACCEPT, &op);
return rc == 0 ? op.rc : rc;
}
int citp_pipe_splice_read(citp_fdinfo* fdi, int alien_fd, loff_t* alien_off,
size_t len, int flags,
citp_lib_context_t* lib_context)
{
citp_pipe_fdi* epi = fdi_to_pipe_fdi(fdi);
int rc;
int read_len = 0;
int non_block = (flags & SPLICE_F_NONBLOCK) || (epi->pipe->aflags &
(CI_PFD_AFLAG_NONBLOCK << CI_PFD_AFLAG_READER_SHIFT));
if( ! fdi_is_reader(fdi) ) {
errno = EINVAL;
return -1;
}
if( alien_off ) {
/* TODO support this */
errno = ENOTSUP;
return -1;
}
if( len == 0 )
return 0;
do {
struct oo_splice_read_context ctx = {
.alien_fd = alien_fd,
.len = len,
.lib_context = lib_context
};
rc = ci_pipe_zc_read(epi->ni, epi->pipe, len,
non_block ? MSG_DONTWAIT : 0,
oo_splice_read_cb, &ctx);
if( rc > 0 )
read_len += rc;
} while(0);
if( rc < 0 && errno == EPIPE && ! (flags & MSG_NOSIGNAL) ) {
ci_sys_ioctl(ci_netif_get_driver_handle(epi->ni),
OO_IOC_KILL_SELF_SIGPIPE, NULL);
return rc;
}
if( rc > 0 )
return read_len;
return rc;
}
ci_fd_t ci_tcp_ep_ctor(citp_socket* ep, ci_netif* netif, int domain, int type)
{
ci_tcp_state* ts;
ci_fd_t fd;
ci_assert(ep);
ci_assert(netif);
ci_netif_lock(netif);
ts = ci_tcp_get_state_buf(netif);
if( ts == NULL ) {
ci_netif_unlock(netif);
LOG_E(ci_log("%s: [%d] out of socket buffers", __FUNCTION__,NI_ID(netif)));
return -ENOMEM;
}
fd = ci_tcp_helper_sock_attach(ci_netif_get_driver_handle(netif), S_SP(ts),
domain, type);
if( fd < 0 ) {
if( fd == -EAFNOSUPPORT )
LOG_U(ci_log("%s: ci_tcp_helper_sock_attach" \
"(domain=%d, type=%d) failed %d",
__FUNCTION__, domain, type, fd));
else
LOG_E(ci_log("%s: ci_tcp_helper_sock_attach" \
"(domain=%d, type=%d) failed %d",
__FUNCTION__, domain, type, fd));
}
else {
ci_assert(~ts->s.b.sb_aflags & CI_SB_AFLAG_ORPHAN);
/* Apply default sockbuf sizes now we've updated them from the kernel
** defaults. */
ts->s.so.sndbuf = NI_OPTS(netif).tcp_sndbuf_def;
ts->s.so.rcvbuf = NI_OPTS(netif).tcp_rcvbuf_def;
ep->netif = netif;
ep->s = &ts->s;
CHECK_TEP(ep);
}
ci_netif_unlock(netif);
return fd;
}
int oo_os_sock_sendmsg_raw(ci_netif* ni, oo_sp sock_p,
const struct msghdr* msg, int flags)
{
unsigned long socketcall_args[8];
oo_os_sock_sendmsg_raw_t op;
int rc;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg, msg);
CI_USER_PTR_SET(op.socketcall_args, socketcall_args);
oo_rwlock_lock_read(&citp_dup2_lock);
rc = oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_SENDMSG_RAW, &op);
oo_rwlock_unlock_read (&citp_dup2_lock);
return rc;
}
static int oo_pipe_ctor(ci_netif* netif, struct oo_pipe** out_pipe,
int fds[2], int flags)
{
struct oo_pipe* p;
int rc;
ci_assert(netif);
ci_netif_lock(netif);
p = oo_pipe_buf_get(netif);
if( !p ) {
rc = -1;
errno = ENOMEM;
goto out;
}
if( flags & O_NONBLOCK ) {
p->aflags = (CI_PFD_AFLAG_NONBLOCK << CI_PFD_AFLAG_READER_SHIFT) |
(CI_PFD_AFLAG_NONBLOCK << CI_PFD_AFLAG_WRITER_SHIFT);
}
/* attach */
rc = ci_tcp_helper_pipe_attach(ci_netif_get_driver_handle(netif),
W_SP(&p->b), flags, fds);
if( rc < 0 ) {
LOG_E(ci_log("%s: ci_tcp_helper_pipe_attach %d", __FUNCTION__, rc));
errno = -rc;
rc = -1;
goto out;
}
*out_pipe = p;
out:
ci_netif_unlock(netif);
return rc;
}
int oo_os_sock_sendmsg(ci_netif* ni, oo_sp sock_p,
const struct msghdr* msg, int flags)
{
oo_os_sock_sendmsg_t op;
op.sock_id = OO_SP_TO_INT(sock_p);
op.sizeof_ptr = sizeof(void*);
op.flags = flags;
CI_USER_PTR_SET(op.msg_iov, msg->msg_iov);
op.msg_iovlen = msg->msg_iovlen;
CI_USER_PTR_SET(op.msg_name, msg->msg_name);
op.msg_namelen = msg->msg_namelen;
#ifdef __i386__
/* compat cmsg is not handled in this function */
ci_assert_equal(msg->msg_controllen, 0);
op.msg_controllen = 0;
CI_USER_PTR_SET(op.msg_control, NULL);
#else
CI_USER_PTR_SET(op.msg_control, msg->msg_control);
op.msg_controllen = msg->msg_controllen;
#endif
return oo_resource_op(ci_netif_get_driver_handle(ni),
OO_IOC_OS_SOCK_SENDMSG, &op);
}
int citp_pipe_splice_write(citp_fdinfo* fdi, int alien_fd, loff_t* alien_off,
size_t olen, int flags,
citp_lib_context_t* lib_context)
{
citp_pipe_fdi* epi = fdi_to_pipe_fdi(fdi);
int len_in_bufs = OO_PIPE_SIZE_TO_BUFS(olen);
struct iovec iov_on_stack[CITP_PIPE_SPLICE_WRITE_STACK_IOV_LEN];
struct iovec* iov = iov_on_stack;
int want_buf_count;
int rc;
int bytes_to_read;
int len = olen;
int no_more = 1; /* for now we only run single loop */
int written_total = 0;
int non_block = (flags & SPLICE_F_NONBLOCK) || (epi->pipe->aflags &
(CI_PFD_AFLAG_NONBLOCK << CI_PFD_AFLAG_WRITER_SHIFT));
if( fdi_is_reader(fdi) ) {
errno = EINVAL;
return -1;
}
if( alien_off ) {
/* TODO support this */
errno = ENOTSUP;
return -1;
}
do {
int count;
int iov_num;
int bytes_to_write;
struct ci_pipe_pkt_list pkts = {};
struct ci_pipe_pkt_list pkts2;
want_buf_count = len_in_bufs;
/* We might need to wait for buffers here on the first iteration */
rc = ci_pipe_zc_alloc_buffers(epi->ni, epi->pipe, want_buf_count,
MSG_NOSIGNAL | (non_block || written_total ?
MSG_DONTWAIT : 0),
&pkts);
if( rc < 0 && written_total ) {
/* whatever the error we need to report already written_bytes */
rc = written_total;
break;
}
else if( rc < 0 )
break;
else if( pkts.count == 0 && non_block ) {
errno = EAGAIN;
rc = -1;
break;
}
else
ci_assert_gt(pkts.count, 0);
count = pkts.count;
if( count > CITP_PIPE_SPLICE_WRITE_STACK_IOV_LEN ) {
void* niov = realloc(iov == iov_on_stack ? NULL : iov,
sizeof(*iov) * len_in_bufs);
if( niov == NULL )
/* we can still move quite a few pkts */
count = CITP_PIPE_SPLICE_WRITE_STACK_IOV_LEN;
else
niov = iov;
}
ci_assert_ge(count, 1);
iov_num = count;
pkts2 = pkts;
bytes_to_read = ci_pipe_list_to_iovec(epi->ni, epi->pipe, iov, &iov_num,
&pkts2, len);
citp_exit_lib_if(lib_context, TRUE);
/* Note: the following call might be non-blocking as well as blocking */
rc = readv(alien_fd, iov, count);
citp_reenter_lib(lib_context);
if( rc > 0 ) {
bytes_to_write = rc;
written_total += bytes_to_write;
len -= bytes_to_write;
no_more |= bytes_to_write < bytes_to_read;
}
else {
bytes_to_write = 0;
no_more = 1;
}
{
/* pipe zc_write will write non_empty buffers and release the empty
* ones */
int rc2 = ci_pipe_zc_write(epi->ni, epi->pipe, &pkts, bytes_to_write,
CI_PIPE_ZC_WRITE_FLAG_FORCE | MSG_DONTWAIT | MSG_NOSIGNAL);
(void) rc2;
ci_assert_equal(rc2, bytes_to_write);
}
/* for now we will not be doing second iteration, to allow for that
* we'd need to have guarantee that read will not block
* e.g. insight into type of fd and a nonblokcing operation
* (to name a valid case: socket, recvmsg) */
} while( ! no_more );
if( iov != iov_on_stack )
free(iov);
if( rc > 0 )
return written_total;
if( rc < 0 && errno == EPIPE && ! (flags & MSG_NOSIGNAL) ) {
ci_sys_ioctl(ci_netif_get_driver_handle(epi->ni),
OO_IOC_KILL_SELF_SIGPIPE, NULL);
}
return rc;
}
