/* virtual */ SysStatusUval FileLinuxFile::readv(struct iovec* vec, uval vecCount, ThreadWait **tw, GenState &moreAvail) { FLFDEBUG("readv"); SysStatusUval rc; uval length_read; char* buf_read; uval length = vecLength(vec,vecCount); lock(); moreAvail.state = FileLinux::READ_AVAIL|FileLinux::WRITE_AVAIL; rc = locked_readAlloc(length, buf_read, tw); if (_FAILURE(rc)) { unLock(); //semantics shift from EOF error to 0 length on EOF if (_SCLSCD(rc) == FileLinux::EndOfFile) { moreAvail.state = FileLinux::ENDOFFILE; return 0; } return rc; } length_read = _SGETUVAL(rc); if (length_read) { memcpy_toiovec(vec, buf_read, vecCount, length_read); locked_readFree(buf_read); } unLock(); return rc; }
/* * We have already pinned down the pages we will be using in the iovecs. * Each entry in iov array has corresponding entry in pinned_list->page_list. * Using array indexing to keep iov[] and page_list[] in sync. * Initial elements in iov array's iov->iov_len will be 0 if already copied into * by another call. * iov array length remaining guaranteed to be bigger than len. */ dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) { int iov_byte_offset; int copy; dma_cookie_t dma_cookie = 0; int iovec_idx; int page_idx; if (!chan) return memcpy_toiovec(iov, kdata, len); iovec_idx = 0; while (iovec_idx < pinned_list->nr_iovecs) { struct dma_page_list *page_list; /* skip already used-up iovecs */ while (!iov[iovec_idx].iov_len) iovec_idx++; page_list = &pinned_list->page_list[iovec_idx]; iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; /* break up copies to not cross page boundary */ while (iov[iovec_idx].iov_len) { copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); copy = min_t(int, copy, iov[iovec_idx].iov_len); dma_cookie = dma_async_memcpy_buf_to_pg(chan, page_list->pages[page_idx], iov_byte_offset, kdata, copy); /* poll for a descriptor slot */ if (unlikely(dma_cookie < 0)) { dma_async_issue_pending(chan); continue; } len -= copy; iov[iovec_idx].iov_len -= copy; iov[iovec_idx].iov_base += copy; if (!len) return dma_cookie; kdata += copy; iov_byte_offset = 0; page_idx++; } iovec_idx++; } /* really bad if we ever run out of iovecs */ BUG(); return -EFAULT; }
/* Put packet to the user space buffer (already verified) */ static __inline__ ssize_t tun_put_user(struct tun_struct *tun, struct sk_buff *skb, struct iovec *iv, int len) { struct tun_pi pi = { 0, skb->protocol }; ssize_t total = 0; if (!(tun->flags & TUN_NO_PI)) { if ((len -= sizeof(pi)) < 0) return -EINVAL; if (len < skb->len) { /* Packet will be striped */ pi.flags |= TUN_PKT_STRIP; } memcpy_toiovec(iv, (void *) &pi, sizeof(pi)); total += sizeof(pi); } len = MIN(skb->len, len); skb_copy_datagram_iovec(skb, 0, iv, len); total += len; tun->stats.tx_packets++; tun->stats.tx_bytes += len; return total; }
static int raw_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t size, int flags) { struct sock *sk = sock->sk; struct raw_sock *ro = raw_sk(sk); struct sk_buff *skb; int rxmtu; int err = 0; int noblock; noblock = flags & MSG_DONTWAIT; flags &= ~MSG_DONTWAIT; skb = skb_recv_datagram(sk, flags, noblock, &err); if (!skb) return err; /* * when serving a legacy socket the DLC <= 8 is already checked inside * raw_rcv(). Now check if we need to pass a canfd_frame to a legacy * socket and cut the possible CANFD_MTU/CAN_MTU length to CAN_MTU */ if (!ro->fd_frames) rxmtu = CAN_MTU; else rxmtu = skb->len; if (size < rxmtu) msg->msg_flags |= MSG_TRUNC; else size = rxmtu; err = memcpy_toiovec(msg->msg_iov, skb->data, size); if (err < 0) { skb_free_datagram(sk, skb); return err; } sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } /* assign the flags that have been recorded in raw_rcv() */ msg->msg_flags |= *(raw_flags(skb)); skb_free_datagram(sk, skb); return size; }
static int raw_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t size, int flags) { struct sock *sk = sock->sk; struct sk_buff *skb; int error = 0; int noblock; DBG("socket %p, sk %p\n", sock, sk); noblock = flags & MSG_DONTWAIT; flags &= ~MSG_DONTWAIT; skb = skb_recv_datagram(sk, flags, noblock, &error); if (!skb) return error; DBG("delivering skbuff %p\n", skb); DBG_SKB(skb); if (size < skb->len) msg->msg_flags |= MSG_TRUNC; else size = skb->len; error = memcpy_toiovec(msg->msg_iov, skb->data, size); if (error < 0) { skb_free_datagram(sk, skb); return error; } sock_recv_timestamp(msg, sk, skb); if (msg->msg_name) { msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } DBG("freeing sock %p, skbuff %p\n", sk, skb); skb_free_datagram(sk, skb); return size; }
int __VMCIMemcpyFromQueue(void *dest, // OUT: const VMCIQueue *queue, // IN: uint64 queueOffset, // IN: size_t size, // IN: Bool isIovec) // IN: if dest is a struct iovec * { size_t bytesCopied = 0; while (bytesCopied < size) { uint64 pageIndex = (queueOffset + bytesCopied) / PAGE_SIZE; size_t pageOffset = (queueOffset + bytesCopied) & (PAGE_SIZE - 1); void *va = kmap(queue->page[pageIndex]); size_t toCopy; ASSERT(va); if (size - bytesCopied > PAGE_SIZE - pageOffset) { /* Enough payload to fill up this page. */ toCopy = PAGE_SIZE - pageOffset; } else { toCopy = size - bytesCopied; } if (isIovec) { struct iovec *iov = (struct iovec *)dest; int err; /* The iovec will track bytesCopied internally. */ err = memcpy_toiovec(iov, (uint8 *)va + pageOffset, toCopy); if (err != 0) { kunmap(queue->page[pageIndex]); return err; } } else { memcpy((uint8 *)dest + bytesCopied, (uint8 *)va + pageOffset, toCopy); } bytesCopied += toCopy; kunmap(queue->page[pageIndex]); } return 0; }
static int raw_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t size, int flags) { struct sock *sk = sock->sk; struct sk_buff *skb; int err = 0; int noblock; noblock = flags & MSG_DONTWAIT; flags &= ~MSG_DONTWAIT; skb = skb_recv_datagram(sk, flags, noblock, &err); if (!skb) return err; if (size < skb->len) msg->msg_flags |= MSG_TRUNC; else size = skb->len; err = memcpy_toiovec(msg->msg_iov, skb->data, size); if (err < 0) { skb_free_datagram(sk, skb); return err; } sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } /* assign the flags that have been recorded in raw_rcv() */ msg->msg_flags |= *(raw_flags(skb)); skb_free_datagram(sk, skb); return size; }
/* virtual */ SysStatusUval StreamServerConsole::recvfrom(struct iovec *vec, uval veclen, uval flags, char *addr, uval &addrLen, GenState &moreAvail, void *controlData, uval &controlDataLen, __XHANDLE xhandle) { addrLen = 0; uval copied = 0; controlDataLen = 0; /* setting to zero, since no control data */ lock(); if (buffer.bytesAvail()) { char data[buffer.bufSize()]; copied = buffer.getData(data, buffer.bufSize()); memcpy_toiovec(vec, data, veclen, copied); } calcAvailable(moreAvail); unlock(); clnt(xhandle)->setAvail(moreAvail); return copied; }
static int raw_recvmsg(struct socket *sock, struct msghdr *msg, int size, int flags, struct scm_cookie *scm) { struct sock *sk = sock->sk; struct sk_buff *skb; int err = 0; int noblock; noblock = flags & MSG_DONTWAIT; flags &= ~MSG_DONTWAIT; skb = skb_recv_datagram(sk, flags, noblock, &err); if (!skb) return err; if (size < skb->len) msg->msg_flags |= MSG_TRUNC; else size = skb->len; err = memcpy_toiovec(msg->msg_iov, skb->data, size); if (err < 0) { skb_free_datagram(sk, skb); return err; } sock_recv_timestamp(msg, sk, skb); if (msg->msg_name) { msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } skb_free_datagram(sk, skb); return size; }
dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, struct dma_pinned_list *pinned_list, struct page *page, unsigned int offset, size_t len) { int iov_byte_offset; int copy; dma_cookie_t dma_cookie = 0; int iovec_idx; int page_idx; int err; /* this needs as-yet-unimplemented buf-to-buff, so punt. */ /* TODO: use dma for this */ if (!chan || !pinned_list) { u8 *vaddr = kmap(page); err = memcpy_toiovec(iov, vaddr + offset, len); kunmap(page); return err; } iovec_idx = 0; while (iovec_idx < pinned_list->nr_iovecs) { struct dma_page_list *page_list; /* skip already used-up iovecs */ while (!iov[iovec_idx].iov_len) iovec_idx++; page_list = &pinned_list->page_list[iovec_idx]; iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; /* break up copies to not cross page boundary */ while (iov[iovec_idx].iov_len) { copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); copy = min_t(int, copy, iov[iovec_idx].iov_len); dma_cookie = dma_async_memcpy_pg_to_pg(chan, page_list->pages[page_idx], iov_byte_offset, page, offset, copy); /* poll for a descriptor slot */ if (unlikely(dma_cookie < 0)) { dma_async_issue_pending(chan); continue; } len -= copy; iov[iovec_idx].iov_len -= copy; iov[iovec_idx].iov_base += copy; if (!len) return dma_cookie; offset += copy; iov_byte_offset = 0; page_idx++; } iovec_idx++; } /* really bad if we ever run out of iovecs */ BUG(); return -EFAULT; }
static int econet_recvmsg(struct socket *sock, struct msghdr *msg, int len, int flags, struct scm_cookie *scm) { struct sock *sk = sock->sk; struct sk_buff *skb; int copied, err; msg->msg_namelen = sizeof(struct sockaddr_ec); /* * Call the generic datagram receiver. This handles all sorts * of horrible races and re-entrancy so we can forget about it * in the protocol layers. * * Now it will return ENETDOWN, if device have just gone down, * but then it will block. */ skb=skb_recv_datagram(sk,flags,flags&MSG_DONTWAIT,&err); /* * An error occurred so return it. Because skb_recv_datagram() * handles the blocking we don't see and worry about blocking * retries. */ if(skb==NULL) goto out; /* * You lose any data beyond the buffer you gave. If it worries a * user program they can ask the device for its MTU anyway. */ copied = skb->len; if (copied > len) { copied=len; msg->msg_flags|=MSG_TRUNC; } /* We can't use skb_copy_datagram here */ err = memcpy_toiovec(msg->msg_iov, skb->data, copied); if (err) goto out_free; sk->stamp=skb->stamp; if (msg->msg_name) memcpy(msg->msg_name, skb->cb, msg->msg_namelen); /* * Free or return the buffer as appropriate. Again this * hides all the races and re-entrancy issues from us. */ err = copied; out_free: skb_free_datagram(sk, skb); out: return err; }
int skb_copy_datagram_iovec(struct sk_buff *skb, int offset, struct iovec *to, int size) { return memcpy_toiovec(to, skb->h.raw + offset, size); }
static void handle_tx(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX]; unsigned out, in, s; int head; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; size_t len, total_len = 0; int err, wmem; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock) return; wmem = atomic_read(&sock->sk->sk_wmem_alloc); if (wmem >= sock->sk->sk_sndbuf) { mutex_lock(&vq->mutex); tx_poll_start(net, sock); mutex_unlock(&vq->mutex); return; } use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); if (wmem < sock->sk->sk_sndbuf / 2) tx_poll_stop(net); hdr_size = vq->hdr_size; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, NULL, NULL); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* Nothing new? Wait for eventfd to tell us they refilled. */ if (head == vq->num) { wmem = atomic_read(&sock->sk->sk_wmem_alloc); if (wmem >= sock->sk->sk_sndbuf * 3 / 4) { tx_poll_start(net, sock); set_bit(SOCK_ASYNC_NOSPACE, &sock->flags); break; } if (unlikely(vhost_enable_notify(vq))) { vhost_disable_notify(vq); continue; } break; } if (in) { vq_err(vq, "Unexpected descriptor format for TX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out); msg.msg_iovlen = out; len = iov_length(vq->iov, out); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for TX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } /* TODO: Check specific error and bomb out unless ENOBUFS? */ err = sock->ops->sendmsg(NULL, sock, &msg, len); if (unlikely(err < 0)) { vhost_discard_vq_desc(vq); tx_poll_start(net, sock); break; } if (err != len) pr_debug("Truncated TX packet: " " len %d != %zd\n", err, len); vhost_add_used_and_signal(&net->dev, vq, head, 0); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_rx(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned out, in, log, s; int head; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr hdr = { .flags = 0, .gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t len, total_len = 0; int err; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); hdr_size = vq->hdr_size; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* OK, now we need to know about added descriptors. */ if (head == vq->num) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (out) { vq_err(vq, "Unexpected descriptor format for RX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO/mergeable rx buffers. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in); msg.msg_iovlen = in; len = iov_length(vq->iov, in); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for RX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } err = sock->ops->recvmsg(NULL, sock, &msg, len, MSG_DONTWAIT | MSG_TRUNC); /* TODO: Check specific error and bomb out unless EAGAIN? */ if (err < 0) { vhost_discard_vq_desc(vq); break; } /* TODO: Should check and handle checksum. */ if (err > len) { pr_debug("Discarded truncated rx packet: " " len %d > %zd\n", err, len); vhost_discard_vq_desc(vq); continue; } len = err; err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size); if (err) { vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n", vq->iov->iov_base, err); break; } len += hdr_size; vhost_add_used_and_signal(&net->dev, vq, head, len); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, len); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_tx_kick(struct work_struct *work) { struct vhost_virtqueue *vq; struct vhost_net *net; vq = container_of(work, struct vhost_virtqueue, poll.work); net = container_of(vq->dev, struct vhost_net, dev); handle_tx(net); } static void handle_rx_kick(struct work_struct *work) { struct vhost_virtqueue *vq; struct vhost_net *net; vq = container_of(work, struct vhost_virtqueue, poll.work); net = container_of(vq->dev, struct vhost_net, dev); handle_rx(net); } static void handle_tx_net(struct work_struct *work) { struct vhost_net *net; net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work); handle_tx(net); } static void handle_rx_net(struct work_struct *work) { struct vhost_net *net; net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work); handle_rx(net); } static int vhost_net_open(struct inode *inode, struct file *f) { struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL); int r; if (!n) return -ENOMEM; n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick; n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick; r = vhost_dev_init(&n->dev, n->vqs, VHOST_NET_VQ_MAX); if (r < 0) { kfree(n); return r; } vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN); n->tx_poll_state = VHOST_NET_POLL_DISABLED; f->private_data = n; return 0; } static void vhost_net_disable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { if (!vq->private_data) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { tx_poll_stop(n); n->tx_poll_state = VHOST_NET_POLL_DISABLED; } else vhost_poll_stop(n->poll + VHOST_NET_VQ_RX); } static void vhost_net_enable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock = vq->private_data; if (!sock) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { n->tx_poll_state = VHOST_NET_POLL_STOPPED; tx_poll_start(n, sock); } else vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file); } static struct socket *vhost_net_stop_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; mutex_lock(&vq->mutex); sock = vq->private_data; vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, NULL); mutex_unlock(&vq->mutex); return sock; } static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, struct socket **rx_sock) { *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX); *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX); } static void vhost_net_flush_vq(struct vhost_net *n, int index) { vhost_poll_flush(n->poll + index); vhost_poll_flush(&n->dev.vqs[index].poll); } static void vhost_net_flush(struct vhost_net *n) { vhost_net_flush_vq(n, VHOST_NET_VQ_TX); vhost_net_flush_vq(n, VHOST_NET_VQ_RX); }
/* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_rx_big(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned out, in, log, s; int head; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr hdr = { .flags = 0, .gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t len, total_len = 0; int err; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); hdr_size = vq->vhost_hlen; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* OK, now we need to know about added descriptors. */ if (head == vq->num) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (out) { vq_err(vq, "Unexpected descriptor format for RX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO/mergeable rx buffers. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in); msg.msg_iovlen = in; len = iov_length(vq->iov, in); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for RX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } err = sock->ops->recvmsg(NULL, sock, &msg, len, MSG_DONTWAIT | MSG_TRUNC); /* TODO: Check specific error and bomb out unless EAGAIN? */ if (err < 0) { vhost_discard_vq_desc(vq, 1); break; } /* TODO: Should check and handle checksum. */ if (err > len) { pr_debug("Discarded truncated rx packet: " " len %d > %zd\n", err, len); vhost_discard_vq_desc(vq, 1); continue; } len = err; err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size); if (err) { vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n", vq->iov->iov_base, err); break; } len += hdr_size; vhost_add_used_and_signal(&net->dev, vq, head, len); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, len); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } /* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_rx_mergeable(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned uninitialized_var(in), log; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr_mrg_rxbuf hdr = { .hdr.flags = 0, .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t total_len = 0; int err, headcount; size_t vhost_hlen, sock_hlen; size_t vhost_len, sock_len; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); vhost_hlen = vq->vhost_hlen; sock_hlen = vq->sock_hlen; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; while ((sock_len = peek_head_len(sock->sk))) { sock_len += sock_hlen; vhost_len = sock_len + vhost_hlen; headcount = get_rx_bufs(vq, vq->heads, vhost_len, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(headcount < 0)) break; /* OK, now we need to know about added descriptors. */ if (!headcount) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (unlikely((vhost_hlen))) /* Skip header. TODO: support TSO. */ move_iovec_hdr(vq->iov, vq->hdr, vhost_hlen, in); else /* Copy the header for use in VIRTIO_NET_F_MRG_RXBUF: * needed because sendmsg can modify msg_iov. */ copy_iovec_hdr(vq->iov, vq->hdr, sock_hlen, in); msg.msg_iovlen = in; err = sock->ops->recvmsg(NULL, sock, &msg, sock_len, MSG_DONTWAIT | MSG_TRUNC); /* Userspace might have consumed the packet meanwhile: * it's not supposed to do this usually, but might be hard * to prevent. Discard data we got (if any) and keep going. */ if (unlikely(err != sock_len)) { pr_debug("Discarded rx packet: " " len %d, expected %zd\n", err, sock_len); vhost_discard_vq_desc(vq, headcount); continue; } if (unlikely(vhost_hlen) && memcpy_toiovecend(vq->hdr, (unsigned char *)&hdr, 0, vhost_hlen)) { vq_err(vq, "Unable to write vnet_hdr at addr %p\n", vq->iov->iov_base); break; } /* TODO: Should check and handle checksum. */ if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF) && memcpy_toiovecend(vq->hdr, (unsigned char *)&headcount, offsetof(typeof(hdr), num_buffers), sizeof hdr.num_buffers)) { vq_err(vq, "Failed num_buffers write"); vhost_discard_vq_desc(vq, headcount); break; } vhost_add_used_and_signal_n(&net->dev, vq, vq->heads, headcount); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, vhost_len); total_len += vhost_len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_rx(struct vhost_net *net) { if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF)) handle_rx_mergeable(net); else handle_rx_big(net); } static void handle_tx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_tx(net); } static void handle_rx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_rx(net); } static void handle_tx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work); handle_tx(net); } static void handle_rx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work); handle_rx(net); } static int vhost_net_open(struct inode *inode, struct file *f) { struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL); struct vhost_dev *dev; int r; if (!n) return -ENOMEM; dev = &n->dev; n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick; n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick; r = vhost_dev_init(dev, n->vqs, VHOST_NET_VQ_MAX); if (r < 0) { kfree(n); return r; } vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev); n->tx_poll_state = VHOST_NET_POLL_DISABLED; f->private_data = n; return 0; } static void vhost_net_disable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { if (!vq->private_data) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { tx_poll_stop(n); n->tx_poll_state = VHOST_NET_POLL_DISABLED; } else vhost_poll_stop(n->poll + VHOST_NET_VQ_RX); } static void vhost_net_enable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; sock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); if (!sock) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { n->tx_poll_state = VHOST_NET_POLL_STOPPED; tx_poll_start(n, sock); } else vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file); } static struct socket *vhost_net_stop_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; mutex_lock(&vq->mutex); sock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, NULL); mutex_unlock(&vq->mutex); return sock; } static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, struct socket **rx_sock) { *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX); *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX); } static void vhost_net_flush_vq(struct vhost_net *n, int index) { vhost_poll_flush(n->poll + index); vhost_poll_flush(&n->dev.vqs[index].poll); } static void vhost_net_flush(struct vhost_net *n) { vhost_net_flush_vq(n, VHOST_NET_VQ_TX); vhost_net_flush_vq(n, VHOST_NET_VQ_RX); } static int vhost_net_release(struct inode *inode, struct file *f) { struct vhost_net *n = f->private_data; struct socket *tx_sock; struct socket *rx_sock; vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_cleanup(&n->dev); if (tx_sock) fput(tx_sock->file); if (rx_sock) fput(rx_sock->file); /* We do an extra flush before freeing memory, * since jobs can re-queue themselves. */ vhost_net_flush(n); kfree(n); return 0; } static struct socket *get_raw_socket(int fd) { struct { struct sockaddr_ll sa; char buf[MAX_ADDR_LEN]; } uaddr; int uaddr_len = sizeof uaddr, r; struct socket *sock = sockfd_lookup(fd, &r); if (!sock) return ERR_PTR(-ENOTSOCK); /* Parameter checking */ if (sock->sk->sk_type != SOCK_RAW) { r = -ESOCKTNOSUPPORT; goto err; } r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa, &uaddr_len, 0); if (r) goto err; if (uaddr.sa.sll_family != AF_PACKET) { r = -EPFNOSUPPORT; goto err; } return sock; err: fput(sock->file); return ERR_PTR(r); } static struct socket *get_tap_socket(int fd) { struct file *file = fget(fd); struct socket *sock; if (!file) return ERR_PTR(-EBADF); sock = tun_get_socket(file); if (!IS_ERR(sock)) return sock; sock = macvtap_get_socket(file); if (IS_ERR(sock)) fput(file); return sock; } static struct socket *get_socket(int fd) { struct socket *sock; /* special case to disable backend */ if (fd == -1) return NULL; sock = get_raw_socket(fd); if (!IS_ERR(sock)) return sock; sock = get_tap_socket(fd); if (!IS_ERR(sock)) return sock; return ERR_PTR(-ENOTSOCK); } static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) { struct socket *sock, *oldsock; struct vhost_virtqueue *vq; int r; mutex_lock(&n->dev.mutex); r = vhost_dev_check_owner(&n->dev); if (r) goto err; if (index >= VHOST_NET_VQ_MAX) { r = -ENOBUFS; goto err; } vq = n->vqs + index; mutex_lock(&vq->mutex); /* Verify that ring has been setup correctly. */ if (!vhost_vq_access_ok(vq)) { r = -EFAULT; goto err_vq; } sock = get_socket(fd); if (IS_ERR(sock)) { r = PTR_ERR(sock); goto err_vq; } /* start polling new socket */ oldsock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); if (sock != oldsock) { vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, sock); vhost_net_enable_vq(n, vq); } mutex_unlock(&vq->mutex); if (oldsock) { vhost_net_flush_vq(n, index); fput(oldsock->file); } mutex_unlock(&n->dev.mutex); return 0; err_vq: mutex_unlock(&vq->mutex); err: mutex_unlock(&n->dev.mutex); return r; } static long vhost_net_reset_owner(struct vhost_net *n) { struct socket *tx_sock = NULL; struct socket *rx_sock = NULL; long err; mutex_lock(&n->dev.mutex); err = vhost_dev_check_owner(&n->dev); if (err) goto done; vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); err = vhost_dev_reset_owner(&n->dev); done: mutex_unlock(&n->dev.mutex); if (tx_sock) fput(tx_sock->file); if (rx_sock) fput(rx_sock->file); return err; } static int vhost_net_set_features(struct vhost_net *n, u64 features) { size_t vhost_hlen, sock_hlen, hdr_len; int i; hdr_len = (features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr); if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) { /* vhost provides vnet_hdr */ vhost_hlen = hdr_len; sock_hlen = 0; } else { /* socket provides vnet_hdr */ vhost_hlen = 0; sock_hlen = hdr_len; } mutex_lock(&n->dev.mutex); if ((features & (1 << VHOST_F_LOG_ALL)) && !vhost_log_access_ok(&n->dev)) { mutex_unlock(&n->dev.mutex); return -EFAULT; } n->dev.acked_features = features; smp_wmb(); for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { mutex_lock(&n->vqs[i].mutex); n->vqs[i].vhost_hlen = vhost_hlen; n->vqs[i].sock_hlen = sock_hlen; mutex_unlock(&n->vqs[i].mutex); } vhost_net_flush(n); mutex_unlock(&n->dev.mutex); return 0; }
/* Send message/packet data to user-land */ static int spx_recvmsg(struct socket *sock, struct msghdr *msg, int size, int flags, struct scm_cookie *scm) { struct sk_buff *skb; struct ipxspxhdr *ispxh; struct sock *sk = sock->sk; struct spx_opt *pdata = &sk->tp_pinfo.af_spx; struct sockaddr_ipx *sipx = (struct sockaddr_ipx *)msg->msg_name; int copied, err; if(sk->zapped) return (-ENOTCONN); /* Socket not bound */ lock_sock(sk); restart: while(skb_queue_empty(&pdata->rcv_queue)) /* No data */ { /* Socket errors? */ err = sock_error(sk); if(err) return (err); /* Socket shut down? */ if(sk->shutdown & RCV_SHUTDOWN) return (-ESHUTDOWN); /* handle signals */ if(signal_pending(current)) return (-ERESTARTSYS); /* User doesn't want to wait */ if(flags&MSG_DONTWAIT) return (-EAGAIN); release_sock(sk); save_flags(flags); cli(); if(skb_peek(&pdata->rcv_queue) == NULL) interruptible_sleep_on(sk->sleep); restore_flags(flags); lock_sock(sk); } skb = skb_dequeue(&pdata->rcv_queue); if(skb == NULL) goto restart; ispxh = (struct ipxspxhdr *)skb->nh.raw; copied = ntohs(ispxh->ipx.ipx_pktsize) - SPX_SYS_PKT_LEN; if(copied > size) { copied = size; msg->msg_flags |= MSG_TRUNC; } err = memcpy_toiovec(msg->msg_iov, skb->nh.raw+SPX_SYS_PKT_LEN, copied); if(err) return (-EFAULT); msg->msg_namelen = sizeof(*sipx); if(sipx) { sipx->sipx_family = AF_IPX; sipx->sipx_port = ispxh->ipx.ipx_source.sock; memcpy(sipx->sipx_node,ispxh->ipx.ipx_source.node,IPX_NODE_LEN); sipx->sipx_network = ispxh->ipx.ipx_source.net; sipx->sipx_type = ispxh->ipx.ipx_type; } kfree_skb(skb); release_sock(sk); return (copied); }
/* * Copy a datagram to an iovec. * Note: the iovec is modified during the copy. */ int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, struct iovec *to, int len) { int i, copy; int start = skb->len - skb->data_len; /* Copy header. */ if ((copy = start-offset) > 0) { if (copy > len) copy = len; if (memcpy_toiovec(to, skb->data + offset, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i=0; i<skb_shinfo(skb)->nr_frags; i++) { int end; BUG_TRAP(start <= offset+len); end = start + skb_shinfo(skb)->frags[i].size; if ((copy = end-offset) > 0) { int err; u8 *vaddr; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct page *page = frag->page; if (copy > len) copy = len; vaddr = kmap(page); err = memcpy_toiovec(to, vaddr + frag->page_offset + offset-start, copy); kunmap(page); if (err) goto fault; if (!(len -= copy)) return 0; offset += copy; } start = end; } if (skb_shinfo(skb)->frag_list) { struct sk_buff *list; for (list = skb_shinfo(skb)->frag_list; list; list=list->next) { int end; BUG_TRAP(start <= offset+len); end = start + list->len; if ((copy = end-offset) > 0) { if (copy > len) copy = len; if (skb_copy_datagram_iovec(list, offset-start, to, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; } } if (len == 0) return 0; fault: return -EFAULT; }
/** * skb_copy_datagram_iovec - Copy a datagram to an iovec. * @skb: buffer to copy * @offset: offset in the buffer to start copying from * @to: io vector to copy to * @len: amount of data to copy from buffer to iovec * * Note: the iovec is modified during the copy. */ int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, struct iovec *to, int len) { int start = skb_headlen(skb); int i, copy = start - offset; struct sk_buff *frag_iter; trace_skb_copy_datagram_iovec(skb, len); /* Copy header. */ if (copy > 0) { if (copy > len) copy = len; if (memcpy_toiovec(to, skb->data + offset, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; WARN_ON(start > offset + len); end = start + skb_frag_size(frag); if ((copy = end - offset) > 0) { int err; u8 *vaddr; struct page *page = skb_frag_page(frag); if (copy > len) copy = len; vaddr = kmap(page); err = memcpy_toiovec(to, vaddr + frag->page_offset + offset - start, copy); kunmap(page); if (err) goto fault; if (!(len -= copy)) return 0; offset += copy; } start = end; } skb_walk_frags(skb, frag_iter) { int end; WARN_ON(start > offset + len); end = start + frag_iter->len; if ((copy = end - offset) > 0) { if (copy > len) copy = len; if (skb_copy_datagram_iovec(frag_iter, offset - start, to, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; }
static int MksckDgramRecvMsg(struct kiocb *kiocb, struct socket *sock, struct msghdr *msg, size_t len, int flags) { int err = 0; struct sock *sk = sock->sk; Mksck *mksck; Mksck_Datagram *dg; struct sockaddr_mk *fromAddr; uint32 read; struct iovec *iov; size_t payloadLen, untypedLen; uint32 iovCount; if (flags & MSG_OOB || flags & MSG_ERRQUEUE) return -EOPNOTSUPP; if ((msg->msg_name != NULL) && (msg->msg_namelen < sizeof(*fromAddr))) return -EINVAL; lock_sock(sk); err = MksckTryBind(sk); if (err) { release_sock(sk); return err; } mksck = sk->sk_protinfo; ATOMIC_ADDV(mksck->refCount, 1); release_sock(sk); while (1) { err = Mutex_Lock(&mksck->mutex, MutexModeEX); if (err < 0) goto decRefc; read = mksck->read; if (read != mksck->write) break; if (flags & MSG_DONTWAIT) { Mutex_Unlock(&mksck->mutex, MutexModeEX); err = -EAGAIN; goto decRefc; } mksck->foundEmpty++; err = Mutex_UnlSleep(&mksck->mutex, MutexModeEX, MKSCK_CVAR_FILL); if (err < 0) { PRINTK("MksckDgramRecvMsg: aborted\n"); goto decRefc; } } dg = (void *)&mksck->buff[read]; if (msg->msg_name != NULL) { fromAddr = (void *)msg->msg_name; fromAddr->mk_addr = dg->fromAddr; fromAddr->mk_family = AF_MKSCK; msg->msg_namelen = sizeof(*fromAddr); } else { msg->msg_namelen = 0; } iov = msg->msg_iov; iovCount = msg->msg_iovlen; untypedLen = dg->len - dg->pages * sizeof(Mksck_PageDesc) - dg->pad; payloadLen = untypedLen; if (untypedLen <= len) { err = memcpy_toiovec(iov, dg->data, untypedLen); if (err < 0) { pr_warn("MksckDgramRecvMsg: Failed to " \ "memcpy_to_iovec untyped message component " \ "(buf len %d datagram len %d (untyped %d))\n", len, dg->len, untypedLen); } } else { err = -EINVAL; } if (err >= 0 && dg->pages > 0) { Mksck_PageDesc *pd = (Mksck_PageDesc *)(dg->data + untypedLen + dg->pad); if (msg->msg_controllen > 0) err = MksckPageDescToFd(sock, msg, pd, dg->pages); if ((msg->msg_controllen <= 0) || (err != 0) || (MsgHdrHasAvailableRoom(msg) != 0)) { down_write(¤t->mm->mmap_sem); payloadLen += MksckPageDescMap(pd, dg->pages, iov, iovCount, NULL); up_write(¤t->mm->mmap_sem); } } if ((err >= 0) && Mksck_IncReadIndex(mksck, read, dg)) Mutex_UnlWake(&mksck->mutex, MutexModeEX, MKSCK_CVAR_ROOM, true); else Mutex_Unlock(&mksck->mutex, MutexModeEX); if (err >= 0) err = payloadLen; decRefc: Mksck_DecRefc(mksck); return err; }