static void pci_vtrnd_notify(void *vsc, struct vqueue_info *vq) { struct iovec iov; struct pci_vtrnd_softc *sc; int len; uint16_t idx; sc = vsc; if (sc->vrsc_fd < 0) { vq_endchains(vq, 0); return; } while (vq_has_descs(vq)) { vq_getchain(vq, &idx, &iov, 1, NULL); len = (int) read(sc->vrsc_fd, iov.iov_base, iov.iov_len); DPRINTF(("vtrnd: vtrnd_notify(): %d\r\n", len)); /* Catastrophe if unable to read from /dev/random */ assert(len > 0); /* * Release this chain and handle more */ vq_relchain(vq, idx, (uint32_t)len); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ }
static void pci_vtnet_proctx(struct pci_vtnet_softc *sc, struct vqueue_info *vq) { struct iovec iov[VTNET_MAXSEGS + 1]; int i, n; int plen, tlen; /* * Obtain chain of descriptors. The first one is * really the header descriptor, so we need to sum * up two lengths: packet length and transfer length. */ n = vq_getchain(vq, iov, VTNET_MAXSEGS, NULL); assert(n >= 1 && n <= VTNET_MAXSEGS); plen = 0; tlen = iov[0].iov_len; for (i = 1; i < n; i++) { plen += iov[i].iov_len; tlen += iov[i].iov_len; } DPRINTF(("virtio: packet send, %d bytes, %d segs\n\r", plen, n)); pci_vtnet_tap_tx(sc, &iov[1], n - 1, plen); /* chain is processed, release it and set tlen */ vq_relchain(vq, tlen); }
static void pci_vtcon_control_send(struct pci_vtcon_softc *sc, struct pci_vtcon_control *ctrl, const void *payload, size_t len) { struct vqueue_info *vq; struct iovec iov; uint16_t idx; int n; vq = pci_vtcon_port_to_vq(&sc->vsc_control_port, true); if (!vq_has_descs(vq)) return; n = vq_getchain(vq, &idx, &iov, 1, NULL); assert(n == 1); memcpy(iov.iov_base, ctrl, sizeof(struct pci_vtcon_control)); if (payload != NULL && len > 0) memcpy(iov.iov_base + sizeof(struct pci_vtcon_control), payload, len); vq_relchain(vq, idx, sizeof(struct pci_vtcon_control) + len); vq_endchains(vq, 1); }
static void pci_vtcon_sock_rx(int fd __unused, enum ev_type t __unused, void *arg) { struct pci_vtcon_port *port; struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg; struct vqueue_info *vq; struct iovec iov; static char dummybuf[2048]; int len, n; uint16_t idx; port = sock->vss_port; vq = pci_vtcon_port_to_vq(port, true); if (!sock->vss_open || !port->vsp_rx_ready) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); if (len == 0) goto close; return; } if (!vq_has_descs(vq)) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); vq_endchains(vq, 1); if (len == 0) goto close; return; } do { n = vq_getchain(vq, &idx, &iov, 1, NULL); len = readv(sock->vss_conn_fd, &iov, n); if (len == 0 || (len < 0 && errno == EWOULDBLOCK)) { vq_retchain(vq); vq_endchains(vq, 0); if (len == 0) goto close; return; } vq_relchain(vq, idx, len); } while (vq_has_descs(vq)); vq_endchains(vq, 1); close: mevent_delete_close(sock->vss_conn_evp); sock->vss_conn_fd = -1; sock->vss_open = false; }
static void vtbe_proc_rx(struct vtbe_softc *sc, struct vqueue_info *vq) { struct iovec iov[DESC_COUNT]; struct ifnet *ifp; struct uio uio; struct mbuf *m; int iolen; int i; int n; ifp = sc->ifp; n = vq_getchain(sc->beri_mem_offset, vq, iov, DESC_COUNT, NULL); KASSERT(n >= 1 && n <= DESC_COUNT, ("wrong n %d", n)); iolen = 0; for (i = 1; i < n; i++) { iolen += iov[i].iov_len; } uio.uio_resid = iolen; uio.uio_iov = &iov[1]; uio.uio_segflg = UIO_SYSSPACE; uio.uio_iovcnt = (n - 1); uio.uio_rw = UIO_WRITE; if ((m = m_uiotombuf(&uio, M_NOWAIT, 0, ETHER_ALIGN, M_PKTHDR)) == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); goto done; } m->m_pkthdr.rcvif = ifp; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); VTBE_UNLOCK(sc); (*ifp->if_input)(ifp, m); VTBE_LOCK(sc); CURVNET_RESTORE(); done: vq_relchain(vq, iov, n, iolen + sc->hdrsize); }
static void pci_vtcon_notify_tx(void *vsc, struct vqueue_info *vq) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *port; struct iovec iov[1]; uint16_t idx, n; uint16_t flags[8]; sc = vsc; port = pci_vtcon_vq_to_port(sc, vq); while (vq_has_descs(vq)) { n = vq_getchain(vq, &idx, iov, 1, flags); if (port != NULL) port->vsp_cb(port, port->vsp_arg, iov, 1); /* * Release this chain and handle more */ vq_relchain(vq, idx, 0); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ }
static void pci_vtnet_tap_rx(struct pci_vtnet_softc *sc) { struct iovec iov[VTNET_MAXSEGS], *riov; struct vqueue_info *vq; void *vrx; int len, n; /* * Should never be called without a valid tap fd */ assert(sc->vsc_tapfd != -1); /* * But, will be called when the rx ring hasn't yet * been set up or the guest is resetting the device. */ if (!sc->vsc_rx_ready || sc->resetting) { /* * Drop the packet and try later. */ (void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf)); return; } /* * Check for available rx buffers */ vq = &sc->vsc_queues[VTNET_RXQ]; vq_startchains(vq); if (!vq_has_descs(vq)) { /* * Drop the packet and try later. Interrupt on * empty, if that's negotiated. */ (void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf)); vq_endchains(vq, 1); return; } do { /* * Get descriptor chain. */ n = vq_getchain(vq, iov, VTNET_MAXSEGS, NULL); assert(n >= 1 && n <= VTNET_MAXSEGS); /* * Get a pointer to the rx header, and use the * data immediately following it for the packet buffer. */ vrx = iov[0].iov_base; riov = rx_iov_trim(iov, &n, sc->rx_vhdrlen); len = readv(sc->vsc_tapfd, riov, n); if (len < 0 && errno == EWOULDBLOCK) { /* * No more packets, but still some avail ring * entries. Interrupt if needed/appropriate. */ vq_endchains(vq, 0); return; } /* * The only valid field in the rx packet header is the * number of buffers if merged rx bufs were negotiated. */ memset(vrx, 0, sc->rx_vhdrlen); if (sc->rx_merge) { struct virtio_net_rxhdr *vrxh; vrxh = vrx; vrxh->vrh_bufs = 1; } /* * Release this chain and handle more chains. */ vq_relchain(vq, len + sc->rx_vhdrlen); } while (vq_has_descs(vq)); /* Interrupt if needed, including for NOTIFY_ON_EMPTY. */ vq_endchains(vq, 1); }
static void pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vqueue_info *vq) { struct virtio_blk_hdr *vbh; struct pci_vtblk_ioreq *io; int i, n; int err; ssize_t iolen; int writeop, type; struct iovec iov[BLOCKIF_IOV_MAX + 2]; uint16_t idx, flags[BLOCKIF_IOV_MAX + 2]; n = vq_getchain(vq, &idx, iov, BLOCKIF_IOV_MAX + 2, flags); /* * The first descriptor will be the read-only fixed header, * and the last is for status (hence +2 above and below). * The remaining iov's are the actual data I/O vectors. * * XXX - note - this fails on crash dump, which does a * VIRTIO_BLK_T_FLUSH with a zero transfer length */ assert(n >= 2 && n <= BLOCKIF_IOV_MAX + 2); io = &sc->vbsc_ios[idx]; assert((flags[0] & VRING_DESC_F_WRITE) == 0); assert(iov[0].iov_len == sizeof(struct virtio_blk_hdr)); vbh = iov[0].iov_base; memcpy(&io->io_req.br_iov, &iov[1], sizeof(struct iovec) * ((size_t)n - 2)); io->io_req.br_iovcnt = n - 2; io->io_req.br_offset = (off_t)(vbh->vbh_sector * DEV_BSIZE); io->io_status = iov[--n].iov_base; assert(iov[n].iov_len == 1); assert(flags[n] & VRING_DESC_F_WRITE); /* * XXX * The guest should not be setting the BARRIER flag because * we don't advertise the capability. */ type = vbh->vbh_type & ~VBH_FLAG_BARRIER; writeop = (type == VBH_OP_WRITE); iolen = 0; for (i = 1; i < n; i++) { /* * - write op implies read-only descriptor, * - read/ident op implies write-only descriptor, * therefore test the inverse of the descriptor bit * to the op. */ assert(((flags[i] & VRING_DESC_F_WRITE) == 0) == writeop); iolen += iov[i].iov_len; } io->io_req.br_resid = iolen; DPRINTF(("virtio-block: %s op, %zd bytes, %d segs\n\r", writeop ? "write" : "read/ident", iolen, i - 1)); switch (type) { case VBH_OP_READ: err = blockif_read(sc->bc, &io->io_req); break; case VBH_OP_WRITE: err = blockif_write(sc->bc, &io->io_req); break; case VBH_OP_FLUSH: case VBH_OP_FLUSH_OUT: err = blockif_flush(sc->bc, &io->io_req); break; case VBH_OP_IDENT: /* Assume a single buffer */ /* S/n equal to buffer is not zero-terminated. */ memset(iov[1].iov_base, 0, iov[1].iov_len); strncpy(iov[1].iov_base, sc->vbsc_ident, MIN(iov[1].iov_len, sizeof(sc->vbsc_ident))); /* xhyve: FIXME */ pci_vtblk_done_locked(&io->io_req, 0); return; default: /* xhyve: FIXME */ pci_vtblk_done_locked(&io->io_req, EOPNOTSUPP); return; } assert(err == 0); }
static void pci_vtnet_tap_rx(struct pci_vtnet_softc *sc) { struct vqueue_info *vq; struct virtio_net_rxhdr *vrx; uint8_t *buf; int len; struct iovec iov; /* * Should never be called without a valid tap fd */ assert(sc->vsc_tapfd != -1); /* * But, will be called when the rx ring hasn't yet * been set up or the guest is resetting the device. */ if (!sc->vsc_rx_ready || sc->resetting) { /* * Drop the packet and try later. */ (void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf)); return; } /* * Check for available rx buffers */ vq = &sc->vsc_queues[VTNET_RXQ]; vq_startchains(vq); if (!vq_has_descs(vq)) { /* * Drop the packet and try later. Interrupt on * empty, if that's negotiated. */ (void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf)); vq_endchains(vq, 1); return; } do { /* * Get descriptor chain, which should have just * one descriptor in it. * ??? allow guests to use multiple descs? */ assert(vq_getchain(vq, &iov, 1, NULL) == 1); /* * Get a pointer to the rx header, and use the * data immediately following it for the packet buffer. */ vrx = iov.iov_base; buf = (uint8_t *)(vrx + 1); len = read(sc->vsc_tapfd, buf, iov.iov_len - sizeof(struct virtio_net_rxhdr)); if (len < 0 && errno == EWOULDBLOCK) { /* * No more packets, but still some avail ring * entries. Interrupt if needed/appropriate. */ vq_endchains(vq, 0); return; } /* * The only valid field in the rx packet header is the * number of buffers, which is always 1 without TSO * support. */ memset(vrx, 0, sizeof(struct virtio_net_rxhdr)); vrx->vrh_bufs = 1; /* * Release this chain and handle more chains. */ vq_relchain(vq, len + sizeof(struct virtio_net_rxhdr)); } while (vq_has_descs(vq)); /* Interrupt if needed, including for NOTIFY_ON_EMPTY. */ vq_endchains(vq, 1); }
static void pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vqueue_info *vq) { struct virtio_blk_hdr *vbh; uint8_t *status; int i, n; int err; int iolen; int writeop, type; off_t offset; struct iovec iov[VTBLK_MAXSEGS + 2]; uint16_t flags[VTBLK_MAXSEGS + 2]; n = vq_getchain(vq, iov, VTBLK_MAXSEGS + 2, flags); /* * The first descriptor will be the read-only fixed header, * and the last is for status (hence +2 above and below). * The remaining iov's are the actual data I/O vectors. * * XXX - note - this fails on crash dump, which does a * VIRTIO_BLK_T_FLUSH with a zero transfer length */ assert(n >= 2 && n <= VTBLK_MAXSEGS + 2); assert((flags[0] & VRING_DESC_F_WRITE) == 0); assert(iov[0].iov_len == sizeof(struct virtio_blk_hdr)); vbh = iov[0].iov_base; status = iov[--n].iov_base; assert(iov[n].iov_len == 1); assert(flags[n] & VRING_DESC_F_WRITE); /* * XXX * The guest should not be setting the BARRIER flag because * we don't advertise the capability. */ type = vbh->vbh_type & ~VBH_FLAG_BARRIER; writeop = (type == VBH_OP_WRITE); offset = vbh->vbh_sector * DEV_BSIZE; iolen = 0; for (i = 1; i < n; i++) { /* * - write op implies read-only descriptor, * - read/ident op implies write-only descriptor, * therefore test the inverse of the descriptor bit * to the op. */ assert(((flags[i] & VRING_DESC_F_WRITE) == 0) == writeop); iolen += iov[i].iov_len; } DPRINTF(("virtio-block: %s op, %d bytes, %d segs, offset %ld\n\r", writeop ? "write" : "read/ident", iolen, i - 1, offset)); switch (type) { case VBH_OP_WRITE: err = pwritev(sc->vbsc_fd, iov + 1, i - 1, offset); break; case VBH_OP_READ: err = preadv(sc->vbsc_fd, iov + 1, i - 1, offset); break; case VBH_OP_IDENT: /* Assume a single buffer */ strlcpy(iov[1].iov_base, sc->vbsc_ident, MIN(iov[1].iov_len, sizeof(sc->vbsc_ident))); err = 0; break; case VBH_OP_FLUSH: case VBH_OP_FLUSH_OUT: err = fsync(sc->vbsc_fd); break; default: err = -ENOSYS; break; } /* convert errno into a virtio block error return */ if (err < 0) { if (err == -ENOSYS) *status = VTBLK_S_UNSUPP; else *status = VTBLK_S_IOERR; } else *status = VTBLK_S_OK; /* * Return the descriptor back to the host. * We wrote 1 byte (our status) to host. */ vq_relchain(vq, 1); }
static void vtbe_txstart_locked(struct vtbe_softc *sc) { struct virtio_net_hdr_mrg_rxbuf *vnh; struct iovec iov[DESC_COUNT]; struct vqueue_info *vq; struct iovec *riov; struct ifnet *ifp; struct mbuf *m; struct uio uio; int enqueued; int iolen; int error; int *addr; int reg; int len; int n; VTBE_ASSERT_LOCKED(sc); /* RX queue */ vq = &sc->vs_queues[0]; if (!vq_has_descs(vq)) { return; } ifp = sc->ifp; if (ifp->if_drv_flags & IFF_DRV_OACTIVE) { return; } enqueued = 0; if (!vq_ring_ready(vq)) return; vq->vq_save_used = be16toh(vq->vq_used->idx); for (;;) { if (!vq_has_descs(vq)) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) { break; } n = vq_getchain(sc->beri_mem_offset, vq, iov, DESC_COUNT, NULL); KASSERT(n >= 1 && n <= DESC_COUNT, ("wrong descriptors num %d", n)); addr = iov[0].iov_base; len = iov[0].iov_len; vnh = iov[0].iov_base; memset(vnh, 0, sc->hdrsize); vnh->num_buffers = htobe16(1); iov[0].iov_len -= sc->hdrsize; iov[0].iov_base = (void *)((uintptr_t)iov[0].iov_base + sc->hdrsize); riov = &iov[0]; uio.uio_resid = iov[0].iov_len; uio.uio_iov = riov; uio.uio_segflg = UIO_SYSSPACE; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_rw = UIO_READ; error = m_mbuftouio(&uio, m, 0); if (error) panic("m_mbuftouio failed\n"); iolen = (len - iov[0].iov_len - sc->hdrsize); vq_relchain(vq, iov, 0, iolen + sc->hdrsize); paddr_unmap((void *)addr, len); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); BPF_MTAP(ifp, m); m_freem(m); ++enqueued; } if (enqueued != 0) { reg = htobe32(VIRTIO_MMIO_INT_VRING); WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); PIO_SET(sc->pio_send, Q_INTR, 1); } }