static void
pci_vtcon_sock_rx(int fd __unused, enum ev_type t __unused, void *arg)
{
struct pci_vtcon_port *port;
struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg;
struct vqueue_info *vq;
struct iovec iov;
static char dummybuf[2048];
int len, n;
uint16_t idx;
port = sock->vss_port;
vq = pci_vtcon_port_to_vq(port, true);
if (!sock->vss_open || !port->vsp_rx_ready) {
len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf));
if (len == 0)
goto close;
return;
}
if (!vq_has_descs(vq)) {
len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf));
vq_endchains(vq, 1);
if (len == 0)
goto close;
return;
}
do {
n = vq_getchain(vq, &idx, &iov, 1, NULL);
len = readv(sock->vss_conn_fd, &iov, n);
if (len == 0 || (len < 0 && errno == EWOULDBLOCK)) {
vq_retchain(vq);
vq_endchains(vq, 0);
if (len == 0)
goto close;
return;
}
vq_relchain(vq, idx, len);
} while (vq_has_descs(vq));
vq_endchains(vq, 1);
close:
mevent_delete_close(sock->vss_conn_evp);
sock->vss_conn_fd = -1;
sock->vss_open = false;
}
/*
* Thread which will handle processing of TX desc
*/
static void *
pci_vtnet_tx_thread(void *param)
{
struct pci_vtnet_softc *sc = param;
struct vqueue_info *vq;
int have_work, error;
vq = &sc->vsc_queues[VTNET_TXQ];
/*
* Let us wait till the tx queue pointers get initialised &
* first tx signaled
*/
pthread_mutex_lock(&sc->tx_mtx);
error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx);
assert(error == 0);
for (;;) {
/* note - tx mutex is locked here */
do {
if (sc->resetting)
have_work = 0;
else
have_work = vq_has_descs(vq);
if (!have_work) {
sc->tx_in_progress = 0;
error = pthread_cond_wait(&sc->tx_cond,
&sc->tx_mtx);
assert(error == 0);
}
} while (!have_work);
sc->tx_in_progress = 1;
pthread_mutex_unlock(&sc->tx_mtx);
vq_startchains(vq);
do {
/*
* Run through entries, placing them into
* iovecs and sending when an end-of-packet
* is found
*/
pci_vtnet_proctx(sc, vq);
} while (vq_has_descs(vq));
/*
* Generate an interrupt if needed.
*/
vq_endchains(vq, 1);
pthread_mutex_lock(&sc->tx_mtx);
}
}
/*
* Thread which will handle processing of TX desc
*/
static void *
pci_vtnet_tx_thread(void *param)
{
struct pci_vtnet_softc *sc = param;
struct vqueue_info *vq;
int error;
pthread_setname_np("net:tap:tx");
vq = &sc->vsc_queues[VTNET_TXQ];
/*
* Let us wait till the tx queue pointers get initialised &
* first tx signaled
*/
pthread_mutex_lock(&sc->tx_mtx);
error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx);
assert(error == 0);
for (;;) {
/* note - tx mutex is locked here */
while (sc->resetting || !vq_has_descs(vq)) {
vq->vq_used->vu_flags &= ~VRING_USED_F_NO_NOTIFY;
mb();
if (!sc->resetting && vq_has_descs(vq))
break;
sc->tx_in_progress = 0;
error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx);
assert(error == 0);
}
vq->vq_used->vu_flags |= VRING_USED_F_NO_NOTIFY;
sc->tx_in_progress = 1;
pthread_mutex_unlock(&sc->tx_mtx);
do {
/*
* Run through entries, placing them into
* iovecs and sending when an end-of-packet
* is found
*/
pci_vtnet_proctx(sc, vq);
} while (vq_has_descs(vq));
/*
* Generate an interrupt if needed.
*/
vq_endchains(vq, 1);
pthread_mutex_lock(&sc->tx_mtx);
}
}
static void
pci_vtrnd_notify(void *vsc, struct vqueue_info *vq)
{
struct iovec iov;
struct pci_vtrnd_softc *sc;
int len;
uint16_t idx;
sc = vsc;
if (sc->vrsc_fd < 0) {
vq_endchains(vq, 0);
return;
}
while (vq_has_descs(vq)) {
vq_getchain(vq, &idx, &iov, 1, NULL);
len = (int) read(sc->vrsc_fd, iov.iov_base, iov.iov_len);
DPRINTF(("vtrnd: vtrnd_notify(): %d\r\n", len));
/* Catastrophe if unable to read from /dev/random */
assert(len > 0);
/*
* Release this chain and handle more
*/
vq_relchain(vq, idx, (uint32_t)len);
}
vq_endchains(vq, 1); /* Generate interrupt if appropriate. */
}
static void
pci_vtcon_control_send(struct pci_vtcon_softc *sc,
struct pci_vtcon_control *ctrl, const void *payload, size_t len)
{
struct vqueue_info *vq;
struct iovec iov;
uint16_t idx;
int n;
vq = pci_vtcon_port_to_vq(&sc->vsc_control_port, true);
if (!vq_has_descs(vq))
return;
n = vq_getchain(vq, &idx, &iov, 1, NULL);
assert(n == 1);
memcpy(iov.iov_base, ctrl, sizeof(struct pci_vtcon_control));
if (payload != NULL && len > 0)
memcpy(iov.iov_base + sizeof(struct pci_vtcon_control),
payload, len);
vq_relchain(vq, idx, sizeof(struct pci_vtcon_control) + len);
vq_endchains(vq, 1);
}
static void
pci_vtblk_notify(void *vsc, struct vqueue_info *vq)
{
struct pci_vtblk_softc *sc = vsc;
while (vq_has_descs(vq))
pci_vtblk_proc(sc, vq);
}
static void
pci_vtblk_notify(void *vsc, struct vqueue_info *vq)
{
struct pci_vtblk_softc *sc = vsc;
vq_startchains(vq);
while (vq_has_descs(vq))
pci_vtblk_proc(sc, vq);
vq_endchains(vq, 1); /* Generate interrupt if appropriate. */
}
static void
pci_vtnet_ping_txq(void *vsc, struct vqueue_info *vq)
{
struct pci_vtnet_softc *sc = vsc;
/*
* Any ring entries to process?
*/
if (!vq_has_descs(vq))
return;
/* Signal the tx thread for processing */
pthread_mutex_lock(&sc->tx_mtx);
if (sc->tx_in_progress == 0)
pthread_cond_signal(&sc->tx_cond);
pthread_mutex_unlock(&sc->tx_mtx);
}
static void
vtbe_rxfinish_locked(struct vtbe_softc *sc)
{
struct vqueue_info *vq;
int reg;
/* TX queue */
vq = &sc->vs_queues[1];
if (!vq_ring_ready(vq))
return;
/* Process new descriptors */
vq->vq_save_used = be16toh(vq->vq_used->idx);
while (vq_has_descs(vq)) {
vtbe_proc_rx(sc, vq);
}
/* Interrupt the other side */
reg = htobe32(VIRTIO_MMIO_INT_VRING);
WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg);
PIO_SET(sc->pio_send, Q_INTR, 1);
}
static void
pci_vtcon_notify_tx(void *vsc, struct vqueue_info *vq)
{
struct pci_vtcon_softc *sc;
struct pci_vtcon_port *port;
struct iovec iov[1];
uint16_t idx, n;
uint16_t flags[8];
sc = vsc;
port = pci_vtcon_vq_to_port(sc, vq);
while (vq_has_descs(vq)) {
n = vq_getchain(vq, &idx, iov, 1, flags);
if (port != NULL)
port->vsp_cb(port, port->vsp_arg, iov, 1);
/*
* Release this chain and handle more
*/
vq_relchain(vq, idx, 0);
}
vq_endchains(vq, 1); /* Generate interrupt if appropriate. */
}
static void
pci_vtnet_tap_rx(struct pci_vtnet_softc *sc)
{
struct iovec iov[VTNET_MAXSEGS], *riov;
struct vqueue_info *vq;
void *vrx;
int len, n;
/*
* Should never be called without a valid tap fd
*/
assert(sc->vsc_tapfd != -1);
/*
* But, will be called when the rx ring hasn't yet
* been set up or the guest is resetting the device.
*/
if (!sc->vsc_rx_ready || sc->resetting) {
/*
* Drop the packet and try later.
*/
(void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf));
return;
}
/*
* Check for available rx buffers
*/
vq = &sc->vsc_queues[VTNET_RXQ];
vq_startchains(vq);
if (!vq_has_descs(vq)) {
/*
* Drop the packet and try later. Interrupt on
* empty, if that's negotiated.
*/
(void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf));
vq_endchains(vq, 1);
return;
}
do {
/*
* Get descriptor chain.
*/
n = vq_getchain(vq, iov, VTNET_MAXSEGS, NULL);
assert(n >= 1 && n <= VTNET_MAXSEGS);
/*
* Get a pointer to the rx header, and use the
* data immediately following it for the packet buffer.
*/
vrx = iov[0].iov_base;
riov = rx_iov_trim(iov, &n, sc->rx_vhdrlen);
len = readv(sc->vsc_tapfd, riov, n);
if (len < 0 && errno == EWOULDBLOCK) {
/*
* No more packets, but still some avail ring
* entries. Interrupt if needed/appropriate.
*/
vq_endchains(vq, 0);
return;
}
/*
* The only valid field in the rx packet header is the
* number of buffers if merged rx bufs were negotiated.
*/
memset(vrx, 0, sc->rx_vhdrlen);
if (sc->rx_merge) {
struct virtio_net_rxhdr *vrxh;
vrxh = vrx;
vrxh->vrh_bufs = 1;
}
/*
* Release this chain and handle more chains.
*/
vq_relchain(vq, len + sc->rx_vhdrlen);
} while (vq_has_descs(vq));
/* Interrupt if needed, including for NOTIFY_ON_EMPTY. */
vq_endchains(vq, 1);
}
static void
pci_vtnet_tap_rx(struct pci_vtnet_softc *sc)
{
struct vqueue_info *vq;
struct virtio_net_rxhdr *vrx;
uint8_t *buf;
int len;
struct iovec iov;
/*
* Should never be called without a valid tap fd
*/
assert(sc->vsc_tapfd != -1);
/*
* But, will be called when the rx ring hasn't yet
* been set up or the guest is resetting the device.
*/
if (!sc->vsc_rx_ready || sc->resetting) {
/*
* Drop the packet and try later.
*/
(void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf));
return;
}
/*
* Check for available rx buffers
*/
vq = &sc->vsc_queues[VTNET_RXQ];
vq_startchains(vq);
if (!vq_has_descs(vq)) {
/*
* Drop the packet and try later. Interrupt on
* empty, if that's negotiated.
*/
(void) read(sc->vsc_tapfd, dummybuf, sizeof(dummybuf));
vq_endchains(vq, 1);
return;
}
do {
/*
* Get descriptor chain, which should have just
* one descriptor in it.
* ??? allow guests to use multiple descs?
*/
assert(vq_getchain(vq, &iov, 1, NULL) == 1);
/*
* Get a pointer to the rx header, and use the
* data immediately following it for the packet buffer.
*/
vrx = iov.iov_base;
buf = (uint8_t *)(vrx + 1);
len = read(sc->vsc_tapfd, buf,
iov.iov_len - sizeof(struct virtio_net_rxhdr));
if (len < 0 && errno == EWOULDBLOCK) {
/*
* No more packets, but still some avail ring
* entries. Interrupt if needed/appropriate.
*/
vq_endchains(vq, 0);
return;
}
/*
* The only valid field in the rx packet header is the
* number of buffers, which is always 1 without TSO
* support.
*/
memset(vrx, 0, sizeof(struct virtio_net_rxhdr));
vrx->vrh_bufs = 1;
/*
* Release this chain and handle more chains.
*/
vq_relchain(vq, len + sizeof(struct virtio_net_rxhdr));
} while (vq_has_descs(vq));
/* Interrupt if needed, including for NOTIFY_ON_EMPTY. */
vq_endchains(vq, 1);
}
static void
vtbe_txstart_locked(struct vtbe_softc *sc)
{
struct virtio_net_hdr_mrg_rxbuf *vnh;
struct iovec iov[DESC_COUNT];
struct vqueue_info *vq;
struct iovec *riov;
struct ifnet *ifp;
struct mbuf *m;
struct uio uio;
int enqueued;
int iolen;
int error;
int *addr;
int reg;
int len;
int n;
VTBE_ASSERT_LOCKED(sc);
/* RX queue */
vq = &sc->vs_queues[0];
if (!vq_has_descs(vq)) {
return;
}
ifp = sc->ifp;
if (ifp->if_drv_flags & IFF_DRV_OACTIVE) {
return;
}
enqueued = 0;
if (!vq_ring_ready(vq))
return;
vq->vq_save_used = be16toh(vq->vq_used->idx);
for (;;) {
if (!vq_has_descs(vq)) {
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
break;
}
IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
if (m == NULL) {
break;
}
n = vq_getchain(sc->beri_mem_offset, vq, iov,
DESC_COUNT, NULL);
KASSERT(n >= 1 && n <= DESC_COUNT,
("wrong descriptors num %d", n));
addr = iov[0].iov_base;
len = iov[0].iov_len;
vnh = iov[0].iov_base;
memset(vnh, 0, sc->hdrsize);
vnh->num_buffers = htobe16(1);
iov[0].iov_len -= sc->hdrsize;
iov[0].iov_base = (void *)((uintptr_t)iov[0].iov_base +
sc->hdrsize);
riov = &iov[0];
uio.uio_resid = iov[0].iov_len;
uio.uio_iov = riov;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_iovcnt = 1;
uio.uio_offset = 0;
uio.uio_rw = UIO_READ;
error = m_mbuftouio(&uio, m, 0);
if (error)
panic("m_mbuftouio failed\n");
iolen = (len - iov[0].iov_len - sc->hdrsize);
vq_relchain(vq, iov, 0, iolen + sc->hdrsize);
paddr_unmap((void *)addr, len);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
BPF_MTAP(ifp, m);
m_freem(m);
++enqueued;
}
if (enqueued != 0) {
reg = htobe32(VIRTIO_MMIO_INT_VRING);
WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg);
PIO_SET(sc->pio_send, Q_INTR, 1);
}
}
