/* Transmit DMA interrupt service */
static inline void sca_tx_intr(port_t *port)
{
struct net_device *dev = port_to_dev(port);
u16 dmac = get_dmac_tx(port);
card_t* card = port_to_card(port);
u8 stat;
spin_lock(&port->lock);
stat = sca_in(DSR_TX(phy_node(port)), card); /* read DMA Status */
/* Reset DSR status bits */
sca_out((stat & (DSR_EOT | DSR_EOM | DSR_BOF | DSR_COF)) | DSR_DWE,
DSR_TX(phy_node(port)), card);
while (1) {
pkt_desc __iomem *desc;
u32 desc_off = desc_offset(port, port->txlast, 1);
u32 cda = sca_inw(dmac + CDAL, card);
if ((cda >= desc_off) && (cda < desc_off + sizeof(pkt_desc)))
break; /* Transmitter is/will_be sending this frame */
desc = desc_address(port, port->txlast, 1);
dev->stats.tx_packets++;
dev->stats.tx_bytes += readw(&desc->len);
writeb(0, &desc->stat); /* Free descriptor */
port->txlast = next_desc(port, port->txlast, 1);
}
netif_wake_queue(dev);
spin_unlock(&port->lock);
}
static inline void sca_rx_intr(port_t *port)
{
struct net_device *dev = port_to_dev(port);
u16 dmac = get_dmac_rx(port);
card_t *card = port_to_card(port);
u8 stat = sca_in(DSR_RX(phy_node(port)), card);
sca_out((stat & (DSR_EOT | DSR_EOM | DSR_BOF | DSR_COF)) | DSR_DWE,
DSR_RX(phy_node(port)), card);
if (stat & DSR_BOF)
dev->stats.rx_over_errors++;
while (1) {
u32 desc_off = desc_offset(port, port->rxin, 0);
pkt_desc __iomem *desc;
u32 cda = sca_inw(dmac + CDAL, card);
if ((cda >= desc_off) && (cda < desc_off + sizeof(pkt_desc)))
break;
desc = desc_address(port, port->rxin, 0);
stat = readb(&desc->stat);
if (!(stat & ST_RX_EOM))
port->rxpart = 1;
else if ((stat & ST_ERROR_MASK) || port->rxpart) {
dev->stats.rx_errors++;
if (stat & ST_RX_OVERRUN)
dev->stats.rx_fifo_errors++;
else if ((stat & (ST_RX_SHORT | ST_RX_ABORT |
ST_RX_RESBIT)) || port->rxpart)
dev->stats.rx_frame_errors++;
else if (stat & ST_RX_CRC)
dev->stats.rx_crc_errors++;
if (stat & ST_RX_EOM)
port->rxpart = 0;
} else
sca_rx(card, port, desc, port->rxin);
sca_outw(desc_off, dmac + EDAL, card);
port->rxin = next_desc(port, port->rxin, 0);
}
sca_out(DSR_DE, DSR_RX(phy_node(port)), card);
}
/* Receive DMA interrupt service */
static inline void sca_rx_intr(port_t *port)
{
struct net_device *dev = port_to_dev(port);
u16 dmac = get_dmac_rx(port);
card_t *card = port_to_card(port);
u8 stat = sca_in(DSR_RX(phy_node(port)), card); /* read DMA Status */
/* Reset DSR status bits */
sca_out((stat & (DSR_EOT | DSR_EOM | DSR_BOF | DSR_COF)) | DSR_DWE,
DSR_RX(phy_node(port)), card);
if (stat & DSR_BOF)
/* Dropped one or more frames */
dev->stats.rx_over_errors++;
while (1) {
u32 desc_off = desc_offset(port, port->rxin, 0);
pkt_desc __iomem *desc;
u32 cda = sca_inw(dmac + CDAL, card);
if ((cda >= desc_off) && (cda < desc_off + sizeof(pkt_desc)))
break; /* No frame received */
desc = desc_address(port, port->rxin, 0);
stat = readb(&desc->stat);
if (!(stat & ST_RX_EOM))
port->rxpart = 1; /* partial frame received */
else if ((stat & ST_ERROR_MASK) || port->rxpart) {
dev->stats.rx_errors++;
if (stat & ST_RX_OVERRUN)
dev->stats.rx_fifo_errors++;
else if ((stat & (ST_RX_SHORT | ST_RX_ABORT |
ST_RX_RESBIT)) || port->rxpart)
dev->stats.rx_frame_errors++;
else if (stat & ST_RX_CRC)
dev->stats.rx_crc_errors++;
if (stat & ST_RX_EOM)
port->rxpart = 0; /* received last fragment */
} else
sca_rx(card, port, desc, port->rxin);
/* Set new error descriptor address */
sca_outw(desc_off, dmac + EDAL, card);
port->rxin = next_desc(port, port->rxin, 0);
}
/* make sure RX DMA is enabled */
sca_out(DSR_DE, DSR_RX(phy_node(port)), card);
}
u16 virtio_queue_get_head_iovec(struct virtio_queue *vq,
u16 head, struct virtio_iovec *iov,
u32 *ret_iov_cnt, u32 *ret_total_len)
{
struct vring_desc *desc;
u16 idx, max;
int i = 0;
if (!vq->addr) {
*ret_iov_cnt = 0;
*ret_total_len = 0;
return 0;
}
idx = head;
*ret_iov_cnt = 0;
*ret_total_len = 0;
max = vq->vring.num;
desc = vq->vring.desc;
if (desc[idx].flags & VRING_DESC_F_INDIRECT) {
#if 0
max = desc[idx].len / sizeof(struct vring_desc);
desc = guest_flat_to_host(kvm, desc[idx].addr);
idx = 0;
#endif
}
do {
iov[i].addr = desc[idx].addr;
iov[i].len = desc[idx].len;
*ret_total_len += desc[idx].len;
if (desc[idx].flags & VRING_DESC_F_WRITE) {
iov[i].flags = 1; /* Write */
} else {
iov[i].flags = 0; /* Read */
}
i++;
} while ((idx = next_desc(desc, idx, max)) != max);
*ret_iov_cnt = i;
return head;
}
static unsigned wait_for_vq_desc(struct virtqueue *vq,
struct iovec iov[],
unsigned int *out_num, unsigned int *in_num)
{
unsigned int i, head, max;
struct vring_desc *desc;
u16 last_avail = lg_last_avail(vq);
while (last_avail == vq->vring.avail->idx) {
u64 event;
trigger_irq(vq);
vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
/*
* They could have slipped one in as we were doing that: make
* sure it's written, then check again.
*/
mb();
if (last_avail != vq->vring.avail->idx) {
vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
break;
}
if (read(vq->eventfd, &event, sizeof(event)) != sizeof(event))
errx(1, "Event read failed?");
vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
}
if ((u16)(vq->vring.avail->idx - last_avail) > vq->vring.num)
errx(1, "Guest moved used index from %u to %u",
last_avail, vq->vring.avail->idx);
head = vq->vring.avail->ring[last_avail % vq->vring.num];
lg_last_avail(vq)++;
if (head >= vq->vring.num)
errx(1, "Guest says index %u is available", head);
*out_num = *in_num = 0;
max = vq->vring.num;
desc = vq->vring.desc;
i = head;
if (desc[i].flags & VRING_DESC_F_INDIRECT) {
if (desc[i].len % sizeof(struct vring_desc))
errx(1, "Invalid size for indirect buffer table");
max = desc[i].len / sizeof(struct vring_desc);
desc = check_pointer(desc[i].addr, desc[i].len);
i = 0;
}
do {
iov[*out_num + *in_num].iov_len = desc[i].len;
iov[*out_num + *in_num].iov_base
= check_pointer(desc[i].addr, desc[i].len);
if (desc[i].flags & VRING_DESC_F_WRITE)
(*in_num)++;
else {
if (*in_num)
errx(1, "Descriptor has out after in");
(*out_num)++;
}
if (*out_num + *in_num > max)
errx(1, "Looped descriptor");
} while ((i = next_desc(desc, i, max)) != max);
return head;
}
/* This looks in the virtqueue and for the first available buffer, and converts
* it to an iovec for convenient access. Since descriptors consist of some
* number of output then some number of input descriptors, it's actually two
* iovecs, but we pack them into one and note how many of each there were.
*
* This function returns the descriptor number found, or vq->num (which
* is never a valid descriptor number) if none was found. */
unsigned vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
struct iovec iov[], unsigned int iov_size,
unsigned int *out_num, unsigned int *in_num,
struct vhost_log *log, unsigned int *log_num)
{
struct vring_desc desc;
unsigned int i, head, found = 0;
u16 last_avail_idx;
int ret;
/* Check it isn't doing very strange things with descriptor numbers. */
last_avail_idx = vq->last_avail_idx;
if (get_user(vq->avail_idx, &vq->avail->idx)) {
vq_err(vq, "Failed to access avail idx at %p\n",
&vq->avail->idx);
return vq->num;
}
if ((u16)(vq->avail_idx - last_avail_idx) > vq->num) {
vq_err(vq, "Guest moved used index from %u to %u",
last_avail_idx, vq->avail_idx);
return vq->num;
}
/* If there's nothing new since last we looked, return invalid. */
if (vq->avail_idx == last_avail_idx)
return vq->num;
/* Only get avail ring entries after they have been exposed by guest. */
smp_rmb();
/* Grab the next descriptor number they're advertising, and increment
* the index we've seen. */
if (get_user(head, &vq->avail->ring[last_avail_idx % vq->num])) {
vq_err(vq, "Failed to read head: idx %d address %p\n",
last_avail_idx,
&vq->avail->ring[last_avail_idx % vq->num]);
return vq->num;
}
/* If their number is silly, that's an error. */
if (head >= vq->num) {
vq_err(vq, "Guest says index %u > %u is available",
head, vq->num);
return vq->num;
}
/* When we start there are none of either input nor output. */
*out_num = *in_num = 0;
if (unlikely(log))
*log_num = 0;
i = head;
do {
unsigned iov_count = *in_num + *out_num;
if (i >= vq->num) {
vq_err(vq, "Desc index is %u > %u, head = %u",
i, vq->num, head);
return vq->num;
}
if (++found > vq->num) {
vq_err(vq, "Loop detected: last one at %u "
"vq size %u head %u\n",
i, vq->num, head);
return vq->num;
}
ret = copy_from_user(&desc, vq->desc + i, sizeof desc);
if (ret) {
vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
i, vq->desc + i);
return vq->num;
}
if (desc.flags & VRING_DESC_F_INDIRECT) {
ret = get_indirect(dev, vq, iov, iov_size,
out_num, in_num,
log, log_num, &desc);
if (ret < 0) {
vq_err(vq, "Failure detected "
"in indirect descriptor at idx %d\n", i);
return vq->num;
}
continue;
}
ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
iov_size - iov_count);
if (ret < 0) {
vq_err(vq, "Translation failure %d descriptor idx %d\n",
ret, i);
return vq->num;
}
if (desc.flags & VRING_DESC_F_WRITE) {
/* If this is an input descriptor,
* increment that count. */
*in_num += ret;
if (unlikely(log)) {
log[*log_num].addr = desc.addr;
log[*log_num].len = desc.len;
++*log_num;
}
} else {
/* If it's an output descriptor, they're all supposed
* to come before any input descriptors. */
if (*in_num) {
vq_err(vq, "Descriptor has out after in: "
"idx %d\n", i);
return vq->num;
}
*out_num += ret;
}
} while ((i = next_desc(&desc)) != -1);
/* On success, increment avail index. */
vq->last_avail_idx++;
return head;
}
static unsigned get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
struct iovec iov[], unsigned int iov_size,
unsigned int *out_num, unsigned int *in_num,
struct vhost_log *log, unsigned int *log_num,
struct vring_desc *indirect)
{
struct vring_desc desc;
unsigned int i = 0, count, found = 0;
int ret;
/* Sanity check */
if (indirect->len % sizeof desc) {
vq_err(vq, "Invalid length in indirect descriptor: "
"len 0x%llx not multiple of 0x%zx\n",
(unsigned long long)indirect->len,
sizeof desc);
return -EINVAL;
}
ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
ARRAY_SIZE(vq->indirect));
if (ret < 0) {
vq_err(vq, "Translation failure %d in indirect.\n", ret);
return ret;
}
/* We will use the result as an address to read from, so most
* architectures only need a compiler barrier here. */
read_barrier_depends();
count = indirect->len / sizeof desc;
/* Buffers are chained via a 16 bit next field, so
* we can have at most 2^16 of these. */
if (count > USHORT_MAX + 1) {
vq_err(vq, "Indirect buffer length too big: %d\n",
indirect->len);
return -E2BIG;
}
do {
unsigned iov_count = *in_num + *out_num;
if (++found > count) {
vq_err(vq, "Loop detected: last one at %u "
"indirect size %u\n",
i, count);
return -EINVAL;
}
if (memcpy_fromiovec((unsigned char *)&desc, vq->indirect,
sizeof desc)) {
vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
i, (size_t)indirect->addr + i * sizeof desc);
return -EINVAL;
}
if (desc.flags & VRING_DESC_F_INDIRECT) {
vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
i, (size_t)indirect->addr + i * sizeof desc);
return -EINVAL;
}
ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
iov_size - iov_count);
if (ret < 0) {
vq_err(vq, "Translation failure %d indirect idx %d\n",
ret, i);
return ret;
}
/* If this is an input descriptor, increment that count. */
if (desc.flags & VRING_DESC_F_WRITE) {
*in_num += ret;
if (unlikely(log)) {
log[*log_num].addr = desc.addr;
log[*log_num].len = desc.len;
++*log_num;
}
} else {
/* If it's an output descriptor, they're all supposed
* to come before any input descriptors. */
if (*in_num) {
vq_err(vq, "Indirect descriptor "
"has out after in: idx %d\n", i);
return -EINVAL;
}
*out_num += ret;
}
} while ((i = next_desc(&desc)) != -1);
return 0;
}
/*
* This looks in the virtqueue for the first available buffer, and converts
* it to an iovec for convenient access. Since descriptors consist of some
* number of output then some number of input descriptors, it's actually two
* iovecs, but we pack them into one and note how many of each there were.
*
* This function waits if necessary, and returns the descriptor number found.
*/
static unsigned wait_for_vq_desc(struct virtqueue *vq,
struct iovec iov[],
unsigned int *out_num, unsigned int *in_num)
{
unsigned int i, head, max;
struct vring_desc *desc;
u16 last_avail = lg_last_avail(vq);
/* There's nothing available? */
while (last_avail == vq->vring.avail->idx) {
u64 event;
/*
* Since we're about to sleep, now is a good time to tell the
* Guest about what we've used up to now.
*/
trigger_irq(vq);
/* OK, now we need to know about added descriptors. */
vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
/*
* They could have slipped one in as we were doing that: make
* sure it's written, then check again.
*/
mb();
if (last_avail != vq->vring.avail->idx) {
vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
break;
}
/* Nothing new? Wait for eventfd to tell us they refilled. */
if (read(vq->eventfd, &event, sizeof(event)) != sizeof(event))
errx(1, "Event read failed?");
/* We don't need to be notified again. */
vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
}
/* Check it isn't doing very strange things with descriptor numbers. */
if ((u16)(vq->vring.avail->idx - last_avail) > vq->vring.num)
errx(1, "Guest moved used index from %u to %u",
last_avail, vq->vring.avail->idx);
/*
* Grab the next descriptor number they're advertising, and increment
* the index we've seen.
*/
head = vq->vring.avail->ring[last_avail % vq->vring.num];
lg_last_avail(vq)++;
/* If their number is silly, that's a fatal mistake. */
if (head >= vq->vring.num)
errx(1, "Guest says index %u is available", head);
/* When we start there are none of either input nor output. */
*out_num = *in_num = 0;
max = vq->vring.num;
desc = vq->vring.desc;
i = head;
/*
* If this is an indirect entry, then this buffer contains a descriptor
* table which we handle as if it's any normal descriptor chain.
*/
if (desc[i].flags & VRING_DESC_F_INDIRECT) {
if (desc[i].len % sizeof(struct vring_desc))
errx(1, "Invalid size for indirect buffer table");
max = desc[i].len / sizeof(struct vring_desc);
desc = check_pointer(desc[i].addr, desc[i].len);
i = 0;
}
do {
/* Grab the first descriptor, and check it's OK. */
iov[*out_num + *in_num].iov_len = desc[i].len;
iov[*out_num + *in_num].iov_base
= check_pointer(desc[i].addr, desc[i].len);
/* If this is an input descriptor, increment that count. */
if (desc[i].flags & VRING_DESC_F_WRITE)
(*in_num)++;
else {
/*
* If it's an output descriptor, they're all supposed
* to come before any input descriptors.
*/
if (*in_num)
errx(1, "Descriptor has out after in");
(*out_num)++;
}
/* If we've got too many, that implies a descriptor loop. */
if (*out_num + *in_num > max)
errx(1, "Looped descriptor");
} while ((i = next_desc(desc, i, max)) != max);
return head;
}
