int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
{
unsigned int i, head, max;
target_phys_addr_t desc_pa = vq->vring.desc;
target_phys_addr_t len;
if (!virtqueue_num_heads(vq, vq->last_avail_idx))
return 0;
/* When we start there are none of either input nor output. */
elem->out_num = elem->in_num = 0;
max = vq->vring.num;
i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
fprintf(stderr, "Invalid size for indirect buffer table\n");
exit(1);
}
/* loop over the indirect descriptor table */
max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
desc_pa = vring_desc_addr(desc_pa, i);
i = 0;
}
do {
struct iovec *sg;
int is_write = 0;
if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
elem->in_addr[elem->in_num] = vring_desc_addr(desc_pa, i);
sg = &elem->in_sg[elem->in_num++];
is_write = 1;
} else
sg = &elem->out_sg[elem->out_num++];
/* Grab the first descriptor, and check it's OK. */
sg->iov_len = vring_desc_len(desc_pa, i);
len = sg->iov_len;
sg->iov_base = cpu_physical_memory_map(vring_desc_addr(desc_pa, i),
&len, is_write);
if (sg->iov_base == NULL || len != sg->iov_len) {
fprintf(stderr, "virtio: trying to map MMIO memory\n");
exit(1);
}
/* If we've got too many, that implies a descriptor loop. */
if ((elem->in_num + elem->out_num) > max) {
fprintf(stderr, "Looped descriptor");
exit(1);
}
} while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
elem->index = head;
vq->inuse++;
return elem->in_num + elem->out_num;
}
void
vu_queue_get_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int *in_bytes,
unsigned int *out_bytes,
unsigned max_in_bytes, unsigned max_out_bytes)
{
unsigned int idx;
unsigned int total_bufs, in_total, out_total;
int rc;
idx = vq->last_avail_idx;
total_bufs = in_total = out_total = 0;
if (unlikely(dev->broken) ||
unlikely(!vq->vring.avail)) {
goto done;
}
while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
unsigned int max, desc_len, num_bufs, indirect = 0;
uint64_t desc_addr, read_len;
struct vring_desc *desc;
struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
unsigned int i;
max = vq->vring.num;
num_bufs = total_bufs;
if (!virtqueue_get_head(dev, vq, idx++, &i)) {
goto err;
}
desc = vq->vring.desc;
if (desc[i].flags & VRING_DESC_F_INDIRECT) {
if (desc[i].len % sizeof(struct vring_desc)) {
vu_panic(dev, "Invalid size for indirect buffer table");
goto err;
}
/* If we've got too many, that implies a descriptor loop. */
if (num_bufs >= max) {
vu_panic(dev, "Looped descriptor");
goto err;
}
/* loop over the indirect descriptor table */
indirect = 1;
desc_addr = desc[i].addr;
desc_len = desc[i].len;
max = desc_len / sizeof(struct vring_desc);
read_len = desc_len;
desc = vu_gpa_to_va(dev, &read_len, desc_addr);
if (unlikely(desc && read_len != desc_len)) {
/* Failed to use zero copy */
desc = NULL;
if (!virtqueue_read_indirect_desc(dev, desc_buf,
desc_addr,
desc_len)) {
desc = desc_buf;
}
}
if (!desc) {
vu_panic(dev, "Invalid indirect buffer table");
goto err;
}
num_bufs = i = 0;
}
do {
/* If we've got too many, that implies a descriptor loop. */
if (++num_bufs > max) {
vu_panic(dev, "Looped descriptor");
goto err;
}
if (desc[i].flags & VRING_DESC_F_WRITE) {
in_total += desc[i].len;
} else {
out_total += desc[i].len;
}
if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
goto done;
}
rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
} while (rc == VIRTQUEUE_READ_DESC_MORE);
if (rc == VIRTQUEUE_READ_DESC_ERROR) {
goto err;
}
if (!indirect) {
total_bufs = num_bufs;
} else {
total_bufs++;
}
}
if (rc < 0) {
goto err;
}
done:
if (in_bytes) {
*in_bytes = in_total;
}
if (out_bytes) {
*out_bytes = out_total;
}
return;
err:
in_total = out_total = 0;
goto done;
}
int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
{
unsigned int idx;
int total_bufs, in_total, out_total;
idx = vq->last_avail_idx;
total_bufs = in_total = out_total = 0;
while (virtqueue_num_heads(vq, idx)) {
unsigned int max, num_bufs, indirect = 0;
target_phys_addr_t desc_pa;
int i;
max = vq->vring.num;
num_bufs = total_bufs;
i = virtqueue_get_head(vq, idx++);
desc_pa = vq->vring.desc;
if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
fprintf(stderr, "Invalid size for indirect buffer table\n");
exit(1);
}
/* If we've got too many, that implies a descriptor loop. */
if (num_bufs >= max) {
fprintf(stderr, "Looped descriptor");
exit(1);
}
/* loop over the indirect descriptor table */
indirect = 1;
max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
num_bufs = i = 0;
desc_pa = vring_desc_addr(desc_pa, i);
}
do {
/* If we've got too many, that implies a descriptor loop. */
if (++num_bufs > max) {
fprintf(stderr, "Looped descriptor");
exit(1);
}
if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
if (in_bytes > 0 &&
(in_total += vring_desc_len(desc_pa, i)) >= in_bytes)
return 1;
} else {
if (out_bytes > 0 &&
(out_total += vring_desc_len(desc_pa, i)) >= out_bytes)
return 1;
}
} while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
if (!indirect)
total_bufs = num_bufs;
else
total_bufs++;
}
return 0;
}