static void
vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
{
struct vring_desc *dp;
struct vq_desc_extra *dxp;
VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
dp = &vq->vq_ring.desc[desc_idx];
dxp = &vq->vq_descx[desc_idx];
if (vq->vq_free_cnt == 0)
VQ_RING_ASSERT_CHAIN_TERM(vq);
vq->vq_free_cnt += dxp->ndescs;
dxp->ndescs--;
if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
while (dp->flags & VRING_DESC_F_NEXT) {
VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
dp = &vq->vq_ring.desc[dp->next];
dxp->ndescs--;
}
}
VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
/*
* We must append the existing free chain, if any, to the end of
* newly freed chain. If the virtqueue was completely used, then
* head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
*/
dp->next = vq->vq_desc_head_idx;
vq->vq_desc_head_idx = desc_idx;
}
/**
* virtqueue_add_buffer() - Enqueues new buffer in vring for consumption
* by other side. Readable buffers are always
* inserted before writable buffers
*
* @param vq - Pointer to VirtIO queue control block.
* @param buffer - Pointer to buffer list
* @param readable - Number of readable buffers
* @param writable - Number of writable buffers
* @param cookie - Pointer to hold call back data
*
* @return - Function status
*/
int virtqueue_add_buffer(struct virtqueue *vq, struct llist *buffer,
int readable, int writable, void *cookie)
{
struct vq_desc_extra *dxp = VQ_NULL;
int status = VQUEUE_SUCCESS;
uint16_t head_idx;
uint16_t idx;
int needed;
needed = readable + writable;
VQ_PARAM_CHK(vq == VQ_NULL, status, ERROR_VQUEUE_INVLD_PARAM);
VQ_PARAM_CHK(needed < 1, status, ERROR_VQUEUE_INVLD_PARAM);
VQ_PARAM_CHK(vq->vq_free_cnt == 0, status, ERROR_VRING_FULL);
//TODO: Add parameters validation for indirect buffer addition
VQUEUE_BUSY(vq);
if (status == VQUEUE_SUCCESS) {
//TODO : Indirect buffer addition support
VQASSERT(vq, cookie != VQ_NULL, "enqueuing with no cookie");
head_idx = vq->vq_desc_head_idx;
VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
dxp = &vq->vq_descx[head_idx];
VQASSERT(vq, (dxp->cookie == VQ_NULL),
"cookie already exists for index");
dxp->cookie = cookie;
dxp->ndescs = needed;
/* Enqueue buffer onto the ring. */
idx = vq_ring_add_buffer(vq, vq->vq_ring.desc, head_idx, buffer,
readable, writable);
vq->vq_desc_head_idx = idx;
vq->vq_free_cnt -= needed;
if (vq->vq_free_cnt == 0) {
VQ_RING_ASSERT_CHAIN_TERM(vq);
} else {
VQ_RING_ASSERT_VALID_IDX(vq, idx);
}
/*
* Update vring_avail control block fields so that other
* side can get buffer using it.
*/
vq_ring_update_avail(vq, head_idx);
}
VQUEUE_IDLE(vq);
return (status);
}
/**
* virtqueue_add_single_buffer - Enqueues single buffer in vring
*
* @param vq - Pointer to VirtIO queue control block
* @param cookie - Pointer to hold call back data
* @param buffer_addr - Address of buffer
* @param len - Length of buffer
* @param writable - If buffer writable
* @param has_next - If buffers for subsequent call are
* to be chained
*
* @return - Function status
*/
int virtqueue_add_single_buffer(struct virtqueue *vq, void *cookie,
void *buffer_addr, uint32_t len, int writable,
boolean has_next)
{
struct vq_desc_extra *dxp;
struct vring_desc *dp;
uint16_t head_idx;
uint16_t idx;
int status = VQUEUE_SUCCESS;
VQ_PARAM_CHK(vq == VQ_NULL, status, ERROR_VQUEUE_INVLD_PARAM);
VQ_PARAM_CHK(vq->vq_free_cnt == 0, status, ERROR_VRING_FULL);
VQUEUE_BUSY(vq);
if (status == VQUEUE_SUCCESS) {
VQASSERT(vq, cookie != VQ_NULL, "enqueuing with no cookie");
head_idx = vq->vq_desc_head_idx;
dxp = &vq->vq_descx[head_idx];
dxp->cookie = cookie;
dxp->ndescs = 1;
idx = head_idx;
dp = &vq->vq_ring.desc[idx];
dp->addr = env_map_vatopa(buffer_addr);
dp->len = len;
dp->flags = 0;
idx = dp->next;
if (has_next)
dp->flags |= VRING_DESC_F_NEXT;
if (writable)
dp->flags |= VRING_DESC_F_WRITE;
vq->vq_desc_head_idx = idx;
vq->vq_free_cnt--;
if (vq->vq_free_cnt == 0) {
VQ_RING_ASSERT_CHAIN_TERM(vq);
} else {
VQ_RING_ASSERT_VALID_IDX(vq, idx);
}
vq_ring_update_avail(vq, head_idx);
}
VQUEUE_IDLE(vq);
return (status);
}
int
virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
int readable, int writable)
{
struct vq_desc_extra *dxp;
int needed;
uint16_t head_idx, idx;
needed = readable + writable;
VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
VQASSERT(vq, needed == sg->sg_nseg,
"segment count mismatch, %d, %d", needed, sg->sg_nseg);
VQASSERT(vq,
needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
"too many segments to enqueue: %d, %d/%d", needed,
vq->vq_nentries, vq->vq_max_indirect_size);
if (needed < 1)
return (EINVAL);
if (vq->vq_free_cnt == 0)
return (ENOSPC);
if (vq_ring_use_indirect(vq, needed)) {
vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
return (0);
} else if (vq->vq_free_cnt < needed)
return (EMSGSIZE);
head_idx = vq->vq_desc_head_idx;
VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
dxp = &vq->vq_descx[head_idx];
VQASSERT(vq, dxp->cookie == NULL,
"cookie already exists for index %d", head_idx);
dxp->cookie = cookie;
dxp->ndescs = needed;
idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
sg, readable, writable);
vq->vq_desc_head_idx = idx;
vq->vq_free_cnt -= needed;
if (vq->vq_free_cnt == 0)
VQ_RING_ASSERT_CHAIN_TERM(vq);
else
VQ_RING_ASSERT_VALID_IDX(vq, idx);
vq_ring_update_avail(vq, head_idx);
return (0);
}
static void
vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
struct sglist *sg, int readable, int writable)
{
struct vring_desc *dp;
struct vq_desc_extra *dxp;
int needed;
uint16_t head_idx;
needed = readable + writable;
VQASSERT(vq, needed <= vq->vq_max_indirect_size,
"enqueuing too many indirect descriptors");
head_idx = vq->vq_desc_head_idx;
VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
dp = &vq->vq_ring.desc[head_idx];
dxp = &vq->vq_descx[head_idx];
VQASSERT(vq, dxp->cookie == NULL,
"cookie already exists for index %d", head_idx);
dxp->cookie = cookie;
dxp->ndescs = 1;
dp->addr = dxp->indirect_paddr;
dp->len = needed * sizeof(struct vring_desc);
dp->flags = VRING_DESC_F_INDIRECT;
vq_ring_enqueue_segments(vq, dxp->indirect, 0,
sg, readable, writable);
vq->vq_desc_head_idx = dp->next;
vq->vq_free_cnt--;
if (vq->vq_free_cnt == 0)
VQ_RING_ASSERT_CHAIN_TERM(vq);
else
VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
vq_ring_update_avail(vq, head_idx);
}
