static void net_tx_response(struct XenNetDev *netdev, netif_tx_request_t *txp, int8_t st) { RING_IDX i = netdev->tx_ring.rsp_prod_pvt; netif_tx_response_t *resp; int notify; resp = RING_GET_RESPONSE(&netdev->tx_ring, i); resp->id = txp->id; resp->status = st; #if 0 if (txp->flags & NETTXF_extra_info) { RING_GET_RESPONSE(&netdev->tx_ring, ++i)->status = NETIF_RSP_NULL; } #endif netdev->tx_ring.rsp_prod_pvt = ++i; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netdev->tx_ring, notify); if (notify) { xen_be_send_notify(&netdev->xendev); } if (i == netdev->tx_ring.req_cons) { int more_to_do; RING_FINAL_CHECK_FOR_REQUESTS(&netdev->tx_ring, more_to_do); if (more_to_do) { netdev->tx_work++; } } }
static int xen_block_send_response(XenBlockRequest *request) { XenBlockDataPlane *dataplane = request->dataplane; int send_notify = 0; int have_requests = 0; blkif_response_t *resp; /* Place on the response ring for the relevant domain. */ switch (dataplane->protocol) { case BLKIF_PROTOCOL_NATIVE: resp = (blkif_response_t *)RING_GET_RESPONSE( &dataplane->rings.native, dataplane->rings.native.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_32: resp = (blkif_response_t *)RING_GET_RESPONSE( &dataplane->rings.x86_32_part, dataplane->rings.x86_32_part.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_64: resp = (blkif_response_t *)RING_GET_RESPONSE( &dataplane->rings.x86_64_part, dataplane->rings.x86_64_part.rsp_prod_pvt); break; default: return 0; } resp->id = request->req.id; resp->operation = request->req.operation; resp->status = request->status; dataplane->rings.common.rsp_prod_pvt++; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane->rings.common, send_notify); if (dataplane->rings.common.rsp_prod_pvt == dataplane->rings.common.req_cons) { /* * Tail check for pending requests. Allows frontend to avoid * notifications if requests are already in flight (lower * overheads and promotes batching). */ RING_FINAL_CHECK_FOR_REQUESTS(&dataplane->rings.common, have_requests); } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane->rings.common)) { have_requests = 1; } if (have_requests) { dataplane->more_work++; } return send_notify; }
static int blk_send_response_one(struct ioreq *ioreq) { struct XenBlkDev *blkdev = ioreq->blkdev; int send_notify = 0; int have_requests = 0; blkif_response_t resp; void *dst; resp.id = ioreq->req.id; resp.operation = ioreq->req.operation; resp.status = ioreq->status; /* Place on the response ring for the relevant domain. */ switch (blkdev->protocol) { case BLKIF_PROTOCOL_NATIVE: dst = RING_GET_RESPONSE(&blkdev->rings.native, blkdev->rings.native.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_32: dst = RING_GET_RESPONSE(&blkdev->rings.x86_32_part, blkdev->rings.x86_32_part.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_64: dst = RING_GET_RESPONSE(&blkdev->rings.x86_64_part, blkdev->rings.x86_64_part.rsp_prod_pvt); break; default: dst = NULL; } memcpy(dst, &resp, sizeof(resp)); blkdev->rings.common.rsp_prod_pvt++; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blkdev->rings.common, send_notify); if (blkdev->rings.common.rsp_prod_pvt == blkdev->rings.common.req_cons) { /* * Tail check for pending requests. Allows frontend to avoid * notifications if requests are already in flight (lower * overheads and promotes batching). */ RING_FINAL_CHECK_FOR_REQUESTS(&blkdev->rings.common, have_requests); } else if (RING_HAS_UNCONSUMED_REQUESTS(&blkdev->rings.common)) { have_requests = 1; } if (have_requests) { blkdev->more_work++; } return send_notify; }
static int xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp, uint_t *uid) { blkif_front_ring_t *uring; blkif_response_t *target; uring = &state->bt_user_ring.ur_ring; if (!RING_HAS_UNCONSUMED_RESPONSES(uring)) { return (B_FALSE); } target = NULL; target = RING_GET_RESPONSE(uring, uring->rsp_cons); if (target == NULL) { return (B_FALSE); } /* copy out the user app response */ bcopy(target, resp, sizeof (*resp)); uring->rsp_cons++; /* restore the quests id from the original request */ *uid = (uint_t)resp->id; resp->id = state->bt_map.um_outstanding_reqs[*uid].id; return (B_TRUE); }
// What if someone calls waiton the same desc several times? int waiton_syscall(syscall_desc_t* desc) { int retval = 0; if (desc == NULL || desc->channel == NULL){ errno = EFAIL; return -1; } // Make sure we were given a desc with a non-NULL frontring. This could // happen if someone forgot to check the error code on the paired syscall. syscall_front_ring_t *fr = &desc->channel->sysfr; if (!fr){ errno = EFAIL; return -1; } printf("waiting %d\n", vcore_id()); syscall_rsp_t* rsp = RING_GET_RESPONSE(fr, desc->idx); // ignoring the ring push response from the kernel side now while (atomic_read(&rsp->sc->flags) != SC_DONE) cpu_relax(); // memcpy(rsp, rsp_inring, sizeof(*rsp)); // run a cleanup function for this desc, if available if (rsp->cleanup) rsp->cleanup(rsp->data); if (RSP_ERRNO(rsp)){ errno = RSP_ERRNO(rsp); retval = -1; } else retval = RSP_RESULT(rsp); atomic_inc((atomic_t*) &(fr->rsp_cons)); return retval; }
void netfront_rx(struct netfront_dev *dev) { RING_IDX rp, cons; struct netif_rx_response *rsp = &(dev->rsp); int more, flags; local_irq_save(flags); netfront_tx_buf_gc(dev); local_irq_restore(flags); #ifdef CONFIG_NETMAP if (dev->netmap) { netmap_netfront_rx(dev); return; } #endif moretodo: rp = dev->rx.sring->rsp_prod; rmb(); /* Ensure we see queued responses up to 'rp'. */ cons = dev->rx.rsp_cons; while (cons != rp) { NETIF_MEMCPY(rsp, RING_GET_RESPONSE(&dev->rx, cons), sizeof(*rsp)); netfront_get_responses(dev, cons); cons = dev->rx.rsp_cons; } dev->rx.rsp_cons = cons; RING_FINAL_CHECK_FOR_RESPONSES(&dev->rx, more); if(more) goto moretodo; netfront_fillup_rx_buffers(dev); }
static void net_rx_response(struct XenNetDev *netdev, netif_rx_request_t *req, int8_t st, uint16_t offset, uint16_t size, uint16_t flags) { RING_IDX i = netdev->rx_ring.rsp_prod_pvt; netif_rx_response_t *resp; int notify; resp = RING_GET_RESPONSE(&netdev->rx_ring, i); resp->offset = offset; resp->flags = flags; resp->id = req->id; resp->status = (int16_t)size; if (st < 0) { resp->status = (int16_t)st; } xen_be_printf(&netdev->xendev, 3, "rx response: idx %d, status %d, flags 0x%x\n", i, resp->status, resp->flags); netdev->rx_ring.rsp_prod_pvt = ++i; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netdev->rx_ring, notify); if (notify) { xen_be_send_notify(&netdev->xendev); } }
static void blktap_read_ring(struct blktap *tap) { struct blktap_ring *ring = &tap->ring; struct blktap_ring_response rsp; RING_IDX rc, rp; down_read(¤t->mm->mmap_sem); if (!ring->vma) { up_read(¤t->mm->mmap_sem); return; } /* for each outstanding message on the ring */ rp = ring->ring.sring->rsp_prod; rmb(); for (rc = ring->ring.rsp_cons; rc != rp; rc++) { memcpy(&rsp, RING_GET_RESPONSE(&ring->ring, rc), sizeof(rsp)); blktap_ring_read_response(tap, &rsp); } ring->ring.rsp_cons = rc; up_read(¤t->mm->mmap_sem); }
/** * Poll for completed packets * * @v netdev Network device */ static void netfront_poll_tx ( struct net_device *netdev ) { struct netfront_nic *netfront = netdev->priv; struct xen_device *xendev = netfront->xendev; struct netif_tx_response *response; struct io_buffer *iobuf; unsigned int status; int rc; /* Consume any unconsumed responses */ while ( RING_HAS_UNCONSUMED_RESPONSES ( &netfront->tx_fring ) ) { /* Get next response */ response = RING_GET_RESPONSE ( &netfront->tx_fring, netfront->tx_fring.rsp_cons++ ); /* Retrieve from descriptor ring */ iobuf = netfront_pull ( netfront, &netfront->tx, response->id ); status = response->status; if ( status == NETIF_RSP_OKAY ) { DBGC2 ( netfront, "NETFRONT %s TX id %d complete\n", xendev->key, response->id ); netdev_tx_complete ( netdev, iobuf ); } else { rc = -EIO_NETIF_RSP ( status ); DBGC2 ( netfront, "NETFRONT %s TX id %d error %d: %s\n", xendev->key, response->id, status, strerror ( rc ) ); netdev_tx_complete_err ( netdev, iobuf, rc ); } } }
int mem_event_get_response(struct domain *d, struct mem_event_domain *med, mem_event_response_t *rsp) { mem_event_front_ring_t *front_ring; RING_IDX rsp_cons; mem_event_ring_lock(med); front_ring = &med->front_ring; rsp_cons = front_ring->rsp_cons; if ( !RING_HAS_UNCONSUMED_RESPONSES(front_ring) ) { mem_event_ring_unlock(med); return 0; } /* Copy response */ memcpy(rsp, RING_GET_RESPONSE(front_ring, rsp_cons), sizeof(*rsp)); rsp_cons++; /* Update ring */ front_ring->rsp_cons = rsp_cons; front_ring->sring->rsp_event = rsp_cons + 1; /* Kick any waiters -- since we've just consumed an event, * there may be additional space available in the ring. */ mem_event_wake(d, med); mem_event_ring_unlock(med); return 1; }
static irqreturn_t as_int (int irq, void *dev_id) { struct as_response *ring_resp; RING_IDX i, rp; printk("\nxen:DomU: as_int called"); again: rp = info.ring.sring->rsp_prod; printk(KERN_DEBUG "\nxen:DomU:ring pointers %d to %d", info.ring.rsp_cons, rp); for (i = info.ring.rsp_cons; i != rp; i++) { unsigned long id; ring_resp = RING_GET_RESPONSE(&(info.ring), i); printk(KERN_DEBUG "\nxen:domU: Recvd in IDX-%d, with id=%d, op=%d, st=%d", i, ring_resp->id, ring_resp->operation, ring_resp->status); id = ring_resp->id; switch(ring_resp->operation) { case 0: printk(KERN_DEBUG "\nxen:DomU: operation: 0"); break; default: break; } } info.ring.rsp_cons = i; if (i != info.ring.req_prod_pvt) { int more_to_do; RING_FINAL_CHECK_FOR_RESPONSES(&info.ring, more_to_do); if (more_to_do) goto again; } else info.ring.sring->rsp_event = i + 1; return IRQ_HANDLED; }
static void netfront_tx_buf_gc(struct netfront_dev *dev) { RING_IDX cons, prod; unsigned short id; #ifndef CONFIG_NETFRONT_PERSISTENT_GRANTS struct net_txbuffer *buf; #endif do { prod = dev->tx.sring->rsp_prod; rmb(); /* Ensure we see responses up to 'rp'. */ for (cons = dev->tx.rsp_cons; cons != prod; cons++) { struct netif_tx_response *txrsp; txrsp = RING_GET_RESPONSE(&dev->tx, cons); if (txrsp->status == NETIF_RSP_NULL) continue; if (txrsp->status == NETIF_RSP_DROPPED) printk("netif drop for tx\n"); if (txrsp->status == NETIF_RSP_ERROR) printk("netif error for tx\n"); id = txrsp->id; BUG_ON(id >= NET_TX_RING_SIZE); #ifndef CONFIG_NETFRONT_PERSISTENT_GRANTS buf = &dev->tx_buffers[id]; gnttab_end_access(buf->gref); buf->gref = GRANT_INVALID_REF; #ifdef HAVE_LWIP if (buf->pbuf) { pbuf_free(buf->pbuf); buf->pbuf = NULL; } #endif /* HAVE_LWIP */ #endif /* CONFIG_NETFRONT_PERSISTENT_GRANTS */ add_id_to_freelist(id, dev->tx_freelist); up(&dev->tx_sem); } dev->tx.rsp_cons = prod; /* * Set a new event, then check for race with update of tx_cons. * Note that it is essential to schedule a callback, no matter * how few tx_buffers are pending. Even if there is space in the * transmit ring, higher layers may be blocked because too much * data is outstanding: in such cases notification from Xen is * likely to be the only kick that we'll get. */ dev->tx.sring->rsp_event = prod + ((dev->tx.sring->req_prod - prod) >> 1) + 1; mb(); } while ((cons == prod) && (prod != dev->tx.sring->rsp_prod)); }
static inline void write_rsp_to_ring(struct td_state *s, blkif_response_t *rsp) { tapdev_info_t *info = s->ring_info; blkif_response_t *rsp_d; rsp_d = RING_GET_RESPONSE(&info->fe_ring, info->fe_ring.rsp_prod_pvt); memcpy(rsp_d, rsp, sizeof(blkif_response_t)); info->fe_ring.rsp_prod_pvt++; }
static void xennet_tx_buf_gc(struct net_device *dev) { RING_IDX cons, prod; unsigned short id; struct netfront_info *np = netdev_priv(dev); struct sk_buff *skb; BUG_ON(!netif_carrier_ok(dev)); do { prod = np->tx.sring->rsp_prod; rmb(); /* Ensure we see responses up to 'rp'. */ for (cons = np->tx.rsp_cons; cons != prod; cons++) { struct xen_netif_tx_response *txrsp; txrsp = RING_GET_RESPONSE(&np->tx, cons); if (txrsp->status == NETIF_RSP_NULL) continue; id = txrsp->id; skb = np->tx_skbs[id].skb; if (unlikely(gnttab_query_foreign_access( np->grant_tx_ref[id]) != 0)) { printk(KERN_ALERT "xennet_tx_buf_gc: warning " "-- grant still in use by backend " "domain.\n"); BUG(); } gnttab_end_foreign_access_ref( np->grant_tx_ref[id], GNTMAP_readonly); gnttab_release_grant_reference( &np->gref_tx_head, np->grant_tx_ref[id]); np->grant_tx_ref[id] = GRANT_INVALID_REF; add_id_to_freelist(&np->tx_skb_freelist, np->tx_skbs, id); dev_kfree_skb_irq(skb); } np->tx.rsp_cons = prod; /* * Set a new event, then check for race with update of tx_cons. * Note that it is essential to schedule a callback, no matter * how few buffers are pending. Even if there is space in the * transmit ring, higher layers may be blocked because too much * data is outstanding: in such cases notification from Xen is * likely to be the only kick that we'll get. */ np->tx.sring->rsp_event = prod + ((np->tx.sring->req_prod - prod) >> 1) + 1; mb(); /* update shared area */ } while ((cons == prod) && (prod != np->tx.sring->rsp_prod)); xennet_maybe_wake_tx(dev); }
static irqreturn_t ixp_interrupt(int irq, void *dev_id) { struct ixp_response *bret; RING_IDX i, rp; struct ixpfront_info *info = (struct ixpfront_info *)dev_id; int error; if (unlikely(info->connected != IXP_STATE_CONNECTED)) { return IRQ_HANDLED; } again: rp = info->ring.sring->rsp_prod; rmb(); /* Ensure we see queued responses up to 'rp'. */ for (i = info->ring.rsp_cons; i != rp; i++) { unsigned long id; bret = RING_GET_RESPONSE(&info->ring, i); id = bret->id; ixp_install_response(info, bret); ixp_completion(&info->shadow[id]); add_id_to_freelist(info, id); error = (bret->status == IXPIF_RSP_OKAY) ? 0 : -EIO; switch (bret->operation) { case IXP_OP_3DES_ENCRYPT: if (unlikely(bret->status != IXPIF_RSP_OKAY)) dev_dbg(&info->xbdev->dev, "Bad return from blkdev data " "request: %x\n", bret->status); break; default: BUG(); } } info->ring.rsp_cons = i; if (i != info->ring.req_prod_pvt) { int more_to_do; RING_FINAL_CHECK_FOR_RESPONSES(&info->ring, more_to_do); if (more_to_do) goto again; } else info->ring.sring->rsp_event = i + 1; return IRQ_HANDLED; }
static int write_block(FILE *p, blkif_sector_t sector, size_t amt) { static uint64_t next_reqid = 1; blkif_response_t *rsp; blkif_request_t *req; int notify, work_to_do; uint64_t reqid; RING_IDX i; /* wait until we can write something */ while(RING_FULL(&p->ring)) runtime_block(1); /* write out the request */ i = p->ring.req_prod_pvt++; req = RING_GET_REQUEST(&p->ring, i); memset(req, 0, sizeof(blkif_request_t)); req->operation = BLKIF_OP_WRITE; req->nr_segments = 1; req->handle = p->disk_handle; req->id = reqid = next_reqid++; req->sector_number = sector; req->seg[0].gref = p->block_grant; req->seg[0].first_sect = 0; req->seg[0].last_sect = (amt - 1) / 512; wmb(); RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&p->ring, notify); if(notify) channel_send(p->chan); /* wait for it to be satisfied */ do { while(!RING_HAS_UNCONSUMED_RESPONSES(&p->ring)) runtime_block(1); i = p->ring.rsp_cons++; rsp = RING_GET_RESPONSE(&p->ring, i); } while(rsp->id != reqid); /* was it successful? */ if(rsp->status != BLKIF_RSP_OKAY) { printf("PROFILING: Block write failed!\n"); return 0; } /* we do writes one at a time, synchronously, so work_to_do should always be false */ RING_FINAL_CHECK_FOR_RESPONSES(&p->ring, work_to_do); assert(!work_to_do); return 1; }
void network_tx_buf_gc(struct netfront_dev *dev) { RING_IDX cons, prod; unsigned short id; do { prod = dev->tx.sring->rsp_prod; rmb(); /* Ensure we see responses up to 'rp'. */ //printk("cons = %ld, prod = %ld\n", dev->tx.rsp_cons, prod); //farewellkou for (cons = dev->tx.rsp_cons; cons != prod; cons++) { struct netif_tx_response *txrsp; struct net_buffer *buf; txrsp = RING_GET_RESPONSE(&dev->tx, cons); if (txrsp->status == NETIF_RSP_NULL){ continue; } if (txrsp->status == NETIF_RSP_ERROR){ printk("packet error\n"); } id = txrsp->id; BUG_ON(id >= NET_TX_RING_SIZE); buf = &dev->tx_buffers[id]; gnttab_end_access(buf->gref); buf->gref=GRANT_INVALID_REF; add_id_to_freelist(id,dev->tx_freelist); up(&dev->tx_sem); } dev->tx.rsp_cons = prod; /* * Set a new event, then check for race with update of tx_cons. * Note that it is essential to schedule a callback, no matter * how few tx_buffers are pending. Even if there is space in the * transmit ring, higher layers may be blocked because too much * data is outstanding: in such cases notification from Xen is * likely to be the only kick that we'll get. */ dev->tx.sring->rsp_event = prod + ((dev->tx.sring->req_prod - prod) >> 1) + 1; mb(); } while ((cons == prod) && (prod != dev->tx.sring->rsp_prod)); }
static void xennet_tx_buf_gc(struct net_device *dev) { RING_IDX cons, prod; unsigned short id; struct netfront_info *np = netdev_priv(dev); struct sk_buff *skb; BUG_ON(!netif_carrier_ok(dev)); do { prod = np->tx.sring->rsp_prod; rmb(); for (cons = np->tx.rsp_cons; cons != prod; cons++) { struct xen_netif_tx_response *txrsp; txrsp = RING_GET_RESPONSE(&np->tx, cons); if (txrsp->status == NETIF_RSP_NULL) continue; id = txrsp->id; skb = np->tx_skbs[id].skb; if (unlikely(gnttab_query_foreign_access( np->grant_tx_ref[id]) != 0)) { printk(KERN_ALERT "xennet_tx_buf_gc: warning " "-- grant still in use by backend " "domain.\n"); BUG(); } gnttab_end_foreign_access_ref( np->grant_tx_ref[id], GNTMAP_readonly); gnttab_release_grant_reference( &np->gref_tx_head, np->grant_tx_ref[id]); np->grant_tx_ref[id] = GRANT_INVALID_REF; add_id_to_freelist(&np->tx_skb_freelist, np->tx_skbs, id); dev_kfree_skb_irq(skb); } np->tx.rsp_cons = prod; np->tx.sring->rsp_event = prod + ((np->tx.sring->req_prod - prod) >> 1) + 1; mb(); } while ((cons == prod) && (prod != np->tx.sring->rsp_prod)); xennet_maybe_wake_tx(dev); }
/* * Computes the size of the pbuf to allocate based * on how many slots the (possible GSO) frame requires. */ static int netfront_get_size(struct netfront_dev *dev, RING_IDX ri) { struct netif_rx_response *rx; int32_t len = 0; int slots = 1; do { rx = RING_GET_RESPONSE(&dev->rx, ++ri); dprintk("rx: scan: slot %d len %d (more %s)\n", slots, rx->status, (rx->flags & NETRXF_more_data ? "true": "false")); len += rx->status; slots++; } while (rx->flags & NETRXF_more_data); return len; }
static int writelog_dequeue_responses(struct writelog* wl) { RING_IDX rstart, rend; log_response_t rsp; rstart = wl->fring.rsp_cons; rend = wl->sring->rsp_prod; BDPRINTF("ring kicked (start = %u, end = %u)", rstart, rend); while (rstart != rend) { memcpy(&rsp, RING_GET_RESPONSE(&wl->fring, rstart), sizeof(rsp)); BDPRINTF("ctl: read response %"PRIu64":%u", rsp.sector, rsp.count); wl->fring.rsp_cons = ++rstart; wl->inflight--; } return 0; }
static irqreturn_t as_int(int irq, void *dev_id) { RING_IDX rc, rp; as_request_t req; as_response_t resp; int more_to_do, notify; printk(KERN_DEBUG "\nxen:Dom0: as_int called with dev_id=%x info=%x", (unsigned int)dev_id, (unsigned int) &info); rc = info.ring.req_cons; rp = info.ring.sring->req_prod; printk(KERN_DEBUG " rc = %d rp = %d", rc, rp); while (rc != rp) { if (RING_REQUEST_CONS_OVERFLOW(&info.ring, rc)) break; memcpy(&req, RING_GET_REQUEST(&info.ring, rc), sizeof(req)); resp.id = req.id; resp.operation = req.operation; resp.status = req.status + 1; printk(KERN_DEBUG "\nxen:Dom0:Recvd at IDX-%d: id = %d, op=%d, status=%d", rc, req.id, req.operation, req.status); info.ring.req_cons = ++rc; barrier(); switch(req.operation) { case 0: printk(KERN_DEBUG "\nxen:dom0:req.operation = 0"); break; default: printk(KERN_DEBUG "\nxen:dom0:req.operation = %d", req.operation); break; } memcpy(RING_GET_RESPONSE(&info.ring, info.ring.rsp_prod_pvt), &resp, sizeof(resp)); info.ring.rsp_prod_pvt++; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&info.ring, notify); if (info.ring.rsp_prod_pvt == info.ring.req_cons) { RING_FINAL_CHECK_FOR_REQUESTS(&info.ring, more_to_do); } else if (RING_HAS_UNCONSUMED_REQUESTS(&info.ring)) { more_to_do = 1; } if (notify) { printk(KERN_DEBUG "\nxen:dom0:send notify to domu"); notify_remote_via_irq(info.irq); } } return IRQ_HANDLED; }
LOCAL void mcd_event_get_response(struct domain *d, mcd_event_response_t *rsp) { mcd_event_front_ring_t *front_ring; RING_IDX rsp_cons; mcd_event_ring_lock(d); front_ring = &d->mcd_event.front_ring; rsp_cons = front_ring->rsp_cons; /* Copy response */ memcpy(rsp, RING_GET_RESPONSE(front_ring, rsp_cons), sizeof(*rsp)); rsp_cons++; /* Update ring */ front_ring->rsp_cons = rsp_cons; front_ring->sring->rsp_event = rsp_cons + 1; mcd_event_ring_unlock(d); }
static FORCEINLINE VOID FrontendCompleteResponses( IN PXENVBD_FRONTEND Frontend ) { for (;;) { ULONG rsp_prod; ULONG rsp_cons; rsp_prod = Frontend->SharedRing->rsp_prod; rsp_cons = Frontend->FrontRing.rsp_cons; __MemoryBarrier(); while (rsp_cons != rsp_prod) { blkif_response_t* Response; PXENVBD_REQUEST Request; SHORT Status; Response = RING_GET_RESPONSE(&Frontend->FrontRing, rsp_cons); Status = Response->status; Request = (PXENVBD_REQUEST)(ULONG_PTR)(Response->id); ++rsp_cons; if (Request) { PdoCompleteSubmittedRequest(Frontend->Pdo, Request, Status); } // zero request slot now its read RtlZeroMemory(Response, sizeof(blkif_response_t)); } Frontend->FrontRing.rsp_cons = rsp_cons; __MemoryBarrier(); if (!__FinalCheckForResponses(Frontend)) break; } }
static int scsifront_ring_drain(struct vscsifrnt_info *info) { struct vscsiif_response *ring_rsp; RING_IDX i, rp; int more_to_do = 0; rp = info->ring.sring->rsp_prod; rmb(); /* ordering required respective to dom0 */ for (i = info->ring.rsp_cons; i != rp; i++) { ring_rsp = RING_GET_RESPONSE(&info->ring, i); scsifront_do_response(info, ring_rsp); } info->ring.rsp_cons = i; if (i != info->ring.req_prod_pvt) RING_FINAL_CHECK_FOR_RESPONSES(&info->ring, more_to_do); else info->ring.sring->rsp_event = i + 1; return more_to_do; }
/* * Reads extra slots to check for a GSO packet */ static int netfront_get_extras(struct netfront_dev *dev, struct netif_extra_info *extras, RING_IDX ri) { struct netif_extra_info *extra; RING_IDX cons = dev->rx.rsp_cons; int err = 0; #ifndef CONFIG_NETFRONT_PERSISTENT_GRANTS struct net_rxbuffer *buf; #endif do { extra = (struct netif_extra_info *) RING_GET_RESPONSE(&dev->rx, ++cons); if (unlikely(!extra->type || extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) { printk("Invalid extra type: %d\n", extra->type); err = -EINVAL; } else { dprintk("rx: scan: extra %u %s\n", extra->type, (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE ? "(more true)": "")); NETIF_MEMCPY(&extras[extra->type - 1], extra, sizeof(*extra)); } #ifndef CONFIG_NETFRONT_PERSISTENT_GRANTS buf = dev->rx_buffers[netfront_rxidx(cons)]; gnttab_end_access(buf->gref); buf->gref = GRANT_INVALID_REF; dev->rx_buffers[netfront_rxidx(cons)] = NULL; netfront_release_rxbuffer(buf, dev); #endif } while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE); dev->rx.rsp_cons = cons; return err; }
/** * Poll for received packets * * @v netdev Network device */ static void netfront_poll_rx ( struct net_device *netdev ) { struct netfront_nic *netfront = netdev->priv; struct xen_device *xendev = netfront->xendev; struct netif_rx_response *response; struct io_buffer *iobuf; int status; size_t len; int rc; /* Consume any unconsumed responses */ while ( RING_HAS_UNCONSUMED_RESPONSES ( &netfront->rx_fring ) ) { /* Get next response */ response = RING_GET_RESPONSE ( &netfront->rx_fring, netfront->rx_fring.rsp_cons++ ); /* Retrieve from descriptor ring */ iobuf = netfront_pull ( netfront, &netfront->rx, response->id ); status = response->status; if ( status >= 0 ) { len = status; iob_reserve ( iobuf, response->offset ); iob_put ( iobuf, len ); DBGC2 ( netfront, "NETFRONT %s RX id %d complete " "%#08lx+%zx\n", xendev->key, response->id, virt_to_phys ( iobuf->data ), len ); netdev_rx ( netdev, iobuf ); } else { rc = -EIO_NETIF_RSP ( status ); DBGC2 ( netfront, "NETFRONT %s RX id %d error %d: %s\n", xendev->key, response->id, status, strerror ( rc ) ); netdev_rx_err ( netdev, iobuf, rc ); } } }
static void netfe_tx_buf_gc(netfe_t *fe) { RING_IDX prod, cons; do { prod = fe->tx_ring.sring->rsp_prod; rmb(); // dark for (cons = fe->tx_ring.rsp_cons; cons != prod; cons++) { netif_tx_response_t *rsp = RING_GET_RESPONSE(&fe->tx_ring, cons); fe->free_tx_bufs[rsp->id] = fe->free_tx_head; fe->free_tx_head = rsp->id; } fe->tx_ring.rsp_cons = prod; // mindlessly copied from netfront.c fe->tx_ring.sring->rsp_event = prod + ((fe->tx_ring.sring->req_prod - prod) >> 1) +1; mb(); } while ((cons == prod) && (prod != fe->tx_ring.sring->rsp_prod)); }
void network_rx(struct netfront_dev *dev) { RING_IDX rp,cons,req_prod; struct netif_rx_response *rx; int nr_consumed, some, more, i, notify; moretodo: rp = dev->rx.sring->rsp_prod; rmb(); /* Ensure we see queued responses up to 'rp'. */ cons = dev->rx.rsp_cons; nr_consumed = 0; some = 0; while ((cons != rp) && !some) { struct net_buffer* buf; unsigned char* page; int id; rx = RING_GET_RESPONSE(&dev->rx, cons); if (rx->flags & NETRXF_extra_info) { printk("+++++++++++++++++++++ we have extras!\n"); continue; } if (rx->status == NETIF_RSP_NULL) continue; id = rx->id; BUG_ON(id >= NET_TX_RING_SIZE); buf = &dev->rx_buffers[id]; page = (unsigned char*)buf->page; gnttab_end_access(buf->gref); if(rx->status>0) { #ifdef HAVE_LIBC if (dev->netif_rx == NETIF_SELECT_RX) { int len = rx->status; ASSERT(current == main_thread); if (len > dev->len) len = dev->len; memcpy(dev->data, page+rx->offset, len); dev->rlen = len; some = 1; } else #endif dev->netif_rx(page+rx->offset,rx->status); } nr_consumed++; ++cons; } dev->rx.rsp_cons=cons; RING_FINAL_CHECK_FOR_RESPONSES(&dev->rx,more); if(more && !some) goto moretodo; req_prod = dev->rx.req_prod_pvt; for(i=0; i<nr_consumed; i++) { int id = xennet_rxidx(req_prod + i); netif_rx_request_t *req = RING_GET_REQUEST(&dev->rx, req_prod + i); struct net_buffer* buf = &dev->rx_buffers[id]; void* page = buf->page; /* We are sure to have free gnttab entries since they got released above */ buf->gref = req->gref = gnttab_grant_access(dev->dom,virt_to_mfn(page),0); req->id = id; } wmb(); dev->rx.req_prod_pvt = req_prod + i; RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&dev->rx, notify); if (notify) notify_remote_via_evtchn(dev->evtchn); }
static irqreturn_t chrif_int(int irq, void *dev_id) { int err; RING_IDX rc, rp; int more_to_do, notify; chrif_request_t req; chrif_response_t resp; printk(KERN_INFO "\n------------------------------start response-------------------------------------"); printk(KERN_DEBUG "\nxen: Dom0: chrif_int called with dev_id=%x info=%x", (unsigned int)dev_id, (unsigned int) &info); rc = info.ring.req_cons; rp = info.ring.sring->req_prod; printk(KERN_DEBUG "\nxen: Dom0: rc = %d rp = %d", rc, rp); while (rc != rp) { if (RING_REQUEST_CONS_OVERFLOW(&info.ring, rc)) break; memcpy(&req, RING_GET_REQUEST(&info.ring, rc), sizeof(req)); resp.id = req.id; resp.operation = req.operation; resp.status = req.status + 1; printk(KERN_DEBUG "\nxen: Dom0: Recvd at IDX-%d: id = %d, op=%d, status=%d", rc, req.id, req.operation, req.status); info.ring.req_cons = ++rc; barrier(); printk(KERN_DEBUG "\nxen: Dom0: operation: %s", op_name(resp.operation)); switch(resp.operation) { case CHRIF_OP_OPEN: info.chrif_filp = filp_open(DEVICE_PATH, O_RDWR, 0); printk(KERN_DEBUG "\nxen: dom0: response open"); break; case CHRIF_OP_READ: { resp.rdwr.len = req.rdwr.len; //struct pdma_info pdma_info; //memset(op_page->addr, 0, resp.rdwr.len); old_fs = get_fs(); set_fs(get_ds()); //get read size of block //err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_INFO, (unsigned long)&pdma_info); //read data from device to page //err =info.chrif_filp->f_op->read(info.chrif_filp, op_page->addr, resp.rdwr.len, &info.chrif_filp->f_pos); set_fs(old_fs); if(err < 0) printk(KERN_DEBUG "\nxen: Dom0: read %u bytes error", resp.rdwr.len); printk(KERN_DEBUG "\nxen: dom0: response read"); break; } case CHRIF_OP_WRITE: { int i = 0, count, ret; struct vm_struct *op_page; struct gnttab_map_grant_ref op_page_ops; struct gnttab_unmap_grant_ref op_page_unmap_ops; resp.rdwr.len = req.rdwr.len; count = resp.rdwr.len/4096; printk(KERN_DEBUG "\nxen: Dom0: write %u bytes %d times", resp.rdwr.len, count); block_buf = (char *)kmalloc(resp.rdwr.len, GFP_KERNEL); memset(block_buf, 0, resp.rdwr.len); while(i < count) { resp.op_gref[i] = req.op_gref[i]; printk(KERN_DEBUG "\nxen: dom0: req.op_gref[0]: %d", resp.op_gref[i]); op_page = alloc_vm_area(PAGE_SIZE, NULL); if(op_page == 0) { free_vm_area(op_page); printk("\nxen: dom0: could not allocate shared_page"); return -EFAULT; } /*gnttab_set_map_op(&op_page_ops, (unsigned long)op_page->addr, GNTMAP_host_map, resp.op_gref[i], info.remoteDomain); op_page_unmap_ops.host_addr = (unsigned long)(op_page->addr); unmap_ops.handle = op_page_ops.handle; if(HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &op_page_ops, 1)){ printk(KERN_DEBUG "\nxen: dom0: HYPERVISOR map grant ref failed"); return -EFAULT; } if (op_page_ops.status) { printk(KERN_DEBUG "\nxen: dom0: HYPERVISOR map grant ref failed status = %d", op_page_ops.status); return -EFAULT; } printk(KERN_DEBUG "\nxen: dom0: map shared page success, shared_page=%x, handle = %x, status = %x", (unsigned int)op_page->addr, op_page_ops.handle, op_page_ops.status); memcpy(block_buf+i*4096, op_page->addr, 4096); ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &op_page_unmap_ops, 1); if (ret == 0) { printk(KERN_DEBUG "\nxen: dom0: dom0_exit: unmapped shared frame"); } else { printk(KERN_DEBUG "\nxen: dom0: dom0_exit: unmapped shared frame failed"); } free_vm_area(op_page);*/ i++; } /* old_fs = get_fs(); set_fs(get_ds()); //write data from page to device //err = info.chrif_filp->f_op->write(info.chrif_filp, block_buf, resp.rdwr.len, &info.chrif_filp->f_pos); set_fs(old_fs); if(err < 0) printk(KERN_DEBUG "\nxen: Dom0: write %u bytes error", resp.rdwr.len); */ //kfree(block_buf); printk(KERN_DEBUG "\nxen: dom0: response write"); break; } case CHRIF_OP_IOCTL: { resp.ioc_parm.cmd = req.ioc_parm.cmd; switch(resp.ioc_parm.cmd) { case PDMA_IOC_START_DMA: { old_fs = get_fs(); set_fs(get_ds()); err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_START_DMA, NULL); set_fs(old_fs); if(err) { printk(KERN_DEBUG "\nxen: Dom0: start-dma ioctl failed"); resp.status = 0; } else printk(KERN_DEBUG "\nxen: Dom0: start-dma ioctl success"); //err = call_ioctl(info.chrif_filp, PDMA_IOC_START_DMA, NULL); break; } case PDMA_IOC_STOP_DMA: { //err = call_ioctl(info.chrif_filp, PDMA_IOC_STOP_DMA, NULL); old_fs = get_fs(); set_fs(get_ds()); err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_STOP_DMA, NULL); set_fs(old_fs); if(err) { printk(KERN_DEBUG "\nxen: Dom0: stop-dma ioctl failed"); resp.status = 0; } else printk(KERN_DEBUG "\nxen: Dom0: stop-dma ioctl success"); break; } case PDMA_IOC_INFO: { struct pdma_info pdma_info; //err = call_ioctl(info.chrif_filp, PDMA_IOC_INFO, (unsigned long)&pdma_info); old_fs = get_fs(); set_fs(get_ds()); err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_INFO, (unsigned long)&pdma_info); set_fs(old_fs); if(err) { printk(KERN_DEBUG "\nxen: Dom0: info ioctl failed"); resp.status = 0; } else printk(KERN_DEBUG "\nxen: Dom0: info ioctl success"); resp.ioc_parm.info = pdma_info; break; } case PDMA_IOC_STAT: { struct pdma_stat pdma_stat; //err = call_ioctl(info.chrif_filp, PDMA_IOC_STAT, (unsigned long)&pdma_stat); old_fs = get_fs(); set_fs(get_ds()); err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_STAT, (unsigned long)&pdma_stat); set_fs(old_fs); if(err) { printk(KERN_DEBUG "\nxen: Dom0: stat ioctl failed"); resp.status = 0; } else printk(KERN_DEBUG "\nxen: Dom0: stat ioctl success"); resp.ioc_parm.stat = pdma_stat; break; } case PDMA_IOC_RW_REG: { struct pdma_rw_reg ctrl = req.ioc_parm.ctrl; //err = call_ioctl(info.chrif_filp, PDMA_IOC_RW_REG, (unsigned long)&ctrl); old_fs = get_fs(); set_fs(get_ds()); err = info.chrif_filp->f_op->unlocked_ioctl(info.chrif_filp, PDMA_IOC_RW_REG, (unsigned long)&ctrl); set_fs(old_fs); if(err) { printk(KERN_DEBUG "\nxen: Dom0: rw-reg ioctl failed"); resp.status = 0; } else printk(KERN_DEBUG "\nxen: Dom0: rw-reg ioctl success"); resp.ioc_parm.ctrl = ctrl; break; } default: printk(KERN_DEBUG "\nxen: Dom0: unknow the operation"); break; } printk(KERN_INFO "\nxen: Dom0: response ioctl"); break; } case CHRIF_OP_CLOSE: filp_close(info.chrif_filp, NULL); printk(KERN_INFO "\nxen: Dom0: response close"); break; default: printk(KERN_DEBUG "\nxen: Dom0: unknow the operation"); break; } memcpy(RING_GET_RESPONSE(&info.ring, info.ring.rsp_prod_pvt), &resp, sizeof(resp)); info.ring.rsp_prod_pvt++; //put response and check whether or not notify domU RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&info.ring, notify); if (info.ring.rsp_prod_pvt == info.ring.req_cons) { RING_FINAL_CHECK_FOR_REQUESTS(&info.ring, more_to_do); } else if (RING_HAS_UNCONSUMED_REQUESTS(&info.ring)) { more_to_do = 1; } if (notify) { printk(KERN_DEBUG "\nxen:dom0:send notify to domu"); notify_remote_via_irq(info.irq); } } return IRQ_HANDLED; }
static void xenvif_rx_action(struct xenvif *vif) { s8 status; u16 flags; struct xen_netif_rx_response *resp; struct sk_buff_head rxq; struct sk_buff *skb; LIST_HEAD(notify); int ret; unsigned long offset; bool need_to_notify = false; struct netrx_pending_operations npo = { .copy = vif->grant_copy_op, .meta = vif->meta, }; skb_queue_head_init(&rxq); while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) { RING_IDX max_slots_needed; RING_IDX old_req_cons; RING_IDX ring_slots_used; int i; /* We need a cheap worse case estimate for the number of * slots we'll use. */ max_slots_needed = DIV_ROUND_UP(offset_in_page(skb->data) + skb_headlen(skb), PAGE_SIZE); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { unsigned int size; unsigned int offset; size = skb_frag_size(&skb_shinfo(skb)->frags[i]); offset = skb_shinfo(skb)->frags[i].page_offset; /* For a worse-case estimate we need to factor in * the fragment page offset as this will affect the * number of times xenvif_gop_frag_copy() will * call start_new_rx_buffer(). */ max_slots_needed += DIV_ROUND_UP(offset + size, PAGE_SIZE); } /* To avoid the estimate becoming too pessimal for some * frontends that limit posted rx requests, cap the estimate * at MAX_SKB_FRAGS. */ if (max_slots_needed > MAX_SKB_FRAGS) max_slots_needed = MAX_SKB_FRAGS; /* We may need one more slot for GSO metadata */ if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 || skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)) max_slots_needed++; /* If the skb may not fit then bail out now */ if (!xenvif_rx_ring_slots_available(vif, max_slots_needed)) { skb_queue_head(&vif->rx_queue, skb); need_to_notify = true; vif->rx_last_skb_slots = max_slots_needed; break; } else vif->rx_last_skb_slots = 0; old_req_cons = vif->rx.req_cons; XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo); ring_slots_used = vif->rx.req_cons - old_req_cons; BUG_ON(ring_slots_used > max_slots_needed); __skb_queue_tail(&rxq, skb); } BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta)); if (!npo.copy_prod) goto done; BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS); gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod); while ((skb = __skb_dequeue(&rxq)) != NULL) { if ((1 << vif->meta[npo.meta_cons].gso_type) & vif->gso_prefix_mask) { resp = RING_GET_RESPONSE(&vif->rx, vif->rx.rsp_prod_pvt++); resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data; resp->offset = vif->meta[npo.meta_cons].gso_size; resp->id = vif->meta[npo.meta_cons].id; resp->status = XENVIF_RX_CB(skb)->meta_slots_used; npo.meta_cons++; XENVIF_RX_CB(skb)->meta_slots_used--; } vif->dev->stats.tx_bytes += skb->len; vif->dev->stats.tx_packets++; status = xenvif_check_gop(vif, XENVIF_RX_CB(skb)->meta_slots_used, &npo); if (XENVIF_RX_CB(skb)->meta_slots_used == 1) flags = 0; else flags = XEN_NETRXF_more_data; if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */ flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated; else if (skb->ip_summed == CHECKSUM_UNNECESSARY) /* remote but checksummed. */ flags |= XEN_NETRXF_data_validated; offset = 0; resp = make_rx_response(vif, vif->meta[npo.meta_cons].id, status, offset, vif->meta[npo.meta_cons].size, flags); if ((1 << vif->meta[npo.meta_cons].gso_type) & vif->gso_mask) { struct xen_netif_extra_info *gso = (struct xen_netif_extra_info *) RING_GET_RESPONSE(&vif->rx, vif->rx.rsp_prod_pvt++); resp->flags |= XEN_NETRXF_extra_info; gso->u.gso.type = vif->meta[npo.meta_cons].gso_type; gso->u.gso.size = vif->meta[npo.meta_cons].gso_size; gso->u.gso.pad = 0; gso->u.gso.features = 0; gso->type = XEN_NETIF_EXTRA_TYPE_GSO; gso->flags = 0; } xenvif_add_frag_responses(vif, status, vif->meta + npo.meta_cons + 1, XENVIF_RX_CB(skb)->meta_slots_used); RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret); need_to_notify |= !!ret; npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used; dev_kfree_skb(skb); } done: if (need_to_notify) notify_remote_via_irq(vif->rx_irq); } void xenvif_check_rx_xenvif(struct xenvif *vif) { int more_to_do; RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do); if (more_to_do) napi_schedule(&vif->napi); }