/**
* Scan paged fragments array for fragments placed in the same page
* with @frag and check if the page has enough room to add @len bytes.
* The fragments are scanned until @refcnt reaches zero. Otherwise,
* the page is in use outside of the SKB, so give up on checking it.
* @return pointer to the last fragment in the page.
*/
static skb_frag_t *
__check_frag_room(struct sk_buff *skb, skb_frag_t *frag, int len)
{
int i, sz, sz2, refcnt;
struct page *pg = skb_frag_page(frag);
skb_frag_t *frag2, *ret = frag;
if ((refcnt = page_count(pg)) == 1)
return frag; /* no other users */
sz = PAGE_SIZE - ss_skb_frag_len(frag);
for (i = skb_shinfo(skb)->nr_frags - 1; i >= 0 ; --i) {
frag2 = &skb_shinfo(skb)->frags[i];
if (frag2 == frag || pg != skb_frag_page(frag2))
continue;
sz2 = PAGE_SIZE - ss_skb_frag_len(frag2);
if (sz2 < len)
return NULL;
if (sz2 < sz) {
sz = sz2;
ret = frag2;
}
/* Return localy referenced pages only. */
if (--refcnt == 1)
return ret;
}
/* The page is used outside of this SKB. */
return NULL;
}
static int
__new_pgfrag(struct sk_buff *skb, struct sk_buff *pskb, int size, int i,
int shift)
{
int off = 0;
struct page *page = NULL;
skb_frag_t *frag;
BUG_ON(i > MAX_SKB_FRAGS);
frag = __lookup_pgfrag_room(skb, size);
if (frag) {
page = skb_frag_page(frag);
off = ss_skb_frag_len(frag);
__skb_frag_ref(frag);
} else {
page = alloc_page(GFP_ATOMIC);
if (!page)
return -ENOMEM;
}
if (__extend_pgfrags(skb, pskb, i, shift)) {
if (!frag)
__free_page(page);
return -ENOMEM;
}
if (i == MAX_SKB_FRAGS) {
/*
* Insert a new paged fragment right after the last one
* in @skb, i.e. as the first fragment of the next skb.
*/
skb = skb_shinfo(pskb ? : skb)->frag_list;
i = 0;
}
static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
struct hv_page_buffer *pb)
{
u32 slots_used = 0;
char *data = skb->data;
int frags = skb_shinfo(skb)->nr_frags;
int i;
/* The packet is laid out thus:
* 1. hdr
* 2. skb linear data
* 3. skb fragment data
*/
if (hdr != NULL)
slots_used += fill_pg_buf(virt_to_page(hdr),
offset_in_page(hdr),
len, &pb[slots_used]);
slots_used += fill_pg_buf(virt_to_page(data),
offset_in_page(data),
skb_headlen(skb), &pb[slots_used]);
for (i = 0; i < frags; i++) {
skb_frag_t *frag = skb_shinfo(skb)->frags + i;
slots_used += fill_pg_buf(skb_frag_page(frag),
frag->page_offset,
skb_frag_size(frag), &pb[slots_used]);
}
return slots_used;
}
/*
* Prepare an SKB to be transmitted to the frontend.
*
* This function is responsible for allocating grant operations, meta
* structures, etc.
*
* It returns the number of meta structures consumed. The number of
* ring slots used is always equal to the number of meta slots used
* plus the number of GSO descriptors used. Currently, we use either
* zero GSO descriptors (for non-GSO packets) or one descriptor (for
* frontend-side LRO).
*/
static int netbk_gop_skb(struct sk_buff *skb,
struct netrx_pending_operations *npo)
{
struct xenvif *vif = netdev_priv(skb->dev);
int nr_frags = skb_shinfo(skb)->nr_frags;
int i;
struct xen_netif_rx_request *req;
struct netbk_rx_meta *meta;
unsigned char *data;
int head = 1;
int old_meta_prod;
old_meta_prod = npo->meta_prod;
/* Set up a GSO prefix descriptor, if necessary */
if (skb_shinfo(skb)->gso_size && vif->gso_prefix) {
req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
meta = npo->meta + npo->meta_prod++;
meta->gso_size = skb_shinfo(skb)->gso_size;
meta->size = 0;
meta->id = req->id;
}
req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
meta = npo->meta + npo->meta_prod++;
if (!vif->gso_prefix)
meta->gso_size = skb_shinfo(skb)->gso_size;
else
meta->gso_size = 0;
meta->size = 0;
meta->id = req->id;
npo->copy_off = 0;
npo->copy_gref = req->gref;
data = skb->data;
while (data < skb_tail_pointer(skb)) {
unsigned int offset = offset_in_page(data);
unsigned int len = PAGE_SIZE - offset;
if (data + len > skb_tail_pointer(skb))
len = skb_tail_pointer(skb) - data;
netbk_gop_frag_copy(vif, skb, npo,
virt_to_page(data), len, offset, &head);
data += len;
}
for (i = 0; i < nr_frags; i++) {
netbk_gop_frag_copy(vif, skb, npo,
skb_frag_page(&skb_shinfo(skb)->frags[i]),
skb_frag_size(&skb_shinfo(skb)->frags[i]),
skb_shinfo(skb)->frags[i].page_offset,
&head);
}
return npo->meta_prod - old_meta_prod;
}
/*
* Make room for @shift fragments starting with slot @i. Then make
* a new fragment in slot @i that can hold @size bytes, and it set up.
*/
static int
__new_pgfrag(struct sk_buff *skb, int size, int i, int shift, TfwStr *it)
{
int off = 0;
struct page *page = NULL;
skb_frag_t *frag;
BUG_ON(i > MAX_SKB_FRAGS);
/*
* Try to find room for @size bytes in SKB fragments.
* If none found, then allocate a new page for the fragment.
*/
frag = __lookup_pgfrag_room(skb, size);
if (frag) {
page = skb_frag_page(frag);
off = ss_skb_frag_len(frag);
__skb_frag_ref(frag); /* get_page(page); */
} else {
page = alloc_page(GFP_ATOMIC);
if (!page)
return -ENOMEM;
}
/* Make room for @shift fragments starting with slot @i. */
if (__extend_pgfrags(skb, i, shift, it)) {
if (frag)
__skb_frag_unref(frag); /* put_page(page); */
else
__free_page(page);
return -ENOMEM;
}
/*
* When the requested slot is right outside the range of the
* array of paged fragments, then the new fragment is put as
* the first fragment of the next SKB.
*/
if (i == MAX_SKB_FRAGS) {
i = 0;
skb = it->skb;
}
/* Set up the new fragment in slot @i to hold @size bytes. */
__skb_fill_page_desc(skb, i, page, off, size);
ss_skb_adjust_data_len(skb, size);
return 0;
}
/**
* dma_skb_copy_datagram_iovec - Copy a datagram to an iovec.
* @skb - buffer to copy
* @offset - offset in the buffer to start copying from
* @iovec - io vector to copy to
* @len - amount of data to copy from buffer to iovec
* @pinned_list - locked iovec buffer data
*
* Note: the iovec is modified during the copy.
*/
int dma_skb_copy_datagram_iovec(struct dma_chan *chan,
struct sk_buff *skb, int offset, struct iovec *to,
size_t len, struct dma_pinned_list *pinned_list)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
dma_cookie_t cookie = 0;
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
cookie = dma_memcpy_to_iovec(chan, to, pinned_list,
skb->data + offset, copy);
if (cookie < 0)
goto fault;
len -= copy;
if (len == 0)
goto end;
offset += copy;
}
/* Copy paged appendix. Hmm... why does this look so complicated? */
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
WARN_ON(start > offset + len);
end = start + skb_shinfo(skb)->frags[i].size;
copy = end - offset;
if (copy > 0) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
struct page *page = skb_frag_page(frag);
if (copy > len)
copy = len;
cookie = dma_memcpy_pg_to_iovec(chan, to, pinned_list, page,
frag->page_offset + offset - start, copy);
if (cookie < 0)
goto fault;
len -= copy;
if (len == 0)
goto end;
offset += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
copy = end - offset;
if (copy > 0) {
if (copy > len)
copy = len;
cookie = dma_skb_copy_datagram_iovec(chan, frag_iter,
offset - start,
to, copy,
pinned_list);
if (cookie < 0)
goto fault;
len -= copy;
if (len == 0)
goto end;
offset += copy;
}
start = end;
}
/**
* Delete @len (the value is positive now) bytes from @frag.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to data after the deleted area in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
struct skb_shared_info *si = skb_shinfo(skb);
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
if (unlikely(off + len > skb_frag_size(frag))) {
SS_WARN("Attempt to delete too much\n");
return -EFAULT;
}
/* Fast path: delete a full fragment. */
if (!off && len == skb_frag_size(frag)) {
ss_skb_adjust_data_len(skb, -len);
__skb_frag_unref(frag);
if (i + 1 < si->nr_frags)
memmove(&si->frags[i], &si->frags[i + 1],
(si->nr_frags - i - 1) * sizeof(skb_frag_t));
--si->nr_frags;
goto lookup_next_ptr;
}
/* Fast path: delete the head part of a fragment. */
if (!off) {
frag->page_offset += len;
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
it->ptr = skb_frag_address(frag);
return 0;
}
/* Fast path: delete the tail part of a fragment. */
if (off + len == skb_frag_size(frag)) {
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
++i;
goto lookup_next_ptr;
}
/*
* Delete data in the middle of a fragment. After the data
* is deleted the fragment will contain only the head part,
* and the tail part is moved to another fragment.
* [frag @i] [frag @i+1 - tail data]
*
* Make room for a fragment right after the @i fragment
* to move the tail part of data there.
*/
if (__extend_pgfrags(skb, i + 1, 1, it))
return -EFAULT;
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off - len;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, len + tail_len);
/* Make the fragment with the tail part. */
i = (i + 1) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off + len, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
ss_skb_adjust_data_len(skb, -len);
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address of data after the deleted area. */
it->flags = (skb != skb_dst);
it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]);
return 0;
lookup_next_ptr:
/* Get the next fragment after the deleted fragment. */
if (i < si->nr_frags)
it->ptr = skb_frag_address(&si->frags[i]);
return 0;
}
/**
* Get room for @len bytes of data starting from offset @off
* in fragment @i.
*
* The room may be found in the preceding fragment if @off is zero.
* Otherwise, a new fragment is allocated and fragments around the
* fragment @i are rearranged so that data is not actually split
* and copied.
*
* Note: @off is always within the borders of fragment @i. It can
* point at the start of a fragment, but it can never point at the
* location right after the end of a fragment. In other words, @off
* can be zero, but it can not be equal to the size of fragment @i.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to the room for new data in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i];
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
/*
* If @off is zero and there's a preceding page fragment,
* then try to append data to that fragment. Go for other
* solutions if there's no room.
*/
if (!off && i) {
frag_dst = __check_frag_room(skb, frag - 1, len);
if (frag_dst) {
/* Coalesce new data with the fragment. */
off = skb_frag_size(frag_dst);
skb_frag_size_add(frag_dst, len);
ss_skb_adjust_data_len(skb, len);
it->ptr = (char *)skb_frag_address(frag_dst) + off;
return 0;
}
}
/*
* Make a fragment that can hold @len bytes. If @off is
* zero, then data is added at the start of fragment @i.
* Make a fragment in slot @i, and the original fragment
* is shifted forward. If @off is not zero, then make
* a fragment in slot @i+1, and make an extra fragment
* in slot @i+2 to hold the tail data.
*/
if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it))
return -EFAULT;
/* If @off is zero, the job is done in __new_pgfrag(). */
if (!off) {
it->ptr = skb_frag_address(frag);
return 0;
}
/*
* If data is added in the middle of a fragment, then split
* the fragment. The head of the fragment stays there, and
* the tail of the fragment is moved to a new fragment.
* The fragment for new data is placed in between.
* [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data]
* If @i is close to MAX_SKB_FRAGS, then new fragments may
* be located in another SKB.
*/
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, tail_len);
/* Make the fragment with the tail part. */
i = (i + 2) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address for new data. */
it->flags = !(i < MAX_SKB_FRAGS - 1);
frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1;
it->ptr = skb_frag_address(frag_dst);
return 0;
}
static int fill_sg_in(struct scatterlist *sg_in,
struct sk_buff *skb,
struct tls_offload_context_tx *ctx,
u64 *rcd_sn,
s32 *sync_size,
int *resync_sgs)
{
int tcp_payload_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
int payload_len = skb->len - tcp_payload_offset;
u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
struct tls_record_info *record;
unsigned long flags;
int remaining;
int i;
spin_lock_irqsave(&ctx->lock, flags);
record = tls_get_record(ctx, tcp_seq, rcd_sn);
if (!record) {
spin_unlock_irqrestore(&ctx->lock, flags);
WARN(1, "Record not found for seq %u\n", tcp_seq);
return -EINVAL;
}
*sync_size = tcp_seq - tls_record_start_seq(record);
if (*sync_size < 0) {
int is_start_marker = tls_record_is_start_marker(record);
spin_unlock_irqrestore(&ctx->lock, flags);
/* This should only occur if the relevant record was
* already acked. In that case it should be ok
* to drop the packet and avoid retransmission.
*
* There is a corner case where the packet contains
* both an acked and a non-acked record.
* We currently don't handle that case and rely
* on TCP to retranmit a packet that doesn't contain
* already acked payload.
*/
if (!is_start_marker)
*sync_size = 0;
return -EINVAL;
}
remaining = *sync_size;
for (i = 0; remaining > 0; i++) {
skb_frag_t *frag = &record->frags[i];
__skb_frag_ref(frag);
sg_set_page(sg_in + i, skb_frag_page(frag),
skb_frag_size(frag), frag->page_offset);
remaining -= skb_frag_size(frag);
if (remaining < 0)
sg_in[i].length += remaining;
}
*resync_sgs = i;
spin_unlock_irqrestore(&ctx->lock, flags);
if (skb_to_sgvec(skb, &sg_in[i], tcp_payload_offset, payload_len) < 0)
return -EINVAL;
return 0;
}
static void xennet_alloc_rx_buffers(struct net_device *dev)
{
unsigned short id;
struct netfront_info *np = netdev_priv(dev);
struct sk_buff *skb;
struct page *page;
int i, batch_target, notify;
RING_IDX req_prod = np->rx.req_prod_pvt;
grant_ref_t ref;
unsigned long pfn;
void *vaddr;
struct xen_netif_rx_request *req;
if (unlikely(!netif_carrier_ok(dev)))
return;
/*
* Allocate skbuffs greedily, even though we batch updates to the
* receive ring. This creates a less bursty demand on the memory
* allocator, so should reduce the chance of failed allocation requests
* both for ourself and for other kernel subsystems.
*/
batch_target = np->rx_target - (req_prod - np->rx.rsp_cons);
for (i = skb_queue_len(&np->rx_batch); i < batch_target; i++) {
skb = __netdev_alloc_skb(dev, RX_COPY_THRESHOLD + NET_IP_ALIGN,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
goto no_skb;
/* Align ip header to a 16 bytes boundary */
skb_reserve(skb, NET_IP_ALIGN);
page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
if (!page) {
kfree_skb(skb);
no_skb:
/* Any skbuffs queued for refill? Force them out. */
if (i != 0)
goto refill;
/* Could not allocate any skbuffs. Try again later. */
mod_timer(&np->rx_refill_timer,
jiffies + (HZ/10));
break;
}
__skb_fill_page_desc(skb, 0, page, 0, 0);
skb_shinfo(skb)->nr_frags = 1;
__skb_queue_tail(&np->rx_batch, skb);
}
/* Is the batch large enough to be worthwhile? */
if (i < (np->rx_target/2)) {
if (req_prod > np->rx.sring->req_prod)
goto push;
return;
}
/* Adjust our fill target if we risked running out of buffers. */
if (((req_prod - np->rx.sring->rsp_prod) < (np->rx_target / 4)) &&
((np->rx_target *= 2) > np->rx_max_target))
np->rx_target = np->rx_max_target;
refill:
for (i = 0; ; i++) {
skb = __skb_dequeue(&np->rx_batch);
if (skb == NULL)
break;
skb->dev = dev;
id = xennet_rxidx(req_prod + i);
BUG_ON(np->rx_skbs[id]);
np->rx_skbs[id] = skb;
ref = gnttab_claim_grant_reference(&np->gref_rx_head);
BUG_ON((signed short)ref < 0);
np->grant_rx_ref[id] = ref;
pfn = page_to_pfn(skb_frag_page(&skb_shinfo(skb)->frags[0]));
vaddr = page_address(skb_frag_page(&skb_shinfo(skb)->frags[0]));
req = RING_GET_REQUEST(&np->rx, req_prod + i);
gnttab_grant_foreign_access_ref(ref,
np->xbdev->otherend_id,
pfn_to_mfn(pfn),
0);
req->id = id;
req->gref = ref;
}
wmb(); /* barrier so backend seens requests */
/* Above is a suitable barrier to ensure backend will see requests. */
np->rx.req_prod_pvt = req_prod + i;
push:
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->rx, notify);
if (notify)
notify_remote_via_irq(np->netdev->irq);
}
/*
* Prepare an SKB to be transmitted to the frontend.
*
* This function is responsible for allocating grant operations, meta
* structures, etc.
*
* It returns the number of meta structures consumed. The number of
* ring slots used is always equal to the number of meta slots used
* plus the number of GSO descriptors used. Currently, we use either
* zero GSO descriptors (for non-GSO packets) or one descriptor (for
* frontend-side LRO).
*/
static int xenvif_gop_skb(struct sk_buff *skb,
struct netrx_pending_operations *npo)
{
struct xenvif *vif = netdev_priv(skb->dev);
int nr_frags = skb_shinfo(skb)->nr_frags;
int i;
struct xen_netif_rx_request *req;
struct xenvif_rx_meta *meta;
unsigned char *data;
int head = 1;
int old_meta_prod;
int gso_type;
struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
grant_ref_t foreign_grefs[MAX_SKB_FRAGS];
struct xenvif *foreign_vif = NULL;
old_meta_prod = npo->meta_prod;
gso_type = XEN_NETIF_GSO_TYPE_NONE;
if (skb_is_gso(skb)) {
if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
}
/* Set up a GSO prefix descriptor, if necessary */
if ((1 << gso_type) & vif->gso_prefix_mask) {
req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
meta = npo->meta + npo->meta_prod++;
meta->gso_type = gso_type;
meta->gso_size = skb_shinfo(skb)->gso_size;
meta->size = 0;
meta->id = req->id;
}
req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
meta = npo->meta + npo->meta_prod++;
if ((1 << gso_type) & vif->gso_mask) {
meta->gso_type = gso_type;
meta->gso_size = skb_shinfo(skb)->gso_size;
} else {
meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
meta->gso_size = 0;
}
meta->size = 0;
meta->id = req->id;
npo->copy_off = 0;
npo->copy_gref = req->gref;
if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
(ubuf->callback == &xenvif_zerocopy_callback)) {
int i = 0;
foreign_vif = ubuf_to_vif(ubuf);
do {
u16 pending_idx = ubuf->desc;
foreign_grefs[i++] =
foreign_vif->pending_tx_info[pending_idx].req.gref;
ubuf = (struct ubuf_info *) ubuf->ctx;
} while (ubuf);
}
data = skb->data;
while (data < skb_tail_pointer(skb)) {
unsigned int offset = offset_in_page(data);
unsigned int len = PAGE_SIZE - offset;
if (data + len > skb_tail_pointer(skb))
len = skb_tail_pointer(skb) - data;
xenvif_gop_frag_copy(vif, skb, npo,
virt_to_page(data), len, offset, &head,
NULL,
0);
data += len;
}
for (i = 0; i < nr_frags; i++) {
xenvif_gop_frag_copy(vif, skb, npo,
skb_frag_page(&skb_shinfo(skb)->frags[i]),
skb_frag_size(&skb_shinfo(skb)->frags[i]),
skb_shinfo(skb)->frags[i].page_offset,
&head,
foreign_vif,
foreign_grefs[i]);
}
return npo->meta_prod - old_meta_prod;
}
static void xennet_make_frags(struct sk_buff *skb, struct net_device *dev,
struct xen_netif_tx_request *tx)
{
struct netfront_info *np = netdev_priv(dev);
char *data = skb->data;
unsigned long mfn;
RING_IDX prod = np->tx.req_prod_pvt;
int frags = skb_shinfo(skb)->nr_frags;
unsigned int offset = offset_in_page(data);
unsigned int len = skb_headlen(skb);
unsigned int id;
grant_ref_t ref;
int i;
/* While the header overlaps a page boundary (including being
larger than a page), split it it into page-sized chunks. */
while (len > PAGE_SIZE - offset) {
tx->size = PAGE_SIZE - offset;
tx->flags |= XEN_NETTXF_more_data;
len -= tx->size;
data += tx->size;
offset = 0;
id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = virt_to_mfn(data);
gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = offset;
tx->size = len;
tx->flags = 0;
}
/* Grant backend access to each skb fragment page. */
for (i = 0; i < frags; i++) {
skb_frag_t *frag = skb_shinfo(skb)->frags + i;
tx->flags |= XEN_NETTXF_more_data;
id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = pfn_to_mfn(page_to_pfn(skb_frag_page(frag)));
gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = frag->page_offset;
tx->size = skb_frag_size(frag);
tx->flags = 0;
}
np->tx.req_prod_pvt = prod;
}
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
int offset, int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6,
struct rt6_info *rt, unsigned int flags, int dontfrag)
{
struct inet_sock *inet = inet_sk(sk);
struct ipv6_pinfo *np = inet6_sk(sk);
struct inet_cork *cork;
struct sk_buff *skb, *skb_prev = NULL;
unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu;
int exthdrlen;
int dst_exthdrlen;
int hh_len;
int copy;
int err;
int offset = 0;
int csummode = CHECKSUM_NONE;
__u8 tx_flags = 0;
if (flags&MSG_PROBE)
return 0;
cork = &inet->cork.base;
if (skb_queue_empty(&sk->sk_write_queue)) {
/*
* setup for corking
*/
if (opt) {
if (WARN_ON(np->cork.opt))
return -EINVAL;
np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation);
if (unlikely(np->cork.opt == NULL))
return -ENOBUFS;
np->cork.opt->tot_len = opt->tot_len;
np->cork.opt->opt_flen = opt->opt_flen;
np->cork.opt->opt_nflen = opt->opt_nflen;
np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt,
sk->sk_allocation);
if (opt->dst0opt && !np->cork.opt->dst0opt)
return -ENOBUFS;
np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt,
sk->sk_allocation);
if (opt->dst1opt && !np->cork.opt->dst1opt)
return -ENOBUFS;
np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt,
sk->sk_allocation);
if (opt->hopopt && !np->cork.opt->hopopt)
return -ENOBUFS;
np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt,
sk->sk_allocation);
if (opt->srcrt && !np->cork.opt->srcrt)
return -ENOBUFS;
/* need source address above miyazawa*/
}
dst_hold(&rt->dst);
cork->dst = &rt->dst;
inet->cork.fl.u.ip6 = *fl6;
np->cork.hop_limit = hlimit;
np->cork.tclass = tclass;
if (rt->dst.flags & DST_XFRM_TUNNEL)
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(&rt->dst);
else
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(rt->dst.path);
if (np->frag_size < mtu) {
if (np->frag_size)
mtu = np->frag_size;
}
cork->fragsize = mtu;
if (dst_allfrag(rt->dst.path))
cork->flags |= IPCORK_ALLFRAG;
cork->length = 0;
sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
exthdrlen = (opt ? opt->opt_flen : 0);
length += exthdrlen;
transhdrlen += exthdrlen;
dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
} else {
rt = (struct rt6_info *)cork->dst;
fl6 = &inet->cork.fl.u.ip6;
opt = np->cork.opt;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
mtu = cork->fragsize;
}
orig_mtu = mtu;
hh_len = LL_RESERVED_SPACE(rt->dst.dev);
fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
(opt ? opt->opt_nflen : 0);
maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
}
/* For UDP, check if TX timestamp is enabled */
if (sk->sk_type == SOCK_DGRAM) {
err = sock_tx_timestamp(sk, &tx_flags);
if (err)
goto error;
}
/*
* Let's try using as much space as possible.
* Use MTU if total length of the message fits into the MTU.
* Otherwise, we need to reserve fragment header and
* fragment alignment (= 8-15 octects, in total).
*
* Note that we may need to "move" the data from the tail of
* of the buffer to the new fragment when we split
* the message.
*
* FIXME: It may be fragmented into multiple chunks
* at once if non-fragmentable extension headers
* are too large.
* --yoshfuji
*/
if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP ||
sk->sk_protocol == IPPROTO_RAW)) {
ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
skb = skb_peek_tail(&sk->sk_write_queue);
cork->length += length;
if (((length > mtu) ||
(skb && skb_has_frags(skb))) &&
(sk->sk_protocol == IPPROTO_UDP) &&
(rt->dst.dev->features & NETIF_F_UFO) &&
(sk->sk_type == SOCK_DGRAM)) {
err = ip6_ufo_append_data(sk, getfrag, from, length,
hh_len, fragheaderlen,
transhdrlen, mtu, flags, rt);
if (err)
goto error;
return 0;
}
if (!skb)
goto alloc_new_skb;
while (length > 0) {
/* Check if the remaining data fits into current packet. */
copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
if (copy < length)
copy = maxfraglen - skb->len;
if (copy <= 0) {
char *data;
unsigned int datalen;
unsigned int fraglen;
unsigned int fraggap;
unsigned int alloclen;
alloc_new_skb:
/* There's no room in the current skb */
if (skb)
fraggap = skb->len - maxfraglen;
else
fraggap = 0;
/* update mtu and maxfraglen if necessary */
if (skb == NULL || skb_prev == NULL)
ip6_append_data_mtu(&mtu, &maxfraglen,
fragheaderlen, skb, rt,
orig_mtu);
skb_prev = skb;
/*
* If remaining data exceeds the mtu,
* we know we need more fragment(s).
*/
datalen = length + fraggap;
if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
if ((flags & MSG_MORE) &&
!(rt->dst.dev->features&NETIF_F_SG))
alloclen = mtu;
else
alloclen = datalen + fragheaderlen;
alloclen += dst_exthdrlen;
if (datalen != length + fraggap) {
/*
* this is not the last fragment, the trailer
* space is regarded as data space.
*/
datalen += rt->dst.trailer_len;
}
alloclen += rt->dst.trailer_len;
fraglen = datalen + fragheaderlen;
/*
* We just reserve space for fragment header.
* Note: this may be overallocation if the message
* (without MSG_MORE) fits into the MTU.
*/
alloclen += sizeof(struct frag_hdr);
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len,
(flags & MSG_DONTWAIT), &err);
} else {
skb = NULL;
if (atomic_read(&sk->sk_wmem_alloc) <=
2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len, 1,
sk->sk_allocation);
if (unlikely(skb == NULL))
err = -ENOBUFS;
else {
/* Only the initial fragment
* is time stamped.
*/
tx_flags = 0;
}
}
if (skb == NULL)
goto error;
/*
* Fill in the control structures
*/
skb->ip_summed = csummode;
skb->csum = 0;
/* reserve for fragmentation and ipsec header */
skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
dst_exthdrlen);
if (sk->sk_type == SOCK_DGRAM)
skb_shinfo(skb)->tx_flags = tx_flags;
/*
* Find where to start putting bytes
*/
data = skb_put(skb, fraglen);
skb_set_network_header(skb, exthdrlen);
data += fragheaderlen;
skb->transport_header = (skb->network_header +
fragheaderlen);
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
data + transhdrlen, fraggap, 0);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
pskb_trim_unique(skb_prev, maxfraglen);
}
copy = datalen - transhdrlen - fraggap;
if (copy < 0) {
err = -EINVAL;
kfree_skb(skb);
goto error;
} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
err = -EFAULT;
kfree_skb(skb);
goto error;
}
offset += copy;
length -= datalen - fraggap;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
csummode = CHECKSUM_NONE;
/*
* Put the packet on the pending queue
*/
__skb_queue_tail(&sk->sk_write_queue, skb);
continue;
}
if (copy > length)
copy = length;
if (!(rt->dst.dev->features&NETIF_F_SG)) {
unsigned int off;
off = skb->len;
if (getfrag(from, skb_put(skb, copy),
offset, copy, off, skb) < 0) {
__skb_trim(skb, off);
err = -EFAULT;
goto error;
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = sk->sk_sndmsg_page;
int off = sk->sk_sndmsg_off;
unsigned int left;
if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != skb_frag_page(frag)) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
skb_frag_ref(skb, i);
frag = &skb_shinfo(skb)->frags[i];
}
} else if(i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->sk_allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
sk->sk_sndmsg_page = page;
sk->sk_sndmsg_off = 0;
skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from,
skb_frag_address(frag) + skb_frag_size(frag),
offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
goto error;
}
sk->sk_sndmsg_off += copy;
skb_frag_size_add(frag, copy);
skb->len += copy;
skb->data_len += copy;
skb->truesize += copy;
atomic_add(copy, &sk->sk_wmem_alloc);
}
offset += copy;
length -= copy;
}
return 0;
error:
cork->length -= length;
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
return err;
}
/**
* skb_copy_datagram_iter - Copy a datagram to an iovec iterator.
* @skb: buffer to copy
* @offset: offset in the buffer to start copying from
* @to: iovec iterator to copy to
* @len: amount of data to copy from buffer to iovec
*/
int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len)
{
int start = skb_headlen(skb);
int i, copy = start - offset, start_off = offset, n;
struct sk_buff *frag_iter;
trace_skb_copy_datagram_iovec(skb, len);
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
n = copy_to_iter(skb->data + offset, copy, to);
offset += n;
if (n != copy)
goto short_copy;
if ((len -= copy) == 0)
return 0;
}
/* Copy paged appendix. Hmm... why does this look so complicated? */
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
WARN_ON(start > offset + len);
end = start + skb_frag_size(frag);
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
n = copy_page_to_iter(skb_frag_page(frag),
frag->page_offset + offset -
start, copy, to);
offset += n;
if (n != copy)
goto short_copy;
if (!(len -= copy))
return 0;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
if (skb_copy_datagram_iter(frag_iter, offset - start,
to, copy))
goto fault;
if ((len -= copy) == 0)
return 0;
offset += copy;
}
start = end;
}
static inline int
roq_eth_hw_tx(struct sk_buff *skb, struct net_device *ndev, int num_slots)
{
struct ib_send_wr sq_wr, *bad_wr;
struct ib_sge send_sgl[MAX_TX_SEGS], *sge = send_sgl;
struct ib_qp *qp;
struct roq_eth_priv *priv = netdev_priv(ndev);
struct ethhdr *eth = (struct ethhdr *)skb->data;
unsigned int len = skb_headlen(skb),
mapped_slots = 0;
tx_slot_desc_t slot_desc;
dma_addr_t bus_addr;
int rank, rv = -EINVAL;
if (likely(len)) {
bus_addr = dma_map_single(&ndev->dev, skb->data, len,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(&ndev->dev, bus_addr))) {
pr_warn("DMA map of TX Data Buffer addr "
"0x%016lx len %d failed\n",
(unsigned long)skb->data, len);
rv = -ENOMEM;
goto out;
}
mapped_slots = 1;
} else {
/*
* Shall we handle that case ?
* For now we just disable fragmented TX
*/
ndev->features &= ~NETIF_F_SG;
pr_info("Cannot handle skb w/o headlen, SG TX disabled\n");
rv = -ENOBUFS;
goto out;
}
/* Get free slots */
roq_get_tx_slots(priv, &slot_desc, num_slots);
priv->tx_buf.post[slot_desc.slot[0]].skb = skb;
priv->tx_buf.post[slot_desc.slot[0]].addr = bus_addr;
priv->tx_buf.post[slot_desc.slot[0]].len = len;
sge->addr = bus_addr;
sge->length = len;
sge->lkey = 0;
sq_wr.next = NULL;
sq_wr.wr_id = slot_desc.id;
sq_wr.sg_list = sge;
sq_wr.num_sge = num_slots;
sq_wr.opcode = IB_WR_SEND;
sq_wr.send_flags = 0;
if (--num_slots > 0) {
struct skb_shared_info *sk_info = skb_shinfo(skb);
struct skb_frag_struct *frag = sk_info->frags;
int i = 1;
do {
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0)
bus_addr = dma_map_page(&ndev->dev, frag->page,
frag->page_offset, frag->size,
DMA_TO_DEVICE);
#else
bus_addr = dma_map_page(&ndev->dev, skb_frag_page(frag),
frag->page_offset, frag->size,
DMA_TO_DEVICE);
#endif
if (unlikely(dma_mapping_error(&ndev->dev, bus_addr))) {
rv = -ENOBUFS;
goto out_unmap;
}
priv->tx_buf.post[slot_desc.slot[i]].addr = bus_addr;
priv->tx_buf.post[slot_desc.slot[i]].len = frag->size;
sge++;
sge->addr = bus_addr;
sge->length = frag->size;
sge->lkey = 0;
mapped_slots++;
frag++;
i++;
} while (--num_slots > 0);
}
if (mapped_slots > 1) {
int i;
for (i = 0; i < mapped_slots; i++)
pr_info("sge %d:: addr 0x%llx, len %u\n",
i, send_sgl[i].addr, send_sgl[i].length);
ndev->features &= ~NETIF_F_SG;
}
if ((priv->is_ionode && eth->h_dest[3] & 0x80)
|| (!priv->is_ionode && eth->h_dest[3] & 0x40)) {
rank = roq_tcoords_to_rank(priv->netdesc_rem,
eth->h_dest[3] & 0xf,
eth->h_dest[4] >> 4,
eth->h_dest[4] & 0xf,
eth->h_dest[5] >> 4,
eth->h_dest[5] & 0xf);
if (rank < 0)
goto out_unmap;
qp = priv->qps_rem[rank];
} else {
static void xennet_make_frags(struct sk_buff *skb, struct net_device *dev,
struct xen_netif_tx_request *tx)
{
struct netfront_info *np = netdev_priv(dev);
char *data = skb->data;
unsigned long mfn;
RING_IDX prod = np->tx.req_prod_pvt;
int frags = skb_shinfo(skb)->nr_frags;
unsigned int offset = offset_in_page(data);
unsigned int len = skb_headlen(skb);
unsigned int id;
grant_ref_t ref;
int i;
/* While the header overlaps a page boundary (including being
larger than a page), split it it into page-sized chunks. */
while (len > PAGE_SIZE - offset) {
tx->size = PAGE_SIZE - offset;
tx->flags |= XEN_NETTXF_more_data;
len -= tx->size;
data += tx->size;
offset = 0;
id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = virt_to_mfn(data);
gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = offset;
tx->size = len;
tx->flags = 0;
}
/* Grant backend access to each skb fragment page. */
for (i = 0; i < frags; i++) {
skb_frag_t *frag = skb_shinfo(skb)->frags + i;
struct page *page = skb_frag_page(frag);
len = skb_frag_size(frag);
offset = frag->page_offset;
/* Data must not cross a page boundary. */
BUG_ON(len + offset > PAGE_SIZE<<compound_order(page));
/* Skip unused frames from start of page */
page += offset >> PAGE_SHIFT;
offset &= ~PAGE_MASK;
while (len > 0) {
unsigned long bytes;
BUG_ON(offset >= PAGE_SIZE);
bytes = PAGE_SIZE - offset;
if (bytes > len)
bytes = len;
tx->flags |= XEN_NETTXF_more_data;
id = get_id_from_freelist(&np->tx_skb_freelist,
np->tx_skbs);
np->tx_skbs[id].skb = skb_get(skb);
tx = RING_GET_REQUEST(&np->tx, prod++);
tx->id = id;
ref = gnttab_claim_grant_reference(&np->gref_tx_head);
BUG_ON((signed short)ref < 0);
mfn = pfn_to_mfn(page_to_pfn(page));
gnttab_grant_foreign_access_ref(ref,
np->xbdev->otherend_id,
mfn, GNTMAP_readonly);
tx->gref = np->grant_tx_ref[id] = ref;
tx->offset = offset;
tx->size = bytes;
tx->flags = 0;
offset += bytes;
len -= bytes;
/* Next frame */
if (offset == PAGE_SIZE && len) {
BUG_ON(!PageCompound(page));
page++;
offset = 0;
}
}
}
np->tx.req_prod_pvt = prod;
}
/**
* skb_copy_datagram_iovec - Copy a datagram to an iovec.
* @skb: buffer to copy
* @offset: offset in the buffer to start copying from
* @to: io vector to copy to
* @len: amount of data to copy from buffer to iovec
*
* Note: the iovec is modified during the copy.
*/
int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset,
struct iovec *to, int len)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
trace_skb_copy_datagram_iovec(skb, len);
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
if (memcpy_toiovec(to, skb->data + offset, copy))
goto fault;
if ((len -= copy) == 0)
return 0;
offset += copy;
}
/* Copy paged appendix. Hmm... why does this look so complicated? */
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
WARN_ON(start > offset + len);
end = start + skb_frag_size(frag);
if ((copy = end - offset) > 0) {
int err;
u8 *vaddr;
struct page *page = skb_frag_page(frag);
if (copy > len)
copy = len;
vaddr = kmap(page);
err = memcpy_toiovec(to, vaddr + frag->page_offset +
offset - start, copy);
kunmap(page);
if (err)
goto fault;
if (!(len -= copy))
return 0;
offset += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
if (skb_copy_datagram_iovec(frag_iter,
offset - start,
to, copy))
goto fault;
if ((len -= copy) == 0)
return 0;
offset += copy;
}
start = end;
}
static void xennet_alloc_rx_buffers(struct net_device *dev)
{
unsigned short id;
struct netfront_info *np = netdev_priv(dev);
struct sk_buff *skb;
struct page *page;
int i, batch_target, notify;
RING_IDX req_prod = np->rx.req_prod_pvt;
grant_ref_t ref;
unsigned long pfn;
void *vaddr;
struct xen_netif_rx_request *req;
if (unlikely(!netif_carrier_ok(dev)))
return;
batch_target = np->rx_target - (req_prod - np->rx.rsp_cons);
for (i = skb_queue_len(&np->rx_batch); i < batch_target; i++) {
skb = __netdev_alloc_skb(dev, RX_COPY_THRESHOLD + NET_IP_ALIGN,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
goto no_skb;
skb_reserve(skb, NET_IP_ALIGN);
page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
if (!page) {
kfree_skb(skb);
no_skb:
if (i != 0)
goto refill;
mod_timer(&np->rx_refill_timer,
jiffies + (HZ/10));
break;
}
__skb_fill_page_desc(skb, 0, page, 0, 0);
skb_shinfo(skb)->nr_frags = 1;
__skb_queue_tail(&np->rx_batch, skb);
}
if (i < (np->rx_target/2)) {
if (req_prod > np->rx.sring->req_prod)
goto push;
return;
}
if (((req_prod - np->rx.sring->rsp_prod) < (np->rx_target / 4)) &&
((np->rx_target *= 2) > np->rx_max_target))
np->rx_target = np->rx_max_target;
refill:
for (i = 0; ; i++) {
skb = __skb_dequeue(&np->rx_batch);
if (skb == NULL)
break;
skb->dev = dev;
id = xennet_rxidx(req_prod + i);
BUG_ON(np->rx_skbs[id]);
np->rx_skbs[id] = skb;
ref = gnttab_claim_grant_reference(&np->gref_rx_head);
BUG_ON((signed short)ref < 0);
np->grant_rx_ref[id] = ref;
pfn = page_to_pfn(skb_frag_page(&skb_shinfo(skb)->frags[0]));
vaddr = page_address(skb_frag_page(&skb_shinfo(skb)->frags[0]));
req = RING_GET_REQUEST(&np->rx, req_prod + i);
gnttab_grant_foreign_access_ref(ref,
np->xbdev->otherend_id,
pfn_to_mfn(pfn),
0);
req->id = id;
req->gref = ref;
}
wmb();
np->rx.req_prod_pvt = req_prod + i;
push:
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->rx, notify);
if (notify)
notify_remote_via_irq(np->netdev->irq);
}
