static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, u8 __user *to, int len, __wsum *csump) { int start = skb_headlen(skb); int pos = 0; int i, copy = start - offset; /* Copy header. */ if (copy > 0) { int err = 0; if (copy > len) copy = len; *csump = csum_and_copy_to_user(skb->data + offset, to, copy, *csump, &err); if (err) goto fault; if ((len -= copy) == 0) return 0; offset += copy; to += copy; pos = copy; } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; WARN_ON(start > offset + len); end = start + skb_shinfo(skb)->frags[i].size; if ((copy = end - offset) > 0) { __wsum csum2; int err = 0; u8 *vaddr; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct page *page = frag->page; if (copy > len) copy = len; vaddr = kmap(page); csum2 = csum_and_copy_to_user(vaddr + frag->page_offset + offset - start, to, copy, 0, &err); kunmap(page); if (err) goto fault; *csump = csum_block_add(*csump, csum2, pos); if (!(len -= copy)) return 0; offset += copy; to += copy; pos += copy; } start = end; } if (skb_shinfo(skb)->frag_list) { struct sk_buff *list = skb_shinfo(skb)->frag_list; for (; list; list=list->next) { int end; WARN_ON(start > offset + len); end = start + list->len; if ((copy = end - offset) > 0) { __wsum csum2 = 0; if (copy > len) copy = len; if (skb_copy_and_csum_datagram(list, offset - start, to, copy, &csum2)) goto fault; *csump = csum_block_add(*csump, csum2, pos); if ((len -= copy) == 0) return 0; offset += copy; to += copy; pos += copy; } start = end; } } if (!len) return 0; fault: return -EFAULT; }
static void greth_clean_rings(struct greth_private *greth) { int i; struct greth_bd *rx_bdp = greth->rx_bd_base; struct greth_bd *tx_bdp = greth->tx_bd_base; if (greth->gbit_mac) { /* Free and unmap RX buffers */ for (i = 0; i < GRETH_RXBD_NUM; i++, rx_bdp++) { if (greth->rx_skbuff[i] != NULL) { dev_kfree_skb(greth->rx_skbuff[i]); dma_unmap_single(greth->dev, greth_read_bd(&rx_bdp->addr), MAX_FRAME_SIZE+NET_IP_ALIGN, DMA_FROM_DEVICE); } } /* TX buffers */ while (greth->tx_free < GRETH_TXBD_NUM) { struct sk_buff *skb = greth->tx_skbuff[greth->tx_last]; int nr_frags = skb_shinfo(skb)->nr_frags; tx_bdp = greth->tx_bd_base + greth->tx_last; greth->tx_last = NEXT_TX(greth->tx_last); dma_unmap_single(greth->dev, greth_read_bd(&tx_bdp->addr), skb_headlen(skb), DMA_TO_DEVICE); for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; tx_bdp = greth->tx_bd_base + greth->tx_last; dma_unmap_page(greth->dev, greth_read_bd(&tx_bdp->addr), skb_frag_size(frag), DMA_TO_DEVICE); greth->tx_last = NEXT_TX(greth->tx_last); } greth->tx_free += nr_frags+1; dev_kfree_skb(skb); } } else { /* 10/100 Mbps MAC */ for (i = 0; i < GRETH_RXBD_NUM; i++, rx_bdp++) { kfree(greth->rx_bufs[i]); dma_unmap_single(greth->dev, greth_read_bd(&rx_bdp->addr), MAX_FRAME_SIZE, DMA_FROM_DEVICE); } for (i = 0; i < GRETH_TXBD_NUM; i++, tx_bdp++) { kfree(greth->tx_bufs[i]); dma_unmap_single(greth->dev, greth_read_bd(&tx_bdp->addr), MAX_FRAME_SIZE, DMA_TO_DEVICE); } } }
static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, struct net_device *netdev) { struct ibmveth_adapter *adapter = netdev_priv(netdev); unsigned int desc_flags; union ibmveth_buf_desc descs[6]; int last, i; int force_bounce = 0; /* * veth handles a maximum of 6 segments including the header, so * we have to linearize the skb if there are more than this. */ if (skb_shinfo(skb)->nr_frags > 5 && __skb_linearize(skb)) { netdev->stats.tx_dropped++; goto out; } /* veth can't checksum offload UDP */ if (skb->ip_summed == CHECKSUM_PARTIAL && ((skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->protocol != IPPROTO_TCP) || (skb->protocol == htons(ETH_P_IPV6) && ipv6_hdr(skb)->nexthdr != IPPROTO_TCP)) && skb_checksum_help(skb)) { netdev_err(netdev, "tx: failed to checksum packet\n"); netdev->stats.tx_dropped++; goto out; } desc_flags = IBMVETH_BUF_VALID; if (skb->ip_summed == CHECKSUM_PARTIAL) { unsigned char *buf = skb_transport_header(skb) + skb->csum_offset; desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD); /* Need to zero out the checksum */ buf[0] = 0; buf[1] = 0; } retry_bounce: memset(descs, 0, sizeof(descs)); /* * If a linear packet is below the rx threshold then * copy it into the static bounce buffer. This avoids the * cost of a TCE insert and remove. */ if (force_bounce || (!skb_is_nonlinear(skb) && (skb->len < tx_copybreak))) { skb_copy_from_linear_data(skb, adapter->bounce_buffer, skb->len); descs[0].fields.flags_len = desc_flags | skb->len; descs[0].fields.address = adapter->bounce_buffer_dma; if (ibmveth_send(adapter, descs)) { adapter->tx_send_failed++; netdev->stats.tx_dropped++; } else { netdev->stats.tx_packets++; netdev->stats.tx_bytes += skb->len; } goto out; } /* Map the header */ descs[0].fields.address = dma_map_single(&adapter->vdev->dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (dma_mapping_error(&adapter->vdev->dev, descs[0].fields.address)) goto map_failed; descs[0].fields.flags_len = desc_flags | skb_headlen(skb); /* Map the frags */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { unsigned long dma_addr; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; dma_addr = dma_map_page(&adapter->vdev->dev, frag->page, frag->page_offset, frag->size, DMA_TO_DEVICE); if (dma_mapping_error(&adapter->vdev->dev, dma_addr)) goto map_failed_frags; descs[i+1].fields.flags_len = desc_flags | frag->size; descs[i+1].fields.address = dma_addr; } if (ibmveth_send(adapter, descs)) { adapter->tx_send_failed++; netdev->stats.tx_dropped++; } else { netdev->stats.tx_packets++; netdev->stats.tx_bytes += skb->len; } dma_unmap_single(&adapter->vdev->dev, descs[0].fields.address, descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK, DMA_TO_DEVICE); for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++) dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address, descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK, DMA_TO_DEVICE); out: dev_kfree_skb(skb); return NETDEV_TX_OK; map_failed_frags: last = i+1; for (i = 0; i < last; i++) dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address, descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK, DMA_TO_DEVICE); map_failed: if (!firmware_has_feature(FW_FEATURE_CMO)) netdev_err(netdev, "tx: unable to map xmit buffer\n"); adapter->tx_map_failed++; skb_linearize(skb); force_bounce = 1; goto retry_bounce; }
/** * skb_copy_datagram_from_iovec - Copy a datagram from an iovec. * @skb: buffer to copy * @offset: offset in the buffer to start copying to * @from: io vector to copy to * @len: amount of data to copy to buffer from iovec * * Returns 0 or -EFAULT. * Note: the iovec is modified during the copy. */ int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset, struct iovec *from, int len) { int start = skb_headlen(skb); int i, copy = start - offset; /* Copy header. */ if (copy > 0) { if (copy > len) copy = len; if (memcpy_fromiovec(skb->data + offset, from, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; WARN_ON(start > offset + len); end = start + skb_shinfo(skb)->frags[i].size; if ((copy = end - offset) > 0) { int err; u8 *vaddr; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct page *page = frag->page; if (copy > len) copy = len; vaddr = kmap(page); err = memcpy_fromiovec(vaddr + frag->page_offset + offset - start, from, copy); kunmap(page); if (err) goto fault; if (!(len -= copy)) return 0; offset += copy; } start = end; } if (skb_shinfo(skb)->frag_list) { struct sk_buff *list = skb_shinfo(skb)->frag_list; for (; list; list = list->next) { int end; WARN_ON(start > offset + len); end = start + list->len; if ((copy = end - offset) > 0) { if (copy > len) copy = len; if (skb_copy_datagram_from_iovec(list, offset - start, from, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; } } if (!len) return 0; fault: return -EFAULT; }
static int xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) { unsigned short id; struct netfront_info *np = netdev_priv(dev); struct xen_netif_tx_request *tx; struct xen_netif_extra_info *extra; char *data = skb->data; RING_IDX i; grant_ref_t ref; unsigned long mfn; int notify; int frags = skb_shinfo(skb)->nr_frags; unsigned int offset = offset_in_page(data); unsigned int len = skb_headlen(skb); /* If skb->len is too big for wire format, drop skb and alert * user about misconfiguration. */ if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { net_alert_ratelimited( "xennet: skb->len = %u, too big for wire format\n", skb->len); goto drop; } frags += DIV_ROUND_UP(offset + len, PAGE_SIZE); if (unlikely(frags > MAX_SKB_FRAGS + 1)) { printk(KERN_ALERT "xennet: skb rides the rocket: %d frags\n", frags); dump_stack(); goto drop; } spin_lock_irq(&np->tx_lock); if (unlikely(!netif_carrier_ok(dev) || (frags > 1 && !xennet_can_sg(dev)) || netif_needs_gso(skb, netif_skb_features(skb)))) { spin_unlock_irq(&np->tx_lock); goto drop; } i = np->tx.req_prod_pvt; id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs); np->tx_skbs[id].skb = skb; tx = RING_GET_REQUEST(&np->tx, i); tx->id = id; ref = gnttab_claim_grant_reference(&np->gref_tx_head); BUG_ON((signed short)ref < 0); mfn = virt_to_mfn(data); gnttab_grant_foreign_access_ref( ref, np->xbdev->otherend_id, mfn, GNTMAP_readonly); tx->gref = np->grant_tx_ref[id] = ref; tx->offset = offset; tx->size = len; extra = NULL; tx->flags = 0; if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */ tx->flags |= NETTXF_csum_blank | NETTXF_data_validated; else if (skb->ip_summed == CHECKSUM_UNNECESSARY) /* remote but checksummed. */ tx->flags |= NETTXF_data_validated; if (skb_shinfo(skb)->gso_size) { struct xen_netif_extra_info *gso; gso = (struct xen_netif_extra_info *) RING_GET_REQUEST(&np->tx, ++i); if (extra) extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE; else tx->flags |= NETTXF_extra_info; gso->u.gso.size = skb_shinfo(skb)->gso_size; gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; gso->u.gso.pad = 0; gso->u.gso.features = 0; gso->type = XEN_NETIF_EXTRA_TYPE_GSO; gso->flags = 0; extra = gso; } np->tx.req_prod_pvt = i + 1; xennet_make_frags(skb, dev, tx); tx->size = skb->len; RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->tx, notify); if (notify) notify_remote_via_irq(np->netdev->irq); dev->stats.tx_bytes += skb->len; dev->stats.tx_packets++; /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ xennet_tx_buf_gc(dev); if (!netfront_tx_slot_available(np)) netif_stop_queue(dev); spin_unlock_irq(&np->tx_lock); return NETDEV_TX_OK; drop: dev->stats.tx_dropped++; dev_kfree_skb(skb); return NETDEV_TX_OK; }
static int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *)) { struct sk_buff *frag; struct rt6_info *rt = (struct rt6_info*)skb_dst(skb); struct ipv6_pinfo *np = skb->sk ? inet6_sk(skb->sk) : NULL; struct ipv6hdr *tmp_hdr; struct frag_hdr *fh; unsigned int mtu, hlen, left, len; __be32 frag_id = 0; int ptr, offset = 0, err=0; u8 *prevhdr, nexthdr = 0; struct net *net = dev_net(skb_dst(skb)->dev); hlen = ip6_find_1stfragopt(skb, &prevhdr); nexthdr = *prevhdr; mtu = ip6_skb_dst_mtu(skb); /* We must not fragment if the socket is set to force MTU discovery * or if the skb it not generated by a local socket. (This last * check should be redundant, but it's free.) */ if (!skb->local_df) { skb->dev = skb_dst(skb)->dev; icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu, skb->dev); IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); kfree_skb(skb); return -EMSGSIZE; } if (np && np->frag_size < mtu) { if (np->frag_size) mtu = np->frag_size; } mtu -= hlen + sizeof(struct frag_hdr); if (skb_has_frags(skb)) { int first_len = skb_pagelen(skb); struct sk_buff *frag2; if (first_len - hlen > mtu || ((first_len - hlen) & 7) || skb_cloned(skb)) goto slow_path; skb_walk_frags(skb, frag) { /* Correct geometry. */ if (frag->len > mtu || ((frag->len & 7) && frag->next) || skb_headroom(frag) < hlen) goto slow_path_clean; /* Partially cloned skb? */ if (skb_shared(frag)) goto slow_path_clean; BUG_ON(frag->sk); if (skb->sk) { frag->sk = skb->sk; frag->destructor = sock_wfree; } skb->truesize -= frag->truesize; } err = 0; offset = 0; frag = skb_shinfo(skb)->frag_list; skb_frag_list_init(skb); /* BUILD HEADER */ *prevhdr = NEXTHDR_FRAGMENT; tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); if (!tmp_hdr) { IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); return -ENOMEM; } __skb_pull(skb, hlen); fh = (struct frag_hdr*)__skb_push(skb, sizeof(struct frag_hdr)); __skb_push(skb, hlen); skb_reset_network_header(skb); memcpy(skb_network_header(skb), tmp_hdr, hlen); ipv6_select_ident(fh, rt); fh->nexthdr = nexthdr; fh->reserved = 0; fh->frag_off = htons(IP6_MF); frag_id = fh->identification; first_len = skb_pagelen(skb); skb->data_len = first_len - skb_headlen(skb); skb->len = first_len; ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr)); dst_hold(&rt->u.dst); for (;;) { /* Prepare header of the next frame, * before previous one went down. */ if (frag) { frag->ip_summed = CHECKSUM_NONE; skb_reset_transport_header(frag); fh = (struct frag_hdr*)__skb_push(frag, sizeof(struct frag_hdr)); __skb_push(frag, hlen); skb_reset_network_header(frag); memcpy(skb_network_header(frag), tmp_hdr, hlen); offset += skb->len - hlen - sizeof(struct frag_hdr); fh->nexthdr = nexthdr; fh->reserved = 0; fh->frag_off = htons(offset); if (frag->next != NULL) fh->frag_off |= htons(IP6_MF); fh->identification = frag_id; ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); ip6_copy_metadata(frag, skb); } err = output(skb); if(!err) IP6_INC_STATS(net, ip6_dst_idev(&rt->u.dst), IPSTATS_MIB_FRAGCREATES); if (err || !frag) break; skb = frag; frag = skb->next; skb->next = NULL; } kfree(tmp_hdr); if (err == 0) { IP6_INC_STATS(net, ip6_dst_idev(&rt->u.dst), IPSTATS_MIB_FRAGOKS); dst_release(&rt->u.dst); return 0; } while (frag) { skb = frag->next; kfree_skb(frag); frag = skb; } IP6_INC_STATS(net, ip6_dst_idev(&rt->u.dst), IPSTATS_MIB_FRAGFAILS); dst_release(&rt->u.dst); return err; slow_path_clean: skb_walk_frags(skb, frag2) { if (frag2 == frag) break; frag2->sk = NULL; frag2->destructor = NULL; skb->truesize += frag2->truesize; } } slow_path: left = skb->len - hlen; /* Space per frame */ ptr = hlen; /* Where to start from */ /* * Fragment the datagram. */ *prevhdr = NEXTHDR_FRAGMENT; /* * Keep copying data until we run out. */ while(left > 0) { len = left; /* IF: it doesn't fit, use 'mtu' - the data space left */ if (len > mtu) len = mtu; /* IF: we are not sending upto and including the packet end then align the next start on an eight byte boundary */ if (len < left) { len &= ~7; } /* * Allocate buffer. */ if ((frag = alloc_skb(len+hlen+sizeof(struct frag_hdr)+LL_ALLOCATED_SPACE(rt->u.dst.dev), GFP_ATOMIC)) == NULL) { NETDEBUG(KERN_INFO "IPv6: frag: no memory for new fragment!\n"); IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); err = -ENOMEM; goto fail; } /* * Set up data on packet */ ip6_copy_metadata(frag, skb); skb_reserve(frag, LL_RESERVED_SPACE(rt->u.dst.dev)); skb_put(frag, len + hlen + sizeof(struct frag_hdr)); skb_reset_network_header(frag); fh = (struct frag_hdr *)(skb_network_header(frag) + hlen); frag->transport_header = (frag->network_header + hlen + sizeof(struct frag_hdr)); /* * Charge the memory for the fragment to any owner * it might possess */ if (skb->sk) skb_set_owner_w(frag, skb->sk); /* * Copy the packet header into the new buffer. */ skb_copy_from_linear_data(skb, skb_network_header(frag), hlen); /* * Build fragment header. */ fh->nexthdr = nexthdr; fh->reserved = 0; if (!frag_id) { ipv6_select_ident(fh, rt); frag_id = fh->identification; } else fh->identification = frag_id; /* * Copy a block of the IP datagram. */ if (skb_copy_bits(skb, ptr, skb_transport_header(frag), len)) BUG(); left -= len; fh->frag_off = htons(offset); if (left > 0) fh->frag_off |= htons(IP6_MF); ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); ptr += len; offset += len; /* * Put this fragment into the sending queue. */ err = output(frag); if (err) goto fail; IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGCREATES); } IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGOKS); kfree_skb(skb); return err; fail: IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); kfree_skb(skb); return err; }
/** * skb_copy_datagram_iovec - Copy a datagram to an iovec. * @skb: buffer to copy * @offset: offset in the buffer to start copying from * @to: io vector to copy to * @len: amount of data to copy from buffer to iovec * * Note: the iovec is modified during the copy. */ int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, struct iovec *to, int len) { int start = skb_headlen(skb); int i, copy = start - offset; struct sk_buff *frag_iter; trace_skb_copy_datagram_iovec(skb, len); /* Copy header. */ if (copy > 0) { if (copy > len) copy = len; if (memcpy_toiovec(to, skb->data + offset, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; WARN_ON(start > offset + len); end = start + skb_shinfo(skb)->frags[i].size; if ((copy = end - offset) > 0) { int err; u8 *vaddr; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct page *page = frag->page; if (copy > len) copy = len; vaddr = kmap(page); err = memcpy_toiovec(to, vaddr + frag->page_offset + offset - start, copy); kunmap(page); if (err) goto fault; if (!(len -= copy)) return 0; offset += copy; } start = end; } skb_walk_frags(skb, frag_iter) { int end; WARN_ON(start > offset + len); end = start + frag_iter->len; if ((copy = end - offset) > 0) { if (copy > len) copy = len; if (skb_copy_datagram_iovec(frag_iter, offset - start, to, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; }
static void xennet_make_frags(struct sk_buff *skb, struct net_device *dev, struct xen_netif_tx_request *tx) { struct netfront_info *np = netdev_priv(dev); char *data = skb->data; unsigned long mfn; RING_IDX prod = np->tx.req_prod_pvt; int frags = skb_shinfo(skb)->nr_frags; unsigned int offset = offset_in_page(data); unsigned int len = skb_headlen(skb); unsigned int id; grant_ref_t ref; int i; /* While the header overlaps a page boundary (including being larger than a page), split it it into page-sized chunks. */ while (len > PAGE_SIZE - offset) { tx->size = PAGE_SIZE - offset; tx->flags |= NETTXF_more_data; len -= tx->size; data += tx->size; offset = 0; id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs); np->tx_skbs[id].skb = skb_get(skb); tx = RING_GET_REQUEST(&np->tx, prod++); tx->id = id; ref = gnttab_claim_grant_reference(&np->gref_tx_head); BUG_ON((signed short)ref < 0); mfn = virt_to_mfn(data); gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id, mfn, GNTMAP_readonly); tx->gref = np->grant_tx_ref[id] = ref; tx->offset = offset; tx->size = len; tx->flags = 0; } /* Grant backend access to each skb fragment page. */ for (i = 0; i < frags; i++) { skb_frag_t *frag = skb_shinfo(skb)->frags + i; tx->flags |= NETTXF_more_data; id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs); np->tx_skbs[id].skb = skb_get(skb); tx = RING_GET_REQUEST(&np->tx, prod++); tx->id = id; ref = gnttab_claim_grant_reference(&np->gref_tx_head); BUG_ON((signed short)ref < 0); mfn = pfn_to_mfn(page_to_pfn(frag->page)); gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id, mfn, GNTMAP_readonly); tx->gref = np->grant_tx_ref[id] = ref; tx->offset = frag->page_offset; tx->size = frag->size; tx->flags = 0; } np->tx.req_prod_pvt = prod; }
/* * Add a socket buffer to a TX queue * * This maps all fragments of a socket buffer for DMA and adds them to * the TX queue. The queue's insert pointer will be incremented by * the number of fragments in the socket buffer. * * If any DMA mapping fails, any mapped fragments will be unmapped, * the queue's insert pointer will be restored to its original value. * * This function is split out from efx_hard_start_xmit to allow the * loopback test to direct packets via specific TX queues. * * Returns NETDEV_TX_OK or NETDEV_TX_BUSY * You must hold netif_tx_lock() to call this function. */ netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; struct pci_dev *pci_dev = efx->pci_dev; struct efx_tx_buffer *buffer; skb_frag_t *fragment; unsigned int len, unmap_len = 0, fill_level, insert_ptr; dma_addr_t dma_addr, unmap_addr = 0; unsigned int dma_len; bool unmap_single; int q_space, i = 0; netdev_tx_t rc = NETDEV_TX_OK; EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); if (skb_shinfo(skb)->gso_size) return efx_enqueue_skb_tso(tx_queue, skb); /* Get size of the initial fragment */ len = skb_headlen(skb); /* Pad if necessary */ if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) { EFX_BUG_ON_PARANOID(skb->data_len); len = 32 + 1; if (skb_pad(skb, len - skb->len)) return NETDEV_TX_OK; } fill_level = tx_queue->insert_count - tx_queue->old_read_count; q_space = efx->txq_entries - 1 - fill_level; /* Map for DMA. Use pci_map_single rather than pci_map_page * since this is more efficient on machines with sparse * memory. */ unmap_single = true; dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE); /* Process all fragments */ while (1) { if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr))) goto pci_err; /* Store fields for marking in the per-fragment final * descriptor */ unmap_len = len; unmap_addr = dma_addr; /* Add to TX queue, splitting across DMA boundaries */ do { if (unlikely(q_space-- <= 0)) { /* It might be that completions have * happened since the xmit path last * checked. Update the xmit path's * copy of read_count. */ netif_tx_stop_queue(tx_queue->core_txq); /* This memory barrier protects the * change of queue state from the access * of read_count. */ smp_mb(); tx_queue->old_read_count = ACCESS_ONCE(tx_queue->read_count); fill_level = (tx_queue->insert_count - tx_queue->old_read_count); q_space = efx->txq_entries - 1 - fill_level; if (unlikely(q_space-- <= 0)) { rc = NETDEV_TX_BUSY; goto unwind; } smp_mb(); if (likely(!efx->loopback_selftest)) netif_tx_start_queue( tx_queue->core_txq); } insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->tsoh); EFX_BUG_ON_PARANOID(buffer->skb); EFX_BUG_ON_PARANOID(buffer->len); EFX_BUG_ON_PARANOID(!buffer->continuation); EFX_BUG_ON_PARANOID(buffer->unmap_len); dma_len = efx_max_tx_len(efx, dma_addr); if (likely(dma_len >= len)) dma_len = len; /* Fill out per descriptor fields */ buffer->len = dma_len; buffer->dma_addr = dma_addr; len -= dma_len; dma_addr += dma_len; ++tx_queue->insert_count; } while (len); /* Transfer ownership of the unmapping to the final buffer */ buffer->unmap_single = unmap_single; buffer->unmap_len = unmap_len; unmap_len = 0; /* Get address and size of next fragment */ if (i >= skb_shinfo(skb)->nr_frags) break; fragment = &skb_shinfo(skb)->frags[i]; len = skb_frag_size(fragment); i++; /* Map for DMA */ unmap_single = false; dma_addr = skb_frag_dma_map(&pci_dev->dev, fragment, 0, len, DMA_TO_DEVICE); } /* Transfer ownership of the skb to the final buffer */ buffer->skb = skb; buffer->continuation = false; /* Pass off to hardware */ efx_nic_push_buffers(tx_queue); return NETDEV_TX_OK; pci_err: netif_err(efx, tx_err, efx->net_dev, " TX queue %d could not map skb with %d bytes %d " "fragments for DMA\n", tx_queue->queue, skb->len, skb_shinfo(skb)->nr_frags + 1); /* Mark the packet as transmitted, and free the SKB ourselves */ dev_kfree_skb_any(skb); unwind: /* Work backwards until we hit the original insert pointer value */ while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_dequeue_buffer(tx_queue, buffer); buffer->len = 0; } /* Free the fragment we were mid-way through pushing */ if (unmap_len) { if (unmap_single) pci_unmap_single(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); else pci_unmap_page(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); } return rc; }
static netdev_tx_t greth_start_xmit_gbit(struct sk_buff *skb, struct net_device *dev) { struct greth_private *greth = netdev_priv(dev); struct greth_bd *bdp; u32 status = 0, dma_addr, ctrl; int curr_tx, nr_frags, i, err = NETDEV_TX_OK; unsigned long flags; nr_frags = skb_shinfo(skb)->nr_frags; /* Clean TX Ring */ greth_clean_tx_gbit(dev); if (greth->tx_free < nr_frags + 1) { spin_lock_irqsave(&greth->devlock, flags);/*save from poll/irq*/ ctrl = GRETH_REGLOAD(greth->regs->control); /* Enable TX IRQ only if not already in poll() routine */ if (ctrl & GRETH_RXI) GRETH_REGSAVE(greth->regs->control, ctrl | GRETH_TXI); netif_stop_queue(dev); spin_unlock_irqrestore(&greth->devlock, flags); err = NETDEV_TX_BUSY; goto out; } if (netif_msg_pktdata(greth)) greth_print_tx_packet(skb); if (unlikely(skb->len > MAX_FRAME_SIZE)) { dev->stats.tx_errors++; goto out; } /* Save skb pointer. */ greth->tx_skbuff[greth->tx_next] = skb; /* Linear buf */ if (nr_frags != 0) status = GRETH_TXBD_MORE; if (skb->ip_summed == CHECKSUM_PARTIAL) status |= GRETH_TXBD_CSALL; status |= skb_headlen(skb) & GRETH_BD_LEN; if (greth->tx_next == GRETH_TXBD_NUM_MASK) status |= GRETH_BD_WR; bdp = greth->tx_bd_base + greth->tx_next; greth_write_bd(&bdp->stat, status); dma_addr = dma_map_single(greth->dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(greth->dev, dma_addr))) goto map_error; greth_write_bd(&bdp->addr, dma_addr); curr_tx = NEXT_TX(greth->tx_next); /* Frags */ for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; greth->tx_skbuff[curr_tx] = NULL; bdp = greth->tx_bd_base + curr_tx; status = GRETH_BD_EN; if (skb->ip_summed == CHECKSUM_PARTIAL) status |= GRETH_TXBD_CSALL; status |= skb_frag_size(frag) & GRETH_BD_LEN; /* Wrap around descriptor ring */ if (curr_tx == GRETH_TXBD_NUM_MASK) status |= GRETH_BD_WR; /* More fragments left */ if (i < nr_frags - 1) status |= GRETH_TXBD_MORE; else status |= GRETH_BD_IE; /* enable IRQ on last fragment */ greth_write_bd(&bdp->stat, status); dma_addr = skb_frag_dma_map(greth->dev, frag, 0, skb_frag_size(frag), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(greth->dev, dma_addr))) goto frag_map_error; greth_write_bd(&bdp->addr, dma_addr); curr_tx = NEXT_TX(curr_tx); } wmb(); /* Enable the descriptor chain by enabling the first descriptor */ bdp = greth->tx_bd_base + greth->tx_next; greth_write_bd(&bdp->stat, greth_read_bd(&bdp->stat) | GRETH_BD_EN); greth->tx_next = curr_tx; greth->tx_free -= nr_frags + 1; wmb(); spin_lock_irqsave(&greth->devlock, flags); /*save from poll/irq*/ greth_enable_tx(greth); spin_unlock_irqrestore(&greth->devlock, flags); return NETDEV_TX_OK; frag_map_error: /* Unmap SKB mappings that succeeded and disable descriptor */ for (i = 0; greth->tx_next + i != curr_tx; i++) { bdp = greth->tx_bd_base + greth->tx_next + i; dma_unmap_single(greth->dev, greth_read_bd(&bdp->addr), greth_read_bd(&bdp->stat) & GRETH_BD_LEN, DMA_TO_DEVICE); greth_write_bd(&bdp->stat, 0); } map_error: if (net_ratelimit()) dev_warn(greth->dev, "Could not create TX DMA mapping\n"); dev_kfree_skb(skb); out: return err; }
/** * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer * @tx_queue: Efx TX queue * @skb: Socket buffer * * Context: You must hold netif_tx_lock() to call this function. * * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if * @skb was not enqueued. In all cases @skb is consumed. Return * %NETDEV_TX_OK or %NETDEV_TX_BUSY. */ static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; int frag_i, rc, rc2 = NETDEV_TX_OK; struct tso_state state; /* Since the stack does not limit the number of segments per * skb, we must do so. Otherwise an attacker may be able to * make the TCP produce skbs that will never fit in our TX * queue, causing repeated resets. */ if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) { unsigned int excess = (skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) * skb_shinfo(skb)->gso_size; if (__pskb_trim(skb, skb->len - excess)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; } } /* Find the packet protocol and sanity-check it */ state.protocol = efx_tso_check_protocol(skb); EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); tso_start(&state, skb); /* Assume that skb header area contains exactly the headers, and * all payload is in the frag list. */ if (skb_headlen(skb) == state.header_len) { /* Grab the first payload fragment. */ EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1); frag_i = 0; rc = tso_get_fragment(&state, efx, skb_shinfo(skb)->frags + frag_i); if (rc) goto mem_err; } else { rc = tso_get_head_fragment(&state, efx, skb); if (rc) goto mem_err; frag_i = -1; } if (tso_start_new_packet(tx_queue, skb, &state) < 0) goto mem_err; while (1) { rc = tso_fill_packet_with_fragment(tx_queue, skb, &state); if (unlikely(rc)) { rc2 = NETDEV_TX_BUSY; goto unwind; } /* Move onto the next fragment? */ if (state.in_len == 0) { if (++frag_i >= skb_shinfo(skb)->nr_frags) /* End of payload reached. */ break; rc = tso_get_fragment(&state, efx, skb_shinfo(skb)->frags + frag_i); if (rc) goto mem_err; } /* Start at new packet? */ if (state.packet_space == 0 && tso_start_new_packet(tx_queue, skb, &state) < 0) goto mem_err; } /* Pass off to hardware */ efx_nic_push_buffers(tx_queue); tx_queue->tso_bursts++; return NETDEV_TX_OK; mem_err: netif_err(efx, tx_err, efx->net_dev, "Out of memory for TSO headers, or PCI mapping error\n"); dev_kfree_skb_any(skb); unwind: /* Free the DMA mapping we were in the process of writing out */ if (state.unmap_len) { if (state.unmap_single) pci_unmap_single(efx->pci_dev, state.unmap_addr, state.unmap_len, PCI_DMA_TODEVICE); else pci_unmap_page(efx->pci_dev, state.unmap_addr, state.unmap_len, PCI_DMA_TODEVICE); } efx_enqueue_unwind(tx_queue); return rc2; }
static void xenvif_rx_action(struct xenvif *vif) { s8 status; u16 flags; struct xen_netif_rx_response *resp; struct sk_buff_head rxq; struct sk_buff *skb; LIST_HEAD(notify); int ret; unsigned long offset; struct skb_cb_overlay *sco; bool need_to_notify = false; struct netrx_pending_operations npo = { .copy = vif->grant_copy_op, .meta = vif->meta, }; skb_queue_head_init(&rxq); while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) { RING_IDX max_slots_needed; int i; /* We need a cheap worse case estimate for the number of * slots we'll use. */ max_slots_needed = DIV_ROUND_UP(offset_in_page(skb->data) + skb_headlen(skb), PAGE_SIZE); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { unsigned int size; size = skb_frag_size(&skb_shinfo(skb)->frags[i]); max_slots_needed += DIV_ROUND_UP(size, PAGE_SIZE); } if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 || skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) max_slots_needed++; /* If the skb may not fit then bail out now */ if (!xenvif_rx_ring_slots_available(vif, max_slots_needed)) { skb_queue_head(&vif->rx_queue, skb); need_to_notify = true; vif->rx_last_skb_slots = max_slots_needed; break; } else vif->rx_last_skb_slots = 0; sco = (struct skb_cb_overlay *)skb->cb; sco->meta_slots_used = xenvif_gop_skb(skb, &npo); BUG_ON(sco->meta_slots_used > max_slots_needed); __skb_queue_tail(&rxq, skb); } BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta)); if (!npo.copy_prod) goto done; BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS); gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod); while ((skb = __skb_dequeue(&rxq)) != NULL) { sco = (struct skb_cb_overlay *)skb->cb; if ((1 << vif->meta[npo.meta_cons].gso_type) & vif->gso_prefix_mask) { resp = RING_GET_RESPONSE(&vif->rx, vif->rx.rsp_prod_pvt++); resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data; resp->offset = vif->meta[npo.meta_cons].gso_size; resp->id = vif->meta[npo.meta_cons].id; resp->status = sco->meta_slots_used; npo.meta_cons++; sco->meta_slots_used--; } vif->dev->stats.tx_bytes += skb->len; vif->dev->stats.tx_packets++; status = xenvif_check_gop(vif, sco->meta_slots_used, &npo); if (sco->meta_slots_used == 1) flags = 0; else flags = XEN_NETRXF_more_data; if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */ flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated; else if (skb->ip_summed == CHECKSUM_UNNECESSARY) /* remote but checksummed. */ flags |= XEN_NETRXF_data_validated; offset = 0; resp = make_rx_response(vif, vif->meta[npo.meta_cons].id, status, offset, vif->meta[npo.meta_cons].size, flags); if ((1 << vif->meta[npo.meta_cons].gso_type) & vif->gso_mask) { struct xen_netif_extra_info *gso = (struct xen_netif_extra_info *) RING_GET_RESPONSE(&vif->rx, vif->rx.rsp_prod_pvt++); resp->flags |= XEN_NETRXF_extra_info; gso->u.gso.type = vif->meta[npo.meta_cons].gso_type; gso->u.gso.size = vif->meta[npo.meta_cons].gso_size; gso->u.gso.pad = 0; gso->u.gso.features = 0; gso->type = XEN_NETIF_EXTRA_TYPE_GSO; gso->flags = 0; } xenvif_add_frag_responses(vif, status, vif->meta + npo.meta_cons + 1, sco->meta_slots_used); RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret); need_to_notify |= !!ret; npo.meta_cons += sco->meta_slots_used; dev_kfree_skb(skb); } done: if (need_to_notify) notify_remote_via_irq(vif->rx_irq); } void xenvif_check_rx_xenvif(struct xenvif *vif) { int more_to_do; RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do); if (more_to_do) napi_schedule(&vif->napi); }
static int xenvif_tx_submit(struct xenvif *vif) { struct gnttab_copy *gop = vif->tx_copy_ops; struct sk_buff *skb; int work_done = 0; while ((skb = __skb_dequeue(&vif->tx_queue)) != NULL) { struct xen_netif_tx_request *txp; u16 pending_idx; unsigned data_len; pending_idx = *((u16 *)skb->data); txp = &vif->pending_tx_info[pending_idx].req; /* Check the remap error code. */ if (unlikely(xenvif_tx_check_gop(vif, skb, &gop))) { netdev_dbg(vif->dev, "netback grant failed.\n"); skb_shinfo(skb)->nr_frags = 0; kfree_skb(skb); continue; } data_len = skb->len; memcpy(skb->data, (void *)(idx_to_kaddr(vif, pending_idx)|txp->offset), data_len); if (data_len < txp->size) { /* Append the packet payload as a fragment. */ txp->offset += data_len; txp->size -= data_len; } else { /* Schedule a response immediately. */ xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY); } if (txp->flags & XEN_NETTXF_csum_blank) skb->ip_summed = CHECKSUM_PARTIAL; else if (txp->flags & XEN_NETTXF_data_validated) skb->ip_summed = CHECKSUM_UNNECESSARY; xenvif_fill_frags(vif, skb); if (skb_is_nonlinear(skb) && skb_headlen(skb) < PKT_PROT_LEN) { int target = min_t(int, skb->len, PKT_PROT_LEN); __pskb_pull_tail(skb, target - skb_headlen(skb)); } skb->dev = vif->dev; skb->protocol = eth_type_trans(skb, skb->dev); skb_reset_network_header(skb); if (checksum_setup(vif, skb)) { netdev_dbg(vif->dev, "Can't setup checksum in net_tx_action\n"); kfree_skb(skb); continue; } skb_probe_transport_header(skb, 0); /* If the packet is GSO then we will have just set up the * transport header offset in checksum_setup so it's now * straightforward to calculate gso_segs. */ if (skb_is_gso(skb)) { int mss = skb_shinfo(skb)->gso_size; int hdrlen = skb_transport_header(skb) - skb_mac_header(skb) + tcp_hdrlen(skb); skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len - hdrlen, mss); } vif->dev->stats.rx_bytes += skb->len; vif->dev->stats.rx_packets++; work_done++; netif_receive_skb(skb); }
static netdev_tx_t bgmac_dma_tx_add(struct bgmac *bgmac, struct bgmac_dma_ring *ring, struct sk_buff *skb) { struct device *dma_dev = bgmac->dma_dev; struct net_device *net_dev = bgmac->net_dev; int index = ring->end % BGMAC_TX_RING_SLOTS; struct bgmac_slot_info *slot = &ring->slots[index]; int nr_frags; u32 flags; int i; if (skb->len > BGMAC_DESC_CTL1_LEN) { netdev_err(bgmac->net_dev, "Too long skb (%d)\n", skb->len); goto err_drop; } if (skb->ip_summed == CHECKSUM_PARTIAL) skb_checksum_help(skb); nr_frags = skb_shinfo(skb)->nr_frags; /* ring->end - ring->start will return the number of valid slots, * even when ring->end overflows */ if (ring->end - ring->start + nr_frags + 1 >= BGMAC_TX_RING_SLOTS) { netdev_err(bgmac->net_dev, "TX ring is full, queue should be stopped!\n"); netif_stop_queue(net_dev); return NETDEV_TX_BUSY; } slot->dma_addr = dma_map_single(dma_dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dma_dev, slot->dma_addr))) goto err_dma_head; flags = BGMAC_DESC_CTL0_SOF; if (!nr_frags) flags |= BGMAC_DESC_CTL0_EOF | BGMAC_DESC_CTL0_IOC; bgmac_dma_tx_add_buf(bgmac, ring, index, skb_headlen(skb), flags); flags = 0; for (i = 0; i < nr_frags; i++) { struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; int len = skb_frag_size(frag); index = (index + 1) % BGMAC_TX_RING_SLOTS; slot = &ring->slots[index]; slot->dma_addr = skb_frag_dma_map(dma_dev, frag, 0, len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dma_dev, slot->dma_addr))) goto err_dma; if (i == nr_frags - 1) flags |= BGMAC_DESC_CTL0_EOF | BGMAC_DESC_CTL0_IOC; bgmac_dma_tx_add_buf(bgmac, ring, index, len, flags); } slot->skb = skb; ring->end += nr_frags + 1; netdev_sent_queue(net_dev, skb->len); wmb(); /* Increase ring->end to point empty slot. We tell hardware the first * slot it should *not* read. */ bgmac_write(bgmac, ring->mmio_base + BGMAC_DMA_TX_INDEX, ring->index_base + (ring->end % BGMAC_TX_RING_SLOTS) * sizeof(struct bgmac_dma_desc)); if (ring->end - ring->start >= BGMAC_TX_RING_SLOTS - 8) netif_stop_queue(net_dev); return NETDEV_TX_OK; err_dma: dma_unmap_single(dma_dev, slot->dma_addr, skb_headlen(skb), DMA_TO_DEVICE); while (i-- > 0) { int index = (ring->end + i) % BGMAC_TX_RING_SLOTS; struct bgmac_slot_info *slot = &ring->slots[index]; u32 ctl1 = le32_to_cpu(ring->cpu_base[index].ctl1); int len = ctl1 & BGMAC_DESC_CTL1_LEN; dma_unmap_page(dma_dev, slot->dma_addr, len, DMA_TO_DEVICE); } err_dma_head: netdev_err(bgmac->net_dev, "Mapping error of skb on ring 0x%X\n", ring->mmio_base); err_drop: dev_kfree_skb(skb); net_dev->stats.tx_dropped++; net_dev->stats.tx_errors++; return NETDEV_TX_OK; }
static int c2_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct c2_port *c2_port = netdev_priv(netdev); struct c2_dev *c2dev = c2_port->c2dev; struct c2_ring *tx_ring = &c2_port->tx_ring; struct c2_element *elem; dma_addr_t mapaddr; u32 maplen; unsigned long flags; unsigned int i; spin_lock_irqsave(&c2_port->tx_lock, flags); if (unlikely(c2_port->tx_avail < (skb_shinfo(skb)->nr_frags + 1))) { netif_stop_queue(netdev); spin_unlock_irqrestore(&c2_port->tx_lock, flags); pr_debug("%s: Tx ring full when queue awake!\n", netdev->name); return NETDEV_TX_BUSY; } maplen = skb_headlen(skb); mapaddr = pci_map_single(c2dev->pcidev, skb->data, maplen, PCI_DMA_TODEVICE); elem = tx_ring->to_use; elem->skb = skb; elem->mapaddr = mapaddr; elem->maplen = maplen; /* Tell HW to xmit */ __raw_writeq((__force u64) cpu_to_be64(mapaddr), elem->hw_desc + C2_TXP_ADDR); __raw_writew((__force u16) cpu_to_be16(maplen), elem->hw_desc + C2_TXP_LEN); __raw_writew((__force u16) cpu_to_be16(TXP_HTXD_READY), elem->hw_desc + C2_TXP_FLAGS); netdev->stats.tx_packets++; netdev->stats.tx_bytes += maplen; /* Loop thru additional data fragments and queue them */ if (skb_shinfo(skb)->nr_frags) { for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; maplen = frag->size; mapaddr = pci_map_page(c2dev->pcidev, frag->page, frag->page_offset, maplen, PCI_DMA_TODEVICE); elem = elem->next; elem->skb = NULL; elem->mapaddr = mapaddr; elem->maplen = maplen; /* Tell HW to xmit */ __raw_writeq((__force u64) cpu_to_be64(mapaddr), elem->hw_desc + C2_TXP_ADDR); __raw_writew((__force u16) cpu_to_be16(maplen), elem->hw_desc + C2_TXP_LEN); __raw_writew((__force u16) cpu_to_be16(TXP_HTXD_READY), elem->hw_desc + C2_TXP_FLAGS); netdev->stats.tx_packets++; netdev->stats.tx_bytes += maplen; } } tx_ring->to_use = elem->next; c2_port->tx_avail -= (skb_shinfo(skb)->nr_frags + 1); if (c2_port->tx_avail <= MAX_SKB_FRAGS + 1) { netif_stop_queue(netdev); if (netif_msg_tx_queued(c2_port)) pr_debug("%s: transmit queue full\n", netdev->name); } spin_unlock_irqrestore(&c2_port->tx_lock, flags); netdev->trans_start = jiffies; return NETDEV_TX_OK; }