static struct sk_buff *lro_gen_skb(struct net_lro_mgr *lro_mgr, struct skb_frag_struct *frags, int len, int true_size, void *mac_hdr, int hlen, __wsum sum, u32 ip_summed) { struct sk_buff *skb; struct skb_frag_struct *skb_frags; int data_len = len; int hdr_len = min(len, hlen); skb = netdev_alloc_skb(lro_mgr->dev, hlen + lro_mgr->frag_align_pad); if (!skb) return NULL; skb_reserve(skb, lro_mgr->frag_align_pad); skb->len = len; skb->data_len = len - hdr_len; skb->truesize += true_size; skb->tail += hdr_len; memcpy(skb->data, mac_hdr, hdr_len); skb_frags = skb_shinfo(skb)->frags; while (data_len > 0) { *skb_frags = *frags; data_len -= skb_frag_size(frags); skb_frags++; frags++; skb_shinfo(skb)->nr_frags++; } skb_shinfo(skb)->frags[0].page_offset += hdr_len; skb_frag_size_sub(&skb_shinfo(skb)->frags[0], hdr_len); skb->ip_summed = ip_summed; skb->csum = sum; skb->protocol = eth_type_trans(skb, lro_mgr->dev); return skb; }
static void lro_add_frags(struct net_lro_desc *lro_desc, int len, int hlen, int truesize, struct skb_frag_struct *skb_frags, struct iphdr *iph, struct tcphdr *tcph) { struct sk_buff *skb = lro_desc->parent; int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph); lro_add_common(lro_desc, iph, tcph, tcp_data_len); skb->truesize += truesize; skb_frags[0].page_offset += hlen; skb_frag_size_sub(&skb_frags[0], hlen); while (tcp_data_len > 0) { *(lro_desc->next_frag) = *skb_frags; tcp_data_len -= skb_frag_size(skb_frags); lro_desc->next_frag++; skb_frags++; skb_shinfo(skb)->nr_frags++; } }
/** * Delete @len (the value is positive now) bytes from @frag. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to data after the deleted area in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct skb_shared_info *si = skb_shinfo(skb); SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); if (unlikely(off + len > skb_frag_size(frag))) { SS_WARN("Attempt to delete too much\n"); return -EFAULT; } /* Fast path: delete a full fragment. */ if (!off && len == skb_frag_size(frag)) { ss_skb_adjust_data_len(skb, -len); __skb_frag_unref(frag); if (i + 1 < si->nr_frags) memmove(&si->frags[i], &si->frags[i + 1], (si->nr_frags - i - 1) * sizeof(skb_frag_t)); --si->nr_frags; goto lookup_next_ptr; } /* Fast path: delete the head part of a fragment. */ if (!off) { frag->page_offset += len; skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); it->ptr = skb_frag_address(frag); return 0; } /* Fast path: delete the tail part of a fragment. */ if (off + len == skb_frag_size(frag)) { skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); ++i; goto lookup_next_ptr; } /* * Delete data in the middle of a fragment. After the data * is deleted the fragment will contain only the head part, * and the tail part is moved to another fragment. * [frag @i] [frag @i+1 - tail data] * * Make room for a fragment right after the @i fragment * to move the tail part of data there. */ if (__extend_pgfrags(skb, i + 1, 1, it)) return -EFAULT; /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off - len; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, len + tail_len); /* Make the fragment with the tail part. */ i = (i + 1) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off + len, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ ss_skb_adjust_data_len(skb, -len); if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address of data after the deleted area. */ it->flags = (skb != skb_dst); it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]); return 0; lookup_next_ptr: /* Get the next fragment after the deleted fragment. */ if (i < si->nr_frags) it->ptr = skb_frag_address(&si->frags[i]); return 0; }
/** * Get room for @len bytes of data starting from offset @off * in fragment @i. * * The room may be found in the preceding fragment if @off is zero. * Otherwise, a new fragment is allocated and fragments around the * fragment @i are rearranged so that data is not actually split * and copied. * * Note: @off is always within the borders of fragment @i. It can * point at the start of a fragment, but it can never point at the * location right after the end of a fragment. In other words, @off * can be zero, but it can not be equal to the size of fragment @i. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to the room for new data in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i]; SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); /* * If @off is zero and there's a preceding page fragment, * then try to append data to that fragment. Go for other * solutions if there's no room. */ if (!off && i) { frag_dst = __check_frag_room(skb, frag - 1, len); if (frag_dst) { /* Coalesce new data with the fragment. */ off = skb_frag_size(frag_dst); skb_frag_size_add(frag_dst, len); ss_skb_adjust_data_len(skb, len); it->ptr = (char *)skb_frag_address(frag_dst) + off; return 0; } } /* * Make a fragment that can hold @len bytes. If @off is * zero, then data is added at the start of fragment @i. * Make a fragment in slot @i, and the original fragment * is shifted forward. If @off is not zero, then make * a fragment in slot @i+1, and make an extra fragment * in slot @i+2 to hold the tail data. */ if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it)) return -EFAULT; /* If @off is zero, the job is done in __new_pgfrag(). */ if (!off) { it->ptr = skb_frag_address(frag); return 0; } /* * If data is added in the middle of a fragment, then split * the fragment. The head of the fragment stays there, and * the tail of the fragment is moved to a new fragment. * The fragment for new data is placed in between. * [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data] * If @i is close to MAX_SKB_FRAGS, then new fragments may * be located in another SKB. */ /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, tail_len); /* Make the fragment with the tail part. */ i = (i + 2) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address for new data. */ it->flags = !(i < MAX_SKB_FRAGS - 1); frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1; it->ptr = skb_frag_address(frag_dst); return 0; }