static int iwl_pcie_gen2_tx_add_frags(struct iwl_trans *trans,
struct sk_buff *skb,
struct iwl_tfh_tfd *tfd,
struct iwl_cmd_meta *out_meta)
{
int i;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
dma_addr_t tb_phys;
int tb_idx;
if (!skb_frag_size(frag))
continue;
tb_phys = skb_frag_dma_map(trans->dev, frag, 0,
skb_frag_size(frag), DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(trans->dev, tb_phys)))
return -ENOMEM;
tb_idx = iwl_pcie_gen2_set_tb(trans, tfd, tb_phys,
skb_frag_size(frag));
trace_iwlwifi_dev_tx_tb(trans->dev, skb,
skb_frag_address(frag),
skb_frag_size(frag));
if (tb_idx < 0)
return tb_idx;
out_meta->tbs |= BIT(tb_idx);
}
return 0;
}
/*
* Determine the address of data in @skb.
*
* Note that @skb is not expected to have
* SKB fragments without page fragments.
*/
static inline void *
ss_skb_data_address(struct sk_buff *skb)
{
if (skb == NULL)
return NULL;
if (skb_headlen(skb))
return skb->data;
BUG_ON(!skb_is_nonlinear(skb));
BUG_ON(!skb_shinfo(skb)->nr_frags);
return skb_frag_address(&skb_shinfo(skb)->frags[0]);
}
static void greth_print_tx_packet(struct sk_buff *skb)
{
int i;
int length;
if (skb_shinfo(skb)->nr_frags == 0)
length = skb->len;
else
length = skb_headlen(skb);
print_hex_dump(KERN_DEBUG, "TX: ", DUMP_PREFIX_OFFSET, 16, 1,
skb->data, length, true);
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
print_hex_dump(KERN_DEBUG, "TX: ", DUMP_PREFIX_OFFSET, 16, 1,
skb_frag_address(&skb_shinfo(skb)->frags[i]),
skb_shinfo(skb)->frags[i].size, true);
}
}
/**
* Fragment @skb to add some room if @len > 0 or delete data otherwise.
*/
static int
__skb_fragment(struct sk_buff *skb, struct sk_buff *pskb,
char *pspt, int len, TfwStr *it)
{
int i, ret;
long offset;
unsigned int d_size;
struct sk_buff *f_skb, **next_fdp;
SS_DBG("[%d]: %s: in: len [%d] pspt [%p], skb [%p]: head [%p]"
" data [%p] tail [%p] end [%p] len [%u] data_len [%u]"
" truesize [%u] nr_frags [%u]\n",
smp_processor_id(), __func__, len, pspt, skb, skb->head,
skb->data, skb_tail_pointer(skb), skb_end_pointer(skb),
skb->len, skb->data_len, skb->truesize,
skb_shinfo(skb)->nr_frags);
BUG_ON(!len);
if (abs(len) > PAGE_SIZE) {
SS_WARN("Attempt to add or delete too much data: %u\n", len);
return -EINVAL;
}
/*
* Use @it to hold the return values from __split_pgfrag()
* and __split_linear_data(). @it->ptr, @it->skb, and
* @it->flags may be set to actual values. If a new SKB is
* allocated, then it is stored in @it->skb. @it->ptr holds
* the pointer either to data after the deleted data, or to
* the area for new data. @it->flags is set when @it->ptr
* points to data in @it->skb. Otherwise, @it->ptr points
* to data in @skb.
*
* Determine where the split begins within the SKB, then do
* the job using the right function.
*/
/* See if the split starts in the linear data. */
d_size = skb_headlen(skb);
offset = pspt - (char *)skb->data;
if ((offset >= 0) && (offset < d_size)) {
int t_size = d_size - offset;
len = max(len, -t_size);
ret = __split_linear_data(skb, pspt, len, it);
goto done;
}
/* See if the split starts in the page fragments data. */
for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
d_size = skb_frag_size(frag);
offset = pspt - (char *)skb_frag_address(frag);
if ((offset >= 0) && (offset < d_size)) {
int t_size = d_size - offset;
len = max(len, -t_size);
ret = __split_pgfrag(skb, i, offset, len, it);
goto done;
}
}
/* See if the split starts in the SKB fragments data. */
skb_walk_frags(skb, f_skb) {
ret = __skb_fragment(f_skb, skb, pspt, len, it);
if (ret != -ENOENT)
return ret;
}
/**
* Delete @len (the value is positive now) bytes from @frag.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to data after the deleted area in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
struct skb_shared_info *si = skb_shinfo(skb);
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
if (unlikely(off + len > skb_frag_size(frag))) {
SS_WARN("Attempt to delete too much\n");
return -EFAULT;
}
/* Fast path: delete a full fragment. */
if (!off && len == skb_frag_size(frag)) {
ss_skb_adjust_data_len(skb, -len);
__skb_frag_unref(frag);
if (i + 1 < si->nr_frags)
memmove(&si->frags[i], &si->frags[i + 1],
(si->nr_frags - i - 1) * sizeof(skb_frag_t));
--si->nr_frags;
goto lookup_next_ptr;
}
/* Fast path: delete the head part of a fragment. */
if (!off) {
frag->page_offset += len;
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
it->ptr = skb_frag_address(frag);
return 0;
}
/* Fast path: delete the tail part of a fragment. */
if (off + len == skb_frag_size(frag)) {
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
++i;
goto lookup_next_ptr;
}
/*
* Delete data in the middle of a fragment. After the data
* is deleted the fragment will contain only the head part,
* and the tail part is moved to another fragment.
* [frag @i] [frag @i+1 - tail data]
*
* Make room for a fragment right after the @i fragment
* to move the tail part of data there.
*/
if (__extend_pgfrags(skb, i + 1, 1, it))
return -EFAULT;
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off - len;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, len + tail_len);
/* Make the fragment with the tail part. */
i = (i + 1) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off + len, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
ss_skb_adjust_data_len(skb, -len);
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address of data after the deleted area. */
it->flags = (skb != skb_dst);
it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]);
return 0;
lookup_next_ptr:
/* Get the next fragment after the deleted fragment. */
if (i < si->nr_frags)
it->ptr = skb_frag_address(&si->frags[i]);
return 0;
}
/**
* Get room for @len bytes of data starting from offset @off
* in fragment @i.
*
* The room may be found in the preceding fragment if @off is zero.
* Otherwise, a new fragment is allocated and fragments around the
* fragment @i are rearranged so that data is not actually split
* and copied.
*
* Note: @off is always within the borders of fragment @i. It can
* point at the start of a fragment, but it can never point at the
* location right after the end of a fragment. In other words, @off
* can be zero, but it can not be equal to the size of fragment @i.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to the room for new data in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i];
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
/*
* If @off is zero and there's a preceding page fragment,
* then try to append data to that fragment. Go for other
* solutions if there's no room.
*/
if (!off && i) {
frag_dst = __check_frag_room(skb, frag - 1, len);
if (frag_dst) {
/* Coalesce new data with the fragment. */
off = skb_frag_size(frag_dst);
skb_frag_size_add(frag_dst, len);
ss_skb_adjust_data_len(skb, len);
it->ptr = (char *)skb_frag_address(frag_dst) + off;
return 0;
}
}
/*
* Make a fragment that can hold @len bytes. If @off is
* zero, then data is added at the start of fragment @i.
* Make a fragment in slot @i, and the original fragment
* is shifted forward. If @off is not zero, then make
* a fragment in slot @i+1, and make an extra fragment
* in slot @i+2 to hold the tail data.
*/
if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it))
return -EFAULT;
/* If @off is zero, the job is done in __new_pgfrag(). */
if (!off) {
it->ptr = skb_frag_address(frag);
return 0;
}
/*
* If data is added in the middle of a fragment, then split
* the fragment. The head of the fragment stays there, and
* the tail of the fragment is moved to a new fragment.
* The fragment for new data is placed in between.
* [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data]
* If @i is close to MAX_SKB_FRAGS, then new fragments may
* be located in another SKB.
*/
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, tail_len);
/* Make the fragment with the tail part. */
i = (i + 2) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address for new data. */
it->flags = !(i < MAX_SKB_FRAGS - 1);
frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1;
it->ptr = skb_frag_address(frag_dst);
return 0;
}
/**
* The kernel may allocate a bit more memory for an SKB than what was
* requested (see ksize() call in __alloc_skb()). Use the extra memory
* if it's enough to hold @n bytes. Otherwise, allocate new linear data.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to the room for new data in @it->ptr if making room.
* @return pointer to data right after the deleted fragment in @it->ptr.
*/
static int
__split_linear_data(struct sk_buff *skb, char *pspt, int len, TfwStr *it)
{
int alloc = len > 0;
int tail_len = (char *)skb_tail_pointer(skb) - pspt;
struct page *page = virt_to_head_page(skb->head);
SS_DBG("[%d]: %s: skb [%p] pspt [%p] len [%d] tail_len [%d]\n",
smp_processor_id(), __func__, skb, pspt, len, tail_len);
BUG_ON(!skb->head_frag);
BUG_ON(tail_len <= 0);
BUG_ON(!(alloc | tail_len));
BUG_ON(-len > tail_len);
/*
* Quick and unlikely path: just advance the skb tail pointer.
* Note that this only works when we make room. When we remove,
* pspt points at the start of the data chunk to remove. In that
* case, tail_len can never be zero.
*/
if (unlikely(!tail_len && len <= ss_skb_tailroom(skb))) {
BUG_ON(len < 0);
it->ptr = ss_skb_put(skb, len);
return 0;
}
/*
* Quick and unlikely path: just move skb tail pointer backward.
* Note that this only works when we remove data, and the data
* is located exactly at the end of the linear part of an skb.
*/
if (unlikely((len < 0) && (tail_len == -len))) {
ss_skb_put(skb, len);
if (skb_is_nonlinear(skb))
it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]);
return 0;
}
/*
* Data is inserted or deleted in the middle of the linear part,
* or there's insufficient room in the linear part of an SKB to
* insert @len bytes.
*
* Don't bother with skb tail room: if the linear part is large,
* then it's likely that we'll do some smaller data insertions
* later and go by the quick path above. Otherwise, the tail size
* is also small.
*
* The inserted data is placed in a fragment. The tail part is
* moved to yet another fragment. The linear part is trimmed to
* exclude the deleted data and the tail part.
*
* Do all allocations before moving the fragments to avoid complex
* rollback.
*/
if (alloc) {
if (__new_pgfrag(skb, len, 0, alloc + !!tail_len, it))
return -EFAULT;
} else {
if (__extend_pgfrags(skb, 0, 1, it))
return -EFAULT;
tail_len += len; /* @len is negative. */
}
if (tail_len) {
int tail_off = pspt - (char *)page_address(page);
/*
* Trim the linear part by |@len| bytes if data
* is deleted. Then trim it further to exclude
* the tail data. Finally, set up the fragment
* allotted above with the tail data.
*/
if (len < 0) {
tail_off -= len;
skb->tail += len;
skb->len += len;
}
skb->tail -= tail_len;
skb->data_len += tail_len;
skb->truesize += tail_len;
__skb_fill_page_desc(skb, alloc, page, tail_off, tail_len);
skb_frag_ref(skb, alloc); /* get_page(page); */
}
it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]);
return 0;
}
static struct sk_buff *__lro_proc_segment(struct net_lro_mgr *lro_mgr,
struct skb_frag_struct *frags,
int len, int true_size,
void *priv, __wsum sum)
{
struct net_lro_desc *lro_desc;
struct iphdr *iph;
struct tcphdr *tcph;
struct sk_buff *skb;
u64 flags;
void *mac_hdr;
int mac_hdr_len;
int hdr_len = LRO_MAX_PG_HLEN;
int vlan_hdr_len = 0;
if (!lro_mgr->get_frag_header ||
lro_mgr->get_frag_header(frags, (void *)&mac_hdr, (void *)&iph,
(void *)&tcph, &flags, priv)) {
mac_hdr = skb_frag_address(frags);
goto out1;
}
if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
goto out1;
hdr_len = (int)((void *)(tcph) + TCP_HDR_LEN(tcph) - mac_hdr);
mac_hdr_len = (int)((void *)(iph) - mac_hdr);
lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
if (!lro_desc)
goto out1;
if (!lro_desc->active) { /* start new lro session */
if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, NULL))
goto out1;
skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
hdr_len, 0, lro_mgr->ip_summed_aggr);
if (!skb)
goto out;
if ((skb->protocol == htons(ETH_P_8021Q)) &&
!(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
vlan_hdr_len = VLAN_HLEN;
iph = (void *)(skb->data + vlan_hdr_len);
tcph = (void *)((u8 *)skb->data + vlan_hdr_len
+ IP_HDR_LEN(iph));
lro_init_desc(lro_desc, skb, iph, tcph);
LRO_INC_STATS(lro_mgr, aggregated);
return NULL;
}
if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
goto out2;
if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, lro_desc))
goto out2;
lro_add_frags(lro_desc, len, hdr_len, true_size, frags, iph, tcph);
LRO_INC_STATS(lro_mgr, aggregated);
if ((skb_shinfo(lro_desc->parent)->nr_frags >= lro_mgr->max_aggr) ||
lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
lro_flush(lro_mgr, lro_desc);
return NULL;
out2: /* send aggregated packets to the stack */
lro_flush(lro_mgr, lro_desc);
out1: /* Original packet has to be posted to the stack */
skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
hdr_len, sum, lro_mgr->ip_summed);
out:
return skb;
}
static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
{
struct ipcomp_data *ipcd = x->data;
const int plen = skb->len;
int dlen = IPCOMP_SCRATCH_SIZE;
const u8 *start = skb->data;
const int cpu = get_cpu();
u8 *scratch = *per_cpu_ptr(ipcomp_scratches, cpu);
struct crypto_comp *tfm = *per_cpu_ptr(ipcd->tfms, cpu);
int err = crypto_comp_decompress(tfm, start, plen, scratch, &dlen);
int len;
if (err)
goto out;
if (dlen < (plen + sizeof(struct ip_comp_hdr))) {
err = -EINVAL;
goto out;
}
len = dlen - plen;
if (len > skb_tailroom(skb))
len = skb_tailroom(skb);
__skb_put(skb, len);
len += plen;
skb_copy_to_linear_data(skb, scratch, len);
while ((scratch += len, dlen -= len) > 0) {
skb_frag_t *frag;
struct page *page;
err = -EMSGSIZE;
if (WARN_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS))
goto out;
frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
page = alloc_page(GFP_ATOMIC);
err = -ENOMEM;
if (!page)
goto out;
__skb_frag_set_page(frag, page);
len = PAGE_SIZE;
if (dlen < len)
len = dlen;
frag->page_offset = 0;
skb_frag_size_set(frag, len);
memcpy(skb_frag_address(frag), scratch, len);
skb->truesize += len;
skb->data_len += len;
skb->len += len;
skb_shinfo(skb)->nr_frags++;
}
err = 0;
out:
put_cpu();
return err;
}
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
int offset, int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6,
struct rt6_info *rt, unsigned int flags, int dontfrag)
{
struct inet_sock *inet = inet_sk(sk);
struct ipv6_pinfo *np = inet6_sk(sk);
struct inet_cork *cork;
struct sk_buff *skb, *skb_prev = NULL;
unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu;
int exthdrlen;
int dst_exthdrlen;
int hh_len;
int copy;
int err;
int offset = 0;
int csummode = CHECKSUM_NONE;
__u8 tx_flags = 0;
if (flags&MSG_PROBE)
return 0;
cork = &inet->cork.base;
if (skb_queue_empty(&sk->sk_write_queue)) {
/*
* setup for corking
*/
if (opt) {
if (WARN_ON(np->cork.opt))
return -EINVAL;
np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation);
if (unlikely(np->cork.opt == NULL))
return -ENOBUFS;
np->cork.opt->tot_len = opt->tot_len;
np->cork.opt->opt_flen = opt->opt_flen;
np->cork.opt->opt_nflen = opt->opt_nflen;
np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt,
sk->sk_allocation);
if (opt->dst0opt && !np->cork.opt->dst0opt)
return -ENOBUFS;
np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt,
sk->sk_allocation);
if (opt->dst1opt && !np->cork.opt->dst1opt)
return -ENOBUFS;
np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt,
sk->sk_allocation);
if (opt->hopopt && !np->cork.opt->hopopt)
return -ENOBUFS;
np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt,
sk->sk_allocation);
if (opt->srcrt && !np->cork.opt->srcrt)
return -ENOBUFS;
/* need source address above miyazawa*/
}
dst_hold(&rt->dst);
cork->dst = &rt->dst;
inet->cork.fl.u.ip6 = *fl6;
np->cork.hop_limit = hlimit;
np->cork.tclass = tclass;
if (rt->dst.flags & DST_XFRM_TUNNEL)
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(&rt->dst);
else
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(rt->dst.path);
if (np->frag_size < mtu) {
if (np->frag_size)
mtu = np->frag_size;
}
cork->fragsize = mtu;
if (dst_allfrag(rt->dst.path))
cork->flags |= IPCORK_ALLFRAG;
cork->length = 0;
sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
exthdrlen = (opt ? opt->opt_flen : 0);
length += exthdrlen;
transhdrlen += exthdrlen;
dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
} else {
rt = (struct rt6_info *)cork->dst;
fl6 = &inet->cork.fl.u.ip6;
opt = np->cork.opt;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
mtu = cork->fragsize;
}
orig_mtu = mtu;
hh_len = LL_RESERVED_SPACE(rt->dst.dev);
fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
(opt ? opt->opt_nflen : 0);
maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
}
/* For UDP, check if TX timestamp is enabled */
if (sk->sk_type == SOCK_DGRAM) {
err = sock_tx_timestamp(sk, &tx_flags);
if (err)
goto error;
}
/*
* Let's try using as much space as possible.
* Use MTU if total length of the message fits into the MTU.
* Otherwise, we need to reserve fragment header and
* fragment alignment (= 8-15 octects, in total).
*
* Note that we may need to "move" the data from the tail of
* of the buffer to the new fragment when we split
* the message.
*
* FIXME: It may be fragmented into multiple chunks
* at once if non-fragmentable extension headers
* are too large.
* --yoshfuji
*/
if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP ||
sk->sk_protocol == IPPROTO_RAW)) {
ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
skb = skb_peek_tail(&sk->sk_write_queue);
cork->length += length;
if (((length > mtu) ||
(skb && skb_has_frags(skb))) &&
(sk->sk_protocol == IPPROTO_UDP) &&
(rt->dst.dev->features & NETIF_F_UFO) &&
(sk->sk_type == SOCK_DGRAM)) {
err = ip6_ufo_append_data(sk, getfrag, from, length,
hh_len, fragheaderlen,
transhdrlen, mtu, flags, rt);
if (err)
goto error;
return 0;
}
if (!skb)
goto alloc_new_skb;
while (length > 0) {
/* Check if the remaining data fits into current packet. */
copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
if (copy < length)
copy = maxfraglen - skb->len;
if (copy <= 0) {
char *data;
unsigned int datalen;
unsigned int fraglen;
unsigned int fraggap;
unsigned int alloclen;
alloc_new_skb:
/* There's no room in the current skb */
if (skb)
fraggap = skb->len - maxfraglen;
else
fraggap = 0;
/* update mtu and maxfraglen if necessary */
if (skb == NULL || skb_prev == NULL)
ip6_append_data_mtu(&mtu, &maxfraglen,
fragheaderlen, skb, rt,
orig_mtu);
skb_prev = skb;
/*
* If remaining data exceeds the mtu,
* we know we need more fragment(s).
*/
datalen = length + fraggap;
if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
if ((flags & MSG_MORE) &&
!(rt->dst.dev->features&NETIF_F_SG))
alloclen = mtu;
else
alloclen = datalen + fragheaderlen;
alloclen += dst_exthdrlen;
if (datalen != length + fraggap) {
/*
* this is not the last fragment, the trailer
* space is regarded as data space.
*/
datalen += rt->dst.trailer_len;
}
alloclen += rt->dst.trailer_len;
fraglen = datalen + fragheaderlen;
/*
* We just reserve space for fragment header.
* Note: this may be overallocation if the message
* (without MSG_MORE) fits into the MTU.
*/
alloclen += sizeof(struct frag_hdr);
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len,
(flags & MSG_DONTWAIT), &err);
} else {
skb = NULL;
if (atomic_read(&sk->sk_wmem_alloc) <=
2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len, 1,
sk->sk_allocation);
if (unlikely(skb == NULL))
err = -ENOBUFS;
else {
/* Only the initial fragment
* is time stamped.
*/
tx_flags = 0;
}
}
if (skb == NULL)
goto error;
/*
* Fill in the control structures
*/
skb->ip_summed = csummode;
skb->csum = 0;
/* reserve for fragmentation and ipsec header */
skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
dst_exthdrlen);
if (sk->sk_type == SOCK_DGRAM)
skb_shinfo(skb)->tx_flags = tx_flags;
/*
* Find where to start putting bytes
*/
data = skb_put(skb, fraglen);
skb_set_network_header(skb, exthdrlen);
data += fragheaderlen;
skb->transport_header = (skb->network_header +
fragheaderlen);
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
data + transhdrlen, fraggap, 0);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
pskb_trim_unique(skb_prev, maxfraglen);
}
copy = datalen - transhdrlen - fraggap;
if (copy < 0) {
err = -EINVAL;
kfree_skb(skb);
goto error;
} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
err = -EFAULT;
kfree_skb(skb);
goto error;
}
offset += copy;
length -= datalen - fraggap;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
csummode = CHECKSUM_NONE;
/*
* Put the packet on the pending queue
*/
__skb_queue_tail(&sk->sk_write_queue, skb);
continue;
}
if (copy > length)
copy = length;
if (!(rt->dst.dev->features&NETIF_F_SG)) {
unsigned int off;
off = skb->len;
if (getfrag(from, skb_put(skb, copy),
offset, copy, off, skb) < 0) {
__skb_trim(skb, off);
err = -EFAULT;
goto error;
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = sk->sk_sndmsg_page;
int off = sk->sk_sndmsg_off;
unsigned int left;
if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != skb_frag_page(frag)) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
skb_frag_ref(skb, i);
frag = &skb_shinfo(skb)->frags[i];
}
} else if(i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->sk_allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
sk->sk_sndmsg_page = page;
sk->sk_sndmsg_off = 0;
skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from,
skb_frag_address(frag) + skb_frag_size(frag),
offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
goto error;
}
sk->sk_sndmsg_off += copy;
skb_frag_size_add(frag, copy);
skb->len += copy;
skb->data_len += copy;
skb->truesize += copy;
atomic_add(copy, &sk->sk_wmem_alloc);
}
offset += copy;
length -= copy;
}
return 0;
error:
cork->length -= length;
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
return err;
}
