static int iwl_pcie_gen2_tx_add_frags(struct iwl_trans *trans, struct sk_buff *skb, struct iwl_tfh_tfd *tfd, struct iwl_cmd_meta *out_meta) { int i; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; dma_addr_t tb_phys; int tb_idx; if (!skb_frag_size(frag)) continue; tb_phys = skb_frag_dma_map(trans->dev, frag, 0, skb_frag_size(frag), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(trans->dev, tb_phys))) return -ENOMEM; tb_idx = iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, skb_frag_size(frag)); trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag), skb_frag_size(frag)); if (tb_idx < 0) return tb_idx; out_meta->tbs |= BIT(tb_idx); } return 0; }
/* * Determine the address of data in @skb. * * Note that @skb is not expected to have * SKB fragments without page fragments. */ static inline void * ss_skb_data_address(struct sk_buff *skb) { if (skb == NULL) return NULL; if (skb_headlen(skb)) return skb->data; BUG_ON(!skb_is_nonlinear(skb)); BUG_ON(!skb_shinfo(skb)->nr_frags); return skb_frag_address(&skb_shinfo(skb)->frags[0]); }
static void greth_print_tx_packet(struct sk_buff *skb) { int i; int length; if (skb_shinfo(skb)->nr_frags == 0) length = skb->len; else length = skb_headlen(skb); print_hex_dump(KERN_DEBUG, "TX: ", DUMP_PREFIX_OFFSET, 16, 1, skb->data, length, true); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { print_hex_dump(KERN_DEBUG, "TX: ", DUMP_PREFIX_OFFSET, 16, 1, skb_frag_address(&skb_shinfo(skb)->frags[i]), skb_shinfo(skb)->frags[i].size, true); } }
/** * Fragment @skb to add some room if @len > 0 or delete data otherwise. */ static int __skb_fragment(struct sk_buff *skb, struct sk_buff *pskb, char *pspt, int len, TfwStr *it) { int i, ret; long offset; unsigned int d_size; struct sk_buff *f_skb, **next_fdp; SS_DBG("[%d]: %s: in: len [%d] pspt [%p], skb [%p]: head [%p]" " data [%p] tail [%p] end [%p] len [%u] data_len [%u]" " truesize [%u] nr_frags [%u]\n", smp_processor_id(), __func__, len, pspt, skb, skb->head, skb->data, skb_tail_pointer(skb), skb_end_pointer(skb), skb->len, skb->data_len, skb->truesize, skb_shinfo(skb)->nr_frags); BUG_ON(!len); if (abs(len) > PAGE_SIZE) { SS_WARN("Attempt to add or delete too much data: %u\n", len); return -EINVAL; } /* * Use @it to hold the return values from __split_pgfrag() * and __split_linear_data(). @it->ptr, @it->skb, and * @it->flags may be set to actual values. If a new SKB is * allocated, then it is stored in @it->skb. @it->ptr holds * the pointer either to data after the deleted data, or to * the area for new data. @it->flags is set when @it->ptr * points to data in @it->skb. Otherwise, @it->ptr points * to data in @skb. * * Determine where the split begins within the SKB, then do * the job using the right function. */ /* See if the split starts in the linear data. */ d_size = skb_headlen(skb); offset = pspt - (char *)skb->data; if ((offset >= 0) && (offset < d_size)) { int t_size = d_size - offset; len = max(len, -t_size); ret = __split_linear_data(skb, pspt, len, it); goto done; } /* See if the split starts in the page fragments data. */ for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; d_size = skb_frag_size(frag); offset = pspt - (char *)skb_frag_address(frag); if ((offset >= 0) && (offset < d_size)) { int t_size = d_size - offset; len = max(len, -t_size); ret = __split_pgfrag(skb, i, offset, len, it); goto done; } } /* See if the split starts in the SKB fragments data. */ skb_walk_frags(skb, f_skb) { ret = __skb_fragment(f_skb, skb, pspt, len, it); if (ret != -ENOENT) return ret; }
/** * Delete @len (the value is positive now) bytes from @frag. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to data after the deleted area in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct skb_shared_info *si = skb_shinfo(skb); SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); if (unlikely(off + len > skb_frag_size(frag))) { SS_WARN("Attempt to delete too much\n"); return -EFAULT; } /* Fast path: delete a full fragment. */ if (!off && len == skb_frag_size(frag)) { ss_skb_adjust_data_len(skb, -len); __skb_frag_unref(frag); if (i + 1 < si->nr_frags) memmove(&si->frags[i], &si->frags[i + 1], (si->nr_frags - i - 1) * sizeof(skb_frag_t)); --si->nr_frags; goto lookup_next_ptr; } /* Fast path: delete the head part of a fragment. */ if (!off) { frag->page_offset += len; skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); it->ptr = skb_frag_address(frag); return 0; } /* Fast path: delete the tail part of a fragment. */ if (off + len == skb_frag_size(frag)) { skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); ++i; goto lookup_next_ptr; } /* * Delete data in the middle of a fragment. After the data * is deleted the fragment will contain only the head part, * and the tail part is moved to another fragment. * [frag @i] [frag @i+1 - tail data] * * Make room for a fragment right after the @i fragment * to move the tail part of data there. */ if (__extend_pgfrags(skb, i + 1, 1, it)) return -EFAULT; /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off - len; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, len + tail_len); /* Make the fragment with the tail part. */ i = (i + 1) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off + len, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ ss_skb_adjust_data_len(skb, -len); if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address of data after the deleted area. */ it->flags = (skb != skb_dst); it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]); return 0; lookup_next_ptr: /* Get the next fragment after the deleted fragment. */ if (i < si->nr_frags) it->ptr = skb_frag_address(&si->frags[i]); return 0; }
/** * Get room for @len bytes of data starting from offset @off * in fragment @i. * * The room may be found in the preceding fragment if @off is zero. * Otherwise, a new fragment is allocated and fragments around the * fragment @i are rearranged so that data is not actually split * and copied. * * Note: @off is always within the borders of fragment @i. It can * point at the start of a fragment, but it can never point at the * location right after the end of a fragment. In other words, @off * can be zero, but it can not be equal to the size of fragment @i. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to the room for new data in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i]; SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); /* * If @off is zero and there's a preceding page fragment, * then try to append data to that fragment. Go for other * solutions if there's no room. */ if (!off && i) { frag_dst = __check_frag_room(skb, frag - 1, len); if (frag_dst) { /* Coalesce new data with the fragment. */ off = skb_frag_size(frag_dst); skb_frag_size_add(frag_dst, len); ss_skb_adjust_data_len(skb, len); it->ptr = (char *)skb_frag_address(frag_dst) + off; return 0; } } /* * Make a fragment that can hold @len bytes. If @off is * zero, then data is added at the start of fragment @i. * Make a fragment in slot @i, and the original fragment * is shifted forward. If @off is not zero, then make * a fragment in slot @i+1, and make an extra fragment * in slot @i+2 to hold the tail data. */ if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it)) return -EFAULT; /* If @off is zero, the job is done in __new_pgfrag(). */ if (!off) { it->ptr = skb_frag_address(frag); return 0; } /* * If data is added in the middle of a fragment, then split * the fragment. The head of the fragment stays there, and * the tail of the fragment is moved to a new fragment. * The fragment for new data is placed in between. * [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data] * If @i is close to MAX_SKB_FRAGS, then new fragments may * be located in another SKB. */ /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, tail_len); /* Make the fragment with the tail part. */ i = (i + 2) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address for new data. */ it->flags = !(i < MAX_SKB_FRAGS - 1); frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1; it->ptr = skb_frag_address(frag_dst); return 0; }
/** * The kernel may allocate a bit more memory for an SKB than what was * requested (see ksize() call in __alloc_skb()). Use the extra memory * if it's enough to hold @n bytes. Otherwise, allocate new linear data. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to the room for new data in @it->ptr if making room. * @return pointer to data right after the deleted fragment in @it->ptr. */ static int __split_linear_data(struct sk_buff *skb, char *pspt, int len, TfwStr *it) { int alloc = len > 0; int tail_len = (char *)skb_tail_pointer(skb) - pspt; struct page *page = virt_to_head_page(skb->head); SS_DBG("[%d]: %s: skb [%p] pspt [%p] len [%d] tail_len [%d]\n", smp_processor_id(), __func__, skb, pspt, len, tail_len); BUG_ON(!skb->head_frag); BUG_ON(tail_len <= 0); BUG_ON(!(alloc | tail_len)); BUG_ON(-len > tail_len); /* * Quick and unlikely path: just advance the skb tail pointer. * Note that this only works when we make room. When we remove, * pspt points at the start of the data chunk to remove. In that * case, tail_len can never be zero. */ if (unlikely(!tail_len && len <= ss_skb_tailroom(skb))) { BUG_ON(len < 0); it->ptr = ss_skb_put(skb, len); return 0; } /* * Quick and unlikely path: just move skb tail pointer backward. * Note that this only works when we remove data, and the data * is located exactly at the end of the linear part of an skb. */ if (unlikely((len < 0) && (tail_len == -len))) { ss_skb_put(skb, len); if (skb_is_nonlinear(skb)) it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]); return 0; } /* * Data is inserted or deleted in the middle of the linear part, * or there's insufficient room in the linear part of an SKB to * insert @len bytes. * * Don't bother with skb tail room: if the linear part is large, * then it's likely that we'll do some smaller data insertions * later and go by the quick path above. Otherwise, the tail size * is also small. * * The inserted data is placed in a fragment. The tail part is * moved to yet another fragment. The linear part is trimmed to * exclude the deleted data and the tail part. * * Do all allocations before moving the fragments to avoid complex * rollback. */ if (alloc) { if (__new_pgfrag(skb, len, 0, alloc + !!tail_len, it)) return -EFAULT; } else { if (__extend_pgfrags(skb, 0, 1, it)) return -EFAULT; tail_len += len; /* @len is negative. */ } if (tail_len) { int tail_off = pspt - (char *)page_address(page); /* * Trim the linear part by |@len| bytes if data * is deleted. Then trim it further to exclude * the tail data. Finally, set up the fragment * allotted above with the tail data. */ if (len < 0) { tail_off -= len; skb->tail += len; skb->len += len; } skb->tail -= tail_len; skb->data_len += tail_len; skb->truesize += tail_len; __skb_fill_page_desc(skb, alloc, page, tail_off, tail_len); skb_frag_ref(skb, alloc); /* get_page(page); */ } it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]); return 0; }
static struct sk_buff *__lro_proc_segment(struct net_lro_mgr *lro_mgr, struct skb_frag_struct *frags, int len, int true_size, void *priv, __wsum sum) { struct net_lro_desc *lro_desc; struct iphdr *iph; struct tcphdr *tcph; struct sk_buff *skb; u64 flags; void *mac_hdr; int mac_hdr_len; int hdr_len = LRO_MAX_PG_HLEN; int vlan_hdr_len = 0; if (!lro_mgr->get_frag_header || lro_mgr->get_frag_header(frags, (void *)&mac_hdr, (void *)&iph, (void *)&tcph, &flags, priv)) { mac_hdr = skb_frag_address(frags); goto out1; } if (!(flags & LRO_IPV4) || !(flags & LRO_TCP)) goto out1; hdr_len = (int)((void *)(tcph) + TCP_HDR_LEN(tcph) - mac_hdr); mac_hdr_len = (int)((void *)(iph) - mac_hdr); lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph); if (!lro_desc) goto out1; if (!lro_desc->active) { /* start new lro session */ if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, NULL)) goto out1; skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr, hdr_len, 0, lro_mgr->ip_summed_aggr); if (!skb) goto out; if ((skb->protocol == htons(ETH_P_8021Q)) && !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID)) vlan_hdr_len = VLAN_HLEN; iph = (void *)(skb->data + vlan_hdr_len); tcph = (void *)((u8 *)skb->data + vlan_hdr_len + IP_HDR_LEN(iph)); lro_init_desc(lro_desc, skb, iph, tcph); LRO_INC_STATS(lro_mgr, aggregated); return NULL; } if (lro_desc->tcp_next_seq != ntohl(tcph->seq)) goto out2; if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, lro_desc)) goto out2; lro_add_frags(lro_desc, len, hdr_len, true_size, frags, iph, tcph); LRO_INC_STATS(lro_mgr, aggregated); if ((skb_shinfo(lro_desc->parent)->nr_frags >= lro_mgr->max_aggr) || lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu)) lro_flush(lro_mgr, lro_desc); return NULL; out2: /* send aggregated packets to the stack */ lro_flush(lro_mgr, lro_desc); out1: /* Original packet has to be posted to the stack */ skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr, hdr_len, sum, lro_mgr->ip_summed); out: return skb; }
static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb) { struct ipcomp_data *ipcd = x->data; const int plen = skb->len; int dlen = IPCOMP_SCRATCH_SIZE; const u8 *start = skb->data; const int cpu = get_cpu(); u8 *scratch = *per_cpu_ptr(ipcomp_scratches, cpu); struct crypto_comp *tfm = *per_cpu_ptr(ipcd->tfms, cpu); int err = crypto_comp_decompress(tfm, start, plen, scratch, &dlen); int len; if (err) goto out; if (dlen < (plen + sizeof(struct ip_comp_hdr))) { err = -EINVAL; goto out; } len = dlen - plen; if (len > skb_tailroom(skb)) len = skb_tailroom(skb); __skb_put(skb, len); len += plen; skb_copy_to_linear_data(skb, scratch, len); while ((scratch += len, dlen -= len) > 0) { skb_frag_t *frag; struct page *page; err = -EMSGSIZE; if (WARN_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) goto out; frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; page = alloc_page(GFP_ATOMIC); err = -ENOMEM; if (!page) goto out; __skb_frag_set_page(frag, page); len = PAGE_SIZE; if (dlen < len) len = dlen; frag->page_offset = 0; skb_frag_size_set(frag, len); memcpy(skb_frag_address(frag), scratch, len); skb->truesize += len; skb->data_len += len; skb->len += len; skb_shinfo(skb)->nr_frags++; } err = 0; out: put_cpu(); return err; }
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), void *from, int length, int transhdrlen, int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6, struct rt6_info *rt, unsigned int flags, int dontfrag) { struct inet_sock *inet = inet_sk(sk); struct ipv6_pinfo *np = inet6_sk(sk); struct inet_cork *cork; struct sk_buff *skb, *skb_prev = NULL; unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu; int exthdrlen; int dst_exthdrlen; int hh_len; int copy; int err; int offset = 0; int csummode = CHECKSUM_NONE; __u8 tx_flags = 0; if (flags&MSG_PROBE) return 0; cork = &inet->cork.base; if (skb_queue_empty(&sk->sk_write_queue)) { /* * setup for corking */ if (opt) { if (WARN_ON(np->cork.opt)) return -EINVAL; np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation); if (unlikely(np->cork.opt == NULL)) return -ENOBUFS; np->cork.opt->tot_len = opt->tot_len; np->cork.opt->opt_flen = opt->opt_flen; np->cork.opt->opt_nflen = opt->opt_nflen; np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation); if (opt->dst0opt && !np->cork.opt->dst0opt) return -ENOBUFS; np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation); if (opt->dst1opt && !np->cork.opt->dst1opt) return -ENOBUFS; np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation); if (opt->hopopt && !np->cork.opt->hopopt) return -ENOBUFS; np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation); if (opt->srcrt && !np->cork.opt->srcrt) return -ENOBUFS; /* need source address above miyazawa*/ } dst_hold(&rt->dst); cork->dst = &rt->dst; inet->cork.fl.u.ip6 = *fl6; np->cork.hop_limit = hlimit; np->cork.tclass = tclass; if (rt->dst.flags & DST_XFRM_TUNNEL) mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ? rt->dst.dev->mtu : dst_mtu(&rt->dst); else mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ? rt->dst.dev->mtu : dst_mtu(rt->dst.path); if (np->frag_size < mtu) { if (np->frag_size) mtu = np->frag_size; } cork->fragsize = mtu; if (dst_allfrag(rt->dst.path)) cork->flags |= IPCORK_ALLFRAG; cork->length = 0; sk->sk_sndmsg_page = NULL; sk->sk_sndmsg_off = 0; exthdrlen = (opt ? opt->opt_flen : 0); length += exthdrlen; transhdrlen += exthdrlen; dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; } else { rt = (struct rt6_info *)cork->dst; fl6 = &inet->cork.fl.u.ip6; opt = np->cork.opt; transhdrlen = 0; exthdrlen = 0; dst_exthdrlen = 0; mtu = cork->fragsize; } orig_mtu = mtu; hh_len = LL_RESERVED_SPACE(rt->dst.dev); fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + (opt ? opt->opt_nflen : 0); maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr); if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) { if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) { ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen); return -EMSGSIZE; } } /* For UDP, check if TX timestamp is enabled */ if (sk->sk_type == SOCK_DGRAM) { err = sock_tx_timestamp(sk, &tx_flags); if (err) goto error; } /* * Let's try using as much space as possible. * Use MTU if total length of the message fits into the MTU. * Otherwise, we need to reserve fragment header and * fragment alignment (= 8-15 octects, in total). * * Note that we may need to "move" the data from the tail of * of the buffer to the new fragment when we split * the message. * * FIXME: It may be fragmented into multiple chunks * at once if non-fragmentable extension headers * are too large. * --yoshfuji */ if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP || sk->sk_protocol == IPPROTO_RAW)) { ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen); return -EMSGSIZE; } skb = skb_peek_tail(&sk->sk_write_queue); cork->length += length; if (((length > mtu) || (skb && skb_has_frags(skb))) && (sk->sk_protocol == IPPROTO_UDP) && (rt->dst.dev->features & NETIF_F_UFO) && (sk->sk_type == SOCK_DGRAM)) { err = ip6_ufo_append_data(sk, getfrag, from, length, hh_len, fragheaderlen, transhdrlen, mtu, flags, rt); if (err) goto error; return 0; } if (!skb) goto alloc_new_skb; while (length > 0) { /* Check if the remaining data fits into current packet. */ copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; if (copy < length) copy = maxfraglen - skb->len; if (copy <= 0) { char *data; unsigned int datalen; unsigned int fraglen; unsigned int fraggap; unsigned int alloclen; alloc_new_skb: /* There's no room in the current skb */ if (skb) fraggap = skb->len - maxfraglen; else fraggap = 0; /* update mtu and maxfraglen if necessary */ if (skb == NULL || skb_prev == NULL) ip6_append_data_mtu(&mtu, &maxfraglen, fragheaderlen, skb, rt, orig_mtu); skb_prev = skb; /* * If remaining data exceeds the mtu, * we know we need more fragment(s). */ datalen = length + fraggap; if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; if ((flags & MSG_MORE) && !(rt->dst.dev->features&NETIF_F_SG)) alloclen = mtu; else alloclen = datalen + fragheaderlen; alloclen += dst_exthdrlen; if (datalen != length + fraggap) { /* * this is not the last fragment, the trailer * space is regarded as data space. */ datalen += rt->dst.trailer_len; } alloclen += rt->dst.trailer_len; fraglen = datalen + fragheaderlen; /* * We just reserve space for fragment header. * Note: this may be overallocation if the message * (without MSG_MORE) fits into the MTU. */ alloclen += sizeof(struct frag_hdr); if (transhdrlen) { skb = sock_alloc_send_skb(sk, alloclen + hh_len, (flags & MSG_DONTWAIT), &err); } else { skb = NULL; if (atomic_read(&sk->sk_wmem_alloc) <= 2 * sk->sk_sndbuf) skb = sock_wmalloc(sk, alloclen + hh_len, 1, sk->sk_allocation); if (unlikely(skb == NULL)) err = -ENOBUFS; else { /* Only the initial fragment * is time stamped. */ tx_flags = 0; } } if (skb == NULL) goto error; /* * Fill in the control structures */ skb->ip_summed = csummode; skb->csum = 0; /* reserve for fragmentation and ipsec header */ skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + dst_exthdrlen); if (sk->sk_type == SOCK_DGRAM) skb_shinfo(skb)->tx_flags = tx_flags; /* * Find where to start putting bytes */ data = skb_put(skb, fraglen); skb_set_network_header(skb, exthdrlen); data += fragheaderlen; skb->transport_header = (skb->network_header + fragheaderlen); if (fraggap) { skb->csum = skb_copy_and_csum_bits( skb_prev, maxfraglen, data + transhdrlen, fraggap, 0); skb_prev->csum = csum_sub(skb_prev->csum, skb->csum); data += fraggap; pskb_trim_unique(skb_prev, maxfraglen); } copy = datalen - transhdrlen - fraggap; if (copy < 0) { err = -EINVAL; kfree_skb(skb); goto error; } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { err = -EFAULT; kfree_skb(skb); goto error; } offset += copy; length -= datalen - fraggap; transhdrlen = 0; exthdrlen = 0; dst_exthdrlen = 0; csummode = CHECKSUM_NONE; /* * Put the packet on the pending queue */ __skb_queue_tail(&sk->sk_write_queue, skb); continue; } if (copy > length) copy = length; if (!(rt->dst.dev->features&NETIF_F_SG)) { unsigned int off; off = skb->len; if (getfrag(from, skb_put(skb, copy), offset, copy, off, skb) < 0) { __skb_trim(skb, off); err = -EFAULT; goto error; } } else { int i = skb_shinfo(skb)->nr_frags; skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1]; struct page *page = sk->sk_sndmsg_page; int off = sk->sk_sndmsg_off; unsigned int left; if (page && (left = PAGE_SIZE - off) > 0) { if (copy >= left) copy = left; if (page != skb_frag_page(frag)) { if (i == MAX_SKB_FRAGS) { err = -EMSGSIZE; goto error; } skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); skb_frag_ref(skb, i); frag = &skb_shinfo(skb)->frags[i]; } } else if(i < MAX_SKB_FRAGS) { if (copy > PAGE_SIZE) copy = PAGE_SIZE; page = alloc_pages(sk->sk_allocation, 0); if (page == NULL) { err = -ENOMEM; goto error; } sk->sk_sndmsg_page = page; sk->sk_sndmsg_off = 0; skb_fill_page_desc(skb, i, page, 0, 0); frag = &skb_shinfo(skb)->frags[i]; } else { err = -EMSGSIZE; goto error; } if (getfrag(from, skb_frag_address(frag) + skb_frag_size(frag), offset, copy, skb->len, skb) < 0) { err = -EFAULT; goto error; } sk->sk_sndmsg_off += copy; skb_frag_size_add(frag, copy); skb->len += copy; skb->data_len += copy; skb->truesize += copy; atomic_add(copy, &sk->sk_wmem_alloc); } offset += copy; length -= copy; } return 0; error: cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); return err; }