static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); int nr_frags = shinfo->nr_frags; int i; for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = shinfo->frags + i; struct xen_netif_tx_request *txp; struct page *page; u16 pending_idx; pending_idx = frag_get_pending_idx(frag); txp = &vif->pending_tx_info[pending_idx].req; page = virt_to_page(idx_to_kaddr(vif, pending_idx)); __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); skb->len += txp->size; skb->data_len += txp->size; skb->truesize += txp->size; /* Take an extra reference to offset xenvif_idx_release */ get_page(vif->mmap_pages[pending_idx]); xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY); } }
/** * Allocate a new skb that can hold data of length @len. * * An SKB is created complely headerless. The linear part of an SKB * is set apart for headers, and stream data is placed in paged fragments. * Lower layers will take care of prepending all required headers. */ struct sk_buff * ss_skb_alloc_pages(size_t len) { int i_frag, nr_frags = DIV_ROUND_UP(len, PAGE_SIZE); struct sk_buff *skb; BUG_ON(nr_frags > MAX_SKB_FRAGS); skb = ss_skb_alloc(); if (!skb) return NULL; for (i_frag = 0; i_frag < nr_frags; ++i_frag) { struct page *page = alloc_page(GFP_ATOMIC); if (!page) { kfree_skb(skb); return NULL; } #if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,12) /* See __skb_alloc_pages() in include/linux/skbuff.h. */ if (page->pfmemalloc) skb->pfmemalloc = true; #endif __skb_fill_page_desc(skb, i_frag, page, 0, 0); skb_shinfo(skb)->nr_frags++; } return skb; }
/* * Make room for @shift fragments starting with slot @i. Then make * a new fragment in slot @i that can hold @size bytes, and it set up. */ static int __new_pgfrag(struct sk_buff *skb, int size, int i, int shift, TfwStr *it) { int off = 0; struct page *page = NULL; skb_frag_t *frag; BUG_ON(i > MAX_SKB_FRAGS); /* * Try to find room for @size bytes in SKB fragments. * If none found, then allocate a new page for the fragment. */ frag = __lookup_pgfrag_room(skb, size); if (frag) { page = skb_frag_page(frag); off = ss_skb_frag_len(frag); __skb_frag_ref(frag); /* get_page(page); */ } else { page = alloc_page(GFP_ATOMIC); if (!page) return -ENOMEM; } /* Make room for @shift fragments starting with slot @i. */ if (__extend_pgfrags(skb, i, shift, it)) { if (frag) __skb_frag_unref(frag); /* put_page(page); */ else __free_page(page); return -ENOMEM; } /* * When the requested slot is right outside the range of the * array of paged fragments, then the new fragment is put as * the first fragment of the next SKB. */ if (i == MAX_SKB_FRAGS) { i = 0; skb = it->skb; } /* Set up the new fragment in slot @i to hold @size bytes. */ __skb_fill_page_desc(skb, i, page, off, size); ss_skb_adjust_data_len(skb, size); return 0; }
static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); int nr_frags = shinfo->nr_frags; int i; u16 prev_pending_idx = INVALID_PENDING_IDX; for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = shinfo->frags + i; struct xen_netif_tx_request *txp; struct page *page; u16 pending_idx; pending_idx = frag_get_pending_idx(frag); /* If this is not the first frag, chain it to the previous*/ if (prev_pending_idx == INVALID_PENDING_IDX) skb_shinfo(skb)->destructor_arg = &callback_param(vif, pending_idx); else callback_param(vif, prev_pending_idx).ctx = &callback_param(vif, pending_idx); callback_param(vif, pending_idx).ctx = NULL; prev_pending_idx = pending_idx; txp = &vif->pending_tx_info[pending_idx].req; page = virt_to_page(idx_to_kaddr(vif, pending_idx)); __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); skb->len += txp->size; skb->data_len += txp->size; skb->truesize += txp->size; /* Take an extra reference to offset network stack's put_page */ get_page(vif->mmap_pages[pending_idx]); } /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc * overlaps with "index", and "mapping" is not set. I think mapping * should be set. If delivered to local stack, it would drop this * skb in sk_filter unless the socket has the right to use it. */ skb->pfmemalloc = false; }
/** * Delete @len (the value is positive now) bytes from @frag. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to data after the deleted area in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; struct skb_shared_info *si = skb_shinfo(skb); SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); if (unlikely(off + len > skb_frag_size(frag))) { SS_WARN("Attempt to delete too much\n"); return -EFAULT; } /* Fast path: delete a full fragment. */ if (!off && len == skb_frag_size(frag)) { ss_skb_adjust_data_len(skb, -len); __skb_frag_unref(frag); if (i + 1 < si->nr_frags) memmove(&si->frags[i], &si->frags[i + 1], (si->nr_frags - i - 1) * sizeof(skb_frag_t)); --si->nr_frags; goto lookup_next_ptr; } /* Fast path: delete the head part of a fragment. */ if (!off) { frag->page_offset += len; skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); it->ptr = skb_frag_address(frag); return 0; } /* Fast path: delete the tail part of a fragment. */ if (off + len == skb_frag_size(frag)) { skb_frag_size_sub(frag, len); ss_skb_adjust_data_len(skb, -len); ++i; goto lookup_next_ptr; } /* * Delete data in the middle of a fragment. After the data * is deleted the fragment will contain only the head part, * and the tail part is moved to another fragment. * [frag @i] [frag @i+1 - tail data] * * Make room for a fragment right after the @i fragment * to move the tail part of data there. */ if (__extend_pgfrags(skb, i + 1, 1, it)) return -EFAULT; /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off - len; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, len + tail_len); /* Make the fragment with the tail part. */ i = (i + 1) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off + len, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ ss_skb_adjust_data_len(skb, -len); if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address of data after the deleted area. */ it->flags = (skb != skb_dst); it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]); return 0; lookup_next_ptr: /* Get the next fragment after the deleted fragment. */ if (i < si->nr_frags) it->ptr = skb_frag_address(&si->frags[i]); return 0; }
/** * Get room for @len bytes of data starting from offset @off * in fragment @i. * * The room may be found in the preceding fragment if @off is zero. * Otherwise, a new fragment is allocated and fragments around the * fragment @i are rearranged so that data is not actually split * and copied. * * Note: @off is always within the borders of fragment @i. It can * point at the start of a fragment, but it can never point at the * location right after the end of a fragment. In other words, @off * can be zero, but it can not be equal to the size of fragment @i. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to the room for new data in @it->ptr. * @return @it->flags is set if @it->ptr points to data in it->skb. */ static int __split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it) { int tail_len; struct sk_buff *skb_dst; skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i]; SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n", smp_processor_id(), __func__, skb, i, off, len, skb_frag_size(frag)); /* * If @off is zero and there's a preceding page fragment, * then try to append data to that fragment. Go for other * solutions if there's no room. */ if (!off && i) { frag_dst = __check_frag_room(skb, frag - 1, len); if (frag_dst) { /* Coalesce new data with the fragment. */ off = skb_frag_size(frag_dst); skb_frag_size_add(frag_dst, len); ss_skb_adjust_data_len(skb, len); it->ptr = (char *)skb_frag_address(frag_dst) + off; return 0; } } /* * Make a fragment that can hold @len bytes. If @off is * zero, then data is added at the start of fragment @i. * Make a fragment in slot @i, and the original fragment * is shifted forward. If @off is not zero, then make * a fragment in slot @i+1, and make an extra fragment * in slot @i+2 to hold the tail data. */ if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it)) return -EFAULT; /* If @off is zero, the job is done in __new_pgfrag(). */ if (!off) { it->ptr = skb_frag_address(frag); return 0; } /* * If data is added in the middle of a fragment, then split * the fragment. The head of the fragment stays there, and * the tail of the fragment is moved to a new fragment. * The fragment for new data is placed in between. * [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data] * If @i is close to MAX_SKB_FRAGS, then new fragments may * be located in another SKB. */ /* Find the SKB for tail data. */ skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb; /* Calculate the length of the tail part. */ tail_len = skb_frag_size(frag) - off; /* Trim the fragment with the head part. */ skb_frag_size_sub(frag, tail_len); /* Make the fragment with the tail part. */ i = (i + 2) % MAX_SKB_FRAGS; __skb_fill_page_desc(skb_dst, i, skb_frag_page(frag), frag->page_offset + off, tail_len); __skb_frag_ref(frag); /* Adjust SKB data lengths. */ if (skb != skb_dst) { ss_skb_adjust_data_len(skb, -tail_len); ss_skb_adjust_data_len(skb_dst, tail_len); } /* Get the SKB and the address for new data. */ it->flags = !(i < MAX_SKB_FRAGS - 1); frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1; it->ptr = skb_frag_address(frag_dst); return 0; }
/** * The kernel may allocate a bit more memory for an SKB than what was * requested (see ksize() call in __alloc_skb()). Use the extra memory * if it's enough to hold @n bytes. Otherwise, allocate new linear data. * * @return 0 on success, -errno on failure. * @return SKB in @it->skb if new SKB is allocated. * @return pointer to the room for new data in @it->ptr if making room. * @return pointer to data right after the deleted fragment in @it->ptr. */ static int __split_linear_data(struct sk_buff *skb, char *pspt, int len, TfwStr *it) { int alloc = len > 0; int tail_len = (char *)skb_tail_pointer(skb) - pspt; struct page *page = virt_to_head_page(skb->head); SS_DBG("[%d]: %s: skb [%p] pspt [%p] len [%d] tail_len [%d]\n", smp_processor_id(), __func__, skb, pspt, len, tail_len); BUG_ON(!skb->head_frag); BUG_ON(tail_len <= 0); BUG_ON(!(alloc | tail_len)); BUG_ON(-len > tail_len); /* * Quick and unlikely path: just advance the skb tail pointer. * Note that this only works when we make room. When we remove, * pspt points at the start of the data chunk to remove. In that * case, tail_len can never be zero. */ if (unlikely(!tail_len && len <= ss_skb_tailroom(skb))) { BUG_ON(len < 0); it->ptr = ss_skb_put(skb, len); return 0; } /* * Quick and unlikely path: just move skb tail pointer backward. * Note that this only works when we remove data, and the data * is located exactly at the end of the linear part of an skb. */ if (unlikely((len < 0) && (tail_len == -len))) { ss_skb_put(skb, len); if (skb_is_nonlinear(skb)) it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]); return 0; } /* * Data is inserted or deleted in the middle of the linear part, * or there's insufficient room in the linear part of an SKB to * insert @len bytes. * * Don't bother with skb tail room: if the linear part is large, * then it's likely that we'll do some smaller data insertions * later and go by the quick path above. Otherwise, the tail size * is also small. * * The inserted data is placed in a fragment. The tail part is * moved to yet another fragment. The linear part is trimmed to * exclude the deleted data and the tail part. * * Do all allocations before moving the fragments to avoid complex * rollback. */ if (alloc) { if (__new_pgfrag(skb, len, 0, alloc + !!tail_len, it)) return -EFAULT; } else { if (__extend_pgfrags(skb, 0, 1, it)) return -EFAULT; tail_len += len; /* @len is negative. */ } if (tail_len) { int tail_off = pspt - (char *)page_address(page); /* * Trim the linear part by |@len| bytes if data * is deleted. Then trim it further to exclude * the tail data. Finally, set up the fragment * allotted above with the tail data. */ if (len < 0) { tail_off -= len; skb->tail += len; skb->len += len; } skb->tail -= tail_len; skb->data_len += tail_len; skb->truesize += tail_len; __skb_fill_page_desc(skb, alloc, page, tail_off, tail_len); skb_frag_ref(skb, alloc); /* get_page(page); */ } it->ptr = skb_frag_address(&skb_shinfo(skb)->frags[0]); return 0; }
static int __ip6_append_data(struct sock *sk, struct flowi6 *fl6, struct sk_buff_head *queue, struct inet_cork *cork, struct inet6_cork *v6_cork, struct page_frag *pfrag, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), void *from, int length, int transhdrlen, unsigned int flags, int dontfrag) { struct sk_buff *skb, *skb_prev = NULL; unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu; int exthdrlen = 0; int dst_exthdrlen = 0; int hh_len; int copy; int err; int offset = 0; __u8 tx_flags = 0; u32 tskey = 0; struct rt6_info *rt = (struct rt6_info *)cork->dst; struct ipv6_txoptions *opt = v6_cork->opt; int csummode = CHECKSUM_NONE; unsigned int maxnonfragsize, headersize; skb = skb_peek_tail(queue); if (!skb) { exthdrlen = opt ? opt->opt_flen : 0; dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; } mtu = cork->fragsize; orig_mtu = mtu; hh_len = LL_RESERVED_SPACE(rt->dst.dev); fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + (opt ? opt->opt_nflen : 0); maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr); headersize = sizeof(struct ipv6hdr) + (opt ? opt->opt_flen + opt->opt_nflen : 0) + (dst_allfrag(&rt->dst) ? sizeof(struct frag_hdr) : 0) + rt->rt6i_nfheader_len; if (cork->length + length > mtu - headersize && dontfrag && (sk->sk_protocol == IPPROTO_UDP || sk->sk_protocol == IPPROTO_RAW)) { ipv6_local_rxpmtu(sk, fl6, mtu - headersize + sizeof(struct ipv6hdr)); goto emsgsize; } if (ip6_sk_ignore_df(sk)) maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; else maxnonfragsize = mtu; if (cork->length + length > maxnonfragsize - headersize) { emsgsize: ipv6_local_error(sk, EMSGSIZE, fl6, mtu - headersize + sizeof(struct ipv6hdr)); return -EMSGSIZE; } /* CHECKSUM_PARTIAL only with no extension headers and when * we are not going to fragment */ if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && headersize == sizeof(struct ipv6hdr) && length < mtu - headersize && !(flags & MSG_MORE) && rt->dst.dev->features & NETIF_F_V6_CSUM) csummode = CHECKSUM_PARTIAL; if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) { sock_tx_timestamp(sk, &tx_flags); if (tx_flags & SKBTX_ANY_SW_TSTAMP && sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) tskey = sk->sk_tskey++; } /* * Let's try using as much space as possible. * Use MTU if total length of the message fits into the MTU. * Otherwise, we need to reserve fragment header and * fragment alignment (= 8-15 octects, in total). * * Note that we may need to "move" the data from the tail of * of the buffer to the new fragment when we split * the message. * * FIXME: It may be fragmented into multiple chunks * at once if non-fragmentable extension headers * are too large. * --yoshfuji */ cork->length += length; if (((length > mtu) || (skb && skb_is_gso(skb))) && (sk->sk_protocol == IPPROTO_UDP) && (rt->dst.dev->features & NETIF_F_UFO) && (sk->sk_type == SOCK_DGRAM) && !udp_get_no_check6_tx(sk)) { err = ip6_ufo_append_data(sk, queue, getfrag, from, length, hh_len, fragheaderlen, exthdrlen, transhdrlen, mtu, flags, fl6); if (err) goto error; return 0; } if (!skb) goto alloc_new_skb; while (length > 0) { /* Check if the remaining data fits into current packet. */ copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; if (copy < length) copy = maxfraglen - skb->len; if (copy <= 0) { char *data; unsigned int datalen; unsigned int fraglen; unsigned int fraggap; unsigned int alloclen; alloc_new_skb: /* There's no room in the current skb */ if (skb) fraggap = skb->len - maxfraglen; else fraggap = 0; /* update mtu and maxfraglen if necessary */ if (!skb || !skb_prev) ip6_append_data_mtu(&mtu, &maxfraglen, fragheaderlen, skb, rt, orig_mtu); skb_prev = skb; /* * If remaining data exceeds the mtu, * we know we need more fragment(s). */ datalen = length + fraggap; if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; if ((flags & MSG_MORE) && !(rt->dst.dev->features&NETIF_F_SG)) alloclen = mtu; else alloclen = datalen + fragheaderlen; alloclen += dst_exthdrlen; if (datalen != length + fraggap) { /* * this is not the last fragment, the trailer * space is regarded as data space. */ datalen += rt->dst.trailer_len; } alloclen += rt->dst.trailer_len; fraglen = datalen + fragheaderlen; /* * We just reserve space for fragment header. * Note: this may be overallocation if the message * (without MSG_MORE) fits into the MTU. */ alloclen += sizeof(struct frag_hdr); if (transhdrlen) { skb = sock_alloc_send_skb(sk, alloclen + hh_len, (flags & MSG_DONTWAIT), &err); } else { skb = NULL; if (atomic_read(&sk->sk_wmem_alloc) <= 2 * sk->sk_sndbuf) skb = sock_wmalloc(sk, alloclen + hh_len, 1, sk->sk_allocation); if (unlikely(!skb)) err = -ENOBUFS; } if (!skb) goto error; /* * Fill in the control structures */ skb->protocol = htons(ETH_P_IPV6); skb->ip_summed = csummode; skb->csum = 0; /* reserve for fragmentation and ipsec header */ skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + dst_exthdrlen); /* Only the initial fragment is time stamped */ skb_shinfo(skb)->tx_flags = tx_flags; tx_flags = 0; skb_shinfo(skb)->tskey = tskey; tskey = 0; /* * Find where to start putting bytes */ data = skb_put(skb, fraglen); skb_set_network_header(skb, exthdrlen); data += fragheaderlen; skb->transport_header = (skb->network_header + fragheaderlen); if (fraggap) { skb->csum = skb_copy_and_csum_bits( skb_prev, maxfraglen, data + transhdrlen, fraggap, 0); skb_prev->csum = csum_sub(skb_prev->csum, skb->csum); data += fraggap; pskb_trim_unique(skb_prev, maxfraglen); } copy = datalen - transhdrlen - fraggap; if (copy < 0) { err = -EINVAL; kfree_skb(skb); goto error; } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { err = -EFAULT; kfree_skb(skb); goto error; } offset += copy; length -= datalen - fraggap; transhdrlen = 0; exthdrlen = 0; dst_exthdrlen = 0; /* * Put the packet on the pending queue */ __skb_queue_tail(queue, skb); continue; } if (copy > length) copy = length; if (!(rt->dst.dev->features&NETIF_F_SG)) { unsigned int off; off = skb->len; if (getfrag(from, skb_put(skb, copy), offset, copy, off, skb) < 0) { __skb_trim(skb, off); err = -EFAULT; goto error; } } else { int i = skb_shinfo(skb)->nr_frags; err = -ENOMEM; if (!sk_page_frag_refill(sk, pfrag)) goto error; if (!skb_can_coalesce(skb, i, pfrag->page, pfrag->offset)) { err = -EMSGSIZE; if (i == MAX_SKB_FRAGS) goto error; __skb_fill_page_desc(skb, i, pfrag->page, pfrag->offset, 0); skb_shinfo(skb)->nr_frags = ++i; get_page(pfrag->page); } copy = min_t(int, copy, pfrag->size - pfrag->offset); if (getfrag(from, page_address(pfrag->page) + pfrag->offset, offset, copy, skb->len, skb) < 0) goto error_efault; pfrag->offset += copy; skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); skb->len += copy; skb->data_len += copy; skb->truesize += copy; atomic_add(copy, &sk->sk_wmem_alloc); } offset += copy; length -= copy; } return 0; error_efault: err = -EFAULT; error: cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); return err; }
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), void *from, int length, int transhdrlen, int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6, struct rt6_info *rt, unsigned int flags, int dontfrag) { struct inet_sock *inet = inet_sk(sk); struct ipv6_pinfo *np = inet6_sk(sk); struct inet_cork *cork; struct sk_buff *skb, *skb_prev = NULL; unsigned int maxfraglen, fragheaderlen, mtu; int exthdrlen; int dst_exthdrlen; int hh_len; int copy; int err; int offset = 0; __u8 tx_flags = 0; if (flags&MSG_PROBE) return 0; cork = &inet->cork.base; if (skb_queue_empty(&sk->sk_write_queue)) { /* * setup for corking */ if (opt) { if (WARN_ON(np->cork.opt)) return -EINVAL; np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation); if (unlikely(np->cork.opt == NULL)) return -ENOBUFS; np->cork.opt->tot_len = opt->tot_len; np->cork.opt->opt_flen = opt->opt_flen; np->cork.opt->opt_nflen = opt->opt_nflen; np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation); if (opt->dst0opt && !np->cork.opt->dst0opt) return -ENOBUFS; np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation); if (opt->dst1opt && !np->cork.opt->dst1opt) return -ENOBUFS; np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation); if (opt->hopopt && !np->cork.opt->hopopt) return -ENOBUFS; np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation); if (opt->srcrt && !np->cork.opt->srcrt) return -ENOBUFS; /* need source address above miyazawa*/ } dst_hold(&rt->dst); cork->dst = &rt->dst; inet->cork.fl.u.ip6 = *fl6; np->cork.hop_limit = hlimit; np->cork.tclass = tclass; if (rt->dst.flags & DST_XFRM_TUNNEL) mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ? rt->dst.dev->mtu : dst_mtu(&rt->dst); else mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ? rt->dst.dev->mtu : dst_mtu(rt->dst.path); if (np->frag_size < mtu) { if (np->frag_size) mtu = np->frag_size; } cork->fragsize = mtu; if (dst_allfrag(rt->dst.path)) cork->flags |= IPCORK_ALLFRAG; cork->length = 0; exthdrlen = (opt ? opt->opt_flen : 0); length += exthdrlen; transhdrlen += exthdrlen; dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; } else { rt = (struct rt6_info *)cork->dst; fl6 = &inet->cork.fl.u.ip6; opt = np->cork.opt; transhdrlen = 0; exthdrlen = 0; dst_exthdrlen = 0; mtu = cork->fragsize; } hh_len = LL_RESERVED_SPACE(rt->dst.dev); fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + (opt ? opt->opt_nflen : 0); maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr); if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) { if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) { ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen); return -EMSGSIZE; } } /* For UDP, check if TX timestamp is enabled */ if (sk->sk_type == SOCK_DGRAM) sock_tx_timestamp(sk, &tx_flags); /* * Let's try using as much space as possible. * Use MTU if total length of the message fits into the MTU. * Otherwise, we need to reserve fragment header and * fragment alignment (= 8-15 octects, in total). * * Note that we may need to "move" the data from the tail of * of the buffer to the new fragment when we split * the message. * * FIXME: It may be fragmented into multiple chunks * at once if non-fragmentable extension headers * are too large. * --yoshfuji */ if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP || sk->sk_protocol == IPPROTO_RAW)) { ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen); return -EMSGSIZE; } skb = skb_peek_tail(&sk->sk_write_queue); cork->length += length; if (((length > mtu) || (skb && skb_has_frags(skb))) && (sk->sk_protocol == IPPROTO_UDP) && (rt->dst.dev->features & NETIF_F_UFO)) { err = ip6_ufo_append_data(sk, getfrag, from, length, hh_len, fragheaderlen, transhdrlen, mtu, flags, rt); if (err) goto error; return 0; } if (!skb) goto alloc_new_skb; while (length > 0) { /* Check if the remaining data fits into current packet. */ copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; if (copy < length) copy = maxfraglen - skb->len; if (copy <= 0) { char *data; unsigned int datalen; unsigned int fraglen; unsigned int fraggap; unsigned int alloclen; alloc_new_skb: /* There's no room in the current skb */ if (skb) fraggap = skb->len - maxfraglen; else fraggap = 0; /* update mtu and maxfraglen if necessary */ if (skb == NULL || skb_prev == NULL) ip6_append_data_mtu(&mtu, &maxfraglen, fragheaderlen, skb, rt, np->pmtudisc == IPV6_PMTUDISC_PROBE); skb_prev = skb; /* * If remaining data exceeds the mtu, * we know we need more fragment(s). */ datalen = length + fraggap; if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; if ((flags & MSG_MORE) && !(rt->dst.dev->features&NETIF_F_SG)) alloclen = mtu; else alloclen = datalen + fragheaderlen; alloclen += dst_exthdrlen; if (datalen != length + fraggap) { /* * this is not the last fragment, the trailer * space is regarded as data space. */ datalen += rt->dst.trailer_len; } alloclen += rt->dst.trailer_len; fraglen = datalen + fragheaderlen; /* * We just reserve space for fragment header. * Note: this may be overallocation if the message * (without MSG_MORE) fits into the MTU. */ alloclen += sizeof(struct frag_hdr); if (transhdrlen) { skb = sock_alloc_send_skb(sk, alloclen + hh_len, (flags & MSG_DONTWAIT), &err); } else { skb = NULL; if (atomic_read(&sk->sk_wmem_alloc) <= 2 * sk->sk_sndbuf) skb = sock_wmalloc(sk, alloclen + hh_len, 1, sk->sk_allocation); if (unlikely(skb == NULL)) err = -ENOBUFS; else { /* Only the initial fragment * is time stamped. */ tx_flags = 0; } } if (skb == NULL) goto error; /* * Fill in the control structures */ skb->ip_summed = CHECKSUM_NONE; skb->csum = 0; /* reserve for fragmentation and ipsec header */ skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + dst_exthdrlen); if (sk->sk_type == SOCK_DGRAM) skb_shinfo(skb)->tx_flags = tx_flags; /* * Find where to start putting bytes */ data = skb_put(skb, fraglen); skb_set_network_header(skb, exthdrlen); data += fragheaderlen; skb->transport_header = (skb->network_header + fragheaderlen); if (fraggap) { skb->csum = skb_copy_and_csum_bits( skb_prev, maxfraglen, data + transhdrlen, fraggap, 0); skb_prev->csum = csum_sub(skb_prev->csum, skb->csum); data += fraggap; pskb_trim_unique(skb_prev, maxfraglen); } copy = datalen - transhdrlen - fraggap; if (copy < 0) { err = -EINVAL; kfree_skb(skb); goto error; } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { err = -EFAULT; kfree_skb(skb); goto error; } offset += copy; length -= datalen - fraggap; transhdrlen = 0; exthdrlen = 0; dst_exthdrlen = 0; /* * Put the packet on the pending queue */ __skb_queue_tail(&sk->sk_write_queue, skb); continue; } if (copy > length) copy = length; if (!(rt->dst.dev->features&NETIF_F_SG)) { unsigned int off; off = skb->len; if (getfrag(from, skb_put(skb, copy), offset, copy, off, skb) < 0) { __skb_trim(skb, off); err = -EFAULT; goto error; } } else { int i = skb_shinfo(skb)->nr_frags; struct page_frag *pfrag = sk_page_frag(sk); err = -ENOMEM; if (!sk_page_frag_refill(sk, pfrag)) goto error; if (!skb_can_coalesce(skb, i, pfrag->page, pfrag->offset)) { err = -EMSGSIZE; if (i == MAX_SKB_FRAGS) goto error; __skb_fill_page_desc(skb, i, pfrag->page, pfrag->offset, 0); skb_shinfo(skb)->nr_frags = ++i; get_page(pfrag->page); } copy = min_t(int, copy, pfrag->size - pfrag->offset); if (getfrag(from, page_address(pfrag->page) + pfrag->offset, offset, copy, skb->len, skb) < 0) goto error_efault; pfrag->offset += copy; skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); skb->len += copy; skb->data_len += copy; skb->truesize += copy; atomic_add(copy, &sk->sk_wmem_alloc); } offset += copy; length -= copy; } return 0; error_efault: err = -EFAULT; error: cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); return err; }
static void xennet_alloc_rx_buffers(struct net_device *dev) { unsigned short id; struct netfront_info *np = netdev_priv(dev); struct sk_buff *skb; struct page *page; int i, batch_target, notify; RING_IDX req_prod = np->rx.req_prod_pvt; grant_ref_t ref; unsigned long pfn; void *vaddr; struct xen_netif_rx_request *req; if (unlikely(!netif_carrier_ok(dev))) return; /* * Allocate skbuffs greedily, even though we batch updates to the * receive ring. This creates a less bursty demand on the memory * allocator, so should reduce the chance of failed allocation requests * both for ourself and for other kernel subsystems. */ batch_target = np->rx_target - (req_prod - np->rx.rsp_cons); for (i = skb_queue_len(&np->rx_batch); i < batch_target; i++) { skb = __netdev_alloc_skb(dev, RX_COPY_THRESHOLD + NET_IP_ALIGN, GFP_ATOMIC | __GFP_NOWARN); if (unlikely(!skb)) goto no_skb; /* Align ip header to a 16 bytes boundary */ skb_reserve(skb, NET_IP_ALIGN); page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); if (!page) { kfree_skb(skb); no_skb: /* Any skbuffs queued for refill? Force them out. */ if (i != 0) goto refill; /* Could not allocate any skbuffs. Try again later. */ mod_timer(&np->rx_refill_timer, jiffies + (HZ/10)); break; } __skb_fill_page_desc(skb, 0, page, 0, 0); skb_shinfo(skb)->nr_frags = 1; __skb_queue_tail(&np->rx_batch, skb); } /* Is the batch large enough to be worthwhile? */ if (i < (np->rx_target/2)) { if (req_prod > np->rx.sring->req_prod) goto push; return; } /* Adjust our fill target if we risked running out of buffers. */ if (((req_prod - np->rx.sring->rsp_prod) < (np->rx_target / 4)) && ((np->rx_target *= 2) > np->rx_max_target)) np->rx_target = np->rx_max_target; refill: for (i = 0; ; i++) { skb = __skb_dequeue(&np->rx_batch); if (skb == NULL) break; skb->dev = dev; id = xennet_rxidx(req_prod + i); BUG_ON(np->rx_skbs[id]); np->rx_skbs[id] = skb; ref = gnttab_claim_grant_reference(&np->gref_rx_head); BUG_ON((signed short)ref < 0); np->grant_rx_ref[id] = ref; pfn = page_to_pfn(skb_frag_page(&skb_shinfo(skb)->frags[0])); vaddr = page_address(skb_frag_page(&skb_shinfo(skb)->frags[0])); req = RING_GET_REQUEST(&np->rx, req_prod + i); gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id, pfn_to_mfn(pfn), 0); req->id = id; req->gref = ref; } wmb(); /* barrier so backend seens requests */ /* Above is a suitable barrier to ensure backend will see requests. */ np->rx.req_prod_pvt = req_prod + i; push: RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->rx, notify); if (notify) notify_remote_via_irq(np->netdev->irq); }
static void xennet_alloc_rx_buffers(struct net_device *dev) { unsigned short id; struct netfront_info *np = netdev_priv(dev); struct sk_buff *skb; struct page *page; int i, batch_target, notify; RING_IDX req_prod = np->rx.req_prod_pvt; grant_ref_t ref; unsigned long pfn; void *vaddr; struct xen_netif_rx_request *req; if (unlikely(!netif_carrier_ok(dev))) return; batch_target = np->rx_target - (req_prod - np->rx.rsp_cons); for (i = skb_queue_len(&np->rx_batch); i < batch_target; i++) { skb = __netdev_alloc_skb(dev, RX_COPY_THRESHOLD + NET_IP_ALIGN, GFP_ATOMIC | __GFP_NOWARN); if (unlikely(!skb)) goto no_skb; skb_reserve(skb, NET_IP_ALIGN); page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); if (!page) { kfree_skb(skb); no_skb: if (i != 0) goto refill; mod_timer(&np->rx_refill_timer, jiffies + (HZ/10)); break; } __skb_fill_page_desc(skb, 0, page, 0, 0); skb_shinfo(skb)->nr_frags = 1; __skb_queue_tail(&np->rx_batch, skb); } if (i < (np->rx_target/2)) { if (req_prod > np->rx.sring->req_prod) goto push; return; } if (((req_prod - np->rx.sring->rsp_prod) < (np->rx_target / 4)) && ((np->rx_target *= 2) > np->rx_max_target)) np->rx_target = np->rx_max_target; refill: for (i = 0; ; i++) { skb = __skb_dequeue(&np->rx_batch); if (skb == NULL) break; skb->dev = dev; id = xennet_rxidx(req_prod + i); BUG_ON(np->rx_skbs[id]); np->rx_skbs[id] = skb; ref = gnttab_claim_grant_reference(&np->gref_rx_head); BUG_ON((signed short)ref < 0); np->grant_rx_ref[id] = ref; pfn = page_to_pfn(skb_frag_page(&skb_shinfo(skb)->frags[0])); vaddr = page_address(skb_frag_page(&skb_shinfo(skb)->frags[0])); req = RING_GET_REQUEST(&np->rx, req_prod + i); gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id, pfn_to_mfn(pfn), 0); req->id = id; req->gref = ref; } wmb(); np->rx.req_prod_pvt = req_prod + i; push: RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->rx, notify); if (notify) notify_remote_via_irq(np->netdev->irq); }