static int
__new_pgfrag(struct sk_buff *skb, struct sk_buff *pskb, int size, int i,
int shift)
{
int off = 0;
struct page *page = NULL;
skb_frag_t *frag;
BUG_ON(i > MAX_SKB_FRAGS);
frag = __lookup_pgfrag_room(skb, size);
if (frag) {
page = skb_frag_page(frag);
off = ss_skb_frag_len(frag);
__skb_frag_ref(frag);
} else {
page = alloc_page(GFP_ATOMIC);
if (!page)
return -ENOMEM;
}
if (__extend_pgfrags(skb, pskb, i, shift)) {
if (!frag)
__free_page(page);
return -ENOMEM;
}
if (i == MAX_SKB_FRAGS) {
/*
* Insert a new paged fragment right after the last one
* in @skb, i.e. as the first fragment of the next skb.
*/
skb = skb_shinfo(pskb ? : skb)->frag_list;
i = 0;
}
/*
* Make room for @shift fragments starting with slot @i. Then make
* a new fragment in slot @i that can hold @size bytes, and it set up.
*/
static int
__new_pgfrag(struct sk_buff *skb, int size, int i, int shift, TfwStr *it)
{
int off = 0;
struct page *page = NULL;
skb_frag_t *frag;
BUG_ON(i > MAX_SKB_FRAGS);
/*
* Try to find room for @size bytes in SKB fragments.
* If none found, then allocate a new page for the fragment.
*/
frag = __lookup_pgfrag_room(skb, size);
if (frag) {
page = skb_frag_page(frag);
off = ss_skb_frag_len(frag);
__skb_frag_ref(frag); /* get_page(page); */
} else {
page = alloc_page(GFP_ATOMIC);
if (!page)
return -ENOMEM;
}
/* Make room for @shift fragments starting with slot @i. */
if (__extend_pgfrags(skb, i, shift, it)) {
if (frag)
__skb_frag_unref(frag); /* put_page(page); */
else
__free_page(page);
return -ENOMEM;
}
/*
* When the requested slot is right outside the range of the
* array of paged fragments, then the new fragment is put as
* the first fragment of the next SKB.
*/
if (i == MAX_SKB_FRAGS) {
i = 0;
skb = it->skb;
}
/* Set up the new fragment in slot @i to hold @size bytes. */
__skb_fill_page_desc(skb, i, page, off, size);
ss_skb_adjust_data_len(skb, size);
return 0;
}
/**
* Delete @len (the value is positive now) bytes from @frag.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to data after the deleted area in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_del(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
struct skb_shared_info *si = skb_shinfo(skb);
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
if (unlikely(off + len > skb_frag_size(frag))) {
SS_WARN("Attempt to delete too much\n");
return -EFAULT;
}
/* Fast path: delete a full fragment. */
if (!off && len == skb_frag_size(frag)) {
ss_skb_adjust_data_len(skb, -len);
__skb_frag_unref(frag);
if (i + 1 < si->nr_frags)
memmove(&si->frags[i], &si->frags[i + 1],
(si->nr_frags - i - 1) * sizeof(skb_frag_t));
--si->nr_frags;
goto lookup_next_ptr;
}
/* Fast path: delete the head part of a fragment. */
if (!off) {
frag->page_offset += len;
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
it->ptr = skb_frag_address(frag);
return 0;
}
/* Fast path: delete the tail part of a fragment. */
if (off + len == skb_frag_size(frag)) {
skb_frag_size_sub(frag, len);
ss_skb_adjust_data_len(skb, -len);
++i;
goto lookup_next_ptr;
}
/*
* Delete data in the middle of a fragment. After the data
* is deleted the fragment will contain only the head part,
* and the tail part is moved to another fragment.
* [frag @i] [frag @i+1 - tail data]
*
* Make room for a fragment right after the @i fragment
* to move the tail part of data there.
*/
if (__extend_pgfrags(skb, i + 1, 1, it))
return -EFAULT;
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 1) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off - len;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, len + tail_len);
/* Make the fragment with the tail part. */
i = (i + 1) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off + len, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
ss_skb_adjust_data_len(skb, -len);
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address of data after the deleted area. */
it->flags = (skb != skb_dst);
it->ptr = skb_frag_address(&skb_shinfo(skb_dst)->frags[i]);
return 0;
lookup_next_ptr:
/* Get the next fragment after the deleted fragment. */
if (i < si->nr_frags)
it->ptr = skb_frag_address(&si->frags[i]);
return 0;
}
/**
* Get room for @len bytes of data starting from offset @off
* in fragment @i.
*
* The room may be found in the preceding fragment if @off is zero.
* Otherwise, a new fragment is allocated and fragments around the
* fragment @i are rearranged so that data is not actually split
* and copied.
*
* Note: @off is always within the borders of fragment @i. It can
* point at the start of a fragment, but it can never point at the
* location right after the end of a fragment. In other words, @off
* can be zero, but it can not be equal to the size of fragment @i.
*
* @return 0 on success, -errno on failure.
* @return SKB in @it->skb if new SKB is allocated.
* @return pointer to the room for new data in @it->ptr.
* @return @it->flags is set if @it->ptr points to data in it->skb.
*/
static int
__split_pgfrag_add(struct sk_buff *skb, int i, int off, int len, TfwStr *it)
{
int tail_len;
struct sk_buff *skb_dst;
skb_frag_t *frag_dst, *frag = &skb_shinfo(skb)->frags[i];
SS_DBG("[%d]: %s: skb [%p] i [%d] off [%d] len [%d] fragsize [%d]\n",
smp_processor_id(), __func__,
skb, i, off, len, skb_frag_size(frag));
/*
* If @off is zero and there's a preceding page fragment,
* then try to append data to that fragment. Go for other
* solutions if there's no room.
*/
if (!off && i) {
frag_dst = __check_frag_room(skb, frag - 1, len);
if (frag_dst) {
/* Coalesce new data with the fragment. */
off = skb_frag_size(frag_dst);
skb_frag_size_add(frag_dst, len);
ss_skb_adjust_data_len(skb, len);
it->ptr = (char *)skb_frag_address(frag_dst) + off;
return 0;
}
}
/*
* Make a fragment that can hold @len bytes. If @off is
* zero, then data is added at the start of fragment @i.
* Make a fragment in slot @i, and the original fragment
* is shifted forward. If @off is not zero, then make
* a fragment in slot @i+1, and make an extra fragment
* in slot @i+2 to hold the tail data.
*/
if (__new_pgfrag(skb, len, i + !!off, 1 + !!off, it))
return -EFAULT;
/* If @off is zero, the job is done in __new_pgfrag(). */
if (!off) {
it->ptr = skb_frag_address(frag);
return 0;
}
/*
* If data is added in the middle of a fragment, then split
* the fragment. The head of the fragment stays there, and
* the tail of the fragment is moved to a new fragment.
* The fragment for new data is placed in between.
* [frag @i] [frag @i+1 - new data] [frag @i+2 - tail data]
* If @i is close to MAX_SKB_FRAGS, then new fragments may
* be located in another SKB.
*/
/* Find the SKB for tail data. */
skb_dst = (i < MAX_SKB_FRAGS - 2) ? skb : it->skb;
/* Calculate the length of the tail part. */
tail_len = skb_frag_size(frag) - off;
/* Trim the fragment with the head part. */
skb_frag_size_sub(frag, tail_len);
/* Make the fragment with the tail part. */
i = (i + 2) % MAX_SKB_FRAGS;
__skb_fill_page_desc(skb_dst, i, skb_frag_page(frag),
frag->page_offset + off, tail_len);
__skb_frag_ref(frag);
/* Adjust SKB data lengths. */
if (skb != skb_dst) {
ss_skb_adjust_data_len(skb, -tail_len);
ss_skb_adjust_data_len(skb_dst, tail_len);
}
/* Get the SKB and the address for new data. */
it->flags = !(i < MAX_SKB_FRAGS - 1);
frag_dst = it->flags ? &skb_shinfo(it->skb)->frags[0] : frag + 1;
it->ptr = skb_frag_address(frag_dst);
return 0;
}
static int fill_sg_in(struct scatterlist *sg_in,
struct sk_buff *skb,
struct tls_offload_context_tx *ctx,
u64 *rcd_sn,
s32 *sync_size,
int *resync_sgs)
{
int tcp_payload_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
int payload_len = skb->len - tcp_payload_offset;
u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
struct tls_record_info *record;
unsigned long flags;
int remaining;
int i;
spin_lock_irqsave(&ctx->lock, flags);
record = tls_get_record(ctx, tcp_seq, rcd_sn);
if (!record) {
spin_unlock_irqrestore(&ctx->lock, flags);
WARN(1, "Record not found for seq %u\n", tcp_seq);
return -EINVAL;
}
*sync_size = tcp_seq - tls_record_start_seq(record);
if (*sync_size < 0) {
int is_start_marker = tls_record_is_start_marker(record);
spin_unlock_irqrestore(&ctx->lock, flags);
/* This should only occur if the relevant record was
* already acked. In that case it should be ok
* to drop the packet and avoid retransmission.
*
* There is a corner case where the packet contains
* both an acked and a non-acked record.
* We currently don't handle that case and rely
* on TCP to retranmit a packet that doesn't contain
* already acked payload.
*/
if (!is_start_marker)
*sync_size = 0;
return -EINVAL;
}
remaining = *sync_size;
for (i = 0; remaining > 0; i++) {
skb_frag_t *frag = &record->frags[i];
__skb_frag_ref(frag);
sg_set_page(sg_in + i, skb_frag_page(frag),
skb_frag_size(frag), frag->page_offset);
remaining -= skb_frag_size(frag);
if (remaining < 0)
sg_in[i].length += remaining;
}
*resync_sgs = i;
spin_unlock_irqrestore(&ctx->lock, flags);
if (skb_to_sgvec(skb, &sg_in[i], tcp_payload_offset, payload_len) < 0)
return -EINVAL;
return 0;
}
