int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
int offset, int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6,
struct rt6_info *rt, unsigned int flags, int dontfrag)
{
struct inet_sock *inet = inet_sk(sk);
struct ipv6_pinfo *np = inet6_sk(sk);
struct inet_cork *cork;
struct sk_buff *skb, *skb_prev = NULL;
unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu;
int exthdrlen;
int dst_exthdrlen;
int hh_len;
int copy;
int err;
int offset = 0;
int csummode = CHECKSUM_NONE;
__u8 tx_flags = 0;
if (flags&MSG_PROBE)
return 0;
cork = &inet->cork.base;
if (skb_queue_empty(&sk->sk_write_queue)) {
/*
* setup for corking
*/
if (opt) {
if (WARN_ON(np->cork.opt))
return -EINVAL;
np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation);
if (unlikely(np->cork.opt == NULL))
return -ENOBUFS;
np->cork.opt->tot_len = opt->tot_len;
np->cork.opt->opt_flen = opt->opt_flen;
np->cork.opt->opt_nflen = opt->opt_nflen;
np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt,
sk->sk_allocation);
if (opt->dst0opt && !np->cork.opt->dst0opt)
return -ENOBUFS;
np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt,
sk->sk_allocation);
if (opt->dst1opt && !np->cork.opt->dst1opt)
return -ENOBUFS;
np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt,
sk->sk_allocation);
if (opt->hopopt && !np->cork.opt->hopopt)
return -ENOBUFS;
np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt,
sk->sk_allocation);
if (opt->srcrt && !np->cork.opt->srcrt)
return -ENOBUFS;
/* need source address above miyazawa*/
}
dst_hold(&rt->dst);
cork->dst = &rt->dst;
inet->cork.fl.u.ip6 = *fl6;
np->cork.hop_limit = hlimit;
np->cork.tclass = tclass;
if (rt->dst.flags & DST_XFRM_TUNNEL)
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(&rt->dst);
else
mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
rt->dst.dev->mtu : dst_mtu(rt->dst.path);
if (np->frag_size < mtu) {
if (np->frag_size)
mtu = np->frag_size;
}
cork->fragsize = mtu;
if (dst_allfrag(rt->dst.path))
cork->flags |= IPCORK_ALLFRAG;
cork->length = 0;
sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
exthdrlen = (opt ? opt->opt_flen : 0);
length += exthdrlen;
transhdrlen += exthdrlen;
dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
} else {
rt = (struct rt6_info *)cork->dst;
fl6 = &inet->cork.fl.u.ip6;
opt = np->cork.opt;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
mtu = cork->fragsize;
}
orig_mtu = mtu;
hh_len = LL_RESERVED_SPACE(rt->dst.dev);
fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
(opt ? opt->opt_nflen : 0);
maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
}
/* For UDP, check if TX timestamp is enabled */
if (sk->sk_type == SOCK_DGRAM) {
err = sock_tx_timestamp(sk, &tx_flags);
if (err)
goto error;
}
/*
* Let's try using as much space as possible.
* Use MTU if total length of the message fits into the MTU.
* Otherwise, we need to reserve fragment header and
* fragment alignment (= 8-15 octects, in total).
*
* Note that we may need to "move" the data from the tail of
* of the buffer to the new fragment when we split
* the message.
*
* FIXME: It may be fragmented into multiple chunks
* at once if non-fragmentable extension headers
* are too large.
* --yoshfuji
*/
if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP ||
sk->sk_protocol == IPPROTO_RAW)) {
ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen);
return -EMSGSIZE;
}
skb = skb_peek_tail(&sk->sk_write_queue);
cork->length += length;
if (((length > mtu) ||
(skb && skb_has_frags(skb))) &&
(sk->sk_protocol == IPPROTO_UDP) &&
(rt->dst.dev->features & NETIF_F_UFO) &&
(sk->sk_type == SOCK_DGRAM)) {
err = ip6_ufo_append_data(sk, getfrag, from, length,
hh_len, fragheaderlen,
transhdrlen, mtu, flags, rt);
if (err)
goto error;
return 0;
}
if (!skb)
goto alloc_new_skb;
while (length > 0) {
/* Check if the remaining data fits into current packet. */
copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
if (copy < length)
copy = maxfraglen - skb->len;
if (copy <= 0) {
char *data;
unsigned int datalen;
unsigned int fraglen;
unsigned int fraggap;
unsigned int alloclen;
alloc_new_skb:
/* There's no room in the current skb */
if (skb)
fraggap = skb->len - maxfraglen;
else
fraggap = 0;
/* update mtu and maxfraglen if necessary */
if (skb == NULL || skb_prev == NULL)
ip6_append_data_mtu(&mtu, &maxfraglen,
fragheaderlen, skb, rt,
orig_mtu);
skb_prev = skb;
/*
* If remaining data exceeds the mtu,
* we know we need more fragment(s).
*/
datalen = length + fraggap;
if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
if ((flags & MSG_MORE) &&
!(rt->dst.dev->features&NETIF_F_SG))
alloclen = mtu;
else
alloclen = datalen + fragheaderlen;
alloclen += dst_exthdrlen;
if (datalen != length + fraggap) {
/*
* this is not the last fragment, the trailer
* space is regarded as data space.
*/
datalen += rt->dst.trailer_len;
}
alloclen += rt->dst.trailer_len;
fraglen = datalen + fragheaderlen;
/*
* We just reserve space for fragment header.
* Note: this may be overallocation if the message
* (without MSG_MORE) fits into the MTU.
*/
alloclen += sizeof(struct frag_hdr);
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len,
(flags & MSG_DONTWAIT), &err);
} else {
skb = NULL;
if (atomic_read(&sk->sk_wmem_alloc) <=
2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len, 1,
sk->sk_allocation);
if (unlikely(skb == NULL))
err = -ENOBUFS;
else {
/* Only the initial fragment
* is time stamped.
*/
tx_flags = 0;
}
}
if (skb == NULL)
goto error;
/*
* Fill in the control structures
*/
skb->ip_summed = csummode;
skb->csum = 0;
/* reserve for fragmentation and ipsec header */
skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
dst_exthdrlen);
if (sk->sk_type == SOCK_DGRAM)
skb_shinfo(skb)->tx_flags = tx_flags;
/*
* Find where to start putting bytes
*/
data = skb_put(skb, fraglen);
skb_set_network_header(skb, exthdrlen);
data += fragheaderlen;
skb->transport_header = (skb->network_header +
fragheaderlen);
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
data + transhdrlen, fraggap, 0);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
pskb_trim_unique(skb_prev, maxfraglen);
}
copy = datalen - transhdrlen - fraggap;
if (copy < 0) {
err = -EINVAL;
kfree_skb(skb);
goto error;
} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
err = -EFAULT;
kfree_skb(skb);
goto error;
}
offset += copy;
length -= datalen - fraggap;
transhdrlen = 0;
exthdrlen = 0;
dst_exthdrlen = 0;
csummode = CHECKSUM_NONE;
/*
* Put the packet on the pending queue
*/
__skb_queue_tail(&sk->sk_write_queue, skb);
continue;
}
if (copy > length)
copy = length;
if (!(rt->dst.dev->features&NETIF_F_SG)) {
unsigned int off;
off = skb->len;
if (getfrag(from, skb_put(skb, copy),
offset, copy, off, skb) < 0) {
__skb_trim(skb, off);
err = -EFAULT;
goto error;
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = sk->sk_sndmsg_page;
int off = sk->sk_sndmsg_off;
unsigned int left;
if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != skb_frag_page(frag)) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
skb_frag_ref(skb, i);
frag = &skb_shinfo(skb)->frags[i];
}
} else if(i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->sk_allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
sk->sk_sndmsg_page = page;
sk->sk_sndmsg_off = 0;
skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from,
skb_frag_address(frag) + skb_frag_size(frag),
offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
goto error;
}
sk->sk_sndmsg_off += copy;
skb_frag_size_add(frag, copy);
skb->len += copy;
skb->data_len += copy;
skb->truesize += copy;
atomic_add(copy, &sk->sk_wmem_alloc);
}
offset += copy;
length -= copy;
}
return 0;
error:
cork->length -= length;
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
return err;
}
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
int hlimit, struct ipv6_txoptions *opt, struct flowi *fl, struct rt6_info *rt,
unsigned int flags)
{
struct inet_sock *inet = inet_sk(sk);
struct ipv6_pinfo *np = inet6_sk(sk);
struct sk_buff *skb;
unsigned int maxfraglen, fragheaderlen;
int exthdrlen;
int hh_len;
int mtu;
int copy;
int err;
int offset = 0;
int csummode = CHECKSUM_NONE;
if (flags&MSG_PROBE)
return 0;
if (skb_queue_empty(&sk->sk_write_queue)) {
/*
* setup for corking
*/
if (opt) {
if (np->cork.opt == NULL) {
np->cork.opt = kmalloc(opt->tot_len,
sk->sk_allocation);
if (unlikely(np->cork.opt == NULL))
return -ENOBUFS;
} else if (np->cork.opt->tot_len < opt->tot_len) {
printk(KERN_DEBUG "ip6_append_data: invalid option length\n");
return -EINVAL;
}
memcpy(np->cork.opt, opt, opt->tot_len);
inet->cork.flags |= IPCORK_OPT;
/* need source address above miyazawa*/
}
dst_hold(&rt->u.dst);
np->cork.rt = rt;
inet->cork.fl = *fl;
np->cork.hop_limit = hlimit;
inet->cork.fragsize = mtu = dst_mtu(rt->u.dst.path);
if (dst_allfrag(rt->u.dst.path))
inet->cork.flags |= IPCORK_ALLFRAG;
inet->cork.length = 0;
sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
exthdrlen = rt->u.dst.header_len + (opt ? opt->opt_flen : 0);
length += exthdrlen;
transhdrlen += exthdrlen;
} else {
rt = np->cork.rt;
fl = &inet->cork.fl;
if (inet->cork.flags & IPCORK_OPT)
opt = np->cork.opt;
transhdrlen = 0;
exthdrlen = 0;
mtu = inet->cork.fragsize;
}
hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
fragheaderlen = sizeof(struct ipv6hdr) + (opt ? opt->opt_nflen : 0);
maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
if (inet->cork.length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
ipv6_local_error(sk, EMSGSIZE, fl, mtu-exthdrlen);
return -EMSGSIZE;
}
}
/*
* Let's try using as much space as possible.
* Use MTU if total length of the message fits into the MTU.
* Otherwise, we need to reserve fragment header and
* fragment alignment (= 8-15 octects, in total).
*
* Note that we may need to "move" the data from the tail of
* of the buffer to the new fragment when we split
* the message.
*
* FIXME: It may be fragmented into multiple chunks
* at once if non-fragmentable extension headers
* are too large.
* --yoshfuji
*/
inet->cork.length += length;
if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
goto alloc_new_skb;
while (length > 0) {
/* Check if the remaining data fits into current packet. */
copy = (inet->cork.length <= mtu && !(inet->cork.flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
if (copy < length)
copy = maxfraglen - skb->len;
if (copy <= 0) {
char *data;
unsigned int datalen;
unsigned int fraglen;
unsigned int fraggap;
unsigned int alloclen;
struct sk_buff *skb_prev;
alloc_new_skb:
skb_prev = skb;
/* There's no room in the current skb */
if (skb_prev)
fraggap = skb_prev->len - maxfraglen;
else
fraggap = 0;
/*
* If remaining data exceeds the mtu,
* we know we need more fragment(s).
*/
datalen = length + fraggap;
if (datalen > (inet->cork.length <= mtu && !(inet->cork.flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
datalen = maxfraglen - fragheaderlen;
fraglen = datalen + fragheaderlen;
if ((flags & MSG_MORE) &&
!(rt->u.dst.dev->features&NETIF_F_SG))
alloclen = mtu;
else
alloclen = datalen + fragheaderlen;
/*
* The last fragment gets additional space at tail.
* Note: we overallocate on fragments with MSG_MODE
* because we have no idea if we're the last one.
*/
if (datalen == length + fraggap)
alloclen += rt->u.dst.trailer_len;
/*
* We just reserve space for fragment header.
* Note: this may be overallocation if the message
* (without MSG_MORE) fits into the MTU.
*/
alloclen += sizeof(struct frag_hdr);
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len,
(flags & MSG_DONTWAIT), &err);
} else {
skb = NULL;
if (atomic_read(&sk->sk_wmem_alloc) <=
2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len, 1,
sk->sk_allocation);
if (unlikely(skb == NULL))
err = -ENOBUFS;
}
if (skb == NULL)
goto error;
/*
* Fill in the control structures
*/
skb->ip_summed = csummode;
skb->csum = 0;
/* reserve for fragmentation */
skb_reserve(skb, hh_len+sizeof(struct frag_hdr));
/*
* Find where to start putting bytes
*/
data = skb_put(skb, fraglen);
skb->nh.raw = data + exthdrlen;
data += fragheaderlen;
skb->h.raw = data + exthdrlen;
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
data + transhdrlen, fraggap, 0);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
skb_trim(skb_prev, maxfraglen);
}
copy = datalen - transhdrlen - fraggap;
if (copy < 0) {
err = -EINVAL;
kfree_skb(skb);
goto error;
} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
err = -EFAULT;
kfree_skb(skb);
goto error;
}
offset += copy;
length -= datalen - fraggap;
transhdrlen = 0;
exthdrlen = 0;
csummode = CHECKSUM_NONE;
/*
* Put the packet on the pending queue
*/
__skb_queue_tail(&sk->sk_write_queue, skb);
continue;
}
if (copy > length)
copy = length;
if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
unsigned int off;
off = skb->len;
if (getfrag(from, skb_put(skb, copy),
offset, copy, off, skb) < 0) {
__skb_trim(skb, off);
err = -EFAULT;
goto error;
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = sk->sk_sndmsg_page;
int off = sk->sk_sndmsg_off;
unsigned int left;
if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != frag->page) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
get_page(page);
skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
frag = &skb_shinfo(skb)->frags[i];
}
} else if(i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->sk_allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
sk->sk_sndmsg_page = page;
sk->sk_sndmsg_off = 0;
skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
skb->truesize += PAGE_SIZE;
atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
goto error;
}
sk->sk_sndmsg_off += copy;
frag->size += copy;
skb->len += copy;
skb->data_len += copy;
}
offset += copy;
length -= copy;
}
return 0;
error:
inet->cork.length -= length;
IP6_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
return err;
}
/*
* Deliver read data back to initiator.
* XXX TBD handle resource problems later.
*/
int ft_queue_data_in(struct se_cmd *se_cmd)
{
struct ft_cmd *cmd = container_of(se_cmd, struct ft_cmd, se_cmd);
struct fc_frame *fp = NULL;
struct fc_exch *ep;
struct fc_lport *lport;
struct scatterlist *sg = NULL;
size_t remaining;
u32 f_ctl = FC_FC_EX_CTX | FC_FC_REL_OFF;
u32 mem_off = 0;
u32 fh_off = 0;
u32 frame_off = 0;
size_t frame_len = 0;
size_t mem_len = 0;
size_t tlen;
size_t off_in_page;
struct page *page = NULL;
int use_sg;
int error;
void *page_addr;
void *from;
void *to = NULL;
ep = fc_seq_exch(cmd->seq);
lport = ep->lp;
cmd->seq = lport->tt.seq_start_next(cmd->seq);
remaining = se_cmd->data_length;
/*
* Setup to use first mem list entry, unless no data.
*/
BUG_ON(remaining && !se_cmd->t_data_sg);
if (remaining) {
sg = se_cmd->t_data_sg;
mem_len = sg->length;
mem_off = sg->offset;
page = sg_page(sg);
}
/* no scatter/gather in skb for odd word length due to fc_seq_send() */
use_sg = !(remaining % 4);
while (remaining) {
if (!mem_len) {
sg = sg_next(sg);
mem_len = min((size_t)sg->length, remaining);
mem_off = sg->offset;
page = sg_page(sg);
}
if (!frame_len) {
/*
* If lport's has capability of Large Send Offload LSO)
* , then allow 'frame_len' to be as big as 'lso_max'
* if indicated transfer length is >= lport->lso_max
*/
frame_len = (lport->seq_offload) ? lport->lso_max :
cmd->sess->max_frame;
frame_len = min(frame_len, remaining);
fp = fc_frame_alloc(lport, use_sg ? 0 : frame_len);
if (!fp)
return -ENOMEM;
to = fc_frame_payload_get(fp, 0);
fh_off = frame_off;
frame_off += frame_len;
/*
* Setup the frame's max payload which is used by base
* driver to indicate HW about max frame size, so that
* HW can do fragmentation appropriately based on
* "gso_max_size" of underline netdev.
*/
fr_max_payload(fp) = cmd->sess->max_frame;
}
tlen = min(mem_len, frame_len);
if (use_sg) {
off_in_page = mem_off;
BUG_ON(!page);
get_page(page);
skb_fill_page_desc(fp_skb(fp),
skb_shinfo(fp_skb(fp))->nr_frags,
page, off_in_page, tlen);
fr_len(fp) += tlen;
fp_skb(fp)->data_len += tlen;
fp_skb(fp)->truesize +=
PAGE_SIZE << compound_order(page);
} else {
BUG_ON(!page);
from = kmap_atomic(page + (mem_off >> PAGE_SHIFT),
KM_SOFTIRQ0);
page_addr = from;
from += mem_off & ~PAGE_MASK;
tlen = min(tlen, (size_t)(PAGE_SIZE -
(mem_off & ~PAGE_MASK)));
memcpy(to, from, tlen);
kunmap_atomic(page_addr, KM_SOFTIRQ0);
to += tlen;
}
mem_off += tlen;
mem_len -= tlen;
frame_len -= tlen;
remaining -= tlen;
if (frame_len &&
(skb_shinfo(fp_skb(fp))->nr_frags < FC_FRAME_SG_LEN))
continue;
if (!remaining)
f_ctl |= FC_FC_END_SEQ;
fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
FC_TYPE_FCP, f_ctl, fh_off);
error = lport->tt.seq_send(lport, cmd->seq, fp);
if (error) {
/* XXX For now, initiator will retry */
pr_err_ratelimited("%s: Failed to send frame %p, "
"xid <0x%x>, remaining %zu, "
"lso_max <0x%x>\n",
__func__, fp, ep->xid,
remaining, lport->lso_max);
}
}
return ft_queue_status(se_cmd);
}
/*
* Send read data back to initiator.
*/
int ft_send_read_data(struct scst_cmd *cmd)
{
struct ft_cmd *fcmd;
struct fc_frame *fp = NULL;
struct fc_exch *ep;
struct fc_lport *lport;
size_t remaining;
u32 fh_off = 0;
u32 frame_off;
size_t frame_len = 0;
size_t mem_len;
u32 mem_off;
size_t tlen;
struct page *page;
int use_sg;
int error;
void *to = NULL;
u8 *from = NULL;
int loop_limit = 10000;
fcmd = scst_cmd_get_tgt_priv(cmd);
ep = fc_seq_exch(fcmd->seq);
lport = ep->lp;
frame_off = fcmd->read_data_len;
tlen = scst_cmd_get_resp_data_len(cmd);
FT_IO_DBG("oid %x oxid %x resp_len %zd frame_off %u\n",
ep->oid, ep->oxid, tlen, frame_off);
if (tlen <= frame_off)
return SCST_TGT_RES_SUCCESS;
remaining = tlen - frame_off;
if (remaining > UINT_MAX)
FT_ERR("oid %x oxid %x resp_len %zd frame_off %u\n",
ep->oid, ep->oxid, tlen, frame_off);
mem_len = scst_get_buf_first(cmd, &from);
mem_off = 0;
if (!mem_len) {
FT_IO_DBG("mem_len 0\n");
return SCST_TGT_RES_SUCCESS;
}
FT_IO_DBG("sid %x oxid %x mem_len %zd frame_off %u remaining %zd\n",
ep->sid, ep->oxid, mem_len, frame_off, remaining);
/*
* If we've already transferred some of the data, skip through
* the buffer over the data already sent and continue with the
* same sequence. Otherwise, get a new sequence for the data.
*/
if (frame_off) {
tlen = frame_off;
while (mem_len <= tlen) {
tlen -= mem_len;
scst_put_buf(cmd, from);
mem_len = scst_get_buf_next(cmd, &from);
if (!mem_len)
return SCST_TGT_RES_SUCCESS;
}
mem_len -= tlen;
mem_off = tlen;
} else
fcmd->seq = lport->tt.seq_start_next(fcmd->seq);
/* no scatter/gather in skb for odd word length due to fc_seq_send() */
use_sg = !(remaining % 4) && lport->sg_supp;
while (remaining) {
if (!loop_limit) {
FT_ERR("hit loop limit. remaining %zx mem_len %zx "
"frame_len %zx tlen %zx\n",
remaining, mem_len, frame_len, tlen);
break;
}
loop_limit--;
if (!mem_len) {
scst_put_buf(cmd, from);
mem_len = scst_get_buf_next(cmd, &from);
mem_off = 0;
if (!mem_len) {
FT_ERR("mem_len 0 from get_buf_next\n");
break;
}
}
if (!frame_len) {
frame_len = fcmd->max_lso_payload;
frame_len = min(frame_len, remaining);
fp = fc_frame_alloc(lport, use_sg ? 0 : frame_len);
if (!fp) {
FT_IO_DBG("frame_alloc failed. "
"use_sg %d frame_len %zd\n",
use_sg, frame_len);
break;
}
fr_max_payload(fp) = fcmd->max_payload;
to = fc_frame_payload_get(fp, 0);
fh_off = frame_off;
}
tlen = min(mem_len, frame_len);
BUG_ON(!tlen);
BUG_ON(tlen > remaining);
BUG_ON(tlen > mem_len);
BUG_ON(tlen > frame_len);
if (use_sg) {
page = virt_to_page(from + mem_off);
get_page(page);
tlen = min_t(size_t, tlen,
PAGE_SIZE - (mem_off & ~PAGE_MASK));
skb_fill_page_desc(fp_skb(fp),
skb_shinfo(fp_skb(fp))->nr_frags,
page, offset_in_page(from + mem_off),
tlen);
fr_len(fp) += tlen;
fp_skb(fp)->data_len += tlen;
fp_skb(fp)->truesize +=
PAGE_SIZE << compound_order(page);
frame_len -= tlen;
if (skb_shinfo(fp_skb(fp))->nr_frags >= FC_FRAME_SG_LEN)
frame_len = 0;
} else {
memcpy(to, from + mem_off, tlen);
to += tlen;
frame_len -= tlen;
}
mem_len -= tlen;
mem_off += tlen;
remaining -= tlen;
frame_off += tlen;
if (frame_len)
continue;
fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
FC_TYPE_FCP,
remaining ? (FC_FC_EX_CTX | FC_FC_REL_OFF) :
(FC_FC_EX_CTX | FC_FC_REL_OFF | FC_FC_END_SEQ),
fh_off);
error = lport->tt.seq_send(lport, fcmd->seq, fp);
if (error) {
WARN_ON(1);
/* XXX For now, initiator will retry */
} else
fcmd->read_data_len = frame_off;
}
if (mem_len)
scst_put_buf(cmd, from);
if (remaining) {
FT_IO_DBG("remaining read data %zd\n", remaining);
return SCST_TGT_RES_QUEUE_FULL;
}
return SCST_TGT_RES_SUCCESS;
}
static bool ftmac100_rx_packet(struct ftmac100 *priv, int *processed)
{
struct net_device *netdev = priv->netdev;
struct ftmac100_rxdes *rxdes;
struct sk_buff *skb;
struct page *page;
dma_addr_t map;
int length;
rxdes = ftmac100_rx_locate_first_segment(priv);
if (!rxdes)
return false;
if (unlikely(ftmac100_rx_packet_error(priv, rxdes))) {
ftmac100_rx_drop_packet(priv);
return true;
}
/*
*/
if (unlikely(!ftmac100_rxdes_last_segment(rxdes)))
BUG();
/* */
skb = netdev_alloc_skb_ip_align(netdev, 128);
if (unlikely(!skb)) {
if (net_ratelimit())
netdev_err(netdev, "rx skb alloc failed\n");
ftmac100_rx_drop_packet(priv);
return true;
}
if (unlikely(ftmac100_rxdes_multicast(rxdes)))
netdev->stats.multicast++;
map = ftmac100_rxdes_get_dma_addr(rxdes);
dma_unmap_page(priv->dev, map, RX_BUF_SIZE, DMA_FROM_DEVICE);
length = ftmac100_rxdes_frame_length(rxdes);
page = ftmac100_rxdes_get_page(rxdes);
skb_fill_page_desc(skb, 0, page, 0, length);
skb->len += length;
skb->data_len += length;
/* */
if (length > 64)
skb->truesize += PAGE_SIZE;
__pskb_pull_tail(skb, min(length, 64));
ftmac100_alloc_rx_page(priv, rxdes, GFP_ATOMIC);
ftmac100_rx_pointer_advance(priv);
skb->protocol = eth_type_trans(skb, netdev);
netdev->stats.rx_packets++;
netdev->stats.rx_bytes += skb->len;
/* */
netif_receive_skb(skb);
(*processed)++;
return true;
}
int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
int hlimit, struct ipv6_txoptions *opt, struct flowi *fl, struct rt6_info *rt,
unsigned int flags)
{
struct inet_opt *inet = inet_sk(sk);
struct ipv6_pinfo *np = inet6_sk(sk);
struct sk_buff *skb;
unsigned int maxfraglen, fragheaderlen;
int exthdrlen;
int hh_len;
int mtu;
int copy = 0;
int err;
int offset = 0;
int csummode = CHECKSUM_NONE;
if (flags&MSG_PROBE)
return 0;
if (skb_queue_empty(&sk->write_queue)) {
/*
* setup for corking
*/
if (opt) {
if (np->cork.opt == NULL)
np->cork.opt = kmalloc(opt->tot_len, sk->allocation);
memcpy(np->cork.opt, opt, opt->tot_len);
inet->cork.flags |= IPCORK_OPT;
/* need source address above miyazawa*/
}
dst_hold(&rt->u.dst);
np->cork.rt = rt;
np->cork.fl = fl;
np->cork.hop_limit = hlimit;
inet->cork.fragsize = mtu = dst_pmtu(&rt->u.dst);
inet->cork.length = 0;
inet->sndmsg_page = NULL;
inet->sndmsg_off = 0;
exthdrlen = rt->u.dst.header_len + (opt ? opt->opt_flen : 0);
length += exthdrlen;
transhdrlen += exthdrlen;
} else {
rt = np->cork.rt;
if (inet->cork.flags & IPCORK_OPT)
opt = np->cork.opt;
transhdrlen = 0;
exthdrlen = 0;
mtu = inet->cork.fragsize;
}
hh_len = (rt->u.dst.dev->hard_header_len&~15) + 16;
fragheaderlen = sizeof(struct ipv6hdr) + (opt ? opt->opt_nflen : 0);
maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
if (inet->cork.length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
ipv6_local_error(sk, EMSGSIZE, fl, mtu-exthdrlen);
return -EMSGSIZE;
}
}
inet->cork.length += length;
if ((skb = skb_peek_tail(&sk->write_queue)) == NULL)
goto alloc_new_skb;
while (length > 0) {
if ((copy = maxfraglen - skb->len) <= 0) {
char *data;
unsigned int datalen;
unsigned int fraglen;
unsigned int alloclen;
BUG_TRAP(copy == 0);
alloc_new_skb:
datalen = maxfraglen - fragheaderlen;
if (datalen > length)
datalen = length;
fraglen = datalen + fragheaderlen;
if ((flags & MSG_MORE) &&
!(rt->u.dst.dev->features&NETIF_F_SG))
alloclen = maxfraglen;
else
alloclen = fraglen;
alloclen += sizeof(struct frag_hdr);
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len + 15,
(flags & MSG_DONTWAIT), &err);
} else {
skb = NULL;
if (atomic_read(&sk->wmem_alloc) <= 2*sk->sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len + 15, 1,
sk->allocation);
if (unlikely(skb == NULL))
err = -ENOBUFS;
}
if (skb == NULL)
goto error;
/*
* Fill in the control structures
*/
skb->ip_summed = csummode;
skb->csum = 0;
/* reserve 8 byte for fragmentation */
skb_reserve(skb, hh_len+sizeof(struct frag_hdr));
/*
* Find where to start putting bytes
*/
data = skb_put(skb, fraglen);
skb->nh.raw = data + exthdrlen;
data += fragheaderlen;
skb->h.raw = data + exthdrlen;
copy = datalen - transhdrlen;
if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, 0, skb) < 0) {
err = -EFAULT;
kfree_skb(skb);
goto error;
}
offset += copy;
length -= datalen;
transhdrlen = 0;
exthdrlen = 0;
csummode = CHECKSUM_NONE;
/*
* Put the packet on the pending queue
*/
__skb_queue_tail(&sk->write_queue, skb);
continue;
}
if (copy > length)
copy = length;
if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
unsigned int off;
off = skb->len;
if (getfrag(from, skb_put(skb, copy),
offset, copy, off, skb) < 0) {
__skb_trim(skb, off);
err = -EFAULT;
goto error;
}
} else {
int i = skb_shinfo(skb)->nr_frags;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
struct page *page = inet->sndmsg_page;
int off = inet->sndmsg_off;
unsigned int left;
if (page && (left = PAGE_SIZE - off) > 0) {
if (copy >= left)
copy = left;
if (page != frag->page) {
if (i == MAX_SKB_FRAGS) {
err = -EMSGSIZE;
goto error;
}
get_page(page);
skb_fill_page_desc(skb, i, page, inet->sndmsg_off, 0);
frag = &skb_shinfo(skb)->frags[i];
}
} else if(i < MAX_SKB_FRAGS) {
if (copy > PAGE_SIZE)
copy = PAGE_SIZE;
page = alloc_pages(sk->allocation, 0);
if (page == NULL) {
err = -ENOMEM;
goto error;
}
inet->sndmsg_page = page;
inet->sndmsg_off = 0;
skb_fill_page_desc(skb, i, page, 0, 0);
frag = &skb_shinfo(skb)->frags[i];
skb->truesize += PAGE_SIZE;
atomic_add(PAGE_SIZE, &sk->wmem_alloc);
} else {
err = -EMSGSIZE;
goto error;
}
if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
err = -EFAULT;
goto error;
}
inet->sndmsg_off += copy;
frag->size += copy;
skb->len += copy;
skb->data_len += copy;
}
offset += copy;
length -= copy;
}
return 0;
error:
inet->cork.length -= length;
IP6_INC_STATS(Ip6OutDiscards);
return err;
}
static void
aoecmd_ata_rw(struct aoedev *d, struct frame *f)
{
struct aoe_hdr *h;
struct aoe_atahdr *ah;
struct buf *buf;
struct sk_buff *skb;
ulong bcnt;
register sector_t sector;
char writebit, extbit;
writebit = 0x10;
extbit = 0x4;
buf = d->inprocess;
sector = buf->sector;
bcnt = buf->bv_resid;
if (bcnt > d->maxbcnt)
bcnt = d->maxbcnt;
/* initialize the headers & frame */
skb = f->skb;
h = aoe_hdr(skb);
ah = (struct aoe_atahdr *) (h+1);
skb_put(skb, sizeof *h + sizeof *ah);
memset(h, 0, skb->len);
f->tag = aoehdr_atainit(d, h);
f->waited = 0;
f->buf = buf;
f->bufaddr = buf->bufaddr;
f->bcnt = bcnt;
f->lba = sector;
/* set up ata header */
ah->scnt = bcnt >> 9;
put_lba(ah, sector);
if (d->flags & DEVFL_EXT) {
ah->aflags |= AOEAFL_EXT;
} else {
extbit = 0;
ah->lba3 &= 0x0f;
ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */
}
if (bio_data_dir(buf->bio) == WRITE) {
skb_fill_page_desc(skb, 0, virt_to_page(f->bufaddr),
offset_in_page(f->bufaddr), bcnt);
ah->aflags |= AOEAFL_WRITE;
skb->len += bcnt;
skb->data_len = bcnt;
} else {
writebit = 0;
}
ah->cmdstat = WIN_READ | writebit | extbit;
/* mark all tracking fields and load out */
buf->nframesout += 1;
buf->bufaddr += bcnt;
buf->bv_resid -= bcnt;
/* printk(KERN_DEBUG "aoe: bv_resid=%ld\n", buf->bv_resid); */
buf->resid -= bcnt;
buf->sector += bcnt >> 9;
if (buf->resid == 0) {
d->inprocess = NULL;
} else if (buf->bv_resid == 0) {
buf->bv++;
WARN_ON(buf->bv->bv_len == 0);
buf->bv_resid = buf->bv->bv_len;
buf->bufaddr = page_address(buf->bv->bv_page) + buf->bv->bv_offset;
}
skb->dev = d->ifp;
skb = skb_clone(skb, GFP_ATOMIC);
if (skb == NULL)
return;
if (d->sendq_hd)
d->sendq_tl->next = skb;
else
d->sendq_hd = skb;
d->sendq_tl = skb;
}
int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
unsigned int count)
{
struct iscsi_conn *conn = task->conn;
struct scsi_cmnd *sc = task->sc;
struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgb3i_task_data *tdata = tcp_task->dd_data;
struct sk_buff *skb = tdata->skb;
unsigned int datalen = count;
int i, padlen = iscsi_padding(count);
struct page *pg;
char *dbuf = sc ? sc->request_buffer : task->data;
cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
task, task->sc, offset, count, skb);
skb_put(skb, task->hdr_len);
tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
if (!count)
return 0;
tdata->offset = offset;
tdata->count = count;
if (sc && sc->use_sg) {
struct scatterlist *sg = NULL;
int err;
tdata->offset = offset;
tdata->count = count;
err = sgl_seek_offset(sc->request_buffer, sc->use_sg,
tdata->offset, &tdata->sgoffset, &sg);
if (err < 0) {
cxgb3i_log_warn("tpdu, sgl %u, bad offset %u/%u.\n",
sc->use_sg, tdata->offset,
sc->request_bufflen);
return err;
}
err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
tdata->frags, MAX_PDU_FRAGS);
if (err < 0) {
cxgb3i_log_warn("tpdu, sgl %u, bad offset %u + %u.\n",
sc->use_sg, tdata->offset,
sc->request_bufflen);
return err;
}
tdata->nr_frags = err;
if (tdata->nr_frags > MAX_SKB_FRAGS ||
(padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
char *dst = skb->data + task->hdr_len;
skb_frag_t *frag = tdata->frags;
/* data fits in the skb's headroom */
for (i = 0; i < tdata->nr_frags; i++, frag++) {
char *src = kmap_atomic(frag->page,
KM_SOFTIRQ0);
memcpy(dst, src+frag->page_offset, frag->size);
dst += frag->size;
kunmap_atomic(src, KM_SOFTIRQ0);
}
if (padlen) {
memset(dst, 0, padlen);
padlen = 0;
}
skb_put(skb, count + padlen);
} else {
/* data fit into frag_list */
for (i = 0; i < tdata->nr_frags; i++)
get_page(tdata->frags[i].page);
memcpy(skb_shinfo(skb)->frags, tdata->frags,
sizeof(skb_frag_t) * tdata->nr_frags);
skb_shinfo(skb)->nr_frags = tdata->nr_frags;
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
} else {
pg = virt_to_page(dbuf);
get_page(pg);
skb_fill_page_desc(skb, 0, pg, offset_in_page(dbuf) + offset,
count);
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
if (padlen) {
i = skb_shinfo(skb)->nr_frags;
get_page(pad_page);
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, pad_page, 0,
padlen);
skb->data_len += padlen;
skb->truesize += padlen;
skb->len += padlen;
}
return 0;
}
static bool ftmac100_rx_packet(struct ftmac100 *priv, int *processed)
{
struct net_device *netdev = priv->netdev;
struct ftmac100_rxdes *rxdes;
struct sk_buff *skb;
struct page *page;
dma_addr_t map;
int length;
rxdes = ftmac100_rx_locate_first_segment(priv);
if (!rxdes)
return false;
if (unlikely(ftmac100_rx_packet_error(priv, rxdes))) {
ftmac100_rx_drop_packet(priv);
return true;
}
/*
* It is impossible to get multi-segment packets
* because we always provide big enough receive buffers.
*/
if (unlikely(!ftmac100_rxdes_last_segment(rxdes)))
BUG();
/* start processing */
skb = netdev_alloc_skb_ip_align(netdev, 128);
if (unlikely(!skb)) {
if (net_ratelimit())
netdev_err(netdev, "rx skb alloc failed\n");
ftmac100_rx_drop_packet(priv);
return true;
}
if (unlikely(ftmac100_rxdes_multicast(rxdes)))
netdev->stats.multicast++;
map = ftmac100_rxdes_get_dma_addr(rxdes);
dma_unmap_page(priv->dev, map, RX_BUF_SIZE, DMA_FROM_DEVICE);
length = ftmac100_rxdes_frame_length(rxdes);
page = ftmac100_rxdes_get_page(rxdes);
skb_fill_page_desc(skb, 0, page, 0, length);
skb->len += length;
skb->data_len += length;
if (length > 128) {
skb->truesize += PAGE_SIZE;
/* We pull the minimum amount into linear part */
__pskb_pull_tail(skb, ETH_HLEN);
} else {
/* Small frames are copied into linear part to free one page */
__pskb_pull_tail(skb, length);
}
ftmac100_alloc_rx_page(priv, rxdes, GFP_ATOMIC);
ftmac100_rx_pointer_advance(priv);
skb->protocol = eth_type_trans(skb, netdev);
netdev->stats.rx_packets++;
netdev->stats.rx_bytes += skb->len;
/* push packet to protocol stack */
netif_receive_skb(skb);
(*processed)++;
return true;
}