static ssize_t
_usdf_dgram_send_iov_copy(struct usdf_ep *ep, struct usd_dest *dest,
const struct iovec *iov, size_t count, void *context,
uint8_t cq_entry)
{
struct usd_wq *wq;
struct usd_qp_impl *qp;
struct usd_udp_hdr *hdr;
uint32_t last_post;
size_t len;
unsigned i;
qp = to_qpi(ep->e.dg.ep_qp);
wq = &qp->uq_wq;
hdr = _usdf_find_hdr(wq);
memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
len = 0;
for (i = 0; i < count; i++) {
memcpy((char *) hdr + sizeof(*hdr) + len, iov[i].iov_base,
iov[i].iov_len);
len += iov[i].iov_len;
}
_usdf_adjust_hdr(hdr, qp, len);
last_post = _usd_post_send_one(wq, hdr, len + sizeof(*hdr), cq_entry);
_usdf_adjust_post_info(wq, last_post, context, len);
return 0;
}
ssize_t
usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, void *context)
{
struct usd_udp_hdr *hdr;
struct usd_qp_impl *qp;
struct usdf_dest *dest;
struct usdf_ep *ep;
struct usd_wq *wq;
uint32_t last_post;
uint32_t flags;
size_t padding;
ep = ep_ftou(fep);
dest = (struct usdf_dest *)(uintptr_t) dest_addr;
padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
flags = (ep->ep_tx_completion) ? USD_SF_SIGNAL : 0;
if (ep->e.dg.tx_op_flags & FI_INJECT) {
if ((len - padding) > USD_SEND_MAX_COPY) {
USDF_DBG_SYS(EP_DATA,
"given inject length (%zu) exceeds max inject length (%d)\n",
len, USD_SEND_MAX_COPY);
return -FI_ENOSPC;
}
return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest,
buf + USDF_HDR_BUF_ENTRY, len -
USDF_HDR_BUF_ENTRY, flags,
context);
}
qp = to_qpi(ep->e.dg.ep_qp);
wq = &qp->uq_wq;
hdr = (struct usd_udp_hdr *) ((char *) buf + padding);
memcpy(hdr, &dest->ds_dest.ds_dest.ds_udp.u_hdr, sizeof(*hdr));
_usdf_adjust_prefix_hdr(hdr, qp, len, padding);
last_post = _usd_post_send_one(wq, hdr, len - padding,
ep->ep_tx_completion);
_usdf_adjust_post_info(wq, last_post, context, len - USDF_HDR_BUF_ENTRY);
return FI_SUCCESS;
}
ssize_t
usdf_dgram_inject(struct fid_ep *fep, const void *buf, size_t len,
fi_addr_t dest_addr)
{
struct usdf_ep *ep;
struct usdf_dest *dest;
struct usd_wq *wq;
struct usd_qp_impl *qp;
struct usd_udp_hdr *hdr;
uint32_t last_post;
struct usd_wq_post_info *info;
uint8_t *copybuf;
if (len + sizeof(struct usd_udp_hdr) > USD_SEND_MAX_COPY) {
return -FI_ENOSPC;
}
ep = ep_ftou(fep);
dest = (struct usdf_dest *)(uintptr_t)dest_addr;
qp = to_qpi(ep->e.dg.ep_qp);
wq = &qp->uq_wq;
copybuf = wq->uwq_copybuf +
wq->uwq_post_index * USD_SEND_MAX_COPY;
hdr = (struct usd_udp_hdr *)copybuf;
memcpy(hdr, &dest->ds_dest.ds_dest.ds_udp.u_hdr, sizeof(*hdr));
hdr->uh_udp.source =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
hdr->uh_ip.tot_len = htons(len + sizeof(*hdr)
- sizeof(struct ether_header));
hdr->uh_udp.len = htons(len + sizeof(*hdr) -
sizeof(struct ether_header) -
sizeof(struct iphdr));
memcpy(hdr + 1, buf, len);
last_post = _usd_post_send_one(wq, hdr, len + sizeof(*hdr), 1);
info = &wq->uwq_post_info[last_post];
info->wp_context = NULL;
info->wp_len = len;
return 0;
}
static inline void
usdf_msg_send_ack(struct usdf_tx *tx, struct usdf_ep *ep)
{
struct rudp_pkt *hdr;
struct usd_wq *wq;
uint32_t last_post;
struct usd_wq_post_info *info;
uint16_t seq;
wq = &(to_qpi(tx->tx_qp)->uq_wq);
hdr = (struct rudp_pkt *) (wq->uwq_copybuf +
wq->uwq_post_index * USD_SEND_MAX_COPY);
memcpy(hdr, &ep->e.msg.ep_dest->ds_dest.ds_udp.u_hdr,
sizeof(struct usd_udp_hdr));
hdr->msg.src_peer_id = htons(ep->e.msg.ep_lcl_peer_id);
if (ep->e.msg.ep_send_nak) {
hdr->msg.opcode = htons(RUDP_OP_NAK);
seq = ep->e.msg.ep_next_rx_seq;
hdr->msg.m.nak.nak_seq = htons(seq);
ep->e.msg.ep_send_nak = 0;
} else {
hdr->msg.opcode = htons(RUDP_OP_ACK);
seq = ep->e.msg.ep_next_rx_seq - 1;
hdr->msg.m.ack.ack_seq = htons(seq);
}
/* add packet lengths */
hdr->hdr.uh_ip.tot_len = htons(
sizeof(struct rudp_pkt) -
sizeof(struct ether_header));
hdr->hdr.uh_udp.len = htons(sizeof(struct rudp_pkt) -
sizeof(struct ether_header) - sizeof(struct iphdr));
last_post = _usd_post_send_one(wq, hdr, sizeof(*hdr), 1);
info = &wq->uwq_post_info[last_post];
info->wp_context = tx;
info->wp_len = 0;
}
static ssize_t
_usdf_dgram_send_iov_copy(struct usdf_ep *ep, struct usd_dest *dest,
const struct iovec *iov, size_t count, void *context)
{
struct usd_wq *wq;
struct usd_qp_impl *qp;
struct usd_udp_hdr *hdr;
uint32_t last_post;
struct usd_wq_post_info *info;
uint8_t *copybuf;
size_t len;
unsigned i;
qp = to_qpi(ep->e.dg.ep_qp);
wq = &qp->uq_wq;
copybuf = wq->uwq_copybuf +
wq->uwq_post_index * USD_SEND_MAX_COPY;
hdr = (struct usd_udp_hdr *)copybuf;
memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
hdr->uh_udp.source =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
len = sizeof(*hdr);
for (i = 0; i < count; i++) {
memcpy(copybuf + len, iov[i].iov_base, iov[i].iov_len);
len += iov[i].iov_len;
}
/* adjust lengths */
hdr->uh_ip.tot_len = htons(len - sizeof(struct ether_header));
hdr->uh_udp.len = htons(len - sizeof(struct ether_header) -
sizeof(struct iphdr));
last_post = _usd_post_send_one(wq, hdr, len, 1);
info = &wq->uwq_post_info[last_post];
info->wp_context = context;
info->wp_len = len;
return 0;
}
ssize_t
usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, void *context)
{
struct usdf_ep *ep;
struct usd_dest *dest;
struct usd_qp_impl *qp;
struct usd_udp_hdr *hdr;
struct usd_wq *wq;
uint32_t last_post;
struct usd_wq_post_info *info;
ep = ep_ftou(fep);
dest = (struct usd_dest *)(uintptr_t)dest_addr;
qp = to_qpi(ep->e.dg.ep_qp);
wq = &qp->uq_wq;
hdr = (struct usd_udp_hdr *) buf - 1;
memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
/* adjust lengths and insert source port */
hdr->uh_ip.tot_len = htons(len + sizeof(struct usd_udp_hdr) -
sizeof(struct ether_header));
hdr->uh_udp.len = htons((sizeof(struct usd_udp_hdr) -
sizeof(struct ether_header) -
sizeof(struct iphdr)) + len);
hdr->uh_udp.source =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
last_post = _usd_post_send_one(wq, hdr,
len + sizeof(struct usd_udp_hdr), 1);
info = &wq->uwq_post_info[last_post];
info->wp_context = context;
info->wp_len = len;
return 0;
}
static inline void
usdf_msg_send_segment(struct usdf_tx *tx, struct usdf_ep *ep)
{
struct usdf_msg_qe *msg;
struct rudp_pkt *hdr;
struct usd_wq *wq;
uint32_t index;
size_t cur_iov;
size_t cur_resid;
size_t resid;
const uint8_t *cur_ptr;
const uint8_t *send_ptr;
size_t sge_len;
uint8_t *ptr;
struct usd_wq_post_info *info;
msg = TAILQ_FIRST(&ep->e.msg.ep_posted_wqe);
wq = &(to_qpi(tx->tx_qp)->uq_wq);
index = wq->uwq_post_index;
hdr = (struct rudp_pkt *)(wq->uwq_copybuf + index * USD_SEND_MAX_COPY);
memcpy(hdr, &ep->e.msg.ep_dest->ds_dest.ds_udp.u_hdr,
sizeof(struct usd_udp_hdr));
hdr->msg.src_peer_id = htons(ep->e.msg.ep_lcl_peer_id);
resid = msg->ms_resid;
cur_iov = msg->ms_cur_iov;
cur_ptr = msg->ms_cur_ptr;
cur_resid = msg->ms_iov_resid;
/* save first seq for message */
if (cur_iov == 0 && cur_resid == msg->ms_iov[0].iov_len) {
msg->ms_first_seq = ep->e.msg.ep_next_tx_seq;
}
if (resid < USD_SEND_MAX_COPY - sizeof(*hdr)) {
hdr->msg.opcode = htons(RUDP_OP_LAST);
hdr->msg.m.rc_data.length = htons(resid);
hdr->msg.m.rc_data.seqno = htons(ep->e.msg.ep_next_tx_seq);
++ep->e.msg.ep_next_tx_seq;
sge_len = resid;
ptr = (uint8_t *)(hdr + 1);
while (resid > 0) {
memcpy(ptr, cur_ptr, cur_resid);
ptr += cur_resid;
resid -= cur_resid;
++cur_iov;
cur_ptr = msg->ms_iov[cur_iov].iov_base;
cur_resid = msg->ms_iov[cur_iov].iov_len;
}
/* add packet lengths */
hdr->hdr.uh_ip.tot_len = htons(
sge_len + sizeof(struct rudp_pkt) -
sizeof(struct ether_header));
hdr->hdr.uh_udp.len = htons(
(sizeof(struct rudp_pkt) -
sizeof(struct ether_header) -
sizeof(struct iphdr)) + sge_len);
index = _usd_post_send_one(wq, hdr,
sge_len + sizeof(*hdr), 1);
} else {
struct vnic_wq *vwq;
u_int8_t offload_mode = 0, eop;
u_int16_t mss = 7, header_length = 0, vlan_tag = 0;
u_int8_t vlan_tag_insert = 0, loopback = 0, fcoe_encap = 0;
struct wq_enet_desc *desc;
size_t space;
size_t num_sge;
size_t sent;
vwq = &wq->uwq_vnic_wq;
desc = wq->uwq_next_desc;
space = ep->ep_domain->dom_fabric->fab_dev_attrs->uda_mtu -
sizeof(*hdr);
num_sge = 1;
/* encode header desc */
eop = 0;
wq_enet_desc_enc(desc, (uintptr_t)hdr, sizeof(*hdr),
mss, header_length, offload_mode, eop, 0, fcoe_encap,
vlan_tag_insert, vlan_tag, loopback);
do {
desc = (struct wq_enet_desc *)
((uintptr_t)wq->uwq_desc_ring + (index << 4));
index = (index + 1) & wq->uwq_post_index_mask;
send_ptr = cur_ptr;
if (cur_resid >= space) {
sge_len = space;
eop = 1;
cur_resid -= sge_len;
cur_ptr += sge_len;
} else {
sge_len = cur_resid;
if (num_sge == USDF_MSG_MAX_SGE ||
cur_resid == resid) {
eop = 1;
}
++cur_iov;
cur_ptr = msg->ms_iov[cur_iov].iov_base;
cur_resid = msg->ms_iov[cur_iov].iov_len;
}
wq_enet_desc_enc(desc, (uintptr_t)send_ptr, sge_len,
mss, header_length, offload_mode, eop, eop,
fcoe_encap, vlan_tag_insert,
vlan_tag, loopback);
++num_sge;
space -= sge_len;
resid -= sge_len;
} while (space > 0 && num_sge <= USDF_MSG_MAX_SGE && resid > 0);
/* add packet lengths */
sent = ep->ep_domain->dom_fabric->fab_dev_attrs->uda_mtu -
sizeof(*hdr) - space;
hdr->hdr.uh_ip.tot_len = htons(
sent + sizeof(struct rudp_pkt) -
sizeof(struct ether_header));
hdr->hdr.uh_udp.len = htons(
(sizeof(struct rudp_pkt) -
sizeof(struct ether_header) -
sizeof(struct iphdr)) + sent);
#if 0
if ((random() % 177) == 0 && resid == 0) {
hdr->hdr.uh_eth.ether_type = 0;
//printf("BORK seq %u\n", ep->e.msg.ep_next_tx_seq);
}
#endif
if (resid == 0) {
hdr->msg.opcode = htons(RUDP_OP_LAST);
} else {
hdr->msg.opcode = htons(RUDP_OP_FIRST);
}
hdr->msg.m.rc_data.length = htons(sent);
hdr->msg.m.rc_data.seqno = htons(ep->e.msg.ep_next_tx_seq);
++ep->e.msg.ep_next_tx_seq;
wmb();
iowrite64(index, &vwq->ctrl->posted_index);
wq->uwq_next_desc = (struct wq_enet_desc *)
((uintptr_t)wq->uwq_desc_ring + (index << 4));
wq->uwq_post_index = (index + 1) & wq->uwq_post_index_mask;
wq->uwq_send_credits -= num_sge;
}
info = &wq->uwq_post_info[index];
info->wp_context = tx;
info->wp_len = sge_len;
/* If send complete, remove from send list */
if (resid == 0) {
usdf_msg_send_complete(ep, msg);
} else {
msg->ms_resid = resid;
msg->ms_iov_resid = cur_resid;
msg->ms_cur_iov = cur_iov;
msg->ms_cur_ptr = cur_ptr;
}
/* set ACK timer */
usdf_timer_set(ep->ep_domain->dom_fabric, ep->e.msg.ep_ack_timer,
USDF_RUDP_ACK_TIMEOUT);
}
usd_post_send_one_prefixed_raw_normal(
struct usd_qp *uqp,
struct usd_dest
__attribute__ ((unused)) * dest,
const void *buf,
size_t len,
uint32_t flags,
void *context)
{
struct usd_qp_impl *qp;
struct usd_wq *wq;
uint32_t last_post;
struct usd_wq_post_info *info;
qp = to_qpi(uqp);
wq = &qp->uq_wq;
last_post =
_usd_post_send_one(wq, buf, len, USD_SF_ISSET(flags, SIGNAL));
info = &wq->uwq_post_info[last_post];
info->wp_context = context;
info->wp_len = len;
return 0;
}
struct usd_qp_ops usd_qp_ops_raw_normal = {
.qo_post_send_one_prefixed = usd_post_send_one_prefixed_raw_normal,
};
