static ssize_t _usdf_dgram_send_iov_copy(struct usdf_ep *ep, struct usd_dest *dest, const struct iovec *iov, size_t count, void *context, uint8_t cq_entry) { struct usd_wq *wq; struct usd_qp_impl *qp; struct usd_udp_hdr *hdr; uint32_t last_post; size_t len; unsigned i; qp = to_qpi(ep->e.dg.ep_qp); wq = &qp->uq_wq; hdr = _usdf_find_hdr(wq); memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr)); len = 0; for (i = 0; i < count; i++) { memcpy((char *) hdr + sizeof(*hdr) + len, iov[i].iov_base, iov[i].iov_len); len += iov[i].iov_len; } _usdf_adjust_hdr(hdr, qp, len); last_post = _usd_post_send_one(wq, hdr, len + sizeof(*hdr), cq_entry); _usdf_adjust_post_info(wq, last_post, context, len); return 0; }
ssize_t usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, void *context) { struct usd_udp_hdr *hdr; struct usd_qp_impl *qp; struct usdf_dest *dest; struct usdf_ep *ep; struct usd_wq *wq; uint32_t last_post; uint32_t flags; size_t padding; ep = ep_ftou(fep); dest = (struct usdf_dest *)(uintptr_t) dest_addr; padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr); flags = (ep->ep_tx_completion) ? USD_SF_SIGNAL : 0; if (ep->e.dg.tx_op_flags & FI_INJECT) { if ((len - padding) > USD_SEND_MAX_COPY) { USDF_DBG_SYS(EP_DATA, "given inject length (%zu) exceeds max inject length (%d)\n", len, USD_SEND_MAX_COPY); return -FI_ENOSPC; } return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest, buf + USDF_HDR_BUF_ENTRY, len - USDF_HDR_BUF_ENTRY, flags, context); } qp = to_qpi(ep->e.dg.ep_qp); wq = &qp->uq_wq; hdr = (struct usd_udp_hdr *) ((char *) buf + padding); memcpy(hdr, &dest->ds_dest.ds_dest.ds_udp.u_hdr, sizeof(*hdr)); _usdf_adjust_prefix_hdr(hdr, qp, len, padding); last_post = _usd_post_send_one(wq, hdr, len - padding, ep->ep_tx_completion); _usdf_adjust_post_info(wq, last_post, context, len - USDF_HDR_BUF_ENTRY); return FI_SUCCESS; }
ssize_t usdf_dgram_inject(struct fid_ep *fep, const void *buf, size_t len, fi_addr_t dest_addr) { struct usdf_ep *ep; struct usdf_dest *dest; struct usd_wq *wq; struct usd_qp_impl *qp; struct usd_udp_hdr *hdr; uint32_t last_post; struct usd_wq_post_info *info; uint8_t *copybuf; if (len + sizeof(struct usd_udp_hdr) > USD_SEND_MAX_COPY) { return -FI_ENOSPC; } ep = ep_ftou(fep); dest = (struct usdf_dest *)(uintptr_t)dest_addr; qp = to_qpi(ep->e.dg.ep_qp); wq = &qp->uq_wq; copybuf = wq->uwq_copybuf + wq->uwq_post_index * USD_SEND_MAX_COPY; hdr = (struct usd_udp_hdr *)copybuf; memcpy(hdr, &dest->ds_dest.ds_dest.ds_udp.u_hdr, sizeof(*hdr)); hdr->uh_udp.source = qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port; hdr->uh_ip.tot_len = htons(len + sizeof(*hdr) - sizeof(struct ether_header)); hdr->uh_udp.len = htons(len + sizeof(*hdr) - sizeof(struct ether_header) - sizeof(struct iphdr)); memcpy(hdr + 1, buf, len); last_post = _usd_post_send_one(wq, hdr, len + sizeof(*hdr), 1); info = &wq->uwq_post_info[last_post]; info->wp_context = NULL; info->wp_len = len; return 0; }
static inline void usdf_msg_send_ack(struct usdf_tx *tx, struct usdf_ep *ep) { struct rudp_pkt *hdr; struct usd_wq *wq; uint32_t last_post; struct usd_wq_post_info *info; uint16_t seq; wq = &(to_qpi(tx->tx_qp)->uq_wq); hdr = (struct rudp_pkt *) (wq->uwq_copybuf + wq->uwq_post_index * USD_SEND_MAX_COPY); memcpy(hdr, &ep->e.msg.ep_dest->ds_dest.ds_udp.u_hdr, sizeof(struct usd_udp_hdr)); hdr->msg.src_peer_id = htons(ep->e.msg.ep_lcl_peer_id); if (ep->e.msg.ep_send_nak) { hdr->msg.opcode = htons(RUDP_OP_NAK); seq = ep->e.msg.ep_next_rx_seq; hdr->msg.m.nak.nak_seq = htons(seq); ep->e.msg.ep_send_nak = 0; } else { hdr->msg.opcode = htons(RUDP_OP_ACK); seq = ep->e.msg.ep_next_rx_seq - 1; hdr->msg.m.ack.ack_seq = htons(seq); } /* add packet lengths */ hdr->hdr.uh_ip.tot_len = htons( sizeof(struct rudp_pkt) - sizeof(struct ether_header)); hdr->hdr.uh_udp.len = htons(sizeof(struct rudp_pkt) - sizeof(struct ether_header) - sizeof(struct iphdr)); last_post = _usd_post_send_one(wq, hdr, sizeof(*hdr), 1); info = &wq->uwq_post_info[last_post]; info->wp_context = tx; info->wp_len = 0; }
static ssize_t _usdf_dgram_send_iov_copy(struct usdf_ep *ep, struct usd_dest *dest, const struct iovec *iov, size_t count, void *context) { struct usd_wq *wq; struct usd_qp_impl *qp; struct usd_udp_hdr *hdr; uint32_t last_post; struct usd_wq_post_info *info; uint8_t *copybuf; size_t len; unsigned i; qp = to_qpi(ep->e.dg.ep_qp); wq = &qp->uq_wq; copybuf = wq->uwq_copybuf + wq->uwq_post_index * USD_SEND_MAX_COPY; hdr = (struct usd_udp_hdr *)copybuf; memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr)); hdr->uh_udp.source = qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port; len = sizeof(*hdr); for (i = 0; i < count; i++) { memcpy(copybuf + len, iov[i].iov_base, iov[i].iov_len); len += iov[i].iov_len; } /* adjust lengths */ hdr->uh_ip.tot_len = htons(len - sizeof(struct ether_header)); hdr->uh_udp.len = htons(len - sizeof(struct ether_header) - sizeof(struct iphdr)); last_post = _usd_post_send_one(wq, hdr, len, 1); info = &wq->uwq_post_info[last_post]; info->wp_context = context; info->wp_len = len; return 0; }
ssize_t usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, void *context) { struct usdf_ep *ep; struct usd_dest *dest; struct usd_qp_impl *qp; struct usd_udp_hdr *hdr; struct usd_wq *wq; uint32_t last_post; struct usd_wq_post_info *info; ep = ep_ftou(fep); dest = (struct usd_dest *)(uintptr_t)dest_addr; qp = to_qpi(ep->e.dg.ep_qp); wq = &qp->uq_wq; hdr = (struct usd_udp_hdr *) buf - 1; memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr)); /* adjust lengths and insert source port */ hdr->uh_ip.tot_len = htons(len + sizeof(struct usd_udp_hdr) - sizeof(struct ether_header)); hdr->uh_udp.len = htons((sizeof(struct usd_udp_hdr) - sizeof(struct ether_header) - sizeof(struct iphdr)) + len); hdr->uh_udp.source = qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port; last_post = _usd_post_send_one(wq, hdr, len + sizeof(struct usd_udp_hdr), 1); info = &wq->uwq_post_info[last_post]; info->wp_context = context; info->wp_len = len; return 0; }
static inline void usdf_msg_send_segment(struct usdf_tx *tx, struct usdf_ep *ep) { struct usdf_msg_qe *msg; struct rudp_pkt *hdr; struct usd_wq *wq; uint32_t index; size_t cur_iov; size_t cur_resid; size_t resid; const uint8_t *cur_ptr; const uint8_t *send_ptr; size_t sge_len; uint8_t *ptr; struct usd_wq_post_info *info; msg = TAILQ_FIRST(&ep->e.msg.ep_posted_wqe); wq = &(to_qpi(tx->tx_qp)->uq_wq); index = wq->uwq_post_index; hdr = (struct rudp_pkt *)(wq->uwq_copybuf + index * USD_SEND_MAX_COPY); memcpy(hdr, &ep->e.msg.ep_dest->ds_dest.ds_udp.u_hdr, sizeof(struct usd_udp_hdr)); hdr->msg.src_peer_id = htons(ep->e.msg.ep_lcl_peer_id); resid = msg->ms_resid; cur_iov = msg->ms_cur_iov; cur_ptr = msg->ms_cur_ptr; cur_resid = msg->ms_iov_resid; /* save first seq for message */ if (cur_iov == 0 && cur_resid == msg->ms_iov[0].iov_len) { msg->ms_first_seq = ep->e.msg.ep_next_tx_seq; } if (resid < USD_SEND_MAX_COPY - sizeof(*hdr)) { hdr->msg.opcode = htons(RUDP_OP_LAST); hdr->msg.m.rc_data.length = htons(resid); hdr->msg.m.rc_data.seqno = htons(ep->e.msg.ep_next_tx_seq); ++ep->e.msg.ep_next_tx_seq; sge_len = resid; ptr = (uint8_t *)(hdr + 1); while (resid > 0) { memcpy(ptr, cur_ptr, cur_resid); ptr += cur_resid; resid -= cur_resid; ++cur_iov; cur_ptr = msg->ms_iov[cur_iov].iov_base; cur_resid = msg->ms_iov[cur_iov].iov_len; } /* add packet lengths */ hdr->hdr.uh_ip.tot_len = htons( sge_len + sizeof(struct rudp_pkt) - sizeof(struct ether_header)); hdr->hdr.uh_udp.len = htons( (sizeof(struct rudp_pkt) - sizeof(struct ether_header) - sizeof(struct iphdr)) + sge_len); index = _usd_post_send_one(wq, hdr, sge_len + sizeof(*hdr), 1); } else { struct vnic_wq *vwq; u_int8_t offload_mode = 0, eop; u_int16_t mss = 7, header_length = 0, vlan_tag = 0; u_int8_t vlan_tag_insert = 0, loopback = 0, fcoe_encap = 0; struct wq_enet_desc *desc; size_t space; size_t num_sge; size_t sent; vwq = &wq->uwq_vnic_wq; desc = wq->uwq_next_desc; space = ep->ep_domain->dom_fabric->fab_dev_attrs->uda_mtu - sizeof(*hdr); num_sge = 1; /* encode header desc */ eop = 0; wq_enet_desc_enc(desc, (uintptr_t)hdr, sizeof(*hdr), mss, header_length, offload_mode, eop, 0, fcoe_encap, vlan_tag_insert, vlan_tag, loopback); do { desc = (struct wq_enet_desc *) ((uintptr_t)wq->uwq_desc_ring + (index << 4)); index = (index + 1) & wq->uwq_post_index_mask; send_ptr = cur_ptr; if (cur_resid >= space) { sge_len = space; eop = 1; cur_resid -= sge_len; cur_ptr += sge_len; } else { sge_len = cur_resid; if (num_sge == USDF_MSG_MAX_SGE || cur_resid == resid) { eop = 1; } ++cur_iov; cur_ptr = msg->ms_iov[cur_iov].iov_base; cur_resid = msg->ms_iov[cur_iov].iov_len; } wq_enet_desc_enc(desc, (uintptr_t)send_ptr, sge_len, mss, header_length, offload_mode, eop, eop, fcoe_encap, vlan_tag_insert, vlan_tag, loopback); ++num_sge; space -= sge_len; resid -= sge_len; } while (space > 0 && num_sge <= USDF_MSG_MAX_SGE && resid > 0); /* add packet lengths */ sent = ep->ep_domain->dom_fabric->fab_dev_attrs->uda_mtu - sizeof(*hdr) - space; hdr->hdr.uh_ip.tot_len = htons( sent + sizeof(struct rudp_pkt) - sizeof(struct ether_header)); hdr->hdr.uh_udp.len = htons( (sizeof(struct rudp_pkt) - sizeof(struct ether_header) - sizeof(struct iphdr)) + sent); #if 0 if ((random() % 177) == 0 && resid == 0) { hdr->hdr.uh_eth.ether_type = 0; //printf("BORK seq %u\n", ep->e.msg.ep_next_tx_seq); } #endif if (resid == 0) { hdr->msg.opcode = htons(RUDP_OP_LAST); } else { hdr->msg.opcode = htons(RUDP_OP_FIRST); } hdr->msg.m.rc_data.length = htons(sent); hdr->msg.m.rc_data.seqno = htons(ep->e.msg.ep_next_tx_seq); ++ep->e.msg.ep_next_tx_seq; wmb(); iowrite64(index, &vwq->ctrl->posted_index); wq->uwq_next_desc = (struct wq_enet_desc *) ((uintptr_t)wq->uwq_desc_ring + (index << 4)); wq->uwq_post_index = (index + 1) & wq->uwq_post_index_mask; wq->uwq_send_credits -= num_sge; } info = &wq->uwq_post_info[index]; info->wp_context = tx; info->wp_len = sge_len; /* If send complete, remove from send list */ if (resid == 0) { usdf_msg_send_complete(ep, msg); } else { msg->ms_resid = resid; msg->ms_iov_resid = cur_resid; msg->ms_cur_iov = cur_iov; msg->ms_cur_ptr = cur_ptr; } /* set ACK timer */ usdf_timer_set(ep->ep_domain->dom_fabric, ep->e.msg.ep_ack_timer, USDF_RUDP_ACK_TIMEOUT); }
usd_post_send_one_prefixed_raw_normal( struct usd_qp *uqp, struct usd_dest __attribute__ ((unused)) * dest, const void *buf, size_t len, uint32_t flags, void *context) { struct usd_qp_impl *qp; struct usd_wq *wq; uint32_t last_post; struct usd_wq_post_info *info; qp = to_qpi(uqp); wq = &qp->uq_wq; last_post = _usd_post_send_one(wq, buf, len, USD_SF_ISSET(flags, SIGNAL)); info = &wq->uwq_post_info[last_post]; info->wp_context = context; info->wp_len = len; return 0; } struct usd_qp_ops usd_qp_ops_raw_normal = { .qo_post_send_one_prefixed = usd_post_send_one_prefixed_raw_normal, };