static inline int rxm_finish_sar_segment_send(struct rxm_ep *rxm_ep, struct rxm_tx_sar_buf *tx_buf) { int ret = FI_SUCCESS; struct rxm_tx_sar_buf *first_tx_buf; switch (rxm_sar_get_seg_type(&tx_buf->pkt.ctrl_hdr)) { case RXM_SAR_SEG_FIRST: break; case RXM_SAR_SEG_MIDDLE: ofi_buf_free(tx_buf); break; case RXM_SAR_SEG_LAST: ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op), tx_buf->app_context, tx_buf->flags); assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND); ofi_ep_tx_cntr_inc(&rxm_ep->util_ep); first_tx_buf = ofi_bufpool_get_ibuf(rxm_ep-> buf_pools[RXM_BUF_POOL_TX_SAR].pool, tx_buf->pkt.ctrl_hdr.msg_id); ofi_buf_free(first_tx_buf); ofi_buf_free(tx_buf); break; } return ret; }
static inline int rxm_finish_eager_send(struct rxm_ep *rxm_ep, struct rxm_tx_eager_buf *tx_buf) { int ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op), tx_buf->app_context, tx_buf->flags); assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND); ofi_ep_tx_cntr_inc(&rxm_ep->util_ep); return ret; }
static int rxm_rndv_tx_finish(struct rxm_ep *rxm_ep, struct rxm_tx_rndv_buf *tx_buf) { int ret; RXM_LOG_STATE_TX(FI_LOG_CQ, tx_buf, RXM_RNDV_FINISH); tx_buf->hdr.state = RXM_RNDV_FINISH; tx_buf->conn->rndv_tx_credits++; if (!rxm_ep->rxm_mr_local) rxm_ep_msg_mr_closev(tx_buf->mr, tx_buf->count); ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op), tx_buf->app_context, tx_buf->flags); assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND); ofi_ep_tx_cntr_inc(&rxm_ep->util_ep); ofi_buf_free(tx_buf); return ret; }
static ssize_t smr_generic_atomic(struct fid_ep *ep_fid, const struct fi_ioc *ioc, void **desc, size_t count, const struct fi_ioc *compare_ioc, void **compare_desc, size_t compare_count, struct fi_ioc *result_ioc, void **result_desc, size_t result_count, fi_addr_t addr, const struct fi_rma_ioc *rma_ioc, size_t rma_count, enum fi_datatype datatype, enum fi_op atomic_op, void *context, uint32_t op) { struct smr_ep *ep; struct smr_domain *domain; struct smr_region *peer_smr; struct smr_inject_buf *tx_buf; struct smr_cmd *cmd; struct iovec iov[SMR_IOV_LIMIT]; struct iovec compare_iov[SMR_IOV_LIMIT]; struct iovec result_iov[SMR_IOV_LIMIT]; int peer_id, err = 0; uint16_t flags = 0; ssize_t ret = 0; size_t msg_len, total_len; assert(count <= SMR_IOV_LIMIT); assert(result_count <= SMR_IOV_LIMIT); assert(compare_count <= SMR_IOV_LIMIT); assert(rma_count <= SMR_IOV_LIMIT); ep = container_of(ep_fid, struct smr_ep, util_ep.ep_fid.fid); domain = container_of(ep->util_ep.domain, struct smr_domain, util_domain); peer_id = (int) addr; ret = smr_verify_peer(ep, peer_id); if(ret) return ret; peer_smr = smr_peer_region(ep->region, peer_id); fastlock_acquire(&peer_smr->lock); if (peer_smr->cmd_cnt < 2) { ret = -FI_EAGAIN; goto unlock_region; } fastlock_acquire(&ep->util_ep.tx_cq->cq_lock); if (ofi_cirque_isfull(ep->util_ep.tx_cq->cirq)) { ret = -FI_EAGAIN; goto unlock_cq; } cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr)); msg_len = total_len = ofi_datatype_size(datatype) * ofi_total_ioc_cnt(ioc, count); switch (op) { case ofi_op_atomic_compare: assert(compare_ioc); ofi_ioc_to_iov(compare_ioc, compare_iov, compare_count, ofi_datatype_size(datatype)); total_len *= 2; /* fall through */ case ofi_op_atomic_fetch: assert(result_ioc); ofi_ioc_to_iov(result_ioc, result_iov, result_count, ofi_datatype_size(datatype)); if (!domain->fast_rma) flags |= SMR_RMA_REQ; /* fall through */ case ofi_op_atomic: if (atomic_op != FI_ATOMIC_READ) { assert(ioc); ofi_ioc_to_iov(ioc, iov, count, ofi_datatype_size(datatype)); } else { count = 0; } break; default: break; } if (total_len <= SMR_MSG_DATA_LEN && !(flags & SMR_RMA_REQ)) { smr_format_inline_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr, iov, count, compare_iov, compare_count, op, datatype, atomic_op); } else if (total_len <= SMR_INJECT_SIZE) { tx_buf = smr_freestack_pop(smr_inject_pool(peer_smr)); smr_format_inject_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr, iov, count, result_iov, result_count, compare_iov, compare_count, op, datatype, atomic_op, peer_smr, tx_buf); } else { FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "message too large\n"); ret = -FI_EINVAL; goto unlock_cq; } cmd->msg.hdr.op_flags |= flags; ofi_cirque_commit(smr_cmd_queue(peer_smr)); peer_smr->cmd_cnt--; if (op != ofi_op_atomic) { if (flags & SMR_RMA_REQ) { smr_post_fetch_resp(ep, cmd, (const struct iovec *) result_iov, result_count); goto format_rma; } err = smr_fetch_result(ep, peer_smr, result_iov, result_count, rma_ioc, rma_count, datatype, msg_len); if (err) FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "unable to fetch results"); } ret = ep->tx_comp(ep, context, ofi_tx_cq_flags(op), err); if (ret) { FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "unable to process tx completion\n"); } format_rma: cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr)); smr_format_rma_ioc(cmd, rma_ioc, rma_count); ofi_cirque_commit(smr_cmd_queue(peer_smr)); peer_smr->cmd_cnt--; unlock_cq: fastlock_release(&ep->util_ep.tx_cq->cq_lock); unlock_region: fastlock_release(&peer_smr->lock); return ret; }
struct rxd_x_entry *rxd_tx_entry_init(struct rxd_ep *ep, const struct iovec *iov, size_t iov_count, const struct iovec *res_iov, size_t res_count, size_t rma_count, uint64_t data, uint64_t tag, void *context, fi_addr_t addr, uint32_t op, uint32_t flags) { struct rxd_x_entry *tx_entry; struct rxd_domain *rxd_domain = rxd_ep_domain(ep); size_t max_inline; tx_entry = rxd_get_tx_entry(ep, op); if (!tx_entry) { FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not get tx entry\n"); return NULL; } tx_entry->op = op; tx_entry->peer = addr; tx_entry->flags = flags; tx_entry->bytes_done = 0; tx_entry->offset = 0; tx_entry->next_seg_no = 0; tx_entry->iov_count = iov_count; memcpy(&tx_entry->iov[0], iov, sizeof(*iov) * iov_count); if (res_count) { tx_entry->res_count = res_count; memcpy(&tx_entry->res_iov[0], res_iov, sizeof(*res_iov) * res_count); } tx_entry->cq_entry.op_context = context; tx_entry->cq_entry.len = ofi_total_iov_len(iov, iov_count); tx_entry->cq_entry.buf = iov[0].iov_base; tx_entry->cq_entry.flags = ofi_tx_cq_flags(op); tx_entry->cq_entry.tag = tag; tx_entry->pkt = NULL; max_inline = rxd_domain->max_inline_msg; if (tx_entry->cq_entry.flags & FI_RMA) max_inline -= sizeof(struct ofi_rma_iov) * rma_count; if (tx_entry->flags & RXD_TAG_HDR) max_inline -= sizeof(tx_entry->cq_entry.tag); if (tx_entry->flags & RXD_REMOTE_CQ_DATA) { max_inline -= sizeof(tx_entry->cq_entry.data); tx_entry->cq_entry.data = data; } if (rma_count > 1 || tx_entry->cq_entry.flags & FI_READ || tx_entry->cq_entry.len > max_inline) max_inline -= sizeof(struct rxd_sar_hdr); else tx_entry->flags |= RXD_INLINE; if (tx_entry->cq_entry.flags & FI_ATOMIC || tx_entry->cq_entry.len <= max_inline) tx_entry->num_segs = 1; else if (tx_entry->cq_entry.flags & FI_READ) tx_entry->num_segs = ofi_div_ceil(tx_entry->cq_entry.len, rxd_domain->max_seg_sz); else tx_entry->num_segs = ofi_div_ceil(tx_entry->cq_entry.len - max_inline, rxd_domain->max_seg_sz) + 1; if ((tx_entry->op == RXD_READ_REQ || tx_entry->op == RXD_ATOMIC_FETCH || tx_entry->op == RXD_ATOMIC_COMPARE) && ep->peers[tx_entry->peer].unacked_cnt < ep->peers[tx_entry->peer].tx_window && ep->peers[tx_entry->peer].peer_addr != FI_ADDR_UNSPEC) dlist_insert_tail(&tx_entry->entry, &ep->peers[tx_entry->peer].rma_rx_list); else dlist_insert_tail(&tx_entry->entry, &ep->peers[tx_entry->peer].tx_list); return tx_entry; }