static ssize_t mrail_tsend_common(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, size_t count, size_t len, fi_addr_t dest_addr, uint64_t tag, uint64_t data, void *context, uint64_t flags) { struct mrail_ep *mrail_ep = container_of(ep_fid, struct mrail_ep, util_ep.ep_fid.fid); struct mrail_peer_info *peer_info; struct iovec *iov_dest = alloca(sizeof(*iov_dest) * (count + 1)); struct mrail_tx_buf *tx_buf; uint32_t i = mrail_get_tx_rail(mrail_ep); struct fi_msg msg; ssize_t ret; peer_info = ofi_av_get_addr(mrail_ep->util_ep.av, (int) dest_addr); ofi_ep_lock_acquire(&mrail_ep->util_ep); tx_buf = mrail_get_tx_buf(mrail_ep, context, peer_info->seq_no++, ofi_op_tagged, flags | FI_TAGGED); if (OFI_UNLIKELY(!tx_buf)) { ret = -FI_ENOMEM; goto err1; } tx_buf->hdr.tag = tag; mrail_copy_iov_hdr(&tx_buf->hdr, iov_dest, iov, count); msg.msg_iov = iov_dest; msg.desc = desc; msg.iov_count = count + 1; msg.addr = dest_addr; msg.context = tx_buf; msg.data = data; if (len < mrail_ep->rails[i].info->tx_attr->inject_size) flags |= FI_INJECT; FI_DBG(&mrail_prov, FI_LOG_EP_DATA, "Posting tsend of length: %" PRIu64 " dest_addr: 0x%" PRIx64 " tag: 0x%" PRIx64 " seq: %d" " on rail: %d\n", len, dest_addr, tag, peer_info->seq_no - 1, i); ret = fi_sendmsg(mrail_ep->rails[i].ep, &msg, flags); if (ret) { FI_WARN(&mrail_prov, FI_LOG_EP_DATA, "Unable to fi_sendmsg on rail: %" PRIu32 "\n", i); goto err2; } else if (!(flags & FI_COMPLETION)) { ofi_ep_tx_cntr_inc(&mrail_ep->util_ep); } ofi_ep_lock_release(&mrail_ep->util_ep); return ret; err2: util_buf_release(mrail_ep->tx_buf_pool, tx_buf); err1: peer_info->seq_no--; ofi_ep_lock_release(&mrail_ep->util_ep); return ret; }
// TODO go for separate recv functions (recvmsg, recvv, etc) to be optimal static ssize_t mrail_recv_common(struct mrail_ep *mrail_ep, struct mrail_recv_queue *recv_queue, struct iovec *iov, size_t count, void *context, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, uint64_t flags, uint64_t comp_flags) { struct mrail_recv *recv; struct mrail_unexp_msg_entry *unexp_msg_entry; recv = mrail_pop_recv(mrail_ep); if (!recv) return -FI_EAGAIN; recv->count = count + 1; recv->context = context; recv->flags = flags; recv->comp_flags |= comp_flags; recv->addr = src_addr; recv->tag = tag; recv->ignore = ignore; memcpy(&recv->iov[1], iov, sizeof(*iov) * count); FI_DBG(&mrail_prov, FI_LOG_EP_DATA, "Posting recv of length: %zu " "src_addr: 0x%" PRIx64 " tag: 0x%" PRIx64 " ignore: 0x%" PRIx64 "\n", ofi_total_iov_len(iov, count), recv->addr, recv->tag, recv->ignore); ofi_ep_lock_acquire(&mrail_ep->util_ep); unexp_msg_entry = container_of(dlist_remove_first_match( &recv_queue->unexp_msg_list, recv_queue->match_unexp, recv), struct mrail_unexp_msg_entry, entry); if (!unexp_msg_entry) { dlist_insert_tail(&recv->entry, &recv_queue->recv_list); ofi_ep_lock_release(&mrail_ep->util_ep); return 0; } ofi_ep_lock_release(&mrail_ep->util_ep); FI_DBG(recv_queue->prov, FI_LOG_EP_DATA, "Match for posted recv" " with addr: 0x%" PRIx64 ", tag: 0x%" PRIx64 " ignore: " "0x%" PRIx64 " found in unexpected msg queue\n", recv->addr, recv->tag, recv->ignore); return mrail_cq_process_buf_recv((struct fi_cq_tagged_entry *) unexp_msg_entry->data, recv); }
static inline ssize_t rxm_ep_rma_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags, rxm_rma_msg_fn rma_msg, uint64_t comp_flags) { struct rxm_rma_buf *rma_buf; struct fi_msg_rma msg_rma = *msg; struct rxm_conn *rxm_conn; void *mr_desc[RXM_IOV_LIMIT] = { 0 }; int ret; assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit); ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; ofi_ep_lock_acquire(&rxm_ep->util_ep); rma_buf = rxm_rma_buf_alloc(rxm_ep); if (OFI_UNLIKELY(!rma_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from RMA buffer pool\n"); ret = -FI_ENOMEM; goto unlock; } rma_buf->app_context = msg->context; rma_buf->flags = flags; ret = rxm_ep_rma_reg_iov(rxm_ep, msg_rma.msg_iov, msg_rma.desc, mr_desc, msg_rma.iov_count, comp_flags & (FI_WRITE | FI_READ), rma_buf); if (OFI_UNLIKELY(ret)) goto release; msg_rma.desc = mr_desc; msg_rma.context = rma_buf; ret = rma_msg(rxm_conn->msg_ep, &msg_rma, flags); if (OFI_LIKELY(!ret)) goto unlock; if ((rxm_ep->msg_mr_local) && (!rxm_ep->rxm_mr_local)) rxm_ep_msg_mr_closev(rma_buf->mr.mr, rma_buf->mr.count); release: ofi_buf_free(rma_buf); unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; }
static ssize_t rxm_ep_readv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); struct fi_rma_iov rma_iov = { .addr = addr, .len = ofi_total_iov_len(iov, count), .key = key, }; struct fi_msg_rma msg = { .msg_iov = iov, .desc = desc, .iov_count = count, .addr = src_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ); } static ssize_t rxm_ep_read(struct fid_ep *ep_fid, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) { struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = src_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ); } static inline void rxm_ep_format_rma_msg(struct rxm_rma_buf *rma_buf, const struct fi_msg_rma *orig_msg, struct iovec *rxm_iov, struct fi_msg_rma *rxm_msg) { rxm_msg->context = rma_buf; rxm_msg->addr = orig_msg->addr; rxm_msg->data = orig_msg->data; ofi_copy_from_iov(rma_buf->pkt.data, rma_buf->pkt.hdr.size, orig_msg->msg_iov, orig_msg->iov_count, 0); rxm_iov->iov_base = &rma_buf->pkt.data; rxm_iov->iov_len = rma_buf->pkt.hdr.size; rxm_msg->msg_iov = rxm_iov; rxm_msg->desc = &rma_buf->hdr.desc; rxm_msg->iov_count = 1; rxm_msg->rma_iov = orig_msg->rma_iov; rxm_msg->rma_iov_count = orig_msg->rma_iov_count; } static inline ssize_t rxm_ep_rma_emulate_inject_msg(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, size_t total_size, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_rma_buf *rma_buf; ssize_t ret; struct iovec rxm_msg_iov = { 0 }; struct fi_msg_rma rxm_rma_msg = { 0 }; assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit); ofi_ep_lock_acquire(&rxm_ep->util_ep); rma_buf = rxm_rma_buf_alloc(rxm_ep); if (OFI_UNLIKELY(!rma_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from RMA buffer pool\n"); ret = -FI_ENOMEM; goto unlock; } rma_buf->pkt.hdr.size = total_size; rma_buf->app_context = msg->context; rma_buf->flags = flags; rxm_ep_format_rma_msg(rma_buf, msg, &rxm_msg_iov, &rxm_rma_msg); flags = (flags & ~FI_INJECT) | FI_COMPLETION; ret = fi_writemsg(rxm_conn->msg_ep, &rxm_rma_msg, flags); if (OFI_UNLIKELY(ret)) { if (ret == -FI_EAGAIN) rxm_ep_do_progress(&rxm_ep->util_ep); ofi_buf_free(rma_buf); } unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; } static inline ssize_t rxm_ep_rma_emulate_inject(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, uint64_t flags) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = NULL, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = NULL, .data = data, }; return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, len, &msg, flags); } static inline ssize_t rxm_ep_rma_inject_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_conn *rxm_conn; size_t total_size = ofi_total_iov_len(msg->msg_iov, msg->iov_count); ssize_t ret; assert(total_size <= rxm_ep->rxm_info->tx_attr->inject_size); ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if ((total_size <= rxm_ep->msg_info->tx_attr->inject_size) && !(flags & FI_COMPLETION) && (msg->iov_count == 1) && (msg->rma_iov_count == 1)) { if (flags & FI_REMOTE_CQ_DATA) { ret = fi_inject_writedata(rxm_conn->msg_ep, msg->msg_iov->iov_base, msg->msg_iov->iov_len, msg->data, msg->addr, msg->rma_iov->addr, msg->rma_iov->key); } else { ret = fi_inject_write(rxm_conn->msg_ep, msg->msg_iov->iov_base, msg->msg_iov->iov_len, msg->addr, msg->rma_iov->addr, msg->rma_iov->key); } if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_write* for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, total_size, msg, flags); } } static inline ssize_t rxm_ep_generic_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); if (flags & FI_INJECT) return rxm_ep_rma_inject_common(rxm_ep, msg, flags); else return rxm_ep_rma_common(rxm_ep, msg, flags, fi_writemsg, FI_WRITE); } static inline ssize_t rxm_ep_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, msg, flags | rxm_ep->util_ep.tx_msg_flags); } static ssize_t rxm_ep_writev(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = ofi_total_iov_len(iov, count), .key = key, }; struct fi_msg_rma msg = { .msg_iov = iov, .desc = desc, .iov_count = count, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep)); } static ssize_t rxm_ep_writedata(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = data, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep) | FI_REMOTE_CQ_DATA); } static ssize_t rxm_ep_write(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep)); } static ssize_t rxm_ep_inject_write(struct fid_ep *ep_fid, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t addr, uint64_t key) { ssize_t ret; struct rxm_conn *rxm_conn; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if (len <= rxm_ep->msg_info->tx_attr->inject_size) { ret = fi_inject_write(rxm_conn->msg_ep, buf, len, dest_addr, addr, key); if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_write for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len, 0, dest_addr, addr, key, FI_INJECT); } } static ssize_t rxm_ep_inject_writedata(struct fid_ep *ep_fid, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key) { ssize_t ret; struct rxm_conn *rxm_conn; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if (len <= rxm_ep->msg_info->tx_attr->inject_size) { ret = fi_inject_writedata(rxm_conn->msg_ep, buf, len, data, dest_addr, addr, key); if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_writedata for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len, data, dest_addr, addr, key, FI_REMOTE_CQ_DATA | FI_INJECT); } } struct fi_ops_rma rxm_ops_rma = { .size = sizeof (struct fi_ops_rma), .read = rxm_ep_read, .readv = rxm_ep_readv, .readmsg = rxm_ep_readmsg, .write = rxm_ep_write, .writev = rxm_ep_writev, .writemsg = rxm_ep_writemsg, .inject = rxm_ep_inject_write, .writedata = rxm_ep_writedata, .injectdata = rxm_ep_inject_writedata, };
static ssize_t rxm_ep_atomic_common(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, const struct fi_msg_atomic *msg, const struct fi_ioc *comparev, void **compare_desc, size_t compare_iov_count, struct fi_ioc *resultv, void **result_desc, size_t result_iov_count, uint32_t op, uint64_t flags) { struct rxm_tx_atomic_buf *tx_buf; struct rxm_atomic_hdr *atomic_hdr; struct iovec buf_iov[RXM_IOV_LIMIT]; struct iovec cmp_iov[RXM_IOV_LIMIT]; size_t datatype_sz = ofi_datatype_size(msg->datatype); size_t buf_len = 0; size_t cmp_len = 0; size_t tot_len; ssize_t ret; assert(msg->iov_count <= RXM_IOV_LIMIT && msg->rma_iov_count <= RXM_IOV_LIMIT); if (flags & FI_REMOTE_CQ_DATA) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "atomic with remote CQ data not supported\n"); return -FI_EINVAL; } if (msg->op != FI_ATOMIC_READ) { assert(msg->msg_iov); ofi_ioc_to_iov(msg->msg_iov, buf_iov, msg->iov_count, datatype_sz); buf_len = ofi_total_iov_len(buf_iov, msg->iov_count); } if (op == ofi_op_atomic_compare) { assert(comparev); ofi_ioc_to_iov(comparev, cmp_iov, compare_iov_count, datatype_sz); cmp_len = ofi_total_iov_len(cmp_iov, compare_iov_count); assert(buf_len == cmp_len); } tot_len = buf_len + cmp_len + sizeof(struct rxm_atomic_hdr) + sizeof(struct rxm_pkt); if (tot_len > rxm_eager_limit) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "atomic data too large %zu\n", tot_len); return -FI_EINVAL; } ofi_ep_lock_acquire(&rxm_ep->util_ep); tx_buf = (struct rxm_tx_atomic_buf *) rxm_tx_buf_alloc(rxm_ep, RXM_BUF_POOL_TX_ATOMIC); if (OFI_UNLIKELY(!tx_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from Atomic buffer pool\n"); ret = -FI_EAGAIN; goto unlock; } rxm_ep_format_atomic_pkt_hdr(rxm_conn, tx_buf, tot_len, op, msg->datatype, msg->op, flags, msg->data, msg->rma_iov, msg->rma_iov_count); tx_buf->pkt.ctrl_hdr.msg_id = ofi_buf_index(tx_buf); tx_buf->app_context = msg->context; atomic_hdr = (struct rxm_atomic_hdr *) tx_buf->pkt.data; ofi_copy_from_iov(atomic_hdr->data, buf_len, buf_iov, msg->iov_count, 0); if (cmp_len) ofi_copy_from_iov(atomic_hdr->data + buf_len, cmp_len, cmp_iov, compare_iov_count, 0); tx_buf->result_iov_count = result_iov_count; if (resultv) ofi_ioc_to_iov(resultv, tx_buf->result_iov, result_iov_count, datatype_sz); ret = rxm_ep_send_atomic_req(rxm_ep, rxm_conn, tx_buf, tot_len); if (ret) ofi_buf_free(tx_buf); unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; }