ssize_t udpx_recv(struct fid_ep *ep_fid, void *buf, size_t len, void *desc,
fi_addr_t src_addr, void *context)
{
struct udpx_ep *ep;
struct udpx_ep_entry *entry;
ssize_t ret;
ep = container_of(ep_fid, struct udpx_ep, util_ep.ep_fid.fid);
fastlock_acquire(&ep->util_ep.rx_cq->cq_lock);
if (ofi_cirque_isfull(ep->rxq)) {
ret = -FI_EAGAIN;
goto out;
}
entry = ofi_cirque_tail(ep->rxq);
entry->context = context;
entry->iov_count = 1;
entry->iov[0].iov_base = buf;
entry->iov[0].iov_len = len;
entry->flags = 0;
ofi_cirque_commit(ep->rxq);
ret = 0;
out:
fastlock_release(&ep->util_ep.rx_cq->cq_lock);
return ret;
}
int smr_rx_comp(struct smr_ep *ep, void *context, uint64_t flags, size_t len,
void *buf, void *addr, uint64_t tag, uint64_t data,
uint64_t err)
{
struct fi_cq_tagged_entry *comp;
struct util_cq_err_entry *entry;
comp = ofi_cirque_tail(ep->util_ep.rx_cq->cirq);
if (err) {
if (!(entry = calloc(1, sizeof(*entry))))
return -FI_ENOMEM;
entry->err_entry.op_context = context;
entry->err_entry.flags = flags;
entry->err_entry.tag = tag;
entry->err_entry.err = err;
entry->err_entry.prov_errno = -err;
slist_insert_tail(&entry->list_entry,
&ep->util_ep.rx_cq->err_list);
comp->flags = UTIL_FLAG_ERROR;
} else {
comp->op_context = context;
comp->flags = flags;
comp->len = len;
comp->buf = buf;
comp->data = data;
comp->tag = tag;
}
ofi_cirque_commit(ep->util_ep.rx_cq->cirq);
return 0;
}
ssize_t udpx_recvmsg(struct fid_ep *ep_fid, const struct fi_msg *msg,
uint64_t flags)
{
struct udpx_ep *ep;
struct udpx_ep_entry *entry;
ssize_t ret;
ep = container_of(ep_fid, struct udpx_ep, util_ep.ep_fid.fid);
fastlock_acquire(&ep->util_ep.rx_cq->cq_lock);
if (ofi_cirque_isfull(ep->rxq)) {
ret = -FI_EAGAIN;
goto out;
}
entry = ofi_cirque_tail(ep->rxq);
entry->context = msg->context;
for (entry->iov_count = 0; entry->iov_count < msg->iov_count;
entry->iov_count++) {
entry->iov[entry->iov_count] = msg->msg_iov[entry->iov_count];
}
entry->flags = 0;
ofi_cirque_commit(ep->rxq);
ret = 0;
out:
fastlock_release(&ep->util_ep.rx_cq->cq_lock);
return ret;
}
static void udpx_tx_comp(struct udpx_ep *ep, void *context)
{
struct fi_cq_tagged_entry *comp;
comp = ofi_cirque_tail(ep->util_ep.tx_cq->cirq);
comp->op_context = context;
comp->flags = FI_SEND;
comp->len = 0;
comp->buf = NULL;
comp->data = 0;
ofi_cirque_commit(ep->util_ep.tx_cq->cirq);
}
static int rxd_cq_write_ctx(struct rxd_cq *cq,
struct fi_cq_tagged_entry *cq_entry)
{
struct fi_cq_tagged_entry *comp;
if (ofi_cirque_isfull(cq->util_cq.cirq))
return -FI_ENOSPC;
comp = ofi_cirque_tail(cq->util_cq.cirq);
comp->op_context = cq_entry->op_context;
ofi_cirque_commit(cq->util_cq.cirq);
return 0;
}
static void udpx_rx_comp(struct udpx_ep *ep, void *context, uint64_t flags,
size_t len, void *buf, void *addr)
{
struct fi_cq_tagged_entry *comp;
comp = ofi_cirque_tail(ep->util_ep.rx_cq->cirq);
comp->op_context = context;
comp->flags = FI_RECV | flags;
comp->len = len;
comp->buf = buf;
comp->data = 0;
ofi_cirque_commit(ep->util_ep.rx_cq->cirq);
}
static int rxd_cq_write_tagged(struct rxd_cq *cq,
struct fi_cq_tagged_entry *cq_entry)
{
struct fi_cq_tagged_entry *comp;
if (ofi_cirque_isfull(cq->util_cq.cirq))
return -FI_ENOSPC;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"report completion: %p\n", cq_entry->tag);
comp = ofi_cirque_tail(cq->util_cq.cirq);
*comp = *cq_entry;
ofi_cirque_commit(cq->util_cq.cirq);
return 0;
}
static int rxd_cq_write_data(struct rxd_cq *cq,
struct fi_cq_tagged_entry *cq_entry)
{
struct fi_cq_tagged_entry *comp;
if (ofi_cirque_isfull(cq->util_cq.cirq))
return -FI_ENOSPC;
comp = ofi_cirque_tail(cq->util_cq.cirq);
comp->op_context = cq_entry->op_context;
comp->flags = cq_entry->flags;
comp->len = cq_entry->len;
comp->buf = cq_entry->buf;
comp->data = cq_entry->data;
ofi_cirque_commit(cq->util_cq.cirq);
return 0;
}
static void smr_post_fetch_resp(struct smr_ep *ep, struct smr_cmd *cmd,
const struct iovec *result_iov, size_t count)
{
struct smr_cmd *pend;
struct smr_resp *resp;
assert(!ofi_cirque_isfull(smr_resp_queue(ep->region)));
resp = ofi_cirque_tail(smr_resp_queue(ep->region));
cmd->msg.hdr.data = (uint64_t) ((char **) resp -
(char **) ep->region);
pend = freestack_pop(ep->pend_fs);
smr_post_pend_resp(cmd, pend, resp);
memcpy(pend->msg.data.iov, result_iov,
sizeof(*result_iov) * count);
pend->msg.data.iov_count = count;
ofi_cirque_commit(smr_resp_queue(ep->region));
}
static ssize_t smr_atomic_inject(struct fid_ep *ep_fid, const void *buf,
size_t count, fi_addr_t dest_addr, uint64_t addr,
uint64_t key, enum fi_datatype datatype, enum fi_op op)
{
struct smr_ep *ep;
struct smr_region *peer_smr;
struct smr_inject_buf *tx_buf;
struct smr_cmd *cmd;
struct iovec iov;
struct fi_rma_ioc rma_ioc;
int peer_id;
ssize_t ret = 0;
size_t total_len;
assert(count <= SMR_INJECT_SIZE);
ep = container_of(ep_fid, struct smr_ep, util_ep.ep_fid.fid);
peer_id = (int) dest_addr;
ret = smr_verify_peer(ep, peer_id);
if(ret)
return ret;
peer_smr = smr_peer_region(ep->region, peer_id);
fastlock_acquire(&peer_smr->lock);
if (peer_smr->cmd_cnt < 2) {
ret = -FI_EAGAIN;
goto unlock_region;
}
cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr));
total_len = count * ofi_datatype_size(datatype);
iov.iov_base = (void *) buf;
iov.iov_len = total_len;
rma_ioc.addr = addr;
rma_ioc.count = count;
rma_ioc.key = key;
if (total_len <= SMR_MSG_DATA_LEN) {
smr_format_inline_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr,
&iov, 1, NULL, 0, ofi_op_atomic,
datatype, op);
} else if (total_len <= SMR_INJECT_SIZE) {
tx_buf = smr_freestack_pop(smr_inject_pool(peer_smr));
smr_format_inject_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr,
&iov, 1, NULL, 0, NULL, 0, ofi_op_atomic,
datatype, op, peer_smr, tx_buf);
}
ofi_cirque_commit(smr_cmd_queue(peer_smr));
peer_smr->cmd_cnt--;
cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr));
smr_format_rma_ioc(cmd, &rma_ioc, 1);
ofi_cirque_commit(smr_cmd_queue(peer_smr));
peer_smr->cmd_cnt--;
unlock_region:
fastlock_release(&peer_smr->lock);
return ret;
}
static ssize_t smr_generic_atomic(struct fid_ep *ep_fid,
const struct fi_ioc *ioc, void **desc, size_t count,
const struct fi_ioc *compare_ioc, void **compare_desc,
size_t compare_count, struct fi_ioc *result_ioc,
void **result_desc, size_t result_count,
fi_addr_t addr, const struct fi_rma_ioc *rma_ioc,
size_t rma_count, enum fi_datatype datatype,
enum fi_op atomic_op, void *context, uint32_t op)
{
struct smr_ep *ep;
struct smr_domain *domain;
struct smr_region *peer_smr;
struct smr_inject_buf *tx_buf;
struct smr_cmd *cmd;
struct iovec iov[SMR_IOV_LIMIT];
struct iovec compare_iov[SMR_IOV_LIMIT];
struct iovec result_iov[SMR_IOV_LIMIT];
int peer_id, err = 0;
uint16_t flags = 0;
ssize_t ret = 0;
size_t msg_len, total_len;
assert(count <= SMR_IOV_LIMIT);
assert(result_count <= SMR_IOV_LIMIT);
assert(compare_count <= SMR_IOV_LIMIT);
assert(rma_count <= SMR_IOV_LIMIT);
ep = container_of(ep_fid, struct smr_ep, util_ep.ep_fid.fid);
domain = container_of(ep->util_ep.domain, struct smr_domain, util_domain);
peer_id = (int) addr;
ret = smr_verify_peer(ep, peer_id);
if(ret)
return ret;
peer_smr = smr_peer_region(ep->region, peer_id);
fastlock_acquire(&peer_smr->lock);
if (peer_smr->cmd_cnt < 2) {
ret = -FI_EAGAIN;
goto unlock_region;
}
fastlock_acquire(&ep->util_ep.tx_cq->cq_lock);
if (ofi_cirque_isfull(ep->util_ep.tx_cq->cirq)) {
ret = -FI_EAGAIN;
goto unlock_cq;
}
cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr));
msg_len = total_len = ofi_datatype_size(datatype) *
ofi_total_ioc_cnt(ioc, count);
switch (op) {
case ofi_op_atomic_compare:
assert(compare_ioc);
ofi_ioc_to_iov(compare_ioc, compare_iov, compare_count,
ofi_datatype_size(datatype));
total_len *= 2;
/* fall through */
case ofi_op_atomic_fetch:
assert(result_ioc);
ofi_ioc_to_iov(result_ioc, result_iov, result_count,
ofi_datatype_size(datatype));
if (!domain->fast_rma)
flags |= SMR_RMA_REQ;
/* fall through */
case ofi_op_atomic:
if (atomic_op != FI_ATOMIC_READ) {
assert(ioc);
ofi_ioc_to_iov(ioc, iov, count, ofi_datatype_size(datatype));
} else {
count = 0;
}
break;
default:
break;
}
if (total_len <= SMR_MSG_DATA_LEN && !(flags & SMR_RMA_REQ)) {
smr_format_inline_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr,
iov, count, compare_iov, compare_count,
op, datatype, atomic_op);
} else if (total_len <= SMR_INJECT_SIZE) {
tx_buf = smr_freestack_pop(smr_inject_pool(peer_smr));
smr_format_inject_atomic(cmd, smr_peer_addr(ep->region)[peer_id].addr,
iov, count, result_iov, result_count,
compare_iov, compare_count, op, datatype,
atomic_op, peer_smr, tx_buf);
} else {
FI_WARN(&smr_prov, FI_LOG_EP_CTRL,
"message too large\n");
ret = -FI_EINVAL;
goto unlock_cq;
}
cmd->msg.hdr.op_flags |= flags;
ofi_cirque_commit(smr_cmd_queue(peer_smr));
peer_smr->cmd_cnt--;
if (op != ofi_op_atomic) {
if (flags & SMR_RMA_REQ) {
smr_post_fetch_resp(ep, cmd,
(const struct iovec *) result_iov,
result_count);
goto format_rma;
}
err = smr_fetch_result(ep, peer_smr, result_iov, result_count,
rma_ioc, rma_count, datatype, msg_len);
if (err)
FI_WARN(&smr_prov, FI_LOG_EP_CTRL,
"unable to fetch results");
}
ret = ep->tx_comp(ep, context, ofi_tx_cq_flags(op), err);
if (ret) {
FI_WARN(&smr_prov, FI_LOG_EP_CTRL,
"unable to process tx completion\n");
}
format_rma:
cmd = ofi_cirque_tail(smr_cmd_queue(peer_smr));
smr_format_rma_ioc(cmd, rma_ioc, rma_count);
ofi_cirque_commit(smr_cmd_queue(peer_smr));
peer_smr->cmd_cnt--;
unlock_cq:
fastlock_release(&ep->util_ep.tx_cq->cq_lock);
unlock_region:
fastlock_release(&peer_smr->lock);
return ret;
}