static int __gnix_rndzv_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd, *tail_txd = NULL;
gni_return_t status;
int rc;
int use_tx_cq_blk = 0;
struct fid_mr *auto_mr = NULL;
int inject_err = _gnix_req_inject_err(req);
int head_off, head_len, tail_len;
void *tail_data = NULL;
if (!req->msg.recv_md) {
rc = gnix_mr_reg(&ep->domain->domain_fid.fid,
(void *)req->msg.recv_addr, req->msg.recv_len,
FI_READ | FI_WRITE, 0, 0, 0, &auto_mr, NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
return -FI_EAGAIN;
}
req->msg.recv_flags |= FI_LOCAL_MR;
req->msg.recv_md = container_of(auto_mr,
struct gnix_fid_mem_desc,
mr_fid);
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
enum gnix_fab_req_type fr_type,
const struct fi_msg_atomic *msg,
const struct fi_ioc *comparev,
void **compare_desc,
size_t compare_count,
struct fi_ioc *resultv,
void **result_desc,
size_t result_count,
uint64_t flags)
{
struct gnix_vc *vc;
struct gnix_fab_req *req;
struct gnix_fid_mem_desc *md = NULL;
int rc, len;
struct fid_mr *auto_mr = NULL;
void *mdesc = NULL;
uint64_t compare_operand = 0;
void *loc_addr = NULL;
int dt_len, dt_align;
if (!ep || !msg || !msg->msg_iov ||
!msg->msg_iov[0].addr ||
msg->msg_iov[0].count != 1 ||
msg->iov_count != 1 ||
!msg->rma_iov || !msg->rma_iov[0].addr)
return -FI_EINVAL;
if (fr_type == GNIX_FAB_RQ_CAMO) {
if (!comparev || !comparev[0].addr || compare_count != 1)
return -FI_EINVAL;
compare_operand = *(uint64_t *)comparev[0].addr;
}
dt_len = fi_datatype_size(msg->datatype);
dt_align = dt_len - 1;
len = dt_len * msg->msg_iov->count;
if (msg->rma_iov->addr & dt_align) {
GNIX_INFO(FI_LOG_EP_DATA,
"Invalid target alignment: %d (mask 0x%x)\n",
msg->rma_iov->addr, dt_align);
return -FI_EINVAL;
}
/* need a memory descriptor for all fetching and comparison AMOs */
if (fr_type == GNIX_FAB_RQ_FAMO || fr_type == GNIX_FAB_RQ_CAMO) {
if (!resultv || !resultv[0].addr || result_count != 1)
return -FI_EINVAL;
loc_addr = resultv[0].addr;
if ((uint64_t)loc_addr & dt_align) {
GNIX_INFO(FI_LOG_EP_DATA,
"Invalid source alignment: %d (mask 0x%x)\n",
loc_addr, dt_align);
return -FI_EINVAL;
}
if (!result_desc || !result_desc[0]) {
rc = gnix_mr_reg(&ep->domain->domain_fid.fid,
loc_addr, len, FI_READ | FI_WRITE,
0, 0, 0, &auto_mr, NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
return rc;
}
flags |= FI_LOCAL_MR;
mdesc = (void *)auto_mr;
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n",
auto_mr);
} else {
mdesc = result_desc[0];
}
}
/* find VC for target */
rc = _gnix_ep_get_vc(ep, msg->addr, &vc);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_ep_get_vc() failed, addr: %lx, rc:\n",
msg->addr, rc);
goto err_get_vc;
}
/* setup fabric request */
req = _gnix_fr_alloc(ep);
if (!req) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_fr_alloc() failed\n");
rc = -FI_ENOSPC;
goto err_fr_alloc;
}
req->type = fr_type;
req->gnix_ep = ep;
req->vc = vc;
req->user_context = msg->context;
req->work_fn = _gnix_amo_post_req;
if (mdesc) {
md = container_of(mdesc, struct gnix_fid_mem_desc, mr_fid);
}
req->amo.loc_md = (void *)md;
req->amo.loc_addr = (uint64_t)loc_addr;
if (msg->op == FI_ATOMIC_READ) {
/* Atomic reads are the only AMO which write to the operand
* buffer. It's assumed that this is in addition to writing
* fetched data to the result buffer. Make the NIC write to
* the result buffer, like all other AMOS, and copy read data
* to the operand buffer after the completion is received. */
req->amo.first_operand = 0xFFFFFFFFFFFFFFFF; /* operand to FAND */
req->amo.read_buf = msg->msg_iov[0].addr;
} else if (msg->op == FI_CSWAP) {
req->amo.first_operand = compare_operand;
req->amo.second_operand = *(uint64_t *)msg->msg_iov[0].addr;
} else if (msg->op == FI_MSWAP) {
req->amo.first_operand = ~compare_operand;
req->amo.second_operand = *(uint64_t *)msg->msg_iov[0].addr;
req->amo.second_operand &= compare_operand;
} else {
req->amo.first_operand = *(uint64_t *)msg->msg_iov[0].addr;
}
req->amo.rem_addr = msg->rma_iov->addr;
req->amo.rem_mr_key = msg->rma_iov->key;
req->amo.len = len;
req->amo.imm = msg->data;
req->amo.datatype = msg->datatype;
req->amo.op = msg->op;
req->flags = flags;
/* Inject interfaces always suppress completions. If
* SELECTIVE_COMPLETION is set, honor any setting. Otherwise, always
* deliver a completion. */
if ((flags & GNIX_SUPPRESS_COMPLETION) ||
(ep->send_selective_completion && !(flags & FI_COMPLETION))) {
req->flags &= ~FI_COMPLETION;
} else {
req->flags |= FI_COMPLETION;
}
return _gnix_vc_queue_tx_req(req);
err_fr_alloc:
err_get_vc:
if (auto_mr) {
fi_close(&auto_mr->fid);
}
return rc;
}
ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
uint64_t loc_addr, size_t len, void *mdesc,
uint64_t dest_addr, uint64_t rem_addr, uint64_t mkey,
void *context, uint64_t flags, uint64_t data)
{
struct gnix_vc *vc;
struct gnix_fab_req *req;
struct gnix_fid_mem_desc *md = NULL;
int rc;
int rdma;
struct fid_mr *auto_mr = NULL;
if (!ep) {
return -FI_EINVAL;
}
if ((flags & FI_INJECT) && (len > GNIX_INJECT_SIZE)) {
GNIX_INFO(FI_LOG_EP_DATA,
"RMA length %d exceeds inject max size: %d\n",
len, GNIX_INJECT_SIZE);
return -FI_EINVAL;
}
/* find VC for target */
rc = _gnix_ep_get_vc(ep, dest_addr, &vc);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_ep_get_vc() failed, addr: %lx, rc:\n",
dest_addr, rc);
return rc;
}
/* setup fabric request */
req = _gnix_fr_alloc(ep);
if (!req) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_fr_alloc() failed\n");
return -FI_ENOSPC;
}
rdma = len >= ep->domain->params.rma_rdma_thresh;
req->type = fr_type;
req->gnix_ep = ep;
req->vc = vc;
req->user_context = context;
req->work_fn = _gnix_rma_post_req;
atomic_initialize(&req->rma.outstanding_txds, 0);
if (fr_type == GNIX_FAB_RQ_RDMA_READ &&
(rem_addr & GNI_READ_ALIGN_MASK || len & GNI_READ_ALIGN_MASK)) {
if (len >= GNIX_RMA_UREAD_CHAINED_THRESH) {
GNIX_INFO(FI_LOG_EP_DATA,
"Using CT for unaligned GET, req: %p\n",
req);
flags |= GNIX_RMA_CHAINED;
} else {
GNIX_INFO(FI_LOG_EP_DATA,
"Using tmp buf for unaligned GET, req: %p\n",
req);
flags |= GNIX_RMA_INDIRECT;
}
if (rdma)
req->work_fn = _gnix_rma_post_rdma_chain_req;
}
if (!(flags & GNIX_RMA_INDIRECT) && !mdesc &&
(rdma || fr_type == GNIX_FAB_RQ_RDMA_READ)) {
/* We need to auto-register the source buffer. */
rc = gnix_mr_reg(&ep->domain->domain_fid.fid, (void *)loc_addr,
len, FI_READ | FI_WRITE, 0, 0, 0, &auto_mr,
NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
goto err_auto_reg;
}
flags |= FI_LOCAL_MR;
mdesc = (void *)auto_mr;
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
if (mdesc)
md = container_of(mdesc, struct gnix_fid_mem_desc, mr_fid);
req->rma.loc_md = (void *)md;
req->rma.rem_addr = rem_addr;
req->rma.rem_mr_key = mkey;
req->rma.len = len;
req->rma.imm = data;
req->flags = flags;
if (req->flags & FI_INJECT) {
memcpy(req->inject_buf, (void *)loc_addr, len);
req->rma.loc_addr = (uint64_t)req->inject_buf;
} else {
req->rma.loc_addr = loc_addr;
}
/* Inject interfaces always suppress completions. If
* SELECTIVE_COMPLETION is set, honor any setting. Otherwise, always
* deliver a completion. */
if ((flags & GNIX_SUPPRESS_COMPLETION) ||
(ep->send_selective_completion && !(flags & FI_COMPLETION))) {
req->flags &= ~FI_COMPLETION;
} else {
req->flags |= FI_COMPLETION;
}
if (rdma) {
req->flags |= GNIX_RMA_RDMA;
}
GNIX_INFO(FI_LOG_EP_DATA, "Queuing (%p %p %d)\n",
(void *)loc_addr, (void *)rem_addr, len);
return _gnix_vc_queue_tx_req(req);
err_auto_reg:
_gnix_fr_free(req->vc->ep, req);
return rc;
}