static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status)
{
struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
struct gnix_fab_req *req = txd->req;
int rc = FI_SUCCESS;
_gnix_nic_tx_free(req->vc->ep->nic, txd);
if (tx_status != GNI_RC_SUCCESS) {
return __gnix_amo_post_err(req, FI_ECANCELED);
}
if (req->vc->peer_caps & FI_RMA_EVENT) {
/* control message needed for a counter event. */
req->work_fn = __gnix_amo_send_cntr_req;
_gnix_vc_queue_work_req(req);
} else {
/* complete request */
rc = __gnix_amo_send_completion(req->vc->ep, req);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_send_completion() failed: %d\n",
rc);
__gnix_amo_fr_complete(req);
}
return FI_SUCCESS;
}
static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status)
{
struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
struct gnix_fab_req *req = txd->req;
int rc = FI_SUCCESS;
uint32_t read_data32;
uint64_t read_data64;
if (tx_status != GNI_RC_SUCCESS) {
return __gnix_amo_post_err(txd);
}
/* FI_ATOMIC_READ data is delivered to operand buffer in addition to
* the results buffer. */
if (req->amo.op == FI_ATOMIC_READ) {
switch(fi_datatype_size(req->amo.datatype)) {
case sizeof(uint32_t):
read_data32 = *(uint32_t *)req->amo.loc_addr;
*(uint32_t *)req->amo.read_buf = read_data32;
break;
case sizeof(uint64_t):
read_data64 = *(uint64_t *)req->amo.loc_addr;
*(uint64_t *)req->amo.read_buf = read_data64;
break;
default:
GNIX_WARN(FI_LOG_EP_DATA, "Invalid datatype: %d\n",
req->amo.datatype);
assert(0);
break;
}
}
/* complete request */
rc = __gnix_amo_send_completion(req->vc->ep, req);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_send_completion() failed: %d\n",
rc);
__gnix_amo_fr_complete(req, txd);
return FI_SUCCESS;
}
static int __gnix_amo_txd_cntr_complete(void *arg, gni_return_t tx_status)
{
struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
struct gnix_fab_req *req = txd->req;
int rc;
_gnix_nic_tx_free(req->gnix_ep->nic, txd);
if (tx_status != GNI_RC_SUCCESS)
return __gnix_amo_post_err(req, FI_ECANCELED);
/* Successful data delivery. Generate local completion. */
rc = __gnix_amo_send_completion(req->vc->ep, req);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_send_completion() failed: %d\n",
rc);
__gnix_amo_fr_complete(req);
return FI_SUCCESS;
}
int _gnix_amo_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(fab_req);
if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags,
fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS));
GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n",
ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS));
GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n",
fab_req->vc->peer_caps,
fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS));
rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_post_err() failed: %d\n", rc);
return -FI_ECANCELED;
}
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_amo_txd_complete;
txd->req = fab_req;
/* Mem handle CRC is not validated during FMA operations. Skip this
* costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
(gnix_mr_key_t *)&fab_req->amo.rem_mr_key,
&mdh);
loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md;
txd->gni_desc.type = GNI_POST_AMO;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr;
if (loc_md) {
txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr;
txd->gni_desc.remote_mem_hndl = mdh;
txd->gni_desc.length = fab_req->amo.len;
txd->gni_desc.rdma_mode = 0; /* check flags */
txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype,
fab_req->amo.op,
fab_req->type);
txd->gni_desc.first_operand = fab_req->amo.first_operand;
txd->gni_desc.second_operand = fab_req->amo.second_operand;
GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n",
txd->gni_desc.first_operand, txd->gni_desc.second_operand);
GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n",
txd->gni_desc.amo_cmd);
GNIX_LOG_DUMP_TXD(txd);
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (OFI_UNLIKELY(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
