static int __gnix_rma_send_data_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->req = req;
txd->completer_fn = __gnix_rma_txd_data_complete;
txd->rma_data_hdr.flags = FI_RMA | FI_REMOTE_CQ_DATA;
if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
txd->rma_data_hdr.flags |= FI_REMOTE_WRITE;
} else {
txd->rma_data_hdr.flags |= FI_REMOTE_READ;
}
txd->rma_data_hdr.data = req->rma.imm;
fastlock_acquire(&nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->rma_data_hdr,
sizeof(txd->rma_data_hdr),
NULL, 0, txd->id,
GNIX_SMSG_T_RMA_DATA);
}
fastlock_release(&nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else {
GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req);
}
return gnixu_to_fi_errno(status);
}
static int __gnix_amo_send_cntr_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->req = req;
txd->completer_fn = __gnix_amo_txd_cntr_complete;
if (req->type == GNIX_FAB_RQ_AMO) {
txd->amo_cntr_hdr.flags = FI_REMOTE_WRITE;
} else {
txd->amo_cntr_hdr.flags = FI_REMOTE_READ;
}
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->amo_cntr_hdr,
sizeof(txd->amo_cntr_hdr),
NULL, 0, txd->id,
GNIX_SMSG_T_AMO_CNTR);
}
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else {
GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req);
}
return gnixu_to_fi_errno(status);
}
static int __gnix_rndzv_req_send_fin(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_nic *nic;
struct gnix_fid_ep *ep;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
GNIX_TRACE(FI_LOG_EP_DATA, "\n");
ep = req->gnix_ep;
assert(ep != NULL);
nic = ep->nic;
assert(nic != NULL);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->rndzv_fin_hdr.req_addr = req->msg.rma_id;
txd->req = req;
txd->completer_fn = gnix_ep_smsg_completers[GNIX_SMSG_T_RNDZV_FIN];
fastlock_acquire(&nic->lock);
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->rndzv_fin_hdr, sizeof(txd->rndzv_fin_hdr),
NULL, 0, txd->id, GNIX_SMSG_T_RNDZV_FIN);
if ((status == GNI_RC_SUCCESS) &&
(ep->domain->data_progress == FI_PROGRESS_AUTO))
_gnix_rma_post_irq(req->vc);
fastlock_release(&nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
}
GNIX_INFO(FI_LOG_EP_DATA, "Initiated RNDZV_FIN, req: %p\n", req);
return gnixu_to_fi_errno(status);
}
int _gnix_amo_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(fab_req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_amo_txd_complete;
txd->req = fab_req;
/* Mem handle CRC is not validated during FMA operations. Skip this
* costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
(gnix_mr_key_t *)&fab_req->amo.rem_mr_key,
&mdh);
loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md;
txd->gni_desc.type = GNI_POST_AMO;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr;
if (loc_md) {
txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr;
txd->gni_desc.remote_mem_hndl = mdh;
txd->gni_desc.length = fab_req->amo.len;
txd->gni_desc.rdma_mode = 0; /* check flags */
txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype,
fab_req->amo.op,
fab_req->type);
txd->gni_desc.first_operand = fab_req->amo.first_operand;
txd->gni_desc.second_operand = fab_req->amo.second_operand;
{
gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl;
gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl;
GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n",
txd->gni_desc.local_addr, txd->gni_desc.remote_addr,
txd->gni_desc.length);
GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n",
*(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1),
*(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1),
fab_req->amo.rem_mr_key);
}
fastlock_acquire(&nic->lock);
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
fastlock_release(&nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
int _gnix_rma_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int rdma = !!(fab_req->flags & GNIX_RMA_RDMA);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_EAGAIN;
}
txd->desc.completer_fn = __gnix_rma_txd_complete;
txd->desc.req = fab_req;
_gnix_convert_key_to_mhdl((gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
&mdh);
loc_md = (struct gnix_fid_mem_desc *)fab_req->loc_md;
//txd->desc.gni_desc.post_id = (uint64_t)fab_req; /* unused */
txd->desc.gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma);
txd->desc.gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->desc.gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
txd->desc.gni_desc.local_addr = (uint64_t)fab_req->loc_addr;
if (loc_md) {
txd->desc.gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
txd->desc.gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr;
txd->desc.gni_desc.remote_mem_hndl = mdh;
txd->desc.gni_desc.length = fab_req->len;
txd->desc.gni_desc.rdma_mode = 0; /* check flags */
txd->desc.gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
{
gni_mem_handle_t *tl_mdh = &txd->desc.gni_desc.local_mem_hndl;
gni_mem_handle_t *tr_mdh = &txd->desc.gni_desc.remote_mem_hndl;
GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n",
txd->desc.gni_desc.local_addr, txd->desc.gni_desc.remote_addr,
txd->desc.gni_desc.length);
GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n",
*(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1),
*(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1),
fab_req->rma.rem_mr_key);
}
fastlock_acquire(&nic->lock);
if (rdma) {
status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->desc.gni_desc);
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->desc.gni_desc);
}
fastlock_release(&nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
(void *)req->msg.recv_addr, req->msg.recv_len,
FI_READ | FI_WRITE, 0, 0, 0, &auto_mr, NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
return -FI_EAGAIN;
}
req->msg.recv_flags |= FI_LOCAL_MR;
req->msg.recv_md = container_of(auto_mr,
struct gnix_fid_mem_desc,
mr_fid);
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_rndzv_req_complete;
txd->req = req;
use_tx_cq_blk = (ep->domain->data_progress == FI_PROGRESS_AUTO) ? 1 : 0;
txd->gni_desc.type = GNI_POST_RDMA_GET;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
txd->gni_desc.local_mem_hndl = req->msg.recv_md->mem_hndl;
int _gnix_rma_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int rdma = !!(fab_req->flags & GNIX_RMA_RDMA);
int indirect = !!(fab_req->flags & GNIX_RMA_INDIRECT);
int chained = !!(fab_req->flags & GNIX_RMA_CHAINED);
int inject_err = _gnix_req_inject_err(fab_req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_rma_txd_complete;
txd->req = fab_req;
if (rdma) {
_gnix_convert_key_to_mhdl(
(gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
&mdh);
} else {
/* Mem handle CRC is not validated during FMA operations. Skip
* this costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
(gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
&mdh);
}
txd->gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma);
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
if (unlikely(indirect)) {
__gnix_rma_fill_pd_indirect_get(fab_req, txd);
} else if (unlikely(chained)) {
__gnix_rma_fill_pd_chained_get(fab_req, txd, &mdh);
} else {
txd->gni_desc.local_addr = (uint64_t)fab_req->rma.loc_addr;
txd->gni_desc.length = fab_req->rma.len;
txd->gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr;
loc_md = (struct gnix_fid_mem_desc *)fab_req->rma.loc_md;
if (loc_md) {
txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
}
txd->gni_desc.remote_mem_hndl = mdh;
txd->gni_desc.rdma_mode = 0; /* check flags */
txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
{
gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl;
gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl;
GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n",
txd->gni_desc.local_addr, txd->gni_desc.remote_addr,
txd->gni_desc.length);
GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n",
*(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1),
*(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1),
fab_req->rma.rem_mr_key);
}
fastlock_acquire(&nic->lock);
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else if (chained) {
status = GNI_CtPostFma(fab_req->vc->gni_ep, &txd->gni_desc);
} else if (rdma) {
status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->gni_desc);
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
fastlock_release(&nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
int _gnix_rma_post_rdma_chain_req(void *data)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *bte_txd, *ct_txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(req);
int head_off, head_len, tail_len;
int fma_chain = 0;
rc = _gnix_nic_tx_alloc(nic, &bte_txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"BTE _gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
rc = _gnix_nic_tx_alloc(nic, &ct_txd);
if (rc) {
_gnix_nic_tx_free(nic, bte_txd);
GNIX_INFO(FI_LOG_EP_DATA,
"CT _gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
_gnix_convert_key_to_mhdl(
(gnix_mr_key_t *)&req->rma.rem_mr_key,
&mdh);
/* BTE TXD */
bte_txd->completer_fn = __gnix_rma_txd_complete;
bte_txd->req = req;
bte_txd->gni_desc.type = GNI_POST_RDMA_GET;
bte_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
bte_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK;
bte_txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len;
bte_txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len;
bte_txd->gni_desc.length = req->rma.len - head_len - tail_len;
bte_txd->gni_desc.remote_mem_hndl = mdh;
bte_txd->gni_desc.rdma_mode = 0; /* check flags */
bte_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
bte_txd->gni_desc.local_mem_hndl = req->rma.loc_md->mem_hndl;
/* FMA TXD */
ct_txd->completer_fn = __gnix_rma_txd_complete;
ct_txd->req = req;
ct_txd->gni_desc.type = GNI_POST_FMA_GET;
ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
ct_txd->gni_desc.remote_mem_hndl = mdh;
ct_txd->gni_desc.rdma_mode = 0; /* check flags */
ct_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
ct_txd->gni_desc.local_mem_hndl = nic->int_bufs_mdh;
ct_txd->gni_desc.length = GNI_READ_ALIGN;
if (head_off) {
ct_txd->gni_desc.remote_addr =
req->rma.rem_addr & ~GNI_READ_ALIGN_MASK;
ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf;
if (tail_len) {
ct_txd->gni_desc.next_descr = &ct_txd->gni_ct_descs[0];
ct_txd->gni_ct_descs[0].ep_hndl = req->vc->gni_ep;
ct_txd->gni_ct_descs[0].length = GNI_READ_ALIGN;
ct_txd->gni_ct_descs[0].remote_addr =
(req->rma.rem_addr +
req->rma.len) & ~GNI_READ_ALIGN_MASK;
ct_txd->gni_ct_descs[0].remote_mem_hndl = mdh;
ct_txd->gni_ct_descs[0].local_addr =
(uint64_t)ct_txd->int_buf +
GNI_READ_ALIGN;
ct_txd->gni_ct_descs[0].local_mem_hndl =
nic->int_bufs_mdh;
ct_txd->gni_ct_descs[0].next_descr = NULL;
fma_chain = 1;
}
} else {
ct_txd->gni_desc.remote_addr =
(req->rma.rem_addr +
req->rma.len) & ~GNI_READ_ALIGN_MASK;
ct_txd->gni_desc.local_addr =
(uint64_t)ct_txd->int_buf + GNI_READ_ALIGN;
}
fastlock_acquire(&nic->lock);
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, bte_txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_PostRdma(req->vc->gni_ep,
&bte_txd->gni_desc);
}
if (status != GNI_RC_SUCCESS) {
fastlock_release(&nic->lock);
_gnix_nic_tx_free(nic, ct_txd);
_gnix_nic_tx_free(nic, bte_txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
return gnixu_to_fi_errno(status);
}
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, ct_txd);
status = GNI_RC_SUCCESS;
} else if (fma_chain) {
status = GNI_CtPostFma(req->vc->gni_ep,
&ct_txd->gni_desc);
} else {
status = GNI_PostFma(req->vc->gni_ep,
&ct_txd->gni_desc);
}
if (status != GNI_RC_SUCCESS) {
fastlock_release(&nic->lock);
_gnix_nic_tx_free(nic, ct_txd);
/* Wait for the first TX to complete, then retransmit the
* entire thing. */
atomic_set(&req->rma.outstanding_txds, 1);
req->rma.status = GNI_RC_TRANSACTION_ERROR;
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
return FI_SUCCESS;
}
fastlock_release(&nic->lock);
/* Wait for both TXs to complete, then process the request. */
atomic_set(&req->rma.outstanding_txds, 2);
req->rma.status = 0;
return FI_SUCCESS;
}
int _gnix_amo_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(fab_req);
if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags,
fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS));
GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n",
ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS));
GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n",
fab_req->vc->peer_caps,
fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS));
rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_post_err() failed: %d\n", rc);
return -FI_ECANCELED;
}
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_amo_txd_complete;
txd->req = fab_req;
/* Mem handle CRC is not validated during FMA operations. Skip this
* costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
(gnix_mr_key_t *)&fab_req->amo.rem_mr_key,
&mdh);
loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md;
txd->gni_desc.type = GNI_POST_AMO;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr;
if (loc_md) {
txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr;
txd->gni_desc.remote_mem_hndl = mdh;
txd->gni_desc.length = fab_req->amo.len;
txd->gni_desc.rdma_mode = 0; /* check flags */
txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype,
fab_req->amo.op,
fab_req->type);
txd->gni_desc.first_operand = fab_req->amo.first_operand;
txd->gni_desc.second_operand = fab_req->amo.second_operand;
GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n",
txd->gni_desc.first_operand, txd->gni_desc.second_operand);
GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n",
txd->gni_desc.amo_cmd);
GNIX_LOG_DUMP_TXD(txd);
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (OFI_UNLIKELY(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
