static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc = FI_SUCCESS; _gnix_nic_tx_free(req->vc->ep->nic, txd); if (tx_status != GNI_RC_SUCCESS) { return __gnix_amo_post_err(req, FI_ECANCELED); } if (req->vc->peer_caps & FI_RMA_EVENT) { /* control message needed for a counter event. */ req->work_fn = __gnix_amo_send_cntr_req; _gnix_vc_queue_work_req(req); } else { /* complete request */ rc = __gnix_amo_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_send_completion() failed: %d\n", rc); __gnix_amo_fr_complete(req); } return FI_SUCCESS; }
static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc = FI_SUCCESS; uint32_t read_data32; uint64_t read_data64; if (tx_status != GNI_RC_SUCCESS) { return __gnix_amo_post_err(txd); } /* FI_ATOMIC_READ data is delivered to operand buffer in addition to * the results buffer. */ if (req->amo.op == FI_ATOMIC_READ) { switch(fi_datatype_size(req->amo.datatype)) { case sizeof(uint32_t): read_data32 = *(uint32_t *)req->amo.loc_addr; *(uint32_t *)req->amo.read_buf = read_data32; break; case sizeof(uint64_t): read_data64 = *(uint64_t *)req->amo.loc_addr; *(uint64_t *)req->amo.read_buf = read_data64; break; default: GNIX_WARN(FI_LOG_EP_DATA, "Invalid datatype: %d\n", req->amo.datatype); assert(0); break; } } /* complete request */ rc = __gnix_amo_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_send_completion() failed: %d\n", rc); __gnix_amo_fr_complete(req, txd); return FI_SUCCESS; }
static int __gnix_amo_txd_cntr_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc; _gnix_nic_tx_free(req->gnix_ep->nic, txd); if (tx_status != GNI_RC_SUCCESS) return __gnix_amo_post_err(req, FI_ECANCELED); /* Successful data delivery. Generate local completion. */ rc = __gnix_amo_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_send_completion() failed: %d\n", rc); __gnix_amo_fr_complete(req); return FI_SUCCESS; }
int _gnix_amo_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(fab_req); if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) { GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags, fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS)); GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n", ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS)); GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n", fab_req->vc->peer_caps, fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS)); rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_post_err() failed: %d\n", rc); return -FI_ECANCELED; } rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_amo_txd_complete; txd->req = fab_req; /* Mem handle CRC is not validated during FMA operations. Skip this * costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->amo.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md; txd->gni_desc.type = GNI_POST_AMO; txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr; txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.length = fab_req->amo.len; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype, fab_req->amo.op, fab_req->type); txd->gni_desc.first_operand = fab_req->amo.first_operand; txd->gni_desc.second_operand = fab_req->amo.second_operand; GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n", txd->gni_desc.first_operand, txd->gni_desc.second_operand); GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n", txd->gni_desc.amo_cmd); GNIX_LOG_DUMP_TXD(txd); COND_ACQUIRE(nic->requires_lock, &nic->lock); if (OFI_UNLIKELY(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } COND_RELEASE(nic->requires_lock, &nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }