static int __gnix_rma_send_data_req(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->req = req; txd->completer_fn = __gnix_rma_txd_data_complete; txd->rma_data_hdr.flags = FI_RMA | FI_REMOTE_CQ_DATA; if (req->type == GNIX_FAB_RQ_RDMA_WRITE) { txd->rma_data_hdr.flags |= FI_REMOTE_WRITE; } else { txd->rma_data_hdr.flags |= FI_REMOTE_READ; } txd->rma_data_hdr.data = req->rma.imm; fastlock_acquire(&nic->lock); if (inject_err) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->rma_data_hdr, sizeof(txd->rma_data_hdr), NULL, 0, txd->id, GNIX_SMSG_T_RMA_DATA); } fastlock_release(&nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else { GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req); } return gnixu_to_fi_errno(status); }
static int __gnix_amo_send_cntr_req(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->req = req; txd->completer_fn = __gnix_amo_txd_cntr_complete; if (req->type == GNIX_FAB_RQ_AMO) { txd->amo_cntr_hdr.flags = FI_REMOTE_WRITE; } else { txd->amo_cntr_hdr.flags = FI_REMOTE_READ; } COND_ACQUIRE(nic->requires_lock, &nic->lock); if (inject_err) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->amo_cntr_hdr, sizeof(txd->amo_cntr_hdr), NULL, 0, txd->id, GNIX_SMSG_T_AMO_CNTR); } COND_RELEASE(nic->requires_lock, &nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else { GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req); } return gnixu_to_fi_errno(status); }
static int __gnix_rndzv_req_send_fin(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_nic *nic; struct gnix_fid_ep *ep; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; GNIX_TRACE(FI_LOG_EP_DATA, "\n"); ep = req->gnix_ep; assert(ep != NULL); nic = ep->nic; assert(nic != NULL); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->rndzv_fin_hdr.req_addr = req->msg.rma_id; txd->req = req; txd->completer_fn = gnix_ep_smsg_completers[GNIX_SMSG_T_RNDZV_FIN]; fastlock_acquire(&nic->lock); status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->rndzv_fin_hdr, sizeof(txd->rndzv_fin_hdr), NULL, 0, txd->id, GNIX_SMSG_T_RNDZV_FIN); if ((status == GNI_RC_SUCCESS) && (ep->domain->data_progress == FI_PROGRESS_AUTO)) _gnix_rma_post_irq(req->vc); fastlock_release(&nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } GNIX_INFO(FI_LOG_EP_DATA, "Initiated RNDZV_FIN, req: %p\n", req); return gnixu_to_fi_errno(status); }
static int __gnix_rma_post_err(struct gnix_tx_descriptor *txd) { struct gnix_fab_req *req = txd->req; int rc; req->tx_failures++; if (req->tx_failures < req->gnix_ep->domain->params.max_retransmits) { _gnix_nic_tx_free(req->gnix_ep->nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "Requeueing failed request: %p\n", req); return _gnix_vc_queue_work_req(req); } GNIX_INFO(FI_LOG_EP_DATA, "Failed %d transmits: %p\n", req->tx_failures, req); rc = __gnix_rma_send_err(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_rma_send_err() failed: %d\n", rc); __gnix_rma_fr_complete(req, txd); return FI_SUCCESS; }
static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc = FI_SUCCESS; _gnix_nic_tx_free(req->vc->ep->nic, txd); if (tx_status != GNI_RC_SUCCESS) { return __gnix_amo_post_err(req, FI_ECANCELED); } if (req->vc->peer_caps & FI_RMA_EVENT) { /* control message needed for a counter event. */ req->work_fn = __gnix_amo_send_cntr_req; _gnix_vc_queue_work_req(req); } else { /* complete request */ rc = __gnix_amo_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_send_completion() failed: %d\n", rc); __gnix_amo_fr_complete(req); } return FI_SUCCESS; }
static void __gnix_msg_send_fr_complete(struct gnix_fab_req *req, struct gnix_tx_descriptor *txd) { atomic_dec(&req->vc->outstanding_tx_reqs); _gnix_nic_tx_free(req->gnix_ep->nic, txd); /* Schedule VC TX queue in case the VC is 'fenced'. */ _gnix_vc_tx_schedule(req->vc); _gnix_fr_free(req->gnix_ep, req); }
static void __gnix_amo_fr_complete(struct gnix_fab_req *req, struct gnix_tx_descriptor *txd) { if (req->flags & FI_LOCAL_MR) { GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n", req->amo.loc_md); fi_close(&req->amo.loc_md->mr_fid.fid); } atomic_dec(&req->vc->outstanding_tx_reqs); _gnix_nic_tx_free(req->vc->ep->nic, txd); /* Schedule VC TX queue in case the VC is 'fenced'. */ _gnix_vc_tx_schedule(req->vc); _gnix_fr_free(req->vc->ep, req); }
static int __gnix_amo_txd_cntr_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc; _gnix_nic_tx_free(req->gnix_ep->nic, txd); if (tx_status != GNI_RC_SUCCESS) return __gnix_amo_post_err(req, FI_ECANCELED); /* Successful data delivery. Generate local completion. */ rc = __gnix_amo_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_send_completion() failed: %d\n", rc); __gnix_amo_fr_complete(req); return FI_SUCCESS; }
int _gnix_amo_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(fab_req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_amo_txd_complete; txd->req = fab_req; /* Mem handle CRC is not validated during FMA operations. Skip this * costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->amo.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md; txd->gni_desc.type = GNI_POST_AMO; txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr; txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.length = fab_req->amo.len; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype, fab_req->amo.op, fab_req->type); txd->gni_desc.first_operand = fab_req->amo.first_operand; txd->gni_desc.second_operand = fab_req->amo.second_operand; { gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->gni_desc.local_addr, txd->gni_desc.remote_addr, txd->gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->amo.rem_mr_key); } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
int _gnix_rma_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int rdma = !!(fab_req->flags & GNIX_RMA_RDMA); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_EAGAIN; } txd->desc.completer_fn = __gnix_rma_txd_complete; txd->desc.req = fab_req; _gnix_convert_key_to_mhdl((gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->loc_md; //txd->desc.gni_desc.post_id = (uint64_t)fab_req; /* unused */ txd->desc.gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma); txd->desc.gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->desc.gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->desc.gni_desc.local_addr = (uint64_t)fab_req->loc_addr; if (loc_md) { txd->desc.gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->desc.gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr; txd->desc.gni_desc.remote_mem_hndl = mdh; txd->desc.gni_desc.length = fab_req->len; txd->desc.gni_desc.rdma_mode = 0; /* check flags */ txd->desc.gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ { gni_mem_handle_t *tl_mdh = &txd->desc.gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->desc.gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->desc.gni_desc.local_addr, txd->desc.gni_desc.remote_addr, txd->desc.gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->rma.rem_mr_key); } fastlock_acquire(&nic->lock); if (rdma) { status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->desc.gni_desc); } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->desc.gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
static int __gnix_rndzv_req_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int ret; if (req->msg.recv_flags & GNIX_MSG_GET_TAIL) { /* There are two TXDs involved with this request, an RDMA * transfer to move the middle block and an FMA transfer to * move unaligned tail data. If this is the FMA TXD, store the * unaligned bytes. Bytes are copied from the request to the * user buffer once both TXDs arrive. */ if (txd->gni_desc.type == GNI_POST_FMA_GET) req->msg.rndzv_tail = *(uint32_t *)txd->int_buf; /* Remember any failure. Retransmit both TXDs once both are * complete. */ req->msg.status |= tx_status; atomic_dec(&req->msg.outstanding_txds); if (atomic_get(&req->msg.outstanding_txds)) { _gnix_nic_tx_free(req->gnix_ep->nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "Received first RDMA chain TXD, req: %p\n", req); return FI_SUCCESS; } tx_status = req->msg.status; } _gnix_nic_tx_free(req->gnix_ep->nic, txd); if (tx_status != GNI_RC_SUCCESS) { req->tx_failures++; if (req->tx_failures < req->gnix_ep->domain->params.max_retransmits) { GNIX_INFO(FI_LOG_EP_DATA, "Requeueing failed request: %p\n", req); return _gnix_vc_queue_work_req(req); } /* TODO should this be fatal? A request will sit waiting at the * peer. */ return __gnix_msg_recv_err(req->gnix_ep, req); } __gnix_msg_copy_unaligned_get_data(req); GNIX_INFO(FI_LOG_EP_DATA, "Completed RNDZV GET, req: %p\n", req); if (req->msg.recv_flags & FI_LOCAL_MR) { GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n", req->msg.recv_md); fi_close(&req->msg.recv_md->mr_fid.fid); } req->work_fn = __gnix_rndzv_req_send_fin; ret = _gnix_vc_queue_work_req(req); return ret; }
int _gnix_rma_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int rdma = !!(fab_req->flags & GNIX_RMA_RDMA); int indirect = !!(fab_req->flags & GNIX_RMA_INDIRECT); int chained = !!(fab_req->flags & GNIX_RMA_CHAINED); int inject_err = _gnix_req_inject_err(fab_req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_rma_txd_complete; txd->req = fab_req; if (rdma) { _gnix_convert_key_to_mhdl( (gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); } else { /* Mem handle CRC is not validated during FMA operations. Skip * this costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); } txd->gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma); txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ if (unlikely(indirect)) { __gnix_rma_fill_pd_indirect_get(fab_req, txd); } else if (unlikely(chained)) { __gnix_rma_fill_pd_chained_get(fab_req, txd, &mdh); } else { txd->gni_desc.local_addr = (uint64_t)fab_req->rma.loc_addr; txd->gni_desc.length = fab_req->rma.len; txd->gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr; loc_md = (struct gnix_fid_mem_desc *)fab_req->rma.loc_md; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } } txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ { gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->gni_desc.local_addr, txd->gni_desc.remote_addr, txd->gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->rma.rem_mr_key); } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else if (chained) { status = GNI_CtPostFma(fab_req->vc->gni_ep, &txd->gni_desc); } else if (rdma) { status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->gni_desc); } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
int _gnix_rma_post_rdma_chain_req(void *data) { struct gnix_fab_req *req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *bte_txd, *ct_txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); int head_off, head_len, tail_len; int fma_chain = 0; rc = _gnix_nic_tx_alloc(nic, &bte_txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "BTE _gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } rc = _gnix_nic_tx_alloc(nic, &ct_txd); if (rc) { _gnix_nic_tx_free(nic, bte_txd); GNIX_INFO(FI_LOG_EP_DATA, "CT _gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } _gnix_convert_key_to_mhdl( (gnix_mr_key_t *)&req->rma.rem_mr_key, &mdh); /* BTE TXD */ bte_txd->completer_fn = __gnix_rma_txd_complete; bte_txd->req = req; bte_txd->gni_desc.type = GNI_POST_RDMA_GET; bte_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ bte_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK; head_len = head_off ? GNI_READ_ALIGN - head_off : 0; tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK; bte_txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len; bte_txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len; bte_txd->gni_desc.length = req->rma.len - head_len - tail_len; bte_txd->gni_desc.remote_mem_hndl = mdh; bte_txd->gni_desc.rdma_mode = 0; /* check flags */ bte_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ bte_txd->gni_desc.local_mem_hndl = req->rma.loc_md->mem_hndl; /* FMA TXD */ ct_txd->completer_fn = __gnix_rma_txd_complete; ct_txd->req = req; ct_txd->gni_desc.type = GNI_POST_FMA_GET; ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ ct_txd->gni_desc.remote_mem_hndl = mdh; ct_txd->gni_desc.rdma_mode = 0; /* check flags */ ct_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ ct_txd->gni_desc.local_mem_hndl = nic->int_bufs_mdh; ct_txd->gni_desc.length = GNI_READ_ALIGN; if (head_off) { ct_txd->gni_desc.remote_addr = req->rma.rem_addr & ~GNI_READ_ALIGN_MASK; ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf; if (tail_len) { ct_txd->gni_desc.next_descr = &ct_txd->gni_ct_descs[0]; ct_txd->gni_ct_descs[0].ep_hndl = req->vc->gni_ep; ct_txd->gni_ct_descs[0].length = GNI_READ_ALIGN; ct_txd->gni_ct_descs[0].remote_addr = (req->rma.rem_addr + req->rma.len) & ~GNI_READ_ALIGN_MASK; ct_txd->gni_ct_descs[0].remote_mem_hndl = mdh; ct_txd->gni_ct_descs[0].local_addr = (uint64_t)ct_txd->int_buf + GNI_READ_ALIGN; ct_txd->gni_ct_descs[0].local_mem_hndl = nic->int_bufs_mdh; ct_txd->gni_ct_descs[0].next_descr = NULL; fma_chain = 1; } } else { ct_txd->gni_desc.remote_addr = (req->rma.rem_addr + req->rma.len) & ~GNI_READ_ALIGN_MASK; ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf + GNI_READ_ALIGN; } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, bte_txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostRdma(req->vc->gni_ep, &bte_txd->gni_desc); } if (status != GNI_RC_SUCCESS) { fastlock_release(&nic->lock); _gnix_nic_tx_free(nic, ct_txd); _gnix_nic_tx_free(nic, bte_txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); return gnixu_to_fi_errno(status); } if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, ct_txd); status = GNI_RC_SUCCESS; } else if (fma_chain) { status = GNI_CtPostFma(req->vc->gni_ep, &ct_txd->gni_desc); } else { status = GNI_PostFma(req->vc->gni_ep, &ct_txd->gni_desc); } if (status != GNI_RC_SUCCESS) { fastlock_release(&nic->lock); _gnix_nic_tx_free(nic, ct_txd); /* Wait for the first TX to complete, then retransmit the * entire thing. */ atomic_set(&req->rma.outstanding_txds, 1); req->rma.status = GNI_RC_TRANSACTION_ERROR; GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); return FI_SUCCESS; } fastlock_release(&nic->lock); /* Wait for both TXs to complete, then process the request. */ atomic_set(&req->rma.outstanding_txds, 2); req->rma.status = 0; return FI_SUCCESS; }
static int __gnix_rma_txd_complete(void *arg, gni_return_t tx_status) { struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg; struct gnix_fab_req *req = txd->req; int rc = FI_SUCCESS; /* Wait for both TXDs before processing RDMA chained requests. */ if (req->flags & GNIX_RMA_CHAINED && req->flags & GNIX_RMA_RDMA) { /* There are two TXDs involved with this request, an RDMA * transfer to move the middle block and an FMA transfer to * move unaligned head and/or tail. If this is the FMA TXD, * copy the unaligned data to the user buffer. */ if (txd->gni_desc.type == GNI_POST_FMA_GET) __gnix_rma_copy_chained_get_data(txd); /* Remember any failure. Retransmit both TXDs once both are * complete. */ req->rma.status |= tx_status; atomic_dec(&req->rma.outstanding_txds); if (atomic_get(&req->rma.outstanding_txds)) { _gnix_nic_tx_free(req->gnix_ep->nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "Received first RDMA chain TXD, req: %p\n", req); return FI_SUCCESS; } tx_status = req->rma.status; } if (tx_status != GNI_RC_SUCCESS) { return __gnix_rma_post_err(txd); } /* Successful delivery. Progress request. */ if (req->flags & FI_REMOTE_CQ_DATA) { /* initiate immediate data transfer */ req->tx_failures = 0; req->work_fn = __gnix_rma_send_data_req; _gnix_vc_queue_work_req(req); } else { if (req->flags & GNIX_RMA_INDIRECT) { __gnix_rma_copy_indirect_get_data(txd); } else if (req->flags & GNIX_RMA_CHAINED && !(req->flags & GNIX_RMA_RDMA)) { __gnix_rma_copy_chained_get_data(txd); } /* complete request */ rc = __gnix_rma_send_completion(req->vc->ep, req); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_rma_send_completion() failed: %d\n", rc); __gnix_rma_fr_complete(req, txd); } return FI_SUCCESS; }
int _gnix_amo_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(fab_req); if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) { GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags, fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS)); GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n", ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS)); GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n", fab_req->vc->peer_caps, fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS)); rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_post_err() failed: %d\n", rc); return -FI_ECANCELED; } rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_amo_txd_complete; txd->req = fab_req; /* Mem handle CRC is not validated during FMA operations. Skip this * costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->amo.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md; txd->gni_desc.type = GNI_POST_AMO; txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr; txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.length = fab_req->amo.len; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype, fab_req->amo.op, fab_req->type); txd->gni_desc.first_operand = fab_req->amo.first_operand; txd->gni_desc.second_operand = fab_req->amo.second_operand; GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n", txd->gni_desc.first_operand, txd->gni_desc.second_operand); GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n", txd->gni_desc.amo_cmd); GNIX_LOG_DUMP_TXD(txd); COND_ACQUIRE(nic->requires_lock, &nic->lock); if (OFI_UNLIKELY(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } COND_RELEASE(nic->requires_lock, &nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }