static int __gnix_amo_send_completion(struct gnix_fid_ep *ep, struct gnix_fab_req *req) { struct gnix_fid_cntr *cntr = NULL; int rc = FI_SUCCESS; uint64_t flags = req->flags & GNIX_AMO_COMPLETION_FLAGS; if ((req->flags & FI_COMPLETION) && ep->send_cq) { rc = _gnix_cq_add_event(ep->send_cq, req->user_context, flags, 0, 0, 0, 0, FI_ADDR_NOTAVAIL); if (rc) { GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cq_add_event() failed: %d\n", rc); } } if ((req->type == GNIX_FAB_RQ_AMO) && ep->write_cntr) { cntr = ep->write_cntr; } else if ((req->type == GNIX_FAB_RQ_FAMO || req->type == GNIX_FAB_RQ_CAMO) && ep->read_cntr) { cntr = ep->read_cntr; } if (cntr) { rc = _gnix_cntr_inc(cntr); if (rc) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cntr_inc() failed: %d\n", rc); } return FI_SUCCESS; }
static int __gnix_msg_send_completion(struct gnix_fid_ep *ep, struct gnix_fab_req *req) { uint64_t flags = FI_SEND | FI_MSG; int rc; flags |= req->msg.send_flags & FI_TAGGED; if ((req->msg.send_flags & FI_COMPLETION) && ep->send_cq) { rc = _gnix_cq_add_event(ep->send_cq, req->user_context, flags, 0, 0, 0, 0, FI_ADDR_NOTAVAIL); if (rc != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cq_add_event returned %d\n", rc); } } if (ep->send_cntr) { rc = _gnix_cntr_inc(ep->send_cntr); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cntr_inc() failed: %d\n", rc); } return FI_SUCCESS; }
static int __recv_completion( struct gnix_fid_ep *ep, struct gnix_fab_req *req, void *context, uint64_t flags, size_t len, void *addr, uint64_t data, uint64_t tag, fi_addr_t src_addr) { int rc; if ((req->msg.recv_flags & FI_COMPLETION) && ep->recv_cq) { rc = _gnix_cq_add_event(ep->recv_cq, context, flags, len, addr, data, tag, src_addr); if (rc != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cq_add_event returned %d\n", rc); } } if (ep->recv_cntr) { rc = _gnix_cntr_inc(ep->recv_cntr); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cntr_inc() failed: %d\n", rc); } return FI_SUCCESS; }
static int __gnix_rma_fab_req_complete(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_fid_ep *ep = req->gnix_ep; int rc; struct gnix_fid_cntr *cntr = NULL; /* more transaction needed for request? */ if (req->flags & FI_COMPLETION) { rc = _gnix_cq_add_event(ep->send_cq, req->user_context, req->flags, req->len, (void *)req->loc_addr, req->imm, req->msg.tag); if (rc) { GNIX_WARN(FI_LOG_CQ, "_gnix_cq_add_event() failed: %d\n", rc); } } if ((req->type == GNIX_FAB_RQ_RDMA_WRITE) && ep->write_cntr) cntr = ep->write_cntr; if ((req->type == GNIX_FAB_RQ_RDMA_READ) && ep->read_cntr) cntr = ep->read_cntr; if (cntr) { rc = _gnix_cntr_inc(cntr); if (rc) GNIX_WARN(FI_LOG_CQ, "_gnix_cntr_inc() failed: %d\n", rc); } atomic_dec(&req->vc->outstanding_tx_reqs); /* We could have requests waiting for TXDs or FI_FENCE operations. * Schedule this VC to push any such TXs. */ _gnix_vc_schedule_tx(req->vc); _gnix_fr_free(ep, req); return FI_SUCCESS; }