/* SMSG callback for AMO remote counter control message. */
int __smsg_amo_cntr(void *data, void *msg)
{
int ret = FI_SUCCESS;
struct gnix_vc *vc = (struct gnix_vc *)data;
struct gnix_smsg_amo_cntr_hdr *hdr =
(struct gnix_smsg_amo_cntr_hdr *)msg;
struct gnix_fid_ep *ep = vc->ep;
gni_return_t status;
if (hdr->flags & FI_REMOTE_WRITE && ep->rwrite_cntr) {
ret = _gnix_cntr_inc(ep->rwrite_cntr);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
ret);
}
if (hdr->flags & FI_REMOTE_READ && ep->rread_cntr) {
ret = _gnix_cntr_inc(ep->rread_cntr);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
ret);
}
status = GNI_SmsgRelease(vc->gni_ep);
if (OFI_UNLIKELY(status != GNI_RC_SUCCESS)) {
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgRelease returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
}
return ret;
}
static int __gnix_amo_send_completion(struct gnix_fid_ep *ep,
struct gnix_fab_req *req)
{
struct gnix_fid_cntr *cntr = NULL;
int rc = FI_SUCCESS;
uint64_t flags = req->flags & GNIX_AMO_COMPLETION_FLAGS;
if ((req->flags & FI_COMPLETION) && ep->send_cq) {
rc = _gnix_cq_add_event(ep->send_cq, req->user_context,
flags, 0, 0, 0, 0, FI_ADDR_NOTAVAIL);
if (rc) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cq_add_event() failed: %d\n", rc);
}
}
if ((req->type == GNIX_FAB_RQ_AMO) &&
ep->write_cntr) {
cntr = ep->write_cntr;
} else if ((req->type == GNIX_FAB_RQ_FAMO ||
req->type == GNIX_FAB_RQ_CAMO) &&
ep->read_cntr) {
cntr = ep->read_cntr;
}
if (cntr) {
rc = _gnix_cntr_inc(cntr);
if (rc)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n", rc);
}
return FI_SUCCESS;
}
static int __gnix_msg_send_completion(struct gnix_fid_ep *ep,
struct gnix_fab_req *req)
{
uint64_t flags = FI_SEND | FI_MSG;
int rc;
flags |= req->msg.send_flags & FI_TAGGED;
if ((req->msg.send_flags & FI_COMPLETION) && ep->send_cq) {
rc = _gnix_cq_add_event(ep->send_cq,
req->user_context,
flags, 0, 0, 0, 0, FI_ADDR_NOTAVAIL);
if (rc != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cq_add_event returned %d\n",
rc);
}
}
if (ep->send_cntr) {
rc = _gnix_cntr_inc(ep->send_cntr);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
rc);
}
return FI_SUCCESS;
}
static int __recv_completion(
struct gnix_fid_ep *ep,
struct gnix_fab_req *req,
void *context,
uint64_t flags,
size_t len,
void *addr,
uint64_t data,
uint64_t tag,
fi_addr_t src_addr)
{
int rc;
if ((req->msg.recv_flags & FI_COMPLETION) && ep->recv_cq) {
rc = _gnix_cq_add_event(ep->recv_cq, context, flags, len,
addr, data, tag, src_addr);
if (rc != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cq_add_event returned %d\n",
rc);
}
}
if (ep->recv_cntr) {
rc = _gnix_cntr_inc(ep->recv_cntr);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
rc);
}
return FI_SUCCESS;
}
static int __gnix_rma_fab_req_complete(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
int rc;
struct gnix_fid_cntr *cntr = NULL;
/* more transaction needed for request? */
if (req->flags & FI_COMPLETION) {
rc = _gnix_cq_add_event(ep->send_cq, req->user_context,
req->flags, req->len,
(void *)req->loc_addr,
req->imm, req->msg.tag);
if (rc) {
GNIX_WARN(FI_LOG_CQ,
"_gnix_cq_add_event() failed: %d\n", rc);
}
}
if ((req->type == GNIX_FAB_RQ_RDMA_WRITE) &&
ep->write_cntr)
cntr = ep->write_cntr;
if ((req->type == GNIX_FAB_RQ_RDMA_READ) &&
ep->read_cntr)
cntr = ep->read_cntr;
if (cntr) {
rc = _gnix_cntr_inc(cntr);
if (rc)
GNIX_WARN(FI_LOG_CQ,
"_gnix_cntr_inc() failed: %d\n", rc);
}
atomic_dec(&req->vc->outstanding_tx_reqs);
/* We could have requests waiting for TXDs or FI_FENCE operations.
* Schedule this VC to push any such TXs. */
_gnix_vc_schedule_tx(req->vc);
_gnix_fr_free(ep, req);
return FI_SUCCESS;
}