/* SMSG callback for AMO remote counter control message. */
int __smsg_amo_cntr(void *data, void *msg)
{
int ret = FI_SUCCESS;
struct gnix_vc *vc = (struct gnix_vc *)data;
struct gnix_smsg_amo_cntr_hdr *hdr =
(struct gnix_smsg_amo_cntr_hdr *)msg;
struct gnix_fid_ep *ep = vc->ep;
gni_return_t status;
if (hdr->flags & FI_REMOTE_WRITE && ep->rwrite_cntr) {
ret = _gnix_cntr_inc(ep->rwrite_cntr);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
ret);
}
if (hdr->flags & FI_REMOTE_READ && ep->rread_cntr) {
ret = _gnix_cntr_inc(ep->rread_cntr);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_cntr_inc() failed: %d\n",
ret);
}
status = GNI_SmsgRelease(vc->gni_ep);
if (OFI_UNLIKELY(status != GNI_RC_SUCCESS)) {
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgRelease returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
}
return ret;
}
static void process_mbox(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_ep_t *ep){
ucs_status_t status;
uint8_t tag;
void *data_ptr;
gni_return_t ugni_rc;
uct_ugni_smsg_header_t *header;
void *user_data;
pthread_mutex_lock(&uct_ugni_global_lock);
while(1){
tag = GNI_SMSG_ANY_TAG;
ugni_rc = GNI_SmsgGetNextWTag(ep->super.ep, (void **)&data_ptr, &tag);
/* Yes, GNI_RC_NOT_DONE means that you're done with the smsg mailbox */
if(GNI_RC_NOT_DONE == ugni_rc){
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
if(GNI_RC_SUCCESS != ugni_rc){
ucs_error("Unhandled smsg error: %s %d", gni_err_str[ugni_rc], ugni_rc);
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
if(NULL == data_ptr){
ucs_error("Empty data pointer in smsg.");
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
header = (uct_ugni_smsg_header_t *)data_ptr;
user_data = (void *)(header + 1);
void *user_desc = iface->user_desc+1;
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_RECV,
tag, user_data, header->length, "RX: AM");
pthread_mutex_unlock(&uct_ugni_global_lock);
status = uct_iface_invoke_am(&iface->super.super, tag, user_data,
header->length, user_desc);
pthread_mutex_lock(&uct_ugni_global_lock);
if(status != UCS_OK){
uct_recv_desc_iface(user_desc) = &iface->super.super.super;
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
iface->user_desc, iface->user_desc = NULL);
}
ugni_rc = GNI_SmsgRelease(ep->super.ep);
if(GNI_RC_SUCCESS != ugni_rc){
ucs_error("Unhandled smsg error in GNI_SmsgRelease: %s %d", gni_err_str[ugni_rc], ugni_rc);
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
}
}
static int __smsg_eager_msg_w_data(void *data, void *msg)
{
int ret = FI_SUCCESS;
gni_return_t status;
struct gnix_vc *vc = (struct gnix_vc *)data;
struct gnix_smsg_hdr *hdr = (struct gnix_smsg_hdr *)msg;
struct gnix_fid_ep *ep;
ep = vc->ep;
assert(ep);
/*
* the msg data must be consumed either by matching
* a message or allocating a buffer and putting
* on unexpected queue.
*/
ret = _gnix_ep_eager_msg_w_data_match(ep,
(void *)((char *)msg + sizeof(*hdr)),
vc->peer_addr,
hdr->len,
hdr->imm,
hdr->flags);
/*
* we keep on going even if we got an error back because
* we need to release the pending message in the SMSG buffer
* and unlock the nic lock
*/
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_ep_eager_msg_rcv returned %d\n",
ret);
status = GNI_SmsgRelease(vc->gni_ep);
if (unlikely(status != GNI_RC_SUCCESS)) {
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgRelease returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
err:
return ret;
}
/* progress */
int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
{
mca_btl_active_message_callback_t *reg;
mca_btl_ugni_base_frag_t frag;
mca_btl_base_segment_t seg;
bool disconnect = false;
uintptr_t data_ptr;
gni_return_t rc;
uint32_t len;
int count = 0;
if (!opal_atomic_cmpset_32 (&ep->smsg_progressing, 0, 1)) {
/* already progressing (we can't support reentry here) */
return 0;
}
/* per uGNI documentation we loop until the mailbox is empty */
do {
uint8_t tag = GNI_SMSG_ANY_TAG;
rc = GNI_SmsgGetNextWTag (ep->smsg_ep_handle, (void **) &data_ptr, &tag);
if (GNI_RC_NOT_DONE == rc) {
BTL_VERBOSE(("no smsg message waiting. rc = %d", rc));
ep->smsg_progressing = 0;
return count;
}
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
fprintf (stderr, "Unhandled Smsg error: %d\n", rc);
assert (0);
return OMPI_ERROR;
}
if (OPAL_UNLIKELY(0 == data_ptr)) {
BTL_ERROR(("null data ptr!"));
assert (0);
return OMPI_ERROR;
}
count++;
BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag));
switch (tag) {
case MCA_BTL_UGNI_TAG_SEND:
frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];
tag = frag.hdr.send.lag >> 24;
len = frag.hdr.send.lag & 0x00ffffff;
BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));
reg = mca_btl_base_active_message_trigger + tag;
frag.base.des_dst = &seg;
frag.base.des_dst_cnt = 1;
seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
seg.seg_len = len;
assert (NULL != reg->cbfunc);
reg->cbfunc(&ep->btl->super, tag, &(frag.base), reg->cbdata);
break;
case MCA_BTL_UGNI_TAG_GET_INIT:
frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL);
break;
case MCA_BTL_UGNI_TAG_RDMA_COMPLETE:
frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OMPI_SUCCESS);
break;
case MCA_BTL_UGNI_TAG_DISCONNECT:
/* remote endpoint has disconnected */
disconnect = true;
break;
default:
BTL_ERROR(("unknown tag %d\n", tag));
break;
}
rc = GNI_SmsgRelease (ep->smsg_ep_handle);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_ERROR(("Smsg release failed! rc = %d", rc));
return OMPI_ERROR;
}
} while (!disconnect);
ep->smsg_progressing = false;
/* disconnect if we get here */
mca_btl_ugni_ep_disconnect (ep, false);
return count;
}
