int mca_btl_ugni_progress_send_wait_list (mca_btl_base_endpoint_t *endpoint)
{
mca_btl_ugni_base_frag_t *frag;
int rc;
while (NULL !=
(frag = (mca_btl_ugni_base_frag_t *) opal_list_remove_first (&endpoint->frag_wait_list))) {
rc = mca_btl_ugni_send_frag (endpoint, frag);
if (OPAL_UNLIKELY(OMPI_SUCCESS > rc)) {
if (OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) {
opal_list_prepend (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
} else {
mca_btl_ugni_frag_complete (frag, rc);
}
return rc;
}
}
return OMPI_SUCCESS;
}
/* progress */
int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
{
mca_btl_active_message_callback_t *reg;
mca_btl_ugni_base_frag_t frag;
mca_btl_base_segment_t seg;
bool disconnect = false;
uintptr_t data_ptr;
gni_return_t rc;
uint32_t len;
int count = 0;
if (!opal_atomic_cmpset_32 (&ep->smsg_progressing, 0, 1)) {
/* already progressing (we can't support reentry here) */
return 0;
}
/* per uGNI documentation we loop until the mailbox is empty */
do {
uint8_t tag = GNI_SMSG_ANY_TAG;
rc = GNI_SmsgGetNextWTag (ep->smsg_ep_handle, (void **) &data_ptr, &tag);
if (GNI_RC_NOT_DONE == rc) {
BTL_VERBOSE(("no smsg message waiting. rc = %d", rc));
ep->smsg_progressing = 0;
return count;
}
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
fprintf (stderr, "Unhandled Smsg error: %d\n", rc);
assert (0);
return OMPI_ERROR;
}
if (OPAL_UNLIKELY(0 == data_ptr)) {
BTL_ERROR(("null data ptr!"));
assert (0);
return OMPI_ERROR;
}
count++;
BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag));
switch (tag) {
case MCA_BTL_UGNI_TAG_SEND:
frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];
tag = frag.hdr.send.lag >> 24;
len = frag.hdr.send.lag & 0x00ffffff;
BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));
reg = mca_btl_base_active_message_trigger + tag;
frag.base.des_dst = &seg;
frag.base.des_dst_cnt = 1;
seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
seg.seg_len = len;
assert (NULL != reg->cbfunc);
reg->cbfunc(&ep->btl->super, tag, &(frag.base), reg->cbdata);
break;
case MCA_BTL_UGNI_TAG_GET_INIT:
frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL);
break;
case MCA_BTL_UGNI_TAG_RDMA_COMPLETE:
frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OMPI_SUCCESS);
break;
case MCA_BTL_UGNI_TAG_DISCONNECT:
/* remote endpoint has disconnected */
disconnect = true;
break;
default:
BTL_ERROR(("unknown tag %d\n", tag));
break;
}
rc = GNI_SmsgRelease (ep->smsg_ep_handle);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_ERROR(("Smsg release failed! rc = %d", rc));
return OMPI_ERROR;
}
} while (!disconnect);
ep->smsg_progressing = false;
/* disconnect if we get here */
mca_btl_ugni_ep_disconnect (ep, false);
return count;
}
int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct mca_btl_base_descriptor_t *descriptor,
mca_btl_base_tag_t tag)
{
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) descriptor;
size_t size = frag->segments[0].base.seg_len + frag->segments[1].base.seg_len;
mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
int flags_save = frag->base.des_flags;
int rc;
/* tag and len are at the same location in eager and smsg frag hdrs */
frag->hdr.send.lag = (tag << 24) | size;
rc = mca_btl_ugni_check_endpoint_state (endpoint);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
return OMPI_SUCCESS;
}
BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor,
OMPI_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, frag->segments[0].base.seg_len));
/* temporarily disable ownership and callback flags so we can reliably check the complete flag */
frag->base.des_flags &= ~(MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
frag->flags &= ~MCA_BTL_UGNI_FRAG_COMPLETE;
rc = mca_btl_ugni_send_frag (endpoint, frag);
if (OPAL_LIKELY(frag->flags & MCA_BTL_UGNI_FRAG_COMPLETE)) {
/* fast path: remote side has received the frag */
frag->base.des_flags = flags_save;
mca_btl_ugni_frag_complete (frag, OMPI_SUCCESS);
return 1;
}
if ((OMPI_SUCCESS == rc) && (frag->flags & MCA_BTL_UGNI_FRAG_BUFFERED) && (flags_save & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
/* fast(ish) path: btl owned buffered frag. report send as complete */
frag->base.des_flags = flags_save & ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
if (OPAL_LIKELY(flags_save & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) {
frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc);
}
return 1;
}
/* slow(ish) path: remote side hasn't received the frag. call the frag's callback when
we get the local smsg/msgq or remote rdma completion */
frag->base.des_flags = flags_save | MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) {
/* queue up request */
if (false == endpoint->wait_listed) {
opal_list_append (&ugni_module->ep_wait_list, &endpoint->super);
endpoint->wait_listed = true;
}
opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
rc = OMPI_SUCCESS;
}
return rc;
}
/* progress */
int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
{
mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
mca_btl_active_message_callback_t *reg;
mca_btl_ugni_base_frag_t frag;
mca_btl_base_segment_t seg;
bool disconnect = false;
int32_t _tmp_value = 0;
uintptr_t data_ptr;
gni_return_t rc;
uint32_t len;
int count = 0;
if (!opal_atomic_compare_exchange_strong_32 (&ep->smsg_progressing, &_tmp_value, 1)) {
/* already progressing (we can't support reentry here) */
return 0;
}
/* per uGNI documentation we loop until the mailbox is empty */
do {
uint8_t tag = GNI_SMSG_ANY_TAG;
rc = mca_btl_ugni_smsg_get_next_wtag (ep->smsg_ep_handle, &data_ptr, &tag);
if (GNI_RC_SUCCESS != rc) {
if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) {
BTL_VERBOSE(("no smsg message waiting. rc = %s", gni_err_str[rc]));
ep->smsg_progressing = 0;
return count;
}
BTL_ERROR(("unhandled GNI_SmsgGetNextWTag error"));
return OPAL_ERROR;
}
assert (0 != data_ptr);
count++;
BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag));
switch (tag) {
case MCA_BTL_UGNI_TAG_SEND:
frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];
tag = frag.hdr.send.lag >> 24;
len = frag.hdr.send.lag & 0x00ffffff;
BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));
reg = mca_btl_base_active_message_trigger + tag;
frag.base.des_segments = &seg;
frag.base.des_segment_count = 1;
seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
seg.seg_len = len;
assert (NULL != reg->cbfunc);
reg->cbfunc(&ugni_module->super, tag, &(frag.base), reg->cbdata);
break;
case MCA_BTL_UGNI_TAG_GET_INIT:
frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL);
break;
case MCA_BTL_UGNI_TAG_RDMA_COMPLETE:
frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
if (((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE) {
mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OPAL_SUCCESS);
} else {
/* let the local smsg completion finish this frag */
((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags &= ~MCA_BTL_UGNI_FRAG_IGNORE;
}
break;
case MCA_BTL_UGNI_TAG_DISCONNECT:
/* remote endpoint has disconnected */
disconnect = true;
break;
default:
BTL_ERROR(("unknown tag %d\n", tag));
break;
}
rc = mca_btl_ugni_smsg_release (ep->smsg_ep_handle);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_ERROR(("Smsg release failed! rc = %d", rc));
return OPAL_ERROR;
}
} while (!disconnect);
ep->smsg_progressing = 0;
/* disconnect if we get here */
opal_mutex_lock (&ep->lock);
mca_btl_ugni_ep_disconnect (ep, false);
opal_mutex_unlock (&ep->lock);
return count;
}
