int mca_btl_ugni_progress_send_wait_list (mca_btl_base_endpoint_t *endpoint) { mca_btl_ugni_base_frag_t *frag; int rc; while (NULL != (frag = (mca_btl_ugni_base_frag_t *) opal_list_remove_first (&endpoint->frag_wait_list))) { rc = mca_btl_ugni_send_frag (endpoint, frag); if (OPAL_UNLIKELY(OMPI_SUCCESS > rc)) { if (OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) { opal_list_prepend (&endpoint->frag_wait_list, (opal_list_item_t *) frag); } else { mca_btl_ugni_frag_complete (frag, rc); } return rc; } } return OMPI_SUCCESS; }
/* progress */ int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep) { mca_btl_active_message_callback_t *reg; mca_btl_ugni_base_frag_t frag; mca_btl_base_segment_t seg; bool disconnect = false; uintptr_t data_ptr; gni_return_t rc; uint32_t len; int count = 0; if (!opal_atomic_cmpset_32 (&ep->smsg_progressing, 0, 1)) { /* already progressing (we can't support reentry here) */ return 0; } /* per uGNI documentation we loop until the mailbox is empty */ do { uint8_t tag = GNI_SMSG_ANY_TAG; rc = GNI_SmsgGetNextWTag (ep->smsg_ep_handle, (void **) &data_ptr, &tag); if (GNI_RC_NOT_DONE == rc) { BTL_VERBOSE(("no smsg message waiting. rc = %d", rc)); ep->smsg_progressing = 0; return count; } if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) { fprintf (stderr, "Unhandled Smsg error: %d\n", rc); assert (0); return OMPI_ERROR; } if (OPAL_UNLIKELY(0 == data_ptr)) { BTL_ERROR(("null data ptr!")); assert (0); return OMPI_ERROR; } count++; BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag)); switch (tag) { case MCA_BTL_UGNI_TAG_SEND: frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0]; tag = frag.hdr.send.lag >> 24; len = frag.hdr.send.lag & 0x00ffffff; BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag)); reg = mca_btl_base_active_message_trigger + tag; frag.base.des_dst = &seg; frag.base.des_dst_cnt = 1; seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t)); seg.seg_len = len; assert (NULL != reg->cbfunc); reg->cbfunc(&ep->btl->super, tag, &(frag.base), reg->cbdata); break; case MCA_BTL_UGNI_TAG_GET_INIT: frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0]; mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL); break; case MCA_BTL_UGNI_TAG_RDMA_COMPLETE: frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0]; mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OMPI_SUCCESS); break; case MCA_BTL_UGNI_TAG_DISCONNECT: /* remote endpoint has disconnected */ disconnect = true; break; default: BTL_ERROR(("unknown tag %d\n", tag)); break; } rc = GNI_SmsgRelease (ep->smsg_ep_handle); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) { BTL_ERROR(("Smsg release failed! rc = %d", rc)); return OMPI_ERROR; } } while (!disconnect); ep->smsg_progressing = false; /* disconnect if we get here */ mca_btl_ugni_ep_disconnect (ep, false); return count; }
int mca_btl_ugni_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag) { mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) descriptor; size_t size = frag->segments[0].base.seg_len + frag->segments[1].base.seg_len; mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl; int flags_save = frag->base.des_flags; int rc; /* tag and len are at the same location in eager and smsg frag hdrs */ frag->hdr.send.lag = (tag << 24) | size; rc = mca_btl_ugni_check_endpoint_state (endpoint); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag); return OMPI_SUCCESS; } BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor, OMPI_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, frag->segments[0].base.seg_len)); /* temporarily disable ownership and callback flags so we can reliably check the complete flag */ frag->base.des_flags &= ~(MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK); frag->flags &= ~MCA_BTL_UGNI_FRAG_COMPLETE; rc = mca_btl_ugni_send_frag (endpoint, frag); if (OPAL_LIKELY(frag->flags & MCA_BTL_UGNI_FRAG_COMPLETE)) { /* fast path: remote side has received the frag */ frag->base.des_flags = flags_save; mca_btl_ugni_frag_complete (frag, OMPI_SUCCESS); return 1; } if ((OMPI_SUCCESS == rc) && (frag->flags & MCA_BTL_UGNI_FRAG_BUFFERED) && (flags_save & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) { /* fast(ish) path: btl owned buffered frag. report send as complete */ frag->base.des_flags = flags_save & ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK; if (OPAL_LIKELY(flags_save & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) { frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc); } return 1; } /* slow(ish) path: remote side hasn't received the frag. call the frag's callback when we get the local smsg/msgq or remote rdma completion */ frag->base.des_flags = flags_save | MCA_BTL_DES_SEND_ALWAYS_CALLBACK; if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) { /* queue up request */ if (false == endpoint->wait_listed) { opal_list_append (&ugni_module->ep_wait_list, &endpoint->super); endpoint->wait_listed = true; } opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag); rc = OMPI_SUCCESS; } return rc; }
/* progress */ int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep) { mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep); mca_btl_active_message_callback_t *reg; mca_btl_ugni_base_frag_t frag; mca_btl_base_segment_t seg; bool disconnect = false; int32_t _tmp_value = 0; uintptr_t data_ptr; gni_return_t rc; uint32_t len; int count = 0; if (!opal_atomic_compare_exchange_strong_32 (&ep->smsg_progressing, &_tmp_value, 1)) { /* already progressing (we can't support reentry here) */ return 0; } /* per uGNI documentation we loop until the mailbox is empty */ do { uint8_t tag = GNI_SMSG_ANY_TAG; rc = mca_btl_ugni_smsg_get_next_wtag (ep->smsg_ep_handle, &data_ptr, &tag); if (GNI_RC_SUCCESS != rc) { if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) { BTL_VERBOSE(("no smsg message waiting. rc = %s", gni_err_str[rc])); ep->smsg_progressing = 0; return count; } BTL_ERROR(("unhandled GNI_SmsgGetNextWTag error")); return OPAL_ERROR; } assert (0 != data_ptr); count++; BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag)); switch (tag) { case MCA_BTL_UGNI_TAG_SEND: frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0]; tag = frag.hdr.send.lag >> 24; len = frag.hdr.send.lag & 0x00ffffff; BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag)); reg = mca_btl_base_active_message_trigger + tag; frag.base.des_segments = &seg; frag.base.des_segment_count = 1; seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t)); seg.seg_len = len; assert (NULL != reg->cbfunc); reg->cbfunc(&ugni_module->super, tag, &(frag.base), reg->cbdata); break; case MCA_BTL_UGNI_TAG_GET_INIT: frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0]; mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL); break; case MCA_BTL_UGNI_TAG_RDMA_COMPLETE: frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0]; if (((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE) { mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OPAL_SUCCESS); } else { /* let the local smsg completion finish this frag */ ((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags &= ~MCA_BTL_UGNI_FRAG_IGNORE; } break; case MCA_BTL_UGNI_TAG_DISCONNECT: /* remote endpoint has disconnected */ disconnect = true; break; default: BTL_ERROR(("unknown tag %d\n", tag)); break; } rc = mca_btl_ugni_smsg_release (ep->smsg_ep_handle); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) { BTL_ERROR(("Smsg release failed! rc = %d", rc)); return OPAL_ERROR; } } while (!disconnect); ep->smsg_progressing = 0; /* disconnect if we get here */ opal_mutex_lock (&ep->lock); mca_btl_ugni_ep_disconnect (ep, false); opal_mutex_unlock (&ep->lock); return count; }