static inline int
mca_btl_ugni_handle_remote_smsg_overrun (mca_btl_ugni_module_t *btl)
{
gni_cq_entry_t event_data;
unsigned int ep_index;
int count, rc;
BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. "
"processing message backlog..."));
/* we don't know which endpoint lost an smsg completion. clear the
smsg remote cq and check all mailboxes */
/* clear out remote cq */
do {
rc = GNI_CqGetEvent (btl->smsg_remote_cq, &event_data);
} while (GNI_RC_NOT_DONE != rc);
for (ep_index = 0, count = 0 ; ep_index < btl->endpoint_count ; ++ep_index) {
mca_btl_base_endpoint_t *ep = btl->endpoints[ep_index];
if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) {
continue;
}
/* clear out smsg mailbox */
rc = mca_btl_ugni_smsg_process (ep);
if (OPAL_LIKELY(rc >= 0)) {
count += rc;
}
}
return count;
}
static inline int
mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *ugni_module)
{
uint32_t remote_addr, remote_id;
mca_btl_base_endpoint_t *ep;
gni_post_state_t post_state;
gni_ep_handle_t handle;
uint64_t datagram_id;
gni_return_t grc;
int count = 0;
/* check for datagram completion */
grc = GNI_PostDataProbeById (ugni_module->device->dev_handle, &datagram_id);
if (OPAL_LIKELY(GNI_RC_SUCCESS != grc)) {
return 0;
}
if ((datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) ==
MCA_BTL_UGNI_CONNECT_WILDCARD_ID) {
handle = ugni_module->wildcard_ep;
} else {
handle =
ugni_module->endpoints[(uint32_t)(datagram_id & 0xffffffffull)]->smsg_ep_handle;
}
/* wait for the incoming datagram to complete (in case it isn't) */
grc = GNI_EpPostDataWaitById (handle, datagram_id, -1, &post_state,
&remote_addr, &remote_id);
if (GNI_RC_SUCCESS != grc) {
BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", grc));
return ompi_common_rc_ugni_to_ompi (grc);
}
BTL_VERBOSE(("got a datagram completion: id = %" PRIx64 ", state = %d, "
"peer = %d", datagram_id, post_state, remote_id));
ep = ugni_module->endpoints[remote_id];
/* NTH: TODO -- error handling */
(void) mca_btl_ugni_ep_connect_progress (ep);
if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
/* process messages waiting in the endpoint's smsg mailbox */
count = mca_btl_ugni_smsg_process (ep);
}
/* repost the wildcard datagram */
if ((datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) ==
MCA_BTL_UGNI_CONNECT_WILDCARD_ID) {
mca_btl_ugni_wildcard_ep_post (ugni_module);
}
return count;
}
int mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_module_t *btl)
{
mca_btl_base_endpoint_t *ep;
gni_cq_entry_t event_data;
gni_return_t grc;
uint64_t inst_id;
grc = mca_btl_ugni_gni_cq_get_event (btl->devices, btl->smsg_remote_cq, &event_data);
if (GNI_RC_NOT_DONE == grc) {
return 0;
}
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data) ||
GNI_CQ_OVERRUN(event_data))) {
if (GNI_RC_ERROR_RESOURCE == grc ||
(GNI_RC_SUCCESS == grc && GNI_CQ_OVERRUN(event_data))) {
/* recover from smsg cq overrun */
return mca_btl_ugni_handle_remote_smsg_overrun (btl);
}
BTL_ERROR(("unhandled error in GNI_CqGetEvent"));
/* unhandled error: crash */
assert (0);
return mca_btl_rc_ugni_to_opal (grc);
}
BTL_VERBOSE(("REMOTE CQ: Got event 0x%" PRIx64 ". msg id = %" PRIu64
". ok = %d, type = %" PRIu64, (uint64_t) event_data,
GNI_CQ_GET_INST_ID(event_data), GNI_CQ_STATUS_OK(event_data),
GNI_CQ_GET_TYPE(event_data)));
inst_id = GNI_CQ_GET_INST_ID(event_data);
ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&btl->endpoints, inst_id);
if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state)) {
/* due to the nature of datagrams we may get a smsg completion before
we get mailbox info from the peer */
BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", ep->state));
return 0;
}
return mca_btl_ugni_smsg_process (ep);
}
static inline int
mca_btl_ugni_handle_remote_smsg_overrun (mca_btl_ugni_module_t *btl)
{
size_t endpoint_count;
unsigned int ep_index;
int count, rc;
BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. "
"processing message backlog..."));
/* we don't know which endpoint lost an smsg completion. clear the
smsg remote cq and check all mailboxes */
/* clear out remote cq */
mca_btl_ugni_cq_clear (btl->devices, btl->smsg_remote_cq);
endpoint_count = opal_pointer_array_get_size (&btl->endpoints);
for (ep_index = 0, count = 0 ; ep_index < endpoint_count ; ++ep_index) {
mca_btl_base_endpoint_t *ep;
ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&btl->endpoints,
ep_index);
if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) {
continue;
}
/* clear out smsg mailbox */
rc = mca_btl_ugni_smsg_process (ep);
if (OPAL_LIKELY(rc >= 0)) {
count += rc;
}
}
return count;
}
static inline int
mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *ugni_module)
{
uint64_t datagram_id, data, proc_id;
uint32_t remote_addr, remote_id;
mca_btl_base_endpoint_t *ep;
gni_post_state_t post_state;
gni_ep_handle_t handle;
gni_return_t grc;
int count = 0, rc;
/* check for datagram completion */
OPAL_THREAD_LOCK(&ugni_module->device->dev_lock); /* TODO: may not need lock for this function */
grc = GNI_PostDataProbeById (ugni_module->device->dev_handle, &datagram_id);
if (OPAL_LIKELY(GNI_RC_SUCCESS != grc)) {
OPAL_THREAD_UNLOCK(&ugni_module->device->dev_lock);
return 0;
}
data = datagram_id & ~(MCA_BTL_UGNI_DATAGRAM_MASK);
BTL_VERBOSE(("datgram_id: %" PRIx64 ", mask: %" PRIx64, datagram_id, (uint64_t) (datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK)));
if ((datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) == MCA_BTL_UGNI_CONNECT_DIRECTED_ID) {
ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&ugni_module->endpoints, data);
handle = ep->smsg_ep_handle;
} else {
handle = ugni_module->wildcard_ep;
}
/* wait for the incoming datagram to complete (in case it isn't) */
grc = GNI_EpPostDataWaitById (handle, datagram_id, -1, &post_state,
&remote_addr, &remote_id);
OPAL_THREAD_UNLOCK(&ugni_module->device->dev_lock);
if (GNI_RC_SUCCESS != grc) {
BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", grc));
return opal_common_rc_ugni_to_opal (grc);
}
/* if this is a wildcard endpoint lookup the remote peer by the proc id we received */
if (handle == ugni_module->wildcard_ep) {
proc_id = mca_btl_ugni_proc_name_to_id (ugni_module->wc_remote_attr.proc_name);
BTL_VERBOSE(("received connection attempt on wildcard endpoint from proc id: %" PRIx64,
proc_id));
OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
rc = opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
/* check if the endpoint is known */
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc || NULL == ep)) {
struct opal_proc_t *remote_proc = opal_proc_for_name (ugni_module->wc_remote_attr.proc_name);
BTL_VERBOSE(("Got connection request from an unknown peer {jobid = 0x%x, vid = 0x%x}",
ugni_module->wc_remote_attr.proc_name.jobid, ugni_module->wc_remote_attr.proc_name.vpid));
ep = mca_btl_ugni_get_ep (&ugni_module->super, remote_proc);
if (OPAL_UNLIKELY(NULL == ep)) {
return rc;
}
}
} else {
BTL_VERBOSE(("directed datagram complete for endpoint %p", (void *) ep));
}
/* should not have gotten a NULL endpoint */
assert (NULL != ep);
BTL_VERBOSE(("got a datagram completion: id = %" PRIx64 ", state = %d, "
"data = 0x%" PRIx64 ", ep = %p, remote id: %d", datagram_id, post_state,
data, (void *) ep, remote_id));
/* NTH: TODO -- error handling */
opal_mutex_lock (&ep->lock);
if (handle != ugni_module->wildcard_ep) {
/* directed post complete */
ep->dg_posted = false;
}
(void) mca_btl_ugni_ep_connect_progress (ep);
opal_mutex_unlock (&ep->lock);
if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
/* process messages waiting in the endpoint's smsg mailbox */
count = mca_btl_ugni_smsg_process (ep);
}
/* repost the wildcard datagram */
if (handle == ugni_module->wildcard_ep) {
mca_btl_ugni_wildcard_ep_post (ugni_module);
}
return count;
}
int mca_btl_ugni_progress_datagram (mca_btl_ugni_device_t *device)
{
mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_component.modules;
mca_btl_base_endpoint_t *ep;
gni_ep_handle_t handle;
int count = 0, rc;
rc = mca_btl_ugni_get_datagram (ugni_module, device, &handle, &ep);
if (1 != rc) {
return rc;
}
BTL_VERBOSE(("remote datagram completion on handle %p", handle));
/* if this is a wildcard endpoint lookup the remote peer by the proc id we received */
if (handle == ugni_module->wildcard_ep) {
struct opal_proc_t *remote_proc = opal_proc_for_name (ugni_module->wc_remote_attr.proc_name);
BTL_VERBOSE(("received connection attempt on wildcard endpoint from proc: %s",
OPAL_NAME_PRINT(ugni_module->wc_remote_attr.proc_name)));
ep = mca_btl_ugni_get_ep (&ugni_module->super, remote_proc);
if (OPAL_UNLIKELY(NULL == ep)) {
/* there is no way to recover from this error so just abort() */
BTL_ERROR(("could not find/allocate a btl endpoint for peer %s",
OPAL_NAME_PRINT(ugni_module->wc_remote_attr.proc_name)));
abort ();
return OPAL_ERR_NOT_FOUND;
}
}
/* should not have gotten a NULL endpoint */
assert (NULL != ep);
BTL_VERBOSE(("got a datagram completion: ep = %p. wc = %d", (void *) ep, handle == ugni_module->wildcard_ep));
/* NTH: TODO -- error handling */
opal_mutex_lock (&ep->lock);
if (handle != ugni_module->wildcard_ep) {
/* directed post complete */
BTL_VERBOSE(("directed datagram complete for endpoint %p", (void *) ep));
ep->dg_posted = false;
(void) opal_atomic_add_32 (&ugni_module->active_datagrams, -1);
}
(void) mca_btl_ugni_ep_connect_progress (ep);
opal_mutex_unlock (&ep->lock);
if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
/* process messages waiting in the endpoint's smsg mailbox */
count = mca_btl_ugni_smsg_process (ep);
}
/* repost the wildcard datagram */
if (handle == ugni_module->wildcard_ep) {
mca_btl_ugni_wildcard_ep_post (ugni_module);
}
return count;
}
static inline int
mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *ugni_module)
{
uint32_t remote_addr, remote_id;
uint64_t datagram_id, data;
mca_btl_base_endpoint_t *ep;
gni_post_state_t post_state;
gni_ep_handle_t handle;
gni_return_t grc;
int count = 0, rc;
/* check for datagram completion */
grc = GNI_PostDataProbeById (ugni_module->device->dev_handle, &datagram_id);
if (OPAL_LIKELY(GNI_RC_SUCCESS != grc)) {
return 0;
}
data = datagram_id & ~(MCA_BTL_UGNI_DATAGRAM_MASK);
BTL_VERBOSE(("datgram_id: %" PRIx64 ", mask: %" PRIx64, datagram_id, (uint64_t) (datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK)));
if ((datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) == MCA_BTL_UGNI_CONNECT_DIRECTED_ID) {
ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&ugni_module->endpoints, data);
handle = ep->smsg_ep_handle;
} else {
handle = ugni_module->wildcard_ep;
}
/* wait for the incoming datagram to complete (in case it isn't) */
grc = GNI_EpPostDataWaitById (handle, datagram_id, -1, &post_state,
&remote_addr, &remote_id);
if (GNI_RC_SUCCESS != grc) {
BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", grc));
return ompi_common_rc_ugni_to_ompi (grc);
}
/* if this is a wildcard endpoint lookup the remote peer by the proc id we received */
if (handle == ugni_module->wildcard_ep) {
BTL_VERBOSE(("received connection attempt on wildcard endpoint from proc id: %" PRIx64, ugni_module->wc_remote_attr.proc_id));
rc = opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint,
ugni_module->wc_remote_attr.proc_id,
(void *) &ep);
/* check if the endpoint is known */
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc || NULL == ep)) {
BTL_ERROR(("received connection attempt from an unknown peer. rc: %d, ep: %p, id: 0x%" PRIx64,
rc, ep, ugni_module->wc_remote_attr.proc_id));
return OMPI_ERR_NOT_FOUND;
}
} else {
BTL_VERBOSE(("directed datagram complete for endpoint %p", ep));
}
/* should not have gotten a NULL endpoint */
assert (NULL != ep);
BTL_VERBOSE(("got a datagram completion: id = %" PRIx64 ", state = %d, "
"data = 0x%" PRIx64 ", ep = %p, remote id: %d", datagram_id, post_state,
data, ep, remote_id));
/* NTH: TODO -- error handling */
(void) mca_btl_ugni_ep_connect_progress (ep);
if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
/* process messages waiting in the endpoint's smsg mailbox */
count = mca_btl_ugni_smsg_process (ep);
}
/* repost the wildcard datagram */
if (handle == ugni_module->wildcard_ep) {
mca_btl_ugni_wildcard_ep_post (ugni_module);
}
return count;
}
