static inline int
mca_btl_ugni_post_pending (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device)
{
int pending_post_count = opal_list_get_size (&device->pending_post);
mca_btl_ugni_post_descriptor_t *post_desc;
int rc;
/* check if there are any posts pending resources */
if (OPAL_LIKELY(0 == pending_post_count)) {
return 0;
}
BTL_VERBOSE(("progressing %d pending FMA/RDMA operations", pending_post_count));
for (int i = 0 ; i < pending_post_count ; ++i) {
mca_btl_ugni_device_lock (device);
post_desc = (mca_btl_ugni_post_descriptor_t *) opal_list_remove_first (&device->pending_post);
mca_btl_ugni_device_unlock (device);
if (NULL == post_desc) {
break;
}
rc = mca_btl_ugni_repost (ugni_module, post_desc);
if (OPAL_SUCCESS != rc) {
mca_btl_ugni_device_lock (device);
opal_list_prepend (&device->pending_post, (opal_list_item_t *) post_desc);
mca_btl_ugni_device_unlock (device);
break;
}
}
return 1;
}
int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnect)
{
mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
mca_btl_ugni_device_t *device;
int rc;
if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
/* nothing to do */
return OPAL_SUCCESS;
}
device = ep->smsg_ep_handle.device;
while (device->dev_smsg_local_cq.active_operations) {
/* ensure all sends are complete before removing and procs */
rc = mca_btl_ugni_progress_local_smsg (ugni_module, device);
if (OPAL_SUCCESS != rc) {
break;
}
}
if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state && send_disconnect) {
rc = mca_btl_ugni_ep_send_disconnect (ep);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
BTL_VERBOSE(("could not send disconnect message to peer"));
}
/* wait for the disconnect messagse to go */
do {
/* ensure all sends are complete before removing and procs */
rc = mca_btl_ugni_progress_local_smsg (ugni_module, device);
if (OPAL_SUCCESS != rc) {
break;
}
} while (device->dev_smsg_local_cq.active_operations);
(void) opal_atomic_add_fetch_32 (&ep->smsg_ep_handle.device->smsg_connections, -1);
}
mca_btl_ugni_device_lock (device);
/* NTH: this call may not need the device lock. seems to work without it but
* the lock is here to be safe. */
(void) mca_btl_ugni_ep_handle_cleanup (&ep->smsg_ep_handle);
mca_btl_ugni_device_unlock (device);
if (ep->mailbox) {
opal_free_list_return (&ugni_module->smsg_mboxes, ((opal_free_list_item_t *) ep->mailbox));
ep->mailbox = NULL;
}
ep->state = MCA_BTL_UGNI_EP_STATE_INIT;
return OPAL_SUCCESS;
}
static inline int mca_btl_ugni_ep_connect_start (mca_btl_base_endpoint_t *ep) {
mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
mca_btl_ugni_device_t *device = ugni_module->devices;
int rc;
/* protect against re-entry from opal_progress */
if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTING == ep->state)) {
return OPAL_ERR_RESOURCE_BUSY;
}
ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTING;
BTL_VERBOSE(("initiating connection to remote peer with address: %u id: %u proc: %p",
ep->ep_rem_addr, ep->ep_rem_id, (void *)ep->peer_proc));
/* bind endpoint to remote address */
/* we bind two endpoints to seperate out local smsg completion and local fma completion */
mca_btl_ugni_device_lock (device);
rc = mca_btl_ugni_ep_handle_init (ep, device->dev_smsg_local_cq.gni_handle, device, &ep->smsg_ep_handle);
mca_btl_ugni_device_unlock (device);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
return rc;
}
/* build connection data */
rc = mca_btl_ugni_ep_smsg_get_mbox (ep);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
return rc;
}
ep->remote_attr = calloc (1, sizeof (*ep->remote_attr));
if (OPAL_UNLIKELY(NULL == ep->remote_attr)) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
BTL_VERBOSE(("btl/ugni connection to remote peer initiated"));
return OPAL_SUCCESS;
}
