int opal_common_ugni_ep_create (opal_common_ugni_endpoint_t *cep, gni_cq_handle_t cq,
gni_ep_handle_t *ep_handle)
{
gni_return_t grc;
if (OPAL_UNLIKELY(NULL == cep)) {
assert (0);
return OPAL_ERR_BAD_PARAM;
}
/* create a uGNI endpoint handle and bind it to the remote peer */
OPAL_THREAD_LOCK(&cep->dev->dev_lock);
grc = GNI_EpCreate (cep->dev->dev_handle, cq, ep_handle);
OPAL_THREAD_UNLOCK(&cep->dev->dev_lock);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
return opal_common_rc_ugni_to_opal (grc);
}
OPAL_THREAD_LOCK(&cep->dev->dev_lock);
grc = GNI_EpBind (*ep_handle, cep->ep_rem_addr, cep->ep_rem_id);
OPAL_THREAD_UNLOCK(&cep->dev->dev_lock);
if (GNI_RC_SUCCESS != grc) {
OPAL_THREAD_LOCK(&cep->dev->dev_lock);
GNI_EpDestroy (*ep_handle);
OPAL_THREAD_UNLOCK(&cep->dev->dev_lock);
return opal_common_rc_ugni_to_opal (grc);
}
return OPAL_SUCCESS;
}
ucs_status_t ugni_connect_ep(uct_ugni_ep_t *ep,
uct_ugni_iface_t *iface,
const uct_sockaddr_ugni_t *iface_addr,
const uct_devaddr_ugni_t *ugni_dev_addr)
{
gni_return_t ugni_rc;
uct_ugni_cdm_lock(&iface->cdm);
ugni_rc = GNI_EpBind(ep->ep, ugni_dev_addr->nic_addr, iface_addr->domain_id);
uct_ugni_cdm_unlock(&iface->cdm);
if (GNI_RC_SUCCESS != ugni_rc) {
uct_ugni_cdm_lock(&iface->cdm);
(void)GNI_EpDestroy(ep->ep);
uct_ugni_cdm_unlock(&iface->cdm);
ucs_error("GNI_EpBind failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_UNREACHABLE;
}
ucs_debug("Binding ep %p to address (%d %d)", ep, ugni_dev_addr->nic_addr,
iface_addr->domain_id);
ep->flush_group->flush_comp.count = UCT_UGNI_INIT_FLUSH;
return UCS_OK;
}
int mca_btl_ugni_ep_handle_init (mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
mca_btl_ugni_device_t *device, mca_btl_ugni_endpoint_handle_t *ep_handle)
{
gni_return_t grc;
ep_handle->device = device;
/* create a uGNI endpoint handle and bind it to the remote peer */
grc = GNI_EpCreate (device->dev_handle, cq, &ep_handle->gni_handle);
if (OPAL_LIKELY(GNI_RC_SUCCESS == grc)) {
grc = GNI_EpBind (ep_handle->gni_handle, ep->ep_rem_addr, ep->ep_rem_id);
}
return mca_btl_rc_ugni_to_opal (grc);
}
ucs_status_t ugni_connect_ep(uct_ugni_iface_t *iface, const uct_sockaddr_ugni_t *iface_addr, uct_ugni_ep_t *ep){
gni_return_t ugni_rc;
ugni_rc = GNI_EpBind(ep->ep, iface_addr->nic_addr, iface_addr->domain_id);
if (GNI_RC_SUCCESS != ugni_rc) {
(void)GNI_EpDestroy(ep->ep);
ucs_error("GNI_EpBind failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_UNREACHABLE;
}
ucs_debug("Binding ep %p to address (%d %d)", ep, iface_addr->nic_addr,
iface_addr->domain_id);
ep->outstanding = 0;
return UCS_OK;
}
/* Before this function is called, you MUST
* A) Deregister the datagram processing function from the async thread.
* B) Cancel the wildcard datagram.
* C) Drain all other messages from the queue.
*/
static inline void uct_ugni_udt_terminate_thread(uct_ugni_udt_iface_t *iface)
{
gni_return_t ugni_rc;
gni_ep_handle_t ep;
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_EpCreate(uct_ugni_udt_iface_nic_handle(iface), iface->super.local_cq, &ep);
if (GNI_RC_SUCCESS != ugni_rc) {
uct_ugni_device_unlock(&iface->super.cdm);
ucs_error("GNI_EpCreate, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_EpBind(ep, iface->super.cdm.dev->address, iface->super.cdm.domain_id);
if (GNI_RC_SUCCESS != ugni_rc) {
GNI_EpDestroy(ep);
uct_ugni_device_unlock(&iface->super.cdm);
ucs_error("GNI_EpBind failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_EpPostDataWId(ep,
NULL, 0,
NULL, 0,
UCT_UGNI_UDT_CANCEL);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("Couldn't send cancel message to UGNI interface! %s %d",
gni_err_str[ugni_rc], ugni_rc);
}
/* When the gni_ep is destroyed the above post will be canceled */
ugni_rc = GNI_EpDestroy(ep);
uct_ugni_device_unlock(&iface->super.cdm);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_EpDestroy failed, Error status: %s %d\n",
gni_err_str[ugni_rc], ugni_rc);
}
}
/*
* helper function to initialize an SMSG connection
*/
static int __gnix_vc_smsg_init(struct gnix_vc *vc,
int peer_id,
gni_smsg_attr_t *peer_smsg_attr)
{
int ret = FI_SUCCESS;
struct gnix_fid_ep *ep;
struct gnix_fid_domain *dom;
struct gnix_mbox *mbox = NULL;
gni_smsg_attr_t local_smsg_attr;
gni_return_t __attribute__((unused)) status;
ssize_t __attribute__((unused)) len;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
assert(vc);
ep = vc->ep;
assert(ep);
dom = ep->domain;
if (dom == NULL)
return -FI_EINVAL;
mbox = vc->smsg_mbox;
assert (mbox);
local_smsg_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
local_smsg_attr.msg_buffer = mbox->base;
local_smsg_attr.buff_size = vc->ep->nic->mem_per_mbox;
local_smsg_attr.mem_hndl = *mbox->memory_handle;
local_smsg_attr.mbox_offset = (uint64_t)mbox->offset;
local_smsg_attr.mbox_maxcredit = dom->params.mbox_maxcredit;
local_smsg_attr.msg_maxsize = dom->params.mbox_msg_maxsize;
/*
* now build the SMSG connection
*/
fastlock_acquire(&ep->nic->lock);
status = GNI_EpCreate(ep->nic->gni_nic_hndl,
ep->nic->tx_cq,
&vc->gni_ep);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_EpCreate returned %s\n", gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
status = GNI_EpBind(vc->gni_ep,
vc->peer_addr.device_addr,
vc->peer_addr.cdm_id);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_EpBind returned %s\n", gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err1;
}
status = GNI_SmsgInit(vc->gni_ep,
&local_smsg_attr,
peer_smsg_attr);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_SmsgInit returned %s\n", gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err1;
}
status = GNI_EpSetEventData(vc->gni_ep,
vc->vc_id,
peer_id);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_EpSetEventData returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err1;
}
fastlock_release(&ep->nic->lock);
return ret;
err1:
GNI_EpDestroy(vc->gni_ep);
err:
fastlock_release(&ep->nic->lock);
return ret;
}
