static ucs_status_t ugni_smsg_deactivate_iface(uct_ugni_smsg_iface_t *iface)
{
gni_return_t ugni_rc;
if(!iface->super.activated) {
return UCS_OK;
}
ugni_rc = GNI_CqDestroy(iface->super.local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_warn("GNI_CqDestroy failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
ugni_rc = GNI_CqDestroy(iface->remote_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_warn("GNI_CqDestroy failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
ugni_rc = GNI_CdmDestroy(iface->super.cdm_handle);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_warn("GNI_CdmDestroy error status: %s (%d)",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
iface->super.activated = false ;
return UCS_OK;
}
int opal_common_ugni_fini (void)
{
int i, rc;
if (0 == opal_common_ugni_module_ref_count) {
return OPAL_SUCCESS;
}
if (1 == opal_common_ugni_module_ref_count) {
/* tear down component */
if (opal_common_ugni_module.devices) {
/* finalize devices */
for (i = 0 ; i < opal_common_ugni_module.device_count ; ++i) {
opal_common_ugni_device_fini (opal_common_ugni_module.devices + i);
}
free (opal_common_ugni_module.devices);
opal_common_ugni_module.devices = NULL;
}
/* finally, tear down the communication domain */
rc = GNI_CdmDestroy (opal_common_ugni_module.cd_handle);
if (GNI_RC_SUCCESS != rc) {
OPAL_OUTPUT((-1, "error destroying cdm"));
}
}
opal_common_ugni_module_ref_count--;
return OPAL_SUCCESS;
}
int _gnix_cm_nic_free(struct gnix_cm_nic *cm_nic)
{
int ret = FI_SUCCESS;
gni_return_t status;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (cm_nic == NULL)
return -FI_EINVAL;
if (cm_nic->dgram_hndl != NULL) {
ret = _gnix_dgram_hndl_free(cm_nic->dgram_hndl);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_CTRL,
"gnix_dgram_hndl_free returned %d\n",
ret);
}
if (cm_nic->gni_cdm_hndl != NULL) {
status = GNI_CdmDestroy(cm_nic->gni_cdm_hndl);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"cdm destroy failed - %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
}
}
free(cm_nic);
return ret;
}
static void uct_ugni_md_close(uct_md_h md)
{
gni_return_t ugni_rc;
uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t);
pthread_mutex_lock(&uct_ugni_global_lock);
ugni_md->ref_count--;
if (!ugni_md->ref_count) {
ugni_rc = GNI_CdmDestroy(ugni_md->cdm_handle);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_warn("GNI_CdmDestroy error status: %s (%d)",
gni_err_str[ugni_rc], ugni_rc);
}
ucs_debug("MD GNI_CdmDestroy");
}
pthread_mutex_unlock(&uct_ugni_global_lock);
}
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
struct gnix_cm_nic **cm_nic_ptr)
{
int ret = FI_SUCCESS;
struct gnix_cm_nic *cm_nic = NULL;
uint32_t device_addr, cdm_id;
gni_return_t status;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
*cm_nic_ptr = NULL;
cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof(*cm_nic));
if (cm_nic == NULL) {
ret = -FI_ENOMEM;
goto err;
}
ret = _gnix_get_new_cdm_id(domain, &cdm_id);
if (ret != FI_SUCCESS)
goto err;
GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n",
domain->ptag, domain->cookie, cdm_id);
status = GNI_CdmCreate(cdm_id,
domain->ptag,
domain->cookie,
gnix_cdm_modes,
&cm_nic->gni_cdm_hndl);
if (status != GNI_RC_SUCCESS) {
GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmCreate returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
/*
* Okay, now go for the attach
*/
status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr,
&cm_nic->gni_nic_hndl);
if (status != GNI_RC_SUCCESS) {
GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmAttach returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
cm_nic->cdm_id = cdm_id;
cm_nic->ptag = domain->ptag;
cm_nic->cookie = domain->cookie;
cm_nic->device_addr = device_addr;
cm_nic->control_progress = domain->control_progress;
fastlock_init(&cm_nic->lock);
fastlock_init(&cm_nic->wq_lock);
list_head_init(&cm_nic->cm_nic_wq);
/*
* prep the cm nic's dgram component
*/
ret = _gnix_dgram_hndl_alloc(domain->fabric,
cm_nic,
domain->control_progress,
&cm_nic->dgram_hndl);
if (ret != FI_SUCCESS)
goto err;
*cm_nic_ptr = cm_nic;
return ret;
err:
if (cm_nic->dgram_hndl)
_gnix_dgram_hndl_free(cm_nic->dgram_hndl);
if (cm_nic->gni_cdm_hndl)
GNI_CdmDestroy(cm_nic->gni_cdm_hndl);
if (cm_nic != NULL)
free(cm_nic);
return ret;
}
