ucs_status_t ugni_activate_iface(uct_ugni_iface_t *iface)
{
int rc;
gni_return_t ugni_rc;
if(iface->activated) {
return UCS_OK;
}
rc = uct_ugni_init_nic(0, &iface->domain_id,
&iface->cdm_handle, &iface->nic_handle,
&iface->pe_address);
if (UCS_OK != rc) {
ucs_error("Failed to UGNI NIC, Error status: %d", rc);
return rc;
}
ugni_rc = GNI_CqCreate(iface->nic_handle, UCT_UGNI_LOCAL_CQ, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
iface->activated = true;
/* iface is activated */
return UCS_OK;
}
ucs_status_t ugni_activate_iface(uct_ugni_iface_t *iface)
{
ucs_status_t status;
gni_return_t ugni_rc;
uint32_t pe_address;
if(iface->activated) {
return UCS_OK;
}
status = uct_ugni_init_nic(0, &iface->domain_id,
&iface->cdm_handle, &iface->nic_handle,
&pe_address);
if (UCS_OK != status) {
ucs_error("Failed to UGNI NIC, Error status: %d", status);
return status;
}
ucs_debug("Made ugni interface. iface->dev->nic_addr = %i iface->domain_id = %i", iface->dev->address, iface->domain_id);
ugni_rc = GNI_CqCreate(iface->nic_handle, UCT_UGNI_LOCAL_CQ, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
iface->activated = true;
/* iface is activated */
return UCS_OK;
}
static ucs_status_t ugni_smsg_activate_iface(uct_ugni_smsg_iface_t *iface)
{
ucs_status_t status;
gni_return_t ugni_rc;
uint32_t pe_address;
if(iface->super.activated) {
return UCS_OK;
}
/*pull out these chunks into common routines */
status = uct_ugni_init_nic(0, &iface->super.domain_id,
&iface->super.cdm_handle, &iface->super.nic_handle,
&pe_address);
if (UCS_OK != status) {
ucs_error("Failed to UGNI NIC, Error status: %d", status);
return status;
}
ugni_rc = GNI_CqCreate(iface->super.nic_handle, UCT_UGNI_LOCAL_CQ, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->super.local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
ugni_rc = GNI_CqCreate(iface->super.nic_handle, 40000, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->remote_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
iface->super.activated = true;
/* iface is activated */
return UCS_OK;
}
static ucs_status_t uct_ugni_md_open(const char *md_name, const uct_md_config_t *md_config,
uct_md_h *md_p)
{
int domain_id;
ucs_status_t status = UCS_OK;
pthread_mutex_lock(&uct_ugni_global_lock);
static uct_md_ops_t md_ops = {
.close = uct_ugni_md_close,
.query = uct_ugni_md_query,
.mem_alloc = (void*)ucs_empty_function,
.mem_free = (void*)ucs_empty_function,
.mem_reg = uct_ugni_mem_reg,
.mem_dereg = uct_ugni_mem_dereg,
.mkey_pack = uct_ugni_rkey_pack
};
static uct_ugni_md_t md = {
.super.ops = &md_ops,
.super.component = &uct_ugni_md_component,
.ref_count = 0
};
*md_p = &md.super;
if (!md.ref_count) {
status = init_device_list(&job_info);
if (UCS_OK != status) {
ucs_error("Failed to init device list, Error status: %d", status);
goto error;
}
status = uct_ugni_init_nic(0, &domain_id,
&md.cdm_handle, &md.nic_handle,
&md.address);
if (UCS_OK != status) {
ucs_error("Failed to UGNI NIC, Error status: %d", status);
goto error;
}
}
md.ref_count++;
error:
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
uct_ugni_device_t * uct_ugni_device_by_name(const char *dev_name)
{
uct_ugni_device_t *dev;
unsigned dev_index;
if ((NULL == dev_name)) {
ucs_error("Bad parameter. Device name is set to NULL");
return NULL;
}
for (dev_index = 0; dev_index < job_info.num_devices; ++dev_index) {
dev = &job_info.devices[dev_index];
if ((strlen(dev_name) == strlen(dev->fname)) &&
(0 == strncmp(dev_name, dev->fname, strlen(dev->fname)))) {
ucs_info("Device found: %s", dev_name);
return dev;
}
}
/* Device not found */
ucs_error("Cannot find: %s", dev_name);
return NULL;
}
UCT_MD_COMPONENT_DEFINE(uct_ugni_md_component,
UCT_UGNI_MD_NAME,
uct_ugni_query_md_resources,
uct_ugni_md_open,
NULL,
uct_ugni_rkey_unpack,
uct_ugni_rkey_release,
"UGNI_",
uct_md_config_table,
uct_md_config_t);
