ucs_status_t uct_ugni_init_nic(int device_index,
int *domain_id,
gni_cdm_handle_t *cdm_handle,
gni_nic_handle_t *nic_handle,
uint32_t *address)
{
int modes;
ucs_status_t rc;
gni_return_t ugni_rc = GNI_RC_SUCCESS;
rc = uct_ugni_fetch_pmi();
if (UCS_OK != rc) {
ucs_error("Failed to activate context, Error status: %d", rc);
return rc;
}
*domain_id = job_info.pmi_rank_id + job_info.pmi_num_of_ranks * ugni_domain_global_counter;
modes = GNI_CDM_MODE_FORK_FULLCOPY | GNI_CDM_MODE_CACHED_AMO_ENABLED |
GNI_CDM_MODE_ERR_NO_KILL | GNI_CDM_MODE_FAST_DATAGRAM_POLL;
ucs_debug("Creating new PD domain with id %d (%d + %d * %d)",
*domain_id, job_info.pmi_rank_id,
job_info.pmi_num_of_ranks, ugni_domain_global_counter);
ugni_rc = GNI_CdmCreate(*domain_id, job_info.ptag, job_info.cookie,
modes, cdm_handle);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CdmCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
/* For now we use the first device for allocation of the domain */
ugni_rc = GNI_CdmAttach(*cdm_handle, job_info.devices[device_index].device_id,
address, nic_handle);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CdmAttach failed (domain id %d, %d), Error status: %s %d",
*domain_id, ugni_domain_global_counter, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
++ugni_domain_global_counter;
return UCS_OK;
}
static int opal_common_ugni_device_init (opal_common_ugni_device_t *device,
int device_id)
{
int rc;
/* Create a NIC Adress */
device->dev_id = device_id; /* Minor number of the Gemini NIC */
device->dev_addr = opal_common_ugni_get_nic_address (device->dev_id);
OPAL_OUTPUT((-1, "Got NIC Addr: 0x%08x, CPU ID: %d", device->dev_addr, device->dev_id));
OBJ_CONSTRUCT(&device->dev_lock,opal_mutex_t);
/* Attach device to the communication domain */
rc = GNI_CdmAttach (opal_common_ugni_module.cd_handle, device->dev_id,
&device->dev_pe_addr, &device->dev_handle);
if (GNI_RC_SUCCESS != rc) {
OPAL_OUTPUT((0, "Error: Creating communication domain %d\n", rc));
return opal_common_rc_ugni_to_opal (rc);
}
return OPAL_SUCCESS;
}
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
struct gnix_cm_nic **cm_nic_ptr)
{
int ret = FI_SUCCESS;
struct gnix_cm_nic *cm_nic = NULL;
uint32_t device_addr, cdm_id;
gni_return_t status;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
*cm_nic_ptr = NULL;
cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof(*cm_nic));
if (cm_nic == NULL) {
ret = -FI_ENOMEM;
goto err;
}
ret = _gnix_get_new_cdm_id(domain, &cdm_id);
if (ret != FI_SUCCESS)
goto err;
GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n",
domain->ptag, domain->cookie, cdm_id);
status = GNI_CdmCreate(cdm_id,
domain->ptag,
domain->cookie,
gnix_cdm_modes,
&cm_nic->gni_cdm_hndl);
if (status != GNI_RC_SUCCESS) {
GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmCreate returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
/*
* Okay, now go for the attach
*/
status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr,
&cm_nic->gni_nic_hndl);
if (status != GNI_RC_SUCCESS) {
GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmAttach returned %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err;
}
cm_nic->cdm_id = cdm_id;
cm_nic->ptag = domain->ptag;
cm_nic->cookie = domain->cookie;
cm_nic->device_addr = device_addr;
cm_nic->control_progress = domain->control_progress;
fastlock_init(&cm_nic->lock);
fastlock_init(&cm_nic->wq_lock);
list_head_init(&cm_nic->cm_nic_wq);
/*
* prep the cm nic's dgram component
*/
ret = _gnix_dgram_hndl_alloc(domain->fabric,
cm_nic,
domain->control_progress,
&cm_nic->dgram_hndl);
if (ret != FI_SUCCESS)
goto err;
*cm_nic_ptr = cm_nic;
return ret;
err:
if (cm_nic->dgram_hndl)
_gnix_dgram_hndl_free(cm_nic->dgram_hndl);
if (cm_nic->gni_cdm_hndl)
GNI_CdmDestroy(cm_nic->gni_cdm_hndl);
if (cm_nic != NULL)
free(cm_nic);
return ret;
}