int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain, struct gnix_cm_nic **cm_nic_ptr) { int ret = FI_SUCCESS; struct gnix_cm_nic *cm_nic = NULL; uint32_t device_addr, cdm_id; gni_return_t status; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); *cm_nic_ptr = NULL; cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof(*cm_nic)); if (cm_nic == NULL) { ret = -FI_ENOMEM; goto err; } ret = _gnix_get_new_cdm_id(domain, &cdm_id); if (ret != FI_SUCCESS) goto err; GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n", domain->ptag, domain->cookie, cdm_id); status = GNI_CdmCreate(cdm_id, domain->ptag, domain->cookie, gnix_cdm_modes, &cm_nic->gni_cdm_hndl); if (status != GNI_RC_SUCCESS) { GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmCreate returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } /* * Okay, now go for the attach */ status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr, &cm_nic->gni_nic_hndl); if (status != GNI_RC_SUCCESS) { GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmAttach returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } cm_nic->cdm_id = cdm_id; cm_nic->ptag = domain->ptag; cm_nic->cookie = domain->cookie; cm_nic->device_addr = device_addr; cm_nic->control_progress = domain->control_progress; fastlock_init(&cm_nic->lock); fastlock_init(&cm_nic->wq_lock); list_head_init(&cm_nic->cm_nic_wq); /* * prep the cm nic's dgram component */ ret = _gnix_dgram_hndl_alloc(domain->fabric, cm_nic, domain->control_progress, &cm_nic->dgram_hndl); if (ret != FI_SUCCESS) goto err; *cm_nic_ptr = cm_nic; return ret; err: if (cm_nic->dgram_hndl) _gnix_dgram_hndl_free(cm_nic->dgram_hndl); if (cm_nic->gni_cdm_hndl) GNI_CdmDestroy(cm_nic->gni_cdm_hndl); if (cm_nic != NULL) free(cm_nic); return ret; }
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain, struct fi_info *info, uint32_t cdm_id, struct gnix_auth_key *auth_key, struct gnix_cm_nic **cm_nic_ptr) { int ret = FI_SUCCESS; struct gnix_cm_nic *cm_nic = NULL; gnix_hashtable_attr_t gnix_ht_attr = {0}; uint32_t name_type = GNIX_EPN_TYPE_UNBOUND; struct gnix_nic_attr nic_attr = {0}; struct gnix_ep_name ep_name; struct gnix_dgram_hndl_attr dgram_hndl_attr = {0}; struct gnix_dgram_hndl_attr *dgram_hndl_attr_ptr = NULL; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); *cm_nic_ptr = NULL; /* * if app has specified a src_addr in the info * argument and length matches that for gnix_ep_name * we must allocate a cm_nic, otherwise we first * check to see if there is a cm_nic already for this domain * and just use it. */ if (info->src_addr) { /*TODO (optimization): strchr to name_type and strtol */ _gnix_get_ep_name(info->src_addr, 0, &ep_name, domain); name_type = ep_name.name_type; } GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n", auth_key->ptag, auth_key->cookie, cdm_id); cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof(*cm_nic)); if (cm_nic == NULL) { ret = -FI_ENOMEM; goto err; } /* * we have to force allocation of a new nic since we want * an a particulard cdm id */ nic_attr.must_alloc = true; nic_attr.use_cdm_id = true; nic_attr.cdm_id = cdm_id; nic_attr.auth_key = auth_key; ret = gnix_nic_alloc(domain, &nic_attr, &cm_nic->nic); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "gnix_nic_alloc returned %s\n", fi_strerror(-ret)); goto err; } cm_nic->my_name.gnix_addr.cdm_id = cdm_id; cm_nic->ptag = auth_key->ptag; cm_nic->my_name.cookie = auth_key->cookie; cm_nic->my_name.gnix_addr.device_addr = cm_nic->nic->device_addr; cm_nic->domain = domain; cm_nic->ctrl_progress = domain->control_progress; cm_nic->my_name.name_type = name_type; cm_nic->poll_cnt = 0; fastlock_init(&cm_nic->wq_lock); dlist_init(&cm_nic->cm_nic_wq); /* * prep the cm nic's dgram component */ if (domain->control_progress == FI_PROGRESS_AUTO) { dgram_hndl_attr.timeout_needed = __gnix_cm_nic_timeout_needed; dgram_hndl_attr.timeout_progress = __gnix_cm_nic_timeout_progress; dgram_hndl_attr.timeout_data = (void *)cm_nic; dgram_hndl_attr.timeout = domain->params.dgram_progress_timeout; dgram_hndl_attr_ptr = &dgram_hndl_attr; }; ret = _gnix_dgram_hndl_alloc(cm_nic, domain->control_progress, dgram_hndl_attr_ptr, &cm_nic->dgram_hndl); if (ret != FI_SUCCESS) goto err; /* * allocate hash table for translating ep addresses * to ep's. * This table will not be large - how many FI_EP_RDM ep's * will an app create using one domain?, nor in the critical path * so just use defaults. */ cm_nic->addr_to_ep_ht = calloc(1, sizeof(struct gnix_hashtable)); if (cm_nic->addr_to_ep_ht == NULL) goto err; gnix_ht_attr.ht_initial_size = 64; gnix_ht_attr.ht_maximum_size = 1024; gnix_ht_attr.ht_increase_step = 2; gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT; gnix_ht_attr.ht_collision_thresh = 500; gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead; gnix_ht_attr.ht_internal_locking = 1; gnix_ht_attr.destructor = NULL; ret = _gnix_ht_init(cm_nic->addr_to_ep_ht, &gnix_ht_attr); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "gnix_ht_init returned %s\n", fi_strerror(-ret)); goto err; } _gnix_ref_init(&cm_nic->ref_cnt, 1, __cm_nic_destruct); *cm_nic_ptr = cm_nic; pthread_mutex_lock(&gnix_cm_nic_list_lock); dlist_insert_tail(&cm_nic->cm_nic_list, &gnix_cm_nic_list); pthread_mutex_unlock(&gnix_cm_nic_list_lock); return ret; err: if (cm_nic->dgram_hndl) _gnix_dgram_hndl_free(cm_nic->dgram_hndl); if (cm_nic->nic) _gnix_nic_free(cm_nic->nic); if (cm_nic->addr_to_ep_ht) { _gnix_ht_destroy(cm_nic->addr_to_ep_ht); free(cm_nic->addr_to_ep_ht); } if (cm_nic != NULL) free(cm_nic); return ret; }