int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, void *context) { int ret = FI_SUCCESS; struct gnix_fid_domain *domain_priv; struct gnix_fid_ep *ep_priv; gnix_hashtable_attr_t gnix_ht_attr; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); if ((domain == NULL) || (info == NULL) || (ep == NULL)) return -FI_EINVAL; if (info->ep_attr->type != FI_EP_RDM) return -FI_ENOSYS; domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid); ep_priv = calloc(1, sizeof *ep_priv); if (!ep_priv) return -FI_ENOMEM; ep_priv->ep_fid.fid.fclass = FI_CLASS_EP; ep_priv->ep_fid.fid.context = context; ep_priv->ep_fid.fid.ops = &gnix_ep_fi_ops; ep_priv->ep_fid.ops = &gnix_ep_ops; ep_priv->domain = domain_priv; ep_priv->type = info->ep_attr->type; fastlock_init(&ep_priv->vc_list_lock); dlist_init(&ep_priv->wc_vc_list); atomic_initialize(&ep_priv->active_fab_reqs, 0); atomic_initialize(&ep_priv->ref_cnt, 0); fastlock_init(&ep_priv->recv_queue_lock); slist_init(&ep_priv->unexp_recv_queue); slist_init(&ep_priv->posted_recv_queue); slist_init(&ep_priv->pending_recv_comp_queue); if (info->tx_attr) ep_priv->op_flags = info->tx_attr->op_flags; if (info->rx_attr) ep_priv->op_flags |= info->rx_attr->op_flags; ret = __fr_freelist_init(ep_priv); if (ret != FI_SUCCESS) { GNIX_ERR(FI_LOG_EP_CTRL, "Error allocating gnix_fab_req freelist (%s)", fi_strerror(-ret)); goto err1; } ep_priv->ep_fid.msg = &gnix_ep_msg_ops; ep_priv->ep_fid.rma = &gnix_ep_rma_ops; ep_priv->ep_fid.tagged = &gnix_ep_tagged_ops; ep_priv->ep_fid.atomic = NULL; ep_priv->ep_fid.cm = &gnix_cm_ops; /* * TODO, initialize vc hash table */ if (ep_priv->type == FI_EP_RDM) { ret = _gnix_cm_nic_alloc(domain_priv, &ep_priv->cm_nic); if (ret != FI_SUCCESS) goto err; gnix_ht_attr.ht_initial_size = domain_priv->params.ct_init_size; gnix_ht_attr.ht_maximum_size = domain_priv->params.ct_max_size; gnix_ht_attr.ht_increase_step = domain_priv->params.ct_step; gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT; gnix_ht_attr.ht_collision_thresh = 500; gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead; gnix_ht_attr.ht_internal_locking = 1; ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable)); if (ep_priv->vc_ht == NULL) goto err; ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "gnix_ht_init call returned %d\n", ret); goto err; } } else { ep_priv->cm_nic = NULL; ep_priv->vc = NULL; } ep_priv->progress_fn = NULL; ep_priv->rx_progress_fn = NULL; ret = gnix_nic_alloc(domain_priv, &ep_priv->nic); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_nic_alloc call returned %d\n", ret); goto err; } /* * if smsg callbacks not present hook them up now */ if (ep_priv->nic->smsg_callbacks == NULL) ep_priv->nic->smsg_callbacks = gnix_ep_smsg_callbacks; atomic_inc(&domain_priv->ref_cnt); *ep = &ep_priv->ep_fid; return ret; err1: __fr_freelist_destroy(ep_priv); err: if (ep_priv->vc_ht != NULL) { _gnix_ht_destroy(ep_priv->vc_ht); /* may not be initialized but okay */ free(ep_priv->vc_ht); ep_priv->vc_ht = NULL; } if (ep_priv->cm_nic != NULL) ret = _gnix_cm_nic_free(ep_priv->cm_nic); free(ep_priv); return ret; }
int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, void *context) { int ret = FI_SUCCESS; int tsret = FI_SUCCESS; uint32_t cdm_id; struct gnix_fid_domain *domain_priv; struct gnix_fid_ep *ep_priv; gnix_hashtable_attr_t gnix_ht_attr; gnix_ht_key_t *key_ptr; struct gnix_tag_storage_attr untagged_attr = { .type = GNIX_TAG_LIST, .use_src_addr_matching = 1, }; bool free_list_inited = false; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); if ((domain == NULL) || (info == NULL) || (ep == NULL) || (info->ep_attr == NULL)) return -FI_EINVAL; /* * TODO: need to implement other endpoint types */ if (info->ep_attr->type != FI_EP_RDM) return -FI_ENOSYS; domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid); ep_priv = calloc(1, sizeof *ep_priv); if (!ep_priv) return -FI_ENOMEM; /* init untagged storages */ tsret = _gnix_posted_tag_storage_init( &ep_priv->posted_recv_queue, &untagged_attr); if (tsret) return tsret; tsret = _gnix_unexpected_tag_storage_init( &ep_priv->unexp_recv_queue, &untagged_attr); if (tsret) return tsret; /* init tagged storages */ tsret = _gnix_posted_tag_storage_init( &ep_priv->tagged_posted_recv_queue, NULL); if (tsret) return tsret; tsret = _gnix_unexpected_tag_storage_init( &ep_priv->tagged_unexp_recv_queue, NULL); if (tsret) return tsret; ep_priv->ep_fid.fid.fclass = FI_CLASS_EP; ep_priv->ep_fid.fid.context = context; ep_priv->ep_fid.fid.ops = &gnix_ep_fi_ops; ep_priv->ep_fid.ops = &gnix_ep_ops; ep_priv->domain = domain_priv; ep_priv->type = info->ep_attr->type; _gnix_ref_init(&ep_priv->ref_cnt, 1, __ep_destruct); fastlock_init(&ep_priv->recv_comp_lock); fastlock_init(&ep_priv->recv_queue_lock); fastlock_init(&ep_priv->tagged_queue_lock); slist_init(&ep_priv->pending_recv_comp_queue); ep_priv->caps = info->caps & GNIX_EP_RDM_CAPS; if (info->tx_attr) ep_priv->op_flags = info->tx_attr->op_flags; if (info->rx_attr) ep_priv->op_flags |= info->rx_attr->op_flags; ep_priv->op_flags &= GNIX_EP_OP_FLAGS; ep_priv->min_multi_recv = GNIX_OPT_MIN_MULTI_RECV_DEFAULT; ret = __fr_freelist_init(ep_priv); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating gnix_fab_req freelist (%s)", fi_strerror(-ret)); goto err; } else free_list_inited = true; ep_priv->ep_fid.msg = &gnix_ep_msg_ops; ep_priv->ep_fid.rma = &gnix_ep_rma_ops; ep_priv->ep_fid.tagged = &gnix_ep_tagged_ops; ep_priv->ep_fid.atomic = &gnix_ep_atomic_ops; ep_priv->ep_fid.cm = &gnix_cm_ops; if (ep_priv->type == FI_EP_RDM) { if (info->src_addr != NULL) { ret = __gnix_ep_bound_prep(domain_priv, info, ep_priv); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "__gnix_ep_bound_prep returned error (%s)", fi_strerror(-ret)); goto err; } } else { fastlock_acquire(&domain_priv->cm_nic_lock); /* * if a cm_nic has not yet been allocated for this * domain, do it now. Reuse the embedded gnix_nic * in the cm_nic as the nic for this endpoint * to reduce demand on Aries hw resources. */ if (domain_priv->cm_nic == NULL) { ret = _gnix_cm_nic_alloc(domain_priv, info, &domain_priv->cm_nic); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_cm_nic_alloc returned %s\n", fi_strerror(-ret)); fastlock_release( &domain_priv->cm_nic_lock); goto err; } ep_priv->cm_nic = domain_priv->cm_nic; ep_priv->nic = ep_priv->cm_nic->nic; _gnix_ref_get(ep_priv->nic); } else { ep_priv->cm_nic = domain_priv->cm_nic; _gnix_ref_get(ep_priv->cm_nic); } fastlock_release(&domain_priv->cm_nic_lock); ep_priv->my_name.gnix_addr.device_addr = ep_priv->cm_nic->my_name.gnix_addr.device_addr; ep_priv->my_name.cm_nic_cdm_id = ep_priv->cm_nic->my_name.gnix_addr.cdm_id; ret = _gnix_get_new_cdm_id(domain_priv, &cdm_id); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "gnix_get_new_cdm_id call returned %s\n", fi_strerror(-ret)); goto err; } ep_priv->my_name.gnix_addr.cdm_id = cdm_id; } key_ptr = (gnix_ht_key_t *)&ep_priv->my_name.gnix_addr; ret = _gnix_ht_insert(ep_priv->cm_nic->addr_to_ep_ht, *key_ptr, ep_priv); if ((ret != FI_SUCCESS) && (ret != -FI_ENOSPC)) { GNIX_WARN(FI_LOG_EP_CTRL, "__gnix_ht_insert returned %d\n", ret); goto err; } gnix_ht_attr.ht_initial_size = domain_priv->params.ct_init_size; gnix_ht_attr.ht_maximum_size = domain_priv->params.ct_max_size; gnix_ht_attr.ht_increase_step = domain_priv->params.ct_step; gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT; gnix_ht_attr.ht_collision_thresh = 500; gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead; gnix_ht_attr.ht_internal_locking = 0; gnix_ht_attr.destructor = __gnix_vc_destroy_ht_entry; ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable)); if (ep_priv->vc_ht == NULL) goto err; ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "gnix_ht_init call returned %d\n", ret); goto err; } fastlock_init(&ep_priv->vc_ht_lock); } else { ep_priv->cm_nic = NULL; ep_priv->vc = NULL; } ep_priv->progress_fn = NULL; ep_priv->rx_progress_fn = NULL; if (ep_priv->nic == NULL) { ret = gnix_nic_alloc(domain_priv, NULL, &ep_priv->nic); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_nic_alloc call returned %d\n", ret); goto err; } } /* * if smsg callbacks not present hook them up now */ if (ep_priv->nic->smsg_callbacks == NULL) ep_priv->nic->smsg_callbacks = gnix_ep_smsg_callbacks; _gnix_ref_get(ep_priv->domain); *ep = &ep_priv->ep_fid; return ret; err: if (free_list_inited == true) __fr_freelist_destroy(ep_priv); if (ep_priv->vc_ht != NULL) { _gnix_ht_destroy(ep_priv->vc_ht); /* may not be initialized but okay */ free(ep_priv->vc_ht); ep_priv->vc_ht = NULL; } if (ep_priv->cm_nic != NULL) ret = _gnix_cm_nic_free(ep_priv->cm_nic); if (ep_priv->nic != NULL) ret = _gnix_nic_free(ep_priv->nic); free(ep_priv); return ret; } static int __match_context(struct slist_entry *item, const void *arg) { struct gnix_fab_req *req; req = container_of(item, struct gnix_fab_req, slist); return req->user_context == arg; } static inline struct gnix_fab_req *__find_tx_req( struct gnix_fid_ep *ep, void *context) { struct gnix_fab_req *req = NULL; struct slist_entry *entry; struct gnix_vc *vc; GNIX_HASHTABLE_ITERATOR(ep->vc_ht, iter); GNIX_DEBUG(FI_LOG_EP_CTRL, "searching VCs for the correct context to" " cancel, context=%p", context); fastlock_acquire(&ep->vc_ht_lock); while ((vc = _gnix_ht_iterator_next(&iter))) { fastlock_acquire(&vc->tx_queue_lock); entry = slist_remove_first_match(&vc->tx_queue, __match_context, context); fastlock_release(&vc->tx_queue_lock); if (entry) { req = container_of(entry, struct gnix_fab_req, slist); break; } } fastlock_release(&ep->vc_ht_lock); return req; } static inline struct gnix_fab_req *__find_rx_req( struct gnix_fid_ep *ep, void *context) { struct gnix_fab_req *req = NULL; fastlock_acquire(&ep->recv_queue_lock); req = _gnix_remove_req_by_context(&ep->posted_recv_queue, context); fastlock_release(&ep->recv_queue_lock); if (req) return req; fastlock_acquire(&ep->tagged_queue_lock); req = _gnix_remove_req_by_context(&ep->tagged_posted_recv_queue, context); fastlock_release(&ep->tagged_queue_lock); return req; } static ssize_t gnix_ep_cancel(fid_t fid, void *context) { int ret = FI_SUCCESS; struct gnix_fid_ep *ep; struct gnix_fab_req *req; struct gnix_fid_cq *err_cq = NULL; struct gnix_fid_cntr *err_cntr = NULL; void *addr; uint64_t tag, flags; size_t len; int is_send = 0; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid); if (!ep->domain) return -FI_EDOMAIN; /* without context, we will have to find a request that matches * a recv or send request. Try the send requests first. */ GNIX_INFO(FI_LOG_EP_CTRL, "looking for event to cancel\n"); req = __find_tx_req(ep, context); if (!req) { req = __find_rx_req(ep, context); if (req) { err_cq = ep->recv_cq; err_cntr = ep->recv_cntr; } } else { is_send = 1; err_cq = ep->send_cq; err_cntr = ep->send_cntr; } GNIX_INFO(FI_LOG_EP_CTRL, "finished searching\n"); if (!req) return -FI_ENOENT; if (err_cq) { /* add canceled event */ if (!(req->type == GNIX_FAB_RQ_RDMA_READ || req->type == GNIX_FAB_RQ_RDMA_WRITE)) { if (!is_send) { addr = (void *) req->msg.recv_addr; len = req->msg.recv_len; } else { addr = (void *) req->msg.send_addr; len = req->msg.send_len; } tag = req->msg.tag; } else { /* rma information */ addr = (void *) req->rma.loc_addr; len = req->rma.len; tag = 0; } flags = req->flags; _gnix_cq_add_error(err_cq, context, flags, len, addr, 0 /* data */, tag, len, FI_ECANCELED, FI_ECANCELED, 0); } if (err_cntr) { /* signal increase in cntr errs */ _gnix_cntr_inc_err(err_cntr); } _gnix_fr_free(ep, req); return ret; }