static ucs_status_t uct_ugni_smsg_mbox_reg(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_mbox_t *mbox) { gni_return_t ugni_rc; void *address = (mbox+1); if (0 == iface->bytes_per_mbox) { ucs_error("Unexpected length %zu", iface->bytes_per_mbox); return UCS_ERR_INVALID_PARAM; } uct_ugni_cdm_lock(&iface->super.cdm); ugni_rc = GNI_MemRegister(uct_ugni_iface_nic_handle(&iface->super), (uint64_t)address, iface->bytes_per_mbox, iface->remote_cq, GNI_MEM_READWRITE, -1, &(mbox->gni_mem)); uct_ugni_cdm_unlock(&iface->super.cdm); if (GNI_RC_SUCCESS != ugni_rc) { ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d", address, iface->bytes_per_mbox, gni_err_str[ugni_rc], ugni_rc); return UCS_ERR_IO_ERROR; } mbox->base_address = (uintptr_t)address; return UCS_OK; }
static int __nic_setup_irq_cq(struct gnix_nic *nic) { int ret = FI_SUCCESS; size_t len; gni_return_t status; int fd = -1; void *mmap_addr; len = (size_t)sysconf(_SC_PAGESIZE); mmap_addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, fd, 0); if (mmap_addr == MAP_FAILED) { GNIX_WARN(FI_LOG_EP_CTRL, "mmap failed - %s\n", strerror(errno)); ret = -errno; goto err; } nic->irq_mmap_addr = mmap_addr; nic->irq_mmap_len = len; status = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) nic->irq_mmap_addr, len, nic->rx_cq_blk, GNI_MEM_READWRITE, -1, &nic->irq_mem_hndl); if (status != GNI_RC_SUCCESS) { ret = gnixu_to_fi_errno(status); GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister returned %s\n", gni_err_str[status]); goto err_w_mmap; } #if 0 fprintf(stderr,"registered ireq memhndl 0x%016lx 0x%016lx\n", nic->irq_mem_hndl.qword1, nic->irq_mem_hndl.qword2); #endif return ret; err_w_mmap: munmap(mmap_addr, len); err: return ret; }
static ucs_status_t uct_ugni_mem_reg(uct_md_h md, void *address, size_t length, uct_mem_h *memh_p) { ucs_status_t status; gni_return_t ugni_rc; uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t); gni_mem_handle_t * mem_hndl = NULL; pthread_mutex_lock(&uct_ugni_global_lock); if (0 == length) { ucs_error("Unexpected length %zu", length); return UCS_ERR_INVALID_PARAM; } mem_hndl = ucs_malloc(sizeof(gni_mem_handle_t), "gni_mem_handle_t"); if (NULL == mem_hndl) { ucs_error("Failed to allocate memory for gni_mem_handle_t"); status = UCS_ERR_NO_MEMORY; goto mem_err; } ugni_rc = GNI_MemRegister(ugni_md->nic_handle, (uint64_t)address, length, NULL, GNI_MEM_READWRITE | GNI_MEM_RELAXED_PI_ORDERING, -1, mem_hndl); if (GNI_RC_SUCCESS != ugni_rc) { ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d", address, length, gni_err_str[ugni_rc], ugni_rc); status = UCS_ERR_IO_ERROR; goto mem_err; } ucs_debug("Memory registration address %p, len %lu, keys [%"PRIx64" %"PRIx64"]", address, length, mem_hndl->qword1, mem_hndl->qword2); *memh_p = mem_hndl; pthread_mutex_unlock(&uct_ugni_global_lock); return UCS_OK; mem_err: free(mem_hndl); pthread_mutex_unlock(&uct_ugni_global_lock); return status; }
/** * Create a slab from a handle and append to the slab list. * * @param[in] handle Handle to the allocator being used. * * @return FI_SUCCESS On successful slab creation. * * @return -FI_ENOMEM if failure to allocate memory for slab or bitmap. * @return [Unspec] if failure in alloc_bitmap. Will return error code from * alloc_bitmap. * @return [Unspec] if failure in GNI_MemRegister. Converts gni_return_t * status code to FI_ERRNO value. */ static int __create_slab(struct gnix_mbox_alloc_handle *handle) { struct gnix_slab *slab; gni_return_t status; char error_buf[256]; char *error; size_t total_size; int ret; int vmdh_index = -1; int flags = GNI_MEM_READWRITE; struct gnix_auth_key *info; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); slab = calloc(1, sizeof(*slab)); if (!slab) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating slab: %s\n", error); ret = -FI_ENOMEM; goto err_slab_calloc; } total_size = handle->page_size * __page_count(handle); GNIX_DEBUG(FI_LOG_EP_CTRL, "total_size requested for mmap: %zu.\n", total_size); slab->used = calloc(1, sizeof(*(slab->used))); if (!slab->used) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap: %s\n", error); ret = -FI_ENOMEM; goto err_bitmap_calloc; } slab->base = mmap(0, total_size, (PROT_READ | PROT_WRITE), MAP_SHARED, handle->fd, handle->last_offset); if (slab->base == MAP_FAILED) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "%s\n", error); ret = -FI_ENOMEM; goto err_mmap; } ret = _gnix_alloc_bitmap(slab->used, __mbox_count(handle), NULL); if (ret) { GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap.\n"); goto err_alloc_bitmap; } COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock); if (handle->nic_handle->using_vmdh) { info = _gnix_auth_key_lookup(GNIX_PROV_DEFAULT_AUTH_KEY, GNIX_PROV_DEFAULT_AUTH_KEYLEN); assert(info); if (!handle->nic_handle->mdd_resources_set) { /* check to see if the ptag registration limit was set * yet or not -- becomes read-only after success */ _gnix_auth_key_enable(info); status = GNI_SetMddResources( handle->nic_handle->gni_nic_hndl, (info->attr.prov_key_limit + info->attr.user_key_limit)); assert(status == GNI_RC_SUCCESS); handle->nic_handle->mdd_resources_set = 1; } vmdh_index = _gnix_get_next_reserved_key(info); if (vmdh_index <= 0) { GNIX_FATAL(FI_LOG_DOMAIN, "failed to get reserved key for mbox " "registration, rc=%d\n", vmdh_index); } flags |= GNI_MEM_USE_VMDH; } status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl, (uint64_t) slab->base, total_size, handle->cq_handle, flags, vmdh_index, &slab->memory_handle); COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err_memregister; } slab->allocator = handle; gnix_slist_insert_tail(&slab->list_entry, &handle->slab_list); handle->last_offset += total_size; return ret; err_memregister: _gnix_free_bitmap(slab->used); err_alloc_bitmap: munmap(slab->base, total_size); err_mmap: free(slab->used); err_bitmap_calloc: free(slab); err_slab_calloc: return ret; }
static inline void *__gnix_generic_register( struct gnix_fid_domain *domain, struct gnix_fid_mem_desc *md, void *address, size_t length, gni_cq_handle_t dst_cq_hndl, int flags, int vmdh_index) { struct gnix_nic *nic; gni_return_t grc = GNI_RC_SUCCESS; int rc; pthread_mutex_lock(&gnix_nic_list_lock); /* If the nic list is empty, create a nic */ if (unlikely((dlist_empty(&gnix_nic_list_ptag[domain->ptag])))) { /* release the lock because we are not checking the list after this point. Additionally, gnix_nic_alloc takes the lock to add the nic. */ pthread_mutex_unlock(&gnix_nic_list_lock); rc = gnix_nic_alloc(domain, NULL, &nic); if (rc) { GNIX_INFO(FI_LOG_MR, "could not allocate nic to do mr_reg," " ret=%i\n", rc); return NULL; } } else { nic = dlist_first_entry(&gnix_nic_list_ptag[domain->ptag], struct gnix_nic, ptag_nic_list); if (unlikely(nic == NULL)) { GNIX_ERR(FI_LOG_MR, "Failed to find nic on " "ptag list\n"); pthread_mutex_unlock(&gnix_nic_list_lock); return NULL; } _gnix_ref_get(nic); pthread_mutex_unlock(&gnix_nic_list_lock); } COND_ACQUIRE(nic->requires_lock, &nic->lock); grc = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) address, length, dst_cq_hndl, flags, vmdh_index, &md->mem_hndl); COND_RELEASE(nic->requires_lock, &nic->lock); if (unlikely(grc != GNI_RC_SUCCESS)) { GNIX_INFO(FI_LOG_MR, "failed to register memory with uGNI, " "ret=%s\n", gni_err_str[grc]); _gnix_ref_put(nic); return NULL; } /* set up the mem desc */ md->nic = nic; md->domain = domain; /* take references on domain */ _gnix_ref_get(md->domain); return md; }