static ucs_status_t uct_ugni_smsg_mbox_reg(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_mbox_t *mbox)
{
gni_return_t ugni_rc;
void *address = (mbox+1);
if (0 == iface->bytes_per_mbox) {
ucs_error("Unexpected length %zu", iface->bytes_per_mbox);
return UCS_ERR_INVALID_PARAM;
}
uct_ugni_cdm_lock(&iface->super.cdm);
ugni_rc = GNI_MemRegister(uct_ugni_iface_nic_handle(&iface->super), (uint64_t)address,
iface->bytes_per_mbox, iface->remote_cq,
GNI_MEM_READWRITE,
-1, &(mbox->gni_mem));
uct_ugni_cdm_unlock(&iface->super.cdm);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d",
address, iface->bytes_per_mbox, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
mbox->base_address = (uintptr_t)address;
return UCS_OK;
}
static int __nic_setup_irq_cq(struct gnix_nic *nic)
{
int ret = FI_SUCCESS;
size_t len;
gni_return_t status;
int fd = -1;
void *mmap_addr;
len = (size_t)sysconf(_SC_PAGESIZE);
mmap_addr = mmap(NULL, len, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANON, fd, 0);
if (mmap_addr == MAP_FAILED) {
GNIX_WARN(FI_LOG_EP_CTRL, "mmap failed - %s\n",
strerror(errno));
ret = -errno;
goto err;
}
nic->irq_mmap_addr = mmap_addr;
nic->irq_mmap_len = len;
status = GNI_MemRegister(nic->gni_nic_hndl,
(uint64_t) nic->irq_mmap_addr,
len,
nic->rx_cq_blk,
GNI_MEM_READWRITE,
-1,
&nic->irq_mem_hndl);
if (status != GNI_RC_SUCCESS) {
ret = gnixu_to_fi_errno(status);
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_MemRegister returned %s\n",
gni_err_str[status]);
goto err_w_mmap;
}
#if 0
fprintf(stderr,"registered ireq memhndl 0x%016lx 0x%016lx\n",
nic->irq_mem_hndl.qword1,
nic->irq_mem_hndl.qword2);
#endif
return ret;
err_w_mmap:
munmap(mmap_addr, len);
err:
return ret;
}
static ucs_status_t uct_ugni_mem_reg(uct_md_h md, void *address, size_t length,
uct_mem_h *memh_p)
{
ucs_status_t status;
gni_return_t ugni_rc;
uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t);
gni_mem_handle_t * mem_hndl = NULL;
pthread_mutex_lock(&uct_ugni_global_lock);
if (0 == length) {
ucs_error("Unexpected length %zu", length);
return UCS_ERR_INVALID_PARAM;
}
mem_hndl = ucs_malloc(sizeof(gni_mem_handle_t), "gni_mem_handle_t");
if (NULL == mem_hndl) {
ucs_error("Failed to allocate memory for gni_mem_handle_t");
status = UCS_ERR_NO_MEMORY;
goto mem_err;
}
ugni_rc = GNI_MemRegister(ugni_md->nic_handle, (uint64_t)address,
length, NULL,
GNI_MEM_READWRITE | GNI_MEM_RELAXED_PI_ORDERING,
-1, mem_hndl);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d",
address, length, gni_err_str[ugni_rc], ugni_rc);
status = UCS_ERR_IO_ERROR;
goto mem_err;
}
ucs_debug("Memory registration address %p, len %lu, keys [%"PRIx64" %"PRIx64"]",
address, length, mem_hndl->qword1, mem_hndl->qword2);
*memh_p = mem_hndl;
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
mem_err:
free(mem_hndl);
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
/**
* Create a slab from a handle and append to the slab list.
*
* @param[in] handle Handle to the allocator being used.
*
* @return FI_SUCCESS On successful slab creation.
*
* @return -FI_ENOMEM if failure to allocate memory for slab or bitmap.
* @return [Unspec] if failure in alloc_bitmap. Will return error code from
* alloc_bitmap.
* @return [Unspec] if failure in GNI_MemRegister. Converts gni_return_t
* status code to FI_ERRNO value.
*/
static int __create_slab(struct gnix_mbox_alloc_handle *handle)
{
struct gnix_slab *slab;
gni_return_t status;
char error_buf[256];
char *error;
size_t total_size;
int ret;
int vmdh_index = -1;
int flags = GNI_MEM_READWRITE;
struct gnix_auth_key *info;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
slab = calloc(1, sizeof(*slab));
if (!slab) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating slab: %s\n",
error);
ret = -FI_ENOMEM;
goto err_slab_calloc;
}
total_size = handle->page_size * __page_count(handle);
GNIX_DEBUG(FI_LOG_EP_CTRL, "total_size requested for mmap: %zu.\n",
total_size);
slab->used = calloc(1, sizeof(*(slab->used)));
if (!slab->used) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating bitmap: %s\n",
error);
ret = -FI_ENOMEM;
goto err_bitmap_calloc;
}
slab->base = mmap(0, total_size, (PROT_READ | PROT_WRITE), MAP_SHARED,
handle->fd, handle->last_offset);
if (slab->base == MAP_FAILED) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL, "%s\n", error);
ret = -FI_ENOMEM;
goto err_mmap;
}
ret = _gnix_alloc_bitmap(slab->used, __mbox_count(handle), NULL);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap.\n");
goto err_alloc_bitmap;
}
COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (handle->nic_handle->using_vmdh) {
info = _gnix_auth_key_lookup(GNIX_PROV_DEFAULT_AUTH_KEY,
GNIX_PROV_DEFAULT_AUTH_KEYLEN);
assert(info);
if (!handle->nic_handle->mdd_resources_set) {
/* check to see if the ptag registration limit was set
* yet or not -- becomes read-only after success */
_gnix_auth_key_enable(info);
status = GNI_SetMddResources(
handle->nic_handle->gni_nic_hndl,
(info->attr.prov_key_limit +
info->attr.user_key_limit));
assert(status == GNI_RC_SUCCESS);
handle->nic_handle->mdd_resources_set = 1;
}
vmdh_index = _gnix_get_next_reserved_key(info);
if (vmdh_index <= 0) {
GNIX_FATAL(FI_LOG_DOMAIN,
"failed to get reserved key for mbox "
"registration, rc=%d\n",
vmdh_index);
}
flags |= GNI_MEM_USE_VMDH;
}
status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl,
(uint64_t) slab->base, total_size,
handle->cq_handle,
flags, vmdh_index,
&slab->memory_handle);
COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err_memregister;
}
slab->allocator = handle;
gnix_slist_insert_tail(&slab->list_entry, &handle->slab_list);
handle->last_offset += total_size;
return ret;
err_memregister:
_gnix_free_bitmap(slab->used);
err_alloc_bitmap:
munmap(slab->base, total_size);
err_mmap:
free(slab->used);
err_bitmap_calloc:
free(slab);
err_slab_calloc:
return ret;
}
static inline void *__gnix_generic_register(
struct gnix_fid_domain *domain,
struct gnix_fid_mem_desc *md,
void *address,
size_t length,
gni_cq_handle_t dst_cq_hndl,
int flags,
int vmdh_index)
{
struct gnix_nic *nic;
gni_return_t grc = GNI_RC_SUCCESS;
int rc;
pthread_mutex_lock(&gnix_nic_list_lock);
/* If the nic list is empty, create a nic */
if (unlikely((dlist_empty(&gnix_nic_list_ptag[domain->ptag])))) {
/* release the lock because we are not checking the list after
this point. Additionally, gnix_nic_alloc takes the
lock to add the nic. */
pthread_mutex_unlock(&gnix_nic_list_lock);
rc = gnix_nic_alloc(domain, NULL, &nic);
if (rc) {
GNIX_INFO(FI_LOG_MR,
"could not allocate nic to do mr_reg,"
" ret=%i\n", rc);
return NULL;
}
} else {
nic = dlist_first_entry(&gnix_nic_list_ptag[domain->ptag],
struct gnix_nic, ptag_nic_list);
if (unlikely(nic == NULL)) {
GNIX_ERR(FI_LOG_MR, "Failed to find nic on "
"ptag list\n");
pthread_mutex_unlock(&gnix_nic_list_lock);
return NULL;
}
_gnix_ref_get(nic);
pthread_mutex_unlock(&gnix_nic_list_lock);
}
COND_ACQUIRE(nic->requires_lock, &nic->lock);
grc = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) address,
length, dst_cq_hndl, flags,
vmdh_index, &md->mem_hndl);
COND_RELEASE(nic->requires_lock, &nic->lock);
if (unlikely(grc != GNI_RC_SUCCESS)) {
GNIX_INFO(FI_LOG_MR, "failed to register memory with uGNI, "
"ret=%s\n", gni_err_str[grc]);
_gnix_ref_put(nic);
return NULL;
}
/* set up the mem desc */
md->nic = nic;
md->domain = domain;
/* take references on domain */
_gnix_ref_get(md->domain);
return md;
}
