/**
* Destroy a slab.
*
* @param[in] handle Handle to the allocator being used.
* @param[in] slab Slab to be destroyed.
*
* @return FI_SUCCESS On successful slab destruction.
*
* @return -FI_EINVAL On invalid handle or slab being given as parameters.
*/
static int __destroy_slab(struct gnix_mbox_alloc_handle *handle,
struct gnix_slab *slab)
{
size_t total_size;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (!handle || !slab) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Invalid argument handle or slab.\n");
return -FI_EINVAL;
}
total_size = handle->page_size * __page_count(handle);
_gnix_free_bitmap(slab->used);
free(slab->used);
COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
GNI_MemDeregister(handle->nic_handle->gni_nic_hndl,
&slab->memory_handle);
COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
munmap(slab->base, total_size);
free(slab);
return FI_SUCCESS;
}
/**
* Determine how many mboxes are in a requested allocation size.
*
* @param[in] handle Handle to the allocator being used.
*
* @return Number of mail boxes being allocated.
*/
static size_t __mbox_count(struct gnix_mbox_alloc_handle *handle)
{
size_t mbox_count = (__page_count(handle) * handle->page_size) /
handle->mbox_size;
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Mbox_count: %zu.\n", mbox_count);
return mbox_count;
}
/**
* Fill all of the fields of an mbox to be returned to the requester.
*
* @param[in] handle Handle to the allocator being used.
* @param[in] slab Slab which the mbox is allocated from.
* @param[in] position Position of the mbox in the slab.
* @param[out] ptr Contains the allocated mbox upon success.
*
* @return FI_SUCCESS Upon successfully filling an mbox with relevant data.
* @return -FI_EINVAL Upon receiving invalid input, or finding the bitmap in
* a corrupted state.
* @return -FI_ENOMEM Upon failure to create the mbox structure using calloc.
*/
static int __fill_mbox(struct gnix_mbox_alloc_handle *handle,
struct gnix_slab *slab, size_t position,
struct gnix_mbox **ptr)
{
struct gnix_mbox *out;
int ret = FI_SUCCESS;
char error_buf[256];
size_t mapped_size;
char *error;
out = calloc(1, sizeof(*out));
if (!out) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating mbox: %s\n",
error);
ret = -FI_ENOMEM;
goto err_mbox_calloc;
}
mapped_size = handle->page_size * __page_count(handle);
out->slab = slab;
out->base = slab->base;
out->offset = (position * handle->mbox_size);
out->memory_handle = &slab->memory_handle;
if (out->offset > mapped_size) {
GNIX_WARN(FI_LOG_EP_CTRL, "Mbox out of bounds.\n");
ret = -FI_EINVAL;
goto err_invalid;
}
/* On some systems, the page may not be zero'd from first use.
Memset it here */
memset((void *) ((uint64_t) out->base + out->offset),
0x0, handle->mbox_size);
ret = _gnix_test_and_set_bit(slab->used, position);
if (ret != 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Bit already set when creating mbox.\n");
ret = -FI_EINVAL;
goto err_invalid;
}
*ptr = out;
return ret;
err_invalid:
free(out);
err_mbox_calloc:
return ret;
}
/**
* Create a slab from a handle and append to the slab list.
*
* @param[in] handle Handle to the allocator being used.
*
* @return FI_SUCCESS On successful slab creation.
*
* @return -FI_ENOMEM if failure to allocate memory for slab or bitmap.
* @return [Unspec] if failure in alloc_bitmap. Will return error code from
* alloc_bitmap.
* @return [Unspec] if failure in GNI_MemRegister. Converts gni_return_t
* status code to FI_ERRNO value.
*/
static int __create_slab(struct gnix_mbox_alloc_handle *handle)
{
struct gnix_slab *slab;
gni_return_t status;
char error_buf[256];
char *error;
size_t total_size;
int ret;
int vmdh_index = -1;
int flags = GNI_MEM_READWRITE;
struct gnix_auth_key *info;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
slab = calloc(1, sizeof(*slab));
if (!slab) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating slab: %s\n",
error);
ret = -FI_ENOMEM;
goto err_slab_calloc;
}
total_size = handle->page_size * __page_count(handle);
GNIX_DEBUG(FI_LOG_EP_CTRL, "total_size requested for mmap: %zu.\n",
total_size);
slab->used = calloc(1, sizeof(*(slab->used)));
if (!slab->used) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating bitmap: %s\n",
error);
ret = -FI_ENOMEM;
goto err_bitmap_calloc;
}
slab->base = mmap(0, total_size, (PROT_READ | PROT_WRITE), MAP_SHARED,
handle->fd, handle->last_offset);
if (slab->base == MAP_FAILED) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL, "%s\n", error);
ret = -FI_ENOMEM;
goto err_mmap;
}
ret = _gnix_alloc_bitmap(slab->used, __mbox_count(handle), NULL);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap.\n");
goto err_alloc_bitmap;
}
COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (handle->nic_handle->using_vmdh) {
info = _gnix_auth_key_lookup(GNIX_PROV_DEFAULT_AUTH_KEY,
GNIX_PROV_DEFAULT_AUTH_KEYLEN);
assert(info);
if (!handle->nic_handle->mdd_resources_set) {
/* check to see if the ptag registration limit was set
* yet or not -- becomes read-only after success */
_gnix_auth_key_enable(info);
status = GNI_SetMddResources(
handle->nic_handle->gni_nic_hndl,
(info->attr.prov_key_limit +
info->attr.user_key_limit));
assert(status == GNI_RC_SUCCESS);
handle->nic_handle->mdd_resources_set = 1;
}
vmdh_index = _gnix_get_next_reserved_key(info);
if (vmdh_index <= 0) {
GNIX_FATAL(FI_LOG_DOMAIN,
"failed to get reserved key for mbox "
"registration, rc=%d\n",
vmdh_index);
}
flags |= GNI_MEM_USE_VMDH;
}
status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl,
(uint64_t) slab->base, total_size,
handle->cq_handle,
flags, vmdh_index,
&slab->memory_handle);
COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err_memregister;
}
slab->allocator = handle;
gnix_slist_insert_tail(&slab->list_entry, &handle->slab_list);
handle->last_offset += total_size;
return ret;
err_memregister:
_gnix_free_bitmap(slab->used);
err_alloc_bitmap:
munmap(slab->base, total_size);
err_mmap:
free(slab->used);
err_bitmap_calloc:
free(slab);
err_slab_calloc:
return ret;
}
