int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
{
unsigned char* ptr;
ompi_free_list_memory_t *alloc_ptr;
size_t i, alloc_size;
mca_mpool_base_registration_t* user_out = NULL;
if (flist->fl_max_to_alloc > 0)
if (flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
if (num_elements == 0)
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
alloc_size = num_elements * flist->fl_elem_size +
sizeof(ompi_free_list_memory_t) + flist->fl_header_space +
flist->fl_alignment;
if (NULL != flist->fl_mpool)
alloc_ptr = (ompi_free_list_memory_t*)flist->fl_mpool->mpool_alloc(flist->fl_mpool,
alloc_size, 0, MCA_MPOOL_FLAGS_CACHE_BYPASS, &user_out);
else
alloc_ptr = (ompi_free_list_memory_t*)malloc(alloc_size);
if(NULL == alloc_ptr)
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
/* make the alloc_ptr a list item, save the chunk in the allocations list, and
have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_memory_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
alloc_ptr->registration = user_out;
ptr = (unsigned char*) alloc_ptr + sizeof(ompi_free_list_memory_t);
ptr = (unsigned char*)(align_to((size_t)ptr + flist->fl_header_space,
flist->fl_alignment) - flist->fl_header_space);
for(i=0; i<num_elements; i++) {
ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr;
item->user_data = user_out;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class);
opal_atomic_lifo_push(&(flist->super), &(item->super));
ptr += flist->fl_elem_size;
}
flist->fl_num_allocated += num_elements;
return OMPI_SUCCESS;
}
int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements)
{
unsigned char* ptr;
unsigned char* alloc_ptr;
size_t i;
size_t mod;
if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
alloc_ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) +
sizeof(opal_list_item_t) +
CACHE_LINE_SIZE);
if(NULL == alloc_ptr)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
/* make the alloc_ptr a list item, save the chunk in the allocations list, and
have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, opal_list_item_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
ptr = alloc_ptr + sizeof(opal_list_item_t);
mod = (uintptr_t)ptr % CACHE_LINE_SIZE;
if(mod != 0) {
ptr += (CACHE_LINE_SIZE - mod);
}
if (NULL != flist->fl_elem_class) {
for(i=0; i<num_elements; i++) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class);
opal_list_append(&(flist->super), &(item->super));
ptr += flist->fl_elem_size;
}
} else {
for(i=0; i<num_elements; i++) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
opal_list_append(&(flist->super), &(item->super));
ptr += flist->fl_elem_size;
}
}
flist->fl_num_allocated += num_elements;
return OPAL_SUCCESS;
}
int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements)
{
unsigned char *ptr, *mpool_alloc_ptr = NULL, *payload_ptr = NULL;
opal_free_list_memory_t *alloc_ptr;
size_t alloc_size, head_size, elem_size = 0;
mca_mpool_base_registration_t *reg = NULL;
int rc = OPAL_SUCCESS;
if (flist->fl_max_to_alloc && (flist->fl_num_allocated + num_elements) >
flist->fl_max_to_alloc) {
num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
}
if (num_elements == 0) {
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
head_size = OPAL_ALIGN(flist->fl_frag_size, flist->fl_frag_alignment, size_t);
/* calculate head allocation size */
alloc_size = num_elements * head_size + sizeof(opal_free_list_memory_t) +
flist->fl_frag_alignment;
alloc_ptr = (opal_free_list_memory_t *) malloc(alloc_size);
if (OPAL_UNLIKELY(NULL == alloc_ptr)) {
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
if (0 != flist->fl_payload_buffer_size) {
elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size,
flist->fl_payload_buffer_alignment, size_t);
/* elem_size should not be 0 here */
assert (elem_size > 0);
/* allocate the rest from the mpool (or use memalign/malloc) */
if(flist->fl_mpool != NULL) {
payload_ptr = mpool_alloc_ptr =
(unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool,
num_elements * elem_size,
flist->fl_payload_buffer_alignment,
flist->fl_mpool_reg_flags, ®);
} else {
#ifdef HAVE_POSIX_MEMALIGN
posix_memalign ((void **) &mpool_alloc_ptr, flist->fl_payload_buffer_alignment,
num_elements * elem_size);
payload_ptr = mpool_alloc_ptr;
#else
mpool_alloc_ptr = (unsigned char *) malloc (num_elements * elem_size +
flist->fl_payload_buffer_alignment);
payload_ptr = (unsigned char *) OPAL_ALIGN((uintptr_t)mpool_alloc_ptr,
flist->fl_payload_buffer_alignment,
uintptr_t);
#endif
}
if(NULL == mpool_alloc_ptr) {
free(alloc_ptr);
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
}
/* make the alloc_ptr a list item, save the chunk in the allocations list,
* and have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, opal_free_list_item_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr);
alloc_ptr->registration = reg;
alloc_ptr->ptr = mpool_alloc_ptr;
ptr = (unsigned char*)alloc_ptr + sizeof(opal_free_list_memory_t);
ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char*);
for(size_t i = 0; i < num_elements ; ++i) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
item->registration = reg;
item->ptr = payload_ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class);
item->super.item_free = 0;
/* run the initialize function if present */
if (flist->item_init) {
if (OPAL_SUCCESS != (rc = flist->item_init(item, flist->ctx))) {
num_elements = i;
OBJ_DESTRUCT (item);
break;
}
}
/* NTH: in case the free list may be accessed from multiple threads
* use the atomic lifo push. The overhead is small compared to the
* overall overhead of opal_free_list_grow(). */
opal_lifo_push_atomic (&flist->super, &item->super);
ptr += head_size;
payload_ptr += elem_size;
}
if (OPAL_SUCCESS != rc && 0 == num_elements) {
/* couldn't initialize any items */
opal_list_remove_item (&flist->fl_allocations, (opal_list_item_t *) alloc_ptr);
opal_free_list_allocation_release (flist, alloc_ptr);
return OPAL_ERR_OUT_OF_RESOURCE;
}
flist->fl_num_allocated += num_elements;
return OPAL_SUCCESS;
}
