int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements) { unsigned char* ptr; ompi_free_list_memory_t *alloc_ptr; size_t i, alloc_size; mca_mpool_base_registration_t* user_out = NULL; if (flist->fl_max_to_alloc > 0) if (flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc) num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated; if (num_elements == 0) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; alloc_size = num_elements * flist->fl_elem_size + sizeof(ompi_free_list_memory_t) + flist->fl_header_space + flist->fl_alignment; if (NULL != flist->fl_mpool) alloc_ptr = (ompi_free_list_memory_t*)flist->fl_mpool->mpool_alloc(flist->fl_mpool, alloc_size, 0, MCA_MPOOL_FLAGS_CACHE_BYPASS, &user_out); else alloc_ptr = (ompi_free_list_memory_t*)malloc(alloc_size); if(NULL == alloc_ptr) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* make the alloc_ptr a list item, save the chunk in the allocations list, and have ptr point to memory right after the list item structure */ OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_memory_t); opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr); alloc_ptr->registration = user_out; ptr = (unsigned char*) alloc_ptr + sizeof(ompi_free_list_memory_t); ptr = (unsigned char*)(align_to((size_t)ptr + flist->fl_header_space, flist->fl_alignment) - flist->fl_header_space); for(i=0; i<num_elements; i++) { ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr; item->user_data = user_out; OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class); opal_atomic_lifo_push(&(flist->super), &(item->super)); ptr += flist->fl_elem_size; } flist->fl_num_allocated += num_elements; return OMPI_SUCCESS; }
int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements) { unsigned char* ptr; unsigned char* alloc_ptr; size_t i; size_t mod; if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc) return OPAL_ERR_TEMP_OUT_OF_RESOURCE; alloc_ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + sizeof(opal_list_item_t) + CACHE_LINE_SIZE); if(NULL == alloc_ptr) return OPAL_ERR_TEMP_OUT_OF_RESOURCE; /* make the alloc_ptr a list item, save the chunk in the allocations list, and have ptr point to memory right after the list item structure */ OBJ_CONSTRUCT(alloc_ptr, opal_list_item_t); opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr); ptr = alloc_ptr + sizeof(opal_list_item_t); mod = (uintptr_t)ptr % CACHE_LINE_SIZE; if(mod != 0) { ptr += (CACHE_LINE_SIZE - mod); } if (NULL != flist->fl_elem_class) { for(i=0; i<num_elements; i++) { opal_free_list_item_t* item = (opal_free_list_item_t*)ptr; OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class); opal_list_append(&(flist->super), &(item->super)); ptr += flist->fl_elem_size; } } else { for(i=0; i<num_elements; i++) { opal_free_list_item_t* item = (opal_free_list_item_t*)ptr; opal_list_append(&(flist->super), &(item->super)); ptr += flist->fl_elem_size; } } flist->fl_num_allocated += num_elements; return OPAL_SUCCESS; }
int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements) { unsigned char *ptr, *mpool_alloc_ptr = NULL, *payload_ptr = NULL; opal_free_list_memory_t *alloc_ptr; size_t alloc_size, head_size, elem_size = 0; mca_mpool_base_registration_t *reg = NULL; int rc = OPAL_SUCCESS; if (flist->fl_max_to_alloc && (flist->fl_num_allocated + num_elements) > flist->fl_max_to_alloc) { num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated; } if (num_elements == 0) { return OPAL_ERR_TEMP_OUT_OF_RESOURCE; } head_size = OPAL_ALIGN(flist->fl_frag_size, flist->fl_frag_alignment, size_t); /* calculate head allocation size */ alloc_size = num_elements * head_size + sizeof(opal_free_list_memory_t) + flist->fl_frag_alignment; alloc_ptr = (opal_free_list_memory_t *) malloc(alloc_size); if (OPAL_UNLIKELY(NULL == alloc_ptr)) { return OPAL_ERR_TEMP_OUT_OF_RESOURCE; } if (0 != flist->fl_payload_buffer_size) { elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size, flist->fl_payload_buffer_alignment, size_t); /* elem_size should not be 0 here */ assert (elem_size > 0); /* allocate the rest from the mpool (or use memalign/malloc) */ if(flist->fl_mpool != NULL) { payload_ptr = mpool_alloc_ptr = (unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool, num_elements * elem_size, flist->fl_payload_buffer_alignment, flist->fl_mpool_reg_flags, ®); } else { #ifdef HAVE_POSIX_MEMALIGN posix_memalign ((void **) &mpool_alloc_ptr, flist->fl_payload_buffer_alignment, num_elements * elem_size); payload_ptr = mpool_alloc_ptr; #else mpool_alloc_ptr = (unsigned char *) malloc (num_elements * elem_size + flist->fl_payload_buffer_alignment); payload_ptr = (unsigned char *) OPAL_ALIGN((uintptr_t)mpool_alloc_ptr, flist->fl_payload_buffer_alignment, uintptr_t); #endif } if(NULL == mpool_alloc_ptr) { free(alloc_ptr); return OPAL_ERR_TEMP_OUT_OF_RESOURCE; } } /* make the alloc_ptr a list item, save the chunk in the allocations list, * and have ptr point to memory right after the list item structure */ OBJ_CONSTRUCT(alloc_ptr, opal_free_list_item_t); opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr); alloc_ptr->registration = reg; alloc_ptr->ptr = mpool_alloc_ptr; ptr = (unsigned char*)alloc_ptr + sizeof(opal_free_list_memory_t); ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char*); for(size_t i = 0; i < num_elements ; ++i) { opal_free_list_item_t* item = (opal_free_list_item_t*)ptr; item->registration = reg; item->ptr = payload_ptr; OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class); item->super.item_free = 0; /* run the initialize function if present */ if (flist->item_init) { if (OPAL_SUCCESS != (rc = flist->item_init(item, flist->ctx))) { num_elements = i; OBJ_DESTRUCT (item); break; } } /* NTH: in case the free list may be accessed from multiple threads * use the atomic lifo push. The overhead is small compared to the * overall overhead of opal_free_list_grow(). */ opal_lifo_push_atomic (&flist->super, &item->super); ptr += head_size; payload_ptr += elem_size; } if (OPAL_SUCCESS != rc && 0 == num_elements) { /* couldn't initialize any items */ opal_list_remove_item (&flist->fl_allocations, (opal_list_item_t *) alloc_ptr); opal_free_list_allocation_release (flist, alloc_ptr); return OPAL_ERR_OUT_OF_RESOURCE; } flist->fl_num_allocated += num_elements; return OPAL_SUCCESS; }