void* mk_malloc(size_t bytes)
{
assert(hbw_check_available());
void *ptr = NULL;
ptr = memkind_malloc(MEMKIND_HBW_HUGETLB, bytes);
if (ptr) return ptr;
ptr = memkind_malloc(MEMKIND_HBW, bytes);
if (ptr) return ptr;
perror("memkind_malloc()");
return NULL;
}
void *thread_onekind(void *arg)
{
struct arg_struct *args = (struct arg_struct *)arg;
int i;
if (pthread_mutex_lock(&mutex) != 0) {
fprintf(stderr, "Failed to acquire mutex.\n");
return NULL;
}
if (pthread_cond_wait(&cond, &mutex) != 0) {
fprintf(stderr, "Failed to block mutex on condition.\n");
return NULL;
}
if (pthread_mutex_unlock(&mutex) != 0) {
fprintf(stderr, "Failed to release mutex.\n");
return NULL;
}
// Lets alloc int and put there thread ID
for (i = 0; i < NUM_ALLOCS; i++) {
*(args->ptr+i) = (int *)memkind_malloc(args->kind, sizeof(int));
if (*(args->ptr+i) == NULL) {
fprintf(stderr, "Unable to allocate pmem int.\n");
return NULL;
}
**(args->ptr+i) = args->id;
}
return NULL;
}
memory_operation wrapped_malloc(size_t size)
{
START_TEST(type_id, FunctionCalls::MALLOC)
data.ptr = memkind_malloc(kind, size);
data.error_code = errno;
END_TEST
}
void *hbw_malloc(size_t size)
{
memkind_t kind;
kind = hbw_get_kind(HBW_PAGESIZE_4KB);
return memkind_malloc(kind, size);
}
static void put_kind_entry_into_kind_state(struct distmem *dist_kind) {
int hash_key = get_hashkey_from_string(dist_kind->name);
struct dist_intenal_distmem_node *specific_node =
(struct dist_intenal_distmem_node *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct dist_intenal_distmem_node));
specific_node->item = dist_kind;
specific_node->next = dist_memory_kinds[hash_key];
dist_memory_kinds[hash_key] = specific_node;
}
TEST_F(MemkindPmemTests, test_TC_MEMKIND_PmemMalloc)
{
const size_t size = 1024;
char *default_str = NULL;
default_str = (char *)memkind_malloc(pmem_kind, size);
EXPECT_TRUE(NULL != default_str);
sprintf(default_str, "memkind_malloc MEMKIND_PMEM\n");
printf("%s", default_str);
memkind_free(pmem_kind, default_str);
// Out of memory
default_str = (char *)memkind_malloc(pmem_kind, 2 * PMEM_PART_SIZE);
EXPECT_EQ(NULL, default_str);
}
void init_distmem() {
dist_memory_kinds =
(struct dist_intenal_distmem_node **)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct dist_intenal_distmem_node *) * HASH_SIZE);
int i;
for (i = 0; i < HASH_SIZE; i++) {
dist_memory_kinds[i] = NULL;
}
}
int distmem_arena_create(struct distmem *dist_kind, struct distmem_ops *ops, const char *name) {
int i;
dist_kind->internal_state = (struct distmem_memory_information_generic **)memkind_malloc(
MEMKIND_DEFAULT, sizeof(struct distmem_memory_information_generic *) * HASH_SIZE);
for (i = 0; i < HASH_SIZE; i++) {
dist_kind->internal_state[i] = NULL;
}
return 0;
}
void test_allocation(memkind_t kind, size_t size)
{
ASSERT_TRUE(kind != NULL);
void* ptr = memkind_malloc(kind, size);
ASSERT_TRUE(ptr != NULL);
void* memset_ret = memset(ptr, 3, size);
ASSERT_TRUE(memset_ret != NULL);
memkind_free(kind, ptr);
}
void * HBWSpace::allocate( const size_t arg_alloc_size ) const
{
static_assert( sizeof(void*) == sizeof(uintptr_t)
, "Error sizeof(void*) != sizeof(uintptr_t)" );
static_assert( Kokkos::Impl::power_of_two< Kokkos::Impl::MEMORY_ALIGNMENT >::value
, "Memory alignment must be power of two" );
constexpr uintptr_t alignment = Kokkos::Impl::MEMORY_ALIGNMENT ;
constexpr uintptr_t alignment_mask = alignment - 1 ;
void * ptr = 0 ;
if ( arg_alloc_size ) {
if ( m_alloc_mech == STD_MALLOC ) {
// Over-allocate to and round up to guarantee proper alignment.
size_t size_padded = arg_alloc_size + sizeof(void*) + alignment ;
void * alloc_ptr = memkind_malloc(MEMKIND_TYPE, size_padded );
if (alloc_ptr) {
uintptr_t address = reinterpret_cast<uintptr_t>(alloc_ptr);
// offset enough to record the alloc_ptr
address += sizeof(void *);
uintptr_t rem = address % alignment;
uintptr_t offset = rem ? (alignment - rem) : 0u;
address += offset;
ptr = reinterpret_cast<void *>(address);
// record the alloc'd pointer
address -= sizeof(void *);
*reinterpret_cast<void **>(address) = alloc_ptr;
}
}
}
if ( ( ptr == 0 ) || ( reinterpret_cast<uintptr_t>(ptr) == ~uintptr_t(0) )
|| ( reinterpret_cast<uintptr_t>(ptr) & alignment_mask ) ) {
std::ostringstream msg ;
msg << "Kokkos::Experimental::HBWSpace::allocate[ " ;
switch( m_alloc_mech ) {
case STD_MALLOC: msg << "STD_MALLOC" ; break ;
}
msg << " ]( " << arg_alloc_size << " ) FAILED" ;
if ( ptr == NULL ) { msg << " NULL" ; }
else { msg << " NOT ALIGNED " << ptr ; }
std::cerr << msg.str() << std::endl ;
std::cerr.flush();
Kokkos::Impl::throw_runtime_exception( msg.str() );
}
return ptr;
}
int distmem_create(struct distmem_ops *ops, const char *name, memkind_t *kind) {
int err = memkind_create(ops->memkind_operations, name, kind);
if (err) {
char err_msg[ERROR_MESSAGE_SIZE];
memkind_error_message(err, err_msg, ERROR_MESSAGE_SIZE);
fprintf(stderr, "%s", err_msg);
}
struct distmem *dist_kind = (struct distmem *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct distmem));
dist_kind->memkind = *kind;
dist_kind->name = (char *)memkind_malloc(MEMKIND_DEFAULT, strlen(name) + 1);
dist_kind->operations = ops;
strcpy(dist_kind->name, name);
err = ops->dist_create(dist_kind, ops, name);
if (err) {
fprintf(stderr, "Error in dist memory creation\n");
}
put_kind_entry_into_kind_state(dist_kind);
return 0;
}
void distmem_put_specific_entry_into_state(struct distmem *dist_kind, void *specific_info, void *ptr,
void (*deallocate_specific_information)(void *)) {
int hashkey = get_hashkey(ptr);
struct distmem_memory_information_generic *generic_info =
(struct distmem_memory_information_generic *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct distmem_memory_information_generic));
generic_info->ptr = ptr;
generic_info->specific_information = specific_info;
generic_info->deallocate_specific_information = deallocate_specific_information;
generic_info->next = dist_kind->internal_state[hashkey];
dist_kind->internal_state[hashkey] = generic_info;
}
TEST_F(FreeingMemorySegfault, test_TC_MEMKIND_freeing_memory_after_thread_finish)
{
void* ptr = nullptr;
std::thread t([&] {
ptr = memkind_malloc(MEMKIND_DEFAULT, 32);
ASSERT_TRUE(ptr != NULL);
});
t.join();
memkind_free(0, ptr);
SUCCEED();
}
////////////////////////////////////////////////////////////////////////////
// My malloc implementation calling memkind_malloc.
////////////////////////////////////////////////////////////////////////////
void *myMemkindMalloc(size_t size)
{
DBG(2) printf("In my memkind malloc sz:%ld .. ", size);
void *pp;
// if we have not initialized memkind HBW arena yet, call default kind
// Similarly, if the hueristic decides not to alloc in HBW, use default
//
if (!MemkindInitDone || !isAllocInHBW(size))
pp = memkind_malloc(MEMKIND_DEFAULT, size);
else {
DBG(2) printf("\tHBW");
pp = memkind_malloc(HBW_Type, size);
logHBW(pp, size);
}
DBG(2) printf("\tptr:%p\n", pp);
return pp;
}
////////////////////////////////////////////////////////////////////////////
// This function is executed at library load time.
// Initilize HBW arena by making a dummy allocation/free at library load
// time. Until HBW initialization is complete, we must not call any
// allocation routines with HBW as kind.
////////////////////////////////////////////////////////////////////////////
void __attribute__ ((constructor)) autohbw_load(void)
{
// First set the default memory type this library allocates. This can
// be overridden by env variable
// Note: 'memkind_hbw_preferred' will allow falling back to DDR but
// 'memkind_hbw will not'
// Note: If HBM is not installed on a system, memkind_hbw_preferred call
// woudl fail. Therefore, we need to check for availability first.
//
int ret = 0;
if (memkind_check_available(MEMKIND_HBW) == 0) {
ret = memkind_get_kind_by_name("memkind_hbw_preferred", &HBW_Type);
}
else {
printf("WARN: *** No HBM found in system. Will use default (DDR) "
"OR user specifid type ***\n");
ret = memkind_get_kind_by_name("memkind_default", &HBW_Type);
}
assert(!ret && "FATAL: Could not find default memory type\n");
// Read any env variables. This has to be done first because DbgLevel
// is set using env variables and debug printing is used below
//
setEnvValues(); // read any env variables
DBG(1) printf("INFO: autohbw.so loaded!\n");
// dummy HBW call to initialize HBW arena
//
void *pp = memkind_malloc(HBW_Type, 16);
//
if (pp) {
// We have successfully initilized HBW arena
//
DBG(2) printf("\t-HBW int call succeeded\n");
memkind_free(0, pp);
MemkindInitDone = TRUE; // enable HBW allocation
}
else {
errPrn("\t-HBW init call FAILED. Is required memory type present on your system?\n");
assert(0 && "HBW/memkind initialization faild");
}
}
TEST_F(MemkindDefaultTests, test_TC_MEMKIND_DefaultGetSize)
{
const size_t size = 512;
char *default_str = NULL;
int err = 0;
size_t *total, *free;
total = (size_t*) malloc(sizeof(size_t));
free = (size_t*) malloc(sizeof(size_t));
default_str = (char *)memkind_malloc(MEMKIND_DEFAULT, size);
EXPECT_TRUE(NULL != default_str);
err = memkind_get_size(MEMKIND_DEFAULT, total, free);
EXPECT_EQ(0,err);
memkind_free(MEMKIND_DEFAULT, default_str);
}
TEST_F(MemkindPmemTests, PmemGetSize)
{
const size_t size = 512;
char *default_str = NULL;
int err = 0;
size_t total;
size_t free;
default_str = (char *)memkind_malloc(pmem_kind, size);
EXPECT_TRUE(NULL != default_str);
err = memkind_get_size(pmem_kind, &total, &free);
EXPECT_EQ(0, err);
// requested PMEM partition size is internally aligned to 4MB
EXPECT_EQ(total, (size_t)roundup(PMEM_PART_SIZE, CHUNK_SIZE));
memkind_free(pmem_kind, default_str);
}
int distmem_create_default(struct distmem_ops *ops, const char *name, memkind_t *kind) {
ops->memkind_operations = (struct memkind_ops *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct memkind_ops));
memcpy(ops->memkind_operations, &MEMKIND_DEFAULT_OPS, sizeof(struct memkind_ops));
ops->memkind_operations->free = distmem_free;
return distmem_create(ops, name, kind);
}
MEMKIND_EXPORT void *hbw_malloc(size_t size)
{
return memkind_malloc(hbw_get_kind(HBW_PAGESIZE_4KB), size);
}
/**
* Initialises the local heap space, the master will determine the local heap start address for each process. This is broadcast to all
* processes who will then set up their own space via memkind, pin it & allocate the RMA window. All heaps are added into the directory
* here. Note that there might be slight gaps between local heaps and the start of the global address space and the first heap, this is
* because jemalloc requires the start address to divide into its chunk size so we have to round up to this.
*/
void *gvi_localHeap_initialise(int myRank, int totalRanks, void *global_base_address) {
int i;
struct memkind_ops *my_memkind_ops = (struct memkind_ops *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct memkind_ops));
memcpy(my_memkind_ops, &MEMKIND_PMEM_OPS, sizeof(struct memkind_ops));
my_memkind_ops->mmap = my_pmem_mmap;
struct distmem_ops *localheap_vtable = (struct distmem_ops *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct distmem_ops));
localheap_vtable->dist_malloc = NULL;
localheap_vtable->dist_create = distmem_arena_create;
localheap_vtable->memkind_operations = my_memkind_ops;
localheap_vtable->dist_determine_distribution = NULL;
distmem_create(localheap_vtable, "localheap", &LOCALHEAP_KIND);
struct memkind_ops *my_memkind_ops_cache = (struct memkind_ops *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct memkind_ops));
memcpy(my_memkind_ops_cache, &MEMKIND_PMEM_OPS, sizeof(struct memkind_ops));
my_memkind_ops_cache->mmap = my_pmem_mmap;
struct distmem_ops *localheap_vtable_cache = (struct distmem_ops *)memkind_malloc(MEMKIND_DEFAULT, sizeof(struct distmem_ops));
localheap_vtable_cache->dist_malloc = NULL;
localheap_vtable_cache->dist_create = distmem_arena_create;
localheap_vtable_cache->memkind_operations = my_memkind_ops_cache;
localheap_vtable_cache->dist_determine_distribution = NULL;
distmem_create(localheap_vtable_cache, "localheap_cache", &INTERNAL_LOCALCACHE_KIND);
size_t jemk_chunksize_exponent;
size_t s = sizeof(jemk_chunksize_exponent);
jemk_mallctl("opt.lg_chunk", &jemk_chunksize_exponent, &s, NULL, 0);
int jemk_chunksize = (int)pow(2.0, jemk_chunksize_exponent);
unsigned long start_addresses[totalRanks];
if (myRank == MASTER_RANK) {
for (i = 0; i < totalRanks; i++) {
if (i == 0) {
start_addresses[i] = (unsigned long)global_base_address;
} else {
start_addresses[i] = start_addresses[i - 1] + LOCAL_HEAP_SIZE;
}
start_addresses[i] = roundup(start_addresses[i], jemk_chunksize);
}
}
MPI_Bcast(start_addresses, totalRanks, MPI_UNSIGNED_LONG, MASTER_RANK, MPI_COMM_WORLD);
unsigned long local_cache_start_address = roundup(start_addresses[myRank] + (LOCAL_HEAP_SIZE - LOCAL_CACHE_SIZE), jemk_chunksize);
struct memkind_pmem *priv = (struct memkind_pmem *)INTERNAL_LOCALCACHE_KIND->priv;
priv->fd = 0;
priv->addr = (void *)local_cache_start_address;
priv->max_size = LOCAL_HEAP_SIZE - (local_cache_start_address - 1 - start_addresses[myRank]);
priv->offset = 0;
priv = (struct memkind_pmem *)LOCALHEAP_KIND->priv;
priv->fd = 0;
priv->addr = (void *)start_addresses[myRank];
priv->max_size = local_cache_start_address - 1 - start_addresses[myRank];
priv->offset = 0;
mlock(priv->addr, LOCAL_HEAP_SIZE);
for (i = 0; i < totalRanks; i++) {
gvi_directory_registerMemory((void *)start_addresses[i], LOCAL_HEAP_SIZE, i);
}
MPI_Win win;
MPI_Win_create(priv->addr, LOCAL_HEAP_SIZE, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
gvi_cache_registerLocalHeap(win, global_base_address, start_addresses, totalRanks);
return priv->addr;
}
