static void memkind_hbw_closest_numanode_init(void)
{
struct memkind_hbw_closest_numanode_t *g =
&memkind_hbw_closest_numanode_g;
int *bandwidth = NULL;
int num_unique = 0;
int high_bandwidth = 0;
int i;
struct bandwidth_nodes_t *bandwidth_nodes = NULL;
g->num_cpu = numa_num_configured_cpus();
g->closest_numanode = (int *)jemk_malloc(sizeof(int) * g->num_cpu);
bandwidth = (int *)jemk_malloc(sizeof(int) * NUMA_NUM_NODES);
if (!(g->closest_numanode && bandwidth)) {
g->init_err = MEMKIND_ERROR_MALLOC;
log_err("jemk_malloc() failed.");
goto exit;
}
g->init_err = fill_nodes_bandwidth(bandwidth, NUMA_NUM_NODES);
if (g->init_err)
goto exit;
g->init_err = create_bandwidth_nodes(NUMA_NUM_NODES, bandwidth,
&num_unique, &bandwidth_nodes);
if (g->init_err)
goto exit;
if (num_unique == 1) {
g->init_err = MEMKIND_ERROR_UNAVAILABLE;
goto exit;
}
high_bandwidth = bandwidth_nodes[num_unique-1].bandwidth;
g->init_err = set_closest_numanode(num_unique, bandwidth_nodes,
high_bandwidth, g->num_cpu,
g->closest_numanode);
for(i=0; i<bandwidth_nodes[num_unique-1].num_numanodes; i++) {
log_info("NUMA node %d is high-bandwidth memory.",
bandwidth_nodes[num_unique-1].numanodes[i]);
}
exit:
jemk_free(bandwidth_nodes);
jemk_free(bandwidth);
if (g->init_err) {
jemk_free(g->closest_numanode);
g->closest_numanode = NULL;
}
}
int memkind_thread_get_arena(struct memkind *kind, unsigned int *arena, size_t size)
{
int err = 0;
unsigned int *arena_tsd;
arena_tsd = pthread_getspecific(kind->arena_key);
if (arena_tsd == NULL) {
arena_tsd = jemk_malloc(sizeof(unsigned int));
if (arena_tsd == NULL) {
err = MEMKIND_ERROR_MALLOC;
}
if (!err) {
*arena_tsd = _mm_crc32_u64(0, (uint64_t)pthread_self()) %
kind->arena_map_len;
err = pthread_setspecific(kind->arena_key, arena_tsd) ?
MEMKIND_ERROR_PTHREAD : 0;
}
}
if (!err) {
*arena = kind->arena_map[*arena_tsd];
if (*arena == UINT_MAX) {
err = MEMKIND_ERROR_MALLCTL;
}
}
return err;
}
MEMKIND_EXPORT void *memkind_default_malloc(struct memkind *kind, size_t size)
{
if(MEMKIND_UNLIKELY(size_out_of_bounds(size))) {
return NULL;
}
return jemk_malloc(size);
}
int memkind_arena_create_map(struct memkind *kind)
{
int err = 0;
int i;
size_t unsigned_size = sizeof(unsigned int);
if (kind->arena_map_len == 0) {
err = memkind_set_arena_map_len(kind);
}
#ifndef MEMKIND_TLS
if (kind->ops->get_arena == memkind_thread_get_arena) {
pthread_key_create(&(kind->arena_key), jemk_free);
}
#endif
if (kind->arena_map_len) {
kind->arena_map = (unsigned int *)jemk_malloc(sizeof(unsigned int) * kind->arena_map_len);
if (kind->arena_map == NULL) {
err = MEMKIND_ERROR_MALLOC;
}
}
if (!err) {
for (i = 0; i < kind->arena_map_len; ++i) {
kind->arena_map[i] = UINT_MAX;
}
for (i = 0; !err && i < kind->arena_map_len; ++i) {
err = jemk_mallctl("arenas.extendk", kind->arena_map + i,
&unsigned_size, &(kind->partition),
unsigned_size);
}
if (err) {
if (kind->arena_map) {
jemk_free(kind->arena_map);
}
err = MEMKIND_ERROR_MALLCTL;
}
}
return err;
}
static int create_bandwidth_nodes(int num_bandwidth, const int *bandwidth,
int *num_unique, struct bandwidth_nodes_t **bandwidth_nodes)
{
/***************************************************************************
* num_bandwidth (IN): *
* number of numa nodes and length of bandwidth vector. *
* bandwidth (IN): *
* A vector of length num_bandwidth that gives bandwidth for *
* each numa node, zero if numa node has unknown bandwidth. *
* num_unique (OUT): *
* number of unique non-zero bandwidth values in bandwidth *
* vector. *
* bandwidth_nodes (OUT): *
* A list of length num_unique sorted by bandwidth value where *
* each element gives a list of the numa nodes that have the *
* given bandwidth. *
* RETURNS zero on success, error code on failure *
***************************************************************************/
int err = 0;
int i, j, k, l, last_bandwidth;
struct numanode_bandwidth_t *numanode_bandwidth = NULL;
*bandwidth_nodes = NULL;
/* allocate space for sorting array */
numanode_bandwidth = jemk_malloc(sizeof(struct numanode_bandwidth_t) *
num_bandwidth);
if (!numanode_bandwidth) {
err = MEMKIND_ERROR_MALLOC;
log_err("jemk_malloc() failed.");
}
if (!err) {
/* set sorting array */
j = 0;
for (i = 0; i < num_bandwidth; ++i) {
if (bandwidth[i] != 0) {
numanode_bandwidth[j].numanode = i;
numanode_bandwidth[j].bandwidth = bandwidth[i];
++j;
}
}
/* ignore zero bandwidths */
num_bandwidth = j;
if (num_bandwidth == 0) {
err = MEMKIND_ERROR_UNAVAILABLE;
}
}
if (!err) {
qsort(numanode_bandwidth, num_bandwidth,
sizeof(struct numanode_bandwidth_t), numanode_bandwidth_compare);
/* calculate the number of unique bandwidths */
*num_unique = 1;
last_bandwidth = numanode_bandwidth[0].bandwidth;
for (i = 1; i < num_bandwidth; ++i) {
if (numanode_bandwidth[i].bandwidth != last_bandwidth) {
last_bandwidth = numanode_bandwidth[i].bandwidth;
++*num_unique;
}
}
/* allocate output array */
*bandwidth_nodes = (struct bandwidth_nodes_t*)jemk_malloc(
sizeof(struct bandwidth_nodes_t) * (*num_unique) +
sizeof(int) * num_bandwidth);
if (!*bandwidth_nodes) {
err = MEMKIND_ERROR_MALLOC;
log_err("jemk_malloc() failed.");
}
}
if (!err) {
/* populate output */
(*bandwidth_nodes)[0].numanodes = (int*)(*bandwidth_nodes + *num_unique);
last_bandwidth = numanode_bandwidth[0].bandwidth;
k = 0;
l = 0;
for (i = 0; i < num_bandwidth; ++i, ++l) {
(*bandwidth_nodes)[0].numanodes[i] = numanode_bandwidth[i].numanode;
if (numanode_bandwidth[i].bandwidth != last_bandwidth) {
(*bandwidth_nodes)[k].num_numanodes = l;
(*bandwidth_nodes)[k].bandwidth = last_bandwidth;
l = 0;
++k;
(*bandwidth_nodes)[k].numanodes = (*bandwidth_nodes)[0].numanodes + i;
last_bandwidth = numanode_bandwidth[i].bandwidth;
}
}
(*bandwidth_nodes)[k].num_numanodes = l;
(*bandwidth_nodes)[k].bandwidth = last_bandwidth;
}
if (numanode_bandwidth) {
jemk_free(numanode_bandwidth);
}
if (err) {
if (*bandwidth_nodes) {
jemk_free(*bandwidth_nodes);
}
}
return err;
}
void *memkind_default_malloc(struct memkind *kind, size_t size)
{
return jemk_malloc(size);
}
static int memkind_store(void *memptr, void **mmapptr, struct memkind **kind,
size_t *req_size, size_t *size, int mode)
{
static int table_len = 0;
static int is_init = 0;
static memkind_table_node_t *table = NULL;
static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
int err = 0;
int hash, i;
memkind_list_node_t *storeptr, *lastptr;
if (!is_init && *mmapptr == NULL) {
return -1;
}
if (!is_init) {
pthread_mutex_lock(&init_mutex);
if (!is_init) {
table_len = numa_num_configured_cpus();
table = jemk_malloc(sizeof(memkind_table_node_t) * table_len);
if (table == NULL) {
err = MEMKIND_ERROR_MALLOC;
}
else {
for (i = 0; i < table_len; ++i) {
pthread_mutex_init(&(table[i].mutex), NULL);
table[i].list = NULL;
}
is_init = 1;
}
}
pthread_mutex_unlock(&init_mutex);
}
if (is_init) {
hash = ptr_hash(memptr, table_len);
pthread_mutex_lock(&(table[hash].mutex));
if (mode == GBTLB_STORE_REMOVE || mode == GBTLB_STORE_QUERY) {
/*
memkind_store() call is a query
GBTLB_STORE_REMOVE -> Query if found remove and
return the address and size;
GBTLB_STORE_QUERTY -> Query if found and return;
*/
storeptr = table[hash].list;
lastptr = NULL;
while (storeptr && storeptr->ptr != memptr) {
lastptr = storeptr;
storeptr = storeptr->next;
}
if (storeptr == NULL) {
err = MEMKIND_ERROR_RUNTIME;
}
if (!err) {
*mmapptr = storeptr->mmapptr;
*size = storeptr->size;
*req_size = storeptr->requested_size;
*kind = storeptr->kind;
}
if (!err && mode == GBTLB_STORE_REMOVE) {
if (lastptr) {
lastptr->next = storeptr->next;
}
else {
table[hash].list = storeptr->next;
}
jemk_free(storeptr);
}
}
else { /* memkind_store() call is a store */
storeptr = table[hash].list;
table[hash].list = (memkind_list_node_t*)jemk_malloc(sizeof(memkind_list_node_t));
table[hash].list->ptr = memptr;
table[hash].list->mmapptr = *mmapptr;
table[hash].list->size = *size;
table[hash].list->requested_size = *req_size;
table[hash].list->kind = *kind;
table[hash].list->next = storeptr;
}
pthread_mutex_unlock(&(table[hash].mutex));
}
else {
err = MEMKIND_ERROR_MALLOC;
}
return err;
}