void* mmap_1g(void* addr /* = nullptr */, int node /* = -1 */) {
#ifdef __linux__
if (s_num1GPages >= kMaxNum1GPages) return nullptr;
if (get_huge1g_info(node).free_hugepages <= 0) return nullptr;
if (node >= 0 && !numa_node_allowed(node)) return nullptr;
#ifdef HAVE_NUMA
bitmask* memMask = nullptr;
bitmask* interleaveMask = nullptr;
if (node >= 0 && numa_num_nodes > 1) {
memMask = numa_get_membind();
interleaveMask = numa_get_interleave_mask();
bitmask* mask = numa_allocate_nodemask();
numa_bitmask_setbit(mask, node);
numa_set_membind(mask);
numa_bitmask_free(mask);
}
#endif
void* ret = mmap_1g_impl(addr);
if (ret != nullptr) {
s_1GPages[s_num1GPages++] = ret;
}
#ifdef HAVE_NUMA
if (memMask) {
assert(interleaveMask);
numa_set_membind(memMask);
numa_set_interleave_mask(interleaveMask);
numa_bitmask_free(memMask);
numa_bitmask_free(interleaveMask);
}
#endif
return ret;
#else
return nullptr;
#endif
}
void show(void)
{
unsigned long prefnode;
struct bitmask *membind, *interleave, *cpubind;
unsigned long cur;
int policy;
int numa_num_nodes = numa_num_possible_nodes();
if (numa_available() < 0) {
show_physcpubind();
printf("No NUMA support available on this system.\n");
exit(1);
}
cpubind = numa_get_run_node_mask();
prefnode = numa_preferred();
interleave = numa_get_interleave_mask();
membind = numa_get_membind();
cur = numa_get_interleave_node();
policy = 0;
if (get_mempolicy(&policy, NULL, 0, 0, 0) < 0)
perror("get_mempolicy");
printf("policy: %s\n", policy_name(policy));
printf("preferred node: ");
switch (policy) {
case MPOL_PREFERRED:
if (prefnode != -1) {
printf("%ld\n", prefnode);
break;
}
/*FALL THROUGH*/
case MPOL_DEFAULT:
printf("current\n");
break;
case MPOL_INTERLEAVE:
printf("%ld (interleave next)\n",cur);
break;
case MPOL_BIND:
printf("%d\n", find_first_bit(&membind, numa_num_nodes));
break;
}
if (policy == MPOL_INTERLEAVE) {
printmask("interleavemask", interleave);
printf("interleavenode: %ld\n", cur);
}
show_physcpubind();
printmask("cpubind", cpubind); // for compatibility
printmask("nodebind", cpubind);
printmask("membind", membind);
}
void* mmap_2m(void* addr, int prot, int node /* = -1 */,
bool map_shared /* = false */, bool map_fixed /* = false */) {
#ifdef __linux__
if (get_huge2m_info(node).free_hugepages <= 0) return nullptr;
#ifdef HAVE_NUMA
bitmask* memMask = nullptr;
bitmask* interleaveMask = nullptr;
if (node >= 0 && numa_num_nodes > 1) {
assert(numa_node_set != 0);
if ((numa_node_set & (1u << node)) == 0) {
// Numa policy forbids allocation on the node.
return nullptr;
}
memMask = numa_get_membind();
interleaveMask = numa_get_interleave_mask();
bitmask* mask = numa_allocate_nodemask();
numa_bitmask_setbit(mask, node);
numa_set_membind(mask);
numa_bitmask_free(mask);
}
#endif
void* ret = mmap_2m_impl(addr, prot, map_shared, map_fixed);
s_num2MPages += !!ret;
#ifdef HAVE_NUMA
if (memMask) {
numa_set_membind(memMask);
numa_set_interleave_mask(interleaveMask);
numa_bitmask_free(memMask);
numa_bitmask_free(interleaveMask);
}
#endif
return ret;
#else // not linux
return nullptr;
#endif
}
size_t remap_interleaved_2m_pages(void* addr, size_t pages, int prot,
bool shared /* = false */) {
#ifdef __linux__
assert(reinterpret_cast<uintptr_t>(addr) % size2m == 0);
assert(addr != nullptr);
if (pages == 0) return 0;
#ifdef HAVE_NUMA
const int maxNode = numa_max_node();
bitmask* memMask = nullptr;
bitmask* interleaveMask = nullptr;
bitmask* mask = nullptr;
if (maxNode > 0) {
memMask = numa_get_membind();
interleaveMask = numa_get_interleave_mask();
mask = numa_allocate_nodemask();
}
#else
constexpr int maxNode = 0;
#endif
int node = -1;
int failed = 0; // consecutive failure count
int mapped_count = 0;
do {
#ifdef HAVE_NUMA
if (maxNode > 0) {
if (++node > maxNode) node = 0;
if (!numa_node_allowed(node)) {
// Numa policy forbids allocation on node
if (++failed > maxNode) break;
continue;
}
numa_bitmask_setbit(mask, node);
numa_set_membind(mask);
numa_bitmask_clearbit(mask, node);
}
#endif
// Fail early if we don't have huge pages reserved.
if (get_huge2m_info(node).free_hugepages > 0 &&
mmap_2m_impl(addr, prot, shared, true /* MAP_FIXED */)) {
addr = (char*)addr + size2m;
++mapped_count;
failed = 0;
continue;
}
// We failed on node, give up if we have failed on all nodes
if (++failed > maxNode) break;
} while (mapped_count < pages);
#ifdef HAVE_NUMA
if (mask) {
numa_set_membind(memMask);
numa_set_interleave_mask(interleaveMask);
numa_bitmask_free(mask);
numa_bitmask_free(interleaveMask);
numa_bitmask_free(memMask);
}
#endif
return mapped_count;
#else // not linux
return 0;
#endif
}