size_t remap_interleaved_2m_pages(void* addr, size_t pages, int prot,
bool shared /* = false */) {
#ifdef __linux__
assert(reinterpret_cast<uintptr_t>(addr) % size2m == 0);
assert(addr != nullptr);
if (pages == 0) return 0;
#ifdef HAVE_NUMA
const int maxNode = numa_max_node();
bitmask* memMask = nullptr;
bitmask* interleaveMask = nullptr;
bitmask* mask = nullptr;
if (maxNode > 0) {
memMask = numa_get_membind();
interleaveMask = numa_get_interleave_mask();
mask = numa_allocate_nodemask();
}
#else
constexpr int maxNode = 0;
#endif
int node = -1;
int failed = 0; // consecutive failure count
int mapped_count = 0;
do {
#ifdef HAVE_NUMA
if (maxNode > 0) {
if (++node > maxNode) node = 0;
if (!numa_node_allowed(node)) {
// Numa policy forbids allocation on node
if (++failed > maxNode) break;
continue;
}
numa_bitmask_setbit(mask, node);
numa_set_membind(mask);
numa_bitmask_clearbit(mask, node);
}
#endif
// Fail early if we don't have huge pages reserved.
if (get_huge2m_info(node).free_hugepages > 0 &&
mmap_2m_impl(addr, prot, shared, true /* MAP_FIXED */)) {
addr = (char*)addr + size2m;
++mapped_count;
failed = 0;
continue;
}
// We failed on node, give up if we have failed on all nodes
if (++failed > maxNode) break;
} while (mapped_count < pages);
#ifdef HAVE_NUMA
if (mask) {
numa_set_membind(memMask);
numa_set_interleave_mask(interleaveMask);
numa_bitmask_free(mask);
numa_bitmask_free(interleaveMask);
numa_bitmask_free(memMask);
}
#endif
return mapped_count;
#else // not linux
return 0;
#endif
}
int init_virtual_topology(config_t* cfg, cpu_model_t* cpu_model, virtual_topology_t** virtual_topologyp)
{
char* mc_pci_file;
char* str;
char* saveptr;
char* token = "NULL";
int* physical_node_ids;
physical_node_t** physical_nodes;
int num_physical_nodes;
int n, v, i, j, sibling_idx, node_i_idx;
int node_id;
physical_node_t* node_i, *node_j, *sibling_node;
int ret;
int min_distance;
int hyperthreading;
struct bitmask* mem_nodes;
virtual_topology_t* virtual_topology;
__cconfig_lookup_string(cfg, "topology.physical_nodes", &str);
// parse the physical nodes string
physical_node_ids = calloc(numa_num_possible_nodes(), sizeof(*physical_node_ids));
num_physical_nodes = 0;
while (token = strtok_r(str, ",", &saveptr)) {
physical_node_ids[num_physical_nodes] = atoi(token);
str = NULL;
if (++num_physical_nodes > numa_num_possible_nodes()) {
// we re being asked to run on more nodes than available
free(physical_node_ids);
ret = E_ERROR;
goto done;
}
}
physical_nodes = calloc(num_physical_nodes, sizeof(*physical_nodes));
// select those nodes we can run on (e.g. not constrained by any numactl)
mem_nodes = numa_get_mems_allowed();
for (i=0, n=0; i<num_physical_nodes; i++) {
node_id = physical_node_ids[i];
if (numa_bitmask_isbitset(mem_nodes, node_id)) {
physical_nodes[n] = malloc(sizeof(**physical_nodes));
physical_nodes[n]->node_id = node_id;
// TODO: what if we want to avoid using only a single hardware contexts of a hyperthreaded core?
physical_nodes[n]->cpu_bitmask = numa_allocate_cpumask();
numa_node_to_cpus(node_id, physical_nodes[n]->cpu_bitmask);
__cconfig_lookup_bool(cfg, "topology.hyperthreading", &hyperthreading);
if (hyperthreading) {
physical_nodes[n]->num_cpus = num_cpus(physical_nodes[n]->cpu_bitmask);
} else {
DBG_LOG(INFO, "Not using hyperthreading.\n");
// disable the upper half of the processors in the bitmask
physical_nodes[n]->num_cpus = num_cpus(physical_nodes[n]->cpu_bitmask) / 2;
int fc = first_cpu(physical_nodes[n]->cpu_bitmask);
for (j=fc+system_num_cpus()/2; j<fc+system_num_cpus()/2+physical_nodes[n]->num_cpus; j++) {
if (numa_bitmask_isbitset(physical_nodes[n]->cpu_bitmask, j)) {
numa_bitmask_clearbit(physical_nodes[n]->cpu_bitmask, j);
}
}
}
n++;
}
}
free(physical_node_ids);
num_physical_nodes = n;
// if pci bus topology of each physical node is not provided then discover it
if (__cconfig_lookup_string(cfg, "topology.mc_pci", &mc_pci_file) == CONFIG_FALSE ||
(__cconfig_lookup_string(cfg, "topology.mc_pci", &mc_pci_file) == CONFIG_TRUE &&
load_mc_pci_topology(mc_pci_file, physical_nodes, num_physical_nodes) != E_SUCCESS))
{
discover_mc_pci_topology(cpu_model, physical_nodes, num_physical_nodes);
save_mc_pci_topology(mc_pci_file, physical_nodes, num_physical_nodes);
}
// form virtual nodes by grouping physical nodes that are close to each other
virtual_topology = malloc(sizeof(*virtual_topology));
virtual_topology->num_virtual_nodes = num_physical_nodes / 2 + num_physical_nodes % 2;
virtual_topology->virtual_nodes = calloc(virtual_topology->num_virtual_nodes,
sizeof(*(virtual_topology->virtual_nodes)));
for (i=0, v=0; i<num_physical_nodes; i++) {
min_distance = INT_MAX;
sibling_node = NULL;
sibling_idx = -1;
if ((node_i = physical_nodes[i]) == NULL) {
continue;
}
for (j=i+1; j<num_physical_nodes; j++) {
if ((node_j = physical_nodes[j]) == NULL) {
continue;
}
if (numa_distance(node_i->node_id,node_j->node_id) < min_distance) {
sibling_node = node_j;
sibling_idx = j;
}
}
if (sibling_node) {
physical_nodes[i] = physical_nodes[sibling_idx] = NULL;
virtual_node_t* virtual_node = &virtual_topology->virtual_nodes[v];
virtual_node->dram_node = node_i;
virtual_node->nvram_node = sibling_node;
virtual_node->node_id = v;
virtual_node->cpu_model = cpu_model;
DBG_LOG(INFO, "Fusing physical nodes %d %d into virtual node %d\n",
node_i->node_id, sibling_node->node_id, virtual_node->node_id);
v++;
}
}
// any physical node that is not paired with another physical node is
// formed into a virtual node on its own
if (2*v < num_physical_nodes) {
for (i=0; i<num_physical_nodes; i++) {
node_i = physical_nodes[i];
virtual_node_t* virtual_node = &virtual_topology->virtual_nodes[v];
virtual_node->dram_node = virtual_node->nvram_node = node_i;
virtual_node->node_id = v;
DBG_LOG(WARNING, "Forming physical node %d into virtual node %d without a sibling node.\n",
node_i->node_id, virtual_node->node_id);
}
}
*virtual_topologyp = virtual_topology;
ret = E_SUCCESS;
done:
free(physical_nodes);
return ret;
}
