void qrm_hwloc_topo(int *nodes, int *topo)
{
int depth, ret;
unsigned i, n, j, ncores, nnodes, cnode;
int topodepth, numa;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj, cobj;
hwloc_obj_type_t otype;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
/* get the number os cores and NUMA nodes */
ncores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
printf("ncores: %d\n",ncores);
nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NODE);
if(nnodes == 0){
otype = HWLOC_OBJ_SOCKET;
printf("grouping with sockets\n");
nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_SOCKET);
} else {
otype = HWLOC_OBJ_NODE;
printf("grouping with NUMA nodes\n");
}
/* get the handle for the first NUMA node */
obj = hwloc_get_obj_by_type(topology, otype, 0);
/* get the number of cores in one NUMA node (supposedly the same for all nodes) */
cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);
for(i=0; i<nnodes; i++){
/* get the handle for the first i-th node */
obj = hwloc_get_obj_by_type(topology, otype, i);
/* get the number of cores in i-th NUMA node (supposedly the same for all nodes) */
cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);
/* get the first core in this node */
cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, NULL);
topo[(i*cnode)] = cobj->logical_index;
/* printf("%2d -- group: %2d",i,cobj->logical_index); */
for(j=1; j<cnode; j++){
cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, cobj);
topo[(i*cnode)+j] = cobj->logical_index;
/* printf(" %2d",cobj->logical_index); */
}
/* printf("\n"); */
}
hwloc_topology_destroy(topology);
return;
}
static HYD_status handle_bitmap_binding(const char *binding, const char *mapping)
{
int i, j, k, bind_count, map_count, cache_depth = 0, bind_depth = 0, map_depth = 0;
int total_map_objs, total_bind_objs, num_pus_in_map_domain, num_pus_in_bind_domain,
total_map_domains;
hwloc_obj_t map_obj, bind_obj, *start_pu;
hwloc_cpuset_t *map_domains;
char *bind_str, *map_str;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
/* split out the count fields */
status = split_count_field(binding, &bind_str, &bind_count);
HYDU_ERR_POP(status, "error splitting count field\n");
status = split_count_field(mapping, &map_str, &map_count);
HYDU_ERR_POP(status, "error splitting count field\n");
/* get the binding object */
if (!strcmp(bind_str, "board"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE);
else if (!strcmp(bind_str, "numa"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE);
else if (!strcmp(bind_str, "socket"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET);
else if (!strcmp(bind_str, "core"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE);
else if (!strcmp(bind_str, "hwthread"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU);
else {
/* check if it's in the l*cache format */
cache_depth = parse_cache_string(bind_str);
if (!cache_depth) {
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"unrecognized binding string \"%s\"\n", binding);
}
bind_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
}
/* get the mapping */
if (!strcmp(map_str, "board"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE);
else if (!strcmp(map_str, "numa"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE);
else if (!strcmp(map_str, "socket"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET);
else if (!strcmp(map_str, "core"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE);
else if (!strcmp(map_str, "hwthread"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU);
else {
cache_depth = parse_cache_string(map_str);
if (!cache_depth) {
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"unrecognized mapping string \"%s\"\n", mapping);
}
map_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
}
/*
* Process Affinity Algorithm:
*
* The code below works in 3 stages. The end result is an array of all the possible
* binding bitmaps for a system, based on the options specified.
*
* 1. Define all possible mapping "domains" in a system. A mapping domain is a group
* of hardware elements found by traversing the topology. Each traversal skips the
* number of elements the user specified in the mapping string. The traversal ends
* when the next mapping domain == the first mapping domain. Note that if the
* mapping string defines a domain that is larger than the system size, we exit
* with an error.
*
* 2. Define the number of possible binding domains within a mapping domain. This
* process is similar to step 1, in that we traverse the mapping domain finding
* all possible bind combinations, stopping when a duplicate of the first binding
* is reached. If a binding is larger (in # of PUs) than the mapping domain,
* the number of possible bindings for that domain is 1. In this stage, we also
* locate the first PU in each mapping domain for use later during binding.
*
* 3. Create the binding bitmaps. We allocate an array of bitmaps and fill them in
* with all possible bindings. The starting PU in each mapping domain is advanced
* if and when we wrap around to the beginning of the mapping domains. This ensures
* that we do not repeat.
*
*/
/* calculate the number of map domains */
total_map_objs = hwloc_get_nbobjs_by_depth(topology, map_depth);
num_pus_in_map_domain = (HYDT_topo_hwloc_info.total_num_pus / total_map_objs) * map_count;
HYDU_ERR_CHKANDJUMP(status, num_pus_in_map_domain > HYDT_topo_hwloc_info.total_num_pus,
HYD_INTERNAL_ERROR, "mapping option \"%s\" larger than total system size\n",
mapping);
/* The number of total_map_domains should be large enough to
* contain all contiguous map object collections of length
* map_count. For example, if the map object is "socket" and the
* map_count is 3, on a system with 4 sockets, the following map
* domains should be included: (0,1,2), (3,0,1), (2,3,0), (1,2,3).
* We do this by finding how many times we need to replicate the
* list of the map objects so that an integral number of map
* domains can map to them. In the above case, the list of map
* objects is replicated 3 times. */
for (i = 1; (i * total_map_objs) % map_count; i++);
total_map_domains = (i * total_map_objs) / map_count;
/* initialize the map domains */
HYDU_MALLOC_OR_JUMP(map_domains, hwloc_bitmap_t *, total_map_domains * sizeof(hwloc_bitmap_t),
status);
HYDU_MALLOC_OR_JUMP(start_pu, hwloc_obj_t *, total_map_domains * sizeof(hwloc_obj_t), status);
/* For each map domain, find the next map object (first map object
* for the first map domain) and add the following "map_count"
* number of contiguous map objects, wrapping to the first one if
* needed, to the map domain. Store the first PU in the first map
* object of the map domain as "start_pu". This is needed later
* for the actual binding. */
map_obj = NULL;
for (i = 0; i < total_map_domains; i++) {
map_domains[i] = hwloc_bitmap_alloc();
hwloc_bitmap_zero(map_domains[i]);
for (j = 0; j < map_count; j++) {
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
/* map_obj will be NULL if it reaches the end. call again to wrap around */
if (!map_obj)
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
if (j == 0)
start_pu[i] =
hwloc_get_obj_inside_cpuset_by_type(topology, map_obj->cpuset, HWLOC_OBJ_PU, 0);
hwloc_bitmap_or(map_domains[i], map_domains[i], map_obj->cpuset);
}
}
/* Find the possible binding domains is similar to that of map
* domains. But if a binding domain is larger (in # of PUs) than
* the mapping domain, the number of possible bindings for that
* domain is 1. */
/* calculate the number of possible bindings and allocate bitmaps for them */
total_bind_objs = hwloc_get_nbobjs_by_depth(topology, bind_depth);
num_pus_in_bind_domain = (HYDT_topo_hwloc_info.total_num_pus / total_bind_objs) * bind_count;
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
for (i = 1; (i * num_pus_in_map_domain) % num_pus_in_bind_domain; i++);
HYDT_topo_hwloc_info.num_bitmaps =
(i * num_pus_in_map_domain * total_map_domains) / num_pus_in_bind_domain;
}
else {
HYDT_topo_hwloc_info.num_bitmaps = total_map_domains;
}
/* initialize bitmaps */
HYDU_MALLOC_OR_JUMP(HYDT_topo_hwloc_info.bitmap, hwloc_bitmap_t *,
HYDT_topo_hwloc_info.num_bitmaps * sizeof(hwloc_bitmap_t), status);
for (i = 0; i < HYDT_topo_hwloc_info.num_bitmaps; i++) {
HYDT_topo_hwloc_info.bitmap[i] = hwloc_bitmap_alloc();
hwloc_bitmap_zero(HYDT_topo_hwloc_info.bitmap[i]);
}
/* do bindings */
i = 0;
while (i < HYDT_topo_hwloc_info.num_bitmaps) {
for (j = 0; j < total_map_domains; j++) {
bind_obj = hwloc_get_ancestor_obj_by_depth(topology, bind_depth, start_pu[j]);
for (k = 0; k < bind_count; k++) {
hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i],
bind_obj->cpuset);
/* if the binding is smaller than the mapping domain, wrap around inside that domain */
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
bind_obj =
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
bind_depth, bind_obj);
if (!bind_obj)
bind_obj =
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
bind_depth, bind_obj);
}
else {
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
if (!bind_obj)
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
}
}
i++;
/* advance the starting position for this map domain, if needed */
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
for (k = 0; k < num_pus_in_bind_domain; k++) {
start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
HWLOC_OBJ_PU,
start_pu[j]);
if (!start_pu[j])
start_pu[j] =
hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
HWLOC_OBJ_PU, start_pu[j]);
}
}
}
}
/* free temporary memory */
MPL_free(map_domains);
MPL_free(start_pu);
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
int
main (void)
{
hwloc_topology_t topology;
hwloc_obj_t obj, root;
int err;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "nodes:2 sockets:3 caches:4 cores:5 6");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
/* there is no second system object */
root = hwloc_get_root_obj (topology);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, 1);
assert(!obj);
/* first system object is the top-level object of the topology */
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, 0);
assert(obj == hwloc_get_root_obj(topology));
/* first next-object object is the top-level object of the topology */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, NULL);
assert(obj == hwloc_get_root_obj(topology));
/* there is no next object after the system object */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, obj);
assert(!obj);
/* check last PU */
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 2*3*4*5*6-1);
assert(obj == hwloc_get_obj_by_depth(topology, 5, 2*3*4*5*6-1));
/* there is no next PU after the last one */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, obj);
assert(!obj);
/* check there are 20 cores inside first socket */
root = hwloc_get_obj_by_depth(topology, 2, 0);
assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE) == 20);
/* check there are 12 caches inside last node */
root = hwloc_get_obj_by_depth(topology, 1, 1);
assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CACHE) == 12);
/* check first PU of second socket */
root = hwloc_get_obj_by_depth(topology, 2, 1);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 0);
assert(obj == hwloc_get_obj_by_depth(topology, 5, 4*5*6));
/* check third core of third socket */
root = hwloc_get_obj_by_depth(topology, 2, 2);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE, 2);
assert(obj == hwloc_get_obj_by_depth(topology, 4, 2*4*5+2));
/* check first socket of second node */
root = hwloc_get_obj_by_depth(topology, 1, 1);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SOCKET, 0);
assert(obj == hwloc_get_obj_by_depth(topology, 2, 3));
/* there is no node inside sockets */
root = hwloc_get_obj_by_depth(topology, 2, 0);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_NODE, 0);
assert(!obj);
hwloc_topology_destroy (topology);
return EXIT_SUCCESS;
}
