void qrm_hwloc_topo(int *nodes, int *topo) { int depth, ret; unsigned i, n, j, ncores, nnodes, cnode; int topodepth, numa; hwloc_topology_t topology; hwloc_cpuset_t cpuset; hwloc_obj_t obj, cobj; hwloc_obj_type_t otype; hwloc_topology_init(&topology); hwloc_topology_load(topology); /* get the number os cores and NUMA nodes */ ncores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE); printf("ncores: %d\n",ncores); nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NODE); if(nnodes == 0){ otype = HWLOC_OBJ_SOCKET; printf("grouping with sockets\n"); nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_SOCKET); } else { otype = HWLOC_OBJ_NODE; printf("grouping with NUMA nodes\n"); } /* get the handle for the first NUMA node */ obj = hwloc_get_obj_by_type(topology, otype, 0); /* get the number of cores in one NUMA node (supposedly the same for all nodes) */ cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE); for(i=0; i<nnodes; i++){ /* get the handle for the first i-th node */ obj = hwloc_get_obj_by_type(topology, otype, i); /* get the number of cores in i-th NUMA node (supposedly the same for all nodes) */ cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE); /* get the first core in this node */ cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, NULL); topo[(i*cnode)] = cobj->logical_index; /* printf("%2d -- group: %2d",i,cobj->logical_index); */ for(j=1; j<cnode; j++){ cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, cobj); topo[(i*cnode)+j] = cobj->logical_index; /* printf(" %2d",cobj->logical_index); */ } /* printf("\n"); */ } hwloc_topology_destroy(topology); return; }
static HYD_status handle_bitmap_binding(const char *binding, const char *mapping) { int i, j, k, bind_count, map_count, cache_depth = 0, bind_depth = 0, map_depth = 0; int total_map_objs, total_bind_objs, num_pus_in_map_domain, num_pus_in_bind_domain, total_map_domains; hwloc_obj_t map_obj, bind_obj, *start_pu; hwloc_cpuset_t *map_domains; char *bind_str, *map_str; HYD_status status = HYD_SUCCESS; HYDU_FUNC_ENTER(); /* split out the count fields */ status = split_count_field(binding, &bind_str, &bind_count); HYDU_ERR_POP(status, "error splitting count field\n"); status = split_count_field(mapping, &map_str, &map_count); HYDU_ERR_POP(status, "error splitting count field\n"); /* get the binding object */ if (!strcmp(bind_str, "board")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE); else if (!strcmp(bind_str, "numa")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE); else if (!strcmp(bind_str, "socket")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET); else if (!strcmp(bind_str, "core")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE); else if (!strcmp(bind_str, "hwthread")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU); else { /* check if it's in the l*cache format */ cache_depth = parse_cache_string(bind_str); if (!cache_depth) { HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "unrecognized binding string \"%s\"\n", binding); } bind_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1); } /* get the mapping */ if (!strcmp(map_str, "board")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE); else if (!strcmp(map_str, "numa")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE); else if (!strcmp(map_str, "socket")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET); else if (!strcmp(map_str, "core")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE); else if (!strcmp(map_str, "hwthread")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU); else { cache_depth = parse_cache_string(map_str); if (!cache_depth) { HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "unrecognized mapping string \"%s\"\n", mapping); } map_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1); } /* * Process Affinity Algorithm: * * The code below works in 3 stages. The end result is an array of all the possible * binding bitmaps for a system, based on the options specified. * * 1. Define all possible mapping "domains" in a system. A mapping domain is a group * of hardware elements found by traversing the topology. Each traversal skips the * number of elements the user specified in the mapping string. The traversal ends * when the next mapping domain == the first mapping domain. Note that if the * mapping string defines a domain that is larger than the system size, we exit * with an error. * * 2. Define the number of possible binding domains within a mapping domain. This * process is similar to step 1, in that we traverse the mapping domain finding * all possible bind combinations, stopping when a duplicate of the first binding * is reached. If a binding is larger (in # of PUs) than the mapping domain, * the number of possible bindings for that domain is 1. In this stage, we also * locate the first PU in each mapping domain for use later during binding. * * 3. Create the binding bitmaps. We allocate an array of bitmaps and fill them in * with all possible bindings. The starting PU in each mapping domain is advanced * if and when we wrap around to the beginning of the mapping domains. This ensures * that we do not repeat. * */ /* calculate the number of map domains */ total_map_objs = hwloc_get_nbobjs_by_depth(topology, map_depth); num_pus_in_map_domain = (HYDT_topo_hwloc_info.total_num_pus / total_map_objs) * map_count; HYDU_ERR_CHKANDJUMP(status, num_pus_in_map_domain > HYDT_topo_hwloc_info.total_num_pus, HYD_INTERNAL_ERROR, "mapping option \"%s\" larger than total system size\n", mapping); /* The number of total_map_domains should be large enough to * contain all contiguous map object collections of length * map_count. For example, if the map object is "socket" and the * map_count is 3, on a system with 4 sockets, the following map * domains should be included: (0,1,2), (3,0,1), (2,3,0), (1,2,3). * We do this by finding how many times we need to replicate the * list of the map objects so that an integral number of map * domains can map to them. In the above case, the list of map * objects is replicated 3 times. */ for (i = 1; (i * total_map_objs) % map_count; i++); total_map_domains = (i * total_map_objs) / map_count; /* initialize the map domains */ HYDU_MALLOC_OR_JUMP(map_domains, hwloc_bitmap_t *, total_map_domains * sizeof(hwloc_bitmap_t), status); HYDU_MALLOC_OR_JUMP(start_pu, hwloc_obj_t *, total_map_domains * sizeof(hwloc_obj_t), status); /* For each map domain, find the next map object (first map object * for the first map domain) and add the following "map_count" * number of contiguous map objects, wrapping to the first one if * needed, to the map domain. Store the first PU in the first map * object of the map domain as "start_pu". This is needed later * for the actual binding. */ map_obj = NULL; for (i = 0; i < total_map_domains; i++) { map_domains[i] = hwloc_bitmap_alloc(); hwloc_bitmap_zero(map_domains[i]); for (j = 0; j < map_count; j++) { map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj); /* map_obj will be NULL if it reaches the end. call again to wrap around */ if (!map_obj) map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj); if (j == 0) start_pu[i] = hwloc_get_obj_inside_cpuset_by_type(topology, map_obj->cpuset, HWLOC_OBJ_PU, 0); hwloc_bitmap_or(map_domains[i], map_domains[i], map_obj->cpuset); } } /* Find the possible binding domains is similar to that of map * domains. But if a binding domain is larger (in # of PUs) than * the mapping domain, the number of possible bindings for that * domain is 1. */ /* calculate the number of possible bindings and allocate bitmaps for them */ total_bind_objs = hwloc_get_nbobjs_by_depth(topology, bind_depth); num_pus_in_bind_domain = (HYDT_topo_hwloc_info.total_num_pus / total_bind_objs) * bind_count; if (num_pus_in_bind_domain < num_pus_in_map_domain) { for (i = 1; (i * num_pus_in_map_domain) % num_pus_in_bind_domain; i++); HYDT_topo_hwloc_info.num_bitmaps = (i * num_pus_in_map_domain * total_map_domains) / num_pus_in_bind_domain; } else { HYDT_topo_hwloc_info.num_bitmaps = total_map_domains; } /* initialize bitmaps */ HYDU_MALLOC_OR_JUMP(HYDT_topo_hwloc_info.bitmap, hwloc_bitmap_t *, HYDT_topo_hwloc_info.num_bitmaps * sizeof(hwloc_bitmap_t), status); for (i = 0; i < HYDT_topo_hwloc_info.num_bitmaps; i++) { HYDT_topo_hwloc_info.bitmap[i] = hwloc_bitmap_alloc(); hwloc_bitmap_zero(HYDT_topo_hwloc_info.bitmap[i]); } /* do bindings */ i = 0; while (i < HYDT_topo_hwloc_info.num_bitmaps) { for (j = 0; j < total_map_domains; j++) { bind_obj = hwloc_get_ancestor_obj_by_depth(topology, bind_depth, start_pu[j]); for (k = 0; k < bind_count; k++) { hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i], bind_obj->cpuset); /* if the binding is smaller than the mapping domain, wrap around inside that domain */ if (num_pus_in_bind_domain < num_pus_in_map_domain) { bind_obj = hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j], bind_depth, bind_obj); if (!bind_obj) bind_obj = hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j], bind_depth, bind_obj); } else { bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj); if (!bind_obj) bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj); } } i++; /* advance the starting position for this map domain, if needed */ if (num_pus_in_bind_domain < num_pus_in_map_domain) { for (k = 0; k < num_pus_in_bind_domain; k++) { start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j], HWLOC_OBJ_PU, start_pu[j]); if (!start_pu[j]) start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j], HWLOC_OBJ_PU, start_pu[j]); } } } } /* free temporary memory */ MPL_free(map_domains); MPL_free(start_pu); fn_exit: HYDU_FUNC_EXIT(); return status; fn_fail: goto fn_exit; }
int main (void) { hwloc_topology_t topology; hwloc_obj_t obj, root; int err; err = hwloc_topology_init (&topology); if (err) return EXIT_FAILURE; hwloc_topology_set_synthetic (topology, "nodes:2 sockets:3 caches:4 cores:5 6"); err = hwloc_topology_load (topology); if (err) return EXIT_FAILURE; /* there is no second system object */ root = hwloc_get_root_obj (topology); obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, 1); assert(!obj); /* first system object is the top-level object of the topology */ obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, 0); assert(obj == hwloc_get_root_obj(topology)); /* first next-object object is the top-level object of the topology */ obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, NULL); assert(obj == hwloc_get_root_obj(topology)); /* there is no next object after the system object */ obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, obj); assert(!obj); /* check last PU */ obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 2*3*4*5*6-1); assert(obj == hwloc_get_obj_by_depth(topology, 5, 2*3*4*5*6-1)); /* there is no next PU after the last one */ obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, obj); assert(!obj); /* check there are 20 cores inside first socket */ root = hwloc_get_obj_by_depth(topology, 2, 0); assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE) == 20); /* check there are 12 caches inside last node */ root = hwloc_get_obj_by_depth(topology, 1, 1); assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CACHE) == 12); /* check first PU of second socket */ root = hwloc_get_obj_by_depth(topology, 2, 1); obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 0); assert(obj == hwloc_get_obj_by_depth(topology, 5, 4*5*6)); /* check third core of third socket */ root = hwloc_get_obj_by_depth(topology, 2, 2); obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE, 2); assert(obj == hwloc_get_obj_by_depth(topology, 4, 2*4*5+2)); /* check first socket of second node */ root = hwloc_get_obj_by_depth(topology, 1, 1); obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SOCKET, 0); assert(obj == hwloc_get_obj_by_depth(topology, 2, 3)); /* there is no node inside sockets */ root = hwloc_get_obj_by_depth(topology, 2, 0); obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_NODE, 0); assert(!obj); hwloc_topology_destroy (topology); return EXIT_SUCCESS; }