tm_topology_t* get_local_topo_with_hwloc(void)
{
hwloc_topology_t topology;
tm_topology_t *res = NULL;
hwloc_obj_t *objs = NULL;
unsigned topodepth,depth;
int nb_nodes,i;
/* Build the topology */
hwloc_topology_init(&topology);
hwloc_topology_ignore_all_keep_structure(topology);
hwloc_topology_load(topology);
/* Test if symetric */
if(!symetric(topology)){
if(get_verbose_level() >= CRITICAL)
fprintf(stderr,"Local toplogy not symetric!\n");
exit(-1);
}
/* work on depth */
topodepth = hwloc_topology_get_depth(topology);
res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
res->nb_levels = topodepth;
res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels);
res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);
/* Build TreeMatch topology */
for( depth = 0 ; depth < topodepth ; depth++ ){
nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
res->nb_nodes[depth] = nb_nodes;
res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
res->arity[depth] = objs[0]->arity;
/* printf("%d:",res->arity[depth]); */
/* Build process id tab */
for (i = 0; i < nb_nodes; i++){
res->node_id[depth][i] = objs[i]->os_index;
/* if(depth==topodepth-1) */
}
FREE(objs);
}
/* Destroy HWLOC topology object. */
hwloc_topology_destroy(topology);
/* printf("\n"); */
return res;
}
int symetric(hwloc_topology_t topology)
{
int depth,i,topodepth = hwloc_topology_get_depth(topology);
unsigned int arity;
hwloc_obj_t obj;
for ( depth = 0; depth < topodepth-1 ; depth++ ) {
int N = hwloc_get_nbobjs_by_depth(topology, depth);
obj = hwloc_get_next_obj_by_depth (topology,depth,NULL);
arity = obj->arity;
/* printf("Depth=%d, N=%d, Arity:%d\n",depth,N,arity); */
for (i = 1; i < N; i++ ){
obj = hwloc_get_next_obj_by_depth (topology,depth,obj);
if( obj->arity != arity){
/* printf("[%d]: obj->arity=%d, arity=%d\n",i,obj->arity,arity); */
return 0;
}
}
}
return 1;
}
static HYD_status handle_bitmap_binding(const char *binding, const char *mapping)
{
int i, j, k, bind_count, map_count, cache_depth = 0, bind_depth = 0, map_depth = 0;
int total_map_objs, total_bind_objs, num_pus_in_map_domain, num_pus_in_bind_domain,
total_map_domains;
hwloc_obj_t map_obj, bind_obj, *start_pu;
hwloc_cpuset_t *map_domains;
char *bind_str, *map_str;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
/* split out the count fields */
status = split_count_field(binding, &bind_str, &bind_count);
HYDU_ERR_POP(status, "error splitting count field\n");
status = split_count_field(mapping, &map_str, &map_count);
HYDU_ERR_POP(status, "error splitting count field\n");
/* get the binding object */
if (!strcmp(bind_str, "board"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE);
else if (!strcmp(bind_str, "numa"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE);
else if (!strcmp(bind_str, "socket"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET);
else if (!strcmp(bind_str, "core"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE);
else if (!strcmp(bind_str, "hwthread"))
bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU);
else {
/* check if it's in the l*cache format */
cache_depth = parse_cache_string(bind_str);
if (!cache_depth) {
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"unrecognized binding string \"%s\"\n", binding);
}
bind_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
}
/* get the mapping */
if (!strcmp(map_str, "board"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE);
else if (!strcmp(map_str, "numa"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE);
else if (!strcmp(map_str, "socket"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET);
else if (!strcmp(map_str, "core"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE);
else if (!strcmp(map_str, "hwthread"))
map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU);
else {
cache_depth = parse_cache_string(map_str);
if (!cache_depth) {
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"unrecognized mapping string \"%s\"\n", mapping);
}
map_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1);
}
/*
* Process Affinity Algorithm:
*
* The code below works in 3 stages. The end result is an array of all the possible
* binding bitmaps for a system, based on the options specified.
*
* 1. Define all possible mapping "domains" in a system. A mapping domain is a group
* of hardware elements found by traversing the topology. Each traversal skips the
* number of elements the user specified in the mapping string. The traversal ends
* when the next mapping domain == the first mapping domain. Note that if the
* mapping string defines a domain that is larger than the system size, we exit
* with an error.
*
* 2. Define the number of possible binding domains within a mapping domain. This
* process is similar to step 1, in that we traverse the mapping domain finding
* all possible bind combinations, stopping when a duplicate of the first binding
* is reached. If a binding is larger (in # of PUs) than the mapping domain,
* the number of possible bindings for that domain is 1. In this stage, we also
* locate the first PU in each mapping domain for use later during binding.
*
* 3. Create the binding bitmaps. We allocate an array of bitmaps and fill them in
* with all possible bindings. The starting PU in each mapping domain is advanced
* if and when we wrap around to the beginning of the mapping domains. This ensures
* that we do not repeat.
*
*/
/* calculate the number of map domains */
total_map_objs = hwloc_get_nbobjs_by_depth(topology, map_depth);
num_pus_in_map_domain = (HYDT_topo_hwloc_info.total_num_pus / total_map_objs) * map_count;
HYDU_ERR_CHKANDJUMP(status, num_pus_in_map_domain > HYDT_topo_hwloc_info.total_num_pus,
HYD_INTERNAL_ERROR, "mapping option \"%s\" larger than total system size\n",
mapping);
/* The number of total_map_domains should be large enough to
* contain all contiguous map object collections of length
* map_count. For example, if the map object is "socket" and the
* map_count is 3, on a system with 4 sockets, the following map
* domains should be included: (0,1,2), (3,0,1), (2,3,0), (1,2,3).
* We do this by finding how many times we need to replicate the
* list of the map objects so that an integral number of map
* domains can map to them. In the above case, the list of map
* objects is replicated 3 times. */
for (i = 1; (i * total_map_objs) % map_count; i++);
total_map_domains = (i * total_map_objs) / map_count;
/* initialize the map domains */
HYDU_MALLOC_OR_JUMP(map_domains, hwloc_bitmap_t *, total_map_domains * sizeof(hwloc_bitmap_t),
status);
HYDU_MALLOC_OR_JUMP(start_pu, hwloc_obj_t *, total_map_domains * sizeof(hwloc_obj_t), status);
/* For each map domain, find the next map object (first map object
* for the first map domain) and add the following "map_count"
* number of contiguous map objects, wrapping to the first one if
* needed, to the map domain. Store the first PU in the first map
* object of the map domain as "start_pu". This is needed later
* for the actual binding. */
map_obj = NULL;
for (i = 0; i < total_map_domains; i++) {
map_domains[i] = hwloc_bitmap_alloc();
hwloc_bitmap_zero(map_domains[i]);
for (j = 0; j < map_count; j++) {
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
/* map_obj will be NULL if it reaches the end. call again to wrap around */
if (!map_obj)
map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj);
if (j == 0)
start_pu[i] =
hwloc_get_obj_inside_cpuset_by_type(topology, map_obj->cpuset, HWLOC_OBJ_PU, 0);
hwloc_bitmap_or(map_domains[i], map_domains[i], map_obj->cpuset);
}
}
/* Find the possible binding domains is similar to that of map
* domains. But if a binding domain is larger (in # of PUs) than
* the mapping domain, the number of possible bindings for that
* domain is 1. */
/* calculate the number of possible bindings and allocate bitmaps for them */
total_bind_objs = hwloc_get_nbobjs_by_depth(topology, bind_depth);
num_pus_in_bind_domain = (HYDT_topo_hwloc_info.total_num_pus / total_bind_objs) * bind_count;
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
for (i = 1; (i * num_pus_in_map_domain) % num_pus_in_bind_domain; i++);
HYDT_topo_hwloc_info.num_bitmaps =
(i * num_pus_in_map_domain * total_map_domains) / num_pus_in_bind_domain;
}
else {
HYDT_topo_hwloc_info.num_bitmaps = total_map_domains;
}
/* initialize bitmaps */
HYDU_MALLOC_OR_JUMP(HYDT_topo_hwloc_info.bitmap, hwloc_bitmap_t *,
HYDT_topo_hwloc_info.num_bitmaps * sizeof(hwloc_bitmap_t), status);
for (i = 0; i < HYDT_topo_hwloc_info.num_bitmaps; i++) {
HYDT_topo_hwloc_info.bitmap[i] = hwloc_bitmap_alloc();
hwloc_bitmap_zero(HYDT_topo_hwloc_info.bitmap[i]);
}
/* do bindings */
i = 0;
while (i < HYDT_topo_hwloc_info.num_bitmaps) {
for (j = 0; j < total_map_domains; j++) {
bind_obj = hwloc_get_ancestor_obj_by_depth(topology, bind_depth, start_pu[j]);
for (k = 0; k < bind_count; k++) {
hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i],
bind_obj->cpuset);
/* if the binding is smaller than the mapping domain, wrap around inside that domain */
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
bind_obj =
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
bind_depth, bind_obj);
if (!bind_obj)
bind_obj =
hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j],
bind_depth, bind_obj);
}
else {
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
if (!bind_obj)
bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj);
}
}
i++;
/* advance the starting position for this map domain, if needed */
if (num_pus_in_bind_domain < num_pus_in_map_domain) {
for (k = 0; k < num_pus_in_bind_domain; k++) {
start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
HWLOC_OBJ_PU,
start_pu[j]);
if (!start_pu[j])
start_pu[j] =
hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j],
HWLOC_OBJ_PU, start_pu[j]);
}
}
}
}
/* free temporary memory */
MPL_free(map_domains);
MPL_free(start_pu);
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
tm_topology_t* hwloc_to_tm(char *filename,double **pcost)
{
hwloc_topology_t topology;
tm_topology_t *res = NULL;
hwloc_obj_t *objs = NULL;
unsigned topodepth,depth;
int nb_nodes,i;
double *cost;
int err;
/* Build the topology */
hwloc_topology_init(&topology);
err = hwloc_topology_set_xml(topology,filename);
if(err == -1){
if(get_verbose_level() >= CRITICAL)
fprintf(stderr,"Error: %s is a bad xml topology file!\n",filename);
exit(-1);
}
hwloc_topology_ignore_all_keep_structure(topology);
hwloc_topology_load(topology);
/* Test if symetric */
if(!symetric(topology)){
if(get_verbose_level() >= CRITICAL)
fprintf(stderr,"%s not symetric!\n",filename);
exit(-1);
}
/* work on depth */
topodepth = hwloc_topology_get_depth(topology);
res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
res->nb_levels = topodepth;
res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels);
res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);
if(get_verbose_level() >= INFO)
printf("topodepth = %d\n",topodepth);
/* Build TreeMatch topology */
for( depth = 0 ; depth < topodepth ; depth++ ){
nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
res->nb_nodes[depth] = nb_nodes;
res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
res->arity[depth] = objs[0]->arity;
if(get_verbose_level() >= INFO)
printf("%d(%d):",res->arity[depth],nb_nodes);
/* Build process id tab */
for (i = 0; i < nb_nodes; i++){
res->node_id[depth][i] = objs[i]->os_index;
/* if(depth==topodepth-1) */
}
FREE(objs);
}
cost = (double*)CALLOC(res->nb_levels,sizeof(double));
for(i=0; i<res->nb_levels; i++){
cost[i] = speed(i);
}
*pcost = cost;
/* Destroy topology object. */
hwloc_topology_destroy(topology);
if(get_verbose_level() >= INFO)
printf("\n");
return res;
}
tm_topology_t* get_local_topo_with_hwloc(void)
{
hwloc_topology_t topology;
tm_topology_t *res = NULL;
hwloc_obj_t *objs = NULL;
unsigned topodepth,depth;
int nb_nodes,i;
/* Build the topology */
hwloc_topology_init(&topology);
#if HWLOC_API_VERSION >= 0x00020000
hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_STRUCTURE);
#else /* HWLOC_API_VERSION >= 0x00020000 */
hwloc_topology_ignore_all_keep_structure(topology);
#endif /* HWLOC_API_VERSION >= 0x00020000 */
hwloc_topology_load(topology);
/* Test if symetric */
if(!symetric(topology)){
if(tm_get_verbose_level() >= CRITICAL)
fprintf(stderr,"Local toplogy not symetric!\n");
exit(-1);
}
/* work on depth */
topodepth = hwloc_topology_get_depth(topology);
res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
res->nb_constraints = 0;
res->constraints = NULL;
res->nb_levels = topodepth;
res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->node_rank = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->nb_nodes = (size_t*)MALLOC(sizeof(size_t)*res->nb_levels);
res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);
/* Build TreeMatch topology */
for( depth = 0 ; depth < topodepth ; depth++ ){
nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
res->nb_nodes[depth] = nb_nodes;
res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
res->node_rank[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
res->arity[depth] = objs[0]->arity;
if (depth == topodepth -1){
res->nb_constraints = nb_nodes;
res->nb_proc_units = nb_nodes;
}
/* printf("%d:",res->arity[depth]); */
/* Build process id tab */
for (i = 0; i < nb_nodes; i++){
res->node_id[depth][i] = objs[i]->os_index;
res->node_rank[depth][objs[i]->os_index] = i;
/* if(depth==topodepth-1) */
}
FREE(objs);
}
/* Destroy HWLOC topology object. */
hwloc_topology_destroy(topology);
/* printf("\n"); */
return res;
}
tm_topology_t* hwloc_to_tm(char *filename)
{
hwloc_topology_t topology;
tm_topology_t *res = NULL;
hwloc_obj_t *objs = NULL;
unsigned topodepth,depth;
unsigned int nb_nodes;
double *cost;
int err, l;
unsigned int i;
int vl = tm_get_verbose_level();
/* Build the topology */
hwloc_topology_init(&topology);
err = hwloc_topology_set_xml(topology,filename);
if(err == -1){
if(vl >= CRITICAL)
fprintf(stderr,"Error: %s is a bad xml topology file!\n",filename);
exit(-1);
}
#if HWLOC_API_VERSION >= 0x00020000
hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_STRUCTURE);
#else /* HWLOC_API_VERSION >= 0x00020000 */
hwloc_topology_ignore_all_keep_structure(topology);
#endif /* HWLOC_API_VERSION >= 0x00020000 */
hwloc_topology_load(topology);
/* Test if symetric */
if(!symetric(topology)){
if(tm_get_verbose_level() >= CRITICAL)
fprintf(stderr,"%s not symetric!\n",filename);
exit(-1);
}
/* work on depth */
topodepth = hwloc_topology_get_depth(topology);
res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
res->oversub_fact = 1;
res->nb_constraints = 0;
res->constraints = NULL;
res->nb_levels = topodepth;
res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->node_rank = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->nb_nodes = (size_t*)MALLOC(sizeof(size_t)*res->nb_levels);
res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);
if(vl >= INFO)
printf("topodepth = %d\n",topodepth);
/* Build TreeMatch topology */
for( depth = 0 ; depth < topodepth ; depth++ ){
nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
res->nb_nodes[depth] = nb_nodes;
res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
res->node_rank[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
res->arity[depth] = objs[0]->arity;
if (depth == topodepth -1){
res->nb_constraints = nb_nodes;
res->nb_proc_units = nb_nodes;
}
if(vl >= DEBUG)
printf("\n--%d(%d) **%d**:--\n",res->arity[depth],nb_nodes,res->arity[0]);
/* Build process id tab */
for (i = 0; i < nb_nodes; i++){
if(objs[i]->os_index > nb_nodes){
if(vl >= CRITICAL){
fprintf(stderr, "Index of object %d of level %d is %d and larger than number of nodes : %d\n",
i, depth, objs[i]->os_index, nb_nodes);
}
exit(-1);
}
res->node_id[depth][i] = objs[i]->os_index;
res->node_rank[depth][objs[i]->os_index] = i;
/* if(depth==topodepth-1) */
}
FREE(objs);
}
cost = (double*)CALLOC(res->nb_levels,sizeof(double));
for(l=0; l<res->nb_levels; l++){
cost[l] = link_cost(l);
}
res->cost = cost;
/* Destroy topology object. */
hwloc_topology_destroy(topology);
if(tm_get_verbose_level() >= INFO)
printf("\n");
return res;
}
