tm_topology_t* get_local_topo_with_hwloc(void) { hwloc_topology_t topology; tm_topology_t *res = NULL; hwloc_obj_t *objs = NULL; unsigned topodepth,depth; int nb_nodes,i; /* Build the topology */ hwloc_topology_init(&topology); hwloc_topology_ignore_all_keep_structure(topology); hwloc_topology_load(topology); /* Test if symetric */ if(!symetric(topology)){ if(get_verbose_level() >= CRITICAL) fprintf(stderr,"Local toplogy not symetric!\n"); exit(-1); } /* work on depth */ topodepth = hwloc_topology_get_depth(topology); res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t)); res->nb_levels = topodepth; res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels); res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels); /* Build TreeMatch topology */ for( depth = 0 ; depth < topodepth ; depth++ ){ nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth); res->nb_nodes[depth] = nb_nodes; res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes); objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL); hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1); res->arity[depth] = objs[0]->arity; /* printf("%d:",res->arity[depth]); */ /* Build process id tab */ for (i = 0; i < nb_nodes; i++){ res->node_id[depth][i] = objs[i]->os_index; /* if(depth==topodepth-1) */ } FREE(objs); } /* Destroy HWLOC topology object. */ hwloc_topology_destroy(topology); /* printf("\n"); */ return res; }
int symetric(hwloc_topology_t topology) { int depth,i,topodepth = hwloc_topology_get_depth(topology); unsigned int arity; hwloc_obj_t obj; for ( depth = 0; depth < topodepth-1 ; depth++ ) { int N = hwloc_get_nbobjs_by_depth(topology, depth); obj = hwloc_get_next_obj_by_depth (topology,depth,NULL); arity = obj->arity; /* printf("Depth=%d, N=%d, Arity:%d\n",depth,N,arity); */ for (i = 1; i < N; i++ ){ obj = hwloc_get_next_obj_by_depth (topology,depth,obj); if( obj->arity != arity){ /* printf("[%d]: obj->arity=%d, arity=%d\n",i,obj->arity,arity); */ return 0; } } } return 1; }
static HYD_status handle_bitmap_binding(const char *binding, const char *mapping) { int i, j, k, bind_count, map_count, cache_depth = 0, bind_depth = 0, map_depth = 0; int total_map_objs, total_bind_objs, num_pus_in_map_domain, num_pus_in_bind_domain, total_map_domains; hwloc_obj_t map_obj, bind_obj, *start_pu; hwloc_cpuset_t *map_domains; char *bind_str, *map_str; HYD_status status = HYD_SUCCESS; HYDU_FUNC_ENTER(); /* split out the count fields */ status = split_count_field(binding, &bind_str, &bind_count); HYDU_ERR_POP(status, "error splitting count field\n"); status = split_count_field(mapping, &map_str, &map_count); HYDU_ERR_POP(status, "error splitting count field\n"); /* get the binding object */ if (!strcmp(bind_str, "board")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE); else if (!strcmp(bind_str, "numa")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE); else if (!strcmp(bind_str, "socket")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET); else if (!strcmp(bind_str, "core")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE); else if (!strcmp(bind_str, "hwthread")) bind_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU); else { /* check if it's in the l*cache format */ cache_depth = parse_cache_string(bind_str); if (!cache_depth) { HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "unrecognized binding string \"%s\"\n", binding); } bind_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1); } /* get the mapping */ if (!strcmp(map_str, "board")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_MACHINE); else if (!strcmp(map_str, "numa")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_NODE); else if (!strcmp(map_str, "socket")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_SOCKET); else if (!strcmp(map_str, "core")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_CORE); else if (!strcmp(map_str, "hwthread")) map_depth = hwloc_get_type_or_above_depth(topology, HWLOC_OBJ_PU); else { cache_depth = parse_cache_string(map_str); if (!cache_depth) { HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "unrecognized mapping string \"%s\"\n", mapping); } map_depth = hwloc_get_cache_type_depth(topology, cache_depth, -1); } /* * Process Affinity Algorithm: * * The code below works in 3 stages. The end result is an array of all the possible * binding bitmaps for a system, based on the options specified. * * 1. Define all possible mapping "domains" in a system. A mapping domain is a group * of hardware elements found by traversing the topology. Each traversal skips the * number of elements the user specified in the mapping string. The traversal ends * when the next mapping domain == the first mapping domain. Note that if the * mapping string defines a domain that is larger than the system size, we exit * with an error. * * 2. Define the number of possible binding domains within a mapping domain. This * process is similar to step 1, in that we traverse the mapping domain finding * all possible bind combinations, stopping when a duplicate of the first binding * is reached. If a binding is larger (in # of PUs) than the mapping domain, * the number of possible bindings for that domain is 1. In this stage, we also * locate the first PU in each mapping domain for use later during binding. * * 3. Create the binding bitmaps. We allocate an array of bitmaps and fill them in * with all possible bindings. The starting PU in each mapping domain is advanced * if and when we wrap around to the beginning of the mapping domains. This ensures * that we do not repeat. * */ /* calculate the number of map domains */ total_map_objs = hwloc_get_nbobjs_by_depth(topology, map_depth); num_pus_in_map_domain = (HYDT_topo_hwloc_info.total_num_pus / total_map_objs) * map_count; HYDU_ERR_CHKANDJUMP(status, num_pus_in_map_domain > HYDT_topo_hwloc_info.total_num_pus, HYD_INTERNAL_ERROR, "mapping option \"%s\" larger than total system size\n", mapping); /* The number of total_map_domains should be large enough to * contain all contiguous map object collections of length * map_count. For example, if the map object is "socket" and the * map_count is 3, on a system with 4 sockets, the following map * domains should be included: (0,1,2), (3,0,1), (2,3,0), (1,2,3). * We do this by finding how many times we need to replicate the * list of the map objects so that an integral number of map * domains can map to them. In the above case, the list of map * objects is replicated 3 times. */ for (i = 1; (i * total_map_objs) % map_count; i++); total_map_domains = (i * total_map_objs) / map_count; /* initialize the map domains */ HYDU_MALLOC_OR_JUMP(map_domains, hwloc_bitmap_t *, total_map_domains * sizeof(hwloc_bitmap_t), status); HYDU_MALLOC_OR_JUMP(start_pu, hwloc_obj_t *, total_map_domains * sizeof(hwloc_obj_t), status); /* For each map domain, find the next map object (first map object * for the first map domain) and add the following "map_count" * number of contiguous map objects, wrapping to the first one if * needed, to the map domain. Store the first PU in the first map * object of the map domain as "start_pu". This is needed later * for the actual binding. */ map_obj = NULL; for (i = 0; i < total_map_domains; i++) { map_domains[i] = hwloc_bitmap_alloc(); hwloc_bitmap_zero(map_domains[i]); for (j = 0; j < map_count; j++) { map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj); /* map_obj will be NULL if it reaches the end. call again to wrap around */ if (!map_obj) map_obj = hwloc_get_next_obj_by_depth(topology, map_depth, map_obj); if (j == 0) start_pu[i] = hwloc_get_obj_inside_cpuset_by_type(topology, map_obj->cpuset, HWLOC_OBJ_PU, 0); hwloc_bitmap_or(map_domains[i], map_domains[i], map_obj->cpuset); } } /* Find the possible binding domains is similar to that of map * domains. But if a binding domain is larger (in # of PUs) than * the mapping domain, the number of possible bindings for that * domain is 1. */ /* calculate the number of possible bindings and allocate bitmaps for them */ total_bind_objs = hwloc_get_nbobjs_by_depth(topology, bind_depth); num_pus_in_bind_domain = (HYDT_topo_hwloc_info.total_num_pus / total_bind_objs) * bind_count; if (num_pus_in_bind_domain < num_pus_in_map_domain) { for (i = 1; (i * num_pus_in_map_domain) % num_pus_in_bind_domain; i++); HYDT_topo_hwloc_info.num_bitmaps = (i * num_pus_in_map_domain * total_map_domains) / num_pus_in_bind_domain; } else { HYDT_topo_hwloc_info.num_bitmaps = total_map_domains; } /* initialize bitmaps */ HYDU_MALLOC_OR_JUMP(HYDT_topo_hwloc_info.bitmap, hwloc_bitmap_t *, HYDT_topo_hwloc_info.num_bitmaps * sizeof(hwloc_bitmap_t), status); for (i = 0; i < HYDT_topo_hwloc_info.num_bitmaps; i++) { HYDT_topo_hwloc_info.bitmap[i] = hwloc_bitmap_alloc(); hwloc_bitmap_zero(HYDT_topo_hwloc_info.bitmap[i]); } /* do bindings */ i = 0; while (i < HYDT_topo_hwloc_info.num_bitmaps) { for (j = 0; j < total_map_domains; j++) { bind_obj = hwloc_get_ancestor_obj_by_depth(topology, bind_depth, start_pu[j]); for (k = 0; k < bind_count; k++) { hwloc_bitmap_or(HYDT_topo_hwloc_info.bitmap[i], HYDT_topo_hwloc_info.bitmap[i], bind_obj->cpuset); /* if the binding is smaller than the mapping domain, wrap around inside that domain */ if (num_pus_in_bind_domain < num_pus_in_map_domain) { bind_obj = hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j], bind_depth, bind_obj); if (!bind_obj) bind_obj = hwloc_get_next_obj_inside_cpuset_by_depth(topology, map_domains[j], bind_depth, bind_obj); } else { bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj); if (!bind_obj) bind_obj = hwloc_get_next_obj_by_depth(topology, bind_depth, bind_obj); } } i++; /* advance the starting position for this map domain, if needed */ if (num_pus_in_bind_domain < num_pus_in_map_domain) { for (k = 0; k < num_pus_in_bind_domain; k++) { start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j], HWLOC_OBJ_PU, start_pu[j]); if (!start_pu[j]) start_pu[j] = hwloc_get_next_obj_inside_cpuset_by_type(topology, map_domains[j], HWLOC_OBJ_PU, start_pu[j]); } } } } /* free temporary memory */ MPL_free(map_domains); MPL_free(start_pu); fn_exit: HYDU_FUNC_EXIT(); return status; fn_fail: goto fn_exit; }
tm_topology_t* hwloc_to_tm(char *filename,double **pcost) { hwloc_topology_t topology; tm_topology_t *res = NULL; hwloc_obj_t *objs = NULL; unsigned topodepth,depth; int nb_nodes,i; double *cost; int err; /* Build the topology */ hwloc_topology_init(&topology); err = hwloc_topology_set_xml(topology,filename); if(err == -1){ if(get_verbose_level() >= CRITICAL) fprintf(stderr,"Error: %s is a bad xml topology file!\n",filename); exit(-1); } hwloc_topology_ignore_all_keep_structure(topology); hwloc_topology_load(topology); /* Test if symetric */ if(!symetric(topology)){ if(get_verbose_level() >= CRITICAL) fprintf(stderr,"%s not symetric!\n",filename); exit(-1); } /* work on depth */ topodepth = hwloc_topology_get_depth(topology); res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t)); res->nb_levels = topodepth; res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels); res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels); if(get_verbose_level() >= INFO) printf("topodepth = %d\n",topodepth); /* Build TreeMatch topology */ for( depth = 0 ; depth < topodepth ; depth++ ){ nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth); res->nb_nodes[depth] = nb_nodes; res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes); objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL); hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1); res->arity[depth] = objs[0]->arity; if(get_verbose_level() >= INFO) printf("%d(%d):",res->arity[depth],nb_nodes); /* Build process id tab */ for (i = 0; i < nb_nodes; i++){ res->node_id[depth][i] = objs[i]->os_index; /* if(depth==topodepth-1) */ } FREE(objs); } cost = (double*)CALLOC(res->nb_levels,sizeof(double)); for(i=0; i<res->nb_levels; i++){ cost[i] = speed(i); } *pcost = cost; /* Destroy topology object. */ hwloc_topology_destroy(topology); if(get_verbose_level() >= INFO) printf("\n"); return res; }
tm_topology_t* get_local_topo_with_hwloc(void) { hwloc_topology_t topology; tm_topology_t *res = NULL; hwloc_obj_t *objs = NULL; unsigned topodepth,depth; int nb_nodes,i; /* Build the topology */ hwloc_topology_init(&topology); #if HWLOC_API_VERSION >= 0x00020000 hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_STRUCTURE); #else /* HWLOC_API_VERSION >= 0x00020000 */ hwloc_topology_ignore_all_keep_structure(topology); #endif /* HWLOC_API_VERSION >= 0x00020000 */ hwloc_topology_load(topology); /* Test if symetric */ if(!symetric(topology)){ if(tm_get_verbose_level() >= CRITICAL) fprintf(stderr,"Local toplogy not symetric!\n"); exit(-1); } /* work on depth */ topodepth = hwloc_topology_get_depth(topology); res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t)); res->nb_constraints = 0; res->constraints = NULL; res->nb_levels = topodepth; res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->node_rank = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->nb_nodes = (size_t*)MALLOC(sizeof(size_t)*res->nb_levels); res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels); /* Build TreeMatch topology */ for( depth = 0 ; depth < topodepth ; depth++ ){ nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth); res->nb_nodes[depth] = nb_nodes; res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); res->node_rank[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes); objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL); hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1); res->arity[depth] = objs[0]->arity; if (depth == topodepth -1){ res->nb_constraints = nb_nodes; res->nb_proc_units = nb_nodes; } /* printf("%d:",res->arity[depth]); */ /* Build process id tab */ for (i = 0; i < nb_nodes; i++){ res->node_id[depth][i] = objs[i]->os_index; res->node_rank[depth][objs[i]->os_index] = i; /* if(depth==topodepth-1) */ } FREE(objs); } /* Destroy HWLOC topology object. */ hwloc_topology_destroy(topology); /* printf("\n"); */ return res; }
tm_topology_t* hwloc_to_tm(char *filename) { hwloc_topology_t topology; tm_topology_t *res = NULL; hwloc_obj_t *objs = NULL; unsigned topodepth,depth; unsigned int nb_nodes; double *cost; int err, l; unsigned int i; int vl = tm_get_verbose_level(); /* Build the topology */ hwloc_topology_init(&topology); err = hwloc_topology_set_xml(topology,filename); if(err == -1){ if(vl >= CRITICAL) fprintf(stderr,"Error: %s is a bad xml topology file!\n",filename); exit(-1); } #if HWLOC_API_VERSION >= 0x00020000 hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_STRUCTURE); #else /* HWLOC_API_VERSION >= 0x00020000 */ hwloc_topology_ignore_all_keep_structure(topology); #endif /* HWLOC_API_VERSION >= 0x00020000 */ hwloc_topology_load(topology); /* Test if symetric */ if(!symetric(topology)){ if(tm_get_verbose_level() >= CRITICAL) fprintf(stderr,"%s not symetric!\n",filename); exit(-1); } /* work on depth */ topodepth = hwloc_topology_get_depth(topology); res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t)); res->oversub_fact = 1; res->nb_constraints = 0; res->constraints = NULL; res->nb_levels = topodepth; res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->node_rank = (int**)MALLOC(sizeof(int*)*res->nb_levels); res->nb_nodes = (size_t*)MALLOC(sizeof(size_t)*res->nb_levels); res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels); if(vl >= INFO) printf("topodepth = %d\n",topodepth); /* Build TreeMatch topology */ for( depth = 0 ; depth < topodepth ; depth++ ){ nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth); res->nb_nodes[depth] = nb_nodes; res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); res->node_rank[depth] = (int*)MALLOC(sizeof(int)*nb_nodes); objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes); objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL); hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1); res->arity[depth] = objs[0]->arity; if (depth == topodepth -1){ res->nb_constraints = nb_nodes; res->nb_proc_units = nb_nodes; } if(vl >= DEBUG) printf("\n--%d(%d) **%d**:--\n",res->arity[depth],nb_nodes,res->arity[0]); /* Build process id tab */ for (i = 0; i < nb_nodes; i++){ if(objs[i]->os_index > nb_nodes){ if(vl >= CRITICAL){ fprintf(stderr, "Index of object %d of level %d is %d and larger than number of nodes : %d\n", i, depth, objs[i]->os_index, nb_nodes); } exit(-1); } res->node_id[depth][i] = objs[i]->os_index; res->node_rank[depth][objs[i]->os_index] = i; /* if(depth==topodepth-1) */ } FREE(objs); } cost = (double*)CALLOC(res->nb_levels,sizeof(double)); for(l=0; l<res->nb_levels; l++){ cost[l] = link_cost(l); } res->cost = cost; /* Destroy topology object. */ hwloc_topology_destroy(topology); if(tm_get_verbose_level() >= INFO) printf("\n"); return res; }