int hwloc_topology_diff_build(hwloc_topology_t topo1,
hwloc_topology_t topo2,
unsigned long flags,
hwloc_topology_diff_t *diffp)
{
hwloc_topology_diff_t lastdiff, tmpdiff;
int err;
if (flags != 0) {
errno = EINVAL;
return -1;
}
*diffp = NULL;
err = hwloc_diff_trees(topo1, hwloc_get_root_obj(topo1),
topo2, hwloc_get_root_obj(topo2),
flags,
diffp, &lastdiff);
if (!err) {
tmpdiff = *diffp;
while (tmpdiff) {
if (tmpdiff->generic.type == HWLOC_TOPOLOGY_DIFF_TOO_COMPLEX) {
err = 1;
break;
}
tmpdiff = tmpdiff->generic.next;
}
}
return err;
}
void hw_high_level_show(struct machine_output *ma_out)
{
if(local_topo->nnodes > local_topo->nsockets || local_topo->nnodes == 0){
strcpy(ma_out->console_output,"\n MACHINE| Socket organization\n");
print_machine_branch(topology, hwloc_get_root_obj(topology), 0, HWLOC_OBJ_SOCKET);}
else{
strcpy(ma_out->console_output,"\n MACHINE| NUMA node organization\n");
print_machine_branch(topology, hwloc_get_root_obj(topology), 0, HWLOC_OBJ_NODE);}
strcat(ma_out->console_output, console_output);
strcat(ma_out->console_output, "\0");
ma_out->len = strlen(ma_out->console_output);
}
static void check(unsigned nbnodes, unsigned nbcores, unsigned nbpus)
{
unsigned depth;
unsigned nb;
unsigned long long total_memory;
/* sanity checks */
depth = hwloc_topology_get_depth(topology);
assert(depth == 4);
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NODE);
assert(depth == 1);
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_CORE);
assert(depth == 2);
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU);
assert(depth == 3);
/* actual checks */
nb = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NODE);
assert(nb == nbnodes);
nb = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
assert(nb == nbcores);
nb = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
assert(nb == nbpus);
total_memory = hwloc_get_root_obj(topology)->memory.total_memory;
assert(total_memory == nbnodes * 1024*1024*1024); /* synthetic topology puts 1GB per node */
}
void output_synthetic(hwloc_topology_t topology, const char *filename, int overwrite, int logical __hwloc_attribute_unused, int legend __hwloc_attribute_unused, int verbose_mode __hwloc_attribute_unused)
{
FILE *output;
hwloc_obj_t obj = hwloc_get_root_obj(topology);
int arity;
if (!obj->symmetric_subtree) {
fprintf(stderr, "Cannot output assymetric topology in synthetic format.\n");
fprintf(stderr, "Adding --no-io may help making the topology symmetric.\n");
return;
}
output = open_output(filename, overwrite);
if (!output) {
fprintf(stderr, "Failed to open %s for writing (%s)\n", filename, strerror(errno));
return;
}
arity = obj->arity;
while (arity) {
char types[64];
obj = obj->first_child;
hwloc_obj_type_snprintf(types, sizeof(types), obj, 1);
fprintf(output, "%s:%u ", types, arity);
arity = obj->arity;
}
fprintf(output, "\n");
if (output != stdout)
fclose(output);
}
void LpelHwLocInit(lpel_config_t *cfg)
{
#ifdef HAVE_HWLOC
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
pu_count = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
cpu_sets = malloc(pu_count * sizeof(hwloc_cpuset_t));
hw_places = malloc(pu_count * sizeof(lpel_hw_place_t));
traverse(hwloc_get_root_obj(topology));
#elif defined(HAVE_SYSCONF)
/* query the number of CPUs */
pu_count = sysconf(_SC_NPROCESSORS_ONLN);
#else
char *p = getenv("LPEL_NUM_WORKERS");
if (p) {
errno = 0;
pu_count = strtoul(p, 0, 0);
if (errno) pu_count = 0;
}
#endif
cfg->num_workers = pu_count;
cfg->proc_workers = pu_count;
cfg->proc_others = 0;
}
void output_synthetic(hwloc_topology_t topology, const char *filename, int logical __hwloc_attribute_unused, int legend __hwloc_attribute_unused, int verbose_mode __hwloc_attribute_unused)
{
FILE *output;
hwloc_obj_t obj = hwloc_get_root_obj(topology);
int arity;
if (!obj->symmetric_subtree) {
fprintf(stderr, "Cannot output assymetric topology in synthetic format.\n");
fprintf(stderr, "Adding --no-io may help making the topology symmetric.\n");
return;
}
if (!filename || !strcmp(filename, "-"))
output = stdout;
else {
output = open_file(filename, "w");
if (!output) {
fprintf(stderr, "Failed to open %s for writing (%s)\n", filename, strerror(errno));
return;
}
}
arity = obj->arity;
while (arity) {
obj = obj->first_child;
fprintf(output, "%s:%u ", hwloc_obj_type_string(obj->type), arity);
arity = obj->arity;
}
fprintf(output, "\n");
if (output != stdout)
fclose(output);
}
static void find_and_assign_combinations_with_hwloc(int *workerids, int nworkers)
{
struct _starpu_machine_config *config = _starpu_get_machine_config();
struct _starpu_machine_topology *topology = &config->topology;
int synthesize_arity = starpu_get_env_number("STARPU_SYNTHESIZE_ARITY_COMBINED_WORKER");
int min = starpu_get_env_number("STARPU_MIN_WORKERSIZE");
if (min < 2)
min = 2;
int max = starpu_get_env_number("STARPU_MAX_WORKERSIZE");
if (max == -1)
max = INT_MAX;
if (synthesize_arity == -1)
synthesize_arity = 2;
/* First, mark nodes which contain CPU workers, simply by setting their userdata field */
int i;
for (i = 0; i < nworkers; i++)
{
struct _starpu_worker *worker = _starpu_get_worker_struct(workerids[i]);
if (worker->perf_arch.devices[0].type == STARPU_CPU_WORKER && worker->perf_arch.devices[0].ncores == 1)
{
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology->hwtopology, config->pu_depth, worker->bindid);
obj = obj->parent;
while (obj)
{
obj->userdata = (void*) -1;
obj = obj->parent;
}
}
}
find_and_assign_combinations(hwloc_get_root_obj(topology->hwtopology), min, max, synthesize_arity);
}
static xmlDocPtr
hwloc__libxml2_prepare_export(hwloc_topology_t topology)
{
struct hwloc__xml_export_state_s state;
hwloc__libxml_export_state_data_t data = (void *) state.data;
xmlDocPtr doc = NULL; /* document pointer */
xmlNodePtr root_node = NULL; /* root pointer */
assert(sizeof(*data) <= sizeof(state.data));
LIBXML_TEST_VERSION;
hwloc_libxml2_disable_stderrwarnings();
/* Creates a new document, a node and set it as a root node. */
doc = xmlNewDoc(BAD_CAST "1.0");
root_node = xmlNewNode(NULL, BAD_CAST "topology");
xmlDocSetRootElement(doc, root_node);
/* Creates a DTD declaration. Isn't mandatory. */
(void) xmlCreateIntSubset(doc, BAD_CAST "topology", NULL, BAD_CAST "hwloc.dtd");
state.new_child = hwloc__libxml_export_new_child;
state.new_prop = hwloc__libxml_export_new_prop;
state.add_content = hwloc__libxml_export_add_content;
state.end_object = hwloc__libxml_export_end_object;
data->current_node = root_node;
hwloc__xml_export_object (&state, topology, hwloc_get_root_obj(topology));
return doc;
}
int main(int argc, char **argv) {
if (argc != 2) {
std::cerr << "usage: " << std::string(argv[0]) << " output-file" <<
std::endl;
return 1;
}
std::ofstream output;
output.open(argv[1]);
hwloc_topology_t topology;
hwloc_topology_init(&topology); // initialization
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology); // actual detection
hwloc_obj_t root = hwloc_get_root_obj(topology);
write_hpt_header(output);
write_hpt_tree(output, root, 1);
write_hpt_footer(output);
output.close();
hwloc_topology_destroy(topology);
return 0;
}
/*
*Call the function that prints the processing units hierarchy
*/
void hw_pu_topology (struct machine_output *ma_out)
{
console_output[0] = '\0';
strcpy(ma_out->console_output, "\n MACHINE| Processing units organization\n");
print_children_pu(topology, hwloc_get_root_obj(topology), 0);
strcat(ma_out->console_output, console_output);
strcat(ma_out->console_output, "\0");
ma_out->len = strlen(ma_out->console_output);
}
int
main (void)
{
hwloc_topology_t topology;
unsigned depth;
hwloc_obj_t last;
hwloc_obj_t *closest;
unsigned found;
int err;
unsigned numprocs;
hwloc_obj_t ancestor;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "2 3 4 5");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
depth = hwloc_topology_get_depth(topology);
/* get the last object of last level */
numprocs = hwloc_get_nbobjs_by_depth(topology, depth-1);
last = hwloc_get_obj_by_depth(topology, depth-1, numprocs-1);
/* allocate the array of closest objects */
closest = malloc(numprocs * sizeof(*closest));
assert(closest);
/* get closest levels */
found = hwloc_get_closest_objs (topology, last, closest, numprocs);
printf("looked for %u closest entries, found %u\n", numprocs, found);
assert(found == numprocs-1);
/* check first found is closest */
assert(closest[0] == hwloc_get_obj_by_depth(topology, depth-1, numprocs-5 /* arity is 5 on last level */));
/* check some other expected positions */
assert(closest[found-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5-1 /* last of first half */));
assert(closest[found/2-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5-1 /* last of second third of second half */));
assert(closest[found/2/3-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5+3*5-1 /* last of third quarter of third third of second half */));
/* get ancestor of last and less close object */
ancestor = hwloc_get_common_ancestor_obj(topology, last, closest[found-1]);
assert(hwloc_obj_is_in_subtree(topology, last, ancestor));
assert(hwloc_obj_is_in_subtree(topology, closest[found-1], ancestor));
assert(ancestor == hwloc_get_root_obj(topology)->first_child);
printf("ancestor type %u depth %u number %u is system level\n",
ancestor->type, ancestor->depth, ancestor->logical_index);
free(closest);
hwloc_topology_destroy (topology);
return EXIT_SUCCESS;
}
static inline
hwloc_obj_t getNumaObj(c_sublocid_t subloc) {
// could easily imagine this being a bit slow, but it's okay for now
return
hwloc_get_obj_inside_cpuset_by_depth(topology,
hwloc_get_root_obj(topology)->cpuset,
numaLevel,
subloc);
}
void output_synthetic(struct lstopo_output *loutput, const char *filename)
{
hwloc_topology_t topology = loutput->topology;
FILE *output;
int length;
char sbuffer[1024];
char * dbuffer = NULL;
unsigned nb1, nb2, nb3;
if (!hwloc_get_root_obj(topology)->symmetric_subtree) {
fprintf(stderr, "Cannot output assymetric topology in synthetic format.\n");
return;
}
nb1 = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_MISC);
if (nb1) {
fprintf(stderr, "Ignoring %u Misc objects.\n", nb1);
fprintf(stderr, "Passing --ignore Misc may remove them.\n");
}
nb1 = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_BRIDGE);
nb2 = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PCI_DEVICE);
nb3 = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
if (nb1 || nb2 || nb3) {
fprintf(stderr, "Ignoring %u Bridge, %u PCI device and %u OS device objects\n", nb1, nb2, nb3);
fprintf(stderr, "Passing --no-io may remove them.\n");
}
length = hwloc_topology_export_synthetic(topology, sbuffer, sizeof(sbuffer), lstopo_export_synthetic_flags);
if (length < 0)
return;
if (length >= sizeof(sbuffer)) {
dbuffer = malloc(length+1 /* \0 */);
if (!dbuffer)
return;
length = hwloc_topology_export_synthetic(topology, dbuffer, length+1, lstopo_export_synthetic_flags);
if (length < 0)
goto out;
}
output = open_output(filename, loutput->overwrite);
if (!output) {
fprintf(stderr, "Failed to open %s for writing (%s)\n", filename, strerror(errno));
goto out;
}
fprintf(output, "%s\n", dbuffer ? dbuffer : sbuffer);
if (output != stdout)
fclose(output);
out:
if (dbuffer)
free(dbuffer);
}
int opal_hwloc_print(char **output, char *prefix, hwloc_topology_t src, opal_data_type_t type)
{
hwloc_obj_t obj;
char *tmp=NULL;
/* get root object */
obj = hwloc_get_root_obj(src);
/* print it */
print_hwloc_obj(&tmp, prefix, src, obj);
*output = tmp;
return OPAL_SUCCESS;
}
pmix_status_t pmix_bfrop_print_topo(char **output, char *prefix,
hwloc_topology_t src, pmix_data_type_t type)
{
hwloc_obj_t obj;
char *tmp=NULL;
/* get root object */
obj = hwloc_get_root_obj(src);
/* print it */
print_hwloc_obj(&tmp, prefix, src, obj);
*output = tmp;
return PMIX_SUCCESS;
}
static void assert_foo_bar(hwloc_topology_t topo, int setornot)
{
hwloc_obj_t root = hwloc_get_root_obj(topo);
const char *found = hwloc_obj_get_info_by_name(root, "Foo");
if (!setornot) {
assert(!found);
} else {
int diff;
assert(found);
diff = strcmp(found, "Bar");
assert(!diff);
}
}
/*
*Call the recursive function print_children_physical with the HWLOC object
*/
void hw_phys_pu_topology(struct machine_output *ma_out)
{
hwloc_topology_t topo;
int error;
error = hwloc_topology_init(&topo);
if (!error){
hwloc_topology_load(topo);
console_output[0] = '\0';
strcpy(ma_out->console_output, "\n MACHINE| Physical processing units organization\n");
print_children_physical(topo, hwloc_get_root_obj(topo));
strcat(ma_out->console_output, console_output);
strcat(ma_out->console_output, "\0");
ma_out->len = strlen(ma_out->console_output);
}
}
/*
*Call the function that prints the memory hierarchy of the machine
*/
void hw_mem_topology (struct machine_output *ma_out)
{
hwloc_topology_t topo;
int error;
error = hwloc_topology_init(&topo);
if (!error){
hwloc_topology_load(topo);
console_output[0] = '\0';
strcpy(ma_out->console_output, "\n MACHINE| Physical memory subsystem organization\n");
print_children_mem(topo, hwloc_get_root_obj(topo), 0);
strcat(ma_out->console_output, console_output);
strcat(ma_out->console_output, "\0");
ma_out->len = strlen(ma_out->console_output);
}
}
int main(void) {
hwloc_topology_t topology;
hwloc_topology_init(&topology); // initialization
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology); // actual detection
hwloc_obj_t root = hwloc_get_root_obj(topology);
print_hwloc_obj(root, 0);
hwloc_topology_destroy(topology);
return 0;
}
/*
* Where is this process currently bound? (layout string)
*/
static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret;
hwloc_obj_t root;
hwloc_cpuset_t boundset, rootset;
bool bound = false;
/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* get our bindings */
boundset = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, boundset,
HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound
- the environment does not support it */
bound = false;
} else {
/* we are bound if the two cpusets are not equal, or if there
is only ONE PU available to us */
if (0 != hwloc_bitmap_compare(boundset, rootset) ||
opal_hwloc_base_single_cpu(rootset) ||
opal_hwloc_base_single_cpu(boundset)) {
bound = true;
}
}
/* If we are not bound, indicate that */
if (!bound) {
strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
ret = OMPI_SUCCESS;
}
/* If we are bound, print it out */
else {
ret = opal_hwloc_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX,
opal_hwloc_topology,
boundset);
if (OPAL_ERR_NOT_BOUND == ret) {
strncpy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
ret = OMPI_SUCCESS;
}
}
hwloc_bitmap_free(boundset);
return ret;
}
/*
*Call the function that prints the processing units hierarchy
*/
void hw_machine_topology (struct machine_output *ma_out)
{
hwloc_topology_t topo;
int error;
error = hwloc_topology_init(&topo);
hwloc_topology_set_flags(topo, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
if (!error){
hwloc_topology_load(topo);
console_output[0] = '\0';
strcpy(ma_out->console_output, "\n MACHINE| Architecture organization\n\n");
print_machine(topo, hwloc_get_root_obj(topo), 0);
strcat(ma_out->console_output, console_output);
strcat(ma_out->console_output, "\0");
ma_out->len = strlen(ma_out->console_output);
}
}
static int rmaps_lama_convert_hwloc_tree_to_opal_tree(opal_tree_t *opal_tree, hwloc_topology_t *hwloc_topo)
{
hwloc_obj_t topo_root;
if( 15 <= opal_output_get_verbosity(orte_rmaps_base_framework.framework_output) ) {
opal_output_verbose(15, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Converting Topology:");
/* opal_dss.dump(0, opal_hwloc_topology, OPAL_HWLOC_TOPO); */
opal_dss.dump(0, *hwloc_topo, OPAL_HWLOC_TOPO);
}
topo_root = hwloc_get_root_obj(*hwloc_topo);
rmaps_lama_convert_hwloc_subtree(topo_root,
opal_tree_get_root(opal_tree));
return ORTE_SUCCESS;
}
int main(void)
{
hwloc_topology_t topology;
char *string = NULL;
hwloc_obj_t obj;
hwloc_bitmap_t set;
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, SYNTHETIC_TOPOLOGY_DESCRIPTION);
hwloc_topology_load(topology);
set = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(set, GIVEN_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(obj);
fprintf(stderr, "found covering object type %s covering cpuset %s\n",
hwloc_obj_type_string(obj->type), GIVEN_CPUSET_STRING);
assert(hwloc_bitmap_isincluded(set, obj->cpuset));
hwloc_bitmap_asprintf(&string, obj->cpuset);
fprintf(stderr, "covering object of %s is %s, expected %s\n",
GIVEN_CPUSET_STRING, string, EXPECTED_CPUSET_STRING);
assert(!strcmp(EXPECTED_CPUSET_STRING, string));
free(string);
hwloc_bitmap_sscanf(set, GIVEN_LARGESPLIT_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(obj == hwloc_get_root_obj(topology));
fprintf(stderr, "found system as covering object of first+last cpus cpuset %s\n",
GIVEN_LARGESPLIT_CPUSET_STRING);
hwloc_bitmap_sscanf(set, GIVEN_TOOLARGE_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(!obj);
fprintf(stderr, "found no covering object for too-large cpuset %s\n",
GIVEN_TOOLARGE_CPUSET_STRING);
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
void output_synthetic(hwloc_topology_t topology, const char *filename, int overwrite, int logical __hwloc_attribute_unused, int legend __hwloc_attribute_unused, int verbose_mode __hwloc_attribute_unused)
{
FILE *output;
int length;
char sbuffer[1024];
char * dbuffer = NULL;
if (!hwloc_get_root_obj(topology)->symmetric_subtree) {
fprintf(stderr, "Cannot output assymetric topology in synthetic format.\n");
fprintf(stderr, "Adding --no-io may help making the topology symmetric.\n");
return;
}
length = hwloc_topology_export_synthetic(topology, sbuffer, sizeof(sbuffer), lstopo_export_synthetic_flags);
if (length < 0)
return;
if (length >= sizeof(sbuffer)) {
dbuffer = malloc(length+1 /* \0 */);
if (!dbuffer)
return;
length = hwloc_topology_export_synthetic(topology, dbuffer, length+1, lstopo_export_synthetic_flags);
if (length < 0)
goto out;
}
output = open_output(filename, overwrite);
if (!output) {
fprintf(stderr, "Failed to open %s for writing (%s)\n", filename, strerror(errno));
goto out;
}
fprintf(output, "%s\n", dbuffer ? dbuffer : sbuffer);
if (output != stdout)
fclose(output);
out:
if (dbuffer)
free(dbuffer);
}
void LpelHwLocInit(lpel_config_t *cfg)
{
#ifdef HAVE_HWLOC
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
pu_count = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
cpu_sets = malloc(pu_count * sizeof(hwloc_cpuset_t));
hw_places = malloc(pu_count * sizeof(lpel_hw_place_t));
traverse(hwloc_get_root_obj(topology));
#elif defined(HAVE_SYSCONF)
/* query the number of CPUs */
pu_count = sysconf(_SC_NPROCESSORS_ONLN);
#else
int res = LpelGetNumCores( &pu_count);
if(res == 0)
pu_count = res;
#endif
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t root;
const char *cluster_mode;
const char *memory_mode;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
root = hwloc_get_root_obj(topology);
cluster_mode = hwloc_obj_get_info_by_name(root, "ClusterMode");
memory_mode = hwloc_obj_get_info_by_name(root, "MemoryMode");
printf ("ClusterMode is '%s' MemoryMode is '%s'\n",
cluster_mode ? cluster_mode : "NULL",
memory_mode ? memory_mode : "NULL");
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t obj;
unsigned indexes[5];
float distances[5*5];
unsigned depth;
unsigned width;
/* group 3 numa nodes as 1 group of 2 and 1 on the side */
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, "node:3 pu:1");
indexes[0] = 0; indexes[1] = 1; indexes[2] = 2;
distances[0] = 1; distances[1] = 4; distances[2] = 4;
distances[3] = 4; distances[4] = 1; distances[5] = 2;
distances[6] = 4; distances[7] = 2; distances[8] = 1;
hwloc_topology_set_distance_matrix(topology, HWLOC_OBJ_PU, 3, indexes, distances);
hwloc_topology_load(topology);
/* 2 groups at depth 1 */
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_GROUP);
assert(depth == 1);
width = hwloc_get_nbobjs_by_depth(topology, depth);
assert(width == 1);
/* 3 node at depth 2 */
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
assert(depth == 2);
width = hwloc_get_nbobjs_by_depth(topology, depth);
assert(width == 3);
/* find the root obj */
obj = hwloc_get_root_obj(topology);
assert(obj->arity == 2);
/* check its children */
assert(obj->children[0]->type == HWLOC_OBJ_NUMANODE);
assert(obj->children[0]->depth == 2);
assert(obj->children[0]->arity == 1);
assert(obj->children[1]->type == HWLOC_OBJ_GROUP);
assert(obj->children[1]->depth == 1);
assert(obj->children[1]->arity == 2);
hwloc_topology_destroy(topology);
/* group 5 packages as 2 group of 2 and 1 on the side, all of them below a common node object */
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, "node:1 pack:5 pu:1");
indexes[0] = 0; indexes[1] = 1; indexes[2] = 2; indexes[3] = 3; indexes[4] = 4;
distances[ 0] = 1; distances[ 1] = 2; distances[ 2] = 4; distances[ 3] = 4; distances[ 4] = 4;
distances[ 5] = 2; distances[ 6] = 1; distances[ 7] = 4; distances[ 8] = 4; distances[ 9] = 4;
distances[10] = 4; distances[11] = 4; distances[12] = 1; distances[13] = 4; distances[14] = 4;
distances[15] = 4; distances[16] = 4; distances[17] = 4; distances[18] = 1; distances[19] = 2;
distances[20] = 4; distances[21] = 4; distances[22] = 4; distances[23] = 2; distances[24] = 1;
hwloc_topology_set_distance_matrix(topology, HWLOC_OBJ_PACKAGE, 5, indexes, distances);
hwloc_topology_load(topology);
/* 1 node at depth 1 */
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
assert(depth == 1);
width = hwloc_get_nbobjs_by_depth(topology, depth);
assert(width == 1);
/* 2 groups at depth 2 */
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_GROUP);
assert(depth == 2);
width = hwloc_get_nbobjs_by_depth(topology, depth);
assert(width == 2);
/* 5 packages at depth 3 */
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
assert(depth == 3);
width = hwloc_get_nbobjs_by_depth(topology, depth);
assert(width == 5);
/* find the node obj */
obj = hwloc_get_root_obj(topology);
assert(obj->arity == 1);
obj = obj->children[0];
assert(obj->type == HWLOC_OBJ_NUMANODE);
assert(obj->arity == 3);
/* check its children */
assert(obj->children[0]->type == HWLOC_OBJ_GROUP);
assert(obj->children[0]->depth == 2);
assert(obj->children[0]->arity == 2);
assert(obj->children[1]->type == HWLOC_OBJ_PACKAGE);
assert(obj->children[1]->depth == 3);
assert(obj->children[1]->arity == 1);
assert(obj->children[2]->type == HWLOC_OBJ_GROUP);
assert(obj->children[2]->depth == 2);
assert(obj->children[2]->arity == 2);
hwloc_topology_destroy(topology);
return 0;
}
static int
hwloc_cuda_discover(struct hwloc_backend *backend)
{
struct hwloc_topology *topology = backend->topology;
enum hwloc_type_filter_e filter;
cudaError_t cures;
int nb, i;
hwloc_topology_get_type_filter(topology, HWLOC_OBJ_OS_DEVICE, &filter);
if (filter == HWLOC_TYPE_FILTER_KEEP_NONE)
return 0;
cures = cudaGetDeviceCount(&nb);
if (cures)
return -1;
for (i = 0; i < nb; i++) {
int domain, bus, dev;
char cuda_name[32];
char number[32];
struct cudaDeviceProp prop;
hwloc_obj_t cuda_device, parent;
unsigned cores;
cuda_device = hwloc_alloc_setup_object(HWLOC_OBJ_OS_DEVICE, -1);
snprintf(cuda_name, sizeof(cuda_name), "cuda%d", i);
cuda_device->name = strdup(cuda_name);
cuda_device->depth = (unsigned) HWLOC_TYPE_DEPTH_UNKNOWN;
cuda_device->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
cuda_device->subtype = strdup("CUDA");
hwloc_obj_add_info(cuda_device, "Backend", "CUDA");
hwloc_obj_add_info(cuda_device, "GPUVendor", "NVIDIA Corporation");
cures = cudaGetDeviceProperties(&prop, i);
if (!cures)
hwloc_obj_add_info(cuda_device, "GPUModel", prop.name);
snprintf(number, sizeof(number), "%llu", ((unsigned long long) prop.totalGlobalMem) >> 10);
hwloc_obj_add_info(cuda_device, "CUDAGlobalMemorySize", number);
snprintf(number, sizeof(number), "%llu", ((unsigned long long) prop.l2CacheSize) >> 10);
hwloc_obj_add_info(cuda_device, "CUDAL2CacheSize", number);
snprintf(number, sizeof(number), "%d", prop.multiProcessorCount);
hwloc_obj_add_info(cuda_device, "CUDAMultiProcessors", number);
cores = hwloc_cuda_cores_per_MP(prop.major, prop.minor);
if (cores) {
snprintf(number, sizeof(number), "%u", cores);
hwloc_obj_add_info(cuda_device, "CUDACoresPerMP", number);
}
snprintf(number, sizeof(number), "%llu", ((unsigned long long) prop.sharedMemPerBlock) >> 10);
hwloc_obj_add_info(cuda_device, "CUDASharedMemorySizePerMP", number);
parent = NULL;
if (hwloc_cudart_get_device_pci_ids(NULL /* topology unused */, i, &domain, &bus, &dev) == 0) {
parent = hwloc_pci_belowroot_find_by_busid(topology, domain, bus, dev, 0);
if (!parent)
parent = hwloc_pci_find_busid_parent(topology, domain, bus, dev, 0);
}
if (!parent)
parent = hwloc_get_root_obj(topology);
hwloc_insert_object_by_parent(topology, parent, cuda_device);
}
return 0;
}
static void add_process_objects(hwloc_topology_t topology)
{
#ifdef HAVE_DIRENT_H
hwloc_obj_t root;
hwloc_bitmap_t cpuset;
#ifdef HWLOC_LINUX_SYS
hwloc_bitmap_t task_cpuset;
#endif /* HWLOC_LINUX_SYS */
DIR *dir;
struct dirent *dirent;
const struct hwloc_topology_support *support;
root = hwloc_get_root_obj(topology);
support = hwloc_topology_get_support(topology);
if (!support->cpubind->get_proc_cpubind)
return;
dir = opendir("/proc");
if (!dir)
return;
cpuset = hwloc_bitmap_alloc();
#ifdef HWLOC_LINUX_SYS
task_cpuset = hwloc_bitmap_alloc();
#endif /* HWLOC_LINUX_SYS */
while ((dirent = readdir(dir))) {
long local_pid_number;
hwloc_pid_t local_pid;
char *end;
char name[80];
int proc_cpubind;
local_pid_number = strtol(dirent->d_name, &end, 10);
if (*end)
/* Not a number */
continue;
snprintf(name, sizeof(name), "%ld", local_pid_number);
local_pid = hwloc_pid_from_number(local_pid_number, 0);
proc_cpubind = hwloc_get_proc_cpubind(topology, local_pid, cpuset, 0) != -1;
#ifdef HWLOC_LINUX_SYS
{
char comm[16];
char *path;
size_t pathlen = 6 + strlen(dirent->d_name) + 1 + 7 + 1;
path = malloc(pathlen);
{
/* Get the process name */
char cmd[64];
int file;
ssize_t n;
snprintf(path, pathlen, "/proc/%s/cmdline", dirent->d_name);
file = open(path, O_RDONLY);
if (file < 0) {
/* Ignore errors */
free(path);
continue;
}
n = read(file, cmd, sizeof(cmd));
close(file);
if (n <= 0) {
/* Ignore kernel threads and errors */
free(path);
continue;
}
snprintf(path, pathlen, "/proc/%s/comm", dirent->d_name);
file = open(path, O_RDONLY);
if (file >= 0) {
n = read(file, comm, sizeof(comm) - 1);
close(file);
if (n > 0) {
comm[n] = 0;
if (n > 1 && comm[n-1] == '\n')
comm[n-1] = 0;
} else {
snprintf(comm, sizeof(comm), "(unknown)");
}
} else {
/* Old kernel, have to look at old file */
char stats[32];
char *parenl = NULL, *parenr;
snprintf(path, pathlen, "/proc/%s/stat", dirent->d_name);
file = open(path, O_RDONLY);
if (file < 0) {
/* Ignore errors */
free(path);
continue;
}
/* "pid (comm) ..." */
n = read(file, stats, sizeof(stats) - 1);
close(file);
if (n > 0) {
stats[n] = 0;
parenl = strchr(stats, '(');
parenr = strchr(stats, ')');
if (!parenr)
parenr = &stats[sizeof(stats)-1];
*parenr = 0;
}
if (!parenl) {
snprintf(comm, sizeof(comm), "(unknown)");
} else {
snprintf(comm, sizeof(comm), "%s", parenl+1);
}
}
snprintf(name, sizeof(name), "%ld %s", local_pid_number, comm);
}
{
/* Get threads */
DIR *task_dir;
struct dirent *task_dirent;
snprintf(path, pathlen, "/proc/%s/task", dirent->d_name);
task_dir = opendir(path);
if (task_dir) {
while ((task_dirent = readdir(task_dir))) {
long local_tid;
char *task_end;
const size_t tid_len = sizeof(local_tid)*3+1;
size_t task_pathlen = 6 + strlen(dirent->d_name) + 1 + 4 + 1
+ strlen(task_dirent->d_name) + 1 + 4 + 1;
char *task_path;
int comm_file;
char task_comm[16] = "";
char task_name[sizeof(name) + 1 + tid_len + 1 + sizeof(task_comm) + 1];
ssize_t n;
local_tid = strtol(task_dirent->d_name, &task_end, 10);
if (*task_end)
/* Not a number, or the main task */
continue;
task_path = malloc(task_pathlen);
snprintf(task_path, task_pathlen, "/proc/%s/task/%s/comm",
dirent->d_name, task_dirent->d_name);
comm_file = open(task_path, O_RDONLY);
free(task_path);
if (comm_file >= 0) {
n = read(comm_file, task_comm, sizeof(task_comm) - 1);
if (n < 0)
n = 0;
close(comm_file);
task_comm[n] = 0;
if (n > 1 && task_comm[n-1] == '\n')
task_comm[n-1] = 0;
if (!strcmp(comm, task_comm))
/* Same as process comm, do not show it again */
n = 0;
} else {
n = 0;
}
if (hwloc_linux_get_tid_cpubind(topology, local_tid, task_cpuset))
continue;
if (proc_cpubind && hwloc_bitmap_isequal(task_cpuset, cpuset))
continue;
if (n) {
snprintf(task_name, sizeof(task_name), "%s %li %s", name, local_tid, task_comm);
} else {
snprintf(task_name, sizeof(task_name), "%s %li", name, local_tid);
}
insert_task(topology, task_cpuset, task_name);
}
closedir(task_dir);
}
}
free(path);
}
#endif /* HWLOC_LINUX_SYS */
if (!proc_cpubind)
continue;
if (hwloc_bitmap_isincluded(root->cpuset, cpuset))
continue;
insert_task(topology, cpuset, name);
}
hwloc_bitmap_free(cpuset);
#ifdef HWLOC_LINUX_SYS
hwloc_bitmap_free(task_cpuset);
#endif /* HWLOC_LINUX_SYS */
closedir(dir);
#endif /* HAVE_DIRENT_H */
}
static int setup_fork(orte_job_t *jdata,
orte_app_context_t *app)
{
int i;
char *param;
bool oversubscribed;
orte_node_t *node;
char **envcpy, **nps, **firstranks;
char *npstring, *firstrankstring;
char *num_app_ctx;
bool takeus = false;
/* see if we are included */
for (i=0; NULL != jdata->personality[i]; i++) {
if (0 == strcmp(jdata->personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* see if the mapper thinks we are oversubscribed */
oversubscribed = false;
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_OVERSUBSCRIBED)) {
oversubscribed = true;
}
/* setup base environment: copy the current environ and merge
in the app context environ */
if (NULL != app->env) {
/* manually free original context->env to avoid a memory leak */
char **tmp = app->env;
envcpy = opal_environ_merge(orte_launch_environ, app->env);
if (NULL != tmp) {
opal_argv_free(tmp);
}
} else {
envcpy = opal_argv_copy(orte_launch_environ);
}
app->env = envcpy;
/* special case handling for --prefix: this is somewhat icky,
but at least some users do this. :-\ It is possible that
when using --prefix, the user will also "-x PATH" and/or
"-x LD_LIBRARY_PATH", which would therefore clobber the
work that was done in the prior pls to ensure that we have
the prefix at the beginning of the PATH and
LD_LIBRARY_PATH. So examine the context->env and see if we
find PATH or LD_LIBRARY_PATH. If found, that means the
prior work was clobbered, and we need to re-prefix those
variables. */
param = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)¶m, OPAL_STRING);
for (i = 0; NULL != param && NULL != app->env && NULL != app->env[i]; ++i) {
char *newenv;
/* Reset PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
asprintf(&newenv, "%s/bin:%s", param, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
}
/* Reset LD_LIBRARY_PATH */
else if (0 == strncmp("LD_LIBRARY_PATH=", app->env[i], 16)) {
asprintf(&newenv, "%s/lib:%s", param, app->env[i] + 16);
opal_setenv("LD_LIBRARY_PATH", newenv, true, &app->env);
free(newenv);
}
}
if (NULL != param) {
free(param);
}
/* pass my contact info to the local proc so we can talk */
opal_setenv("OMPI_MCA_orte_local_daemon_uri", orte_process_info.my_daemon_uri, true, &app->env);
/* pass the hnp's contact info to the local proc in case it
* needs it
*/
if (NULL != orte_process_info.my_hnp_uri) {
opal_setenv("OMPI_MCA_orte_hnp_uri", orte_process_info.my_hnp_uri, true, &app->env);
}
/* setup yield schedule - do not override any user-supplied directive! */
if (oversubscribed) {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "1", false, &app->env);
} else {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "0", false, &app->env);
}
/* set the app_context number into the environment */
asprintf(¶m, "%ld", (long)app->idx);
opal_setenv("OMPI_MCA_orte_app_num", param, true, &app->env);
free(param);
/* although the total_slots_alloc is the universe size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here. Also required by the ompi_attributes code!
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->total_slots_alloc);
opal_setenv("OMPI_UNIVERSE_SIZE", param, true, &app->env);
free(param);
/* pass the number of nodes involved in this job */
asprintf(¶m, "%ld", (long)(jdata->map->num_nodes));
opal_setenv("OMPI_MCA_orte_num_nodes", param, true, &app->env);
free(param);
/* pass a param telling the child what type and model of cpu we are on,
* if we know it. If hwloc has the value, use what it knows. Otherwise,
* see if we were explicitly given it and use that value.
*/
hwloc_obj_t obj;
char *htmp;
if (NULL != opal_hwloc_topology) {
obj = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUType")) ||
NULL != (htmp = orte_local_cpu_type)) {
opal_setenv("OMPI_MCA_orte_cpu_type", htmp, true, &app->env);
}
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUModel")) ||
NULL != (htmp = orte_local_cpu_model)) {
opal_setenv("OMPI_MCA_orte_cpu_model", htmp, true, &app->env);
}
} else {
if (NULL != orte_local_cpu_type) {
opal_setenv("OMPI_MCA_orte_cpu_type", orte_local_cpu_type, true, &app->env);
}
if (NULL != orte_local_cpu_model) {
opal_setenv("OMPI_MCA_orte_cpu_model", orte_local_cpu_model, true, &app->env);
}
}
/* get shmem's best component name so we can provide a hint to the shmem
* framework. the idea here is to have someone figure out what component to
* select (via the shmem framework) and then have the rest of the
* components in shmem obey that decision. for more details take a look at
* the shmem framework in opal.
*/
if (NULL != (param = opal_shmem_base_best_runnable_component_name())) {
opal_setenv("OMPI_MCA_shmem_RUNTIME_QUERY_hint", param, true, &app->env);
free(param);
}
/* Set an info MCA param that tells the launched processes that
* any binding policy was applied by us (e.g., so that
* MPI_INIT doesn't try to bind itself)
*/
opal_setenv("OMPI_MCA_orte_bound_at_launch", "1", true, &app->env);
/* tell the ESS to avoid the singleton component - but don't override
* anything that may have been provided elsewhere
*/
opal_setenv("OMPI_MCA_ess", "^singleton", false, &app->env);
/* ensure that the spawned process ignores direct launch components,
* but do not overrride anything we were given */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", false, &app->env);
/* since we want to pass the name as separate components, make sure
* that the "name" environmental variable is cleared!
*/
opal_unsetenv("OMPI_MCA_orte_ess_name", &app->env);
asprintf(¶m, "%ld", (long)jdata->num_procs);
opal_setenv("OMPI_MCA_orte_ess_num_procs", param, true, &app->env);
/* although the num_procs is the comm_world size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
opal_setenv("OMPI_COMM_WORLD_SIZE", param, true, &app->env);
free(param);
/* users would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->num_local_procs);
opal_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, &app->env);
free(param);
/* forcibly set the local tmpdir base to match ours */
opal_setenv("OMPI_MCA_orte_tmpdir_base", orte_process_info.tmpdir_base, true, &app->env);
/* MPI-3 requires we provide some further info to the procs,
* so we pass them as envars to avoid introducing further
* ORTE calls in the MPI layer
*/
asprintf(&num_app_ctx, "%lu", (unsigned long)jdata->num_apps);
/* build some common envars we need to pass for MPI-3 compatibility */
nps = NULL;
firstranks = NULL;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
opal_argv_append_nosize(&nps, ORTE_VPID_PRINT(app->num_procs));
opal_argv_append_nosize(&firstranks, ORTE_VPID_PRINT(app->first_rank));
}
npstring = opal_argv_join(nps, ' ');
firstrankstring = opal_argv_join(firstranks, ' ');
opal_argv_free(nps);
opal_argv_free(firstranks);
/* add the MPI-3 envars */
opal_setenv("OMPI_NUM_APP_CTX", num_app_ctx, true, &app->env);
opal_setenv("OMPI_FIRST_RANKS", firstrankstring, true, &app->env);
opal_setenv("OMPI_APP_CTX_NUM_PROCS", npstring, true, &app->env);
free(num_app_ctx);
free(firstrankstring);
free(npstring);
return ORTE_SUCCESS;
}
