int opal_hwloc_copy(hwloc_topology_t *dest, hwloc_topology_t src, opal_data_type_t type)
{
char *xml;
int len;
struct hwloc_topology_support *support, *destsupport;
if (0 != hwloc_topology_export_xmlbuffer(src, &xml, &len)) {
return OPAL_ERROR;
}
if (0 != hwloc_topology_init(dest)) {
free(xml);
return OPAL_ERROR;
}
if (0 != hwloc_topology_set_xmlbuffer(*dest, xml, len)) {
hwloc_topology_destroy(*dest);
free(xml);
return OPAL_ERROR;
}
if (0 != hwloc_topology_load(*dest)) {
hwloc_topology_destroy(*dest);
free(xml);
return OPAL_ERROR;
}
free(xml);
/* get the available support - hwloc unfortunately does
* not include this info in its xml support!
*/
support = (struct hwloc_topology_support*)hwloc_topology_get_support(src);
destsupport = (struct hwloc_topology_support*)hwloc_topology_get_support(*dest);
*destsupport = *support;
return OPAL_SUCCESS;
}
static int _get_cpuinfo(uint32_t *nsockets, uint32_t *ncores,
uint32_t *nthreads, uint32_t *npus)
{
hwloc_topology_t topology;
if (hwloc_topology_init(&topology)) {
/* error in initialize hwloc library */
error("%s: hwloc_topology_init() failed", __func__);
return -1;
}
/* parse full system info */
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM);
/* ignores cache, misc */
hwloc_topology_ignore_type (topology, HWLOC_OBJ_CACHE);
hwloc_topology_ignore_type (topology, HWLOC_OBJ_MISC);
/* load topology */
if (hwloc_topology_load(topology)) {
error("%s: hwloc_topology_load() failed", __func__);
hwloc_topology_destroy(topology);
return -1;
}
*nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_SOCKET);
*ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
*nthreads = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
*npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
static hwloc_topology_t oldtopology, topology;
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
unsigned node_indexes[3], core_indexes[6];
float node_distances[9], core_distances[36];
unsigned i,j;
int err;
for(i=0; i<3; i++) {
node_indexes[i] = i;
for(j=0; j<3; j++)
node_distances[i*3+j] = (i == j ? 10.f : 20.f);
}
for(i=0; i<6; i++) {
core_indexes[i] = i;
for(j=0; j<6; j++)
core_distances[i*6+j] = (i == j ? 4.f : 8.f);
}
hwloc_topology_init(&oldtopology);
printf("building fake 'node:3 core:2 pu:4' topology\n");
hwloc_topology_set_synthetic(oldtopology, "node:3 core:2 pu:4");
printf("adding node and core matrices\n");
hwloc_topology_set_distance_matrix(oldtopology, HWLOC_OBJ_NUMANODE, 3, node_indexes, node_distances);
hwloc_topology_set_distance_matrix(oldtopology, HWLOC_OBJ_CORE, 6, core_indexes, core_distances);
hwloc_topology_load(oldtopology);
printf("duplicating\n");
hwloc_topology_dup(&topology, oldtopology);
printf("destroying the old topology\n");
hwloc_topology_destroy(oldtopology);
/* remove the entire third node */
printf("removing one node\n");
hwloc_bitmap_fill(cpuset);
hwloc_bitmap_clr_range(cpuset, 16, 23);
err = hwloc_topology_restrict(topology, cpuset, HWLOC_RESTRICT_FLAG_ADAPT_DISTANCES);
assert(!err);
printf("checking the result\n");
assert(hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE) == 2);
hwloc_topology_destroy(topology);
hwloc_bitmap_free(cpuset);
return 0;
}
void Hwloc::unloadHwloc ()
{
#ifdef HWLOC
/* Destroy topology object. */
hwloc_topology_destroy( _hwlocTopology );
#endif
}
/*
* Get the number of net cards in a NUMA node
*/
int hw_get_nnetcards(int NUMAnode)
{
int count, net_NUMAnode, error;
hwloc_obj_t obj, obj_anc;
count = 0;
hwloc_topology_t topo_net;
error = hwloc_topology_init(&topo_net);
hwloc_topology_set_flags(topo_net, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
if (!error){
hwloc_topology_load(topo_net);
for (obj = hwloc_get_obj_by_type(topo_net, HWLOC_OBJ_OS_DEVICE, 0);
obj;
obj = hwloc_get_next_osdev(topo_net,obj))
if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK ||
obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS){
obj_anc = hwloc_get_non_io_ancestor_obj(topo_net,obj);
net_NUMAnode = hwloc_bitmap_first(obj_anc->nodeset);
//only if the MPI NUMA node is equal to the found net card
if(NUMAnode == net_NUMAnode)
count++;
}
}
hwloc_topology_destroy(topo_net);
return count;
}
int main(void)
{
hwloc_topology_t topology;
int i;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
for(i=0; ; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, ancestor;
const char *value;
osdev = hwloc_intel_mic_get_device_osdev_by_index(topology, i);
if (!osdev)
break;
assert(osdev);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "mic", 3);
assert(!err);
assert(atoi(osdev->name+3) == (int) i);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "MIC");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "MICFamily");
printf("found MICFamily %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICSKU");
printf("found MICSKU %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICActiveCores");
printf("found MICActiveCores %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICMemorySize");
printf("found MICMemorySize %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_intel_mic_get_device_cpuset(topology, i, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
int main(int argc, char **argv) {
if (argc != 2) {
std::cerr << "usage: " << std::string(argv[0]) << " output-file" <<
std::endl;
return 1;
}
std::ofstream output;
output.open(argv[1]);
hwloc_topology_t topology;
hwloc_topology_init(&topology); // initialization
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology); // actual detection
hwloc_obj_t root = hwloc_get_root_obj(topology);
write_hpt_header(output);
write_hpt_tree(output, root, 1);
write_hpt_footer(output);
output.close();
hwloc_topology_destroy(topology);
return 0;
}
void chpl_topo_exit(void) {
if (!haveTopology) {
return;
}
hwloc_topology_destroy(topology);
}
int rsreader_hwloc_load (resrc_api_ctx_t *rsapi, const char *buf, size_t len,
uint32_t rank, rsreader_t r_mode, machs_t *machs, char **err_str)
{
int rc = -1;
rssig_t *sig = NULL;
hwloc_topology_t topo;
if (!machs)
goto done;
if (hwloc_topology_init (&topo) != 0)
goto done;
if (hwloc_topology_set_xmlbuffer (topo, buf, len) != 0)
goto err;
if (hwloc_topology_load (topo) != 0)
goto err;
if (rs2rank_set_signature ((char*)buf, len, topo, &sig) != 0)
goto err;
if (rs2rank_tab_update (machs, get_hn (topo), sig, rank) != 0)
goto err;
if (r_mode == RSREADER_HWLOC) {
const char *s = rs2rank_get_digest (sig);
if (!resrc_generate_hwloc_resources (rsapi, topo, s, err_str))
goto err;
}
rc = 0;
err:
hwloc_topology_destroy (topo);
done:
return rc;
}
void terminate_thread_pool(){
int id;
int *ret=NULL;
work_t work;
if(pool){
work.task=NULL;
for (id=0;id<pool->nb_threads;id++){
submit_work(&work,id);
}
for (id=0;id<pool->nb_threads;id++){
pthread_join(pool->thread_list[id],(void **) &ret);
FREE(ret);
pthread_cond_destroy(pool->cond_var +id);
pthread_mutex_destroy(pool->list_lock +id);
if (pool->working_list[id].next != NULL)
if(verbose_level >= WARNING)
printf("Working list of thread %d not empty!\n",id);
}
hwloc_topology_destroy(pool->topology);
FREE(pool -> thread_list);
FREE(pool -> working_list);
FREE(pool -> cond_var);
FREE(pool -> list_lock);
FREE(pool -> local);
FREE(pool);
pool = NULL;
}
}
/* Wrapper routines for hwloc */
void qrm_hwloc_bind(int id)
{
int depth, ret;
unsigned i, n;
int topodepth;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_CORE, id);
ret = hwloc_set_cpubind(topology, obj->cpuset, HWLOC_CPUBIND_THREAD);
if (ret) {
printf("Couldn't bind to core %d\n", id);
assert(0);
} else {
printf("Bound to core %d\n", id);
}
hwloc_topology_destroy(topology);
return;
}
JNIEXPORT void JNICALL Java_com_rr_core_os_NativeHooksImpl_jniSetPriority( JNIEnv *env, jclass clazz, jint cpumask, jint priority ) {
int topodepth;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
topodepth = hwloc_topology_get_depth(topology);
cpuset = hwloc_bitmap_alloc();
hwloc_bitmap_from_ulong( cpuset, (unsigned int)cpumask );
char *str;
hwloc_bitmap_asprintf(&str, cpuset);
printf("cpumask [%d] => hwloc [%s]\n", cpumask, str);
if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD)) {
printf("Couldn't bind cpuset %s\n", str);
} else {
printf("BOUND cpuset %s\n", str);
}
free(str);
/* Free our cpuset copy */
hwloc_bitmap_free(cpuset);
/* Destroy topology object. */
hwloc_topology_destroy(topology);
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t cpuset;
int err;
/* check the OS topology */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
assert(hwloc_topology_is_thissystem(topology));
cpuset = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset(topology));
result("Binding with OS backend", hwloc_set_cpubind(topology, cpuset, 0));
hwloc_topology_destroy(topology);
/* We're assume there is a real processor numbered 0 */
hwloc_bitmap_zero(cpuset);
hwloc_bitmap_set(cpuset, 0);
/* check a synthetic topology */
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, "1");
hwloc_topology_load(topology);
assert(!hwloc_topology_is_thissystem(topology));
err = hwloc_set_cpubind(topology, cpuset, 0);
result("Binding with synthetic backend", err);
assert(!err);
hwloc_topology_destroy(topology);
/* check a synthetic topology but assuming it's the system topology */
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
hwloc_topology_set_synthetic(topology, "1");
hwloc_topology_load(topology);
assert(hwloc_topology_is_thissystem(topology));
result("Binding with synthetic backend faking is_thissystem", hwloc_set_cpubind(topology, cpuset, 0));
hwloc_topology_destroy(topology);
hwloc_bitmap_free(cpuset);
return 0;
}
static int adopt(int fd, unsigned long fileoffset, unsigned long mmap_address, unsigned long mmap_length, int synthetic_with_distances)
{
static hwloc_topology_t adopted;
char *xmlbuf;
int xmlbuflen;
char *origxmlbuf;
struct hwloc_distances_s *distances;
unsigned nr = 1;
int err;
int ret = EXIT_SKIP;
err = lseek(fd, 0, SEEK_SET);
assert(!err);
printf(" reading XML dump\n");
origxmlbuf = malloc(fileoffset);
assert(origxmlbuf);
err = read(fd, origxmlbuf, fileoffset);
assert(err > 0);
printf(" adopting from file at offset %lu with addr %lx len %lu\n", fileoffset, mmap_address, mmap_length);
err = hwloc_shmem_topology_adopt(&adopted, fd, fileoffset, (void*)(uintptr_t)mmap_address, mmap_length, 0);
if (err == -1 && errno == EBUSY) {
/* may fail on 32bits and on some OS (e.g. darwin from time to time), and even on Linux/64bits if unlucky */
fprintf(stderr, "Failed to shmem adopt, requested mapping is busy\n");
goto out_with_origxmlbuf;
}
assert(!err);
printf(" adopted OK\n");
err = hwloc_distances_get_by_type(adopted, HWLOC_OBJ_NUMANODE, &nr, &distances, 0, 0);
assert(!err);
if (synthetic_with_distances) {
assert(nr == 1);
assert(distances->nbobjs == 3);
assert(distances->kind == (HWLOC_DISTANCES_KIND_MEANS_LATENCY|HWLOC_DISTANCES_KIND_FROM_USER));
hwloc_distances_release(adopted, distances);
printf(" distances OK\n");
}
err = hwloc_topology_export_xmlbuffer(adopted, &xmlbuf, &xmlbuflen, 0);
assert(!err);
printf(" XML export %d bytes\n", xmlbuflen);
assert((unsigned long) xmlbuflen < fileoffset);
assert(!memcmp(origxmlbuf, xmlbuf, xmlbuflen));
hwloc_free_xmlbuffer(adopted, xmlbuf);
printf(" XML export is identical to original\n");
hwloc_topology_destroy(adopted);
printf(" destroyed\n");
ret = EXIT_SUCCESS;
out_with_origxmlbuf:
free(origxmlbuf);
return ret;
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t obj;
hwloc_topology_init(&topology);
hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_ALL);
hwloc_topology_load(topology);
printf("Found %d bridges\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_BRIDGE));
obj = NULL;
while ((obj = hwloc_get_next_bridge(topology, obj)) != NULL) {
assert(obj->type == HWLOC_OBJ_BRIDGE);
/* only host->pci and pci->pci bridge supported so far */
if (obj->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_HOST) {
assert(obj->attr->bridge.downstream_type == HWLOC_OBJ_BRIDGE_PCI);
printf(" Found host->PCI bridge for domain %04x bus %02x-%02x\n",
obj->attr->bridge.downstream.pci.domain,
obj->attr->bridge.downstream.pci.secondary_bus,
obj->attr->bridge.downstream.pci.subordinate_bus);
} else {
assert(obj->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_PCI);
assert(obj->attr->bridge.downstream_type == HWLOC_OBJ_BRIDGE_PCI);
printf(" Found PCI->PCI bridge [%04x:%04x] for domain %04x bus %02x-%02x\n",
obj->attr->bridge.upstream.pci.vendor_id,
obj->attr->bridge.upstream.pci.device_id,
obj->attr->bridge.downstream.pci.domain,
obj->attr->bridge.downstream.pci.secondary_bus,
obj->attr->bridge.downstream.pci.subordinate_bus);
}
}
printf("Found %d PCI devices\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PCI_DEVICE));
obj = NULL;
while ((obj = hwloc_get_next_pcidev(topology, obj)) != NULL) {
assert(obj->type == HWLOC_OBJ_PCI_DEVICE);
printf(" Found PCI device class %04x vendor %04x model %04x\n",
obj->attr->pcidev.class_id, obj->attr->pcidev.vendor_id, obj->attr->pcidev.device_id);
}
printf("Found %d OS devices\n", hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE));
obj = NULL;
while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
assert(obj->type == HWLOC_OBJ_OS_DEVICE);
printf(" Found OS device %s subtype %d\n", obj->name, obj->attr->osdev.type);
}
assert(HWLOC_TYPE_DEPTH_BRIDGE == hwloc_get_type_depth(topology, HWLOC_OBJ_BRIDGE));
assert(HWLOC_TYPE_DEPTH_PCI_DEVICE == hwloc_get_type_depth(topology, HWLOC_OBJ_PCI_DEVICE));
assert(HWLOC_TYPE_DEPTH_OS_DEVICE == hwloc_get_type_depth(topology, HWLOC_OBJ_OS_DEVICE));
assert(hwloc_compare_types(HWLOC_OBJ_BRIDGE, HWLOC_OBJ_PCI_DEVICE) < 0);
assert(hwloc_compare_types(HWLOC_OBJ_BRIDGE, HWLOC_OBJ_OS_DEVICE) < 0);
assert(hwloc_compare_types(HWLOC_OBJ_PCI_DEVICE, HWLOC_OBJ_OS_DEVICE) < 0);
hwloc_topology_destroy(topology);
return 0;
}
int
main (void)
{
hwloc_topology_t topology;
unsigned depth;
hwloc_obj_t last;
hwloc_obj_t *closest;
unsigned found;
int err;
unsigned numprocs;
hwloc_obj_t ancestor;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "2 3 4 5");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
depth = hwloc_topology_get_depth(topology);
/* get the last object of last level */
numprocs = hwloc_get_nbobjs_by_depth(topology, depth-1);
last = hwloc_get_obj_by_depth(topology, depth-1, numprocs-1);
/* allocate the array of closest objects */
closest = malloc(numprocs * sizeof(*closest));
assert(closest);
/* get closest levels */
found = hwloc_get_closest_objs (topology, last, closest, numprocs);
printf("looked for %u closest entries, found %u\n", numprocs, found);
assert(found == numprocs-1);
/* check first found is closest */
assert(closest[0] == hwloc_get_obj_by_depth(topology, depth-1, numprocs-5 /* arity is 5 on last level */));
/* check some other expected positions */
assert(closest[found-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5-1 /* last of first half */));
assert(closest[found/2-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5-1 /* last of second third of second half */));
assert(closest[found/2/3-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5+3*5-1 /* last of third quarter of third third of second half */));
/* get ancestor of last and less close object */
ancestor = hwloc_get_common_ancestor_obj(topology, last, closest[found-1]);
assert(hwloc_obj_is_in_subtree(topology, last, ancestor));
assert(hwloc_obj_is_in_subtree(topology, closest[found-1], ancestor));
assert(ancestor == hwloc_get_root_obj(topology)->first_child);
printf("ancestor type %u depth %u number %u is system level\n",
ancestor->type, ancestor->depth, ancestor->logical_index);
free(closest);
hwloc_topology_destroy (topology);
return EXIT_SUCCESS;
}
static void tdes(orte_topology_t *t)
{
if (NULL != t->topo) {
hwloc_topology_destroy(t->topo);
}
if (NULL != t->sig) {
free(t->sig);
}
}
int compute_context_nbr(int * ctxnbr, int threadnbr, int verbose)
{
#if (defined WITH_STARPU && defined STARPU_CONTEXT)
if (*ctxnbr == -1)
{
int depth;
unsigned i, n;
int ncpu_per_socket, nsocket, ncpu;
hwloc_topology_t topology;
hwloc_obj_t obj;
/* Allocate and initialize topology object. */
hwloc_topology_init(&topology);
/* ... Optionally, put detection configuration here to ignore
some objects types, define a synthetic topology, etc....
The default is to detect all the objects of the machine that
the caller is allowed to access. See Configure Topology
Detection. */
/* Perform the topology detection. */
hwloc_topology_load(topology);
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET);
if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
/* number of socket is unknow, let say we have quadcore... */
ncpu_per_socket = 4;
} else {
ncpu = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
nsocket = hwloc_get_nbobjs_by_depth(topology, depth);
ncpu_per_socket = ncpu/nsocket;
}
if (verbose > API_VERBOSE_NO)
fprintf(stdout, "ncpu_per_socket %d\n", ncpu_per_socket);
nsocket = threadnbr/ncpu_per_socket;
if (threadnbr%ncpu_per_socket)
nsocket ++;
*ctxnbr = nsocket + 1;
hwloc_topology_destroy(topology);
}
if (threadnbr + 1 < *ctxnbr)
*ctxnbr = threadnbr+1;
if (*ctxnbr == 1)
*ctxnbr = 2;
/* can't have more than STARPU_NMAX_SCHED_CTXS CTX */
{
PASTIX_INT nctx_bot = *ctxnbr - 1;
while (*ctxnbr > STARPU_NMAX_SCHED_CTXS)
{
nctx_bot /= 2;
*ctxnbr = nctx_bot + 1;
}
}
#endif /* WITH_STARPU */
return 0;
}
tm_topology_t* get_local_topo_with_hwloc(void)
{
hwloc_topology_t topology;
tm_topology_t *res = NULL;
hwloc_obj_t *objs = NULL;
unsigned topodepth,depth;
int nb_nodes,i;
/* Build the topology */
hwloc_topology_init(&topology);
hwloc_topology_ignore_all_keep_structure(topology);
hwloc_topology_load(topology);
/* Test if symetric */
if(!symetric(topology)){
if(get_verbose_level() >= CRITICAL)
fprintf(stderr,"Local toplogy not symetric!\n");
exit(-1);
}
/* work on depth */
topodepth = hwloc_topology_get_depth(topology);
res = (tm_topology_t*)MALLOC(sizeof(tm_topology_t));
res->nb_levels = topodepth;
res->node_id = (int**)MALLOC(sizeof(int*)*res->nb_levels);
res->nb_nodes = (int*)MALLOC(sizeof(int)*res->nb_levels);
res->arity = (int*)MALLOC(sizeof(int)*res->nb_levels);
/* Build TreeMatch topology */
for( depth = 0 ; depth < topodepth ; depth++ ){
nb_nodes = hwloc_get_nbobjs_by_depth(topology, depth);
res->nb_nodes[depth] = nb_nodes;
res->node_id[depth] = (int*)MALLOC(sizeof(int)*nb_nodes);
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*nb_nodes);
objs[0] = hwloc_get_next_obj_by_depth(topology,depth,NULL);
hwloc_get_closest_objs(topology,objs[0],objs+1,nb_nodes-1);
res->arity[depth] = objs[0]->arity;
/* printf("%d:",res->arity[depth]); */
/* Build process id tab */
for (i = 0; i < nb_nodes; i++){
res->node_id[depth][i] = objs[i]->os_index;
/* if(depth==topodepth-1) */
}
FREE(objs);
}
/* Destroy HWLOC topology object. */
hwloc_topology_destroy(topology);
/* printf("\n"); */
return res;
}
void qrm_hwloc_topo(int *nodes, int *topo)
{
int depth, ret;
unsigned i, n, j, ncores, nnodes, cnode;
int topodepth, numa;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj, cobj;
hwloc_obj_type_t otype;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
/* get the number os cores and NUMA nodes */
ncores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
printf("ncores: %d\n",ncores);
nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NODE);
if(nnodes == 0){
otype = HWLOC_OBJ_SOCKET;
printf("grouping with sockets\n");
nnodes = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_SOCKET);
} else {
otype = HWLOC_OBJ_NODE;
printf("grouping with NUMA nodes\n");
}
/* get the handle for the first NUMA node */
obj = hwloc_get_obj_by_type(topology, otype, 0);
/* get the number of cores in one NUMA node (supposedly the same for all nodes) */
cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);
for(i=0; i<nnodes; i++){
/* get the handle for the first i-th node */
obj = hwloc_get_obj_by_type(topology, otype, i);
/* get the number of cores in i-th NUMA node (supposedly the same for all nodes) */
cnode = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE);
/* get the first core in this node */
cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, NULL);
topo[(i*cnode)] = cobj->logical_index;
/* printf("%2d -- group: %2d",i,cobj->logical_index); */
for(j=1; j<cnode; j++){
cobj = hwloc_get_next_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_CORE, cobj);
topo[(i*cnode)+j] = cobj->logical_index;
/* printf(" %2d",cobj->logical_index); */
}
/* printf("\n"); */
}
hwloc_topology_destroy(topology);
return;
}
int main(void)
{
hwloc_topology_t topology;
struct ibv_device **dev_list, *dev;
int count, i;
int err;
dev_list = ibv_get_device_list(&count);
if (!dev_list) {
fprintf(stderr, "ibv_get_device_list failed\n");
return 0;
}
printf("ibv_get_device_list found %d devices\n", count);
hwloc_topology_init(&topology);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
for(i=0; i<count; i++) {
hwloc_bitmap_t set;
dev = dev_list[i];
set = hwloc_bitmap_alloc();
err = hwloc_ibv_get_device_cpuset(topology, dev, set);
if (err < 0) {
printf("failed to get cpuset for device %d (%s)\n",
i, ibv_get_device_name(dev));
} else {
char *cpuset_string = NULL;
hwloc_obj_t os;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d (%s)\n",
cpuset_string, i, ibv_get_device_name(dev));
free(cpuset_string);
os = hwloc_ibv_get_device_osdev(topology, dev);
if (os) {
assert(os->type == HWLOC_OBJ_OS_DEVICE);
printf("found OS object subtype %u lindex %u name %s\n",
(unsigned) os->attr->osdev.type, os->logical_index, os->name);
assert(os->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS);
if (strcmp(ibv_get_device_name(dev), os->name))
assert(0);
}
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
ibv_free_device_list(dev_list);
return 0;
}
void orte_util_nidmap_finalize(void)
{
#if OPAL_HAVE_HWLOC
/* destroy the topology */
if (NULL != opal_hwloc_topology) {
hwloc_topology_destroy(opal_hwloc_topology);
opal_hwloc_topology = NULL;
}
#endif
}
void LpelHwLocCleanup(void)
{
#ifdef HAVE_HWLOC
if (topology) {
free(cpu_sets);
free(hw_places);
hwloc_topology_destroy(topology);
}
#endif
}
magma_int_t magma_get_parallel_numthreads()
{
// query number of cores
magma_int_t ncores = 0;
#ifdef HAVE_HWLOC
// hwloc gives physical cores, not hyperthreads
// from http://stackoverflow.com/questions/12483399/getting-number-of-cores-not-ht-threads
hwloc_topology_t topology;
hwloc_topology_init( &topology );
hwloc_topology_load( topology );
magma_int_t depth = hwloc_get_type_depth( topology, HWLOC_OBJ_CORE );
if (depth != HWLOC_TYPE_DEPTH_UNKNOWN) {
ncores = hwloc_get_nbobjs_by_depth( topology, depth );
}
hwloc_topology_destroy( topology );
#endif
if ( ncores == 0 ) {
#ifdef _MSC_VER // Windows
SYSTEM_INFO sysinfo;
GetSystemInfo( &sysinfo );
ncores = sysinfo.dwNumberOfProcessors;
#else
ncores = sysconf( _SC_NPROCESSORS_ONLN );
#endif
}
// query MAGMA_NUM_THREADS or OpenMP
const char *threads_str = getenv("MAGMA_NUM_THREADS");
magma_int_t threads = 0;
if ( threads_str != NULL ) {
char* endptr;
threads = strtol( threads_str, &endptr, 10 );
if ( threads < 1 || *endptr != '\0' ) {
threads = 1;
fprintf( stderr, "$MAGMA_NUM_THREADS='%s' is an invalid number; using %d thread.\n",
threads_str, (int) threads );
}
}
else {
#if defined(_OPENMP)
#pragma omp parallel
{
threads = omp_get_num_threads();
}
#else
threads = ncores;
#endif
}
// limit to range [1, number of cores]
threads = max( 1, min( ncores, threads ));
return threads;
}
/*
*Deallocate HWLOC and the machine architecture representation
* topo: the HWLOC object to be deallocated
* return: -1 in case of topo is not valid object
*/
int hw_topology_destroy (struct arch_topology *topo)
{
if(topo){
hwloc_topology_destroy(topology);
topo->nnodes = topo->nsockets = topo->ncores = 0;
topo->nshared_caches = topo->nsiblings = 0;
free(local_topo);
return 0;
}
else
return -1;
}
int main(void) {
hwloc_topology_t topology;
int nbcores;
hwloc_topology_init(&topology); // initialization
hwloc_topology_load(topology); // actual detection
nbcores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
printf("%d cores\n", nbcores);
hwloc_topology_destroy(topology);
return 0;
}
int main(void) {
hwloc_topology_t topology;
hwloc_topology_init(&topology); // initialization
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology); // actual detection
hwloc_obj_t root = hwloc_get_root_obj(topology);
print_hwloc_obj(root, 0);
hwloc_topology_destroy(topology);
return 0;
}
/******************* FUNCTION *********************/
void TopoHwloc::loadTopologyFromFile ( const char* filename )
{
//vars
int status;
//destroy the old topology
hwloc_topology_destroy(topology);
//load the new def
status = hwloc_topology_set_xml(topology,filename);
allocAssume(status == 0,"Failed to load topological information from file ... in hwloc.");//,filename);
//load current topology
status = hwloc_topology_load(topology);
allocAssume(status == 0,"Failed to apply topology from ... with hwloc.");//,filename);
}
std::size_t thread_affinity_worker(std::size_t desired)
{
// Returns the OS-thread number of the worker that is running this
// PX-thread.
std::size_t current = hpx::get_worker_thread_num();
bool numa_sensitive = hpx::is_scheduler_numa_sensitive();
if (current == desired)
{
// extract the desired affinity mask
hpx::threads::topology const& t = hpx::get_runtime().get_topology();
hpx::threads::mask_type desired_mask = t.get_thread_affinity_mask(current, numa_sensitive);
#if defined(HPX_HAVE_HWLOC)
std::size_t idx = hpx::threads::find_first(desired_mask);
hwloc_topology_t topo;
hwloc_topology_init(&topo);
hwloc_topology_load(topo);
// retrieve the current affinity mask
hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
hwloc_bitmap_zero(cpuset);
if (0 == hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD)) {
// sadly get_cpubind is not implemented for Windows based systems
hwloc_cpuset_t cpuset_cmp = hwloc_bitmap_alloc();
hwloc_bitmap_zero(cpuset_cmp);
hwloc_bitmap_only(cpuset_cmp, unsigned(idx));
HPX_TEST(hwloc_bitmap_compare(cpuset, cpuset_cmp) == 0);
hwloc_bitmap_free(cpuset_cmp);
}
else
{
HPX_TEST(false && "hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD) failed!");
}
hwloc_bitmap_free(cpuset);
hwloc_topology_destroy(topo);
#endif
return desired;
}
// This PX-thread has been run by the wrong OS-thread, make the foreman
// try again by rescheduling it.
return std::size_t(-1);
}
int main(void)
{
hwloc_topology_t topology;
char *string = NULL;
hwloc_obj_t obj;
hwloc_bitmap_t set;
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, SYNTHETIC_TOPOLOGY_DESCRIPTION);
hwloc_topology_load(topology);
set = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(set, GIVEN_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(obj);
fprintf(stderr, "found covering object type %s covering cpuset %s\n",
hwloc_obj_type_string(obj->type), GIVEN_CPUSET_STRING);
assert(hwloc_bitmap_isincluded(set, obj->cpuset));
hwloc_bitmap_asprintf(&string, obj->cpuset);
fprintf(stderr, "covering object of %s is %s, expected %s\n",
GIVEN_CPUSET_STRING, string, EXPECTED_CPUSET_STRING);
assert(!strcmp(EXPECTED_CPUSET_STRING, string));
free(string);
hwloc_bitmap_sscanf(set, GIVEN_LARGESPLIT_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(obj == hwloc_get_root_obj(topology));
fprintf(stderr, "found system as covering object of first+last cpus cpuset %s\n",
GIVEN_LARGESPLIT_CPUSET_STRING);
hwloc_bitmap_sscanf(set, GIVEN_TOOLARGE_CPUSET_STRING);
obj = hwloc_get_obj_covering_cpuset(topology, set);
assert(!obj);
fprintf(stderr, "found no covering object for too-large cpuset %s\n",
GIVEN_TOOLARGE_CPUSET_STRING);
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
