int
main (void)
{
hwloc_topology_t topology;
unsigned depth;
hwloc_obj_t last;
hwloc_obj_t *closest;
unsigned found;
int err;
unsigned numprocs;
hwloc_obj_t ancestor;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "2 3 4 5");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
depth = hwloc_topology_get_depth(topology);
/* get the last object of last level */
numprocs = hwloc_get_nbobjs_by_depth(topology, depth-1);
last = hwloc_get_obj_by_depth(topology, depth-1, numprocs-1);
/* allocate the array of closest objects */
closest = malloc(numprocs * sizeof(*closest));
assert(closest);
/* get closest levels */
found = hwloc_get_closest_objs (topology, last, closest, numprocs);
printf("looked for %u closest entries, found %u\n", numprocs, found);
assert(found == numprocs-1);
/* check first found is closest */
assert(closest[0] == hwloc_get_obj_by_depth(topology, depth-1, numprocs-5 /* arity is 5 on last level */));
/* check some other expected positions */
assert(closest[found-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5-1 /* last of first half */));
assert(closest[found/2-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5-1 /* last of second third of second half */));
assert(closest[found/2/3-1] == hwloc_get_obj_by_depth(topology, depth-1, 1*3*4*5+2*4*5+3*5-1 /* last of third quarter of third third of second half */));
/* get ancestor of last and less close object */
ancestor = hwloc_get_common_ancestor_obj(topology, last, closest[found-1]);
assert(hwloc_obj_is_in_subtree(topology, last, ancestor));
assert(hwloc_obj_is_in_subtree(topology, closest[found-1], ancestor));
assert(ancestor == hwloc_get_root_obj(topology)->first_child);
printf("ancestor type %u depth %u number %u is system level\n",
ancestor->type, ancestor->depth, ancestor->logical_index);
free(closest);
hwloc_topology_destroy (topology);
return EXIT_SUCCESS;
}
static void find_and_assign_combinations_with_hwloc(int *workerids, int nworkers)
{
struct _starpu_machine_config *config = _starpu_get_machine_config();
struct _starpu_machine_topology *topology = &config->topology;
int synthesize_arity = starpu_get_env_number("STARPU_SYNTHESIZE_ARITY_COMBINED_WORKER");
int min = starpu_get_env_number("STARPU_MIN_WORKERSIZE");
if (min < 2)
min = 2;
int max = starpu_get_env_number("STARPU_MAX_WORKERSIZE");
if (max == -1)
max = INT_MAX;
if (synthesize_arity == -1)
synthesize_arity = 2;
/* First, mark nodes which contain CPU workers, simply by setting their userdata field */
int i;
for (i = 0; i < nworkers; i++)
{
struct _starpu_worker *worker = _starpu_get_worker_struct(workerids[i]);
if (worker->perf_arch.devices[0].type == STARPU_CPU_WORKER && worker->perf_arch.devices[0].ncores == 1)
{
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology->hwtopology, config->pu_depth, worker->bindid);
obj = obj->parent;
while (obj)
{
obj->userdata = (void*) -1;
obj = obj->parent;
}
}
}
find_and_assign_combinations(hwloc_get_root_obj(topology->hwtopology), min, max, synthesize_arity);
}
static int
hwloc_solaris_get_sth_cpubind(hwloc_topology_t topology, idtype_t idtype, id_t id, hwloc_bitmap_t hwloc_set, int flags __hwloc_attribute_unused)
{
processorid_t binding;
int depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
int n;
int i;
if (depth < 0) {
errno = ENOSYS;
return -1;
}
/* first check if processor_bind() was used to bind to a single processor rather than to an lgroup */
if ( processor_bind(idtype, id, PBIND_QUERY, &binding) == 0 && binding != PBIND_NONE ) {
hwloc_bitmap_only(hwloc_set, binding);
return 0;
}
/* if not, check lgroups */
hwloc_bitmap_zero(hwloc_set);
n = hwloc_get_nbobjs_by_depth(topology, depth);
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
lgrp_affinity_t aff = lgrp_affinity_get(idtype, id, obj->os_index);
if (aff == LGRP_AFF_STRONG)
hwloc_bitmap_or(hwloc_set, hwloc_set, obj->cpuset);
}
if (hwloc_bitmap_iszero(hwloc_set))
hwloc_bitmap_copy(hwloc_set, hwloc_topology_get_complete_cpuset(topology));
return 0;
}
hwloc_obj_t location_parse(hwloc_topology_t topology, const char* location){
hwloc_obj_type_t type;
char * name;
int err, depth;
char * idx;
int logical_index;
char * loc = strdup(location);
name = strtok(loc,":");
if(name==NULL){return NULL;}
err = hwloc_type_sscanf_as_depth(name, &type, topology, &depth);
if(err == HWLOC_TYPE_DEPTH_UNKNOWN){
fprintf(stderr,"type %s cannot be found, level=%d\n",name,depth);
return NULL;
}
if(depth == HWLOC_TYPE_DEPTH_MULTIPLE){
fprintf(stderr,"type %s multiple caches match for\n",name);
return NULL;
}
logical_index = 0;
idx = strtok(NULL,":");
if(idx!=NULL){logical_index = atoi(idx);}
free(loc);
return hwloc_get_obj_by_depth(topology,depth,logical_index);
}
/*
* Distribute cpus to the task using block distribution
*/
static int _task_cgroup_cpuset_dist_block(
hwloc_topology_t topology, hwloc_obj_type_t hwtype,
hwloc_obj_type_t req_hwtype, uint32_t nobj,
slurmd_job_t *job, int bind_verbose, hwloc_bitmap_t cpuset)
{
hwloc_obj_t obj;
uint32_t i, pfirst,plast;
uint32_t taskid = job->envtp->localid;
int hwdepth;
if (bind_verbose)
info("task/cgroup: task[%u] using block distribution, "
"task_dist %u", taskid, job->task_dist);
if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0) {
/* cores or threads granularity */
pfirst = taskid * job->cpus_per_task ;
plast = pfirst + job->cpus_per_task - 1;
} else {
/* sockets or ldoms granularity */
pfirst = taskid;
plast = pfirst;
}
hwdepth = hwloc_get_type_depth(topology,hwtype);
for (i = pfirst; i <= plast && i < nobj ; i++) {
obj = hwloc_get_obj_by_depth(topology, hwdepth, (int)i);
_add_cpuset(hwtype, req_hwtype, obj, taskid, bind_verbose,
cpuset);
}
return XCGROUP_SUCCESS;
}
static void
output_compute_pu_min_textwidth(struct lstopo_output *output)
{
unsigned fontsize = output->fontsize;
char text[64];
int n;
hwloc_topology_t topology = output->topology;
hwloc_obj_t lastpu;
if (!output->methods->textsize) {
output->min_pu_textwidth = 0;
return;
}
if (output->logical) {
int depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU);
lastpu = hwloc_get_obj_by_depth(topology, depth, hwloc_get_nbobjs_by_depth(topology, depth)-1);
} else {
unsigned lastidx = hwloc_bitmap_last(hwloc_topology_get_topology_cpuset(topology));
lastpu = hwloc_get_pu_obj_by_os_index(topology, lastidx);
}
n = lstopo_obj_snprintf(output, text, sizeof(text), lastpu);
output->min_pu_textwidth = get_textwidth(output, text, n, fontsize);
}
static int
hwloc_solaris_get_sth_membind(hwloc_topology_t topology, idtype_t idtype, id_t id, hwloc_nodeset_t nodeset, hwloc_membind_policy_t *policy, int flags __hwloc_attribute_unused)
{
int depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
int n;
int i;
if (depth < 0) {
errno = ENOSYS;
return -1;
}
hwloc_bitmap_zero(nodeset);
n = hwloc_get_nbobjs_by_depth(topology, depth);
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
lgrp_affinity_t aff = lgrp_affinity_get(idtype, id, obj->os_index);
if (aff == LGRP_AFF_STRONG)
hwloc_bitmap_set(nodeset, obj->os_index);
}
if (hwloc_bitmap_iszero(nodeset))
hwloc_bitmap_copy(nodeset, hwloc_topology_get_complete_nodeset(topology));
*policy = HWLOC_MEMBIND_BIND;
return 0;
}
static int rmaps_lama_hwloc_compare_topos(hwloc_topology_t *left, hwloc_topology_t *right)
{
hwloc_obj_t left_root;
hwloc_obj_t right_root;
/*
* Note: I hope that there is a 'better' way of doing this natively with
* HWLOC, but it is not obvious if they have the ability to compare
* topologies. So do a depth first comparison of the trees.
* You may be able to use the below:
* OPAL_EQUAL != opal_dss.compare(*last_topo, topo, OPAL_HWLOC_TOPO);
*/
left_root = hwloc_get_obj_by_depth(*left, 0, 0);
right_root = hwloc_get_obj_by_depth(*right, 0, 0);
return rmaps_lama_hwloc_compare_subtrees(left_root, right_root);
}
static void check_level(hwloc_topology_t topology, unsigned depth, unsigned width, unsigned arity)
{
unsigned j;
assert(hwloc_get_nbobjs_by_depth(topology, depth) == width);
for(j=0; j<width; j++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, j);
assert(obj);
assert(obj->arity == arity);
}
}
static void check(hwloc_topology_t topology)
{
unsigned depth;
unsigned i,j;
depth = hwloc_topology_get_depth(topology);
for(i=0; i<depth; i++) {
for(j=0; j<hwloc_get_nbobjs_by_depth(topology, i); j++) {
assert(hwloc_get_obj_by_depth(topology, i, j)->userdata == NULL);
}
}
}
static int _get_ldom_sched_cpuset(hwloc_topology_t topology,
hwloc_obj_type_t hwtype, hwloc_obj_type_t req_hwtype,
uint32_t ldom, cpu_set_t *mask)
{
hwloc_obj_t obj;
hwloc_bitmap_t cpuset;
int hwdepth;
cpuset = hwloc_bitmap_alloc();
hwdepth = hwloc_get_type_depth(topology, hwtype);
obj = hwloc_get_obj_by_depth(topology, hwdepth, ldom);
_add_hwloc_cpuset(hwtype, req_hwtype, obj, 0, 0, cpuset);
hwloc_cpuset_to_glibc_sched_affinity(topology, cpuset,
mask, sizeof(cpu_set_t));
hwloc_bitmap_free(cpuset);
return true;
}
static int
hwloc_aix_get_sth_membind(hwloc_topology_t topology, rstype_t what, rsid_t who, hwloc_bitmap_t nodeset, hwloc_membind_policy_t *policy, int flags __hwloc_attribute_unused)
{
hwloc_bitmap_t hwloc_set;
rsethandle_t rset;
unsigned cpu, maxcpus;
int res = -1;
int depth, n, i;
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
if (depth < 0) {
errno = EXDEV;
return -1;
}
n = hwloc_get_nbobjs_by_depth(topology, depth);
rset = rs_alloc(RS_EMPTY);
if (ra_getrset(what, who, 0, rset) == -1)
goto out;
hwloc_set = hwloc_bitmap_alloc();
maxcpus = rs_getinfo(rset, R_MAXPROCS, 0);
for (cpu = 0; cpu < maxcpus; cpu++)
if (rs_op(RS_TESTRESOURCE, rset, NULL, R_PROCS, cpu) == 1)
hwloc_bitmap_set(hwloc_set, cpu);
hwloc_bitmap_and(hwloc_set, hwloc_set, hwloc_topology_get_complete_cpuset(topology));
hwloc_bitmap_zero(nodeset);
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
if (hwloc_bitmap_isincluded(obj->cpuset, hwloc_set))
hwloc_bitmap_set(nodeset, obj->os_index);
}
hwloc_bitmap_free(hwloc_set);
*policy = HWLOC_MEMBIND_BIND;
res = 0;
out:
rs_free(rset);
return res;
}
static int
hwloc_solaris_set_sth_membind(hwloc_topology_t topology, idtype_t idtype, id_t id, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags)
{
int depth;
int n, i;
switch (policy) {
case HWLOC_MEMBIND_DEFAULT:
case HWLOC_MEMBIND_BIND:
break;
default:
errno = ENOSYS;
return -1;
}
if (flags & HWLOC_MEMBIND_NOCPUBIND) {
errno = ENOSYS;
return -1;
}
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
if (depth < 0) {
errno = EXDEV;
return -1;
}
n = hwloc_get_nbobjs_by_depth(topology, depth);
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
if (hwloc_bitmap_isset(nodeset, obj->os_index)) {
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_STRONG);
} else {
if (flags & HWLOC_CPUBIND_STRICT)
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_NONE);
else
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_WEAK);
}
}
return 0;
}
void Hwloc::getNumSockets(unsigned int &allowedNodes, int &numSockets, unsigned int &hwThreads) {
#ifdef HWLOC
numSockets = 0;
// Nodes that can be seen by hwloc
allowedNodes = 0;
// Hardware threads
hwThreads = 0;
int depth = hwloc_get_type_depth( _hwlocTopology, HWLOC_OBJ_NODE );
// If there are NUMA nodes in this machine
if ( depth != HWLOC_TYPE_DEPTH_UNKNOWN ) {
//hwloc_const_cpuset_t cpuset = hwloc_topology_get_online_cpuset( _hwlocTopology );
//allowedNodes = hwloc_get_nbobjs_inside_cpuset_by_type( _hwlocTopology, cpuset, HWLOC_OBJ_NODE );
//hwThreads = hwloc_get_nbobjs_inside_cpuset_by_type( _hwlocTopology, cpuset, HWLOC_OBJ_PU );
unsigned nodes = hwloc_get_nbobjs_by_depth( _hwlocTopology, depth );
//hwloc_cpuset_t set = i
// For each node, count how many hardware threads there are below.
for ( unsigned nodeIdx = 0; nodeIdx < nodes; ++nodeIdx )
{
hwloc_obj_t node = hwloc_get_obj_by_depth( _hwlocTopology, depth, nodeIdx );
int localThreads = hwloc_get_nbobjs_inside_cpuset_by_type( _hwlocTopology, node->cpuset, HWLOC_OBJ_PU );
// Increase hw thread count
hwThreads += localThreads;
// If this node has hw threads beneath, increase the number of viewable nodes
if ( localThreads > 0 ) ++allowedNodes;
}
numSockets = nodes;
}
// Otherwise, set it to 1
else {
allowedNodes = 1;
numSockets = 1;
}
#else
numSockets = 0;
allowedNodes = 0;
#endif
}
/* user to have to play with the cgroup hierarchy to modify it */
extern int task_cgroup_cpuset_set_task_affinity(slurmd_job_t *job)
{
int fstatus = SLURM_ERROR;
#ifndef HAVE_HWLOC
error("task/cgroup: plugin not compiled with hwloc support, "
"skipping affinity.");
return fstatus;
#else
uint32_t i;
uint32_t nldoms;
uint32_t nsockets;
uint32_t ncores;
uint32_t npus;
uint32_t nobj;
uint32_t pfirst,plast;
uint32_t taskid = job->envtp->localid;
uint32_t jntasks = job->node_tasks;
uint32_t jnpus = jntasks * job->cpus_per_task;
pid_t pid = job->envtp->task_pid;
cpu_bind_type_t bind_type;
int verbose;
hwloc_topology_t topology;
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_t cpuset,ct;
#else
hwloc_bitmap_t cpuset,ct;
#endif
hwloc_obj_t obj;
struct hwloc_obj *pobj;
hwloc_obj_type_t hwtype;
hwloc_obj_type_t req_hwtype;
int hwdepth;
size_t tssize;
cpu_set_t ts;
bind_type = job->cpu_bind_type ;
if (conf->task_plugin_param & CPU_BIND_VERBOSE ||
bind_type & CPU_BIND_VERBOSE)
verbose = 1 ;
if (bind_type & CPU_BIND_NONE) {
if (verbose)
info("task/cgroup: task[%u] is requesting no affinity",
taskid);
return 0;
} else if (bind_type & CPU_BIND_TO_THREADS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"thread level binding",taskid);
req_hwtype = HWLOC_OBJ_PU;
} else if (bind_type & CPU_BIND_TO_CORES) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"core level binding",taskid);
req_hwtype = HWLOC_OBJ_CORE;
} else if (bind_type & CPU_BIND_TO_SOCKETS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"socket level binding",taskid);
req_hwtype = HWLOC_OBJ_SOCKET;
} else if (bind_type & CPU_BIND_TO_LDOMS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"ldom level binding",taskid);
req_hwtype = HWLOC_OBJ_NODE;
} else {
if (verbose)
info("task/cgroup: task[%u] using core level binding"
" by default",taskid);
req_hwtype = HWLOC_OBJ_CORE;
}
/* Allocate and initialize hwloc objects */
hwloc_topology_init(&topology);
#if HWLOC_API_VERSION <= 0x00010000
cpuset = hwloc_cpuset_alloc() ;
#else
cpuset = hwloc_bitmap_alloc() ;
#endif
/*
* Perform the topology detection. It will only get allowed PUs.
* Detect in the same time the granularity to use for binding.
* The granularity can be relaxed from threads to cores if enough
* cores are available as with hyperthread support, ntasks-per-core
* param can let us have access to more threads per core for each
* task
* Revert back to machine granularity if no finer-grained granularity
* matching the request is found. This will result in no affinity
* applied.
* The detected granularity will be used to find where to best place
* the task, then the cpu_bind option will be used to relax the
* affinity constraint and use more PUs. (i.e. use a core granularity
* to dispatch the tasks across the sockets and then provide access
* to each task to the cores of its socket.)
*/
hwloc_topology_load(topology);
npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_SOCKET);
nldoms = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_NODE);
hwtype = HWLOC_OBJ_MACHINE;
nobj = 1;
if (npus >= jnpus || bind_type & CPU_BIND_TO_THREADS) {
hwtype = HWLOC_OBJ_PU;
nobj = npus;
}
if (ncores >= jnpus || bind_type & CPU_BIND_TO_CORES) {
hwtype = HWLOC_OBJ_CORE;
nobj = ncores;
}
if (nsockets >= jntasks &&
bind_type & CPU_BIND_TO_SOCKETS) {
hwtype = HWLOC_OBJ_SOCKET;
nobj = nsockets;
}
/*
* HWLOC returns all the NUMA nodes available regardless of the
* number of underlying sockets available (regardless of the allowed
* resources). So there is no guarantee that each ldom will be populated
* with usable sockets. So add a simple check that at least ensure that
* we have as many sockets as ldoms before moving to ldoms granularity
*/
if (nldoms >= jntasks &&
nsockets >= nldoms &&
bind_type & CPU_BIND_TO_LDOMS) {
hwtype = HWLOC_OBJ_NODE;
nobj = nldoms;
}
/*
* Perform a block binding on the detected object respecting the
* granularity.
* If not enough objects to do the job, revert to no affinity mode
*/
if (hwloc_compare_types(hwtype,HWLOC_OBJ_MACHINE) == 0) {
info("task/cgroup: task[%u] disabling affinity because of %s "
"granularity",taskid,hwloc_obj_type_string(hwtype));
} else if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0 &&
jnpus > nobj) {
info("task/cgroup: task[%u] not enough %s objects, disabling "
"affinity",taskid,hwloc_obj_type_string(hwtype));
} else {
if (verbose) {
info("task/cgroup: task[%u] using %s granularity",
taskid,hwloc_obj_type_string(hwtype));
}
if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0) {
/* cores or threads granularity */
pfirst = taskid * job->cpus_per_task ;
plast = pfirst + job->cpus_per_task - 1;
} else {
/* sockets or ldoms granularity */
pfirst = taskid;
plast = pfirst;
}
hwdepth = hwloc_get_type_depth(topology,hwtype);
for (i = pfirst; i <= plast && i < nobj ; i++) {
obj = hwloc_get_obj_by_depth(topology,hwdepth,(int)i);
/* if requested binding overlap the granularity */
/* use the ancestor cpuset instead of the object one */
if (hwloc_compare_types(hwtype,req_hwtype) > 0) {
/* Get the parent object of req_hwtype or the */
/* one just above if not found (meaning of >0)*/
/* (useful for ldoms binding with !NUMA nodes)*/
pobj = obj->parent;
while (pobj != NULL &&
hwloc_compare_types(pobj->type,
req_hwtype) > 0)
pobj = pobj->parent;
if (pobj != NULL) {
if (verbose)
info("task/cgroup: task[%u] "
"higher level %s found",
taskid,
hwloc_obj_type_string(
pobj->type));
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(pobj->
allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(pobj->
allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
} else {
/* should not be executed */
if (verbose)
info("task/cgroup: task[%u] "
"no higher level found",
taskid);
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(obj->
allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(obj->
allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
}
} else {
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(obj->allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(obj->allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
}
}
char *str;
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_asprintf(&str,cpuset);
#else
hwloc_bitmap_asprintf(&str,cpuset);
#endif
tssize = sizeof(cpu_set_t);
if (hwloc_cpuset_to_glibc_sched_affinity(topology,cpuset,
&ts,tssize) == 0) {
fstatus = SLURM_SUCCESS;
if (sched_setaffinity(pid,tssize,&ts)) {
error("task/cgroup: task[%u] unable to set "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
} else if (verbose) {
info("task/cgroup: task[%u] taskset '%s' is set"
,taskid,str);
}
} else {
error("task/cgroup: task[%u] unable to build "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
}
free(str);
}
/* Destroy hwloc objects */
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_free(cpuset);
#else
hwloc_bitmap_free(cpuset);
#endif
hwloc_topology_destroy(topology);
return fstatus;
#endif
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t obj, cache;
hwloc_bitmap_t set;
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, SYNTHETIC_TOPOLOGY_DESCRIPTION);
hwloc_topology_load(topology);
/* check the cache above a given cpu */
#define CPUINDEX 180
set = hwloc_bitmap_alloc();
obj = hwloc_get_obj_by_depth(topology, 5, CPUINDEX);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
cache = hwloc_get_cache_covering_cpuset(topology, set);
assert(cache);
assert(cache->type == HWLOC_OBJ_CACHE);
assert(cache->logical_index == CPUINDEX/2/3);
assert(hwloc_obj_is_in_subtree(topology, obj, cache));
hwloc_bitmap_free(set);
/* check the cache above two nearby cpus */
#define CPUINDEX1 180
#define CPUINDEX2 183
set = hwloc_bitmap_alloc();
obj = hwloc_get_obj_by_depth(topology, 5, CPUINDEX1);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
obj = hwloc_get_obj_by_depth(topology, 5, CPUINDEX2);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
cache = hwloc_get_cache_covering_cpuset(topology, set);
assert(cache);
assert(cache->type == HWLOC_OBJ_CACHE);
assert(cache->logical_index == CPUINDEX1/2/3);
assert(cache->logical_index == CPUINDEX2/2/3);
assert(hwloc_obj_is_in_subtree(topology, obj, cache));
hwloc_bitmap_free(set);
/* check no cache above two distant cpus */
#undef CPUINDEX1
#define CPUINDEX1 300
set = hwloc_bitmap_alloc();
obj = hwloc_get_obj_by_depth(topology, 5, CPUINDEX1);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
obj = hwloc_get_obj_by_depth(topology, 5, CPUINDEX2);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
cache = hwloc_get_cache_covering_cpuset(topology, set);
assert(!cache);
hwloc_bitmap_free(set);
/* check no cache above higher level */
set = hwloc_bitmap_alloc();
obj = hwloc_get_obj_by_depth(topology, 2, 0);
assert(obj);
hwloc_bitmap_or(set, set, obj->cpuset);
cache = hwloc_get_cache_covering_cpuset(topology, set);
assert(!cache);
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
int main(void)
{
hwloc_topology_t topology;
unsigned nbobjs;
const struct hwloc_distances_s *distances;
float d1, d2;
unsigned depth, topodepth, i, j;
int err;
hwloc_obj_t obj1, obj2;
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, "node:4 core:4 pu:1");
putenv("HWLOC_NUMANode_DISTANCES=0,1,2,3:2*2");
putenv("HWLOC_PU_DISTANCES=0-15:4*2*2");
hwloc_topology_load(topology);
topodepth = hwloc_topology_get_depth(topology);
for(depth=0; depth<topodepth; depth++) {
distances = hwloc_get_whole_distance_matrix_by_depth(topology, depth);
if (!distances || !distances->latency) {
printf("No distance at depth %u\n", depth);
continue;
}
printf("distance matrix for depth %u:\n", depth);
print_distances(distances);
nbobjs = distances->nbobjs;
obj1 = hwloc_get_obj_by_depth(topology, depth, 0);
obj2 = hwloc_get_obj_by_depth(topology, depth, nbobjs-1);
err = hwloc_get_latency(topology, obj1, obj2, &d1, &d2);
assert(!err);
assert(d1 == distances->latency[0*nbobjs+(nbobjs-1)]);
assert(d2 == distances->latency[(nbobjs-1)*nbobjs+0]);
}
/* check that hwloc_get_latency works fine on numa distances */
distances = hwloc_get_whole_distance_matrix_by_type(topology, HWLOC_OBJ_NUMANODE);
if (!distances || !distances->latency) {
fprintf(stderr, "No NUMA distance matrix!\n");
return -1;
}
printf("distance matrix for NUMA nodes\n");
print_distances(distances);
nbobjs = distances->nbobjs;
for(i=0; i<nbobjs; i++)
for(j=0; j<nbobjs; j++) {
obj1 = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, i);
obj2 = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, j);
err = hwloc_get_latency(topology, obj1, obj2, &d1, &d2);
assert(!err);
assert(d1 == distances->latency[i*nbobjs+j]);
assert(d2 == distances->latency[j*nbobjs+i]);
}
/* check that some random values are ok */
assert(distances->latency[0] == 1.0); /* diagonal */
assert(distances->latency[4] == 4.0); /* same group */
assert(distances->latency[6] == 8.0); /* different group */
assert(distances->latency[9] == 8.0); /* different group */
assert(distances->latency[10] == 1.0); /* diagonal */
assert(distances->latency[14] == 4.0); /* same group */
/* check that hwloc_get_latency works fine on PU distances */
distances = hwloc_get_whole_distance_matrix_by_type(topology, HWLOC_OBJ_PU);
if (!distances || !distances->latency) {
fprintf(stderr, "No PU distance matrix!\n");
return -1;
}
printf("distance matrix for PU nodes\n");
print_distances(distances);
nbobjs = distances->nbobjs;
for(i=0; i<16; i++)
for(j=0; j<16; j++) {
obj1 = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, i);
obj2 = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, j);
err = hwloc_get_latency(topology, obj1, obj2, &d1, &d2);
assert(!err);
assert(d1 == distances->latency[i*nbobjs+j]);
assert(d2 == distances->latency[j*nbobjs+i]);
}
/* check that some random values are ok */
assert(distances->latency[0] == 1.0); /* diagonal */
assert(distances->latency[1] == 2.0); /* same group */
assert(distances->latency[3] == 4.0); /* same biggroup */
assert(distances->latency[15] == 8.0); /* different biggroup */
assert(distances->latency[250] == 8.0); /* different biggroup */
assert(distances->latency[253] == 4.0); /* same group */
assert(distances->latency[254] == 2.0); /* same biggroup */
assert(distances->latency[255] == 1.0); /* diagonal */
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology, reimport;
hwloc_obj_t obj1, obj2, obj3;
char *xmlbuf;
int xmlbuflen;
randomstring = malloc(RANDOMSTRINGLENGTH);
/* keep it uninitialized, we want binary data */
/* check the real topology */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
check(topology);
assert(hwloc_topology_get_userdata(topology) == NULL);
hwloc_topology_destroy(topology);
/* check a synthetic topology */
hwloc_topology_init(&topology);
hwloc_topology_set_userdata(topology, (void *)(uintptr_t)0x987654);
hwloc_topology_set_synthetic(topology, "6 5 4 3 2");
hwloc_topology_load(topology);
check(topology);
/* now place some userdata and see if importing/exporting works well */
obj1 = hwloc_get_root_obj(topology);
assert(obj1);
obj1->userdata = (void *)(uintptr_t) 0x1;
obj2 = hwloc_get_obj_by_depth(topology, 3, 13);
assert(obj2);
obj2->userdata = (void *)(uintptr_t) 0x2;
obj3 = hwloc_get_obj_by_depth(topology, 5, 2*3*4*5*6-1);
assert(obj3);
obj3->userdata = (void *)(uintptr_t) 0x3;
/* export/import without callback, we get nothing */
hwloc_topology_export_xmlbuffer(topology, &xmlbuf, &xmlbuflen);
hwloc_topology_init(&reimport);
hwloc_topology_set_xmlbuffer(reimport, xmlbuf, xmlbuflen);
hwloc_topology_load(reimport);
check(reimport); /* there should be no userdata */
hwloc_topology_destroy(reimport);
/* export/import with callback, we should get three userdata */
hwloc_topology_set_userdata_export_callback(topology, export_cb);
hwloc_topology_export_xmlbuffer(topology, &xmlbuf, &xmlbuflen);
hwloc_topology_init(&reimport);
hwloc_topology_set_userdata_import_callback(reimport, import_cb);
hwloc_topology_set_xmlbuffer(reimport, xmlbuf, xmlbuflen);
hwloc_topology_load(reimport);
obj1 = hwloc_get_root_obj(reimport);
assert(obj1);
assert(obj1->userdata == (void *)(uintptr_t) 0x4);
obj2 = hwloc_get_obj_by_depth(reimport, 3, 13);
assert(obj2);
assert(obj2->userdata == (void *)(uintptr_t) 0x5);
obj3 = hwloc_get_obj_by_depth(reimport, 5, 2*3*4*5*6-1);
assert(obj3);
assert(obj3->userdata == (void *)(uintptr_t) 0x6);
hwloc_topology_destroy(reimport);
assert(hwloc_topology_get_userdata(topology) == (void *)(uintptr_t)0x987654);
hwloc_topology_destroy(topology);
free(randomstring);
return 0;
}
static int
hwloc_apply_diff_one(hwloc_topology_t topology,
hwloc_topology_diff_t diff,
unsigned long flags)
{
int reverse = !!(flags & HWLOC_TOPOLOGY_DIFF_APPLY_REVERSE);
switch (diff->generic.type) {
case HWLOC_TOPOLOGY_DIFF_OBJ_ATTR: {
struct hwloc_topology_diff_obj_attr_s *obj_attr = &diff->obj_attr;
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, obj_attr->obj_depth, obj_attr->obj_index);
if (!obj)
return -1;
switch (obj_attr->diff.generic.type) {
case HWLOC_TOPOLOGY_DIFF_OBJ_ATTR_SIZE: {
hwloc_obj_t tmpobj;
hwloc_uint64_t oldvalue = reverse ? obj_attr->diff.uint64.newvalue : obj_attr->diff.uint64.oldvalue;
hwloc_uint64_t newvalue = reverse ? obj_attr->diff.uint64.oldvalue : obj_attr->diff.uint64.newvalue;
hwloc_uint64_t valuediff = newvalue - oldvalue;
if (obj->memory.local_memory != oldvalue)
return -1;
obj->memory.local_memory = newvalue;
tmpobj = obj;
while (tmpobj) {
tmpobj->memory.total_memory += valuediff;
tmpobj = tmpobj->parent;
}
break;
}
case HWLOC_TOPOLOGY_DIFF_OBJ_ATTR_NAME: {
const char *oldvalue = reverse ? obj_attr->diff.string.newvalue : obj_attr->diff.string.oldvalue;
const char *newvalue = reverse ? obj_attr->diff.string.oldvalue : obj_attr->diff.string.newvalue;
if (!obj->name || strcmp(obj->name, oldvalue))
return -1;
free(obj->name);
obj->name = strdup(newvalue);
break;
}
case HWLOC_TOPOLOGY_DIFF_OBJ_ATTR_INFO: {
const char *name = obj_attr->diff.string.name;
const char *oldvalue = reverse ? obj_attr->diff.string.newvalue : obj_attr->diff.string.oldvalue;
const char *newvalue = reverse ? obj_attr->diff.string.oldvalue : obj_attr->diff.string.newvalue;
unsigned i;
int found = 0;
for(i=0; i<obj->infos_count; i++) {
if (!strcmp(obj->infos[i].name, name)
&& !strcmp(obj->infos[i].value, oldvalue)) {
free(obj->infos[i].value);
obj->infos[i].value = strdup(newvalue);
found = 1;
break;
}
}
if (!found)
return -1;
break;
}
default:
return -1;
}
break;
}
default:
return -1;
}
return 0;
}
static int _task_cgroup_cpuset_dist_block(
hwloc_topology_t topology, hwloc_obj_type_t hwtype,
hwloc_obj_type_t req_hwtype, uint32_t nobj,
stepd_step_rec_t *job, int bind_verbose, hwloc_bitmap_t cpuset)
{
hwloc_obj_t obj;
uint32_t core_loop, ntskip, npdist;
uint32_t i, j, pfirst, plast;
uint32_t taskid = job->envtp->localid;
int hwdepth;
uint32_t npus, ncores, nsockets;
int spec_thread_cnt = 0;
bitstr_t *spec_threads = NULL;
uint32_t *thread_idx;
uint32_t core_idx;
bool core_fcyclic, core_block;
nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_SOCKET);
ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
core_block = (job->task_dist & SLURM_DIST_COREMASK) ==
SLURM_DIST_COREBLOCK ? true : false;
core_fcyclic = (job->task_dist & SLURM_DIST_COREMASK) ==
SLURM_DIST_CORECFULL ? true : false;
thread_idx = xmalloc(ncores * sizeof(uint32_t));
if (bind_verbose) {
info("task/cgroup: task[%u] using block distribution, "
"task_dist 0x%x", taskid, job->task_dist);
}
if ((hwloc_compare_types(hwtype, HWLOC_OBJ_PU) == 0) && !core_block) {
thread_idx = xmalloc(ncores * sizeof(uint32_t));
ntskip = taskid;
npdist = job->cpus_per_task;
i = 0; j = 0;
core_idx = 0;
core_loop = 0;
while (i < ntskip + 1 && core_loop < npdist + 1) {
while ((core_idx < ncores) && (j < npdist)) {
obj = hwloc_get_obj_below_by_type(
topology, HWLOC_OBJ_CORE, core_idx,
hwtype, thread_idx[core_idx]);
if (obj != NULL) {
thread_idx[core_idx]++;
j++;
if (i == ntskip)
_add_hwloc_cpuset(hwtype,
req_hwtype, obj, taskid,
bind_verbose, cpuset);
if ((j < npdist) && core_fcyclic)
core_idx++;
} else {
core_idx++;
}
}
if (j == npdist) {
i++; j = 0;
core_idx++; // no validity check, handled by the while
core_loop = 0;
} else {
core_loop++;
core_idx = 0;
}
}
xfree(thread_idx);
/* should never happen in normal scenario */
if (core_loop > npdist) {
error("task/cgroup: task[%u] infinite loop broken while "
"trying to provision compute elements using %s",
taskid, format_task_dist_states(job->task_dist));
return XCGROUP_ERROR;
} else
return XCGROUP_SUCCESS;
}
if (hwloc_compare_types(hwtype, HWLOC_OBJ_CORE) >= 0) {
/* cores or threads granularity */
pfirst = taskid * job->cpus_per_task ;
plast = pfirst + job->cpus_per_task - 1;
} else {
/* sockets or ldoms granularity */
pfirst = taskid;
plast = pfirst;
}
hwdepth = hwloc_get_type_depth(topology, hwtype);
if ((job->job_core_spec != (uint16_t) NO_VAL) &&
(job->job_core_spec & CORE_SPEC_THREAD) &&
(job->job_core_spec != CORE_SPEC_THREAD) &&
(nsockets != 0)) {
/* Skip specialized threads as needed */
int i, t, c, s;
int cores = MAX(1, (ncores / nsockets));
int threads = npus / cores;
spec_thread_cnt = job->job_core_spec & (~CORE_SPEC_THREAD);
spec_threads = bit_alloc(npus);
for (t = threads - 1;
((t >= 0) && (spec_thread_cnt > 0)); t--) {
for (c = cores - 1;
((c >= 0) && (spec_thread_cnt > 0)); c--) {
for (s = nsockets - 1;
((s >= 0) && (spec_thread_cnt > 0)); s--) {
i = s * cores + c;
i = (i * threads) + t;
bit_set(spec_threads, i);
spec_thread_cnt--;
}
}
}
if (hwtype == HWLOC_OBJ_PU) {
for (i = 0; i <= pfirst && i < npus; i++) {
if (bit_test(spec_threads, i))
pfirst++;
};
}
}
for (i = pfirst; i <= plast && i < nobj ; i++) {
obj = hwloc_get_obj_by_depth(topology, hwdepth, (int)i);
_add_hwloc_cpuset(hwtype, req_hwtype, obj, taskid,
bind_verbose, cpuset);
}
if (spec_threads) {
for (i = 0; i < npus; i++) {
if (bit_test(spec_threads, i)) {
hwloc_bitmap_clr(cpuset, i);
}
};
FREE_NULL_BITMAP(spec_threads);
}
return XCGROUP_SUCCESS;
}
void hwloc_init_cacheTopology(void)
{
int maxNumLevels=0;
int id=0;
CacheLevel* cachePool = NULL;
hwloc_obj_t obj;
int depth;
int d;
/* Sum up all depths with caches */
depth = hwloc_topology_get_depth(hwloc_topology);
for (d = 0; d < depth; d++)
{
if (hwloc_get_depth_type(hwloc_topology, d) == HWLOC_OBJ_CACHE)
maxNumLevels++;
}
cachePool = (CacheLevel*) malloc(maxNumLevels * sizeof(CacheLevel));
/* Start at the bottom of the tree to get all cache levels in order */
depth = hwloc_topology_get_depth(hwloc_topology);
id = 0;
for(d=depth-1;d >= 0; d--)
{
/* We only need caches, so skip other levels */
if (hwloc_get_depth_type(hwloc_topology, d) != HWLOC_OBJ_CACHE)
{
continue;
}
/* Get the cache object */
obj = hwloc_get_obj_by_depth(hwloc_topology, d, 0);
/* All caches have this attribute, so safe to access */
switch (obj->attr->cache.type)
{
case HWLOC_OBJ_CACHE_DATA:
cachePool[id].type = DATACACHE;
break;
case HWLOC_OBJ_CACHE_INSTRUCTION:
cachePool[id].type = INSTRUCTIONCACHE;
break;
case HWLOC_OBJ_CACHE_UNIFIED:
cachePool[id].type = UNIFIEDCACHE;
break;
default:
cachePool[id].type = NOCACHE;
break;
}
cachePool[id].associativity = obj->attr->cache.associativity;
cachePool[id].level = obj->attr->cache.depth;
cachePool[id].lineSize = obj->attr->cache.linesize;
cachePool[id].size = obj->attr->cache.size;
cachePool[id].sets = 0;
if ((cachePool[id].associativity * cachePool[id].lineSize) != 0)
{
cachePool[id].sets = cachePool[id].size /
(cachePool[id].associativity * cachePool[id].lineSize);
}
/* Count all HWThreads below the current cache */
cachePool[id].threads = hwloc_record_objs_of_type_below_obj(
hwloc_topology, obj, HWLOC_OBJ_PU, NULL, NULL);
/* We need to read the inclusiveness from CPUID, no possibility in hwloc */
switch ( cpuid_info.family )
{
case MIC_FAMILY:
case P6_FAMILY:
cachePool[id].inclusive = readCacheInclusiveIntel(cachePool[id].level);
break;
case K16_FAMILY:
case K15_FAMILY:
cachePool[id].inclusive = readCacheInclusiveAMD(cachePool[id].level);
break;
/* For K8 and K10 it is known that they are inclusive */
case K8_FAMILY:
case K10_FAMILY:
cachePool[id].inclusive = 1;
break;
default:
ERROR_PLAIN_PRINT(Processor is not supported);
break;
}
id++;
}
cpuid_topology.numCacheLevels = maxNumLevels;
cpuid_topology.cacheLevels = cachePool;
return;
}
int main(void)
{
int depth;
unsigned i, n;
unsigned long size;
int levels;
char string[128];
int topodepth;
void *m;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
/* Allocate and initialize topology object. */
hwloc_topology_init(&topology);
/* ... Optionally, put detection configuration here to ignore
some objects types, define a synthetic topology, etc....
The default is to detect all the objects of the machine that
the caller is allowed to access. See Configure Topology
Detection. */
/* Perform the topology detection. */
hwloc_topology_load(topology);
/* Optionally, get some additional topology information
in case we need the topology depth later. */
topodepth = hwloc_topology_get_depth(topology);
/*****************************************************************
* First example:
* Walk the topology with an array style, from level 0 (always
* the system level) to the lowest level (always the proc level).
*****************************************************************/
for (depth = 0; depth < topodepth; depth++) {
printf("*** Objects at level %d\n", depth);
for (i = 0; i < hwloc_get_nbobjs_by_depth(topology, depth);
i++) {
hwloc_obj_type_snprintf(string, sizeof(string),
hwloc_get_obj_by_depth(topology, depth, i), 0);
printf("Index %u: %s\n", i, string);
}
}
/*****************************************************************
* Second example:
* Walk the topology with a tree style.
*****************************************************************/
printf("*** Printing overall tree\n");
print_children(topology, hwloc_get_root_obj(topology), 0);
/*****************************************************************
* Third example:
* Print the number of packages.
*****************************************************************/
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
printf("*** The number of packages is unknown\n");
} else {
printf("*** %u package(s)\n",
hwloc_get_nbobjs_by_depth(topology, depth));
}
/*****************************************************************
* Fourth example:
* Compute the amount of cache that the first logical processor
* has above it.
*****************************************************************/
levels = 0;
size = 0;
for (obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0);
obj;
obj = obj->parent)
if (obj->type == HWLOC_OBJ_CACHE) {
levels++;
size += obj->attr->cache.size;
}
printf("*** Logical processor 0 has %d caches totaling %luKB\n",
levels, size / 1024);
/*****************************************************************
* Fifth example:
* Bind to only one thread of the last core of the machine.
*
* First find out where cores are, or else smaller sets of CPUs if
* the OS doesn't have the notion of a "core".
*****************************************************************/
depth = hwloc_get_type_or_below_depth(topology, HWLOC_OBJ_CORE);
/* Get last core. */
obj = hwloc_get_obj_by_depth(topology, depth,
hwloc_get_nbobjs_by_depth(topology, depth) - 1);
if (obj) {
/* Get a copy of its cpuset that we may modify. */
cpuset = hwloc_bitmap_dup(obj->cpuset);
/* Get only one logical processor (in case the core is
SMT/hyper-threaded). */
hwloc_bitmap_singlify(cpuset);
/* And try to bind ourself there. */
if (hwloc_set_cpubind(topology, cpuset, 0)) {
char *str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
free(str);
}
/* Free our cpuset copy */
hwloc_bitmap_free(cpuset);
}
/*****************************************************************
* Sixth example:
* Allocate some memory on the last NUMA node, bind some existing
* memory to the last NUMA node.
*****************************************************************/
/* Get last node. There's always at least one. */
n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE);
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, n - 1);
size = 1024*1024;
m = hwloc_alloc_membind_nodeset(topology, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
hwloc_free(topology, m, size);
m = malloc(size);
hwloc_set_area_membind_nodeset(topology, m, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
free(m);
/* Destroy topology object. */
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology;
unsigned depth;
hwloc_obj_t objs[OBJ_MAX];
hwloc_obj_t obj;
hwloc_bitmap_t set;
int ret;
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, SYNTHETIC_TOPOLOGY_DESCRIPTION);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
/* just get the system object */
obj = hwloc_get_root_obj(topology);
ret = hwloc_get_largest_objs_inside_cpuset(topology, obj->cpuset, objs, 1);
assert(ret == 1);
assert(objs[0] == obj);
objs[0] = hwloc_get_first_largest_obj_inside_cpuset(topology, obj->cpuset);
assert(objs[0] == obj);
/* just get the very last object */
obj = hwloc_get_obj_by_depth(topology, depth-1, hwloc_get_nbobjs_by_depth(topology, depth-1)-1);
ret = hwloc_get_largest_objs_inside_cpuset(topology, obj->cpuset, objs, 1);
assert(ret == 1);
assert(objs[0] == obj);
/* try an empty one */
set = hwloc_bitmap_alloc();
ret = hwloc_get_largest_objs_inside_cpuset(topology, set, objs, 1);
assert(ret == 0);
objs[0] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
assert(objs[0] == NULL);
hwloc_bitmap_free(set);
/* try an impossible one */
set = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(set, GIVEN_TOOLARGE_CPUSET_STRING);
ret = hwloc_get_largest_objs_inside_cpuset(topology, set, objs, 1);
assert(ret == -1);
objs[0] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
assert(objs[0] == NULL);
hwloc_bitmap_free(set);
/* try a harder one with 1 obj instead of 2 needed */
set = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(set, GIVEN_LARGESPLIT_CPUSET_STRING);
ret = hwloc_get_largest_objs_inside_cpuset(topology, set, objs, 1);
assert(ret == 1);
assert(objs[0] == hwloc_get_obj_by_depth(topology, depth-1, 0));
objs[0] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
assert(objs[0] == hwloc_get_obj_by_depth(topology, depth-1, 0));
/* try a harder one with lots of objs instead of 2 needed */
ret = hwloc_get_largest_objs_inside_cpuset(topology, set, objs, 2);
assert(ret == 2);
assert(objs[0] == hwloc_get_obj_by_depth(topology, depth-1, 0));
assert(objs[1] == hwloc_get_obj_by_depth(topology, depth-1, hwloc_get_nbobjs_by_depth(topology, depth-1)-1));
objs[0] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
hwloc_bitmap_andnot(set, set, objs[0]->cpuset);
objs[1] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
hwloc_bitmap_andnot(set, set, objs[1]->cpuset);
objs[2] = hwloc_get_first_largest_obj_inside_cpuset(topology, set);
assert(objs[0] == hwloc_get_obj_by_depth(topology, depth-1, 0));
assert(objs[1] == hwloc_get_obj_by_depth(topology, depth-1, hwloc_get_nbobjs_by_depth(topology, depth-1)-1));
assert(objs[2] == NULL);
assert(hwloc_bitmap_iszero(set));
hwloc_bitmap_free(set);
/* try a very hard one */
set = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(set, GIVEN_HARD_CPUSET_STRING);
ret = hwloc_get_largest_objs_inside_cpuset(topology, set, objs, OBJ_MAX);
assert(objs[0] == hwloc_get_obj_by_depth(topology, 5, 29));
assert(objs[1] == hwloc_get_obj_by_depth(topology, 3, 5));
assert(objs[2] == hwloc_get_obj_by_depth(topology, 3, 6));
assert(objs[3] == hwloc_get_obj_by_depth(topology, 3, 7));
assert(objs[4] == hwloc_get_obj_by_depth(topology, 2, 2));
assert(objs[5] == hwloc_get_obj_by_depth(topology, 4, 36));
assert(objs[6] == hwloc_get_obj_by_depth(topology, 5, 74));
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
int
main (void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set;
hwloc_obj_t obj;
int depth;
int err;
set = hwloc_bitmap_alloc();
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "pack:8 cores:2 1");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
hwloc_bitmap_sscanf(set, "00008f18");
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, NULL);
assert(obj == hwloc_get_obj_by_depth(topology, 1, 1));
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, obj);
assert(obj == hwloc_get_obj_by_depth(topology, 1, 2));
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, obj);
assert(obj == hwloc_get_obj_by_depth(topology, 1, 4));
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, obj);
assert(obj == hwloc_get_obj_by_depth(topology, 1, 5));
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, obj);
assert(obj == hwloc_get_obj_by_depth(topology, 1, 7));
obj = hwloc_get_next_obj_covering_cpuset_by_type(topology, set, HWLOC_OBJ_PACKAGE, obj);
assert(!obj);
hwloc_topology_destroy (topology);
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "nodes:2 pack:5 cores:3 4");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
hwloc_bitmap_sscanf(set, "0ff08000");
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
assert(depth == 2);
obj = hwloc_get_next_obj_covering_cpuset_by_depth(topology, set, depth, NULL);
assert(obj == hwloc_get_obj_by_depth(topology, depth, 1));
obj = hwloc_get_next_obj_covering_cpuset_by_depth(topology, set, depth, obj);
assert(obj == hwloc_get_obj_by_depth(topology, depth, 2));
obj = hwloc_get_next_obj_covering_cpuset_by_depth(topology, set, depth, obj);
assert(!obj);
hwloc_topology_destroy (topology);
hwloc_bitmap_free(set);
return EXIT_SUCCESS;
}
static int
hwloc_solaris_set_sth_cpubind(hwloc_topology_t topology, idtype_t idtype, id_t id, hwloc_const_bitmap_t hwloc_set, int flags)
{
unsigned target_cpu;
/* The resulting binding is always strict */
if (hwloc_bitmap_isequal(hwloc_set, hwloc_topology_get_complete_cpuset(topology))) {
if (processor_bind(idtype, id, PBIND_NONE, NULL) != 0)
return -1;
#ifdef HAVE_LIBLGRP
if (!(flags & HWLOC_CPUBIND_NOMEMBIND)) {
int depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
if (depth >= 0) {
int n = hwloc_get_nbobjs_by_depth(topology, depth);
int i;
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_NONE);
}
}
}
#endif /* HAVE_LIBLGRP */
return 0;
}
#ifdef HAVE_LIBLGRP
if (!(flags & HWLOC_CPUBIND_NOMEMBIND)) {
int depth = hwloc_get_type_depth(topology, HWLOC_OBJ_NUMANODE);
if (depth >= 0) {
int n = hwloc_get_nbobjs_by_depth(topology, depth);
int i;
int ok;
hwloc_bitmap_t target = hwloc_bitmap_alloc();
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
if (hwloc_bitmap_isincluded(obj->cpuset, hwloc_set))
hwloc_bitmap_or(target, target, obj->cpuset);
}
ok = hwloc_bitmap_isequal(target, hwloc_set);
hwloc_bitmap_free(target);
if (ok) {
/* Ok, managed to achieve hwloc_set by just combining NUMA nodes */
for (i = 0; i < n; i++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, depth, i);
if (hwloc_bitmap_isincluded(obj->cpuset, hwloc_set)) {
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_STRONG);
} else {
if (flags & HWLOC_CPUBIND_STRICT)
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_NONE);
else
lgrp_affinity_set(idtype, id, obj->os_index, LGRP_AFF_WEAK);
}
}
return 0;
}
}
}
#endif /* HAVE_LIBLGRP */
if (hwloc_bitmap_weight(hwloc_set) != 1) {
errno = EXDEV;
return -1;
}
target_cpu = hwloc_bitmap_first(hwloc_set);
if (processor_bind(idtype, id,
(processorid_t) (target_cpu), NULL) != 0)
return -1;
return 0;
}
int main(void)
{
hwloc_topology_t topology;
unsigned depth;
unsigned i,j, width;
char buffer[1024];
int err;
/* check a synthetic topology */
hwloc_topology_init(&topology);
err = hwloc_topology_set_synthetic(topology, "2 3 4 5 6");
assert(!err);
hwloc_topology_load(topology);
/* internal checks */
hwloc_topology_check(topology);
/* local checks */
depth = hwloc_topology_get_depth(topology);
assert(depth == 6);
width = 1;
for(i=0; i<6; i++) {
/* check arities */
assert(hwloc_get_nbobjs_by_depth(topology, i) == width);
for(j=0; j<width; j++) {
hwloc_obj_t obj = hwloc_get_obj_by_depth(topology, i, j);
assert(obj);
assert(obj->arity == (i<5 ? i+2 : 0));
}
width *= i+2;
}
err = hwloc_topology_export_synthetic(topology, buffer, sizeof(buffer), 0);
assert(err == 75);
err = strcmp("Package:2 NUMANode:3(memory=1073741824) L2Cache:4(size=4194304) Core:5 PU:6", buffer);
assert(!err);
err = hwloc_topology_export_synthetic(topology, buffer, sizeof(buffer), HWLOC_TOPOLOGY_EXPORT_SYNTHETIC_FLAG_NO_EXTENDED_TYPES|HWLOC_TOPOLOGY_EXPORT_SYNTHETIC_FLAG_NO_ATTRS);
assert(err == 42);
err = strcmp("Package:2 NUMANode:3 L2Cache:4 Core:5 PU:6", buffer);
assert(!err);
hwloc_topology_destroy(topology);
hwloc_topology_init(&topology);
err = hwloc_topology_set_type_filter(topology, HWLOC_OBJ_L1ICACHE, HWLOC_TYPE_FILTER_KEEP_ALL);
err = hwloc_topology_set_synthetic(topology, "pack:2(indexes=3,5) numa:2(memory=256GB indexes=pack) l3u:1(size=20mb) l2:2 l1i:1(size=16kB) l1dcache:2 core:1 pu:2(indexes=l2)");
assert(!err);
hwloc_topology_load(topology);
err = hwloc_topology_export_synthetic(topology, buffer, sizeof(buffer), 0);
assert(err == 181);
err = strcmp("Package:2 NUMANode:2(memory=274877906944 indexes=2*2:1*2) L3Cache:1(size=20971520) L2Cache:2(size=4194304) L1iCache:1(size=16384) L1dCache:2(size=32768) Core:1 PU:2(indexes=4*8:1*4)", buffer);
assert(!err);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PACKAGE, 1)->os_index == 5);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, 1)->os_index == 2);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 12)->os_index == 3);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 13)->os_index == 11);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 14)->os_index == 19);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 15)->os_index == 27);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 16)->os_index == 4);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 17)->os_index == 12);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 18)->os_index == 20);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 19)->os_index == 28);
hwloc_topology_destroy(topology);
hwloc_topology_init(&topology);
err = hwloc_topology_set_synthetic(topology, "pack:2 core:2 pu:2(indexes=0,4,2,6,1,5,3,7)");
assert(!err);
hwloc_topology_load(topology);
err = hwloc_topology_export_synthetic(topology, buffer, sizeof(buffer), 0);
assert(err == 72);
err = strcmp("NUMANode:1(memory=1073741824) Package:2 Core:2 PU:2(indexes=4*2:2*2:1*2)", buffer);
assert(!err);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0)->os_index == 0);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 1)->os_index == 4);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 2)->os_index == 2);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 3)->os_index == 6);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 4)->os_index == 1);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 5)->os_index == 5);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 6)->os_index == 3);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 7)->os_index == 7);
hwloc_topology_destroy(topology);
hwloc_topology_init(&topology);
err = hwloc_topology_set_synthetic(topology, "pack:2 numa:2 core:1 pu:2(indexes=0,4,2,6,1,3,5,7)");
assert(!err);
hwloc_topology_load(topology);
err = hwloc_topology_export_synthetic(topology, buffer, sizeof(buffer), 0);
assert(err == 76);
err = strcmp("Package:2 NUMANode:2(memory=1073741824) Core:1 PU:2(indexes=0,4,2,6,1,3,5,7)", buffer);
assert(!err);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0)->os_index == 0);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 1)->os_index == 4);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 2)->os_index == 2);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 3)->os_index == 6);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 4)->os_index == 1);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 5)->os_index == 3);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 6)->os_index == 5);
assert(hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 7)->os_index == 7);
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology;
#ifdef HWLOC_HAVE_CPU_SET
unsigned depth;
hwloc_bitmap_t hwlocset;
cpu_set_t schedset;
hwloc_obj_t obj;
int err;
#endif /* HWLOC_HAVE_CPU_SET */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
#ifdef HWLOC_HAVE_CPU_SET
depth = hwloc_topology_get_depth(topology);
hwlocset = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset(topology));
hwloc_cpuset_to_glibc_sched_affinity(topology, hwlocset, &schedset, sizeof(schedset));
#ifdef HWLOC_HAVE_OLD_SCHED_SETAFFINITY
err = sched_setaffinity(0, sizeof(schedset));
#else
err = sched_setaffinity(0, sizeof(schedset), &schedset);
#endif
assert(!err);
hwloc_bitmap_free(hwlocset);
#ifdef HWLOC_HAVE_OLD_SCHED_SETAFFINITY
err = sched_getaffinity(0, sizeof(schedset));
#else
err = sched_getaffinity(0, sizeof(schedset), &schedset);
#endif
assert(!err);
hwlocset = hwloc_bitmap_alloc();
hwloc_cpuset_from_glibc_sched_affinity(topology, hwlocset, &schedset, sizeof(schedset));
assert(hwloc_bitmap_isincluded(hwlocset, hwloc_topology_get_complete_cpuset(topology)));
hwloc_bitmap_andnot(hwlocset, hwlocset, hwloc_topology_get_online_cpuset(topology));
hwloc_bitmap_andnot(hwlocset, hwlocset, hwloc_topology_get_allowed_cpuset(topology));
assert(hwloc_bitmap_iszero(hwlocset));
hwloc_bitmap_free(hwlocset);
obj = hwloc_get_obj_by_depth(topology, depth-1, hwloc_get_nbobjs_by_depth(topology, depth-1) - 1);
assert(obj);
assert(obj->type == HWLOC_OBJ_PU);
hwlocset = hwloc_bitmap_dup(obj->cpuset);
hwloc_cpuset_to_glibc_sched_affinity(topology, hwlocset, &schedset, sizeof(schedset));
#ifdef HWLOC_HAVE_OLD_SCHED_SETAFFINITY
err = sched_setaffinity(0, sizeof(schedset));
#else
err = sched_setaffinity(0, sizeof(schedset), &schedset);
#endif
assert(!err);
hwloc_bitmap_free(hwlocset);
#ifdef HWLOC_HAVE_OLD_SCHED_SETAFFINITY
err = sched_getaffinity(0, sizeof(schedset));
#else
err = sched_getaffinity(0, sizeof(schedset), &schedset);
#endif
assert(!err);
hwlocset = hwloc_bitmap_alloc();
hwloc_cpuset_from_glibc_sched_affinity(topology, hwlocset, &schedset, sizeof(schedset));
assert(hwloc_bitmap_isequal(hwlocset, obj->cpuset));
hwloc_bitmap_free(hwlocset);
#endif /* HWLOC_HAVE_CPU_SET */
hwloc_topology_destroy(topology);
return 0;
}
/*********************************
* Function Defintions
*********************************/
int rmaps_lama_build_max_tree(orte_job_t *jdata, opal_list_t *node_list,
opal_tree_t * max_tree, bool *is_homogeneous)
{
int ret;
opal_tree_t *tmp_tree = NULL;
hwloc_topology_t topo, *last_topo = NULL;
orte_node_t *cur_node = NULL;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ---------------------------------");
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Building the Max Tree...");
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ---------------------------------");
/*
* Assume homogeneous system, unless otherwise noted
*/
*is_homogeneous = true;
/*
* Process all other unique trees from remote daemons who are in
* this allocation
*/
for(cur_node = (orte_node_t*)opal_list_get_first(node_list);
cur_node != (orte_node_t*)opal_list_get_end(node_list);
cur_node = (orte_node_t*)opal_list_get_next(cur_node) ) {
if (NULL == (topo = cur_node->topology)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- No Tree Available: %s (skipping)", cur_node->name);
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Converting Remote Tree: %s", cur_node->name);
/*
* Convert to opal_tree
*/
tmp_tree = rmaps_lama_create_empty_max_tree();
rmaps_lama_convert_hwloc_tree_to_opal_tree(tmp_tree, &topo);
if( 11 <= opal_output_get_verbosity(orte_rmaps_base_framework.framework_output) ) {
rmaps_lama_max_tree_pretty_print_tree(tmp_tree);
}
/*
* Compare the current and last topologies if we are still considering
* this max tree to represent a homogeneous system.
*/
if( *is_homogeneous ) {
if( NULL == last_topo ) {
last_topo = &topo;
} else {
if( 0 != rmaps_lama_hwloc_compare_topos(last_topo, &topo) ) {
*is_homogeneous = false;
}
}
}
/*
* Prune the input tree so that is only contains levels that the user
* asked for.
*/
if( 11 <= opal_output_get_verbosity(orte_rmaps_base_framework.framework_output) ) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ---------------------------------");
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Pruning input Tree...");
}
if( ORTE_SUCCESS != (ret = rmaps_lama_prune_max_tree(tmp_tree, opal_tree_get_root(tmp_tree))) ) {
return ret;
}
if( 11 <= opal_output_get_verbosity(orte_rmaps_base_framework.framework_output) ) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Input Tree... - Post Prune");
rmaps_lama_max_tree_pretty_print_tree(tmp_tree);
}
/*
* Merge into max_tree
*/
if( opal_tree_is_empty(max_tree) ) {
opal_tree_dup(tmp_tree, max_tree);
} else {
if( ORTE_SUCCESS != (ret = rmaps_lama_merge_trees(tmp_tree,
max_tree,
opal_tree_get_root(tmp_tree),
opal_tree_get_root(max_tree) ))) {
return ret;
}
}
/*
* Release and move on...
*/
OBJ_RELEASE(tmp_tree);
tmp_tree = NULL;
}
/*
* Fill out the MPPR accounting information for each node
*/
for(cur_node = (orte_node_t*)opal_list_get_first(node_list);
cur_node != (orte_node_t*)opal_list_get_end(node_list);
cur_node = (orte_node_t*)opal_list_get_next(cur_node) ) {
if( ORTE_SUCCESS != (ret = rmaps_lama_annotate_node_for_mppr(cur_node,
hwloc_get_obj_by_depth(cur_node->topology, 0, 0))) ) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/*
* JJH: NEEDS TESTING
* Note: This check is in place, but not used at the moment due to lack of
* system availability. Pending system availability and further testing,
* just assume heterogeneous.
*/
*is_homogeneous = false;
/*
* Display the final Max Tree
*/
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ----- Final Max Tree... - %s system",
(*is_homogeneous ? "Homogeneous" : "Heterogeneous") );
if( 11 <= opal_output_get_verbosity(orte_rmaps_base_framework.framework_output) ) {
rmaps_lama_max_tree_pretty_print_tree(max_tree);
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:lama: ---------------------------------");
return ORTE_SUCCESS;
}
int bind_myself_to_core(hwloc_topology_t topology, int id){
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
char *str;
int binding_res;
int depth = hwloc_topology_get_depth(topology);
int nb_cores = hwloc_get_nbobjs_by_depth(topology, depth-1);
int my_core;
int nb_threads = get_nb_threads();
/* printf("depth=%d\n",depth); */
switch (mapping_policy){
case SCATTER:
my_core = id*(nb_cores/nb_threads);
break;
default:
if(verbose_level>=WARNING){
printf("Wrong scheduling policy. Using COMPACT\n");
}
case COMPACT:
my_core = id%nb_cores;
}
if(verbose_level>=INFO){
printf("Mapping thread %d on core %d\n",id,my_core);
}
/* Get my core. */
obj = hwloc_get_obj_by_depth(topology, depth-1, my_core);
if (obj) {
/* Get a copy of its cpuset that we may modify. */
cpuset = hwloc_bitmap_dup(obj->cpuset);
/* Get only one logical processor (in case the core is
SMT/hyperthreaded). */
hwloc_bitmap_singlify(cpuset);
/*hwloc_bitmap_asprintf(&str, cpuset);
printf("Binding thread %d to cpuset %s\n", my_core,str);
FREE(str);
*/
/* And try to bind ourself there. */
binding_res = hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD);
if (binding_res == -1){
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
if(verbose_level>=WARNING)
printf("Thread %d couldn't bind to cpuset %s: %s.\n This thread is not bound to any core...\n", my_core, str, strerror(error));
free(str); /* str is allocated by hlwoc, free it normally*/
return 0;
}
/* FREE our cpuset copy */
hwloc_bitmap_free(cpuset);
return 1;
}else{
if(verbose_level>=WARNING)
printf("No valid object for core id %d!\n",my_core);
return 0;
}
}
int
main (void)
{
hwloc_topology_t topology;
hwloc_obj_t obj, root;
int err;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
hwloc_topology_set_synthetic (topology, "nodes:2 sockets:3 caches:4 cores:5 6");
err = hwloc_topology_load (topology);
if (err)
return EXIT_FAILURE;
/* there is no second system object */
root = hwloc_get_root_obj (topology);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, 1);
assert(!obj);
/* first system object is the top-level object of the topology */
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, 0);
assert(obj == hwloc_get_root_obj(topology));
/* first next-object object is the top-level object of the topology */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_MACHINE, NULL);
assert(obj == hwloc_get_root_obj(topology));
/* there is no next object after the system object */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SYSTEM, obj);
assert(!obj);
/* check last PU */
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 2*3*4*5*6-1);
assert(obj == hwloc_get_obj_by_depth(topology, 5, 2*3*4*5*6-1));
/* there is no next PU after the last one */
obj = hwloc_get_next_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, obj);
assert(!obj);
/* check there are 20 cores inside first socket */
root = hwloc_get_obj_by_depth(topology, 2, 0);
assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE) == 20);
/* check there are 12 caches inside last node */
root = hwloc_get_obj_by_depth(topology, 1, 1);
assert(hwloc_get_nbobjs_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CACHE) == 12);
/* check first PU of second socket */
root = hwloc_get_obj_by_depth(topology, 2, 1);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_PU, 0);
assert(obj == hwloc_get_obj_by_depth(topology, 5, 4*5*6));
/* check third core of third socket */
root = hwloc_get_obj_by_depth(topology, 2, 2);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_CORE, 2);
assert(obj == hwloc_get_obj_by_depth(topology, 4, 2*4*5+2));
/* check first socket of second node */
root = hwloc_get_obj_by_depth(topology, 1, 1);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_SOCKET, 0);
assert(obj == hwloc_get_obj_by_depth(topology, 2, 3));
/* there is no node inside sockets */
root = hwloc_get_obj_by_depth(topology, 2, 0);
obj = hwloc_get_obj_inside_cpuset_by_type(topology, root->cpuset, HWLOC_OBJ_NODE, 0);
assert(!obj);
hwloc_topology_destroy (topology);
return EXIT_SUCCESS;
}
