/* user to have to play with the cgroup hierarchy to modify it */
extern int task_cgroup_cpuset_set_task_affinity(stepd_step_rec_t *job)
{
int fstatus = SLURM_ERROR;
#ifndef HAVE_HWLOC
error("task/cgroup: plugin not compiled with hwloc support, "
"skipping affinity.");
return fstatus;
#else
char mstr[1 + CPU_SETSIZE / 4];
cpu_bind_type_t bind_type;
cpu_set_t ts;
hwloc_obj_t obj;
hwloc_obj_type_t socket_or_node;
hwloc_topology_t topology;
hwloc_bitmap_t cpuset;
hwloc_obj_type_t hwtype;
hwloc_obj_type_t req_hwtype;
int bind_verbose = 0;
int rc = SLURM_SUCCESS, match;
pid_t pid = job->envtp->task_pid;
size_t tssize;
uint32_t nldoms;
uint32_t nsockets;
uint32_t ncores;
uint32_t npus;
uint32_t nobj;
uint32_t taskid = job->envtp->localid;
uint32_t jntasks = job->node_tasks;
uint32_t jnpus;
/* Allocate and initialize hwloc objects */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
cpuset = hwloc_bitmap_alloc();
int spec_threads = 0;
if (job->batch) {
jnpus = job->cpus;
job->cpus_per_task = job->cpus;
} else
jnpus = jntasks * job->cpus_per_task;
bind_type = job->cpu_bind_type;
if ((conf->task_plugin_param & CPU_BIND_VERBOSE) ||
(bind_type & CPU_BIND_VERBOSE))
bind_verbose = 1 ;
if ( hwloc_get_type_depth(topology, HWLOC_OBJ_NODE) >
hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET) ) {
/* One socket contains multiple NUMA-nodes
* like AMD Opteron 6000 series etc.
* In such case, use NUMA-node instead of socket. */
socket_or_node = HWLOC_OBJ_NODE;
} else {
socket_or_node = HWLOC_OBJ_SOCKET;
}
if (bind_type & CPU_BIND_NONE) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting no affinity",
taskid);
return 0;
} else if (bind_type & CPU_BIND_TO_THREADS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"thread level binding",taskid);
req_hwtype = HWLOC_OBJ_PU;
} else if (bind_type & CPU_BIND_TO_CORES) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"core level binding",taskid);
req_hwtype = HWLOC_OBJ_CORE;
} else if (bind_type & CPU_BIND_TO_SOCKETS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"socket level binding",taskid);
req_hwtype = socket_or_node;
} else if (bind_type & CPU_BIND_TO_LDOMS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"ldom level binding",taskid);
req_hwtype = HWLOC_OBJ_NODE;
} else if (bind_type & CPU_BIND_TO_BOARDS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"board level binding",taskid);
req_hwtype = HWLOC_OBJ_GROUP;
} else if (bind_type & bind_mode_ldom) {
req_hwtype = HWLOC_OBJ_NODE;
} else {
if (bind_verbose)
info("task/cgroup: task[%u] using core level binding"
" by default",taskid);
req_hwtype = HWLOC_OBJ_CORE;
}
/*
* Perform the topology detection. It will only get allowed PUs.
* Detect in the same time the granularity to use for binding.
* The granularity can be relaxed from threads to cores if enough
* cores are available as with hyperthread support, ntasks-per-core
* param can let us have access to more threads per core for each
* task
* Revert back to machine granularity if no finer-grained granularity
* matching the request is found. This will result in no affinity
* applied.
* The detected granularity will be used to find where to best place
* the task, then the cpu_bind option will be used to relax the
* affinity constraint and use more PUs. (i.e. use a core granularity
* to dispatch the tasks across the sockets and then provide access
* to each task to the cores of its socket.)
*/
npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
socket_or_node);
nldoms = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_NODE);
//info("PU:%d CORE:%d SOCK:%d LDOM:%d", npus, ncores, nsockets, nldoms);
hwtype = HWLOC_OBJ_MACHINE;
nobj = 1;
if ((job->job_core_spec != (uint16_t) NO_VAL) &&
(job->job_core_spec & CORE_SPEC_THREAD) &&
(job->job_core_spec != CORE_SPEC_THREAD)) {
spec_threads = job->job_core_spec & (~CORE_SPEC_THREAD);
}
if (npus >= (jnpus + spec_threads) || bind_type & CPU_BIND_TO_THREADS) {
hwtype = HWLOC_OBJ_PU;
nobj = npus;
}
if (ncores >= jnpus || bind_type & CPU_BIND_TO_CORES) {
hwtype = HWLOC_OBJ_CORE;
nobj = ncores;
}
if (nsockets >= jntasks &&
bind_type & CPU_BIND_TO_SOCKETS) {
hwtype = socket_or_node;
nobj = nsockets;
}
/*
* HWLOC returns all the NUMA nodes available regardless of the
* number of underlying sockets available (regardless of the allowed
* resources). So there is no guarantee that each ldom will be populated
* with usable sockets. So add a simple check that at least ensure that
* we have as many sockets as ldoms before moving to ldoms granularity
*/
if (nldoms >= jntasks &&
nsockets >= nldoms &&
bind_type & (CPU_BIND_TO_LDOMS | bind_mode_ldom)) {
hwtype = HWLOC_OBJ_NODE;
nobj = nldoms;
}
/*
* If not enough objects to do the job, revert to no affinity mode
*/
if (hwloc_compare_types(hwtype, HWLOC_OBJ_MACHINE) == 0) {
info("task/cgroup: task[%u] disabling affinity because of %s "
"granularity",taskid, hwloc_obj_type_string(hwtype));
} else if ((hwloc_compare_types(hwtype, HWLOC_OBJ_CORE) >= 0) &&
(nobj < jnpus)) {
info("task/cgroup: task[%u] not enough %s objects (%d < %d), "
"disabling affinity",
taskid, hwloc_obj_type_string(hwtype), nobj, jnpus);
} else if (bind_type & bind_mode) {
/* Explicit binding mode specified by the user
* Bind the taskid in accordance with the specified mode
*/
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE, 0);
match = hwloc_bitmap_isequal(obj->complete_cpuset,
obj->allowed_cpuset);
if ((job->job_core_spec == (uint16_t) NO_VAL) && !match) {
info("task/cgroup: entire node must be allocated, "
"disabling affinity, task[%u]", taskid);
fprintf(stderr, "Requested cpu_bind option requires "
"entire node to be allocated; disabling "
"affinity\n");
} else {
if (bind_verbose) {
info("task/cgroup: task[%u] is requesting "
"explicit binding mode", taskid);
}
_get_sched_cpuset(topology, hwtype, req_hwtype, &ts,
job);
tssize = sizeof(cpu_set_t);
fstatus = SLURM_SUCCESS;
if (job->job_core_spec != (uint16_t) NO_VAL)
_validate_mask(taskid, obj, &ts);
if ((rc = sched_setaffinity(pid, tssize, &ts))) {
error("task/cgroup: task[%u] unable to set "
"mask 0x%s", taskid,
cpuset_to_str(&ts, mstr));
error("sched_setaffinity rc = %d", rc);
fstatus = SLURM_ERROR;
} else if (bind_verbose) {
info("task/cgroup: task[%u] mask 0x%s",
taskid, cpuset_to_str(&ts, mstr));
}
_slurm_chkaffinity(&ts, job, rc);
}
} else {
/* Bind the detected object to the taskid, respecting the
* granularity, using the designated or default distribution
* method (block or cyclic). */
char *str;
if (bind_verbose) {
info("task/cgroup: task[%u] using %s granularity dist %u",
taskid, hwloc_obj_type_string(hwtype),
job->task_dist);
}
/* See srun man page for detailed information on --distribution
* option.
*
* You can see the equivalent code for the
* task/affinity plugin in
* src/plugins/task/affinity/dist_tasks.c, around line 368
*/
switch (job->task_dist & SLURM_DIST_NODESOCKMASK) {
case SLURM_DIST_BLOCK_BLOCK:
case SLURM_DIST_CYCLIC_BLOCK:
case SLURM_DIST_PLANE:
/* tasks are distributed in blocks within a plane */
_task_cgroup_cpuset_dist_block(topology,
hwtype, req_hwtype,
nobj, job, bind_verbose, cpuset);
break;
case SLURM_DIST_ARBITRARY:
case SLURM_DIST_BLOCK:
case SLURM_DIST_CYCLIC:
case SLURM_DIST_UNKNOWN:
if (slurm_get_select_type_param()
& CR_CORE_DEFAULT_DIST_BLOCK) {
_task_cgroup_cpuset_dist_block(topology,
hwtype, req_hwtype,
nobj, job, bind_verbose, cpuset);
break;
}
/* We want to fall through here if we aren't doing a
default dist block.
*/
default:
_task_cgroup_cpuset_dist_cyclic(topology,
hwtype, req_hwtype,
job, bind_verbose, cpuset);
break;
}
hwloc_bitmap_asprintf(&str, cpuset);
tssize = sizeof(cpu_set_t);
if (hwloc_cpuset_to_glibc_sched_affinity(topology, cpuset,
&ts, tssize) == 0) {
fstatus = SLURM_SUCCESS;
if ((rc = sched_setaffinity(pid, tssize, &ts))) {
error("task/cgroup: task[%u] unable to set "
"taskset '%s'", taskid, str);
fstatus = SLURM_ERROR;
} else if (bind_verbose) {
info("task/cgroup: task[%u] set taskset '%s'",
taskid, str);
}
_slurm_chkaffinity(&ts, job, rc);
} else {
error("task/cgroup: task[%u] unable to build "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
}
free(str);
}
/* Destroy hwloc objects */
hwloc_bitmap_free(cpuset);
hwloc_topology_destroy(topology);
return fstatus;
#endif
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set, set2;
hwloc_const_bitmap_t cset_available, cset_all;
hwloc_obj_t obj;
char *buffer;
char type[64];
unsigned i;
int err;
/* create a topology */
err = hwloc_topology_init(&topology);
if (err < 0) {
fprintf(stderr, "failed to initialize the topology\n");
return EXIT_FAILURE;
}
err = hwloc_topology_load(topology);
if (err < 0) {
fprintf(stderr, "failed to load the topology\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* retrieve the entire set of available PUs */
cset_available = hwloc_topology_get_topology_cpuset(topology);
/* retrieve the CPU binding of the current entire process */
set = hwloc_bitmap_alloc();
if (!set) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
if (err < 0) {
fprintf(stderr, "failed to get cpu binding\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
}
/* display the processing units that cannot be used by this process */
if (hwloc_bitmap_isequal(set, cset_available)) {
printf("this process can use all available processing units in the system\n");
} else {
/* compute the set where we currently cannot run.
* we can't modify cset_available because it's a system read-only one,
* so we do set = available &~ set
*/
hwloc_bitmap_andnot(set, cset_available, set);
hwloc_bitmap_asprintf(&buffer, set);
printf("process cannot use %d process units (%s) among %u in the system\n",
hwloc_bitmap_weight(set), buffer, hwloc_bitmap_weight(cset_available));
free(buffer);
/* restore set where it was before the &~ operation above */
hwloc_bitmap_andnot(set, cset_available, set);
}
/* print the smallest object covering the current process binding */
obj = hwloc_get_obj_covering_cpuset(topology, set);
hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
printf("process is bound within object %s logical index %u\n", type, obj->logical_index);
/* retrieve the single PU where the current thread actually runs within this process binding */
set2 = hwloc_bitmap_alloc();
if (!set2) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* sanity checks that are not actually needed but help the reader */
/* this thread runs within the process binding */
assert(hwloc_bitmap_isincluded(set2, set));
/* this thread runs on a single PU at a time */
assert(hwloc_bitmap_weight(set2) == 1);
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is now running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
/* migrate this single thread to where other PUs within the current binding */
hwloc_bitmap_andnot(set2, set, set2);
err = hwloc_set_cpubind(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to set thread binding\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* reprint the PU where that thread runs */
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
/* retrieve the entire set of all PUs */
cset_all = hwloc_topology_get_complete_cpuset(topology);
if (hwloc_bitmap_isequal(cset_all, cset_available)) {
printf("all hardware PUs are available\n");
} else {
printf("only %d hardware PUs are available in the machine among %d\n",
hwloc_bitmap_weight(cset_available), hwloc_bitmap_weight(cset_all));
}
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
int main(void)
{
hwloc_topology_t topology;
int i;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
for(i=0; ; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, ancestor;
const char *value;
osdev = hwloc_intel_mic_get_device_osdev_by_index(topology, i);
if (!osdev)
break;
assert(osdev);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "mic", 3);
assert(!err);
assert(atoi(osdev->name+3) == (int) i);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "MIC");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "MICFamily");
printf("found MICFamily %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICSKU");
printf("found MICSKU %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICActiveCores");
printf("found MICActiveCores %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICMemorySize");
printf("found MICMemorySize %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_intel_mic_get_device_cpuset(topology, i, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
unsigned p, d;
osdev = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
/* we currently insert all OpenCL devices, except CPU devices */
if (!osdev) {
cl_device_type type;
clGetDeviceInfo(device_ids[j], CL_DEVICE_TYPE, sizeof(type), &type, NULL);
assert(type == CL_DEVICE_TYPE_CPU);
continue;
}
/* try to get it from PCI locality (only works with AMD extensions) */
osdev2 = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
if (osdev2) {
assert(osdev == osdev2);
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %u device %u\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %u device %u\n", cpuset_string, i, j);
free(cpuset_string);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = sscanf(osdev->name, "opencl%ud%u", &p, &d);
assert(err == 2);
assert(p == i);
assert(d == j);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = osdev->subtype;
assert(value);
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
int loaded = 0;
int depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int use_nodeset = 0;
int get_last_cpu_location = 0;
unsigned long flags = 0;
int force = 0;
int single = 0;
int verbose = 0;
int only_hbm = -1;
int logical = 1;
int taskset = 0;
unsigned cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int got_mempolicy = 0;
unsigned membind_flags = 0;
int opt;
int ret;
int pid_number = -1;
int tid_number = -1;
hwloc_pid_t pid = 0; /* only valid when pid_number > 0, but gcc-4.8 still reports uninitialized warnings */
char *callname;
struct hwloc_calc_location_context_s lcontext;
struct hwloc_calc_set_context_s scontext;
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
hwloc_utils_check_api_version(callname);
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
/* don't load now, in case some options change the config before the topology is actually used */
#define LOADED() (loaded)
#define ENSURE_LOADED() do { \
if (!loaded) { \
hwloc_topology_init(&topology); \
hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_ALL); \
hwloc_topology_set_flags(topology, flags); \
hwloc_topology_load(topology); \
depth = hwloc_topology_get_depth(topology); \
loaded = 1; \
} \
} while (0)
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
#ifdef HWLOC_LINUX_SYS
if (!strcmp(argv[0], "--tid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
tid_number = atoi(argv[1]);
opt = 1;
goto next;
}
#endif
if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, HWLOC_VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
if (!strcmp (argv[0], "--nodeset")) {
use_nodeset = 1;
goto next;
}
if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
got_mempolicy = 1;
opt = 1;
goto next;
}
if (!strcmp(argv[0], "--hbm")) {
only_hbm = 1;
goto next;
}
if (!strcmp(argv[0], "--no-hbm")) {
only_hbm = 0;
goto next;
}
if (!strcmp (argv[0], "--whole-system")) {
if (loaded) {
fprintf(stderr, "Input option %s disallowed after options using the topology\n", argv[0]);
exit(EXIT_FAILURE);
}
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
goto next;
}
if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
ENSURE_LOADED();
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* FALLTHRU */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ENSURE_LOADED();
lcontext.topology = topology;
lcontext.topodepth = depth;
lcontext.only_hbm = only_hbm;
lcontext.logical = logical;
lcontext.verbose = verbose;
scontext.nodeset_input = use_nodeset;
scontext.nodeset_output = working_on_cpubind ? 0 : 1;
scontext.output_set = working_on_cpubind ? cpubind_set : membind_set;
ret = hwloc_calc_process_location_as_set(&lcontext, &scontext, argv[0]);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
ENSURE_LOADED();
if (pid_number > 0 && tid_number > 0) {
fprintf(stderr, "cannot operate both on tid and pid\n");
return EXIT_FAILURE;
}
if (pid_number > 0) {
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
/* no need to set_pid()
* the doc just says we're operating on pid, not that we're retrieving the topo/cpuset as seen from inside pid
*/
}
if (get_last_cpu_location && !working_on_cpubind) {
fprintf(stderr, "Options --membind and --get-last-cpu-location cannot be combined.\n");
return EXIT_FAILURE;
}
if ((get_binding || get_last_cpu_location) && (got_cpubind || got_membind)) {
/* doesn't work because get_binding/get_last_cpu_location overwrites cpubind_set */
fprintf(stderr, "Cannot display and set binding at the same time.\n");
return EXIT_FAILURE;
}
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number > 0)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
#ifdef HWLOC_LINUX_SYS
else if (tid_number > 0)
err = hwloc_linux_get_tid_last_cpu_location(topology, tid_number, cpubind_set);
#endif
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number > 0)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
#ifdef HWLOC_LINUX_SYS
else if (tid_number > 0)
err = hwloc_linux_get_tid_cpubind(topology, tid_number, cpubind_set);
#endif
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else if (tid_number > 0)
fprintf(stderr, "hwloc_get_tid_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", tid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (use_nodeset) {
hwloc_bitmap_t nset = hwloc_bitmap_alloc();
hwloc_cpuset_to_nodeset(topology, cpubind_set, nset);
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, nset);
else
hwloc_bitmap_asprintf(&s, nset);
hwloc_bitmap_free(nset);
} else {
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
}
} else {
hwloc_membind_policy_t policy;
if (pid_number > 0) {
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, use_nodeset ? HWLOC_MEMBIND_BYNODESET : 0);
} else if (tid_number > 0) {
err = -1; errno = ENOSYS;
} else {
err = hwloc_get_membind(topology, membind_set, &policy, use_nodeset ? HWLOC_MEMBIND_BYNODESET : 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number > 0)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags | HWLOC_MEMBIND_BYNODESET);
else if (tid_number > 0) {
ret = -1; errno = ENOSYS;
} else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags | HWLOC_MEMBIND_BYNODESET);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_membind %s (policy %d flags %x) PID %d failed (errno %d %s)\n",
s, membind_policy, membind_flags, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s (policy %d flags %x) failed (errno %d %s)\n",
s, membind_policy, membind_flags, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
} else {
if (got_mempolicy)
fprintf(stderr, "--mempolicy ignored unless memory binding is also requested with --membind.\n");
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (got_membind && !hwloc_bitmap_isequal(membind_set, cpubind_set)) {
if (verbose)
fprintf(stderr, "Conflicting CPU and memory binding requested, adding HWLOC_CPUBIND_NOMEMBIND flag.\n");
cpubind_flags |= HWLOC_CPUBIND_NOMEMBIND;
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number > 0)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
#ifdef HWLOC_LINUX_SYS
else if (tid_number > 0)
ret = hwloc_linux_set_tid_cpubind(topology, tid_number, cpubind_set);
#endif
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_cpubind %s (flags %x) PID %d failed (errno %d %s)\n",
s, cpubind_flags, pid_number, bind_errno, errmsg);
else if (tid_number > 0)
fprintf(stderr, "hwloc_set_tid_cpubind %s (flags %x) PID %d failed (errno %d %s)\n",
s, cpubind_flags, tid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s (flags %x) failed (errno %d %s)\n",
s, cpubind_flags, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number > 0 || tid_number > 0)
return EXIT_SUCCESS;
if (0 == argc) {
if (get_binding || get_last_cpu_location)
return EXIT_SUCCESS;
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
/* FIXME: check whether Windows execvp() passes INHERIT_PARENT_AFFINITY to CreateProcess()
* because we need to propagate processor group affinity. However process-wide affinity
* isn't supported with processor groups so far.
*/
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
osdev = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
osdev2 = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
assert(osdev == osdev2);
if (!osdev) {
printf("no osdev for platform %d device %d\n", i, j);
continue;
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %d device %d\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %d device %d\n", cpuset_string, i, j);
free(cpuset_string);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "opencl", 6);
assert(!err);
assert(atoi(osdev->name+6) == (int) count);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "Name");
printf("found OSDev name %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
unsigned p, d;
osdev = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
osdev2 = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
assert(osdev == osdev2);
if (!osdev) {
printf("no osdev for platform %u device %u\n", i, j);
continue;
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %u device %u\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %u device %u\n", cpuset_string, i, j);
free(cpuset_string);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = sscanf(osdev->name, "opencl%ud%u", &p, &d);
assert(err == 2);
assert(p == i);
assert(d == j);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
int main(void)
{
int depth;
unsigned i, n;
unsigned long size;
int levels;
char string[128];
int topodepth;
void *m;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
/* Allocate and initialize topology object. */
hwloc_topology_init(&topology);
/* ... Optionally, put detection configuration here to ignore
some objects types, define a synthetic topology, etc....
The default is to detect all the objects of the machine that
the caller is allowed to access. See Configure Topology
Detection. */
/* Perform the topology detection. */
hwloc_topology_load(topology);
/* Optionally, get some additional topology information
in case we need the topology depth later. */
topodepth = hwloc_topology_get_depth(topology);
/*****************************************************************
* First example:
* Walk the topology with an array style, from level 0 (always
* the system level) to the lowest level (always the proc level).
*****************************************************************/
for (depth = 0; depth < topodepth; depth++) {
printf("*** Objects at level %d\n", depth);
for (i = 0; i < hwloc_get_nbobjs_by_depth(topology, depth);
i++) {
hwloc_obj_type_snprintf(string, sizeof(string),
hwloc_get_obj_by_depth(topology, depth, i), 0);
printf("Index %u: %s\n", i, string);
}
}
/*****************************************************************
* Second example:
* Walk the topology with a tree style.
*****************************************************************/
printf("*** Printing overall tree\n");
print_children(topology, hwloc_get_root_obj(topology), 0);
/*****************************************************************
* Third example:
* Print the number of packages.
*****************************************************************/
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
printf("*** The number of packages is unknown\n");
} else {
printf("*** %u package(s)\n",
hwloc_get_nbobjs_by_depth(topology, depth));
}
/*****************************************************************
* Fourth example:
* Compute the amount of cache that the first logical processor
* has above it.
*****************************************************************/
levels = 0;
size = 0;
for (obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0);
obj;
obj = obj->parent)
if (obj->type == HWLOC_OBJ_CACHE) {
levels++;
size += obj->attr->cache.size;
}
printf("*** Logical processor 0 has %d caches totaling %luKB\n",
levels, size / 1024);
/*****************************************************************
* Fifth example:
* Bind to only one thread of the last core of the machine.
*
* First find out where cores are, or else smaller sets of CPUs if
* the OS doesn't have the notion of a "core".
*****************************************************************/
depth = hwloc_get_type_or_below_depth(topology, HWLOC_OBJ_CORE);
/* Get last core. */
obj = hwloc_get_obj_by_depth(topology, depth,
hwloc_get_nbobjs_by_depth(topology, depth) - 1);
if (obj) {
/* Get a copy of its cpuset that we may modify. */
cpuset = hwloc_bitmap_dup(obj->cpuset);
/* Get only one logical processor (in case the core is
SMT/hyper-threaded). */
hwloc_bitmap_singlify(cpuset);
/* And try to bind ourself there. */
if (hwloc_set_cpubind(topology, cpuset, 0)) {
char *str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
free(str);
}
/* Free our cpuset copy */
hwloc_bitmap_free(cpuset);
}
/*****************************************************************
* Sixth example:
* Allocate some memory on the last NUMA node, bind some existing
* memory to the last NUMA node.
*****************************************************************/
/* Get last node. There's always at least one. */
n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE);
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, n - 1);
size = 1024*1024;
m = hwloc_alloc_membind_nodeset(topology, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
hwloc_free(topology, m, size);
m = malloc(size);
hwloc_set_area_membind_nodeset(topology, m, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
free(m);
/* Destroy topology object. */
hwloc_topology_destroy(topology);
return 0;
}
PASTIX_INT sopalin_bindthread(PASTIX_INT cpu)
{
#ifdef MARCEL
{
marcel_vpset_t vpset = MARCEL_VPSET_ZERO;
marcel_vpset_vp(&vpset, cpu);
marcel_apply_vpset(&vpset);
}
#else /* Dans les autres cas on se preoccupe de l'archi */
#ifdef WITH_HWLOC
{
hwloc_topology_t topology; /* Topology object */
hwloc_obj_t obj; /* Hwloc object */
hwloc_cpuset_t cpuset; /* HwLoc cpuset */
/* Allocate and initialize topology object. */
hwloc_topology_init(&topology);
/* Perform the topology detection. */
hwloc_topology_load(topology);
/* Get last one. */
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_CORE, cpu);
if (!obj)
return 0;
/* Get a copy of its cpuset that we may modify. */
/* Get only one logical processor (in case the core is SMT/hyperthreaded). */
#if !defined(HWLOC_BITMAP_H)
cpuset = hwloc_cpuset_dup(obj->cpuset);
hwloc_cpuset_singlify(cpuset);
#else
cpuset = hwloc_bitmap_dup(obj->cpuset);
hwloc_bitmap_singlify(cpuset);
#endif
/* And try to bind ourself there. */
if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD)) {
char *str = NULL;
#if !defined(HWLOC_BITMAP_H)
hwloc_cpuset_asprintf(&str, obj->cpuset);
#else
hwloc_bitmap_asprintf(&str, obj->cpuset);
#endif
printf("Couldn't bind to cpuset %s\n", str);
free(str);
}
/* Get the number at Proc level */
cpu = obj->children[0]->os_index;
/* Free our cpuset copy */
#if !defined(HWLOC_BITMAP_H)
hwloc_cpuset_free(cpuset);
#else
hwloc_bitmap_free(cpuset);
#endif
/* Destroy topology object. */
hwloc_topology_destroy(topology);
}
#else /* WITH_HWLOC */
#ifdef X_ARCHpower_ibm_aix
{
tid_t self_ktid = thread_self ();
bindprocessor(BINDTHREAD, self_ktid, cpu);
}
#elif (defined X_ARCHalpha_compaq_osf1)
{
bind_to_cpu_id(getpid(), cpu, 0);
}
#elif (defined X_ARCHi686_pc_linux)
#ifndef X_ARCHi686_mac
{
cpu_set_t mask;
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
#ifdef HAVE_OLD_SCHED_SETAFFINITY
if(sched_setaffinity(0,&mask) < 0)
#else /* HAVE_OLD_SCHED_SETAFFINITY */
if(sched_setaffinity(0,sizeof(mask),&mask) < 0)
#endif /* HAVE_OLD_SCHED_SETAFFINITY */
{
perror("sched_setaffinity");
EXIT(MOD_SOPALIN, INTERNAL_ERR);
}
}
#else /* X_ARCHi686_mac */
{
thread_affinity_policy_data_t ap;
int ret;
ap.affinity_tag = 1; /* non-null affinity tag */
ret = thread_policy_set(
mach_thread_self(),
THREAD_AFFINITY_POLICY,
(integer_t*) &ap,
THREAD_AFFINITY_POLICY_COUNT
);
if(ret != 0)
{
perror("thread_policy_set");
EXIT(MOD_SOPALIN, INTERNAL_ERR);
}
}
#endif /* X_ARCHi686_mac */
#endif /* X_ACHIxxx */
#endif /* WITH_HWLOC */
#endif /* MARCEL */
return cpu;
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
hwloc_pid_t pid = 0;
char **orig_argv = argv;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_WHOLE_IO);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v")) {
verbose = 1;
goto next;
}
else if (!strcmp(argv[0], "--help")) {
usage(stdout);
return EXIT_SUCCESS;
}
else if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
else if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
else if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage (stderr);
exit(EXIT_FAILURE);
}
pid = atoi(argv[1]);
opt = 1;
goto next;
}
else if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", orig_argv[0], VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
else if (!strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
else if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
else if (!strcmp (argv[0], "--cpubind")) {
cpubind = 1;
goto next;
}
else if (!strcmp (argv[0], "--membind")) {
cpubind = 0;
goto next;
}
else if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "replicate", 2))
membind_policy = HWLOC_MEMBIND_REPLICATE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage (stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage(stderr);
return EXIT_FAILURE;
}
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
next:
argc -= opt+1;
argv += opt+1;
}
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (cpubind) {
if (get_last_cpu_location) {
if (pid)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid)
fprintf(stderr, "hwloc_get_proc_%s %ld failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", (long) pid, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid)
fprintf(stderr, "hwloc_get_proc_membind %ld failed (errno %d %s)\n", (long) pid, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_REPLICATE: policystr = "replicate"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
return EXIT_SUCCESS;
}
if (!hwloc_bitmap_iszero(membind_set)) {
if (verbose) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid)
fprintf(stderr, "hwloc_set_proc_membind %s %ld failed (errno %d %s)\n", s, (long) pid, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
}
if (!hwloc_bitmap_iszero(cpubind_set)) {
if (verbose) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid)
fprintf(stderr, "hwloc_set_proc_cpubind %s %ld failed (errno %d %s)\n", s, (long) pid, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid)
return EXIT_SUCCESS;
if (0 == argc) {
fprintf(stderr, "%s: nothing to do!\n", orig_argv[0]);
return EXIT_FAILURE;
}
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
orig_argv[0], argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
}
/* user to have to play with the cgroup hierarchy to modify it */
extern int task_cgroup_cpuset_set_task_affinity(slurmd_job_t *job)
{
int fstatus = SLURM_ERROR;
#ifndef HAVE_HWLOC
error("task/cgroup: plugin not compiled with hwloc support, "
"skipping affinity.");
return fstatus;
#else
uint32_t i;
uint32_t nldoms;
uint32_t nsockets;
uint32_t ncores;
uint32_t npus;
uint32_t nobj;
uint32_t pfirst,plast;
uint32_t taskid = job->envtp->localid;
uint32_t jntasks = job->node_tasks;
uint32_t jnpus = jntasks * job->cpus_per_task;
pid_t pid = job->envtp->task_pid;
cpu_bind_type_t bind_type;
int verbose;
hwloc_topology_t topology;
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_t cpuset,ct;
#else
hwloc_bitmap_t cpuset,ct;
#endif
hwloc_obj_t obj;
struct hwloc_obj *pobj;
hwloc_obj_type_t hwtype;
hwloc_obj_type_t req_hwtype;
int hwdepth;
size_t tssize;
cpu_set_t ts;
bind_type = job->cpu_bind_type ;
if (conf->task_plugin_param & CPU_BIND_VERBOSE ||
bind_type & CPU_BIND_VERBOSE)
verbose = 1 ;
if (bind_type & CPU_BIND_NONE) {
if (verbose)
info("task/cgroup: task[%u] is requesting no affinity",
taskid);
return 0;
} else if (bind_type & CPU_BIND_TO_THREADS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"thread level binding",taskid);
req_hwtype = HWLOC_OBJ_PU;
} else if (bind_type & CPU_BIND_TO_CORES) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"core level binding",taskid);
req_hwtype = HWLOC_OBJ_CORE;
} else if (bind_type & CPU_BIND_TO_SOCKETS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"socket level binding",taskid);
req_hwtype = HWLOC_OBJ_SOCKET;
} else if (bind_type & CPU_BIND_TO_LDOMS) {
if (verbose)
info("task/cgroup: task[%u] is requesting "
"ldom level binding",taskid);
req_hwtype = HWLOC_OBJ_NODE;
} else {
if (verbose)
info("task/cgroup: task[%u] using core level binding"
" by default",taskid);
req_hwtype = HWLOC_OBJ_CORE;
}
/* Allocate and initialize hwloc objects */
hwloc_topology_init(&topology);
#if HWLOC_API_VERSION <= 0x00010000
cpuset = hwloc_cpuset_alloc() ;
#else
cpuset = hwloc_bitmap_alloc() ;
#endif
/*
* Perform the topology detection. It will only get allowed PUs.
* Detect in the same time the granularity to use for binding.
* The granularity can be relaxed from threads to cores if enough
* cores are available as with hyperthread support, ntasks-per-core
* param can let us have access to more threads per core for each
* task
* Revert back to machine granularity if no finer-grained granularity
* matching the request is found. This will result in no affinity
* applied.
* The detected granularity will be used to find where to best place
* the task, then the cpu_bind option will be used to relax the
* affinity constraint and use more PUs. (i.e. use a core granularity
* to dispatch the tasks across the sockets and then provide access
* to each task to the cores of its socket.)
*/
hwloc_topology_load(topology);
npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_SOCKET);
nldoms = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_NODE);
hwtype = HWLOC_OBJ_MACHINE;
nobj = 1;
if (npus >= jnpus || bind_type & CPU_BIND_TO_THREADS) {
hwtype = HWLOC_OBJ_PU;
nobj = npus;
}
if (ncores >= jnpus || bind_type & CPU_BIND_TO_CORES) {
hwtype = HWLOC_OBJ_CORE;
nobj = ncores;
}
if (nsockets >= jntasks &&
bind_type & CPU_BIND_TO_SOCKETS) {
hwtype = HWLOC_OBJ_SOCKET;
nobj = nsockets;
}
/*
* HWLOC returns all the NUMA nodes available regardless of the
* number of underlying sockets available (regardless of the allowed
* resources). So there is no guarantee that each ldom will be populated
* with usable sockets. So add a simple check that at least ensure that
* we have as many sockets as ldoms before moving to ldoms granularity
*/
if (nldoms >= jntasks &&
nsockets >= nldoms &&
bind_type & CPU_BIND_TO_LDOMS) {
hwtype = HWLOC_OBJ_NODE;
nobj = nldoms;
}
/*
* Perform a block binding on the detected object respecting the
* granularity.
* If not enough objects to do the job, revert to no affinity mode
*/
if (hwloc_compare_types(hwtype,HWLOC_OBJ_MACHINE) == 0) {
info("task/cgroup: task[%u] disabling affinity because of %s "
"granularity",taskid,hwloc_obj_type_string(hwtype));
} else if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0 &&
jnpus > nobj) {
info("task/cgroup: task[%u] not enough %s objects, disabling "
"affinity",taskid,hwloc_obj_type_string(hwtype));
} else {
if (verbose) {
info("task/cgroup: task[%u] using %s granularity",
taskid,hwloc_obj_type_string(hwtype));
}
if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0) {
/* cores or threads granularity */
pfirst = taskid * job->cpus_per_task ;
plast = pfirst + job->cpus_per_task - 1;
} else {
/* sockets or ldoms granularity */
pfirst = taskid;
plast = pfirst;
}
hwdepth = hwloc_get_type_depth(topology,hwtype);
for (i = pfirst; i <= plast && i < nobj ; i++) {
obj = hwloc_get_obj_by_depth(topology,hwdepth,(int)i);
/* if requested binding overlap the granularity */
/* use the ancestor cpuset instead of the object one */
if (hwloc_compare_types(hwtype,req_hwtype) > 0) {
/* Get the parent object of req_hwtype or the */
/* one just above if not found (meaning of >0)*/
/* (useful for ldoms binding with !NUMA nodes)*/
pobj = obj->parent;
while (pobj != NULL &&
hwloc_compare_types(pobj->type,
req_hwtype) > 0)
pobj = pobj->parent;
if (pobj != NULL) {
if (verbose)
info("task/cgroup: task[%u] "
"higher level %s found",
taskid,
hwloc_obj_type_string(
pobj->type));
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(pobj->
allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(pobj->
allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
} else {
/* should not be executed */
if (verbose)
info("task/cgroup: task[%u] "
"no higher level found",
taskid);
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(obj->
allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(obj->
allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
}
} else {
#if HWLOC_API_VERSION <= 0x00010000
ct = hwloc_cpuset_dup(obj->allowed_cpuset);
hwloc_cpuset_or(cpuset,cpuset,ct);
hwloc_cpuset_free(ct);
#else
ct = hwloc_bitmap_dup(obj->allowed_cpuset);
hwloc_bitmap_or(cpuset,cpuset,ct);
hwloc_bitmap_free(ct);
#endif
}
}
char *str;
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_asprintf(&str,cpuset);
#else
hwloc_bitmap_asprintf(&str,cpuset);
#endif
tssize = sizeof(cpu_set_t);
if (hwloc_cpuset_to_glibc_sched_affinity(topology,cpuset,
&ts,tssize) == 0) {
fstatus = SLURM_SUCCESS;
if (sched_setaffinity(pid,tssize,&ts)) {
error("task/cgroup: task[%u] unable to set "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
} else if (verbose) {
info("task/cgroup: task[%u] taskset '%s' is set"
,taskid,str);
}
} else {
error("task/cgroup: task[%u] unable to build "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
}
free(str);
}
/* Destroy hwloc objects */
#if HWLOC_API_VERSION <= 0x00010000
hwloc_cpuset_free(cpuset);
#else
hwloc_bitmap_free(cpuset);
#endif
hwloc_topology_destroy(topology);
return fstatus;
#endif
}
void computeCPUOMP(int threadId, expression_type * expr, im_type * im, element_iterator * elt_it, std::vector<std::pair<element_iterator, element_iterator> > * elts)
{
char * a;
int cid;
std::ostringstream oss;
#if 0
hwloc_cpuset_t set = nullptr;
/* get a cpuset object */
set = hwloc_bitmap_alloc();
/* Get the cpu thread affinity info of the current process/thread */
hwloc_get_cpubind(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ")|";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
/* Get the latest core location of the current process/thread */
hwloc_get_last_cpu_location(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ");";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
#endif
#if defined(FEELPP_HAS_HARTS)
perf_mng.init("cpu") ;
perf_mng.start("cpu") ;
perf_mng.init("1.1") ;
perf_mng.init("1.2") ;
perf_mng.init("2.1") ;
perf_mng.init("2.2") ;
perf_mng.init("3") ;
#endif
//M_gm((*elt_it)->gm());
gm_ptrtype gm = (*elt_it)->gm();
//M_geopc(new typename eval::gmpc_type( M_gm, im->points() ));
typename eval::gmpc_ptrtype __geopc( new typename eval::gmpc_type(gm, im->points()) );
//M_c(new gmc_type( M_gm, *(*elt_it), M_geopc ));
gmc_ptrtype __c( new gmc_type( gm, *(*elt_it), __geopc ) );
//M_expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) );
eval_expr_type __expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( __c ) ) );
for (int i = 0; i < elts->size(); i++)
{
/*
std::cout << Environment::worldComm().rank() << " nbItems: " << elts->size()
<< " nbElts " << std::distance(elts->at(i), elts->at(i+1))
<< " 1st id " << elts->at(i)->id() << std::endl;
*/
//std::cout << Environment::worldComm().rank() << "|" << theadId << " fid=" elts.at(i).first.id() << std::endl;
for ( auto _elt = elts->at(i).first; _elt != elts->at(i).second; ++_elt )
{
//perf_mng.start("1.1") ;
__c->update( *_elt );
//perf_mng.stop("1.1") ;
//perf_mng.start("1.2") ;
map_gmc_type mapgmc( fusion::make_pair<vf::detail::gmc<0> >( __c ) );
//perf_mng.stop("1.2") ;
//perf_mng.start("2.1") ;
__expr.update( mapgmc );
//perf_mng.stop("2.1") ;
//perf_mng.start("2.2") ;
im->update( *__c );
//perf_mng.stop("2.2") ;
//perf_mng.start("3") ;
for ( uint16_type c1 = 0; c1 < eval::shape::M; ++c1 )
{
for ( uint16_type c2 = 0; c2 < eval::shape::N; ++c2 )
{
M_ret( c1,c2 ) += (*im)( __expr, c1, c2 );
}
}
//perf_mng.stop("3") ;
}
}
#if defined(FEELPP_HAS_HARTS)
perf_mng.stop("cpu") ;
M_cpuTime = perf_mng.getValueInSeconds("cpu");
#endif
}
void computeCPU(DataArgsType& args)
{
char * a;
int cid;
hwloc_cpuset_t set = nullptr;
std::ostringstream oss;
/* This initialization takes some time */
/* When using hartsi, the object instanciation is done when creating tasks */
/* and this is not a parallel section, thus we lose time in initialization */
/* doing it the computation step allows to incorporate this init time in the parallel section */
/*
M_threadId( threadId ),
M_gm( new gm_type( *_elt.gm() ) ),
M_geopc( new gmpc_type( M_gm, _im.points() ) ),
M_c( new gmc_type( M_gm, _elt, M_geopc ) ),
M_expr( _expr, map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) ),
M_im( _im ),
M_ret( eval::matrix_type::Zero() ),
M_cpuTime( 0.0 )
*/
#if 0
/* get a cpuset object */
set = hwloc_bitmap_alloc();
/* Get the cpu thread affinity info of the current process/thread */
hwloc_get_cpubind(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ")|";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
/* Get the latest core location of the current process/thread */
hwloc_get_last_cpu_location(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ");";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
#endif
perf_mng.init("1.1") ;
perf_mng.init("1.1") ;
perf_mng.init("2.1") ;
perf_mng.init("2.2") ;
perf_mng.init("3") ;
/* free memory */
if(set != nullptr)
{
hwloc_bitmap_free(set);
}
//perf_mng.init("data") ;
//perf_mng.start("data") ;
// DEFINE the range to be iterated on
std::vector<std::pair<element_iterator, element_iterator> > * elts =
args.get("elements")->get<std::vector<std::pair<element_iterator, element_iterator> > >();
int * threadId = args.get("threadId")->get<int>();
expression_type * expr = args.get("expr")->get<expression_type>();
im_type * im = args.get("im")->get<im_type>();
element_iterator * elt_it = args.get("elt")->get<element_iterator>();
//M_gm((*elt_it)->gm());
gm_ptrtype gm = (*elt_it)->gm();
//M_geopc(new typename eval::gmpc_type( M_gm, im->points() ));
typename eval::gmpc_ptrtype __geopc( new typename eval::gmpc_type(gm, im->points()) );
//M_c(new gmc_type( M_gm, *(*elt_it), M_geopc ));
gmc_ptrtype __c( new gmc_type( gm, *(*elt_it), __geopc ) );
//M_expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) );
eval_expr_type __expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( __c ) ) );
//perf_mng.stop("data");
perf_mng.init("cpu") ;
perf_mng.start("cpu") ;
for (int i = 0; i < elts->size(); i++)
{
//std::cout << Environment::worldComm().rank() << " nbItems: " << elts->size() << " nbElts " << std::distance(elts->at(i), elts->at(i+1)) << std::endl;
for ( auto _elt = elts->at(i).first; _elt != elts->at(i).second; ++_elt )
{
//perf_mng.start("1.1") ;
//M_c->update( *_elt );
__c->update( *_elt );
//perf_mng.stop("1.1") ;
//perf_mng.start("1.2") ;
map_gmc_type mapgmc( fusion::make_pair<vf::detail::gmc<0> >( __c ) );
//perf_mng.stop("1.2") ;
//perf_mng.start("2.1") ;
__expr.update( mapgmc );
//perf_mng.stop("2.1") ;
//perf_mng.start("2.2") ;
im->update( *__c );
//perf_mng.stop("2.2") ;
//perf_mng.start("3") ;
for ( uint16_type c1 = 0; c1 < eval::shape::M; ++c1 )
{
for ( uint16_type c2 = 0; c2 < eval::shape::N; ++c2 )
{
M_ret( c1,c2 ) += (*im)( __expr, c1, c2 );
}
}
//perf_mng.stop("3") ;
}
}
perf_mng.stop("cpu") ;
M_cpuTime = perf_mng.getValueInSeconds("cpu");
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set;
hwloc_const_bitmap_t cset;
hwloc_membind_policy_t policy;
const struct hwloc_topology_support *support;
int nbnodes;
hwloc_obj_t obj;
char *buffer;
unsigned i;
int err;
/* create a topology */
err = hwloc_topology_init(&topology);
if (err < 0) {
fprintf(stderr, "failed to initialize the topology\n");
return EXIT_FAILURE;
}
err = hwloc_topology_load(topology);
if (err < 0) {
fprintf(stderr, "failed to load the topology\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* retrieve the entire set of NUMA nodes and count them */
cset = hwloc_topology_get_topology_nodeset(topology);
nbnodes = hwloc_bitmap_weight(cset);
if (nbnodes <= 0) {
/* nbnodes may be -1 when there's no NUMA information,
* or 0 when the machine is known to be non-NUMA */
printf("this machine is not NUMA, nothing to do\n");
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
printf("there are %d nodes in the machine\n", nbnodes);
/* get the process memory binding as a nodeset */
set = hwloc_bitmap_alloc();
if (!set) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_membind_nodeset(topology, set, &policy, 0);
if (err < 0) {
fprintf(stderr, "failed to retrieve my memory binding and policy\n");
hwloc_topology_destroy(topology);
hwloc_bitmap_free(set);
return EXIT_FAILURE;
}
/* print the corresponding NUMA nodes */
hwloc_bitmap_asprintf(&buffer, set);
printf("bound to nodeset %s with contains:\n", buffer);
free(buffer);
hwloc_bitmap_foreach_begin(i, set) {
obj = hwloc_get_numanode_obj_by_os_index(topology, i);
printf(" node #%u (OS index %u) with %lld bytes of memory\n",
obj->logical_index, i, (unsigned long long) obj->memory.local_memory);
} hwloc_bitmap_foreach_end();
int main(void)
{
const struct hwloc_topology_support *support;
char *buffer;
hwloc_topology_t topology;
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_bitmap_t total = hwloc_bitmap_alloc();
hwloc_obj_t node;
char *s;
int err;
err = hwloc_topology_init(&topology);
assert(!err);
err = hwloc_topology_load(topology);
assert(!err);
support = hwloc_topology_get_support(topology);
if (!support->membind->get_area_memlocation)
goto out;
buffer = hwloc_alloc(topology, LEN);
assert(buffer);
printf("buffer %p length %u\n", buffer, LEN);
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_get_area_memlocation() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN, s);
free(s);
hwloc_bitmap_copy(total, set);
node = NULL;
next1:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_buffer;
if (!node->memory.local_memory)
goto next1;
printf("binding to 1st node and touching 1st quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_set_area_membind() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
memset(buffer, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer, 1, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next2:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next2;
printf("binding to 2nd node and touching 2nd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next3:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next3;
printf("binding to 3rd node and touching 3rd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/2, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/2, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/2, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next4:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next4;
printf("binding to 4th node and touching 4th quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+3*LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+3*LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+3*LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
out_with_nomorenodes:
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u located on %s\n", buffer, LEN, s);
free(s);
assert(hwloc_bitmap_isincluded(total, set));
out_with_buffer:
hwloc_free(topology, buffer, LEN);
out:
hwloc_topology_destroy(topology);
hwloc_bitmap_free(set);
hwloc_bitmap_free(total);
return 0;
}
static void create_hwloc_cpusets() {
#ifdef USE_HWLOC
int i;
int err = hwloc_topology_init(&topology);
assert(err == 0);
err = hwloc_topology_load(topology);
assert(err == 0);
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
assert(cpuset);
err = hwloc_get_cpubind(topology, cpuset, HWLOC_CPUBIND_PROCESS);
assert(err == 0);
const int available_pus = hwloc_bitmap_weight(cpuset);
const int last_set_index = hwloc_bitmap_last(cpuset);
const int num_workers = hc_context->nworkers;
hclib_affinity_t selected_affinity = HCLIB_AFFINITY_STRIDED;
const char *user_selected_affinity = getenv("HCLIB_AFFINITY");
if (user_selected_affinity) {
if (strcmp(user_selected_affinity, "strided") == 0) {
selected_affinity = HCLIB_AFFINITY_STRIDED;
} else if (strcmp(user_selected_affinity, "chunked") == 0) {
selected_affinity = HCLIB_AFFINITY_CHUNKED;
} else {
fprintf(stderr, "Unsupported thread affinity \"%s\" specified with "
"HCLIB_AFFINITY.\n", user_selected_affinity);
exit(1);
}
}
thread_cpusets = (hwloc_bitmap_t *)malloc(hc_context->nworkers *
sizeof(*thread_cpusets));
assert(thread_cpusets);
for (i = 0; i < hc_context->nworkers; i++) {
thread_cpusets[i] = hwloc_bitmap_alloc();
assert(thread_cpusets[i]);
}
switch (selected_affinity) {
case (HCLIB_AFFINITY_STRIDED): {
if (available_pus < num_workers) {
fprintf(stderr, "ERROR Available PUs (%d) was less than number "
"of workers (%d), don't currently support "
"oversubscription with strided thread pinning\n",
available_pus, num_workers);
exit(1);
}
int count = 0;
int index = 0;
while (index <= last_set_index) {
if (hwloc_bitmap_isset(cpuset, index)) {
hwloc_bitmap_set(thread_cpusets[count % num_workers],
index);
count++;
}
index++;
}
break;
}
case (HCLIB_AFFINITY_CHUNKED): {
const int chunk_size = (available_pus + num_workers - 1) /
num_workers;
int count = 0;
int index = 0;
while (index <= last_set_index) {
if (hwloc_bitmap_isset(cpuset, index)) {
hwloc_bitmap_set(thread_cpusets[count / chunk_size], index);
count++;
}
index++;
}
break;
}
default:
assert(false);
}
hwloc_bitmap_t nodeset = hwloc_bitmap_alloc();
hwloc_bitmap_t other_nodeset = hwloc_bitmap_alloc();
assert(nodeset && other_nodeset);
/*
* Here, we look for contiguous ranges of worker threads that share any NUMA
* nodes with us. In theory, this should be more hierarchical but isn't yet.
* This is also super inefficient... O(T^2) where T is the number of
* workers.
*/
bool revert_to_naive_stealing = false;
for (i = 0; i < hc_context->nworkers; i++) {
// Get the NUMA nodes for this CPU set
hwloc_cpuset_to_nodeset(topology, thread_cpusets[i], nodeset);
int base = -1;
int limit = -1;
int j;
for (j = 0; j < hc_context->nworkers; j++) {
hwloc_cpuset_to_nodeset(topology, thread_cpusets[j], other_nodeset);
// Take the intersection, see if there is any overlap
hwloc_bitmap_and(other_nodeset, nodeset, other_nodeset);
if (base < 0) {
// Haven't found a contiguous chunk of workers yet.
if (!hwloc_bitmap_iszero(other_nodeset)) {
base = j;
}
} else {
/*
* Have a contiguous chunk of workers, either still inside it or
* after it.
*/
if (limit < 0) {
// Inside the contiguous chunk of workers
if (hwloc_bitmap_iszero(other_nodeset)) {
// Found the end
limit = j;
}
} else {
// After the contiguous chunk of workers
if (!hwloc_bitmap_iszero(other_nodeset)) {
// No contiguous chunk to find, just do something naive.
revert_to_naive_stealing = true;
break;
}
}
}
}
if (revert_to_naive_stealing) {
fprintf(stderr, "WARNING: Using naive work-stealing patterns.\n");
base = 0;
limit = hc_context->nworkers;
} else {
assert(base >= 0);
if (limit < 0) {
limit = hc_context->nworkers;
}
}
hc_context->workers[i]->base_intra_socket_workers = base;
hc_context->workers[i]->limit_intra_socket_workers = limit;
#ifdef VERBOSE
char *nbuf;
hwloc_bitmap_asprintf(&nbuf, nodeset);
char *buffer;
hwloc_bitmap_asprintf(&buffer, thread_cpusets[i]);
fprintf(stderr, "Worker %d has access to %d PUs (%s), %d NUMA nodes "
"(%s). Shared NUMA nodes with [%d, %d).\n", i,
hwloc_bitmap_weight(thread_cpusets[i]), buffer,
hwloc_bitmap_weight(nodeset), nbuf, base, limit);
free(buffer);
#endif
}
#endif
}
int main(void)
{
hwloc_topology_t topology;
CUresult cres;
CUdevice device;
int count, i;
int err;
cres = cuInit(0);
if (cres != CUDA_SUCCESS) {
printf("cuInit failed %d\n", cres);
return 0;
}
cres = cuDeviceGetCount(&count);
if (cres != CUDA_SUCCESS) {
printf("cuDeviceGetCount failed %d\n", cres);
return 0;
}
printf("cuDeviceGetCount found %d devices\n", count);
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
for(i=0; i<count; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
cres = cuDeviceGet(&device, i);
if (cres != CUDA_SUCCESS) {
printf("failed to get device %d\n", i);
continue;
}
osdev = hwloc_cuda_get_device_osdev(topology, device);
assert(osdev);
osdev2 = hwloc_cuda_get_device_osdev_by_index(topology, i);
assert(osdev == osdev2);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "cuda", 4);
assert(!err);
assert(atoi(osdev->name+4) == (int) i);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "CUDA");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "CUDA");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_cuda_get_device_cpuset(topology, device, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
void output_console(hwloc_topology_t topology, const char *filename, int logical, int legend __hwloc_attribute_unused, int verbose_mode)
{
unsigned topodepth;
FILE *output;
if (!filename || !strcmp(filename, "-"))
output = stdout;
else {
output = open_file(filename, "w");
if (!output) {
fprintf(stderr, "Failed to open %s for writing (%s)\n", filename, strerror(errno));
return;
}
}
topodepth = hwloc_topology_get_depth(topology);
/*
* if verbose_mode == 0, only print the summary.
* if verbose_mode == 1, only print the topology tree.
* if verbose_mode > 1, print both.
*/
if (lstopo_show_only != (hwloc_obj_type_t)-1) {
if (verbose_mode > 1)
fprintf(output, "Only showing %s objects\n", hwloc_obj_type_string(lstopo_show_only));
output_only (topology, hwloc_get_root_obj(topology), output, logical, verbose_mode);
} else if (verbose_mode >= 1) {
output_topology (topology, hwloc_get_root_obj(topology), NULL, output, 0, logical, verbose_mode);
fprintf(output, "\n");
}
if ((verbose_mode > 1 || !verbose_mode) && lstopo_show_only == (hwloc_obj_type_t)-1) {
hwloc_lstopo_show_summary(output, topology);
}
if (verbose_mode > 1 && lstopo_show_only == (hwloc_obj_type_t)-1) {
const struct hwloc_distances_s * distances;
unsigned depth;
for (depth = 0; depth < topodepth; depth++) {
distances = hwloc_get_whole_distance_matrix_by_depth(topology, depth);
if (!distances || !distances->latency)
continue;
printf("latency matrix between %ss (depth %u) by %s indexes:\n",
hwloc_obj_type_string(hwloc_get_depth_type(topology, depth)),
depth,
logical ? "logical" : "physical");
hwloc_utils_print_distance_matrix(topology, hwloc_get_root_obj(topology), distances->nbobjs, depth, distances->latency, logical);
}
}
if (verbose_mode > 1 && lstopo_show_only == (hwloc_obj_type_t)-1) {
hwloc_const_bitmap_t complete = hwloc_topology_get_complete_cpuset(topology);
hwloc_const_bitmap_t topo = hwloc_topology_get_topology_cpuset(topology);
hwloc_const_bitmap_t online = hwloc_topology_get_online_cpuset(topology);
hwloc_const_bitmap_t allowed = hwloc_topology_get_allowed_cpuset(topology);
if (complete && !hwloc_bitmap_isequal(topo, complete)) {
hwloc_bitmap_t unknown = hwloc_bitmap_alloc();
char *unknownstr;
hwloc_bitmap_copy(unknown, complete);
hwloc_bitmap_andnot(unknown, unknown, topo);
hwloc_bitmap_asprintf(&unknownstr, unknown);
fprintf (output, "%d processors not represented in topology: %s\n", hwloc_bitmap_weight(unknown), unknownstr);
free(unknownstr);
hwloc_bitmap_free(unknown);
}
if (complete && !hwloc_bitmap_isequal(online, complete)) {
hwloc_bitmap_t offline = hwloc_bitmap_alloc();
char *offlinestr;
hwloc_bitmap_copy(offline, complete);
hwloc_bitmap_andnot(offline, offline, online);
hwloc_bitmap_asprintf(&offlinestr, offline);
fprintf (output, "%d processors offline: %s\n", hwloc_bitmap_weight(offline), offlinestr);
free(offlinestr);
hwloc_bitmap_free(offline);
}
if (complete && !hwloc_bitmap_isequal(allowed, online)) {
if (!hwloc_bitmap_isincluded(online, allowed)) {
hwloc_bitmap_t forbidden = hwloc_bitmap_alloc();
char *forbiddenstr;
hwloc_bitmap_copy(forbidden, online);
hwloc_bitmap_andnot(forbidden, forbidden, allowed);
hwloc_bitmap_asprintf(&forbiddenstr, forbidden);
fprintf(output, "%d processors online but not allowed: %s\n", hwloc_bitmap_weight(forbidden), forbiddenstr);
free(forbiddenstr);
hwloc_bitmap_free(forbidden);
}
if (!hwloc_bitmap_isincluded(allowed, online)) {
hwloc_bitmap_t potential = hwloc_bitmap_alloc();
char *potentialstr;
hwloc_bitmap_copy(potential, allowed);
hwloc_bitmap_andnot(potential, potential, online);
hwloc_bitmap_asprintf(&potentialstr, potential);
fprintf(output, "%d processors allowed but not online: %s\n", hwloc_bitmap_weight(potential), potentialstr);
free(potentialstr);
hwloc_bitmap_free(potential);
}
}
if (!hwloc_topology_is_thissystem(topology))
fprintf (output, "Topology not from this system\n");
}
if (output != stdout)
fclose(output);
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
int loaded = 0;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
unsigned long flags = HWLOC_TOPOLOGY_FLAG_WHOLE_IO|HWLOC_TOPOLOGY_FLAG_ICACHES;
int force = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
int pid_number = -1;
hwloc_pid_t pid = 0; /* only valid when pid_number > 0, but gcc-4.8 still reports uninitialized warnings */
char *callname;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
/* don't load now, in case some options change the config before the topology is actually used */
#define LOADED() (loaded)
#define ENSURE_LOADED() do { \
if (!loaded) { \
hwloc_topology_init(&topology); \
hwloc_topology_set_flags(topology, flags); \
hwloc_topology_load(topology); \
depth = hwloc_topology_get_depth(topology); \
loaded = 1; \
} \
} while (0)
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, HWLOC_VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "replicate", 2))
membind_policy = HWLOC_MEMBIND_REPLICATE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--whole-system")) {
if (loaded) {
fprintf(stderr, "Input option %s disallowed after options using the topology\n", argv[0]);
exit(EXIT_FAILURE);
}
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
goto next;
}
if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
ENSURE_LOADED();
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ENSURE_LOADED();
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
working_on_cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
ENSURE_LOADED();
if (pid_number > 0) {
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
/* no need to set_pid()
* the doc just says we're operating on pid, not that we're retrieving the topo/cpuset as seen from inside pid
*/
}
if (get_last_cpu_location && !working_on_cpubind) {
fprintf(stderr, "Options --membind and --get-last-cpu-location cannot be combined.\n");
return EXIT_FAILURE;
}
if ((get_binding || get_last_cpu_location) && (got_cpubind || got_membind)) {
/* doesn't work because get_binding/get_last_cpu_location overwrites cpubind_set */
fprintf(stderr, "Cannot display and set binding at the same time.\n");
return EXIT_FAILURE;
}
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number > 0)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number > 0)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid_number > 0)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_REPLICATE: policystr = "replicate"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number > 0)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_membind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number > 0)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_cpubind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number > 0)
return EXIT_SUCCESS;
if (0 == argc) {
if (get_binding || get_last_cpu_location)
return EXIT_SUCCESS;
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* user to have to play with the cgroup hierarchy to modify it */
extern int task_cgroup_cpuset_set_task_affinity(stepd_step_rec_t *job)
{
int fstatus = SLURM_ERROR;
#ifndef HAVE_HWLOC
error("task/cgroup: plugin not compiled with hwloc support, "
"skipping affinity.");
return fstatus;
#else
char mstr[1 + CPU_SETSIZE / 4];
cpu_bind_type_t bind_type;
cpu_set_t ts;
hwloc_obj_t obj;
hwloc_obj_type_t socket_or_node;
hwloc_topology_t topology;
hwloc_bitmap_t cpuset;
hwloc_obj_type_t hwtype;
hwloc_obj_type_t req_hwtype;
int bind_verbose = 0;
int rc = SLURM_SUCCESS;
pid_t pid = job->envtp->task_pid;
size_t tssize;
uint32_t nldoms;
uint32_t nsockets;
uint32_t ncores;
uint32_t npus;
uint32_t nobj;
uint32_t taskid = job->envtp->localid;
uint32_t jntasks = job->node_tasks;
uint32_t jnpus = jntasks * job->cpus_per_task;
bind_type = job->cpu_bind_type;
if (conf->task_plugin_param & CPU_BIND_VERBOSE ||
bind_type & CPU_BIND_VERBOSE)
bind_verbose = 1 ;
/* Allocate and initialize hwloc objects */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
cpuset = hwloc_bitmap_alloc();
if ( hwloc_get_type_depth(topology, HWLOC_OBJ_NODE) >
hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET) ) {
/* One socket contains multiple NUMA-nodes
* like AMD Opteron 6000 series etc.
* In such case, use NUMA-node instead of socket. */
socket_or_node = HWLOC_OBJ_NODE;
} else {
socket_or_node = HWLOC_OBJ_SOCKET;
}
if (bind_type & CPU_BIND_NONE) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting no affinity",
taskid);
return 0;
} else if (bind_type & CPU_BIND_TO_THREADS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"thread level binding",taskid);
req_hwtype = HWLOC_OBJ_PU;
} else if (bind_type & CPU_BIND_TO_CORES) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"core level binding",taskid);
req_hwtype = HWLOC_OBJ_CORE;
} else if (bind_type & CPU_BIND_TO_SOCKETS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"socket level binding",taskid);
req_hwtype = socket_or_node;
} else if (bind_type & CPU_BIND_TO_LDOMS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"ldom level binding",taskid);
req_hwtype = HWLOC_OBJ_NODE;
} else if (bind_type & CPU_BIND_TO_BOARDS) {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"board level binding",taskid);
req_hwtype = HWLOC_OBJ_GROUP;
} else if (bind_type & bind_mode_ldom) {
req_hwtype = HWLOC_OBJ_NODE;
} else {
if (bind_verbose)
info("task/cgroup: task[%u] using core level binding"
" by default",taskid);
req_hwtype = HWLOC_OBJ_CORE;
}
/*
* Perform the topology detection. It will only get allowed PUs.
* Detect in the same time the granularity to use for binding.
* The granularity can be relaxed from threads to cores if enough
* cores are available as with hyperthread support, ntasks-per-core
* param can let us have access to more threads per core for each
* task
* Revert back to machine granularity if no finer-grained granularity
* matching the request is found. This will result in no affinity
* applied.
* The detected granularity will be used to find where to best place
* the task, then the cpu_bind option will be used to relax the
* affinity constraint and use more PUs. (i.e. use a core granularity
* to dispatch the tasks across the sockets and then provide access
* to each task to the cores of its socket.)
*/
npus = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_PU);
ncores = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_CORE);
nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
socket_or_node);
nldoms = (uint32_t) hwloc_get_nbobjs_by_type(topology,
HWLOC_OBJ_NODE);
hwtype = HWLOC_OBJ_MACHINE;
nobj = 1;
if (npus >= jnpus || bind_type & CPU_BIND_TO_THREADS) {
hwtype = HWLOC_OBJ_PU;
nobj = npus;
}
if (ncores >= jnpus || bind_type & CPU_BIND_TO_CORES) {
hwtype = HWLOC_OBJ_CORE;
nobj = ncores;
}
if (nsockets >= jntasks &&
bind_type & CPU_BIND_TO_SOCKETS) {
hwtype = socket_or_node;
nobj = nsockets;
}
/*
* HWLOC returns all the NUMA nodes available regardless of the
* number of underlying sockets available (regardless of the allowed
* resources). So there is no guarantee that each ldom will be populated
* with usable sockets. So add a simple check that at least ensure that
* we have as many sockets as ldoms before moving to ldoms granularity
*/
if (nldoms >= jntasks &&
nsockets >= nldoms &&
bind_type & (CPU_BIND_TO_LDOMS | bind_mode_ldom)) {
hwtype = HWLOC_OBJ_NODE;
nobj = nldoms;
}
/*
* If not enough objects to do the job, revert to no affinity mode
*/
if (hwloc_compare_types(hwtype,HWLOC_OBJ_MACHINE) == 0) {
info("task/cgroup: task[%u] disabling affinity because of %s "
"granularity",taskid,hwloc_obj_type_string(hwtype));
} else if (hwloc_compare_types(hwtype,HWLOC_OBJ_CORE) >= 0 &&
jnpus > nobj) {
info("task/cgroup: task[%u] not enough %s objects, disabling "
"affinity",taskid,hwloc_obj_type_string(hwtype));
} else if (bind_type & bind_mode) {
/* Explicit binding mode specified by the user
* Bind the taskid in accordance with the specified mode
*/
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE, 0);
if (!hwloc_bitmap_isequal(obj->complete_cpuset,
obj->allowed_cpuset)) {
info("task/cgroup: entire node must be allocated, "
"disabling affinity, task[%u]", taskid);
fprintf(stderr, "Requested cpu_bind option requires "
"entire node to be allocated; disabling "
"affinity\n");
} else {
if (bind_verbose)
info("task/cgroup: task[%u] is requesting "
"explicit binding mode",taskid);
_get_sched_cpuset(topology, hwtype, req_hwtype, &ts,
job);
tssize = sizeof(cpu_set_t);
fstatus = SLURM_SUCCESS;
if ((rc = sched_setaffinity(pid, tssize, &ts))) {
error("task/cgroup: task[%u] unable to set "
"mask 0x%s", taskid,
cpuset_to_str(&ts, mstr));
fstatus = SLURM_ERROR;
} else if (bind_verbose)
info("task/cgroup: task[%u] mask 0x%s",
taskid, cpuset_to_str(&ts, mstr));
slurm_chkaffinity(&ts, job, rc);
}
} else {
/* Bind the detected object to the taskid, respecting the
* granularity, using the designated or default distribution
* method (block or cyclic). */
char *str;
if (bind_verbose) {
info("task/cgroup: task[%u] using %s granularity",
taskid,hwloc_obj_type_string(hwtype));
}
/* There are two "distributions," controlled by the
* -m option of srun and friends. The first is the
* distribution of tasks to nodes. The second is the
* distribution of allocated cpus to tasks for
* binding. This code is handling the second
* distribution. Here's how the values get set, based
* on the value of -m
*
* SLURM_DIST_CYCLIC = srun -m cyclic
* SLURM_DIST_BLOCK = srun -m block
* SLURM_DIST_CYCLIC_CYCLIC = srun -m cyclic:cyclic
* SLURM_DIST_BLOCK_CYCLIC = srun -m block:cyclic
*
* In the first two cases, the user only specified the
* first distribution. The second distribution
* defaults to cyclic. In the second two cases, the
* user explicitly requested a second distribution of
* cyclic. So all these four cases correspond to a
* second distribution of cyclic. So we want to call
* _task_cgroup_cpuset_dist_cyclic.
*
* If the user explicitly specifies a second
* distribution of block, or if
* CR_CORE_DEFAULT_DIST_BLOCK is configured and the
* user does not explicitly specify a second
* distribution of cyclic, the second distribution is
* block, and we need to call
* _task_cgroup_cpuset_dist_block. In these cases,
* task_dist would be set to SLURM_DIST_CYCLIC_BLOCK
* or SLURM_DIST_BLOCK_BLOCK.
*
* You can see the equivalent code for the
* task/affinity plugin in
* src/plugins/task/affinity/dist_tasks.c, around line 384.
*/
switch (job->task_dist) {
case SLURM_DIST_CYCLIC:
case SLURM_DIST_BLOCK:
case SLURM_DIST_CYCLIC_CYCLIC:
case SLURM_DIST_BLOCK_CYCLIC:
_task_cgroup_cpuset_dist_cyclic(
topology, hwtype, req_hwtype,
job, bind_verbose, cpuset);
break;
default:
_task_cgroup_cpuset_dist_block(
topology, hwtype, req_hwtype,
nobj, job, bind_verbose, cpuset);
}
hwloc_bitmap_asprintf(&str, cpuset);
tssize = sizeof(cpu_set_t);
if (hwloc_cpuset_to_glibc_sched_affinity(topology,cpuset,
&ts,tssize) == 0) {
fstatus = SLURM_SUCCESS;
if ((rc = sched_setaffinity(pid,tssize,&ts))) {
error("task/cgroup: task[%u] unable to set "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
} else if (bind_verbose) {
info("task/cgroup: task[%u] taskset '%s' is set"
,taskid,str);
}
slurm_chkaffinity(&ts, job, rc);
} else {
error("task/cgroup: task[%u] unable to build "
"taskset '%s'",taskid,str);
fstatus = SLURM_ERROR;
}
free(str);
}
/* Destroy hwloc objects */
hwloc_bitmap_free(cpuset);
hwloc_topology_destroy(topology);
return fstatus;
#endif
}
static void
output_console_obj (hwloc_topology_t topology, hwloc_obj_t l, FILE *output, int logical, int verbose_mode)
{
char type[32], *attr, phys[32] = "";
unsigned idx = logical ? l->logical_index : l->os_index;
const char *indexprefix = logical ? " L#" : " P#";
if (lstopo_show_cpuset < 2) {
int len;
if (l->type == HWLOC_OBJ_MISC && l->name)
fprintf(output, "%s", l->name);
else {
hwloc_obj_type_snprintf (type, sizeof(type), l, verbose_mode-1);
fprintf(output, "%s", type);
}
if (l->depth != 0 && idx != (unsigned)-1
&& l->type != HWLOC_OBJ_PCI_DEVICE
&& (l->type != HWLOC_OBJ_BRIDGE || l->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_HOST))
fprintf(output, "%s%u", indexprefix, idx);
if (logical && l->os_index != (unsigned) -1 &&
(verbose_mode >= 2 || l->type == HWLOC_OBJ_PU || l->type == HWLOC_OBJ_NUMANODE))
snprintf(phys, sizeof(phys), "P#%u", l->os_index);
/* display attributes */
len = hwloc_obj_attr_snprintf (NULL, 0, l, " ", verbose_mode-1);
attr = malloc(len+1);
*attr = '\0';
hwloc_obj_attr_snprintf (attr, len+1, l, " ", verbose_mode-1);
if (*phys || *attr) {
const char *separator = *phys != '\0' && *attr!= '\0' ? " " : "";
fprintf(output, " (%s%s%s)",
phys, separator, attr);
}
free(attr);
/* display the root total_memory if not verbose (already shown)
* and different from the local_memory (already shown) */
if (verbose_mode == 1 && !l->parent && l->memory.total_memory > l->memory.local_memory)
fprintf(output, " (%lu%s total)",
(unsigned long) hwloc_memory_size_printf_value(l->memory.total_memory, 0),
hwloc_memory_size_printf_unit(l->memory.total_memory, 0));
/* append the name */
if ((l->type == HWLOC_OBJ_OS_DEVICE || verbose_mode >= 2) && l->name && l->type != HWLOC_OBJ_MISC)
fprintf(output, " \"%s\"", l->name);
}
if (!l->cpuset)
return;
if (lstopo_show_cpuset == 1)
fprintf(output, " cpuset=");
if (lstopo_show_cpuset) {
char *cpusetstr;
if (lstopo_show_taskset)
hwloc_bitmap_taskset_asprintf(&cpusetstr, l->cpuset);
else
hwloc_bitmap_asprintf(&cpusetstr, l->cpuset);
fprintf(output, "%s", cpusetstr);
free(cpusetstr);
}
/* annotate if the PU is forbidden/offline/running */
if (l->type == HWLOC_OBJ_PU && verbose_mode >= 2) {
if (lstopo_pu_offline(l))
printf(" (offline)");
else if (lstopo_pu_forbidden(l))
printf(" (forbidden)");
else if (lstopo_pu_running(topology, l))
printf(" (running)");
}
}
