hwloc_bitmap_t
thwl_get_bitmap (Tcl_Interp* interp, Tcl_Obj* obj)
{
const char* str = Tcl_GetString (obj);
hwloc_bitmap_t bitmap = hwloc_bitmap_alloc();
if (hwloc_bitmap_list_sscanf(bitmap, str) == -1) {
Tcl_ResetResult (interp);
Tcl_AppendResult(interp, "Expected bitmap but got \"", str, "\"", NULL);
hwloc_bitmap_free(bitmap);
return NULL;
}
return bitmap;
}
static void set(orte_job_t *jobdat,
orte_proc_t *child,
char ***environ_copy,
int write_fd)
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
orte_app_context_t *context;
int rc=ORTE_ERROR;
char *msg, *param;
char *cpu_bitmap;
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set on child %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
if (NULL == jobdat || NULL == child) {
/* nothing for us to do */
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set jobdat %s child %s - nothing to do",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == jobdat) ? "NULL" : ORTE_JOBID_PRINT(jobdat->jobid),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
return;
}
context = (orte_app_context_t*)opal_pointer_array_get_item(jobdat->apps, child->app_idx);
/* Set process affinity, if given */
cpu_bitmap = NULL;
if (!orte_get_attribute(&child->attributes, ORTE_PROC_CPU_BITMAP, (void**)&cpu_bitmap, OPAL_STRING) ||
NULL == cpu_bitmap || 0 == strlen(cpu_bitmap)) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
rc = hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
char *tmp;
(void)hwloc_bitmap_list_asprintf(&tmp, sum->available);
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), tmp);
free(tmp);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
return;
}
}
}
if (0 == rc && opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
} else {
/* convert the list to a cpuset */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
return;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
}
if (NULL != cpu_bitmap) {
free(cpu_bitmap);
}
}
int orte_ess_base_proc_binding(void)
{
#if OPAL_HAVE_HWLOC
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
char *map;
int ret;
char *error;
/* Determine if we were pre-bound or not */
if (NULL != getenv("OMPI_MCA_orte_bound_at_launch")) {
orte_proc_is_bound = true;
if (NULL != (map = getenv("OMPI_MCA_orte_base_applied_binding"))) {
orte_proc_applied_binding = hwloc_bitmap_alloc();
if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) {
error = "applied_binding parse";
goto error;
}
}
}
/* see if we were bound when launched */
if (!orte_proc_is_bound) {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Not bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* we were not bound at launch */
if (NULL != opal_hwloc_topology) {
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Binding not supported",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
orte_proc_is_bound = true;
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process was externally bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
/* try to find a level and index for this location */
opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx);
/* cleanup */
hwloc_bitmap_free(cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound according to slot_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == orte_process_info.my_node_rank) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process not bound - no node rank available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_HWTHREAD_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to hwthread",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = ORTE_ERROR;
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to core",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL;
} else {
ret = ORTE_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology,
obj, OPAL_HWLOC_LOGICAL);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_hwloc_base_print_level(orte_process_info.bind_level)));
break;
}
}
if (!orte_proc_is_bound) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
}
}
}
} else {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* report bindings, if requested */
if (opal_hwloc_report_bindings) {
char bindings[64];
hwloc_obj_t root;
hwloc_cpuset_t cpus;
/* get the root object for this node */
root = hwloc_get_root_obj(opal_hwloc_topology);
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* we are not bound if this equals our cpuset */
if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) {
opal_output(0, "%s is not bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
} else {
hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset);
opal_output(0, "%s is bound to cpus %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
bindings);
}
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
#else
return ORTE_SUCCESS;
#endif
}
static int do_child(orte_app_context_t* context,
orte_proc_t *child,
char **environ_copy,
orte_job_t *jobdat, int write_fd,
orte_iof_base_io_conf_t opts)
{
int i, rc;
sigset_t sigs;
long fd, fdmax = sysconf(_SC_OPEN_MAX);
char *param, *msg;
if (orte_forward_job_control) {
/* Set a new process group for this child, so that a
SIGSTOP can be sent to it without being sent to the
orted. */
setpgid(0, 0);
}
/* Setup the pipe to be close-on-exec */
fcntl(write_fd, F_SETFD, FD_CLOEXEC);
if (NULL != child) {
/* setup stdout/stderr so that any error messages that we
may print out will get displayed back at orterun.
NOTE: Definitely do this AFTER we check contexts so
that any error message from those two functions doesn't
come out to the user. IF we didn't do it in this order,
THEN a user who gives us a bad executable name or
working directory would get N error messages, where
N=num_procs. This would be very annoying for large
jobs, so instead we set things up so that orterun
always outputs a nice, single message indicating what
happened
*/
if (ORTE_SUCCESS != (i = orte_iof_base_setup_child(&opts,
&environ_copy))) {
ORTE_ERROR_LOG(i);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt",
"iof setup failed",
orte_process_info.nodename, context->app);
/* Does not return */
}
#if OPAL_HAVE_HWLOC
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
/* Set process affinity, if given */
if (NULL == child->cpu_bitmap) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
} else {
if (0 == strlen(child->cpu_bitmap)) {
/* this proc is not bound */
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
/* provide a nice string representation of what we bound to */
(void) mca_base_var_env_name ("orte_base_applied_binding", ¶m);
opal_setenv(param, child->cpu_bitmap, true, &environ_copy);
free(param);
goto PROCEED;
}
/* convert the list to a cpu bitmap */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, child->cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
goto PROCEED;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
}
}
#endif
} else if (!(ORTE_JOB_CONTROL_FORWARD_OUTPUT & jobdat->controls)) {
/* tie stdin/out/err/internal to /dev/null */
int fdnull;
for (i=0; i < 3; i++) {
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > i && i != write_fd) {
dup2(fdnull, i);
}
close(fdnull);
}
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > opts.p_internal[1]) {
dup2(fdnull, opts.p_internal[1]);
}
close(fdnull);
}
#if OPAL_HAVE_HWLOC
PROCEED:
#endif
/* if the user requested it, set the system resource limits */
if (OPAL_SUCCESS != (rc = opal_util_init_sys_limits(&msg))) {
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"set limit",
orte_process_info.nodename, context->app,
__FILE__, __LINE__, msg);
}
/* ensure we only do this once */
(void) mca_base_var_env_name("opal_set_max_sys_limits", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
/* close all file descriptors w/ exception of stdin/stdout/stderr,
the pipe used for the IOF INTERNAL messages, and the pipe up to
the parent. */
for(fd=3; fd<fdmax; fd++) {
if (fd != opts.p_internal[1] && fd != write_fd) {
close(fd);
}
}
if (context->argv == NULL) {
context->argv = malloc(sizeof(char*)*2);
context->argv[0] = strdup(context->app);
context->argv[1] = NULL;
}
/* Set signal handlers back to the default. Do this close to
the exev() because the event library may (and likely will)
reset them. If we don't do this, the event library may
have left some set that, at least on some OS's, don't get
reset via fork() or exec(). Hence, the launched process
could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that we
set the default handlers, above. This is noticable on
Linux where the event library blocks SIGTERM, but we don't
want that blocked by the launched process. */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
/* Exec the new executable */
if (10 < opal_output_get_verbosity(orte_odls_base_framework.framework_output)) {
int jout;
opal_output(0, "%s STARTING %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), context->app);
for (jout=0; NULL != context->argv[jout]; jout++) {
opal_output(0, "%s\tARGV[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, context->argv[jout]);
}
for (jout=0; NULL != environ_copy[jout]; jout++) {
opal_output(0, "%s\tENVIRON[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, environ_copy[jout]);
}
}
execve(context->app, context->argv, environ_copy);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt", "execve error",
orte_process_info.nodename, context->app, strerror(errno));
/* Does not return */
}
int orte_ess_base_proc_binding(void)
{
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
char *map;
int ret;
char *error=NULL;
hwloc_cpuset_t mycpus;
/* Determine if we were pre-bound or not */
if (NULL != getenv(OPAL_MCA_PREFIX"orte_bound_at_launch")) {
orte_proc_is_bound = true;
if (NULL != (map = getenv(OPAL_MCA_PREFIX"orte_base_applied_binding"))) {
orte_proc_applied_binding = hwloc_bitmap_alloc();
if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) {
error = "applied_binding parse";
goto error;
}
}
}
/* see if we were bound when launched */
if (!orte_proc_is_bound) {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Not bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* we were not bound at launch */
if (NULL == opal_hwloc_topology) {
/* there is nothing we can do, so just return */
return ORTE_SUCCESS;
}
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Binding not supported",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
orte_proc_is_bound = true;
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process was externally bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology,
OPAL_HWLOC_LOGICAL, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound according to slot_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == orte_process_info.my_node_rank) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process not bound - no node rank available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to hwthread",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = ORTE_ERROR;
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to core",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
} else {
ret = ORTE_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
hwloc_obj_type_string(target)));
break;
}
}
if (!orte_proc_is_bound) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
}
}
} else {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
if (NULL != orte_process_info.cpuset) {
free(orte_process_info.cpuset);
orte_process_info.cpuset = NULL;
}
if (opal_hwloc_report_bindings || 4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
opal_output(0, "MCW rank %d is not bound",
ORTE_PROC_MY_NAME->vpid);
}
} else {
/* store/update the string representation of our local binding */
if (NULL != orte_process_info.cpuset) {
free(orte_process_info.cpuset);
orte_process_info.cpuset = NULL;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, mycpus);
/* report the binding, if requested */
if (opal_hwloc_report_bindings || 4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
char tmp1[1024], tmp2[1024];
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", ORTE_PROC_MY_NAME->vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
ORTE_PROC_MY_NAME->vpid, tmp1, tmp2);
}
}
}
hwloc_bitmap_free(mycpus);
/* push our cpuset so others can calculate our locality */
if (NULL != orte_process_info.cpuset) {
OPAL_MODEX_SEND_VALUE(ret, OPAL_PMIX_GLOBAL, OPAL_PMIX_CPUSET,
orte_process_info.cpuset, OPAL_STRING);
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
}
