static void test(hwloc_const_bitmap_t cpuset, int flags)
{
hwloc_bitmap_t new_cpuset = hwloc_bitmap_alloc();
result_set("Bind this singlethreaded process", hwloc_set_cpubind(topology, cpuset, flags), support->cpubind->set_thisproc_cpubind || support->cpubind->set_thisthread_cpubind);
result_get("Get this singlethreaded process", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags), support->cpubind->get_thisproc_cpubind || support->cpubind->get_thisthread_cpubind);
result_set("Bind this thread", hwloc_set_cpubind(topology, cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->set_thisthread_cpubind);
result_get("Get this thread", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->get_thisthread_cpubind);
result_set("Bind this whole process", hwloc_set_cpubind(topology, cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_thisproc_cpubind);
result_get("Get this whole process", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_thisproc_cpubind);
#ifdef HWLOC_WIN_SYS
result_set("Bind process", hwloc_set_proc_cpubind(topology, GetCurrentProcess(), cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_proc_cpubind);
result_get("Get process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, GetCurrentProcess(), new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_proc_cpubind);
result_set("Bind thread", hwloc_set_thread_cpubind(topology, GetCurrentThread(), cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->set_thread_cpubind);
result_get("Get thread", cpuset, new_cpuset, hwloc_get_thread_cpubind(topology, GetCurrentThread(), new_cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->get_thread_cpubind);
#else /* !HWLOC_WIN_SYS */
result_set("Bind whole process", hwloc_set_proc_cpubind(topology, getpid(), cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_proc_cpubind);
result_get("Get whole process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, getpid(), new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_proc_cpubind);
result_set("Bind process", hwloc_set_proc_cpubind(topology, getpid(), cpuset, flags), support->cpubind->set_proc_cpubind);
result_get("Get process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, getpid(), new_cpuset, flags), support->cpubind->get_proc_cpubind);
#ifdef hwloc_thread_t
result_set("Bind thread", hwloc_set_thread_cpubind(topology, pthread_self(), cpuset, flags), support->cpubind->set_thread_cpubind);
result_get("Get thread", cpuset, new_cpuset, hwloc_get_thread_cpubind(topology, pthread_self(), new_cpuset, flags), support->cpubind->get_thread_cpubind);
#endif
#endif /* !HWLOC_WIN_SYS */
printf("\n");
hwloc_bitmap_free(new_cpuset);
}
/******************* FUNCTION *********************/
int TopoHwloc::getCurrentId ( int level, int depth ) const
{
//vars
int res = -1;
//get absolute depth
int absDepth = getAbsDepth(level,depth);
//search current thread binding
hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
int status = hwloc_get_cpubind (topology, cpuset, 0);
allocAssume(status >= 0,"Failed to get the current thread CPU binding with hwloc.");
//count overlap over current level
int cnt = hwloc_get_nbobjs_inside_cpuset_by_depth(topology,cpuset,absDepth);
//if get only one return its ID
if (cnt == 1)
{
hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_depth(topology,cpuset,absDepth,0);
//TODO maybe their is an accessor for this
res = obj->logical_index;
}
//clean memory
hwloc_bitmap_free(cpuset);
return res;
}
inline void __pact_reuse_add(void *ary, long long start, long long end, long long mem_ac) {
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(__pact_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_get_last_cpu_location(__pact_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_bitmap_singlify(set);
hwloc_set_area_membind ( __pact_topo, (const void*)ary, abs(end-start), (hwloc_const_cpuset_t)set, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_MIGRATE );
hwloc_bitmap_free(set);
}
bool unbind_this_thread()
{
if ( ! sentinel() ) return false ;
#define HWLOC_DEBUG_PRINT 0
#if HWLOC_DEBUG_PRINT
std::cout << "Kokkos::unbind_this_thread() from " ;
hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD );
print_bitmap( std::cout , s_hwloc_location );
#endif
const bool result =
s_hwloc_topology &&
0 == hwloc_set_cpubind( s_hwloc_topology ,
s_process_binding ,
HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT );
#if HWLOC_DEBUG_PRINT
std::cout << " to " ;
hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD );
print_bitmap( std::cout , s_hwloc_location );
std::cout << std::endl ;
#endif
return result ;
#undef HWLOC_DEBUG_PRINT
}
void chpl_topo_touchMemFromSubloc(void* p, size_t size, chpl_bool onlyInside,
c_sublocid_t subloc) {
size_t pgSize;
unsigned char* pPgLo;
size_t nPages;
hwloc_cpuset_t cpuset;
int flags;
_DBG_P("chpl_topo_touchMemFromSubloc(%p, %#zx, onlyIn=%s, %d)\n",
p, size, (onlyInside ? "T" : "F"), (int) subloc);
if (!haveTopology
|| !topoSupport->cpubind->get_thread_cpubind
|| !topoSupport->cpubind->set_thread_cpubind) {
return;
}
alignAddrSize(p, size, onlyInside, &pgSize, &pPgLo, &nPages);
_DBG_P(" localize %p, %#zx bytes (%#zx pages)\n",
pPgLo, nPages * pgSize, nPages);
if (nPages == 0)
return;
if ((cpuset = hwloc_bitmap_alloc()) == NULL) {
report_error("hwloc_bitmap_alloc()", errno);
}
flags = HWLOC_CPUBIND_THREAD;
if (hwloc_get_cpubind(topology, cpuset, flags)) {
report_error("hwloc_get_cpubind()", errno);
}
chpl_topo_setThreadLocality(subloc);
{
size_t pg;
for (pg = 0; pg < nPages; pg++) {
pPgLo[pg * pgSize] = 0;
}
}
flags = HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT;
if (hwloc_set_cpubind(topology, cpuset, flags)) {
report_error("hwloc_set_cpubind()", errno);
}
hwloc_bitmap_free(cpuset);
}
/*
* Don't use show_help() here (or print any error message at all).
* Let the upper layer output a relevant message, because doing so may
* be complicated (e.g., this might be called from the ORTE ODLS,
* which has to do some extra steps to get error messages to be
* displayed).
*/
int opal_hwloc_base_set_process_membind_policy(void)
{
int rc = 0, flags;
hwloc_membind_policy_t policy;
hwloc_cpuset_t cpuset;
/* Make sure opal_hwloc_topology has been set by the time we've
been called */
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
return OPAL_ERR_BAD_PARAM;
}
/* Set the default memory allocation policy according to MCA
param */
switch (opal_hwloc_base_map) {
case OPAL_HWLOC_BASE_MAP_LOCAL_ONLY:
policy = HWLOC_MEMBIND_BIND;
flags = HWLOC_MEMBIND_STRICT;
break;
case OPAL_HWLOC_BASE_MAP_NONE:
default:
policy = HWLOC_MEMBIND_DEFAULT;
flags = 0;
break;
}
cpuset = hwloc_bitmap_alloc();
if (NULL == cpuset) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
} else {
int e;
hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
rc = hwloc_set_membind(opal_hwloc_topology,
cpuset, policy, flags);
e = errno;
hwloc_bitmap_free(cpuset);
/* See if hwloc was able to do it. If hwloc failed due to
ENOSYS, but the base_map == NONE, then it's not really an
error. */
if (0 != rc && ENOSYS == e &&
OPAL_HWLOC_BASE_MAP_NONE == opal_hwloc_base_map) {
rc = 0;
}
}
return (0 == rc) ? OPAL_SUCCESS : OPAL_ERROR;
}
/*
* Where is this process currently bound? (layout string)
*/
static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
{
int ret;
hwloc_obj_t root;
hwloc_cpuset_t boundset, rootset;
bool bound = false;
/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* get our bindings */
boundset = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, boundset,
HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound
- the environment does not support it */
bound = false;
} else {
/* we are bound if the two cpusets are not equal, or if there
is only ONE PU available to us */
if (0 != hwloc_bitmap_compare(boundset, rootset) ||
opal_hwloc_base_single_cpu(rootset) ||
opal_hwloc_base_single_cpu(boundset)) {
bound = true;
}
}
/* If we are not bound, indicate that */
if (!bound) {
strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
ret = OMPI_SUCCESS;
}
/* If we are bound, print it out */
else {
ret = opal_hwloc_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX,
opal_hwloc_topology,
boundset);
if (OPAL_ERR_NOT_BOUND == ret) {
strncpy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
ret = OMPI_SUCCESS;
}
}
hwloc_bitmap_free(boundset);
return ret;
}
int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
size_t num_segments)
{
int rc = OPAL_SUCCESS;
char *msg = NULL;
size_t i;
hwloc_cpuset_t cpuset = NULL;
/* bozo check */
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
msg = "hwloc_set_area_membind() failure - topology not available";
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__,
msg, rc);
}
/* This module won't be used unless the process is already
processor-bound. So find out where we're processor bound, and
bind our memory there, too. */
cpuset = hwloc_bitmap_alloc();
if (NULL == cpuset) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
msg = "hwloc_bitmap_alloc() failure";
goto out;
}
hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
for (i = 0; i < num_segments; ++i) {
if (0 != hwloc_set_area_membind(opal_hwloc_topology,
segments[i].mbs_start_addr,
segments[i].mbs_len, cpuset,
HWLOC_MEMBIND_BIND,
HWLOC_MEMBIND_STRICT)) {
rc = OPAL_ERROR;
msg = "hwloc_set_area_membind() failure";
goto out;
}
}
out:
if (NULL != cpuset) {
hwloc_bitmap_free(cpuset);
}
if (OPAL_SUCCESS != rc) {
return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc);
}
return OPAL_SUCCESS;
}
std::size_t thread_affinity_worker(std::size_t desired)
{
// Returns the OS-thread number of the worker that is running this
// PX-thread.
std::size_t current = hpx::get_worker_thread_num();
bool numa_sensitive = hpx::is_scheduler_numa_sensitive();
if (current == desired)
{
// extract the desired affinity mask
hpx::threads::topology const& t = hpx::get_runtime().get_topology();
hpx::threads::mask_type desired_mask = t.get_thread_affinity_mask(current, numa_sensitive);
#if defined(HPX_HAVE_HWLOC)
std::size_t idx = hpx::threads::find_first(desired_mask);
hwloc_topology_t topo;
hwloc_topology_init(&topo);
hwloc_topology_load(topo);
// retrieve the current affinity mask
hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
hwloc_bitmap_zero(cpuset);
if (0 == hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD)) {
// sadly get_cpubind is not implemented for Windows based systems
hwloc_cpuset_t cpuset_cmp = hwloc_bitmap_alloc();
hwloc_bitmap_zero(cpuset_cmp);
hwloc_bitmap_only(cpuset_cmp, unsigned(idx));
HPX_TEST(hwloc_bitmap_compare(cpuset, cpuset_cmp) == 0);
hwloc_bitmap_free(cpuset_cmp);
}
else
{
HPX_TEST(false && "hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD) failed!");
}
hwloc_bitmap_free(cpuset);
hwloc_topology_destroy(topo);
#endif
return desired;
}
// This PX-thread has been run by the wrong OS-thread, make the foreman
// try again by rescheduling it.
return std::size_t(-1);
}
int main(int argc, char* argv[])
{
int rank, size, rc;
hwloc_cpuset_t cpus;
char *bindings;
cpu_set_t *mask;
int nrcpus, c;
size_t csize;
char hostname[1024];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
gethostname(hostname, 1024);
cpus = hwloc_bitmap_alloc();
rc = hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS);
hwloc_bitmap_list_asprintf(&bindings, cpus);
printf("[%s;%d] Hello, World, I am %d of %d [%d local peers]: get_cpubind: %d bitmap %s\n",
hostname, (int)getpid(), rank, size, orte_process_info.num_local_peers, rc,
(NULL == bindings) ? "NULL" : bindings);
nrcpus = sysconf(_SC_NPROCESSORS_ONLN);
mask = CPU_ALLOC(nrcpus);
csize = CPU_ALLOC_SIZE(nrcpus);
CPU_ZERO_S(csize, mask);
if ( sched_getaffinity(0, csize, mask) == -1 ) {
CPU_FREE(mask);
perror("sched_getaffinity");
return -1;
}
for ( c = 0; c < nrcpus; c++ ) {
if ( CPU_ISSET_S(c, csize, mask) ) {
printf("[%s:%d] CPU %d is set\n", hostname, (int)getpid(), c);
}
}
CPU_FREE(mask);
MPI_Finalize();
return 0;
}
static int chk_cpu_bind(hwloc_topology_t topology, hwloc_cpuset_t cpuset, int print)
{
hwloc_bitmap_t checkset = hwloc_bitmap_alloc();
if(hwloc_get_cpubind(topology, checkset, HWLOC_CPUBIND_THREAD) == -1){
perror("get_cpubind");
hwloc_bitmap_free(checkset);
return -1;
}
if(print){
hwloc_obj_t cpu_obj = hwloc_get_first_largest_obj_inside_cpuset(topology, checkset);
printf("cpubind=%s:%d\n",hwloc_obj_type_string(cpu_obj->type),cpu_obj->logical_index);
}
if(cpuset == NULL)
return -1;
int ret = hwloc_bitmap_isequal(cpuset,checkset);
hwloc_bitmap_free(checkset);
return ret ? 0 : -1;
}
/*
* Get the node where the current thread is running
* return the node of the core
*/
int hw_my_node()
{
int node;
hwloc_cpuset_t set;
hwloc_nodeset_t nset;
if (local_topo->nnodes != 0 ){
set = hwloc_bitmap_alloc();
nset = hwloc_bitmap_alloc();
hwloc_get_cpubind (topology,set,HWLOC_CPUBIND_THREAD);
hwloc_cpuset_to_nodeset(topology,set,nset);
node = hwloc_bitmap_first(nset);
hwloc_bitmap_free(set);
hwloc_bitmap_free(nset);
}
else
node = -1;
return node;
}
/******************* FUNCTION *********************/
int TopoHwloc::getCurrentIdFromThreadBinding(void) const
{
hwloc_nodeset_t nodeset = hwloc_bitmap_alloc();
hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
int res = -1;
int weight;
#if defined(SCTK_ALLOC_DEBUG) && defined(hwloc_bitmap_list_snprintf)
char buffer[4096];
#endif
//get current core binding
//for windows use 0 instead of HWLOC_CPUBIND_THREAD
int status = hwloc_get_cpubind (topology, cpuset, 0);
assert(status == 0);
if (status == 0)
return -1;
#if defined(SCTK_ALLOC_DEBUG) && defined(hwloc_bitmap_list_snprintf)
status = hwloc_bitmap_list_snprintf(buffer,4096,cpuset);
sprintf(stderr,"Current cores : %s\n",buffer);
#endif
//nodes from cores
hwloc_cpuset_to_nodeset(topology,cpuset,nodeset);
#if defined(SCTK_ALLOC_DEBUG) && defined(hwloc_bitmap_list_snprintf)
status = hwloc_bitmap_list_snprintf(buffer,4096,nodeset);
sprintf(stderr,"Current nodes from cores : %s\n",buffer);
#endif
//calc res
weight = hwloc_bitmap_weight(nodeset);
assert(weight != 0);
if (weight == 1)
res = getFirstBitInBitmap(nodeset);
hwloc_bitmap_free(cpuset);
hwloc_bitmap_free(nodeset);
return res;
}
void migrate(long PageStart, long PageEnd) {
SPMR_DEBUG(std::cout << "Runtime: migrate pages: " << PageStart << " to "
<< PageEnd << "\n");
SPMR_DEBUG(std::cout << "Runtime: hwloc call: " << (PageStart << PAGE_EXP)
<< ", " << ((PageEnd - PageStart) << PAGE_EXP) << "\n");
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(__spm_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_get_last_cpu_location(__spm_topo, set, HWLOC_CPUBIND_THREAD);
hwloc_bitmap_singlify(set);
assert(
hwloc_set_area_membind(__spm_topo, (const void*)(PageStart << PAGE_EXP),
(PageEnd - PageStart) << PAGE_EXP,
(hwloc_const_cpuset_t)set, HWLOC_MEMBIND_BIND,
HWLOC_MEMBIND_MIGRATE)
!= -1 && "Unable to migrate requested pages");
hwloc_bitmap_free(set);
}
int main(int argc, char* argv[])
{
int rank, size, rc;
hwloc_cpuset_t cpus;
char *bindings = NULL;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
cpus = hwloc_bitmap_alloc();
rc = hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS);
hwloc_bitmap_list_asprintf(&bindings, cpus);
}
printf("Hello, World, I am %d of %d [%d local peers]: get_cpubind: %d bitmap %s\n",
rank, size, orte_process_info.num_local_peers, rc,
(NULL == bindings) ? "NULL" : bindings);
MPI_Finalize();
return 0;
}
c_sublocid_t chpl_topo_getThreadLocality(void) {
hwloc_cpuset_t cpuset;
hwloc_nodeset_t nodeset;
int flags;
int node;
if (!haveTopology) {
return c_sublocid_any;
}
if (!topoSupport->cpubind->get_thread_cpubind) {
return c_sublocid_any;
}
if ((cpuset = hwloc_bitmap_alloc()) == NULL) {
report_error("hwloc_bitmap_alloc()", errno);
}
if ((nodeset = hwloc_bitmap_alloc()) == NULL) {
report_error("hwloc_bitmap_alloc()", errno);
}
flags = HWLOC_CPUBIND_THREAD;
if (hwloc_get_cpubind(topology, cpuset, flags)) {
report_error("hwloc_get_cpubind()", errno);
}
hwloc_cpuset_to_nodeset(topology, cpuset, nodeset);
node = hwloc_bitmap_first(nodeset);
hwloc_bitmap_free(nodeset);
hwloc_bitmap_free(cpuset);
return node;
}
hwloc::hwloc()
{
s_core_topology = std::pair<unsigned,unsigned>(0,0);
s_core_capacity = 0 ;
s_hwloc_topology = 0 ;
s_hwloc_location = 0 ;
s_process_binding = 0 ;
for ( unsigned i = 0 ; i < MAX_CORE ; ++i ) s_core[i] = 0 ;
hwloc_topology_init( & s_hwloc_topology );
hwloc_topology_load( s_hwloc_topology );
s_hwloc_location = hwloc_bitmap_alloc();
s_process_binding = hwloc_bitmap_alloc();
hwloc_get_cpubind( s_hwloc_topology , s_process_binding , HWLOC_CPUBIND_PROCESS );
// Choose a hwloc object type for the NUMA level, which may not exist.
hwloc_obj_type_t root_type = HWLOC_OBJ_TYPE_MAX ;
{
// Object types to search, in order.
static const hwloc_obj_type_t candidate_root_type[] =
{ HWLOC_OBJ_NODE /* NUMA region */
, HWLOC_OBJ_SOCKET /* hardware socket */
, HWLOC_OBJ_MACHINE /* local machine */
};
enum { CANDIDATE_ROOT_TYPE_COUNT =
sizeof(candidate_root_type) / sizeof(hwloc_obj_type_t) };
for ( int k = 0 ; k < CANDIDATE_ROOT_TYPE_COUNT && HWLOC_OBJ_TYPE_MAX == root_type ; ++k ) {
if ( 0 < hwloc_get_nbobjs_by_type( s_hwloc_topology , candidate_root_type[k] ) ) {
root_type = candidate_root_type[k] ;
}
}
}
// Determine which of these 'root' types are available to this process.
// The process may have been bound (e.g., by MPI) to a subset of these root types.
// Determine current location of the master (calling) process>
hwloc_bitmap_t proc_cpuset_location = hwloc_bitmap_alloc();
hwloc_get_last_cpu_location( s_hwloc_topology , proc_cpuset_location , HWLOC_CPUBIND_THREAD );
const unsigned max_root = hwloc_get_nbobjs_by_type( s_hwloc_topology , root_type );
unsigned root_base = max_root ;
unsigned root_count = 0 ;
unsigned core_per_root = 0 ;
unsigned pu_per_core = 0 ;
bool symmetric = true ;
for ( unsigned i = 0 ; i < max_root ; ++i ) {
const hwloc_obj_t root = hwloc_get_obj_by_type( s_hwloc_topology , root_type , i );
if ( hwloc_bitmap_intersects( s_process_binding , root->allowed_cpuset ) ) {
++root_count ;
// Remember which root (NUMA) object the master thread is running on.
// This will be logical NUMA rank #0 for this process.
if ( hwloc_bitmap_intersects( proc_cpuset_location, root->allowed_cpuset ) ) {
root_base = i ;
}
// Count available cores:
const unsigned max_core =
hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology ,
root->allowed_cpuset ,
HWLOC_OBJ_CORE );
unsigned core_count = 0 ;
for ( unsigned j = 0 ; j < max_core ; ++j ) {
const hwloc_obj_t core =
hwloc_get_obj_inside_cpuset_by_type( s_hwloc_topology ,
root->allowed_cpuset ,
HWLOC_OBJ_CORE , j );
// If process' cpuset intersects core's cpuset then process can access this core.
// Must use intersection instead of inclusion because the Intel-Phi
// MPI may bind the process to only one of the core's hyperthreads.
//
// Assumption: if the process can access any hyperthread of the core
// then it has ownership of the entire core.
// This assumes that it would be performance-detrimental
// to spawn more than one MPI process per core and use nested threading.
if ( hwloc_bitmap_intersects( s_process_binding , core->allowed_cpuset ) ) {
++core_count ;
const unsigned pu_count =
hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology ,
core->allowed_cpuset ,
HWLOC_OBJ_PU );
if ( pu_per_core == 0 ) pu_per_core = pu_count ;
// Enforce symmetry by taking the minimum:
pu_per_core = std::min( pu_per_core , pu_count );
if ( pu_count != pu_per_core ) symmetric = false ;
}
}
if ( 0 == core_per_root ) core_per_root = core_count ;
// Enforce symmetry by taking the minimum:
core_per_root = std::min( core_per_root , core_count );
if ( core_count != core_per_root ) symmetric = false ;
}
}
s_core_topology.first = root_count ;
s_core_topology.second = core_per_root ;
s_core_capacity = pu_per_core ;
// Fill the 's_core' array for fast mapping from a core coordinate to the
// hwloc cpuset object required for thread location querying and binding.
for ( unsigned i = 0 ; i < max_root ; ++i ) {
const unsigned root_rank = ( i + root_base ) % max_root ;
const hwloc_obj_t root = hwloc_get_obj_by_type( s_hwloc_topology , root_type , root_rank );
if ( hwloc_bitmap_intersects( s_process_binding , root->allowed_cpuset ) ) {
const unsigned max_core =
hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology ,
root->allowed_cpuset ,
HWLOC_OBJ_CORE );
unsigned core_count = 0 ;
for ( unsigned j = 0 ; j < max_core && core_count < core_per_root ; ++j ) {
const hwloc_obj_t core =
hwloc_get_obj_inside_cpuset_by_type( s_hwloc_topology ,
root->allowed_cpuset ,
HWLOC_OBJ_CORE , j );
if ( hwloc_bitmap_intersects( s_process_binding , core->allowed_cpuset ) ) {
s_core[ core_count + core_per_root * i ] = core->allowed_cpuset ;
++core_count ;
}
}
}
}
hwloc_bitmap_free( proc_cpuset_location );
if ( ! symmetric ) {
std::cout << "KokkosArray::hwloc WARNING: Using a symmetric subset of a non-symmetric core topology."
<< std::endl ;
}
}
int
main (int argc, char *argv[])
{
int err;
hwloc_topology_t topology;
const char *filename = NULL;
unsigned long flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES | HWLOC_TOPOLOGY_FLAG_ICACHES;
unsigned long restrict_flags = 0;
int merge = 0;
int ignorecache = 0;
char * callname;
char * input = NULL;
enum hwloc_utils_input_format input_format = HWLOC_UTILS_INPUT_DEFAULT;
enum output_format output_format = LSTOPO_OUTPUT_DEFAULT;
char *restrictstring = NULL;
struct lstopo_output loutput;
int opt;
unsigned i;
loutput.overwrite = 0;
loutput.logical = -1;
loutput.legend = 1;
loutput.verbose_mode = LSTOPO_VERBOSE_MODE_DEFAULT;
for(i=0; i<HWLOC_OBJ_TYPE_MAX; i++)
force_orient[i] = LSTOPO_ORIENT_NONE;
force_orient[HWLOC_OBJ_PU] = LSTOPO_ORIENT_HORIZ;
force_orient[HWLOC_OBJ_CACHE] = LSTOPO_ORIENT_HORIZ;
force_orient[HWLOC_OBJ_NUMANODE] = LSTOPO_ORIENT_HORIZ;
/* enable verbose backends */
putenv("HWLOC_XML_VERBOSE=1");
putenv("HWLOC_SYNTHETIC_VERBOSE=1");
#ifdef HAVE_SETLOCALE
setlocale(LC_ALL, "");
#endif
callname = strrchr(argv[0], '/');
if (!callname)
callname = argv[0];
else
callname++;
/* skip argv[0], handle options */
argc--;
argv++;
err = hwloc_topology_init (&topology);
if (err)
return EXIT_FAILURE;
while (argc >= 1)
{
opt = 0;
if (!strcmp (argv[0], "-v") || !strcmp (argv[0], "--verbose")) {
loutput.verbose_mode++;
} else if (!strcmp (argv[0], "-s") || !strcmp (argv[0], "--silent")) {
loutput.verbose_mode--;
} else if (!strcmp (argv[0], "-h") || !strcmp (argv[0], "--help")) {
usage(callname, stdout);
exit(EXIT_SUCCESS);
} else if (!strcmp (argv[0], "-f") || !strcmp (argv[0], "--force"))
loutput.overwrite = 1;
else if (!strcmp (argv[0], "-l") || !strcmp (argv[0], "--logical"))
loutput.logical = 1;
else if (!strcmp (argv[0], "-p") || !strcmp (argv[0], "--physical"))
loutput.logical = 0;
else if (!strcmp (argv[0], "-c") || !strcmp (argv[0], "--cpuset"))
lstopo_show_cpuset = 1;
else if (!strcmp (argv[0], "-C") || !strcmp (argv[0], "--cpuset-only"))
lstopo_show_cpuset = 2;
else if (!strcmp (argv[0], "--taskset")) {
lstopo_show_taskset = 1;
if (!lstopo_show_cpuset)
lstopo_show_cpuset = 1;
} else if (!strcmp (argv[0], "--only")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
if (hwloc_obj_type_sscanf(argv[1], &lstopo_show_only, NULL, NULL, 0) < 0)
fprintf(stderr, "Unsupported type `%s' passed to --only, ignoring.\n", argv[1]);
opt = 1;
}
else if (!strcmp (argv[0], "--ignore")) {
hwloc_obj_type_t type;
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
if (hwloc_obj_type_sscanf(argv[1], &type, NULL, NULL, 0) < 0)
fprintf(stderr, "Unsupported type `%s' passed to --ignore, ignoring.\n", argv[1]);
else if (type == HWLOC_OBJ_PU)
lstopo_ignore_pus = 1;
else
hwloc_topology_ignore_type(topology, type);
opt = 1;
}
else if (!strcmp (argv[0], "--no-caches"))
ignorecache = 2;
else if (!strcmp (argv[0], "--no-useless-caches"))
ignorecache = 1;
else if (!strcmp (argv[0], "--no-icaches"))
flags &= ~HWLOC_TOPOLOGY_FLAG_ICACHES;
else if (!strcmp (argv[0], "--whole-system"))
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
else if (!strcmp (argv[0], "--no-io"))
flags &= ~(HWLOC_TOPOLOGY_FLAG_IO_DEVICES | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES);
else if (!strcmp (argv[0], "--no-bridges"))
flags &= ~(HWLOC_TOPOLOGY_FLAG_IO_BRIDGES);
else if (!strcmp (argv[0], "--whole-io"))
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_IO;
else if (!strcmp (argv[0], "--merge"))
merge = 1;
else if (!strcmp (argv[0], "--no-collapse"))
lstopo_collapse = 0;
else if (!strcmp (argv[0], "--thissystem"))
flags |= HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM;
else if (!strcmp (argv[0], "--restrict")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
restrictstring = strdup(argv[1]);
opt = 1;
}
else if (!strcmp (argv[0], "--restrict-flags")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
restrict_flags = (unsigned long) strtoull(argv[1], NULL, 0);
opt = 1;
}
else if (!strcmp (argv[0], "--export-synthetic-flags")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
lstopo_export_synthetic_flags = (unsigned long) strtoull(argv[1], NULL, 0);
opt = 1;
}
else if (!strcmp (argv[0], "--horiz"))
for(i=0; i<HWLOC_OBJ_TYPE_MAX; i++)
force_orient[i] = LSTOPO_ORIENT_HORIZ;
else if (!strcmp (argv[0], "--vert"))
for(i=0; i<HWLOC_OBJ_TYPE_MAX; i++)
force_orient[i] = LSTOPO_ORIENT_VERT;
else if (!strcmp (argv[0], "--rect"))
for(i=0; i<HWLOC_OBJ_TYPE_MAX; i++)
force_orient[i] = LSTOPO_ORIENT_RECT;
else if (!strncmp (argv[0], "--horiz=", 8)
|| !strncmp (argv[0], "--vert=", 7)
|| !strncmp (argv[0], "--rect=", 7)) {
enum lstopo_orient_e orient = (argv[0][2] == 'h') ? LSTOPO_ORIENT_HORIZ : (argv[0][2] == 'v') ? LSTOPO_ORIENT_VERT : LSTOPO_ORIENT_RECT;
char *tmp = argv[0] + ((argv[0][2] == 'h') ? 8 : 7);
while (tmp) {
char *end = strchr(tmp, ',');
hwloc_obj_type_t type;
if (end)
*end = '\0';
if (hwloc_obj_type_sscanf(tmp, &type, NULL, NULL, 0) < 0)
fprintf(stderr, "Unsupported type `%s' passed to %s, ignoring.\n", tmp, argv[0]);
else
force_orient[type] = orient;
if (!end)
break;
tmp = end+1;
}
}
else if (!strcmp (argv[0], "--fontsize")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
fontsize = atoi(argv[1]);
opt = 1;
}
else if (!strcmp (argv[0], "--gridsize")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
gridsize = atoi(argv[1]);
opt = 1;
}
else if (!strcmp (argv[0], "--no-legend")) {
loutput.legend = 0;
}
else if (!strcmp (argv[0], "--append-legend")) {
char **tmp;
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
tmp = realloc(lstopo_append_legends, (lstopo_append_legends_nr+1) * sizeof(*lstopo_append_legends));
if (!tmp) {
fprintf(stderr, "Failed to realloc legend append array, legend ignored.\n");
} else {
lstopo_append_legends = tmp;
lstopo_append_legends[lstopo_append_legends_nr] = strdup(argv[1]);
lstopo_append_legends_nr++;
}
opt = 1;
}
else if (hwloc_utils_lookup_input_option(argv, argc, &opt,
&input, &input_format,
callname)) {
/* nothing to do anymore */
} else if (!strcmp (argv[0], "--pid")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
lstopo_pid_number = atoi(argv[1]); opt = 1;
} else if (!strcmp (argv[0], "--ps") || !strcmp (argv[0], "--top"))
top = 1;
else if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, HWLOC_VERSION);
exit(EXIT_SUCCESS);
} else if (!strcmp (argv[0], "--output-format") || !strcmp (argv[0], "--of")) {
if (argc < 2) {
usage (callname, stderr);
exit(EXIT_FAILURE);
}
output_format = parse_output_format(argv[1], callname);
opt = 1;
} else {
if (filename) {
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage (callname, stderr);
exit(EXIT_FAILURE);
} else
filename = argv[0];
}
argc -= opt+1;
argv += opt+1;
}
if (lstopo_show_only != (hwloc_obj_type_t)-1)
merge = 0;
hwloc_topology_set_flags(topology, flags);
if (ignorecache > 1) {
hwloc_topology_ignore_type(topology, HWLOC_OBJ_CACHE);
} else if (ignorecache) {
hwloc_topology_ignore_type_keep_structure(topology, HWLOC_OBJ_CACHE);
}
if (merge)
hwloc_topology_ignore_all_keep_structure(topology);
if (input) {
err = hwloc_utils_enable_input_format(topology, input, &input_format, loutput.verbose_mode > 1, callname);
if (err)
return err;
}
if (lstopo_pid_number > 0) {
lstopo_pid = hwloc_pid_from_number(lstopo_pid_number, 0);
if (hwloc_topology_set_pid(topology, lstopo_pid)) {
perror("Setting target pid");
return EXIT_FAILURE;
}
}
/* if the output format wasn't enforced, look at the filename */
if (filename && output_format == LSTOPO_OUTPUT_DEFAULT) {
if (!strcmp(filename, "-")
|| !strcmp(filename, "/dev/stdout")) {
output_format = LSTOPO_OUTPUT_CONSOLE;
} else {
char *dot = strrchr(filename, '.');
if (dot)
output_format = parse_output_format(dot+1, callname);
else {
fprintf(stderr, "Cannot infer output type for file `%s' without any extension, using default output.\n", filename);
filename = NULL;
}
}
}
/* if the output format wasn't enforced, think a bit about what the user probably want */
if (output_format == LSTOPO_OUTPUT_DEFAULT) {
if (lstopo_show_cpuset
|| lstopo_show_only != (hwloc_obj_type_t)-1
|| loutput.verbose_mode != LSTOPO_VERBOSE_MODE_DEFAULT)
output_format = LSTOPO_OUTPUT_CONSOLE;
}
if (input_format == HWLOC_UTILS_INPUT_XML
&& output_format == LSTOPO_OUTPUT_XML) {
/* must be after parsing output format and before loading the topology */
putenv("HWLOC_XML_USERDATA_NOT_DECODED=1");
hwloc_topology_set_userdata_import_callback(topology, hwloc_utils_userdata_import_cb);
hwloc_topology_set_userdata_export_callback(topology, hwloc_utils_userdata_export_cb);
}
err = hwloc_topology_load (topology);
if (err) {
fprintf(stderr, "hwloc_topology_load() failed (%s).\n", strerror(errno));
return EXIT_FAILURE;
}
if (top)
add_process_objects(topology);
if (restrictstring) {
hwloc_bitmap_t restrictset = hwloc_bitmap_alloc();
if (!strcmp (restrictstring, "binding")) {
if (lstopo_pid_number > 0)
hwloc_get_proc_cpubind(topology, lstopo_pid, restrictset, HWLOC_CPUBIND_PROCESS);
else
hwloc_get_cpubind(topology, restrictset, HWLOC_CPUBIND_PROCESS);
} else {
hwloc_bitmap_sscanf(restrictset, restrictstring);
}
err = hwloc_topology_restrict (topology, restrictset, restrict_flags);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
free(restrictstring);
}
if (loutput.logical == -1) {
if (output_format == LSTOPO_OUTPUT_CONSOLE)
loutput.logical = 1;
else if (output_format != LSTOPO_OUTPUT_DEFAULT)
loutput.logical = 0;
}
loutput.topology = topology;
loutput.file = NULL;
lstopo_populate_userdata(hwloc_get_root_obj(topology));
if (output_format != LSTOPO_OUTPUT_XML && lstopo_collapse)
lstopo_add_collapse_attributes(topology);
switch (output_format) {
case LSTOPO_OUTPUT_DEFAULT:
#ifdef LSTOPO_HAVE_GRAPHICS
#if CAIRO_HAS_XLIB_SURFACE && defined HWLOC_HAVE_X11_KEYSYM
if (getenv("DISPLAY")) {
if (loutput.logical == -1)
loutput.logical = 0;
output_x11(&loutput, NULL);
} else
#endif /* CAIRO_HAS_XLIB_SURFACE */
#ifdef HWLOC_WIN_SYS
{
if (loutput.logical == -1)
loutput.logical = 0;
output_windows(&loutput, NULL);
}
#endif
#endif /* !LSTOPO_HAVE_GRAPHICS */
#if !defined HWLOC_WIN_SYS || !defined LSTOPO_HAVE_GRAPHICS
{
if (loutput.logical == -1)
loutput.logical = 1;
output_console(&loutput, NULL);
}
#endif
break;
case LSTOPO_OUTPUT_CONSOLE:
output_console(&loutput, filename);
break;
case LSTOPO_OUTPUT_SYNTHETIC:
output_synthetic(&loutput, filename);
break;
case LSTOPO_OUTPUT_ASCII:
output_ascii(&loutput, filename);
break;
case LSTOPO_OUTPUT_FIG:
output_fig(&loutput, filename);
break;
#ifdef LSTOPO_HAVE_GRAPHICS
# if CAIRO_HAS_PNG_FUNCTIONS
case LSTOPO_OUTPUT_PNG:
output_png(&loutput, filename);
break;
# endif /* CAIRO_HAS_PNG_FUNCTIONS */
# if CAIRO_HAS_PDF_SURFACE
case LSTOPO_OUTPUT_PDF:
output_pdf(&loutput, filename);
break;
# endif /* CAIRO_HAS_PDF_SURFACE */
# if CAIRO_HAS_PS_SURFACE
case LSTOPO_OUTPUT_PS:
output_ps(&loutput, filename);
break;
#endif /* CAIRO_HAS_PS_SURFACE */
#if CAIRO_HAS_SVG_SURFACE
case LSTOPO_OUTPUT_SVG:
output_svg(&loutput, filename);
break;
#endif /* CAIRO_HAS_SVG_SURFACE */
#endif /* LSTOPO_HAVE_GRAPHICS */
case LSTOPO_OUTPUT_XML:
output_xml(&loutput, filename);
break;
default:
fprintf(stderr, "file format not supported\n");
usage(callname, stderr);
exit(EXIT_FAILURE);
}
lstopo_destroy_userdata(hwloc_get_root_obj(topology));
hwloc_utils_userdata_free_recursive(hwloc_get_root_obj(topology));
hwloc_topology_destroy (topology);
for(i=0; i<lstopo_append_legends_nr; i++)
free(lstopo_append_legends[i]);
free(lstopo_append_legends);
return EXIT_SUCCESS;
}
int orte_ess_base_proc_binding(void)
{
#if OPAL_HAVE_HWLOC
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
char *map;
int ret;
char *error;
/* Determine if we were pre-bound or not */
if (NULL != getenv("OMPI_MCA_orte_bound_at_launch")) {
orte_proc_is_bound = true;
if (NULL != (map = getenv("OMPI_MCA_orte_base_applied_binding"))) {
orte_proc_applied_binding = hwloc_bitmap_alloc();
if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) {
error = "applied_binding parse";
goto error;
}
}
}
/* see if we were bound when launched */
if (!orte_proc_is_bound) {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Not bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* we were not bound at launch */
if (NULL != opal_hwloc_topology) {
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Binding not supported",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
orte_proc_is_bound = true;
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process was externally bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
/* try to find a level and index for this location */
opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx);
/* cleanup */
hwloc_bitmap_free(cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound according to slot_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == orte_process_info.my_node_rank) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process not bound - no node rank available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_HWTHREAD_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to hwthread",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = ORTE_ERROR;
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to core",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL;
} else {
ret = ORTE_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology,
obj, OPAL_HWLOC_LOGICAL);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_hwloc_base_print_level(orte_process_info.bind_level)));
break;
}
}
if (!orte_proc_is_bound) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
}
}
}
} else {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* report bindings, if requested */
if (opal_hwloc_report_bindings) {
char bindings[64];
hwloc_obj_t root;
hwloc_cpuset_t cpus;
/* get the root object for this node */
root = hwloc_get_root_obj(opal_hwloc_topology);
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* we are not bound if this equals our cpuset */
if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) {
opal_output(0, "%s is not bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
} else {
hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset);
opal_output(0, "%s is bound to cpus %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
bindings);
}
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
#else
return ORTE_SUCCESS;
#endif
}
static void set(orte_job_t *jobdat,
orte_proc_t *child,
char ***environ_copy,
int write_fd)
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
orte_app_context_t *context;
int rc=ORTE_ERROR;
char *msg, *param;
char *cpu_bitmap;
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set on child %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
if (NULL == jobdat || NULL == child) {
/* nothing for us to do */
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set jobdat %s child %s - nothing to do",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == jobdat) ? "NULL" : ORTE_JOBID_PRINT(jobdat->jobid),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
return;
}
context = (orte_app_context_t*)opal_pointer_array_get_item(jobdat->apps, child->app_idx);
/* Set process affinity, if given */
cpu_bitmap = NULL;
if (!orte_get_attribute(&child->attributes, ORTE_PROC_CPU_BITMAP, (void**)&cpu_bitmap, OPAL_STRING) ||
NULL == cpu_bitmap || 0 == strlen(cpu_bitmap)) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
rc = hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
char *tmp;
(void)hwloc_bitmap_list_asprintf(&tmp, sum->available);
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), tmp);
free(tmp);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
return;
}
}
}
if (0 == rc && opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
} else {
/* convert the list to a cpuset */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
return;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
}
if (NULL != cpu_bitmap) {
free(cpu_bitmap);
}
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
int loaded = 0;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
unsigned long flags = 0;
int force = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
int pid_number = -1;
hwloc_pid_t pid = 0; /* only valid when pid_number > 0, but gcc-4.8 still reports uninitialized warnings */
char *callname;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
/* don't load now, in case some options change the config before the topology is actually used */
#define LOADED() (loaded)
#define ENSURE_LOADED() do { \
if (!loaded) { \
hwloc_topology_init(&topology); \
hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_ALL); \
hwloc_topology_set_flags(topology, flags); \
hwloc_topology_load(topology); \
depth = hwloc_topology_get_depth(topology); \
loaded = 1; \
} \
} while (0)
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, HWLOC_VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--whole-system")) {
if (loaded) {
fprintf(stderr, "Input option %s disallowed after options using the topology\n", argv[0]);
exit(EXIT_FAILURE);
}
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
goto next;
}
if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
ENSURE_LOADED();
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ENSURE_LOADED();
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
working_on_cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
ENSURE_LOADED();
if (pid_number > 0) {
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
/* no need to set_pid()
* the doc just says we're operating on pid, not that we're retrieving the topo/cpuset as seen from inside pid
*/
}
if (get_last_cpu_location && !working_on_cpubind) {
fprintf(stderr, "Options --membind and --get-last-cpu-location cannot be combined.\n");
return EXIT_FAILURE;
}
if ((get_binding || get_last_cpu_location) && (got_cpubind || got_membind)) {
/* doesn't work because get_binding/get_last_cpu_location overwrites cpubind_set */
fprintf(stderr, "Cannot display and set binding at the same time.\n");
return EXIT_FAILURE;
}
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number > 0)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number > 0)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid_number > 0)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number > 0)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_membind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number > 0)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_cpubind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number > 0)
return EXIT_SUCCESS;
if (0 == argc) {
if (get_binding || get_last_cpu_location)
return EXIT_SUCCESS;
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
/* FIXME: check whether Windows execvp() passes INHERIT_PARENT_AFFINITY to CreateProcess()
* because we need to propagate processor group affinity. However process-wide affinity
* isn't supported with processor groups so far.
*/
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* NTH: this is no longer used but may be used if we can determine the binding policy*/
static int mca_sbgp_map_to_logical_socket_id(int *socket)
{
int ret = OMPI_SUCCESS;
hwloc_obj_t obj;
hwloc_obj_t first_pu_object;
hwloc_bitmap_t good;
int pu_os_index = -1, my_logical_socket_id = -1;
int this_pus_logical_socket_id = -1;
*socket = my_logical_socket_id;
/* bozo check */
if (NULL == opal_hwloc_topology) {
return OPAL_ERR_NOT_INITIALIZED;
}
good = hwloc_bitmap_alloc();
if (NULL == good) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* get this process' CPU binding */
if( 0 != hwloc_get_cpubind(opal_hwloc_topology,good, 0)){
/* report some error */
BASESMSOCKET_VERBOSE(10, "The global variable opal_hwloc_topology appears not to have been initialized\n");
hwloc_bitmap_free(good);
return OMPI_ERROR;
}
/* find the first logical PU object in the hwloc tree */
first_pu_object = hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU, 0);
/* get the next bit in the bitmap (note: if pu_os_index == -1, then the
* first bit is returned
*/
/* traverse the hwloc tree */
while( -1 != (pu_os_index = hwloc_bitmap_next(good, pu_os_index) ) ) {
/* Traverse all PUs in the machine in logical order, in the simple case
* there should only be a single PU that this process is bound to, right?
*
*/
for( obj = first_pu_object; obj != NULL; obj = obj->next_cousin ) {/* WTF is a "next_cousin" ? */
/* is this PU the same as the bit I pulled off the mask? */
if( obj->os_index == (unsigned int) pu_os_index) {
/* Then I found it, break out of for loop */
break;
}
}
if( NULL != obj) {
/* if we found the PU, then go upward in the tree
* looking for the enclosing socket
*/
while( (NULL != obj) && ( HWLOC_OBJ_SOCKET != obj->type) ){
obj = obj->parent;
}
if( NULL == obj ) {
/* then we couldn't find an enclosing socket, report this */
} else {
/* We found the enclosing socket */
if( -1 == my_logical_socket_id ){
/* this is the first PU that I'm bound to */
this_pus_logical_socket_id = obj->logical_index;
my_logical_socket_id = this_pus_logical_socket_id;
} else {
/* this is not the first PU that I'm bound to.
* Seems I'm bound to more than a single PU. Question
* is, am I bound to the same socket??
*/
/* in order to get rid of the compiler warning, I had to cast
* "this_pus_logical_socket_id", at a glance this seems ok,
* but if subgrouping problems arise, maybe look here. I shall
* tag this line with the "mark of the beast" for grepability
* 666
*/
if( (unsigned int) this_pus_logical_socket_id != obj->logical_index ){
/* 666 */
/* Then we're bound to more than one socket...fail */
this_pus_logical_socket_id = -1;
my_logical_socket_id = -1;
break;
}
}
}
}
/* end while */
}
*socket = my_logical_socket_id;
hwloc_bitmap_free(good);
return ret;
}
void computeCPU(DataArgsType& args)
{
char * a;
int cid;
hwloc_cpuset_t set = nullptr;
std::ostringstream oss;
/* This initialization takes some time */
/* When using hartsi, the object instanciation is done when creating tasks */
/* and this is not a parallel section, thus we lose time in initialization */
/* doing it the computation step allows to incorporate this init time in the parallel section */
/*
M_threadId( threadId ),
M_gm( new gm_type( *_elt.gm() ) ),
M_geopc( new gmpc_type( M_gm, _im.points() ) ),
M_c( new gmc_type( M_gm, _elt, M_geopc ) ),
M_expr( _expr, map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) ),
M_im( _im ),
M_ret( eval::matrix_type::Zero() ),
M_cpuTime( 0.0 )
*/
#if 0
/* get a cpuset object */
set = hwloc_bitmap_alloc();
/* Get the cpu thread affinity info of the current process/thread */
hwloc_get_cpubind(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ")|";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
/* Get the latest core location of the current process/thread */
hwloc_get_last_cpu_location(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ");";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
#endif
perf_mng.init("1.1") ;
perf_mng.init("1.1") ;
perf_mng.init("2.1") ;
perf_mng.init("2.2") ;
perf_mng.init("3") ;
/* free memory */
if(set != nullptr)
{
hwloc_bitmap_free(set);
}
//perf_mng.init("data") ;
//perf_mng.start("data") ;
// DEFINE the range to be iterated on
std::vector<std::pair<element_iterator, element_iterator> > * elts =
args.get("elements")->get<std::vector<std::pair<element_iterator, element_iterator> > >();
int * threadId = args.get("threadId")->get<int>();
expression_type * expr = args.get("expr")->get<expression_type>();
im_type * im = args.get("im")->get<im_type>();
element_iterator * elt_it = args.get("elt")->get<element_iterator>();
//M_gm((*elt_it)->gm());
gm_ptrtype gm = (*elt_it)->gm();
//M_geopc(new typename eval::gmpc_type( M_gm, im->points() ));
typename eval::gmpc_ptrtype __geopc( new typename eval::gmpc_type(gm, im->points()) );
//M_c(new gmc_type( M_gm, *(*elt_it), M_geopc ));
gmc_ptrtype __c( new gmc_type( gm, *(*elt_it), __geopc ) );
//M_expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) );
eval_expr_type __expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( __c ) ) );
//perf_mng.stop("data");
perf_mng.init("cpu") ;
perf_mng.start("cpu") ;
for (int i = 0; i < elts->size(); i++)
{
//std::cout << Environment::worldComm().rank() << " nbItems: " << elts->size() << " nbElts " << std::distance(elts->at(i), elts->at(i+1)) << std::endl;
for ( auto _elt = elts->at(i).first; _elt != elts->at(i).second; ++_elt )
{
//perf_mng.start("1.1") ;
//M_c->update( *_elt );
__c->update( *_elt );
//perf_mng.stop("1.1") ;
//perf_mng.start("1.2") ;
map_gmc_type mapgmc( fusion::make_pair<vf::detail::gmc<0> >( __c ) );
//perf_mng.stop("1.2") ;
//perf_mng.start("2.1") ;
__expr.update( mapgmc );
//perf_mng.stop("2.1") ;
//perf_mng.start("2.2") ;
im->update( *__c );
//perf_mng.stop("2.2") ;
//perf_mng.start("3") ;
for ( uint16_type c1 = 0; c1 < eval::shape::M; ++c1 )
{
for ( uint16_type c2 = 0; c2 < eval::shape::N; ++c2 )
{
M_ret( c1,c2 ) += (*im)( __expr, c1, c2 );
}
}
//perf_mng.stop("3") ;
}
}
perf_mng.stop("cpu") ;
M_cpuTime = perf_mng.getValueInSeconds("cpu");
}
static int do_child(orte_app_context_t* context,
orte_proc_t *child,
char **environ_copy,
orte_job_t *jobdat, int write_fd,
orte_iof_base_io_conf_t opts)
{
int i, rc;
sigset_t sigs;
long fd, fdmax = sysconf(_SC_OPEN_MAX);
char *param, *msg;
if (orte_forward_job_control) {
/* Set a new process group for this child, so that a
SIGSTOP can be sent to it without being sent to the
orted. */
setpgid(0, 0);
}
/* Setup the pipe to be close-on-exec */
fcntl(write_fd, F_SETFD, FD_CLOEXEC);
if (NULL != child) {
/* setup stdout/stderr so that any error messages that we
may print out will get displayed back at orterun.
NOTE: Definitely do this AFTER we check contexts so
that any error message from those two functions doesn't
come out to the user. IF we didn't do it in this order,
THEN a user who gives us a bad executable name or
working directory would get N error messages, where
N=num_procs. This would be very annoying for large
jobs, so instead we set things up so that orterun
always outputs a nice, single message indicating what
happened
*/
if (ORTE_SUCCESS != (i = orte_iof_base_setup_child(&opts,
&environ_copy))) {
ORTE_ERROR_LOG(i);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt",
"iof setup failed",
orte_process_info.nodename, context->app);
/* Does not return */
}
#if OPAL_HAVE_HWLOC
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
/* Set process affinity, if given */
if (NULL == child->cpu_bitmap) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
} else {
if (0 == strlen(child->cpu_bitmap)) {
/* this proc is not bound */
if (opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
}
/* provide a nice string representation of what we bound to */
(void) mca_base_var_env_name ("orte_base_applied_binding", ¶m);
opal_setenv(param, child->cpu_bitmap, true, &environ_copy);
free(param);
goto PROCEED;
}
/* convert the list to a cpu bitmap */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, child->cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), child->cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
goto PROCEED;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
goto PROCEED;
}
}
}
}
#endif
} else if (!(ORTE_JOB_CONTROL_FORWARD_OUTPUT & jobdat->controls)) {
/* tie stdin/out/err/internal to /dev/null */
int fdnull;
for (i=0; i < 3; i++) {
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > i && i != write_fd) {
dup2(fdnull, i);
}
close(fdnull);
}
fdnull = open("/dev/null", O_RDONLY, 0);
if (fdnull > opts.p_internal[1]) {
dup2(fdnull, opts.p_internal[1]);
}
close(fdnull);
}
#if OPAL_HAVE_HWLOC
PROCEED:
#endif
/* if the user requested it, set the system resource limits */
if (OPAL_SUCCESS != (rc = opal_util_init_sys_limits(&msg))) {
send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"set limit",
orte_process_info.nodename, context->app,
__FILE__, __LINE__, msg);
}
/* ensure we only do this once */
(void) mca_base_var_env_name("opal_set_max_sys_limits", ¶m);
opal_unsetenv(param, &environ_copy);
free(param);
/* close all file descriptors w/ exception of stdin/stdout/stderr,
the pipe used for the IOF INTERNAL messages, and the pipe up to
the parent. */
for(fd=3; fd<fdmax; fd++) {
if (fd != opts.p_internal[1] && fd != write_fd) {
close(fd);
}
}
if (context->argv == NULL) {
context->argv = malloc(sizeof(char*)*2);
context->argv[0] = strdup(context->app);
context->argv[1] = NULL;
}
/* Set signal handlers back to the default. Do this close to
the exev() because the event library may (and likely will)
reset them. If we don't do this, the event library may
have left some set that, at least on some OS's, don't get
reset via fork() or exec(). Hence, the launched process
could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that we
set the default handlers, above. This is noticable on
Linux where the event library blocks SIGTERM, but we don't
want that blocked by the launched process. */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
/* Exec the new executable */
if (10 < opal_output_get_verbosity(orte_odls_base_framework.framework_output)) {
int jout;
opal_output(0, "%s STARTING %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), context->app);
for (jout=0; NULL != context->argv[jout]; jout++) {
opal_output(0, "%s\tARGV[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, context->argv[jout]);
}
for (jout=0; NULL != environ_copy[jout]; jout++) {
opal_output(0, "%s\tENVIRON[%d]: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), jout, environ_copy[jout]);
}
}
execve(context->app, context->argv, environ_copy);
send_error_show_help(write_fd, 1,
"help-orte-odls-default.txt", "execve error",
orte_process_info.nodename, context->app, strerror(errno));
/* Does not return */
}
void computeCPUOMP(int threadId, expression_type * expr, im_type * im, element_iterator * elt_it, std::vector<std::pair<element_iterator, element_iterator> > * elts)
{
char * a;
int cid;
std::ostringstream oss;
#if 0
hwloc_cpuset_t set = nullptr;
/* get a cpuset object */
set = hwloc_bitmap_alloc();
/* Get the cpu thread affinity info of the current process/thread */
hwloc_get_cpubind(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ")|";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
/* Get the latest core location of the current process/thread */
hwloc_get_last_cpu_location(Environment::getHwlocTopology(), set, 0);
hwloc_bitmap_asprintf(&a, set);
oss << a;
free(a);
cid = hwloc_bitmap_first(set);
oss << "(";
while(cid != -1)
{
oss << cid << " ";
cid = hwloc_bitmap_next(set, cid);
}
oss << ");";
std::cout << Environment::worldComm().rank() << "|" << M_threadId << " " << oss.str() << std::endl;
#endif
#if defined(FEELPP_HAS_HARTS)
perf_mng.init("cpu") ;
perf_mng.start("cpu") ;
perf_mng.init("1.1") ;
perf_mng.init("1.2") ;
perf_mng.init("2.1") ;
perf_mng.init("2.2") ;
perf_mng.init("3") ;
#endif
//M_gm((*elt_it)->gm());
gm_ptrtype gm = (*elt_it)->gm();
//M_geopc(new typename eval::gmpc_type( M_gm, im->points() ));
typename eval::gmpc_ptrtype __geopc( new typename eval::gmpc_type(gm, im->points()) );
//M_c(new gmc_type( M_gm, *(*elt_it), M_geopc ));
gmc_ptrtype __c( new gmc_type( gm, *(*elt_it), __geopc ) );
//M_expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( M_c ) ) );
eval_expr_type __expr( (*expr), map_gmc_type( fusion::make_pair<vf::detail::gmc<0> >( __c ) ) );
for (int i = 0; i < elts->size(); i++)
{
/*
std::cout << Environment::worldComm().rank() << " nbItems: " << elts->size()
<< " nbElts " << std::distance(elts->at(i), elts->at(i+1))
<< " 1st id " << elts->at(i)->id() << std::endl;
*/
//std::cout << Environment::worldComm().rank() << "|" << theadId << " fid=" elts.at(i).first.id() << std::endl;
for ( auto _elt = elts->at(i).first; _elt != elts->at(i).second; ++_elt )
{
//perf_mng.start("1.1") ;
__c->update( *_elt );
//perf_mng.stop("1.1") ;
//perf_mng.start("1.2") ;
map_gmc_type mapgmc( fusion::make_pair<vf::detail::gmc<0> >( __c ) );
//perf_mng.stop("1.2") ;
//perf_mng.start("2.1") ;
__expr.update( mapgmc );
//perf_mng.stop("2.1") ;
//perf_mng.start("2.2") ;
im->update( *__c );
//perf_mng.stop("2.2") ;
//perf_mng.start("3") ;
for ( uint16_type c1 = 0; c1 < eval::shape::M; ++c1 )
{
for ( uint16_type c2 = 0; c2 < eval::shape::N; ++c2 )
{
M_ret( c1,c2 ) += (*im)( __expr, c1, c2 );
}
}
//perf_mng.stop("3") ;
}
}
#if defined(FEELPP_HAS_HARTS)
perf_mng.stop("cpu") ;
M_cpuTime = perf_mng.getValueInSeconds("cpu");
#endif
}
static void create_hwloc_cpusets() {
#ifdef USE_HWLOC
int i;
int err = hwloc_topology_init(&topology);
assert(err == 0);
err = hwloc_topology_load(topology);
assert(err == 0);
hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
assert(cpuset);
err = hwloc_get_cpubind(topology, cpuset, HWLOC_CPUBIND_PROCESS);
assert(err == 0);
const int available_pus = hwloc_bitmap_weight(cpuset);
const int last_set_index = hwloc_bitmap_last(cpuset);
const int num_workers = hc_context->nworkers;
hclib_affinity_t selected_affinity = HCLIB_AFFINITY_STRIDED;
const char *user_selected_affinity = getenv("HCLIB_AFFINITY");
if (user_selected_affinity) {
if (strcmp(user_selected_affinity, "strided") == 0) {
selected_affinity = HCLIB_AFFINITY_STRIDED;
} else if (strcmp(user_selected_affinity, "chunked") == 0) {
selected_affinity = HCLIB_AFFINITY_CHUNKED;
} else {
fprintf(stderr, "Unsupported thread affinity \"%s\" specified with "
"HCLIB_AFFINITY.\n", user_selected_affinity);
exit(1);
}
}
thread_cpusets = (hwloc_bitmap_t *)malloc(hc_context->nworkers *
sizeof(*thread_cpusets));
assert(thread_cpusets);
for (i = 0; i < hc_context->nworkers; i++) {
thread_cpusets[i] = hwloc_bitmap_alloc();
assert(thread_cpusets[i]);
}
switch (selected_affinity) {
case (HCLIB_AFFINITY_STRIDED): {
if (available_pus < num_workers) {
fprintf(stderr, "ERROR Available PUs (%d) was less than number "
"of workers (%d), don't currently support "
"oversubscription with strided thread pinning\n",
available_pus, num_workers);
exit(1);
}
int count = 0;
int index = 0;
while (index <= last_set_index) {
if (hwloc_bitmap_isset(cpuset, index)) {
hwloc_bitmap_set(thread_cpusets[count % num_workers],
index);
count++;
}
index++;
}
break;
}
case (HCLIB_AFFINITY_CHUNKED): {
const int chunk_size = (available_pus + num_workers - 1) /
num_workers;
int count = 0;
int index = 0;
while (index <= last_set_index) {
if (hwloc_bitmap_isset(cpuset, index)) {
hwloc_bitmap_set(thread_cpusets[count / chunk_size], index);
count++;
}
index++;
}
break;
}
default:
assert(false);
}
hwloc_bitmap_t nodeset = hwloc_bitmap_alloc();
hwloc_bitmap_t other_nodeset = hwloc_bitmap_alloc();
assert(nodeset && other_nodeset);
/*
* Here, we look for contiguous ranges of worker threads that share any NUMA
* nodes with us. In theory, this should be more hierarchical but isn't yet.
* This is also super inefficient... O(T^2) where T is the number of
* workers.
*/
bool revert_to_naive_stealing = false;
for (i = 0; i < hc_context->nworkers; i++) {
// Get the NUMA nodes for this CPU set
hwloc_cpuset_to_nodeset(topology, thread_cpusets[i], nodeset);
int base = -1;
int limit = -1;
int j;
for (j = 0; j < hc_context->nworkers; j++) {
hwloc_cpuset_to_nodeset(topology, thread_cpusets[j], other_nodeset);
// Take the intersection, see if there is any overlap
hwloc_bitmap_and(other_nodeset, nodeset, other_nodeset);
if (base < 0) {
// Haven't found a contiguous chunk of workers yet.
if (!hwloc_bitmap_iszero(other_nodeset)) {
base = j;
}
} else {
/*
* Have a contiguous chunk of workers, either still inside it or
* after it.
*/
if (limit < 0) {
// Inside the contiguous chunk of workers
if (hwloc_bitmap_iszero(other_nodeset)) {
// Found the end
limit = j;
}
} else {
// After the contiguous chunk of workers
if (!hwloc_bitmap_iszero(other_nodeset)) {
// No contiguous chunk to find, just do something naive.
revert_to_naive_stealing = true;
break;
}
}
}
}
if (revert_to_naive_stealing) {
fprintf(stderr, "WARNING: Using naive work-stealing patterns.\n");
base = 0;
limit = hc_context->nworkers;
} else {
assert(base >= 0);
if (limit < 0) {
limit = hc_context->nworkers;
}
}
hc_context->workers[i]->base_intra_socket_workers = base;
hc_context->workers[i]->limit_intra_socket_workers = limit;
#ifdef VERBOSE
char *nbuf;
hwloc_bitmap_asprintf(&nbuf, nodeset);
char *buffer;
hwloc_bitmap_asprintf(&buffer, thread_cpusets[i]);
fprintf(stderr, "Worker %d has access to %d PUs (%s), %d NUMA nodes "
"(%s). Shared NUMA nodes with [%d, %d).\n", i,
hwloc_bitmap_weight(thread_cpusets[i]), buffer,
hwloc_bitmap_weight(nodeset), nbuf, base, limit);
free(buffer);
#endif
}
#endif
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set, set2;
hwloc_const_bitmap_t cset_available, cset_all;
hwloc_obj_t obj;
char *buffer;
char type[64];
unsigned i;
int err;
/* create a topology */
err = hwloc_topology_init(&topology);
if (err < 0) {
fprintf(stderr, "failed to initialize the topology\n");
return EXIT_FAILURE;
}
err = hwloc_topology_load(topology);
if (err < 0) {
fprintf(stderr, "failed to load the topology\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* retrieve the entire set of available PUs */
cset_available = hwloc_topology_get_topology_cpuset(topology);
/* retrieve the CPU binding of the current entire process */
set = hwloc_bitmap_alloc();
if (!set) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
if (err < 0) {
fprintf(stderr, "failed to get cpu binding\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
}
/* display the processing units that cannot be used by this process */
if (hwloc_bitmap_isequal(set, cset_available)) {
printf("this process can use all available processing units in the system\n");
} else {
/* compute the set where we currently cannot run.
* we can't modify cset_available because it's a system read-only one,
* so we do set = available &~ set
*/
hwloc_bitmap_andnot(set, cset_available, set);
hwloc_bitmap_asprintf(&buffer, set);
printf("process cannot use %d process units (%s) among %u in the system\n",
hwloc_bitmap_weight(set), buffer, hwloc_bitmap_weight(cset_available));
free(buffer);
/* restore set where it was before the &~ operation above */
hwloc_bitmap_andnot(set, cset_available, set);
}
/* print the smallest object covering the current process binding */
obj = hwloc_get_obj_covering_cpuset(topology, set);
hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
printf("process is bound within object %s logical index %u\n", type, obj->logical_index);
/* retrieve the single PU where the current thread actually runs within this process binding */
set2 = hwloc_bitmap_alloc();
if (!set2) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* sanity checks that are not actually needed but help the reader */
/* this thread runs within the process binding */
assert(hwloc_bitmap_isincluded(set2, set));
/* this thread runs on a single PU at a time */
assert(hwloc_bitmap_weight(set2) == 1);
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is now running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
/* migrate this single thread to where other PUs within the current binding */
hwloc_bitmap_andnot(set2, set, set2);
err = hwloc_set_cpubind(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to set thread binding\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* reprint the PU where that thread runs */
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
/* retrieve the entire set of all PUs */
cset_all = hwloc_topology_get_complete_cpuset(topology);
if (hwloc_bitmap_isequal(cset_all, cset_available)) {
printf("all hardware PUs are available\n");
} else {
printf("only %d hardware PUs are available in the machine among %d\n",
hwloc_bitmap_weight(cset_available), hwloc_bitmap_weight(cset_all));
}
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
int main(int argc, char **argv)
{
int rank;
int num_ranks;
MPI_Status status;
hwloc_topology_t topology;
hwloc_cpuset_t set;
int pu_rank = -1;
char name[1024];
int resultlen;
MPI_Init(&argc,&argv);
if (argc != 2) {
fprintf(stderr, "Usage: %s <output file>\n", argv[0]);
exit(1);
}
FILE *output = fopen(argv[1], "w");
if (!output) {
perror("fopen");
exit(2);
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &num_ranks);
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
set = hwloc_bitmap_alloc();
hwloc_get_cpubind(topology, set, 0);
pu_rank = hwloc_bitmap_first(set);
MPI_Get_processor_name(name, &resultlen);
resultlen++;
if (rank == 0) {
node_t *nodes = NULL;
node_t *node;
/* Rank 0 info */
/* Find node */
HASH_FIND_PTR(nodes, &name, node);
/* If node does not exist yet, create it */
if (!node) {
node = (node_t *)malloc(sizeof(node_t));
node->name = name;
utarray_new(node->slots, &ut_int_icd);
utarray_new(node->ranks, &ut_int_icd);
HASH_ADD_KEYPTR(hh, nodes, node->name, strlen(node->name), node);
}
/* Add the slot to the list of slots */
utarray_push_back(node->slots, &pu_rank);
utarray_push_back(node->ranks, &rank);
/* Info about other ranks */
for (int p = 1; p < num_ranks; p++) {
/* Receive node name size, and slot index */
char *nodename;
int buffer[2];
MPI_Recv (buffer, 2, MPI_INT, p, 0, MPI_COMM_WORLD, &status);
int size = buffer[0];
int slot = buffer[1];
/* Receive node name */
nodename = (char *)malloc(sizeof(char[size]));
MPI_Recv(nodename, size, MPI_CHAR, p, 0, MPI_COMM_WORLD, &status);
/* Find node */
HASH_FIND_STR(nodes, nodename, node);
/* If node does not exist yet, create it */
if (!node) {
node = (node_t *)malloc(sizeof(node_t));
node->name = nodename;
utarray_new(node->slots, &ut_int_icd);
utarray_new(node->ranks, &ut_int_icd);
HASH_ADD_KEYPTR(hh, nodes, node->name, strlen(node->name), node);
}
/* Add the slot to the list of slots */
utarray_push_back(node->slots, &slot);
utarray_push_back(node->ranks, &p);
}
/* Write the list of nodes and slots by node */
/* Number of nodes */
int num_nodes = HASH_COUNT(nodes);
fprintf(output, "%d", num_nodes);
/* Names of nodes */
node_t *node_tmp;
HASH_ITER(hh, nodes, node, node_tmp) {
fprintf(output, " %s", node->name);
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
unsigned long flags = HWLOC_TOPOLOGY_FLAG_WHOLE_IO|HWLOC_TOPOLOGY_FLAG_ICACHES;
int force = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
int pid_number = 0;
hwloc_pid_t pid;
char *callname;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, flags);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
else if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
else if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
else if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
else if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
else if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
else if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
else if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
else if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
else if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
else if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
else if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
else if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "replicate", 2))
membind_policy = HWLOC_MEMBIND_REPLICATE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
else if (!strcmp (argv[0], "--whole-system")) {
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
hwloc_topology_destroy(topology);
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, flags);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
goto next;
}
else if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
working_on_cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid_number)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid_number)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_REPLICATE: policystr = "replicate"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
return EXIT_SUCCESS;
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number)
fprintf(stderr, "hwloc_set_proc_membind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number)
fprintf(stderr, "hwloc_set_proc_cpubind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number)
return EXIT_SUCCESS;
if (0 == argc) {
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
int orte_ess_base_proc_binding(void)
{
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
char *map;
int ret;
char *error=NULL;
hwloc_cpuset_t mycpus;
/* Determine if we were pre-bound or not */
if (NULL != getenv(OPAL_MCA_PREFIX"orte_bound_at_launch")) {
orte_proc_is_bound = true;
if (NULL != (map = getenv(OPAL_MCA_PREFIX"orte_base_applied_binding"))) {
orte_proc_applied_binding = hwloc_bitmap_alloc();
if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) {
error = "applied_binding parse";
goto error;
}
}
}
/* see if we were bound when launched */
if (!orte_proc_is_bound) {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Not bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* we were not bound at launch */
if (NULL == opal_hwloc_topology) {
/* there is nothing we can do, so just return */
return ORTE_SUCCESS;
}
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Binding not supported",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
orte_proc_is_bound = true;
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process was externally bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology,
OPAL_HWLOC_LOGICAL, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound according to slot_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == orte_process_info.my_node_rank) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process not bound - no node rank available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to hwthread",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = ORTE_ERROR;
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to core",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
} else {
ret = ORTE_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
hwloc_obj_type_string(target)));
break;
}
}
if (!orte_proc_is_bound) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
}
}
} else {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"%s Process bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
if (NULL != orte_process_info.cpuset) {
free(orte_process_info.cpuset);
orte_process_info.cpuset = NULL;
}
if (opal_hwloc_report_bindings || 4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
opal_output(0, "MCW rank %d is not bound",
ORTE_PROC_MY_NAME->vpid);
}
} else {
/* store/update the string representation of our local binding */
if (NULL != orte_process_info.cpuset) {
free(orte_process_info.cpuset);
orte_process_info.cpuset = NULL;
}
hwloc_bitmap_list_asprintf(&orte_process_info.cpuset, mycpus);
/* report the binding, if requested */
if (opal_hwloc_report_bindings || 4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
char tmp1[1024], tmp2[1024];
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", ORTE_PROC_MY_NAME->vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
ORTE_PROC_MY_NAME->vpid, tmp1, tmp2);
}
}
}
hwloc_bitmap_free(mycpus);
/* push our cpuset so others can calculate our locality */
if (NULL != orte_process_info.cpuset) {
OPAL_MODEX_SEND_VALUE(ret, OPAL_PMIX_GLOBAL, OPAL_PMIX_CPUSET,
orte_process_info.cpuset, OPAL_STRING);
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
}
