static void test(hwloc_const_bitmap_t cpuset, int flags)
{
hwloc_bitmap_t new_cpuset = hwloc_bitmap_alloc();
result_set("Bind this singlethreaded process", hwloc_set_cpubind(topology, cpuset, flags), support->cpubind->set_thisproc_cpubind || support->cpubind->set_thisthread_cpubind);
result_get("Get this singlethreaded process", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags), support->cpubind->get_thisproc_cpubind || support->cpubind->get_thisthread_cpubind);
result_set("Bind this thread", hwloc_set_cpubind(topology, cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->set_thisthread_cpubind);
result_get("Get this thread", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->get_thisthread_cpubind);
result_set("Bind this whole process", hwloc_set_cpubind(topology, cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_thisproc_cpubind);
result_get("Get this whole process", cpuset, new_cpuset, hwloc_get_cpubind(topology, new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_thisproc_cpubind);
#ifdef HWLOC_WIN_SYS
result_set("Bind process", hwloc_set_proc_cpubind(topology, GetCurrentProcess(), cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_proc_cpubind);
result_get("Get process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, GetCurrentProcess(), new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_proc_cpubind);
result_set("Bind thread", hwloc_set_thread_cpubind(topology, GetCurrentThread(), cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->set_thread_cpubind);
result_get("Get thread", cpuset, new_cpuset, hwloc_get_thread_cpubind(topology, GetCurrentThread(), new_cpuset, flags | HWLOC_CPUBIND_THREAD), support->cpubind->get_thread_cpubind);
#else /* !HWLOC_WIN_SYS */
result_set("Bind whole process", hwloc_set_proc_cpubind(topology, getpid(), cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->set_proc_cpubind);
result_get("Get whole process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, getpid(), new_cpuset, flags | HWLOC_CPUBIND_PROCESS), support->cpubind->get_proc_cpubind);
result_set("Bind process", hwloc_set_proc_cpubind(topology, getpid(), cpuset, flags), support->cpubind->set_proc_cpubind);
result_get("Get process", cpuset, new_cpuset, hwloc_get_proc_cpubind(topology, getpid(), new_cpuset, flags), support->cpubind->get_proc_cpubind);
#ifdef hwloc_thread_t
result_set("Bind thread", hwloc_set_thread_cpubind(topology, pthread_self(), cpuset, flags), support->cpubind->set_thread_cpubind);
result_get("Get thread", cpuset, new_cpuset, hwloc_get_thread_cpubind(topology, pthread_self(), new_cpuset, flags), support->cpubind->get_thread_cpubind);
#endif
#endif /* !HWLOC_WIN_SYS */
printf("\n");
hwloc_bitmap_free(new_cpuset);
}
/* Wrapper routines for hwloc */
void qrm_hwloc_bind(int id)
{
int depth, ret;
unsigned i, n;
int topodepth;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_CORE, id);
ret = hwloc_set_cpubind(topology, obj->cpuset, HWLOC_CPUBIND_THREAD);
if (ret) {
printf("Couldn't bind to core %d\n", id);
assert(0);
} else {
printf("Bound to core %d\n", id);
}
hwloc_topology_destroy(topology);
return;
}
static void set_up_worker_thread_affinities(const int wid) {
#ifdef USE_HWLOC
int err = hwloc_set_cpubind(topology, thread_cpusets[wid],
HWLOC_CPUBIND_THREAD);
assert(err == 0);
#endif
}
JNIEXPORT void JNICALL Java_com_rr_core_os_NativeHooksImpl_jniSetPriority( JNIEnv *env, jclass clazz, jint cpumask, jint priority ) {
int topodepth;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
topodepth = hwloc_topology_get_depth(topology);
cpuset = hwloc_bitmap_alloc();
hwloc_bitmap_from_ulong( cpuset, (unsigned int)cpumask );
char *str;
hwloc_bitmap_asprintf(&str, cpuset);
printf("cpumask [%d] => hwloc [%s]\n", cpumask, str);
if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD)) {
printf("Couldn't bind cpuset %s\n", str);
} else {
printf("BOUND cpuset %s\n", str);
}
free(str);
/* Free our cpuset copy */
hwloc_bitmap_free(cpuset);
/* Destroy topology object. */
hwloc_topology_destroy(topology);
}
bool hwloc::bind_this_thread( const std::pair<unsigned,unsigned> coord )
{
#if 0
std::cout << "KokkosArray::hwloc::bind_this_thread() at " ;
hwloc_get_last_cpu_location( s_hwloc_topology ,
s_hwloc_location , HWLOC_CPUBIND_THREAD );
print_bitmap( std::cout , s_hwloc_location );
std::cout << " to " ;
print_bitmap( std::cout , s_core[ coord.second + coord.first * s_core_topology.second ] );
std::cout << std::endl ;
#endif
// As safe and fast as possible.
// Fast-lookup by caching the coordinate -> hwloc cpuset mapping in 's_core'.
return coord.first < s_core_topology.first &&
coord.second < s_core_topology.second &&
0 == hwloc_set_cpubind( s_hwloc_topology ,
s_core[ coord.second + coord.first * s_core_topology.second ] ,
HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT );
}
void Topology::set_thread_cpubind(identifier_type numa_node){
#ifdef M3BP_LOCALITY_ENABLED
const auto num_nodes = m_available_numa_nodes.size();
assert(numa_node < num_nodes);
hwloc_obj_t obj;
if(num_nodes == 1){
obj = hwloc_get_obj_by_type(m_topology, HWLOC_OBJ_MACHINE, 0);
if(!obj){
throw std::runtime_error(
"An error occured on "
"`hwloc_get_obj_by_type(HWLOC_OBJ_MACHINE)`");
}
}else{
obj = hwloc_get_obj_by_type(
m_topology, HWLOC_OBJ_NODE, m_available_numa_nodes[numa_node]);
if(!obj){
throw std::runtime_error(
"An error occured on `hwloc_get_obj_by_type(HWLOC_OBJ_NODE)`");
}
}
if(hwloc_set_cpubind(m_topology, obj->cpuset, HWLOC_CPUBIND_THREAD) != 0){
throw std::runtime_error("An error occured on `hwloc_set_cpubind()`");
}
# ifdef M3BP_NO_THREAD_LOCAL
g_ts_binded_node.get() = numa_node;
# else
g_binded_node = numa_node;
# endif
#else
(void)(numa_node);
#endif
}
c_sublocid_t chpl_topo_getThreadLocality(void) {
hwloc_cpuset_t cpuset;
hwloc_nodeset_t nodeset;
int flags;
int node;
if (!haveTopology) {
return c_sublocid_any;
}
if (!topoSupport->cpubind->get_thread_cpubind) {
return c_sublocid_any;
}
CHK_ERR_ERRNO((cpuset = hwloc_bitmap_alloc()) != NULL);
CHK_ERR_ERRNO((nodeset = hwloc_bitmap_alloc()) != NULL);
flags = HWLOC_CPUBIND_THREAD;
CHK_ERR_ERRNO(hwloc_set_cpubind(topology, cpuset, flags) == 0);
hwloc_cpuset_to_nodeset(topology, cpuset, nodeset);
node = hwloc_bitmap_first(nodeset);
hwloc_bitmap_free(nodeset);
hwloc_bitmap_free(cpuset);
return node;
}
void chpl_topo_setThreadLocality(c_sublocid_t subloc) {
hwloc_cpuset_t cpuset;
int flags;
_DBG_P("chpl_topo_setThreadLocality(%d)\n", (int) subloc);
if (!haveTopology) {
return;
}
if (!topoSupport->cpubind->set_thread_cpubind)
return;
if ((cpuset = hwloc_bitmap_alloc()) == NULL) {
report_error("hwloc_bitmap_alloc()", errno);
}
hwloc_cpuset_from_nodeset(topology, cpuset,
getNumaObj(subloc)->allowed_nodeset);
flags = HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT;
if (hwloc_set_cpubind(topology, cpuset, flags)) {
report_error("hwloc_set_cpubind()", errno);
}
hwloc_bitmap_free(cpuset);
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t cpuset;
int err;
/* check the OS topology */
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
assert(hwloc_topology_is_thissystem(topology));
cpuset = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset(topology));
result("Binding with OS backend", hwloc_set_cpubind(topology, cpuset, 0));
hwloc_topology_destroy(topology);
/* We're assume there is a real processor numbered 0 */
hwloc_bitmap_zero(cpuset);
hwloc_bitmap_set(cpuset, 0);
/* check a synthetic topology */
hwloc_topology_init(&topology);
hwloc_topology_set_synthetic(topology, "1");
hwloc_topology_load(topology);
assert(!hwloc_topology_is_thissystem(topology));
err = hwloc_set_cpubind(topology, cpuset, 0);
result("Binding with synthetic backend", err);
assert(!err);
hwloc_topology_destroy(topology);
/* check a synthetic topology but assuming it's the system topology */
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
hwloc_topology_set_synthetic(topology, "1");
hwloc_topology_load(topology);
assert(hwloc_topology_is_thissystem(topology));
result("Binding with synthetic backend faking is_thissystem", hwloc_set_cpubind(topology, cpuset, 0));
hwloc_topology_destroy(topology);
hwloc_bitmap_free(cpuset);
return 0;
}
int location_cpubind(hwloc_topology_t topology, hwloc_obj_t location)
{
if(hwloc_set_cpubind(topology,location->cpuset, HWLOC_CPUBIND_THREAD|HWLOC_CPUBIND_STRICT|HWLOC_CPUBIND_NOMEMBIND) == -1){
perror("cpubind");
return -1;
}
return chk_cpu_bind(topology, location->cpuset,0);
}
void hwloc_topology::set_thread_affinity_mask(
mask_type mask
, error_code& ec
) const
{ // {{{
hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
for (std::size_t i = 0; i < sizeof(std::size_t) * CHAR_BIT; ++i)
{
if (mask & (static_cast<std::size_t>(1) << i))
{
hwloc_bitmap_set(cpuset, static_cast<unsigned int>(i));
}
}
{
scoped_lock lk(topo_mtx);
if (hwloc_set_cpubind(topo, cpuset,
HWLOC_CPUBIND_STRICT | HWLOC_CPUBIND_THREAD))
{
// Strict binding not supported or failed, try weak binding.
if (hwloc_set_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD))
{
hwloc_bitmap_free(cpuset);
HPX_THROWS_IF(ec, kernel_error
, "hpx::threads::hwloc_topology::set_thread_affinity_mask"
, boost::str(boost::format(
"failed to set thread %x affinity mask")
% mask));
if (ec)
return;
}
}
}
#if defined(__linux) || defined(linux) || defined(__linux__) || defined(__FreeBSD__)
sleep(0); // Allow the OS to pick up the change.
#endif
hwloc_bitmap_free(cpuset);
if (&ec != &throws)
ec = make_success_code();
} // }}}
void chpl_topo_touchMemFromSubloc(void* p, size_t size, chpl_bool onlyInside,
c_sublocid_t subloc) {
size_t pgSize;
unsigned char* pPgLo;
size_t nPages;
hwloc_cpuset_t cpuset;
int flags;
_DBG_P("chpl_topo_touchMemFromSubloc(%p, %#zx, onlyIn=%s, %d)\n",
p, size, (onlyInside ? "T" : "F"), (int) subloc);
if (!haveTopology
|| !topoSupport->cpubind->get_thread_cpubind
|| !topoSupport->cpubind->set_thread_cpubind) {
return;
}
alignAddrSize(p, size, onlyInside, &pgSize, &pPgLo, &nPages);
_DBG_P(" localize %p, %#zx bytes (%#zx pages)\n",
pPgLo, nPages * pgSize, nPages);
if (nPages == 0)
return;
CHK_ERR_ERRNO((cpuset = hwloc_bitmap_alloc()) != NULL);
flags = HWLOC_CPUBIND_THREAD;
CHK_ERR_ERRNO(hwloc_set_cpubind(topology, cpuset, flags) == 0);
chpl_topo_setThreadLocality(subloc);
{
size_t pg;
for (pg = 0; pg < nPages; pg++) {
pPgLo[pg * pgSize] = 0;
}
}
flags = HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT;
CHK_ERR_ERRNO(hwloc_set_cpubind(topology, cpuset, flags) == 0);
hwloc_bitmap_free(cpuset);
}
/****** shepherd_binding/bind_process_to_mask() *************************************
* NAME
* bind_process_to_mask() -- Binds current process to a given cpuset (mask).
*
* SYNOPSIS
* static bool bind_process_to_mask(const hwloc_bitmap_t cpuset)
*
* FUNCTION
* Binds current process to a given cpuset.
*
* INPUTS
* const hwloc_bitmap_t cpuset - Processors to bind processes to
*
* RESULT
* static bool - true if successful, false otherwise
*
* NOTES
* MT-NOTE: bind_process_to_mask() is not MT safe
*
*******************************************************************************/
static bool bind_process_to_mask(const hwloc_bitmap_t cpuset)
{
/* we only need core binding capabilites, no topology is required */
if (!has_core_binding()) return false;
/* Try strict binding first; fall back to non-strict if it isn't
available. */
if (!hwloc_set_cpubind(sge_hwloc_topology, cpuset, HWLOC_CPUBIND_STRICT) ||
!hwloc_set_cpubind(sge_hwloc_topology, cpuset, 0)) {
/* Set the environment variable as for the env type. Done for
for conveniece, e.g. with runtimes like GCC's libgomp which
require an environment variable to be set for thread affinity
rather than using the core binding in effect. */
/* This does not show up in "environment" file! */
if (create_binding_env(cpuset) == true)
shepherd_trace("bind_process_to_mask: SGE_BINDING env var created");
return true;
}
return false;
}
/*
* Set the core where the current thread will run
* return the core
*/
void hw_set_my_core(int cpu)
{
hwloc_cpuset_t set;
cpu = phys_cpus[cpu];
set = hwloc_bitmap_alloc();
hwloc_bitmap_zero(set);
hwloc_bitmap_set(set,cpu);
hwloc_set_cpubind (topology,set,HWLOC_CPUBIND_THREAD);
hwloc_bitmap_free(set);
}
int LpelThreadAssign(int core)
{
int res;
#ifdef HAVE_HWLOC
//FIXME
if (core < 0) return 0;
res = hwloc_set_cpubind(topology, cpu_sets[core],
HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (res == -1) return LPEL_ERR_ASSIGN;
#elif defined(HAVE_PTHREAD_SETAFFINITY_NP)
pthread_t pt = pthread_self();
if (core == -1) {
/* assign an others thread to others cpuset */
res = pthread_setaffinity_np(pt, sizeof(cpu_set_t), &cpuset_others);
if (res != 0) return LPEL_ERR_ASSIGN;
} else if (!LPEL_ICFG(LPEL_FLAG_PINNED)) {
/* assign along all workers */
res = pthread_setaffinity_np(pt, sizeof(cpu_set_t), &cpuset_workers);
if (res != 0) return LPEL_ERR_ASSIGN;
} else { /* LPEL_FLAG_PINNED */
/* assign to specified core */
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET( core, &cpuset);
res = pthread_setaffinity_np(pt, sizeof(cpu_set_t), &cpuset);
if (res != 0) return LPEL_ERR_ASSIGN;
/* make non-preemptible */
if (LPEL_ICFG(LPEL_FLAG_EXCLUSIVE)) {
struct sched_param param;
int sp = SCHED_FIFO;
/* highest real-time */
param.sched_priority = sched_get_priority_max(sp);
res = pthread_setschedparam(pt, sp, ¶m);
if (res != 0) {
/* we do best effort at this point */
return LPEL_ERR_EXCL;
}
}
}
#endif
return 0;
}
bool Core::bind()
{
auto cpuset = hwloc_bitmap_dup(core_->cpuset);
hwloc_bitmap_singlify(cpuset);
if (hwloc_set_cpubind(topology_, cpuset, 0))
{
auto error = errno;
LOG(thread_logger, warning) << "Error setting thread affinity: "
<< strerror(error);
hwloc_bitmap_free(cpuset);
return false;
}
hwloc_bitmap_free(cpuset);
return true;
}
void bindCurrentThreadToNumaNode(int node) {
hwloc_topology_t topology = getHWTopology();
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
// The actual node
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NODE, node);
// obj is nullptr on non NUMA machines
if (obj == nullptr) {
fprintf(stderr, "Couldn't get hwloc object, bindCurrentThreadToNumaNode failed!\n");
return;
}
cpuset = hwloc_bitmap_dup(obj->cpuset);
// hwloc_bitmap_singlify(cpuset);
// bind
if (hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_STRICT | HWLOC_CPUBIND_NOMEMBIND | HWLOC_CPUBIND_THREAD)) {
char* str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
free(str);
throw std::runtime_error(strerror(error));
}
// free duplicated cpuset
hwloc_bitmap_free(cpuset);
// assuming single machine system
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE, 0);
// set membind policy interleave for this thread
if (hwloc_set_membind_nodeset(
topology, obj->nodeset, HWLOC_MEMBIND_INTERLEAVE, HWLOC_MEMBIND_STRICT | HWLOC_MEMBIND_THREAD) && errno != ENOSYS) {
char* str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->nodeset);
fprintf(stderr, "Couldn't membind to nodeset %s: %s\n", str, strerror(error));
fprintf(stderr, "Continuing as normal, however, no guarantees\n");
free(str);
}
}
void THardwareLocalityHelper::BindThreadForDevice(int deviceId) {
if (!HasContext) {
return;
}
THwlocSet deviceCpu;
THwlocSet numaNode;
int errCode = hwloc_cudart_get_device_cpuset(Context, deviceId, deviceCpu.Set);
hwloc_cpuset_to_nodeset(Context, deviceCpu.Set, numaNode.Set);
errCode = hwloc_set_cpubind(Context, deviceCpu.Set, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (errCode == -1) {
MATRIXNET_ERROR_LOG << "Can't bind thread for " << deviceId << " with err " << errno << Endl;
}
errCode = hwloc_set_membind_nodeset(Context, numaNode.Set, HWLOC_MEMBIND_BIND, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (errCode == -1) {
MATRIXNET_ERROR_LOG << "Can't bind memory for " << deviceId << " with err " << errno << Endl;
}
}
bool unbind_this_thread()
{
if ( ! sentinel() ) return false ;
#define HWLOC_DEBUG_PRINT 0
#if HWLOC_DEBUG_PRINT
std::cout << "Kokkos::unbind_this_thread() from " ;
hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD );
print_bitmap( std::cout , s_hwloc_location );
#endif
const bool result =
s_hwloc_topology &&
0 == hwloc_set_cpubind( s_hwloc_topology ,
s_process_binding ,
HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT );
#if HWLOC_DEBUG_PRINT
std::cout << " to " ;
hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD );
print_bitmap( std::cout , s_hwloc_location );
std::cout << std::endl ;
#endif
return result ;
#undef HWLOC_DEBUG_PRINT
}
int bind_myself_to_core(hwloc_topology_t topology, int id){
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
char *str;
int binding_res;
int depth = hwloc_topology_get_depth(topology);
int nb_cores = hwloc_get_nbobjs_by_depth(topology, depth-1);
int my_core;
int nb_threads = get_nb_threads();
/* printf("depth=%d\n",depth); */
switch (mapping_policy){
case SCATTER:
my_core = id*(nb_cores/nb_threads);
break;
default:
if(verbose_level>=WARNING){
printf("Wrong scheduling policy. Using COMPACT\n");
}
case COMPACT:
my_core = id%nb_cores;
}
if(verbose_level>=INFO){
printf("Mapping thread %d on core %d\n",id,my_core);
}
/* Get my core. */
obj = hwloc_get_obj_by_depth(topology, depth-1, my_core);
if (obj) {
/* Get a copy of its cpuset that we may modify. */
cpuset = hwloc_bitmap_dup(obj->cpuset);
/* Get only one logical processor (in case the core is
SMT/hyperthreaded). */
hwloc_bitmap_singlify(cpuset);
/*hwloc_bitmap_asprintf(&str, cpuset);
printf("Binding thread %d to cpuset %s\n", my_core,str);
FREE(str);
*/
/* And try to bind ourself there. */
binding_res = hwloc_set_cpubind(topology, cpuset, HWLOC_CPUBIND_THREAD);
if (binding_res == -1){
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
if(verbose_level>=WARNING)
printf("Thread %d couldn't bind to cpuset %s: %s.\n This thread is not bound to any core...\n", my_core, str, strerror(error));
free(str); /* str is allocated by hlwoc, free it normally*/
return 0;
}
/* FREE our cpuset copy */
hwloc_bitmap_free(cpuset);
return 1;
}else{
if(verbose_level>=WARNING)
printf("No valid object for core id %d!\n",my_core);
return 0;
}
}
int main(void)
{
int depth;
unsigned i, n;
unsigned long size;
int levels;
char string[128];
int topodepth;
void *m;
hwloc_topology_t topology;
hwloc_cpuset_t cpuset;
hwloc_obj_t obj;
/* Allocate and initialize topology object. */
hwloc_topology_init(&topology);
/* ... Optionally, put detection configuration here to ignore
some objects types, define a synthetic topology, etc....
The default is to detect all the objects of the machine that
the caller is allowed to access. See Configure Topology
Detection. */
/* Perform the topology detection. */
hwloc_topology_load(topology);
/* Optionally, get some additional topology information
in case we need the topology depth later. */
topodepth = hwloc_topology_get_depth(topology);
/*****************************************************************
* First example:
* Walk the topology with an array style, from level 0 (always
* the system level) to the lowest level (always the proc level).
*****************************************************************/
for (depth = 0; depth < topodepth; depth++) {
printf("*** Objects at level %d\n", depth);
for (i = 0; i < hwloc_get_nbobjs_by_depth(topology, depth);
i++) {
hwloc_obj_type_snprintf(string, sizeof(string),
hwloc_get_obj_by_depth(topology, depth, i), 0);
printf("Index %u: %s\n", i, string);
}
}
/*****************************************************************
* Second example:
* Walk the topology with a tree style.
*****************************************************************/
printf("*** Printing overall tree\n");
print_children(topology, hwloc_get_root_obj(topology), 0);
/*****************************************************************
* Third example:
* Print the number of packages.
*****************************************************************/
depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PACKAGE);
if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
printf("*** The number of packages is unknown\n");
} else {
printf("*** %u package(s)\n",
hwloc_get_nbobjs_by_depth(topology, depth));
}
/*****************************************************************
* Fourth example:
* Compute the amount of cache that the first logical processor
* has above it.
*****************************************************************/
levels = 0;
size = 0;
for (obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0);
obj;
obj = obj->parent)
if (obj->type == HWLOC_OBJ_CACHE) {
levels++;
size += obj->attr->cache.size;
}
printf("*** Logical processor 0 has %d caches totaling %luKB\n",
levels, size / 1024);
/*****************************************************************
* Fifth example:
* Bind to only one thread of the last core of the machine.
*
* First find out where cores are, or else smaller sets of CPUs if
* the OS doesn't have the notion of a "core".
*****************************************************************/
depth = hwloc_get_type_or_below_depth(topology, HWLOC_OBJ_CORE);
/* Get last core. */
obj = hwloc_get_obj_by_depth(topology, depth,
hwloc_get_nbobjs_by_depth(topology, depth) - 1);
if (obj) {
/* Get a copy of its cpuset that we may modify. */
cpuset = hwloc_bitmap_dup(obj->cpuset);
/* Get only one logical processor (in case the core is
SMT/hyper-threaded). */
hwloc_bitmap_singlify(cpuset);
/* And try to bind ourself there. */
if (hwloc_set_cpubind(topology, cpuset, 0)) {
char *str;
int error = errno;
hwloc_bitmap_asprintf(&str, obj->cpuset);
printf("Couldn't bind to cpuset %s: %s\n", str, strerror(error));
free(str);
}
/* Free our cpuset copy */
hwloc_bitmap_free(cpuset);
}
/*****************************************************************
* Sixth example:
* Allocate some memory on the last NUMA node, bind some existing
* memory to the last NUMA node.
*****************************************************************/
/* Get last node. There's always at least one. */
n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE);
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, n - 1);
size = 1024*1024;
m = hwloc_alloc_membind_nodeset(topology, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
hwloc_free(topology, m, size);
m = malloc(size);
hwloc_set_area_membind_nodeset(topology, m, size, obj->nodeset,
HWLOC_MEMBIND_BIND, 0);
free(m);
/* Destroy topology object. */
hwloc_topology_destroy(topology);
return 0;
}
int orte_daemon(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
char *rml_uri;
int i;
opal_buffer_t *buffer;
char hostname[100];
#if OPAL_ENABLE_FT_CR == 1
char *tmp_env_var = NULL;
#endif
/* initialize the globals */
memset(&orted_globals, 0, sizeof(orted_globals));
/* initialize the singleton died pipe to an illegal value so we can detect it was set */
orted_globals.singleton_died_pipe = -1;
/* init the failure orted vpid to an invalid value */
orted_globals.fail = ORTE_VPID_INVALID;
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
if (OPAL_SUCCESS != opal_cmd_line_create(cmd_line, orte_cmd_line_opts)) {
OBJ_RELEASE(cmd_line);
exit(1);
}
mca_base_cmd_line_setup(cmd_line);
if (ORTE_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false,
argc, argv))) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
fprintf(stderr, "Usage: %s [OPTION]...\n%s\n", argv[0], args);
free(args);
OBJ_RELEASE(cmd_line);
return ret;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* Ensure that enough of OPAL is setup for us to be able to run */
/*
* NOTE: (JJH)
* We need to allow 'mca_base_cmd_line_process_args()' to process command
* line arguments *before* calling opal_init_util() since the command
* line could contain MCA parameters that affect the way opal_init_util()
* functions. AMCA parameters are one such option normally received on the
* command line that affect the way opal_init_util() behaves.
* It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* save the environment for launch purposes. This MUST be
* done so that we can pass it to any local procs we
* spawn - otherwise, those local procs won't see any
* non-MCA envars that were set in the enviro when the
* orted was executed - e.g., by .csh
*/
orte_launch_environ = opal_argv_copy(environ);
/* purge any ess flag set in the environ when we were launched */
opal_unsetenv(OPAL_MCA_PREFIX"ess", &orte_launch_environ);
/* if orte_daemon_debug is set, let someone know we are alive right
* away just in case we have a problem along the way
*/
if (orted_globals.debug) {
gethostname(hostname, 100);
fprintf(stderr, "Daemon was launched on %s - beginning to initialize\n", hostname);
}
/* check for help request */
if (orted_globals.help) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
orte_show_help("help-orted.txt", "orted:usage", false,
argv[0], args);
free(args);
return 1;
}
#if defined(HAVE_SETSID)
/* see if we were directed to separate from current session */
if (orted_globals.set_sid) {
setsid();
}
#endif
/* see if they want us to spin until they can connect a debugger to us */
i=0;
while (orted_spin_flag) {
i++;
if (1000 < i) i=0;
}
#if OPAL_ENABLE_FT_CR == 1
/* Mark as a tool program */
(void) mca_base_var_env_name ("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
#endif
/* if mapreduce set, flag it */
if (orted_globals.mapreduce) {
orte_map_reduce = true;
}
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(!orte_debug_flag &&
!orte_debug_daemons_flag &&
orted_globals.daemonize) {
opal_daemon_init(NULL);
}
/* Set the flag telling OpenRTE that I am NOT a
* singleton, but am "infrastructure" - prevents setting
* up incorrect infrastructure that only a singleton would
* require.
*/
if (orted_globals.hnp) {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
ORTE_ERROR_LOG(ret);
return ret;
}
} else {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_DAEMON))) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/* finalize the OPAL utils. As they are opened again from orte_init->opal_init
* we continue to have a reference count on them. So we have to finalize them twice...
*/
opal_finalize_util();
#if OPAL_HAVE_HWLOC
/* bind ourselves if so directed */
if (NULL != orte_daemon_cores) {
char **cores=NULL, tmp[128];
hwloc_obj_t pu;
hwloc_cpuset_t ours, pucpus, res;
int core;
/* could be a collection of comma-delimited ranges, so
* use our handy utility to parse it
*/
orte_util_parse_range_options(orte_daemon_cores, &cores);
if (NULL != cores) {
ours = hwloc_bitmap_alloc();
hwloc_bitmap_zero(ours);
pucpus = hwloc_bitmap_alloc();
res = hwloc_bitmap_alloc();
for (i=0; NULL != cores[i]; i++) {
core = strtoul(cores[i], NULL, 10);
if (NULL == (pu = opal_hwloc_base_get_pu(opal_hwloc_topology, core, OPAL_HWLOC_LOGICAL))) {
/* turn off the show help forwarding as we won't
* be able to cycle the event library to send
*/
orte_show_help_finalize();
/* the message will now come out locally */
orte_show_help("help-orted.txt", "orted:cannot-bind",
true, orte_process_info.nodename,
orte_daemon_cores);
ret = ORTE_ERR_NOT_SUPPORTED;
goto DONE;
}
hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
hwloc_bitmap_or(res, ours, pucpus);
hwloc_bitmap_copy(ours, res);
}
/* if the result is all zeros, then don't bind */
if (!hwloc_bitmap_iszero(ours)) {
(void)hwloc_set_cpubind(opal_hwloc_topology, ours, 0);
if (opal_hwloc_report_bindings) {
opal_hwloc_base_cset2mapstr(tmp, sizeof(tmp), opal_hwloc_topology, ours);
opal_output(0, "Daemon %s is bound to cores %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp);
}
}
/* cleanup */
hwloc_bitmap_free(ours);
hwloc_bitmap_free(pucpus);
hwloc_bitmap_free(res);
opal_argv_free(cores);
}
}
#endif
if ((int)ORTE_VPID_INVALID != orted_globals.fail) {
orted_globals.abort=false;
/* some vpid was ordered to fail. The value can be positive
* or negative, depending upon the desired method for failure,
* so need to check both here
*/
if (0 > orted_globals.fail) {
orted_globals.fail = -1*orted_globals.fail;
orted_globals.abort = true;
}
/* are we the specified vpid? */
if ((int)ORTE_PROC_MY_NAME->vpid == orted_globals.fail) {
/* if the user specified we delay, then setup a timer
* and have it kill us
*/
if (0 < orted_globals.fail_delay) {
ORTE_TIMER_EVENT(orted_globals.fail_delay, 0, shutdown_callback, ORTE_SYS_PRI);
} else {
opal_output(0, "%s is executing clean %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orted_globals.abort ? "abort" : "abnormal termination");
/* do -not- call finalize as this will send a message to the HNP
* indicating clean termination! Instead, just forcibly cleanup
* the local session_dir tree and exit
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* if we were ordered to abort, do so */
if (orted_globals.abort) {
abort();
}
/* otherwise, return with non-zero status */
ret = ORTE_ERROR_DEFAULT_EXIT_CODE;
goto DONE;
}
}
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
/* if I am also the hnp, then update that contact info field too */
if (ORTE_PROC_IS_HNP) {
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_HNP->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_HNP->vpid = ORTE_PROC_MY_NAME->vpid;
}
/* setup the primary daemon command receive function */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "Daemon %s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* We actually do *not* want the orted to voluntarily yield() the
processor more than necessary. The orted already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the orted, we want the
OS to wake up the orted in a timely fashion (which most OS's
seem good about doing) and then we want the orted to process
the message as fast as possible. If the orted yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the orted to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orted that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, report my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *proc;
orte_node_t *node;
orte_app_context_t *app;
char *tmp, *nptr, *sysinfo;
int32_t ljob;
/* setup the singleton's job */
jdata = OBJ_NEW(orte_job_t);
orte_plm_base_create_jobid(jdata);
ljob = ORTE_LOCAL_JOBID(jdata->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, jdata);
/* must create a map for it (even though it has no
* info in it) so that the job info will be picked
* up in subsequent pidmaps or other daemons won't
* know how to route
*/
jdata->map = OBJ_NEW(orte_job_map_t);
/* setup an app_context for the singleton */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("singleton");
app->num_procs = 1;
opal_pointer_array_add(jdata->apps, app);
/* setup a proc object for the singleton - since we
* -must- be the HNP, and therefore we stored our
* node on the global node pool, and since the singleton
* -must- be on the same node as us, indicate that
*/
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = jdata->jobid;
proc->name.vpid = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_ALIVE);
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
/* obviously, it is on my node */
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
proc->node = node;
OBJ_RETAIN(node); /* keep accounting straight */
opal_pointer_array_add(jdata->procs, proc);
jdata->num_procs = 1;
/* and it obviously is on the node */
OBJ_RETAIN(proc);
opal_pointer_array_add(node->procs, proc);
node->num_procs++;
/* and obviously it is one of my local procs */
OBJ_RETAIN(proc);
opal_pointer_array_add(orte_local_children, proc);
jdata->num_local_procs = 1;
/* set the trivial */
proc->local_rank = 0;
proc->node_rank = 0;
proc->app_rank = 0;
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_LOCAL);
/* create a string that contains our uri + sysinfo + PMIx server URI */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, pmix_server_uri);
free(sysinfo);
/* pass that info to the singleton */
write(orted_globals.uri_pipe, tmp, strlen(tmp)+1); /* need to add 1 to get the NULL */
/* cleanup */
free(tmp);
/* since a singleton spawned us, we need to harvest
* any MCA params from the local environment so
* we can pass them along to any subsequent daemons
* we may start as the result of a comm_spawn
*/
for (i=0; NULL != environ[i]; i++) {
if (0 == strncmp(environ[i], OPAL_MCA_PREFIX, 9)) {
/* make a copy to manipulate */
tmp = strdup(environ[i]);
/* find the equal sign */
nptr = strchr(tmp, '=');
*nptr = '\0';
nptr++;
/* add the mca param to the orted cmd line */
opal_argv_append_nosize(&orted_cmd_line, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append_nosize(&orted_cmd_line, &tmp[9]);
opal_argv_append_nosize(&orted_cmd_line, nptr);
free(tmp);
}
}
}
/* if we were given a pipe to monitor for singleton termination, set that up */
if (orted_globals.singleton_died_pipe > 0) {
/* register shutdown handler */
pipe_handler = (opal_event_t*)malloc(sizeof(opal_event_t));
opal_event_set(orte_event_base, pipe_handler,
orted_globals.singleton_died_pipe,
OPAL_EV_READ,
pipe_closed,
pipe_handler);
opal_event_add(pipe_handler, NULL);
}
/* If I have a parent, then save his contact info so
* any messages we send can flow thru him.
*/
orte_parent_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "parent_uri",
"URI for the parent if tree launch is enabled.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_parent_uri);
if (NULL != orte_parent_uri) {
orte_process_name_t parent;
/* set the contact info into the hash table */
orte_rml.set_contact_info(orte_parent_uri);
ret = orte_rml_base_parse_uris(orte_parent_uri, &parent, NULL);
if (ORTE_SUCCESS != ret) {
ORTE_ERROR_LOG(ret);
free (orte_parent_uri);
orte_parent_uri = NULL;
goto DONE;
}
/* don't need this value anymore */
free(orte_parent_uri);
orte_parent_uri = NULL;
/* tell the routed module that we have a path
* back to the HNP
*/
if (ORTE_SUCCESS != (ret = orte_routed.update_route(ORTE_PROC_MY_HNP, &parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* set the lifeline to point to our parent so that we
* can handle the situation if that lifeline goes away
*/
if (ORTE_SUCCESS != (ret = orte_routed.set_lifeline(&parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* if we are not the HNP...the only time we will be an HNP
* is if we are launched by a singleton to provide support
* for it
*/
if (!ORTE_PROC_IS_HNP) {
/* send the information to the orted report-back point - this function
* will process the data, but also counts the number of
* orteds that reported back so the launch procedure can continue.
* We need to do this at the last possible second as the HNP
* can turn right around and begin issuing orders to us
*/
buffer = OBJ_NEW(opal_buffer_t);
/* insert our name for rollup purposes */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* for now, always include our contact info, even if we are using
* static ports. Eventually, this will be removed
*/
rml_uri = orte_rml.get_contact_info();
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &rml_uri, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* include our node name */
opal_dss.pack(buffer, &orte_process_info.nodename, 1, OPAL_STRING);
/* if requested, include any non-loopback aliases for this node */
if (orte_retain_aliases) {
char **aliases=NULL;
uint8_t naliases, ni;
char hostname[ORTE_MAX_HOSTNAME_SIZE];
/* if we stripped the prefix or removed the fqdn,
* include full hostname as an alias
*/
gethostname(hostname, ORTE_MAX_HOSTNAME_SIZE);
if (strlen(orte_process_info.nodename) < strlen(hostname)) {
opal_argv_append_nosize(&aliases, hostname);
}
opal_ifgetaliases(&aliases);
naliases = opal_argv_count(aliases);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &naliases, 1, OPAL_UINT8))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
for (ni=0; ni < naliases; ni++) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &aliases[ni], 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
opal_argv_free(aliases);
}
#if OPAL_HAVE_HWLOC
{
char *coprocessors;
/* add the local topology */
if (NULL != opal_hwloc_topology &&
(1 == ORTE_PROC_MY_NAME->vpid || orte_hetero_nodes)) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &opal_hwloc_topology, 1, OPAL_HWLOC_TOPO))) {
ORTE_ERROR_LOG(ret);
}
}
/* detect and add any coprocessors */
coprocessors = opal_hwloc_base_find_coprocessors(opal_hwloc_topology);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &coprocessors, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
}
/* see if I am on a coprocessor */
coprocessors = opal_hwloc_base_check_on_coprocessor();
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &coprocessors, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
}
}
#endif
/* send to the HNP's callback - will be routed if routes are available */
if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, buffer,
ORTE_RML_TAG_ORTED_CALLBACK,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
/* if we are tree-spawning, then we need to capture the MCA params
* from our cmd line so we can pass them along to the daemons we spawn -
* otherwise, only the first layer of daemons will ever see them
*/
if (orted_globals.tree_spawn) {
int j, k;
bool ignore;
char *no_keep[] = {
"orte_hnp_uri",
"orte_ess_jobid",
"orte_ess_vpid",
"orte_ess_num_procs",
"orte_parent_uri",
"mca_base_env_list",
NULL
};
for (i=0; i < argc; i++) {
if (0 == strcmp("-"OPAL_MCA_CMD_LINE_ID, argv[i]) ||
0 == strcmp("--"OPAL_MCA_CMD_LINE_ID, argv[i]) ) {
ignore = false;
/* see if this is something we cannot pass along */
for (k=0; NULL != no_keep[k]; k++) {
if (0 == strcmp(no_keep[k], argv[i+1])) {
ignore = true;
break;
}
}
if (!ignore) {
/* see if this is already present so we at least can
* avoid growing the cmd line with duplicates
*/
if (NULL != orted_cmd_line) {
for (j=0; NULL != orted_cmd_line[j]; j++) {
if (0 == strcmp(argv[i+1], orted_cmd_line[j])) {
/* already here - ignore it */
ignore = true;
break;
}
}
}
if (!ignore) {
opal_argv_append_nosize(&orted_cmd_line, argv[i]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+1]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+2]);
}
}
i += 2;
}
}
}
if (orte_debug_daemons_flag) {
opal_output(0, "%s orted: up and running - waiting for commands!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
ret = ORTE_SUCCESS;
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
/* ensure all local procs are dead */
orte_odls.kill_local_procs(NULL);
DONE:
/* update the exit status, in case it wasn't done */
ORTE_UPDATE_EXIT_STATUS(ret);
/* cleanup and leave */
orte_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
int orte_ess_base_proc_binding(void)
{
#if OPAL_HAVE_HWLOC
hwloc_obj_t node, obj;
hwloc_cpuset_t cpus, nodeset;
hwloc_obj_type_t target;
unsigned int cache_level = 0;
struct hwloc_topology_support *support;
char *map;
int ret;
char *error;
/* Determine if we were pre-bound or not */
if (NULL != getenv("OMPI_MCA_orte_bound_at_launch")) {
orte_proc_is_bound = true;
if (NULL != (map = getenv("OMPI_MCA_orte_base_applied_binding"))) {
orte_proc_applied_binding = hwloc_bitmap_alloc();
if (0 != (ret = hwloc_bitmap_list_sscanf(orte_proc_applied_binding, map))) {
error = "applied_binding parse";
goto error;
}
}
}
/* see if we were bound when launched */
if (!orte_proc_is_bound) {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Not bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* we were not bound at launch */
if (NULL != opal_hwloc_topology) {
support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
/* get our node object */
node = hwloc_get_root_obj(opal_hwloc_topology);
nodeset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, node);
/* get our bindings */
cpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology, cpus, HWLOC_CPUBIND_PROCESS) < 0) {
/* we are NOT bound if get_cpubind fails, nor can we be bound - the
* environment does not support it
*/
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Binding not supported",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* we are bound if the two cpusets are not equal,
* or if there is only ONE cpu available to us
*/
if (0 != hwloc_bitmap_compare(cpus, nodeset) ||
opal_hwloc_base_single_cpu(nodeset) ||
opal_hwloc_base_single_cpu(cpus)) {
/* someone external set it - indicate it is set
* so that we know
*/
orte_proc_is_bound = true;
hwloc_bitmap_free(cpus);
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process was externally bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (support->cpubind->set_thisproc_cpubind &&
OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
hwloc_bitmap_zero(cpus);
if (OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_slot_list_parse(opal_hwloc_base_slot_list,
opal_hwloc_topology, cpus))) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
hwloc_bitmap_free(cpus);
goto error;
}
/* try to find a level and index for this location */
opal_hwloc_base_get_level_and_index(cpus, &orte_process_info.bind_level, &orte_process_info.bind_idx);
/* cleanup */
hwloc_bitmap_free(cpus);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound according to slot_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* cleanup */
hwloc_bitmap_free(cpus);
/* get the node rank */
if (ORTE_NODE_RANK_INVALID == orte_process_info.my_node_rank) {
/* this is not an error - could be due to being
* direct launched - so just ignore and leave
* us unbound
*/
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process not bound - no node rank available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto MOVEON;
}
/* if the binding policy is hwthread, then we bind to the nrank-th
* hwthread on this node
*/
if (OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_PU,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting hwthread object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_HWTHREAD_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to hwthread",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else if (OPAL_BIND_TO_CORE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
/* if the binding policy is core, then we bind to the nrank-th
* core on this node
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
error = "Setting processor affinity failed";
ret = ORTE_ERROR;
goto error;
}
orte_process_info.bind_level = OPAL_HWLOC_CORE_LEVEL;
orte_process_info.bind_idx = orte_process_info.my_node_rank;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to core",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
} else {
/* for all higher binding policies, we bind to the specified
* object that the nrank-th core belongs to
*/
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE,
0, orte_process_info.my_node_rank, OPAL_HWLOC_LOGICAL))) {
ret = ORTE_ERR_NOT_FOUND;
error = "Getting core object";
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 1;
orte_process_info.bind_level = OPAL_HWLOC_L1CACHE_LEVEL;
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 2;
orte_process_info.bind_level = OPAL_HWLOC_L2CACHE_LEVEL;
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_CACHE;
cache_level = 3;
orte_process_info.bind_level = OPAL_HWLOC_L3CACHE_LEVEL;
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
orte_process_info.bind_level = OPAL_HWLOC_SOCKET_LEVEL;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_NODE;
orte_process_info.bind_level = OPAL_HWLOC_NUMA_LEVEL;
} else {
ret = ORTE_ERR_NOT_FOUND;
error = "Binding policy not known";
goto error;
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
/* this is the place! */
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
orte_process_info.bind_idx = opal_hwloc_base_get_obj_idx(opal_hwloc_topology,
obj, OPAL_HWLOC_LOGICAL);
orte_proc_is_bound = true;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
opal_hwloc_base_print_level(orte_process_info.bind_level)));
break;
}
}
if (!orte_proc_is_bound) {
ret = ORTE_ERROR;
error = "Setting processor affinity failed";
goto error;
}
}
}
}
}
} else {
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_output,
"%s Process bound at launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
MOVEON:
/* get or update our local cpuset - it will get used multiple
* times, so it's more efficient to keep a global copy
*/
opal_hwloc_base_get_local_cpuset();
/* report bindings, if requested */
if (opal_hwloc_report_bindings) {
char bindings[64];
hwloc_obj_t root;
hwloc_cpuset_t cpus;
/* get the root object for this node */
root = hwloc_get_root_obj(opal_hwloc_topology);
cpus = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
/* we are not bound if this equals our cpuset */
if (0 == hwloc_bitmap_compare(cpus, opal_hwloc_my_cpuset)) {
opal_output(0, "%s is not bound",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
} else {
hwloc_bitmap_list_snprintf(bindings, 64, opal_hwloc_my_cpuset);
opal_output(0, "%s is bound to cpus %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
bindings);
}
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
#else
return ORTE_SUCCESS;
#endif
}
int LpelThreadAssign(int core)
{
int res;
#ifdef HAVE_HWLOC
//FIXME
if (core < 0) return 0;
res = hwloc_set_cpubind(topology, cpu_sets[core],
HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
if (res == -1) return LPEL_ERR_ASSIGN;
#elif defined(HAVE_PTHREAD_SETAFFINITY_NP)
lpel_config_t *cfg = &_lpel_global_config;
pthread_t pt = pthread_self();
cpu_set_t cpuset;
if ( LPEL_ICFG(LPEL_FLAG_PINNED)) {
CPU_ZERO(&cpuset);
switch(core) {
case LPEL_MAP_WRAPPER: /* round robin pinned to cores in the set */
case LPEL_MAP_SOSI:
CPU_SET(rot_others + offset_others, &cpuset);
rot_others = (rot_others + 1) % proc_others;
break;
default: // workers
/* assign to specified core */
assert( 0 <= core && core < cfg->num_workers );
CPU_SET( core % proc_workers, &cpuset);
}
}
else {
switch (core) {
case LPEL_MAP_WRAPPER:
case LPEL_MAP_SOSI:
cpuset = cpuset_others;
break;
default: // workers
cpuset = cpuset_workers;
}
}
res = pthread_setaffinity_np(pt, sizeof(cpu_set_t), &cpuset);
if( res != 0) return LPEL_ERR_ASSIGN;
/* make non-preemptible for workers only */
if ( LPEL_ICFG(LPEL_FLAG_EXCLUSIVE) && core >= 0) {
struct sched_param param;
int sp = SCHED_FIFO;
/* highest real-time */
param.sched_priority = sched_get_priority_max(sp);
res = pthread_setschedparam(pt, sp, ¶m);
if ( res != 0) {
/* we do best effort at this point */
return LPEL_ERR_EXCL;
} else {
fprintf(stderr, "set realtime priority %d for worker %d.\n",
param.sched_priority, core);
}
}
#endif
return 0;
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set, set2;
hwloc_const_bitmap_t cset_available, cset_all;
hwloc_obj_t obj;
char *buffer;
char type[64];
unsigned i;
int err;
/* create a topology */
err = hwloc_topology_init(&topology);
if (err < 0) {
fprintf(stderr, "failed to initialize the topology\n");
return EXIT_FAILURE;
}
err = hwloc_topology_load(topology);
if (err < 0) {
fprintf(stderr, "failed to load the topology\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
/* retrieve the entire set of available PUs */
cset_available = hwloc_topology_get_topology_cpuset(topology);
/* retrieve the CPU binding of the current entire process */
set = hwloc_bitmap_alloc();
if (!set) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
if (err < 0) {
fprintf(stderr, "failed to get cpu binding\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
}
/* display the processing units that cannot be used by this process */
if (hwloc_bitmap_isequal(set, cset_available)) {
printf("this process can use all available processing units in the system\n");
} else {
/* compute the set where we currently cannot run.
* we can't modify cset_available because it's a system read-only one,
* so we do set = available &~ set
*/
hwloc_bitmap_andnot(set, cset_available, set);
hwloc_bitmap_asprintf(&buffer, set);
printf("process cannot use %d process units (%s) among %u in the system\n",
hwloc_bitmap_weight(set), buffer, hwloc_bitmap_weight(cset_available));
free(buffer);
/* restore set where it was before the &~ operation above */
hwloc_bitmap_andnot(set, cset_available, set);
}
/* print the smallest object covering the current process binding */
obj = hwloc_get_obj_covering_cpuset(topology, set);
hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
printf("process is bound within object %s logical index %u\n", type, obj->logical_index);
/* retrieve the single PU where the current thread actually runs within this process binding */
set2 = hwloc_bitmap_alloc();
if (!set2) {
fprintf(stderr, "failed to allocate a bitmap\n");
hwloc_bitmap_free(set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* sanity checks that are not actually needed but help the reader */
/* this thread runs within the process binding */
assert(hwloc_bitmap_isincluded(set2, set));
/* this thread runs on a single PU at a time */
assert(hwloc_bitmap_weight(set2) == 1);
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is now running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
/* migrate this single thread to where other PUs within the current binding */
hwloc_bitmap_andnot(set2, set, set2);
err = hwloc_set_cpubind(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to set thread binding\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* reprint the PU where that thread runs */
err = hwloc_get_last_cpu_location(topology, set2, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "failed to get last cpu location\n");
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
hwloc_topology_destroy(topology);
}
/* print the logical number of the PU where that thread runs */
/* extract the PU OS index from the bitmap */
i = hwloc_bitmap_first(set2);
obj = hwloc_get_pu_obj_by_os_index(topology, i);
printf("thread is running on PU logical index %u (OS/physical index %u)\n",
obj->logical_index, i);
hwloc_bitmap_free(set);
hwloc_bitmap_free(set2);
/* retrieve the entire set of all PUs */
cset_all = hwloc_topology_get_complete_cpuset(topology);
if (hwloc_bitmap_isequal(cset_all, cset_available)) {
printf("all hardware PUs are available\n");
} else {
printf("only %d hardware PUs are available in the machine among %d\n",
hwloc_bitmap_weight(cset_available), hwloc_bitmap_weight(cset_all));
}
hwloc_topology_destroy(topology);
return EXIT_SUCCESS;
}
static void set(orte_job_t *jobdat,
orte_proc_t *child,
char ***environ_copy,
int write_fd)
{
hwloc_cpuset_t cpuset;
hwloc_obj_t root;
opal_hwloc_topo_data_t *sum;
orte_app_context_t *context;
int rc=ORTE_ERROR;
char *msg, *param;
char *cpu_bitmap;
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set on child %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
if (NULL == jobdat || NULL == child) {
/* nothing for us to do */
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc:set jobdat %s child %s - nothing to do",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == jobdat) ? "NULL" : ORTE_JOBID_PRINT(jobdat->jobid),
(NULL == child) ? "NULL" : ORTE_NAME_PRINT(&child->name));
return;
}
context = (orte_app_context_t*)opal_pointer_array_get_item(jobdat->apps, child->app_idx);
/* Set process affinity, if given */
cpu_bitmap = NULL;
if (!orte_get_attribute(&child->attributes, ORTE_PROC_CPU_BITMAP, (void**)&cpu_bitmap, OPAL_STRING) ||
NULL == cpu_bitmap || 0 == strlen(cpu_bitmap)) {
/* if the daemon is bound, then we need to "free" this proc */
if (NULL != orte_daemon_cores) {
root = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL == root->userdata) {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "incorrectly bound",
orte_process_info.nodename, context->app,
__FILE__, __LINE__);
}
sum = (opal_hwloc_topo_data_t*)root->userdata;
/* bind this proc to all available processors */
rc = hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
char *tmp;
(void)hwloc_bitmap_list_asprintf(&tmp, sum->available);
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), tmp);
free(tmp);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
return;
}
}
}
if (0 == rc && opal_hwloc_report_bindings) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
} else {
/* convert the list to a cpuset */
cpuset = hwloc_bitmap_alloc();
if (0 != (rc = hwloc_bitmap_list_sscanf(cpuset, cpu_bitmap))) {
/* See comment above about "This may be a small memory leak" */
asprintf(&msg, "hwloc_bitmap_sscanf returned \"%s\" for the string \"%s\"",
opal_strerror(rc), cpu_bitmap);
if (NULL == msg) {
msg = "failed to convert bitmap list to hwloc bitmap";
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding) &&
OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
/* If binding is required and a binding directive was explicitly
* given (i.e., we are not binding due to a default policy),
* send an error up the pipe (which exits -- it doesn't return).
*/
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename,
context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
/* bind as specified */
rc = hwloc_set_cpubind(opal_hwloc_topology, cpuset, 0);
/* if we got an error and this wasn't a default binding policy, then report it */
if (rc < 0 && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
char *tmp = NULL;
if (errno == ENOSYS) {
msg = "hwloc indicates cpu binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates cpu binding cannot be enforced";
} else {
asprintf(&msg, "hwloc_set_cpubind returned \"%s\" for bitmap \"%s\"",
opal_strerror(rc), cpu_bitmap);
}
if (OPAL_BINDING_REQUIRED(jobdat->map->binding)) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"binding generic error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
if (NULL != tmp) {
free(tmp);
free(msg);
}
return;
}
if (NULL != tmp) {
free(tmp);
free(msg);
}
}
if (0 == rc && opal_hwloc_report_bindings) {
char tmp1[1024], tmp2[1024];
hwloc_cpuset_t mycpus;
/* get the cpus we are bound to */
mycpus = hwloc_bitmap_alloc();
if (hwloc_get_cpubind(opal_hwloc_topology,
mycpus,
HWLOC_CPUBIND_PROCESS) < 0) {
opal_output(0, "MCW rank %d is not bound",
child->name.vpid);
} else {
if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(tmp1, sizeof(tmp1), opal_hwloc_topology, mycpus)) {
opal_output(0, "MCW rank %d is not bound (or bound to all available processors)", child->name.vpid);
} else {
opal_hwloc_base_cset2mapstr(tmp2, sizeof(tmp2), opal_hwloc_topology, mycpus);
opal_output(0, "MCW rank %d bound to %s: %s",
child->name.vpid, tmp1, tmp2);
}
}
hwloc_bitmap_free(mycpus);
/* avoid reporting it twice */
(void) mca_base_var_env_name ("hwloc_base_report_bindings", ¶m);
opal_unsetenv(param, environ_copy);
free(param);
}
/* set memory affinity policy - if we get an error, don't report
* anything unless the user actually specified the binding policy
*/
rc = opal_hwloc_base_set_process_membind_policy();
if (ORTE_SUCCESS != rc && OPAL_BINDING_POLICY_IS_SET(jobdat->map->binding)) {
if (errno == ENOSYS) {
msg = "hwloc indicates memory binding not supported";
} else if (errno == EXDEV) {
msg = "hwloc indicates memory binding cannot be enforced";
} else {
msg = "failed to bind memory";
}
if (OPAL_HWLOC_BASE_MBFA_ERROR == opal_hwloc_base_mbfa) {
/* If binding is required, send an error up the pipe (which exits
-- it doesn't return). */
orte_rtc_base_send_error_show_help(write_fd, 1, "help-orte-odls-default.txt",
"memory binding error",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
} else {
orte_rtc_base_send_warn_show_help(write_fd,
"help-orte-odls-default.txt", "memory not bound",
orte_process_info.nodename, context->app, msg,
__FILE__, __LINE__);
free(cpu_bitmap);
return;
}
}
}
if (NULL != cpu_bitmap) {
free(cpu_bitmap);
}
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
int loaded = 0;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
unsigned long flags = 0;
int force = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
int pid_number = -1;
hwloc_pid_t pid = 0; /* only valid when pid_number > 0, but gcc-4.8 still reports uninitialized warnings */
char *callname;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
/* don't load now, in case some options change the config before the topology is actually used */
#define LOADED() (loaded)
#define ENSURE_LOADED() do { \
if (!loaded) { \
hwloc_topology_init(&topology); \
hwloc_topology_set_all_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_ALL); \
hwloc_topology_set_flags(topology, flags); \
hwloc_topology_load(topology); \
depth = hwloc_topology_get_depth(topology); \
loaded = 1; \
} \
} while (0)
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, HWLOC_VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
if (!strcmp (argv[0], "--whole-system")) {
if (loaded) {
fprintf(stderr, "Input option %s disallowed after options using the topology\n", argv[0]);
exit(EXIT_FAILURE);
}
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
goto next;
}
if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
ENSURE_LOADED();
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ENSURE_LOADED();
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
working_on_cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
ENSURE_LOADED();
if (pid_number > 0) {
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
/* no need to set_pid()
* the doc just says we're operating on pid, not that we're retrieving the topo/cpuset as seen from inside pid
*/
}
if (get_last_cpu_location && !working_on_cpubind) {
fprintf(stderr, "Options --membind and --get-last-cpu-location cannot be combined.\n");
return EXIT_FAILURE;
}
if ((get_binding || get_last_cpu_location) && (got_cpubind || got_membind)) {
/* doesn't work because get_binding/get_last_cpu_location overwrites cpubind_set */
fprintf(stderr, "Cannot display and set binding at the same time.\n");
return EXIT_FAILURE;
}
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number > 0)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number > 0)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid_number > 0)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid_number > 0)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number > 0)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_membind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number > 0)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number > 0)
fprintf(stderr, "hwloc_set_proc_cpubind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number > 0)
return EXIT_SUCCESS;
if (0 == argc) {
if (get_binding || get_last_cpu_location)
return EXIT_SUCCESS;
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
/* FIXME: check whether Windows execvp() passes INHERIT_PARENT_AFFINITY to CreateProcess()
* because we need to propagate processor group affinity. However process-wide affinity
* isn't supported with processor groups so far.
*/
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
int orte_daemon(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
int i;
opal_buffer_t *buffer;
char hostname[OPAL_MAXHOSTNAMELEN];
#if OPAL_ENABLE_FT_CR == 1
char *tmp_env_var = NULL;
#endif
/* initialize the globals */
memset(&orted_globals, 0, sizeof(orted_globals));
/* initialize the singleton died pipe to an illegal value so we can detect it was set */
orted_globals.singleton_died_pipe = -1;
bucket = OBJ_NEW(opal_buffer_t);
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
if (OPAL_SUCCESS != opal_cmd_line_create(cmd_line, orte_cmd_line_opts)) {
OBJ_RELEASE(cmd_line);
exit(1);
}
mca_base_cmd_line_setup(cmd_line);
if (ORTE_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false, false,
argc, argv))) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
fprintf(stderr, "Usage: %s [OPTION]...\n%s\n", argv[0], args);
free(args);
OBJ_RELEASE(cmd_line);
return ret;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* Ensure that enough of OPAL is setup for us to be able to run */
/*
* NOTE: (JJH)
* We need to allow 'mca_base_cmd_line_process_args()' to process command
* line arguments *before* calling opal_init_util() since the command
* line could contain MCA parameters that affect the way opal_init_util()
* functions. AMCA parameters are one such option normally received on the
* command line that affect the way opal_init_util() behaves.
* It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* save the environment for launch purposes. This MUST be
* done so that we can pass it to any local procs we
* spawn - otherwise, those local procs won't see any
* non-MCA envars that were set in the enviro when the
* orted was executed - e.g., by .csh
*/
orte_launch_environ = opal_argv_copy(environ);
/* purge any ess/pmix flags set in the environ when we were launched */
opal_unsetenv(OPAL_MCA_PREFIX"ess", &orte_launch_environ);
opal_unsetenv(OPAL_MCA_PREFIX"pmix", &orte_launch_environ);
/* if orte_daemon_debug is set, let someone know we are alive right
* away just in case we have a problem along the way
*/
if (orted_globals.debug) {
gethostname(hostname, sizeof(hostname));
fprintf(stderr, "Daemon was launched on %s - beginning to initialize\n", hostname);
}
/* check for help request */
if (orted_globals.help) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
orte_show_help("help-orted.txt", "orted:usage", false,
argv[0], args);
free(args);
return 1;
}
#if defined(HAVE_SETSID)
/* see if we were directed to separate from current session */
if (orted_globals.set_sid) {
setsid();
}
#endif
/* see if they want us to spin until they can connect a debugger to us */
i=0;
while (orted_spin_flag) {
i++;
if (1000 < i) i=0;
}
#if OPAL_ENABLE_FT_CR == 1
/* Mark as a tool program */
(void) mca_base_var_env_name ("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
#endif
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(!orte_debug_flag &&
!orte_debug_daemons_flag &&
orted_globals.daemonize) {
opal_daemon_init(NULL);
}
/* Set the flag telling OpenRTE that I am NOT a
* singleton, but am "infrastructure" - prevents setting
* up incorrect infrastructure that only a singleton would
* require.
*/
if (orted_globals.hnp) {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
ORTE_ERROR_LOG(ret);
return ret;
}
} else {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_DAEMON))) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/* finalize the OPAL utils. As they are opened again from orte_init->opal_init
* we continue to have a reference count on them. So we have to finalize them twice...
*/
opal_finalize_util();
/* bind ourselves if so directed */
if (NULL != orte_daemon_cores) {
char **cores=NULL, tmp[128];
hwloc_obj_t pu;
hwloc_cpuset_t ours, res;
int core;
/* could be a collection of comma-delimited ranges, so
* use our handy utility to parse it
*/
orte_util_parse_range_options(orte_daemon_cores, &cores);
if (NULL != cores) {
ours = hwloc_bitmap_alloc();
hwloc_bitmap_zero(ours);
res = hwloc_bitmap_alloc();
for (i=0; NULL != cores[i]; i++) {
core = strtoul(cores[i], NULL, 10);
if (NULL == (pu = opal_hwloc_base_get_pu(opal_hwloc_topology, core, OPAL_HWLOC_LOGICAL))) {
/* turn off the show help forwarding as we won't
* be able to cycle the event library to send
*/
orte_show_help_finalize();
/* the message will now come out locally */
orte_show_help("help-orted.txt", "orted:cannot-bind",
true, orte_process_info.nodename,
orte_daemon_cores);
ret = ORTE_ERR_NOT_SUPPORTED;
hwloc_bitmap_free(ours);
hwloc_bitmap_free(res);
goto DONE;
}
hwloc_bitmap_or(res, ours, pu->cpuset);
hwloc_bitmap_copy(ours, res);
}
/* if the result is all zeros, then don't bind */
if (!hwloc_bitmap_iszero(ours)) {
(void)hwloc_set_cpubind(opal_hwloc_topology, ours, 0);
if (opal_hwloc_report_bindings) {
opal_hwloc_base_cset2mapstr(tmp, sizeof(tmp), opal_hwloc_topology, ours);
opal_output(0, "Daemon %s is bound to cores %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp);
}
}
/* cleanup */
hwloc_bitmap_free(ours);
hwloc_bitmap_free(res);
opal_argv_free(cores);
}
}
if ((int)ORTE_VPID_INVALID != orted_debug_failure) {
orted_globals.abort=false;
/* some vpid was ordered to fail. The value can be positive
* or negative, depending upon the desired method for failure,
* so need to check both here
*/
if (0 > orted_debug_failure) {
orted_debug_failure = -1*orted_debug_failure;
orted_globals.abort = true;
}
/* are we the specified vpid? */
if ((int)ORTE_PROC_MY_NAME->vpid == orted_debug_failure) {
/* if the user specified we delay, then setup a timer
* and have it kill us
*/
if (0 < orted_debug_failure_delay) {
ORTE_TIMER_EVENT(orted_debug_failure_delay, 0, shutdown_callback, ORTE_SYS_PRI);
} else {
opal_output(0, "%s is executing clean %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orted_globals.abort ? "abort" : "abnormal termination");
/* do -not- call finalize as this will send a message to the HNP
* indicating clean termination! Instead, just forcibly cleanup
* the local session_dir tree and exit
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* if we were ordered to abort, do so */
if (orted_globals.abort) {
abort();
}
/* otherwise, return with non-zero status */
ret = ORTE_ERROR_DEFAULT_EXIT_CODE;
goto DONE;
}
}
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_oob_base_get_addr(&orte_process_info.my_daemon_uri);
if (NULL == orte_process_info.my_daemon_uri) {
/* no way to communicate */
ret = ORTE_ERROR;
goto DONE;
}
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
/* if I am also the hnp, then update that contact info field too */
if (ORTE_PROC_IS_HNP) {
orte_process_info.my_hnp_uri = strdup(orte_process_info.my_daemon_uri);
ORTE_PROC_MY_HNP->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_HNP->vpid = ORTE_PROC_MY_NAME->vpid;
}
/* setup the primary daemon command receive function */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "Daemon %s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* We actually do *not* want the orted to voluntarily yield() the
processor more than necessary. The orted already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the orted, we want the
OS to wake up the orted in a timely fashion (which most OS's
seem good about doing) and then we want the orted to process
the message as fast as possible. If the orted yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the orted to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orted that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, report my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *proc;
orte_node_t *node;
orte_app_context_t *app;
char *tmp, *nptr, *sysinfo;
char **singenv=NULL, *string_key, *env_str;
/* setup the singleton's job */
jdata = OBJ_NEW(orte_job_t);
/* default to ompi for now */
opal_argv_append_nosize(&jdata->personality, "ompi");
orte_plm_base_create_jobid(jdata);
opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, jdata);
/* must create a map for it (even though it has no
* info in it) so that the job info will be picked
* up in subsequent pidmaps or other daemons won't
* know how to route
*/
jdata->map = OBJ_NEW(orte_job_map_t);
/* setup an app_context for the singleton */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("singleton");
app->num_procs = 1;
opal_pointer_array_add(jdata->apps, app);
jdata->num_apps = 1;
/* setup a proc object for the singleton - since we
* -must- be the HNP, and therefore we stored our
* node on the global node pool, and since the singleton
* -must- be on the same node as us, indicate that
*/
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = jdata->jobid;
proc->name.vpid = 0;
proc->parent = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_ALIVE);
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
/* obviously, it is on my node */
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
proc->node = node;
OBJ_RETAIN(node); /* keep accounting straight */
opal_pointer_array_add(jdata->procs, proc);
jdata->num_procs = 1;
/* add the node to the job map */
OBJ_RETAIN(node);
opal_pointer_array_add(jdata->map->nodes, node);
jdata->map->num_nodes++;
/* and it obviously is on the node */
OBJ_RETAIN(proc);
opal_pointer_array_add(node->procs, proc);
node->num_procs++;
/* and obviously it is one of my local procs */
OBJ_RETAIN(proc);
opal_pointer_array_add(orte_local_children, proc);
jdata->num_local_procs = 1;
/* set the trivial */
proc->local_rank = 0;
proc->node_rank = 0;
proc->app_rank = 0;
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_LOCAL);
/* set the ORTE_JOB_TRANSPORT_KEY from the environment */
orte_pre_condition_transports(jdata, NULL);
/* register the singleton's nspace with our PMIx server */
if (ORTE_SUCCESS != (ret = orte_pmix_server_register_nspace(jdata, false))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* use setup fork to create the envars needed by the singleton */
if (OPAL_SUCCESS != (ret = opal_pmix.server_setup_fork(&proc->name, &singenv))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* append the transport key to the envars needed by the singleton */
if (!orte_get_attribute(&jdata->attributes, ORTE_JOB_TRANSPORT_KEY, (void**)&string_key, OPAL_STRING) || NULL == string_key) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
goto DONE;
}
asprintf(&env_str, OPAL_MCA_PREFIX"orte_precondition_transports=%s", string_key);
opal_argv_append_nosize(&singenv, env_str);
free(env_str);
nptr = opal_argv_join(singenv, '*');
opal_argv_free(singenv);
/* create a string that contains our uri + sysinfo + PMIx server URI envars */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, nptr);
free(sysinfo);
free(nptr);
/* pass that info to the singleton */
if (OPAL_SUCCESS != (ret = opal_fd_write(orted_globals.uri_pipe, strlen(tmp)+1, tmp))) { ; /* need to add 1 to get the NULL */
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* cleanup */
free(tmp);
close(orted_globals.uri_pipe);
/* since a singleton spawned us, we need to harvest
* any MCA params from the local environment so
* we can pass them along to any subsequent daemons
* we may start as the result of a comm_spawn
*/
for (i=0; NULL != environ[i]; i++) {
if (0 == strncmp(environ[i], OPAL_MCA_PREFIX, 9)) {
/* make a copy to manipulate */
tmp = strdup(environ[i]);
/* find the equal sign */
nptr = strchr(tmp, '=');
*nptr = '\0';
nptr++;
/* add the mca param to the orted cmd line */
opal_argv_append_nosize(&orted_cmd_line, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append_nosize(&orted_cmd_line, &tmp[9]);
opal_argv_append_nosize(&orted_cmd_line, nptr);
free(tmp);
}
}
}
/* if we were given a pipe to monitor for singleton termination, set that up */
if (orted_globals.singleton_died_pipe > 0) {
/* register shutdown handler */
pipe_handler = (opal_event_t*)malloc(sizeof(opal_event_t));
opal_event_set(orte_event_base, pipe_handler,
orted_globals.singleton_died_pipe,
OPAL_EV_READ,
pipe_closed,
pipe_handler);
opal_event_add(pipe_handler, NULL);
}
/* If I have a parent, then save his contact info so
* any messages we send can flow thru him.
*/
orte_parent_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "parent_uri",
"URI for the parent if tree launch is enabled.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_parent_uri);
if (NULL != orte_parent_uri) {
orte_process_name_t parent;
opal_value_t val;
/* set the contact info into our local database */
ret = orte_rml_base_parse_uris(orte_parent_uri, &parent, NULL);
if (ORTE_SUCCESS != ret) {
ORTE_ERROR_LOG(ret);
free (orte_parent_uri);
orte_parent_uri = NULL;
goto DONE;
}
OBJ_CONSTRUCT(&val, opal_value_t);
val.key = OPAL_PMIX_PROC_URI;
val.type = OPAL_STRING;
val.data.string = orte_parent_uri;
if (OPAL_SUCCESS != (ret = opal_pmix.store_local(&parent, &val))) {
ORTE_ERROR_LOG(ret);
OBJ_DESTRUCT(&val);
goto DONE;
}
val.key = NULL;
val.data.string = NULL;
OBJ_DESTRUCT(&val);
/* don't need this value anymore */
free(orte_parent_uri);
orte_parent_uri = NULL;
/* tell the routed module that we have a path
* back to the HNP
*/
if (ORTE_SUCCESS != (ret = orte_routed.update_route(NULL, ORTE_PROC_MY_HNP, &parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* set the lifeline to point to our parent so that we
* can handle the situation if that lifeline goes away
*/
if (ORTE_SUCCESS != (ret = orte_routed.set_lifeline(NULL, &parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* if we are not the HNP...the only time we will be an HNP
* is if we are launched by a singleton to provide support
* for it
*/
if (!ORTE_PROC_IS_HNP) {
orte_process_name_t target;
target.jobid = ORTE_PROC_MY_NAME->jobid;
if (orte_fwd_mpirun_port || orte_static_ports) {
/* setup the rollup callback */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_ORTED_CALLBACK,
ORTE_RML_PERSISTENT, rollup, NULL);
target.vpid = ORTE_PROC_MY_NAME->vpid;
/* since we will be waiting for any children to send us
* their rollup info before sending to our parent, save
* a little time in the launch phase by "warming up" the
* connection to our parent while we wait for our children */
buffer = OBJ_NEW(opal_buffer_t); // zero-byte message
if (0 > (ret = orte_rml.send_buffer_nb(orte_mgmt_conduit,
ORTE_PROC_MY_PARENT, buffer,
ORTE_RML_TAG_WARMUP_CONNECTION,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
} else {
target.vpid = 0;
}
/* send the information to the orted report-back point - this function
* will process the data, but also counts the number of
* orteds that reported back so the launch procedure can continue.
* We need to do this at the last possible second as the HNP
* can turn right around and begin issuing orders to us
*/
buffer = OBJ_NEW(opal_buffer_t);
/* insert our name for rollup purposes */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* get any connection info we may have pushed */
{
opal_value_t *val = NULL, *kv;
opal_list_t *modex;
int32_t flag;
if (OPAL_SUCCESS != (ret = opal_pmix.get(ORTE_PROC_MY_NAME, NULL, NULL, &val)) || NULL == val) {
/* just pack a marker indicating we don't have any to share */
flag = 0;
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &flag, 1, OPAL_INT32))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
} else {
/* the data is returned as a list of key-value pairs in the opal_value_t */
if (OPAL_PTR == val->type) {
modex = (opal_list_t*)val->data.ptr;
flag = (int32_t)opal_list_get_size(modex);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &flag, 1, OPAL_INT32))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
OPAL_LIST_FOREACH(kv, modex, opal_value_t) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &kv, 1, OPAL_VALUE))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
OPAL_LIST_RELEASE(modex);
} else {
opal_output(0, "VAL KEY: %s", (NULL == val->key) ? "NULL" : val->key);
/* single value */
flag = 1;
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &flag, 1, OPAL_INT32))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &val, 1, OPAL_VALUE))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
OBJ_RELEASE(val);
}
static int dump_one_proc(hwloc_topology_t topo, hwloc_obj_t pu, const char *path)
{
unsigned regs[4];
unsigned highest_cpuid, highest_ext_cpuid;
unsigned i;
int has_intel_x2apic = 0;
int has_intel_sgx = 0;
int has_amd_topoext = 0;
FILE *output;
int err;
err = hwloc_set_cpubind(topo, pu->cpuset, HWLOC_CPUBIND_PROCESS);
if (err < 0) {
err = hwloc_set_cpubind(topo, pu->cpuset, HWLOC_CPUBIND_THREAD);
if (err < 0) {
fprintf(stderr, "Cannot bind to PU P#%u\n", pu->os_index);
return -1;
}
}
if (path) {
output = fopen(path, "w");
if (!output) {
fprintf(stderr, "Cannot open file '%s' for writing: %s\n", path, strerror(errno));
return -1;
}
if (verbose)
printf("Gathering CPUID of PU P#%u in path %s ...\n", pu->os_index, path);
} else {
output = stdout;
if (verbose)
printf("Gathering CPUID of PU P#%u on stdout ...\n", pu->os_index);
}
fprintf(output, "# mask e[abcd]x => e[abcd]x\n");
regs[0] = 0;
hwloc_x86_cpuid(®s[0], ®s[1], ®s[2], ®s[3]);
highest_cpuid = regs[0];
regs[0] = 0x80000000;
hwloc_x86_cpuid(®s[0], ®s[1], ®s[2], ®s[3]);
highest_ext_cpuid = regs[0];
/* 0x0 = Highest cpuid + Vendor string */
regs[0] = 0x0;
dump_one_cpuid(output, regs, 0x1);
/* 0x1 = Family, Model, Stepping, Topology, Features */
if (highest_cpuid >= 0x1) {
regs[0] = 0x1;
dump_one_cpuid(output, regs, 0x1);
if (regs[2] & (1 << 21))
has_intel_x2apic = 1;
}
/* 0x2 = Cache + TLB on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x2) {
regs[0] = 0x2;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x3 = Processor serial number on Intel P3, reserved otherwise ; Reserved on AMD */
if (highest_cpuid >= 0x3) {
regs[0] = 0x3;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x4 = Caches on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x4) {
for(i=0; ; i++) {
regs[0] = 0x4; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
if (!(regs[0] & 0x1f))
/* invalid, no more caches */
break;
}
}
/* 0x5 = Monitor/mwait */
if (highest_cpuid >= 0x5) {
regs[0] = 0x5;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x6 = Thermal and Power management */
if (highest_cpuid >= 0x6) {
regs[0] = 0x6;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x7 = Extended features */
if (highest_cpuid >= 0x7) {
unsigned max;
regs[0] = 0x7; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
if (regs[1] & (1<<2))
has_intel_sgx = 1;
max = regs[0];
for(i=1; i<=max; i++) {
regs[0] = 0x7; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
}
}
/* 0x9 = DCA on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x9) {
regs[0] = 0x9;
dump_one_cpuid(output, regs, 0x1);
}
/* 0xa = Perf monitoring on Intel ; Reserved on AMD */
if (highest_cpuid >= 0xa) {
regs[0] = 0xa;
dump_one_cpuid(output, regs, 0x1);
}
/* 0xb = Extended topology on Intel ; Reserved on AMD */
if (has_intel_x2apic && highest_cpuid >= 0xb) {
for(i=0; ; i++) {
regs[0] = 0xb; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
if (!(regs[2] & 0xff00))
/* invalid, no more levels */
break;
}
}
/* 0xd = Extended state enumeration */
if (highest_cpuid >= 0xd) {
unsigned xcr0_l, xcr0_h, ia32xss_l, ia32xss_h;
regs[0] = 0xd; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
xcr0_l = regs[0]; xcr0_h = regs[3];
regs[0] = 0xd; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
ia32xss_l = regs[2]; ia32xss_h = regs[3];
for(i=2; i<32; i++) {
if ((xcr0_l | ia32xss_l) & (1<<i)) {
regs[0] = 0xd; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
}
}
for(i=0; i<32; i++) {
if ((xcr0_h | ia32xss_h) & (1<<i)) {
regs[0] = 0xd; regs[2] = i+32;
dump_one_cpuid(output, regs, 0x5);
}
}
}
/* 0xf = Platform/L3 QoS enumeration on Intel and AMD */
if (highest_cpuid >= 0xf) {
regs[0] = 0xf; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
regs[0] = 0xf; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
}
/* 0x10 = Platform/L3 QoS enforcement enumeration on Intel and AMD */
if (highest_cpuid >= 0x10) {
/* Intel Resource Director Technology (Intel RDT) Allocation */
regs[0] = 0x10; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
/* L3 Cache Allocation Technology */
regs[0] = 0x10; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
/* L2 Cache Allocation Technology */
regs[0] = 0x10; regs[2] = 2;
dump_one_cpuid(output, regs, 0x5);
/* Memory Bandwidth Allocation */
regs[0] = 0x10; regs[2] = 3;
dump_one_cpuid(output, regs, 0x5);
}
/* 0x12 = SGX Attributes Enumeration on Intel ; Reserved on AMD */
if (has_intel_sgx && highest_cpuid >= 0x12) {
regs[0] = 0x12; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
regs[0] = 0x12; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
for(i=2; ; i++) {
regs[0] = 0x12; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
if (!(regs[0] & 0xf))
/* invalid, no more subleaves */
break;
}
}
/* 0x14 = Processor trace enumeration on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x14) {
regs[0] = 0x14; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
regs[0] = 0x14; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
}
/* 0x15 = Timestamp counter/core crystal clock on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x15) {
regs[0] = 0x15;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x16 = Processor frequency on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x16) {
regs[0] = 0x16;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x17 = System-On-Chip Vendor Attribute on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x17) {
unsigned maxsocid;
regs[0] = 0x17; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
maxsocid = regs[0];
if (maxsocid >= 3) {
for(i=1; i<=maxsocid; i++) {
regs[0] = 0x17; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
}
}
}
/* 0x18 = Deterministic Address Translation Parameters on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x18) {
unsigned max;
regs[0] = 0x18; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
max = regs[0];
for(i=1; i<=max; i++) {
regs[0] = 0x18; regs[2] = i;
regs[3] = 0; /* mark as invalid in case the cpuid call doesn't do anything */
dump_one_cpuid(output, regs, 0x5);
if (!(regs[3] & 0x1f))
/* invalid, but it doesn't mean the next subleaf will be invalid */
continue;
}
}
/* 0x1b = (Removed) PCONFIG Information on Intel ; Reserved on AMD */
/* 0x1f = V2 Extended Topology Enumeration on Intel ; Reserved on AMD */
if (highest_cpuid >= 0x1f) {
for(i=0; ; i++) {
regs[0] = 0x1f; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
if (!(regs[2] & 0xff00))
/* invalid, no more levels */
break;
}
}
if (highest_cpuid > 0x1f) {
static int reported = 0;
if (!reported)
fprintf(stderr, "WARNING: Processor supports new CPUID leaves upto 0x%x\n", highest_cpuid);
reported = 1;
}
/* 0x80000000 = Largest extended cpuid */
regs[0] = 0x80000000;
dump_one_cpuid(output, regs, 0x1);
/* 0x80000001 = Extended processor signature and features */
if (highest_ext_cpuid >= 0x80000001) {
regs[0] = 0x80000001;
dump_one_cpuid(output, regs, 0x1);
if (regs[2] & (1 << 22))
has_amd_topoext = 1;
}
/* 0x80000002-4 = Processor name string */
if (highest_ext_cpuid >= 0x80000002) {
regs[0] = 0x80000002;
dump_one_cpuid(output, regs, 0x1);
}
if (highest_ext_cpuid >= 0x80000003) {
regs[0] = 0x80000003;
dump_one_cpuid(output, regs, 0x1);
}
if (highest_ext_cpuid >= 0x80000004) {
regs[0] = 0x80000004;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000005 = L1 and TLB on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x80000005) {
regs[0] = 0x80000005;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000006 = L2, L3 and TLB on AMD ; L2 and reserved on Intel */
if (highest_ext_cpuid >= 0x80000006) {
regs[0] = 0x80000006;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000007 = Advanced power management on AMD ; Almost reserved on Intel */
if (highest_ext_cpuid >= 0x80000007) {
regs[0] = 0x80000007;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000008 = Long mode and topology on AMD ; Long mode on Intel */
if (highest_ext_cpuid >= 0x80000008) {
regs[0] = 0x80000008;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x8000000a = SVM on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x8000000a) {
regs[0] = 0x8000000a;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000019 = TLB1G + Perf optim identifiers on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x80000019) {
regs[0] = 0x80000019;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x8000001b = IBS on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x8000001b) {
regs[0] = 0x8000001b;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x8000001c = Profiling on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x8000001c) {
regs[0] = 0x8000001c;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x8000001d = Cache properties on AMD ; Reserved on Intel */
if (highest_ext_cpuid >= 0x8000001d) {
for(i=0; ; i++) {
regs[0] = 0x8000001d; regs[2] = i;
dump_one_cpuid(output, regs, 0x5);
if (!(regs[0] & 0x1f))
/* no such cache, no more cache */
break;
}
}
/* 0x8000001e = Topoext on AMD ; Reserved on Intel */
if (has_amd_topoext && highest_ext_cpuid >= 0x8000001e) {
regs[0] = 0x8000001e;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x8000001f = Encrypted Memory Capabilities ; Reserved on Intel */
if (highest_ext_cpuid >= 0x8000001f) {
regs[0] = 0x8000001f;
dump_one_cpuid(output, regs, 0x1);
}
/* 0x80000020 = Platform QoS Enforcement for Memory Bandwidth */
if (highest_ext_cpuid >= 0x80000020) {
regs[0] = 0x80000020; regs[2] = 0;
dump_one_cpuid(output, regs, 0x5);
regs[0] = 0x80000020; regs[2] = 1;
dump_one_cpuid(output, regs, 0x5);
}
if (highest_ext_cpuid > 0x8000001f) {
static int reported = 0;
if (!reported)
fprintf(stderr, "WARNING: Processor supports new extended CPUID leaves upto 0x%x\n", highest_ext_cpuid);
reported = 1;
}
if (path)
fclose(output);
return 0;
}
int main(int argc, char *argv[])
{
hwloc_topology_t topology;
unsigned depth;
hwloc_bitmap_t cpubind_set, membind_set;
int got_cpubind = 0, got_membind = 0;
int working_on_cpubind = 1; /* membind if 0 */
int get_binding = 0;
int get_last_cpu_location = 0;
unsigned long flags = HWLOC_TOPOLOGY_FLAG_WHOLE_IO|HWLOC_TOPOLOGY_FLAG_ICACHES;
int force = 0;
int single = 0;
int verbose = 0;
int logical = 1;
int taskset = 0;
int cpubind_flags = 0;
hwloc_membind_policy_t membind_policy = HWLOC_MEMBIND_BIND;
int membind_flags = 0;
int opt;
int ret;
int pid_number = 0;
hwloc_pid_t pid;
char *callname;
cpubind_set = hwloc_bitmap_alloc();
membind_set = hwloc_bitmap_alloc();
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, flags);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
callname = argv[0];
/* skip argv[0], handle options */
argv++;
argc--;
while (argc >= 1) {
if (!strcmp(argv[0], "--")) {
argc--;
argv++;
break;
}
opt = 0;
if (*argv[0] == '-') {
if (!strcmp(argv[0], "-v") || !strcmp(argv[0], "--verbose")) {
verbose++;
goto next;
}
else if (!strcmp(argv[0], "-q") || !strcmp(argv[0], "--quiet")) {
verbose--;
goto next;
}
else if (!strcmp(argv[0], "--help")) {
usage("hwloc-bind", stdout);
return EXIT_SUCCESS;
}
else if (!strcmp(argv[0], "--single")) {
single = 1;
goto next;
}
else if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--force")) {
force = 1;
goto next;
}
else if (!strcmp(argv[0], "--strict")) {
cpubind_flags |= HWLOC_CPUBIND_STRICT;
membind_flags |= HWLOC_MEMBIND_STRICT;
goto next;
}
else if (!strcmp(argv[0], "--pid")) {
if (argc < 2) {
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
pid_number = atoi(argv[1]);
opt = 1;
goto next;
}
else if (!strcmp (argv[0], "--version")) {
printf("%s %s\n", callname, VERSION);
exit(EXIT_SUCCESS);
}
if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--logical")) {
logical = 1;
goto next;
}
if (!strcmp(argv[0], "-p") || !strcmp(argv[0], "--physical")) {
logical = 0;
goto next;
}
if (!strcmp(argv[0], "--taskset")) {
taskset = 1;
goto next;
}
else if (!strcmp (argv[0], "-e") || !strncmp (argv[0], "--get-last-cpu-location", 10)) {
get_last_cpu_location = 1;
goto next;
}
else if (!strcmp (argv[0], "--get")) {
get_binding = 1;
goto next;
}
else if (!strcmp (argv[0], "--cpubind")) {
working_on_cpubind = 1;
goto next;
}
else if (!strcmp (argv[0], "--membind")) {
working_on_cpubind = 0;
goto next;
}
else if (!strcmp (argv[0], "--mempolicy")) {
if (!strncmp(argv[1], "default", 2))
membind_policy = HWLOC_MEMBIND_DEFAULT;
else if (!strncmp(argv[1], "firsttouch", 2))
membind_policy = HWLOC_MEMBIND_FIRSTTOUCH;
else if (!strncmp(argv[1], "bind", 2))
membind_policy = HWLOC_MEMBIND_BIND;
else if (!strncmp(argv[1], "interleave", 2))
membind_policy = HWLOC_MEMBIND_INTERLEAVE;
else if (!strncmp(argv[1], "replicate", 2))
membind_policy = HWLOC_MEMBIND_REPLICATE;
else if (!strncmp(argv[1], "nexttouch", 2))
membind_policy = HWLOC_MEMBIND_NEXTTOUCH;
else {
fprintf(stderr, "Unrecognized memory binding policy %s\n", argv[1]);
usage ("hwloc-bind", stderr);
exit(EXIT_FAILURE);
}
opt = 1;
goto next;
}
else if (!strcmp (argv[0], "--whole-system")) {
flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
hwloc_topology_destroy(topology);
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, flags);
hwloc_topology_load(topology);
depth = hwloc_topology_get_depth(topology);
goto next;
}
else if (!strcmp (argv[0], "--restrict")) {
hwloc_bitmap_t restrictset;
int err;
if (argc < 2) {
usage (callname, stdout);
exit(EXIT_FAILURE);
}
restrictset = hwloc_bitmap_alloc();
hwloc_bitmap_sscanf(restrictset, argv[1]);
err = hwloc_topology_restrict (topology, restrictset, 0);
if (err) {
perror("Restricting the topology");
/* fallthrough */
}
hwloc_bitmap_free(restrictset);
argc--;
argv++;
goto next;
}
fprintf (stderr, "Unrecognized option: %s\n", argv[0]);
usage("hwloc-bind", stderr);
return EXIT_FAILURE;
}
ret = hwloc_calc_process_arg(topology, depth, argv[0], logical,
working_on_cpubind ? cpubind_set : membind_set,
verbose);
if (ret < 0) {
if (verbose > 0)
fprintf(stderr, "assuming the command starts at %s\n", argv[0]);
break;
}
if (working_on_cpubind)
got_cpubind = 1;
else
got_membind = 1;
next:
argc -= opt+1;
argv += opt+1;
}
pid = hwloc_pid_from_number(pid_number, !(get_binding || get_last_cpu_location));
if (get_binding || get_last_cpu_location) {
char *s;
const char *policystr = NULL;
int err;
if (working_on_cpubind) {
if (get_last_cpu_location) {
if (pid_number)
err = hwloc_get_proc_last_cpu_location(topology, pid, cpubind_set, 0);
else
err = hwloc_get_last_cpu_location(topology, cpubind_set, 0);
} else {
if (pid_number)
err = hwloc_get_proc_cpubind(topology, pid, cpubind_set, 0);
else
err = hwloc_get_cpubind(topology, cpubind_set, 0);
}
if (err) {
const char *errmsg = strerror(errno);
if (pid_number)
fprintf(stderr, "hwloc_get_proc_%s %d failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_%s failed (errno %d %s)\n", get_last_cpu_location ? "last_cpu_location" : "cpubind", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, cpubind_set);
else
hwloc_bitmap_asprintf(&s, cpubind_set);
} else {
hwloc_membind_policy_t policy;
if (pid_number)
err = hwloc_get_proc_membind(topology, pid, membind_set, &policy, 0);
else
err = hwloc_get_membind(topology, membind_set, &policy, 0);
if (err) {
const char *errmsg = strerror(errno);
if (pid_number)
fprintf(stderr, "hwloc_get_proc_membind %d failed (errno %d %s)\n", pid_number, errno, errmsg);
else
fprintf(stderr, "hwloc_get_membind failed (errno %d %s)\n", errno, errmsg);
return EXIT_FAILURE;
}
if (taskset)
hwloc_bitmap_taskset_asprintf(&s, membind_set);
else
hwloc_bitmap_asprintf(&s, membind_set);
switch (policy) {
case HWLOC_MEMBIND_DEFAULT: policystr = "default"; break;
case HWLOC_MEMBIND_FIRSTTOUCH: policystr = "firsttouch"; break;
case HWLOC_MEMBIND_BIND: policystr = "bind"; break;
case HWLOC_MEMBIND_INTERLEAVE: policystr = "interleave"; break;
case HWLOC_MEMBIND_REPLICATE: policystr = "replicate"; break;
case HWLOC_MEMBIND_NEXTTOUCH: policystr = "nexttouch"; break;
default: fprintf(stderr, "unknown memory policy %d\n", policy); assert(0); break;
}
}
if (policystr)
printf("%s (%s)\n", s, policystr);
else
printf("%s\n", s);
free(s);
return EXIT_SUCCESS;
}
if (got_membind) {
if (hwloc_bitmap_iszero(membind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot membind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
fprintf(stderr, "binding on memory set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(membind_set);
if (pid_number)
ret = hwloc_set_proc_membind(topology, pid, membind_set, membind_policy, membind_flags);
else
ret = hwloc_set_membind(topology, membind_set, membind_policy, membind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, membind_set);
if (pid_number)
fprintf(stderr, "hwloc_set_proc_membind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_membind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
if (got_cpubind) {
if (hwloc_bitmap_iszero(cpubind_set)) {
if (verbose >= 0)
fprintf(stderr, "cannot cpubind to empty set\n");
if (!force)
goto failed_binding;
}
if (verbose > 0) {
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
fprintf(stderr, "binding on cpu set %s\n", s);
free(s);
}
if (single)
hwloc_bitmap_singlify(cpubind_set);
if (pid_number)
ret = hwloc_set_proc_cpubind(topology, pid, cpubind_set, cpubind_flags);
else
ret = hwloc_set_cpubind(topology, cpubind_set, cpubind_flags);
if (ret && verbose >= 0) {
int bind_errno = errno;
const char *errmsg = strerror(bind_errno);
char *s;
hwloc_bitmap_asprintf(&s, cpubind_set);
if (pid_number)
fprintf(stderr, "hwloc_set_proc_cpubind %s %d failed (errno %d %s)\n", s, pid_number, bind_errno, errmsg);
else
fprintf(stderr, "hwloc_set_cpubind %s failed (errno %d %s)\n", s, bind_errno, errmsg);
free(s);
}
if (ret && !force)
goto failed_binding;
}
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
if (pid_number)
return EXIT_SUCCESS;
if (0 == argc) {
fprintf(stderr, "%s: nothing to do!\n", callname);
return EXIT_FAILURE;
}
ret = execvp(argv[0], argv);
if (ret) {
fprintf(stderr, "%s: Failed to launch executable \"%s\"\n",
callname, argv[0]);
perror("execvp");
}
return EXIT_FAILURE;
failed_binding:
hwloc_bitmap_free(cpubind_set);
hwloc_bitmap_free(membind_set);
hwloc_topology_destroy(topology);
return EXIT_FAILURE;
}
