static int mca_sbgp_map_to_socket_core(int processor_id, int *socket, int *core)
{
int ret = OPAL_ERR_NOT_FOUND;
hwloc_obj_t obj;
hwloc_topology_t *t;
hwloc_bitmap_t good;
/* bozo check */
if (NULL == opal_hwloc_topology) {
return OPAL_ERR_NOT_INITIALIZED;
}
t = &opal_hwloc_topology;
good = hwloc_bitmap_alloc();
if (NULL == good) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* Iterate through every core and find one that contains the
processor_id. Then find the corresponding socket. */
for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, NULL);
NULL != obj;
obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, obj)) {
hwloc_bitmap_and(good, obj->online_cpuset,
obj->allowed_cpuset);
/* Does this core contain the processor_id in question? */
if (hwloc_bitmap_isset(good, processor_id)) {
*core = obj->os_index;
/* Go upward from the core object until we find its parent
socket. */
while (HWLOC_OBJ_SOCKET != obj->type) {
if (NULL == obj->parent) {
/* If we get to the root without finding a socket,
er.. Hmm. Error! */
ret = OPAL_ERR_NOT_FOUND;
goto out;
}
obj = obj->parent;
}
*socket = obj->os_index;
ret = OPAL_SUCCESS;
goto out;
}
}
/* If we didn't even find the right core, we didn't find it. Fall
through. */
ret = OPAL_ERR_NOT_FOUND;
out:
hwloc_bitmap_free(good);
return ret;
}
static int get_total_number_of(hwloc_obj_type_t obj_type) {
int number = 0;
if (!has_topology_information()) return 0;
number = hwloc_get_nbobjs_by_type (sge_hwloc_topology, obj_type);
if (-1 == number) {
/* Only for things like L1, L2 caches, not the types we're
interested in? */
hwloc_obj_t obj;
number = 0;
for (obj = hwloc_get_next_obj_by_type(sge_hwloc_topology, obj_type, NULL);
NULL != obj;
obj = hwloc_get_next_obj_by_type(sge_hwloc_topology, obj_type, obj))
if (obj_type == obj->type) ++number;
}
return number;
}
int main(void)
{
hwloc_topology_t topology;
hwloc_bitmap_t set, set2, nocpunomemnodeset, nocpubutmemnodeset, nomembutcpunodeset, nomembutcpucpuset;
hwloc_obj_t node;
struct bitmask *bitmask, *bitmask2;
unsigned long mask;
unsigned long maxnode;
int i;
if (numa_available() < 0)
/* libnuma has inconsistent behavior when the kernel isn't NUMA-aware.
* don't try to check everything precisely.
*/
exit(77);
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
/* convert full stuff between cpuset and libnuma */
set = hwloc_bitmap_alloc();
nocpunomemnodeset = hwloc_bitmap_alloc();
nocpubutmemnodeset = hwloc_bitmap_alloc();
nomembutcpunodeset = hwloc_bitmap_alloc();
nomembutcpucpuset = hwloc_bitmap_alloc();
/* gather all nodes if any, or the whole system if no nodes */
if (hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE)) {
node = NULL;
while ((node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node)) != NULL) {
hwloc_bitmap_or(set, set, node->cpuset);
if (hwloc_bitmap_iszero(node->cpuset)) {
if (node->memory.local_memory)
hwloc_bitmap_set(nocpubutmemnodeset, node->os_index);
else
hwloc_bitmap_set(nocpunomemnodeset, node->os_index);
} else if (!node->memory.local_memory) {
hwloc_bitmap_set(nomembutcpunodeset, node->os_index);
hwloc_bitmap_or(nomembutcpucpuset, nomembutcpucpuset, node->cpuset);
}
}
} else {
hwloc_bitmap_or(set, set, hwloc_topology_get_complete_cpuset(topology));
}
set2 = hwloc_bitmap_alloc();
hwloc_cpuset_from_linux_libnuma_bitmask(topology, set2, numa_all_nodes_ptr);
/* numa_all_nodes_ptr doesn't contain NODES with CPU but no memory */
hwloc_bitmap_or(set2, set2, nomembutcpucpuset);
assert(hwloc_bitmap_isequal(set, set2));
hwloc_bitmap_free(set2);
bitmask = hwloc_cpuset_to_linux_libnuma_bitmask(topology, set);
/* numa_all_nodes_ptr contains NODES with no CPU but with memory */
hwloc_bitmap_foreach_begin(i, nocpubutmemnodeset) { numa_bitmask_setbit(bitmask, i); } hwloc_bitmap_foreach_end();
int topo_nb_proc(hwloc_topology_t topology,int N)
{
hwloc_obj_t *objs = NULL;
int nb_proc;
objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*N);
objs[0] = hwloc_get_next_obj_by_type(topology,HWLOC_OBJ_PU,NULL);
nb_proc = 1 + hwloc_get_closest_objs(topology,objs[0],objs+1,N-1);
FREE(objs);
return nb_proc;
}
int main(int argc, const char * const argv[])
{
hwloc_topology_t topo;
hwloc_obj_t pu;
const char *basedir;
const char *callname;
char *path;
size_t pathlen;
unsigned idx = (unsigned) -1;
int err;
int ret = EXIT_SUCCESS;
callname = argv[0];
argc--; argv++;
hwloc_utils_check_api_version(callname);
if (!hwloc_have_x86_cpuid()) {
fprintf(stderr, "CPUID not supported.\n");
ret = EXIT_FAILURE;
goto out;
}
while (argc > 0 && argv[0][0] == '-' && argv[0][1] != '\0') {
if (argc >= 2 && !strcmp(argv[0], "-c")) {
idx = atoi(argv[1]);
argc -= 2;
argv += 2;
} else if (argc >= 1 && (!strcmp(argv[0], "-s") || !strcmp(argv[0], "--silent"))) {
verbose--;
argc--;
argv++;
} else if (!strcmp(argv[0], "-h") || !strcmp(argv[0], "--help")) {
usage(callname, stdout);
goto out;
} else {
usage(callname, stderr);
ret = EXIT_FAILURE;
goto out;
}
}
basedir = "./cpuid";
if (argc >= 1)
basedir = argv[0];
if (!getenv("HWLOC_COMPONENTS"))
putenv((char *) "HWLOC_COMPONENTS=no_os,stop");
hwloc_topology_init(&topo);
hwloc_topology_set_all_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_NONE);
err = hwloc_topology_load(topo);
if (err < 0) {
fprintf(stderr, "Failed to load topology\n");
ret = EXIT_FAILURE;
goto out;
}
if (!hwloc_topology_is_thissystem(topo)) {
fprintf(stderr, "%s must run on the current system topology, while this topology doesn't come from this system.\n", callname);
ret = EXIT_FAILURE;
goto out;
}
if (!strcmp(basedir, "-")) {
if (verbose)
printf("Gathering on stdout ...\n");
if (idx == (unsigned) -1) {
fprintf(stderr, "Cannot gather multiple PUs on stdout.\n");
ret = EXIT_FAILURE;
goto out;
}
path = NULL;
pathlen = 0;
} else {
err = mkdir(basedir, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
if (err < 0) {
if (access(basedir, X_OK|W_OK) < 0) {
fprintf(stderr, "Could not create/open destination directory %s\n", basedir);
ret = EXIT_FAILURE;
goto out_with_topo;
}
}
if (verbose)
printf("Gathering in directory %s ...\n", basedir);
pathlen = strlen(basedir) + 20; /* for '/pu%u' or '/hwloc-cpuid-info' */
path = malloc(pathlen);
}
if (idx == (unsigned) -1) {
FILE *file;
pu = NULL;
while ((pu = hwloc_get_next_obj_by_type(topo, HWLOC_OBJ_PU, pu)) != NULL) {
idx = pu->os_index;
if (path)
snprintf(path, pathlen, "%s/pu%u", basedir, idx);
dump_one_proc(topo, pu, path);
}
snprintf(path, pathlen, "%s/hwloc-cpuid-info", basedir);
file = fopen(path, "w");
if (file) {
fprintf(file, "Architecture: x86\n");
fclose(file);
if (verbose)
printf("Summary written to %s\n", path);
} else {
fprintf(stderr, "Failed to open summary file '%s' for writing: %s\n", path, strerror(errno));
}
} else {
pu = hwloc_get_pu_obj_by_os_index(topo, idx);
if (!pu) {
fprintf(stderr, "Cannot find PU P#%u\n", idx);
ret = EXIT_FAILURE;
goto out_with_path;
} else {
if (path)
snprintf(path, pathlen, "%s/pu%u", basedir, idx);
dump_one_proc(topo, pu, path);
}
}
if (verbose)
printf("\n"
"WARNING: Do not post these files on a public list or website unless you\n"
"WARNING: are sure that no information about this platform is sensitive.\n");
out_with_path:
free(path);
out_with_topo:
hwloc_topology_destroy(topo);
out:
return ret;
}
int
pocl_topology_detect_device_info(cl_device_id device)
{
hwloc_topology_t pocl_topology;
int ret = 0;
#ifdef HWLOC_API_2
if (hwloc_get_api_version () < 0x20000)
POCL_MSG_ERR ("pocl was compiled against libhwloc 2.x but is"
"actually running against libhwloc 1.x \n");
#else
if (hwloc_get_api_version () >= 0x20000)
POCL_MSG_ERR ("pocl was compiled against libhwloc 1.x but is"
"actually running against libhwloc 2.x \n");
#endif
/*
* hwloc's OpenCL backend causes problems at the initialization stage
* because it reloads libpocl.so via the ICD loader.
*
* See: https://github.com/pocl/pocl/issues/261
*
* The only trick to stop hwloc from initializing the OpenCL plugin
* I could find is to point the plugin search path to a place where there
* are no plugins to be found.
*/
setenv ("HWLOC_PLUGINS_PATH", "/dev/null", 1);
ret = hwloc_topology_init (&pocl_topology);
if (ret == -1)
{
POCL_MSG_ERR ("Cannot initialize the topology.\n");
return ret;
}
#ifdef HWLOC_API_2
hwloc_topology_set_io_types_filter(pocl_topology, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_SYSTEM, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_GROUP, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_BRIDGE, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_MISC, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_NONE);
hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_NONE);
#else
hwloc_topology_ignore_type (pocl_topology, HWLOC_TOPOLOGY_FLAG_WHOLE_IO);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_SYSTEM);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_GROUP);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_BRIDGE);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_MISC);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_PCI_DEVICE);
hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_OS_DEVICE);
#endif
ret = hwloc_topology_load (pocl_topology);
if (ret == -1)
{
POCL_MSG_ERR ("Cannot load the topology.\n");
goto exit_destroy;
}
#ifdef HWLOC_API_2
device->global_mem_size =
hwloc_get_root_obj(pocl_topology)->total_memory;
#else
device->global_mem_size =
hwloc_get_root_obj(pocl_topology)->memory.total_memory;
#endif
// Try to get the number of CPU cores from topology
int depth = hwloc_get_type_depth(pocl_topology, HWLOC_OBJ_PU);
if(depth != HWLOC_TYPE_DEPTH_UNKNOWN)
device->max_compute_units = hwloc_get_nbobjs_by_depth(pocl_topology, depth);
/* Find information about global memory cache by looking at the first
* cache covering the first PU */
do {
size_t cache_size = 0, cacheline_size = 0;
hwloc_obj_t core
= hwloc_get_next_obj_by_type (pocl_topology, HWLOC_OBJ_CORE, NULL);
if (core)
{
hwloc_obj_t cache
= hwloc_get_shared_cache_covering_obj (pocl_topology, core);
if ((cache) && (cache->attr))
{
cacheline_size = cache->attr->cache.linesize;
cache_size = cache->attr->cache.size;
}
else
core = NULL; /* fallback to L1 cache size */
}
hwloc_obj_t pu
= hwloc_get_next_obj_by_type (pocl_topology, HWLOC_OBJ_PU, NULL);
if (!core && pu)
{
hwloc_obj_t cache
= hwloc_get_shared_cache_covering_obj (pocl_topology, pu);
if ((cache) && (cache->attr))
{
cacheline_size = cache->attr->cache.linesize;
cache_size = cache->attr->cache.size;
}
}
if (!cache_size || !cacheline_size)
break;
device->global_mem_cache_type
= 0x2; // CL_READ_WRITE_CACHE, without including all of CL/cl.h
device->global_mem_cacheline_size = cacheline_size;
device->global_mem_cache_size = cache_size;
} while (0);
// Destroy topology object and return
exit_destroy:
hwloc_topology_destroy (pocl_topology);
return ret;
}
int main(void)
{
const struct hwloc_topology_support *support;
char *buffer;
hwloc_topology_t topology;
hwloc_bitmap_t set = hwloc_bitmap_alloc();
hwloc_bitmap_t total = hwloc_bitmap_alloc();
hwloc_obj_t node;
char *s;
int err;
err = hwloc_topology_init(&topology);
assert(!err);
err = hwloc_topology_load(topology);
assert(!err);
support = hwloc_topology_get_support(topology);
if (!support->membind->get_area_memlocation)
goto out;
buffer = hwloc_alloc(topology, LEN);
assert(buffer);
printf("buffer %p length %u\n", buffer, LEN);
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_get_area_memlocation() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN, s);
free(s);
hwloc_bitmap_copy(total, set);
node = NULL;
next1:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_buffer;
if (!node->memory.local_memory)
goto next1;
printf("binding to 1st node and touching 1st quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
if (err < 0 && errno == ENOSYS) {
fprintf(stderr, "hwloc_set_area_membind() failed with ENOSYS, aborting\n");
goto out_with_buffer;
}
assert(!err);
memset(buffer, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer, 1, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next2:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next2;
printf("binding to 2nd node and touching 2nd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next3:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next3;
printf("binding to 3rd node and touching 3rd quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+LEN/2, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+LEN/2, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/2, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
next4:
node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node);
if (!node)
goto out_with_nomorenodes;
if (!node->memory.local_memory)
goto next4;
printf("binding to 4th node and touching 4th quarter\n");
err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET);
assert(!err);
memset(buffer+3*LEN/4, 0, LEN/4);
err = hwloc_get_area_memlocation(topology, buffer+3*LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u allocated in nodeset %s\n", buffer+3*LEN/4, LEN/4, s);
free(s);
hwloc_bitmap_or(total, total, set);
out_with_nomorenodes:
err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET);
assert(!err);
hwloc_bitmap_asprintf(&s, set);
printf("address %p length %u located on %s\n", buffer, LEN, s);
free(s);
assert(hwloc_bitmap_isincluded(total, set));
out_with_buffer:
hwloc_free(topology, buffer, LEN);
out:
hwloc_topology_destroy(topology);
hwloc_bitmap_free(set);
hwloc_bitmap_free(total);
return 0;
}
/* stuff proc attributes for sending back to a proc */
int orte_pmix_server_register_nspace(orte_job_t *jdata, bool force)
{
int rc;
orte_proc_t *pptr;
int i, k, n;
opal_list_t *info, *pmap;
opal_value_t *kv;
orte_node_t *node, *mynode;
opal_vpid_t vpid;
char **list, **procs, **micro, *tmp, *regex;
orte_job_t *dmns;
orte_job_map_t *map;
orte_app_context_t *app;
uid_t uid;
gid_t gid;
opal_list_t *cache;
hwloc_obj_t machine;
opal_output_verbose(2, orte_pmix_server_globals.output,
"%s register nspace for %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jdata->jobid));
/* setup the info list */
info = OBJ_NEW(opal_list_t);
uid = geteuid();
gid = getegid();
/* pass our nspace/rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_NSPACE);
kv->data.string = strdup(ORTE_JOBID_PRINT(ORTE_PROC_MY_NAME->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_RANK);
kv->data.uint32 = ORTE_PROC_MY_NAME->vpid;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* jobid */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOBID);
kv->data.string = strdup(ORTE_JOBID_PRINT(jdata->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
/* offset */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NPROC_OFFSET);
kv->data.uint32 = jdata->offset;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* check for cached values to add to the job info */
cache = NULL;
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE, (void**)&cache, OPAL_PTR) &&
NULL != cache) {
while (NULL != (kv = (opal_value_t*)opal_list_remove_first(cache))) {
opal_list_append(info, &kv->super);
}
orte_remove_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE);
OBJ_RELEASE(cache);
}
/* assemble the node and proc map info */
list = NULL;
procs = NULL;
map = jdata->map;
for (i=0; i < map->nodes->size; i++) {
micro = NULL;
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
opal_argv_append_nosize(&list, node->name);
/* assemble all the ranks for this job that are on this node */
for (k=0; k < node->procs->size; k++) {
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, k))) {
if (jdata->jobid == pptr->name.jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
}
}
}
/* assemble the rank/node map */
if (NULL != micro) {
tmp = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_argv_append_nosize(&procs, tmp);
free(tmp);
}
}
}
/* let the PMIx server generate the nodemap regex */
if (NULL != list) {
tmp = opal_argv_join(list, ',');
opal_argv_free(list);
list = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_regex(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* let the PMIx server generate the procmap regex */
if (NULL != procs) {
tmp = opal_argv_join(procs, ';');
opal_argv_free(procs);
procs = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_ppn(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* get our local node */
if (NULL == (dmns = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(dmns->procs, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
mynode = pptr->node;
if (NULL == mynode) {
/* cannot happen */
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
/* pass our node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->index;
opal_list_append(info, &kv->super);
/* pass our node size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->num_procs;
opal_list_append(info, &kv->super);
/* pass the number of nodes in the job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NUM_NODES);
kv->type = OPAL_UINT32;
kv->data.uint32 = map->num_nodes;
opal_list_append(info, &kv->super);
/* univ size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_UNIV_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* job size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_procs;
opal_list_append(info, &kv->super);
/* number of apps in this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_NUM_APPS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_apps;
opal_list_append(info, &kv->super);
/* local size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_local_procs;
opal_list_append(info, &kv->super);
/* max procs */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAX_PROCS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* topology signature */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOPOLOGY_SIGNATURE);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_topo_signature);
opal_list_append(info, &kv->super);
/* total available physical memory */
machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
if (NULL != machine) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_AVAIL_PHYS_MEMORY);
kv->type = OPAL_UINT64;
#if HWLOC_API_VERSION < 0x20000
kv->data.uint64 = machine->memory.total_memory;
#else
kv->data.uint64 = machine->total_memory;
#endif
opal_list_append(info, &kv->super);
}
/* pass the mapping policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAPBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_mapping(jdata->map->mapping));
opal_list_append(info, &kv->super);
/* pass the ranking policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANKBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_ranking(jdata->map->ranking));
opal_list_append(info, &kv->super);
/* pass the binding policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_BINDTO);
kv->type = OPAL_STRING;
kv->data.string = strdup(opal_hwloc_base_print_binding(jdata->map->binding));
opal_list_append(info, &kv->super);
/* register any local clients */
vpid = ORTE_VPID_MAX;
micro = NULL;
for (i=0; i < mynode->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(mynode->procs, i))) {
continue;
}
if (pptr->name.jobid == jdata->jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
if (pptr->name.vpid < vpid) {
vpid = pptr->name.vpid;
}
/* go ahead and register this client */
if (OPAL_SUCCESS != (rc = opal_pmix.server_register_client(&pptr->name, uid, gid,
(void*)pptr, NULL, NULL))) {
ORTE_ERROR_LOG(rc);
}
}
}
if (NULL != micro) {
/* pass the local peers */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv->type = OPAL_STRING;
kv->data.string = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_list_append(info, &kv->super);
}
/* pass the local ldr */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = vpid;
opal_list_append(info, &kv->super);
/* for each proc in this job, create an object that
* includes the info describing the proc so the recipient has a complete
* picture. This allows procs to connect to each other without
* any further info exchange, assuming the underlying transports
* support it. We also pass all the proc-specific data here so
* that each proc can lookup info about every other proc in the job */
for (n=0; n < map->nodes->size; n++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) {
continue;
}
/* cycle across each proc on this node, passing all data that
* varies by proc */
for (i=0; i < node->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) {
continue;
}
/* only consider procs from this job */
if (pptr->name.jobid != jdata->jobid) {
continue;
}
/* setup the proc map object */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_DATA);
kv->type = OPAL_PTR;
kv->data.ptr = OBJ_NEW(opal_list_t);
opal_list_append(info, &kv->super);
pmap = kv->data.ptr;
/* must start with rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid;
opal_list_append(pmap, &kv->super);
/* location, for local procs */
if (node == mynode) {
tmp = NULL;
if (orte_get_attribute(&pptr->attributes, ORTE_PROC_CPU_BITMAP, (void**)&tmp, OPAL_STRING) &&
NULL != tmp) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = opal_hwloc_base_get_locality_string(opal_hwloc_topology, tmp);
opal_list_append(pmap, &kv->super);
free(tmp);
} else {
/* the proc is not bound */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = NULL;
opal_list_append(pmap, &kv->super);
}
}
/* global/univ rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_GLOBAL_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid + jdata->offset;
opal_list_append(pmap, &kv->super);
if (1 < jdata->num_apps) {
/* appnum */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPNUM);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->app_idx;
opal_list_append(pmap, &kv->super);
/* app ldr */
app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = app->first_rank;
opal_list_append(pmap, &kv->super);
/* app rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->app_rank;
opal_list_append(pmap, &kv->super);
/* app size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = app->num_procs;
opal_list_append(info, &kv->super);
}
/* local rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_RANK);
kv->type = OPAL_UINT16;
kv->data.uint16 = pptr->local_rank;
opal_list_append(pmap, &kv->super);
/* node rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_RANK);
kv->type = OPAL_UINT16;
kv->data.uint32 = pptr->node_rank;
opal_list_append(pmap, &kv->super);
/* node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->node->index;
opal_list_append(pmap, &kv->super);
if (map->num_nodes < orte_hostname_cutoff) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_HOSTNAME);
kv->type = OPAL_STRING;
kv->data.string = strdup(pptr->node->name);
opal_list_append(pmap, &kv->super);
}
}
}
/* mark the job as registered */
orte_set_attribute(&jdata->attributes, ORTE_JOB_NSPACE_REGISTERED, ORTE_ATTR_LOCAL, NULL, OPAL_BOOL);
/* pass it down */
/* we are in an event, so no need to callback */
rc = opal_pmix.server_register_nspace(jdata->jobid,
jdata->num_local_procs,
info, NULL, NULL);
OPAL_LIST_RELEASE(info);
/* if the user has connected us to an external server, then we must
* assume there is going to be some cross-mpirun exchange, and so
* we protect against that situation by publishing the job info
* for this job - this allows any subsequent "connect" to retrieve
* the job info */
if (NULL != orte_data_server_uri) {
opal_buffer_t buf;
OBJ_CONSTRUCT(&buf, opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &jdata, 1, ORTE_JOB))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
return rc;
}
info = OBJ_NEW(opal_list_t);
/* create a key-value with the key being the string jobid
* and the value being the byte object */
kv = OBJ_NEW(opal_value_t);
orte_util_convert_jobid_to_string(&kv->key, jdata->jobid);
kv->type = OPAL_BYTE_OBJECT;
opal_dss.unload(&buf, (void**)&kv->data.bo.bytes, &kv->data.bo.size);
OBJ_DESTRUCT(&buf);
opal_list_append(info, &kv->super);
/* set the range to be session */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANGE);
kv->type = OPAL_UINT;
kv->data.uint = OPAL_PMIX_RANGE_SESSION;
opal_list_append(info, &kv->super);
/* set the persistence to be app */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PERSISTENCE);
kv->type = OPAL_INT;
kv->data.integer = OPAL_PMIX_PERSIST_APP;
opal_list_append(info, &kv->super);
/* add our effective userid to the directives */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_USERID);
kv->type = OPAL_UINT32;
kv->data.uint32 = geteuid();
opal_list_append(info, &kv->super);
/* now publish it */
if (ORTE_SUCCESS != (rc = pmix_server_publish_fn(ORTE_PROC_MY_NAME,
info, mycbfunc, info))) {
ORTE_ERROR_LOG(rc);
}
}
return rc;
}
