static int mca_sbgp_map_to_socket_core(int processor_id, int *socket, int *core) { int ret = OPAL_ERR_NOT_FOUND; hwloc_obj_t obj; hwloc_topology_t *t; hwloc_bitmap_t good; /* bozo check */ if (NULL == opal_hwloc_topology) { return OPAL_ERR_NOT_INITIALIZED; } t = &opal_hwloc_topology; good = hwloc_bitmap_alloc(); if (NULL == good) { return OPAL_ERR_OUT_OF_RESOURCE; } /* Iterate through every core and find one that contains the processor_id. Then find the corresponding socket. */ for (obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, NULL); NULL != obj; obj = hwloc_get_next_obj_by_type(*t, HWLOC_OBJ_CORE, obj)) { hwloc_bitmap_and(good, obj->online_cpuset, obj->allowed_cpuset); /* Does this core contain the processor_id in question? */ if (hwloc_bitmap_isset(good, processor_id)) { *core = obj->os_index; /* Go upward from the core object until we find its parent socket. */ while (HWLOC_OBJ_SOCKET != obj->type) { if (NULL == obj->parent) { /* If we get to the root without finding a socket, er.. Hmm. Error! */ ret = OPAL_ERR_NOT_FOUND; goto out; } obj = obj->parent; } *socket = obj->os_index; ret = OPAL_SUCCESS; goto out; } } /* If we didn't even find the right core, we didn't find it. Fall through. */ ret = OPAL_ERR_NOT_FOUND; out: hwloc_bitmap_free(good); return ret; }
static int get_total_number_of(hwloc_obj_type_t obj_type) { int number = 0; if (!has_topology_information()) return 0; number = hwloc_get_nbobjs_by_type (sge_hwloc_topology, obj_type); if (-1 == number) { /* Only for things like L1, L2 caches, not the types we're interested in? */ hwloc_obj_t obj; number = 0; for (obj = hwloc_get_next_obj_by_type(sge_hwloc_topology, obj_type, NULL); NULL != obj; obj = hwloc_get_next_obj_by_type(sge_hwloc_topology, obj_type, obj)) if (obj_type == obj->type) ++number; } return number; }
int main(void) { hwloc_topology_t topology; hwloc_bitmap_t set, set2, nocpunomemnodeset, nocpubutmemnodeset, nomembutcpunodeset, nomembutcpucpuset; hwloc_obj_t node; struct bitmask *bitmask, *bitmask2; unsigned long mask; unsigned long maxnode; int i; if (numa_available() < 0) /* libnuma has inconsistent behavior when the kernel isn't NUMA-aware. * don't try to check everything precisely. */ exit(77); hwloc_topology_init(&topology); hwloc_topology_load(topology); /* convert full stuff between cpuset and libnuma */ set = hwloc_bitmap_alloc(); nocpunomemnodeset = hwloc_bitmap_alloc(); nocpubutmemnodeset = hwloc_bitmap_alloc(); nomembutcpunodeset = hwloc_bitmap_alloc(); nomembutcpucpuset = hwloc_bitmap_alloc(); /* gather all nodes if any, or the whole system if no nodes */ if (hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_NUMANODE)) { node = NULL; while ((node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node)) != NULL) { hwloc_bitmap_or(set, set, node->cpuset); if (hwloc_bitmap_iszero(node->cpuset)) { if (node->memory.local_memory) hwloc_bitmap_set(nocpubutmemnodeset, node->os_index); else hwloc_bitmap_set(nocpunomemnodeset, node->os_index); } else if (!node->memory.local_memory) { hwloc_bitmap_set(nomembutcpunodeset, node->os_index); hwloc_bitmap_or(nomembutcpucpuset, nomembutcpucpuset, node->cpuset); } } } else { hwloc_bitmap_or(set, set, hwloc_topology_get_complete_cpuset(topology)); } set2 = hwloc_bitmap_alloc(); hwloc_cpuset_from_linux_libnuma_bitmask(topology, set2, numa_all_nodes_ptr); /* numa_all_nodes_ptr doesn't contain NODES with CPU but no memory */ hwloc_bitmap_or(set2, set2, nomembutcpucpuset); assert(hwloc_bitmap_isequal(set, set2)); hwloc_bitmap_free(set2); bitmask = hwloc_cpuset_to_linux_libnuma_bitmask(topology, set); /* numa_all_nodes_ptr contains NODES with no CPU but with memory */ hwloc_bitmap_foreach_begin(i, nocpubutmemnodeset) { numa_bitmask_setbit(bitmask, i); } hwloc_bitmap_foreach_end();
int topo_nb_proc(hwloc_topology_t topology,int N) { hwloc_obj_t *objs = NULL; int nb_proc; objs = (hwloc_obj_t*)MALLOC(sizeof(hwloc_obj_t)*N); objs[0] = hwloc_get_next_obj_by_type(topology,HWLOC_OBJ_PU,NULL); nb_proc = 1 + hwloc_get_closest_objs(topology,objs[0],objs+1,N-1); FREE(objs); return nb_proc; }
int main(int argc, const char * const argv[]) { hwloc_topology_t topo; hwloc_obj_t pu; const char *basedir; const char *callname; char *path; size_t pathlen; unsigned idx = (unsigned) -1; int err; int ret = EXIT_SUCCESS; callname = argv[0]; argc--; argv++; hwloc_utils_check_api_version(callname); if (!hwloc_have_x86_cpuid()) { fprintf(stderr, "CPUID not supported.\n"); ret = EXIT_FAILURE; goto out; } while (argc > 0 && argv[0][0] == '-' && argv[0][1] != '\0') { if (argc >= 2 && !strcmp(argv[0], "-c")) { idx = atoi(argv[1]); argc -= 2; argv += 2; } else if (argc >= 1 && (!strcmp(argv[0], "-s") || !strcmp(argv[0], "--silent"))) { verbose--; argc--; argv++; } else if (!strcmp(argv[0], "-h") || !strcmp(argv[0], "--help")) { usage(callname, stdout); goto out; } else { usage(callname, stderr); ret = EXIT_FAILURE; goto out; } } basedir = "./cpuid"; if (argc >= 1) basedir = argv[0]; if (!getenv("HWLOC_COMPONENTS")) putenv((char *) "HWLOC_COMPONENTS=no_os,stop"); hwloc_topology_init(&topo); hwloc_topology_set_all_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_NONE); err = hwloc_topology_load(topo); if (err < 0) { fprintf(stderr, "Failed to load topology\n"); ret = EXIT_FAILURE; goto out; } if (!hwloc_topology_is_thissystem(topo)) { fprintf(stderr, "%s must run on the current system topology, while this topology doesn't come from this system.\n", callname); ret = EXIT_FAILURE; goto out; } if (!strcmp(basedir, "-")) { if (verbose) printf("Gathering on stdout ...\n"); if (idx == (unsigned) -1) { fprintf(stderr, "Cannot gather multiple PUs on stdout.\n"); ret = EXIT_FAILURE; goto out; } path = NULL; pathlen = 0; } else { err = mkdir(basedir, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH); if (err < 0) { if (access(basedir, X_OK|W_OK) < 0) { fprintf(stderr, "Could not create/open destination directory %s\n", basedir); ret = EXIT_FAILURE; goto out_with_topo; } } if (verbose) printf("Gathering in directory %s ...\n", basedir); pathlen = strlen(basedir) + 20; /* for '/pu%u' or '/hwloc-cpuid-info' */ path = malloc(pathlen); } if (idx == (unsigned) -1) { FILE *file; pu = NULL; while ((pu = hwloc_get_next_obj_by_type(topo, HWLOC_OBJ_PU, pu)) != NULL) { idx = pu->os_index; if (path) snprintf(path, pathlen, "%s/pu%u", basedir, idx); dump_one_proc(topo, pu, path); } snprintf(path, pathlen, "%s/hwloc-cpuid-info", basedir); file = fopen(path, "w"); if (file) { fprintf(file, "Architecture: x86\n"); fclose(file); if (verbose) printf("Summary written to %s\n", path); } else { fprintf(stderr, "Failed to open summary file '%s' for writing: %s\n", path, strerror(errno)); } } else { pu = hwloc_get_pu_obj_by_os_index(topo, idx); if (!pu) { fprintf(stderr, "Cannot find PU P#%u\n", idx); ret = EXIT_FAILURE; goto out_with_path; } else { if (path) snprintf(path, pathlen, "%s/pu%u", basedir, idx); dump_one_proc(topo, pu, path); } } if (verbose) printf("\n" "WARNING: Do not post these files on a public list or website unless you\n" "WARNING: are sure that no information about this platform is sensitive.\n"); out_with_path: free(path); out_with_topo: hwloc_topology_destroy(topo); out: return ret; }
int pocl_topology_detect_device_info(cl_device_id device) { hwloc_topology_t pocl_topology; int ret = 0; #ifdef HWLOC_API_2 if (hwloc_get_api_version () < 0x20000) POCL_MSG_ERR ("pocl was compiled against libhwloc 2.x but is" "actually running against libhwloc 1.x \n"); #else if (hwloc_get_api_version () >= 0x20000) POCL_MSG_ERR ("pocl was compiled against libhwloc 1.x but is" "actually running against libhwloc 2.x \n"); #endif /* * hwloc's OpenCL backend causes problems at the initialization stage * because it reloads libpocl.so via the ICD loader. * * See: https://github.com/pocl/pocl/issues/261 * * The only trick to stop hwloc from initializing the OpenCL plugin * I could find is to point the plugin search path to a place where there * are no plugins to be found. */ setenv ("HWLOC_PLUGINS_PATH", "/dev/null", 1); ret = hwloc_topology_init (&pocl_topology); if (ret == -1) { POCL_MSG_ERR ("Cannot initialize the topology.\n"); return ret; } #ifdef HWLOC_API_2 hwloc_topology_set_io_types_filter(pocl_topology, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_SYSTEM, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_GROUP, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_BRIDGE, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_MISC, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_NONE); hwloc_topology_set_type_filter (pocl_topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_NONE); #else hwloc_topology_ignore_type (pocl_topology, HWLOC_TOPOLOGY_FLAG_WHOLE_IO); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_SYSTEM); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_GROUP); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_BRIDGE); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_MISC); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_PCI_DEVICE); hwloc_topology_ignore_type (pocl_topology, HWLOC_OBJ_OS_DEVICE); #endif ret = hwloc_topology_load (pocl_topology); if (ret == -1) { POCL_MSG_ERR ("Cannot load the topology.\n"); goto exit_destroy; } #ifdef HWLOC_API_2 device->global_mem_size = hwloc_get_root_obj(pocl_topology)->total_memory; #else device->global_mem_size = hwloc_get_root_obj(pocl_topology)->memory.total_memory; #endif // Try to get the number of CPU cores from topology int depth = hwloc_get_type_depth(pocl_topology, HWLOC_OBJ_PU); if(depth != HWLOC_TYPE_DEPTH_UNKNOWN) device->max_compute_units = hwloc_get_nbobjs_by_depth(pocl_topology, depth); /* Find information about global memory cache by looking at the first * cache covering the first PU */ do { size_t cache_size = 0, cacheline_size = 0; hwloc_obj_t core = hwloc_get_next_obj_by_type (pocl_topology, HWLOC_OBJ_CORE, NULL); if (core) { hwloc_obj_t cache = hwloc_get_shared_cache_covering_obj (pocl_topology, core); if ((cache) && (cache->attr)) { cacheline_size = cache->attr->cache.linesize; cache_size = cache->attr->cache.size; } else core = NULL; /* fallback to L1 cache size */ } hwloc_obj_t pu = hwloc_get_next_obj_by_type (pocl_topology, HWLOC_OBJ_PU, NULL); if (!core && pu) { hwloc_obj_t cache = hwloc_get_shared_cache_covering_obj (pocl_topology, pu); if ((cache) && (cache->attr)) { cacheline_size = cache->attr->cache.linesize; cache_size = cache->attr->cache.size; } } if (!cache_size || !cacheline_size) break; device->global_mem_cache_type = 0x2; // CL_READ_WRITE_CACHE, without including all of CL/cl.h device->global_mem_cacheline_size = cacheline_size; device->global_mem_cache_size = cache_size; } while (0); // Destroy topology object and return exit_destroy: hwloc_topology_destroy (pocl_topology); return ret; }
int main(void) { const struct hwloc_topology_support *support; char *buffer; hwloc_topology_t topology; hwloc_bitmap_t set = hwloc_bitmap_alloc(); hwloc_bitmap_t total = hwloc_bitmap_alloc(); hwloc_obj_t node; char *s; int err; err = hwloc_topology_init(&topology); assert(!err); err = hwloc_topology_load(topology); assert(!err); support = hwloc_topology_get_support(topology); if (!support->membind->get_area_memlocation) goto out; buffer = hwloc_alloc(topology, LEN); assert(buffer); printf("buffer %p length %u\n", buffer, LEN); err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET); if (err < 0 && errno == ENOSYS) { fprintf(stderr, "hwloc_get_area_memlocation() failed with ENOSYS, aborting\n"); goto out_with_buffer; } assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u allocated in nodeset %s\n", buffer, LEN, s); free(s); hwloc_bitmap_copy(total, set); node = NULL; next1: node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node); if (!node) goto out_with_buffer; if (!node->memory.local_memory) goto next1; printf("binding to 1st node and touching 1st quarter\n"); err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET); if (err < 0 && errno == ENOSYS) { fprintf(stderr, "hwloc_set_area_membind() failed with ENOSYS, aborting\n"); goto out_with_buffer; } assert(!err); memset(buffer, 0, LEN/4); err = hwloc_get_area_memlocation(topology, buffer, 1, set, HWLOC_MEMBIND_BYNODESET); assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u allocated in nodeset %s\n", buffer, LEN/4, s); free(s); hwloc_bitmap_or(total, total, set); next2: node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node); if (!node) goto out_with_nomorenodes; if (!node->memory.local_memory) goto next2; printf("binding to 2nd node and touching 2nd quarter\n"); err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET); assert(!err); memset(buffer+LEN/4, 0, LEN/4); err = hwloc_get_area_memlocation(topology, buffer+LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET); assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/4, LEN/4, s); free(s); hwloc_bitmap_or(total, total, set); next3: node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node); if (!node) goto out_with_nomorenodes; if (!node->memory.local_memory) goto next3; printf("binding to 3rd node and touching 3rd quarter\n"); err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET); assert(!err); memset(buffer+LEN/2, 0, LEN/4); err = hwloc_get_area_memlocation(topology, buffer+LEN/2, LEN/4, set, HWLOC_MEMBIND_BYNODESET); assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u allocated in nodeset %s\n", buffer+LEN/2, LEN/4, s); free(s); hwloc_bitmap_or(total, total, set); next4: node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node); if (!node) goto out_with_nomorenodes; if (!node->memory.local_memory) goto next4; printf("binding to 4th node and touching 4th quarter\n"); err = hwloc_set_area_membind(topology, buffer, LEN, node->nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_BYNODESET); assert(!err); memset(buffer+3*LEN/4, 0, LEN/4); err = hwloc_get_area_memlocation(topology, buffer+3*LEN/4, LEN/4, set, HWLOC_MEMBIND_BYNODESET); assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u allocated in nodeset %s\n", buffer+3*LEN/4, LEN/4, s); free(s); hwloc_bitmap_or(total, total, set); out_with_nomorenodes: err = hwloc_get_area_memlocation(topology, buffer, LEN, set, HWLOC_MEMBIND_BYNODESET); assert(!err); hwloc_bitmap_asprintf(&s, set); printf("address %p length %u located on %s\n", buffer, LEN, s); free(s); assert(hwloc_bitmap_isincluded(total, set)); out_with_buffer: hwloc_free(topology, buffer, LEN); out: hwloc_topology_destroy(topology); hwloc_bitmap_free(set); hwloc_bitmap_free(total); return 0; }
/* stuff proc attributes for sending back to a proc */ int orte_pmix_server_register_nspace(orte_job_t *jdata, bool force) { int rc; orte_proc_t *pptr; int i, k, n; opal_list_t *info, *pmap; opal_value_t *kv; orte_node_t *node, *mynode; opal_vpid_t vpid; char **list, **procs, **micro, *tmp, *regex; orte_job_t *dmns; orte_job_map_t *map; orte_app_context_t *app; uid_t uid; gid_t gid; opal_list_t *cache; hwloc_obj_t machine; opal_output_verbose(2, orte_pmix_server_globals.output, "%s register nspace for %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_JOBID_PRINT(jdata->jobid)); /* setup the info list */ info = OBJ_NEW(opal_list_t); uid = geteuid(); gid = getegid(); /* pass our nspace/rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_SERVER_NSPACE); kv->data.string = strdup(ORTE_JOBID_PRINT(ORTE_PROC_MY_NAME->jobid)); kv->type = OPAL_STRING; opal_list_append(info, &kv->super); kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_SERVER_RANK); kv->data.uint32 = ORTE_PROC_MY_NAME->vpid; kv->type = OPAL_UINT32; opal_list_append(info, &kv->super); /* jobid */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_JOBID); kv->data.string = strdup(ORTE_JOBID_PRINT(jdata->jobid)); kv->type = OPAL_STRING; opal_list_append(info, &kv->super); /* offset */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NPROC_OFFSET); kv->data.uint32 = jdata->offset; kv->type = OPAL_UINT32; opal_list_append(info, &kv->super); /* check for cached values to add to the job info */ cache = NULL; if (orte_get_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE, (void**)&cache, OPAL_PTR) && NULL != cache) { while (NULL != (kv = (opal_value_t*)opal_list_remove_first(cache))) { opal_list_append(info, &kv->super); } orte_remove_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE); OBJ_RELEASE(cache); } /* assemble the node and proc map info */ list = NULL; procs = NULL; map = jdata->map; for (i=0; i < map->nodes->size; i++) { micro = NULL; if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) { opal_argv_append_nosize(&list, node->name); /* assemble all the ranks for this job that are on this node */ for (k=0; k < node->procs->size; k++) { if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, k))) { if (jdata->jobid == pptr->name.jobid) { opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid)); } } } /* assemble the rank/node map */ if (NULL != micro) { tmp = opal_argv_join(micro, ','); opal_argv_free(micro); opal_argv_append_nosize(&procs, tmp); free(tmp); } } } /* let the PMIx server generate the nodemap regex */ if (NULL != list) { tmp = opal_argv_join(list, ','); opal_argv_free(list); list = NULL; if (OPAL_SUCCESS != (rc = opal_pmix.generate_regex(tmp, ®ex))) { ORTE_ERROR_LOG(rc); free(tmp); OPAL_LIST_RELEASE(info); return rc; } free(tmp); kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NODE_MAP); kv->type = OPAL_STRING; kv->data.string = regex; opal_list_append(info, &kv->super); } /* let the PMIx server generate the procmap regex */ if (NULL != procs) { tmp = opal_argv_join(procs, ';'); opal_argv_free(procs); procs = NULL; if (OPAL_SUCCESS != (rc = opal_pmix.generate_ppn(tmp, ®ex))) { ORTE_ERROR_LOG(rc); free(tmp); OPAL_LIST_RELEASE(info); return rc; } free(tmp); kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_PROC_MAP); kv->type = OPAL_STRING; kv->data.string = regex; opal_list_append(info, &kv->super); } /* get our local node */ if (NULL == (dmns = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) { ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND); OPAL_LIST_RELEASE(info); return ORTE_ERR_NOT_FOUND; } if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(dmns->procs, ORTE_PROC_MY_NAME->vpid))) { ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND); OPAL_LIST_RELEASE(info); return ORTE_ERR_NOT_FOUND; } mynode = pptr->node; if (NULL == mynode) { /* cannot happen */ ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND); OPAL_LIST_RELEASE(info); return ORTE_ERR_NOT_FOUND; } /* pass our node ID */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NODEID); kv->type = OPAL_UINT32; kv->data.uint32 = mynode->index; opal_list_append(info, &kv->super); /* pass our node size */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NODE_SIZE); kv->type = OPAL_UINT32; kv->data.uint32 = mynode->num_procs; opal_list_append(info, &kv->super); /* pass the number of nodes in the job */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NUM_NODES); kv->type = OPAL_UINT32; kv->data.uint32 = map->num_nodes; opal_list_append(info, &kv->super); /* univ size */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_UNIV_SIZE); kv->type = OPAL_UINT32; kv->data.uint32 = jdata->total_slots_alloc; opal_list_append(info, &kv->super); /* job size */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_JOB_SIZE); kv->type = OPAL_UINT32; kv->data.uint32 = jdata->num_procs; opal_list_append(info, &kv->super); /* number of apps in this job */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_JOB_NUM_APPS); kv->type = OPAL_UINT32; kv->data.uint32 = jdata->num_apps; opal_list_append(info, &kv->super); /* local size */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCAL_SIZE); kv->type = OPAL_UINT32; kv->data.uint32 = jdata->num_local_procs; opal_list_append(info, &kv->super); /* max procs */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_MAX_PROCS); kv->type = OPAL_UINT32; kv->data.uint32 = jdata->total_slots_alloc; opal_list_append(info, &kv->super); /* topology signature */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_TOPOLOGY_SIGNATURE); kv->type = OPAL_STRING; kv->data.string = strdup(orte_topo_signature); opal_list_append(info, &kv->super); /* total available physical memory */ machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL); if (NULL != machine) { kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_AVAIL_PHYS_MEMORY); kv->type = OPAL_UINT64; #if HWLOC_API_VERSION < 0x20000 kv->data.uint64 = machine->memory.total_memory; #else kv->data.uint64 = machine->total_memory; #endif opal_list_append(info, &kv->super); } /* pass the mapping policy used for this job */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_MAPBY); kv->type = OPAL_STRING; kv->data.string = strdup(orte_rmaps_base_print_mapping(jdata->map->mapping)); opal_list_append(info, &kv->super); /* pass the ranking policy used for this job */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_RANKBY); kv->type = OPAL_STRING; kv->data.string = strdup(orte_rmaps_base_print_ranking(jdata->map->ranking)); opal_list_append(info, &kv->super); /* pass the binding policy used for this job */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_BINDTO); kv->type = OPAL_STRING; kv->data.string = strdup(opal_hwloc_base_print_binding(jdata->map->binding)); opal_list_append(info, &kv->super); /* register any local clients */ vpid = ORTE_VPID_MAX; micro = NULL; for (i=0; i < mynode->procs->size; i++) { if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(mynode->procs, i))) { continue; } if (pptr->name.jobid == jdata->jobid) { opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid)); if (pptr->name.vpid < vpid) { vpid = pptr->name.vpid; } /* go ahead and register this client */ if (OPAL_SUCCESS != (rc = opal_pmix.server_register_client(&pptr->name, uid, gid, (void*)pptr, NULL, NULL))) { ORTE_ERROR_LOG(rc); } } } if (NULL != micro) { /* pass the local peers */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCAL_PEERS); kv->type = OPAL_STRING; kv->data.string = opal_argv_join(micro, ','); opal_argv_free(micro); opal_list_append(info, &kv->super); } /* pass the local ldr */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCALLDR); kv->type = OPAL_VPID; kv->data.name.vpid = vpid; opal_list_append(info, &kv->super); /* for each proc in this job, create an object that * includes the info describing the proc so the recipient has a complete * picture. This allows procs to connect to each other without * any further info exchange, assuming the underlying transports * support it. We also pass all the proc-specific data here so * that each proc can lookup info about every other proc in the job */ for (n=0; n < map->nodes->size; n++) { if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) { continue; } /* cycle across each proc on this node, passing all data that * varies by proc */ for (i=0; i < node->procs->size; i++) { if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) { continue; } /* only consider procs from this job */ if (pptr->name.jobid != jdata->jobid) { continue; } /* setup the proc map object */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_PROC_DATA); kv->type = OPAL_PTR; kv->data.ptr = OBJ_NEW(opal_list_t); opal_list_append(info, &kv->super); pmap = kv->data.ptr; /* must start with rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_RANK); kv->type = OPAL_VPID; kv->data.name.vpid = pptr->name.vpid; opal_list_append(pmap, &kv->super); /* location, for local procs */ if (node == mynode) { tmp = NULL; if (orte_get_attribute(&pptr->attributes, ORTE_PROC_CPU_BITMAP, (void**)&tmp, OPAL_STRING) && NULL != tmp) { kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCALITY_STRING); kv->type = OPAL_STRING; kv->data.string = opal_hwloc_base_get_locality_string(opal_hwloc_topology, tmp); opal_list_append(pmap, &kv->super); free(tmp); } else { /* the proc is not bound */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCALITY_STRING); kv->type = OPAL_STRING; kv->data.string = NULL; opal_list_append(pmap, &kv->super); } } /* global/univ rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_GLOBAL_RANK); kv->type = OPAL_VPID; kv->data.name.vpid = pptr->name.vpid + jdata->offset; opal_list_append(pmap, &kv->super); if (1 < jdata->num_apps) { /* appnum */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_APPNUM); kv->type = OPAL_UINT32; kv->data.uint32 = pptr->app_idx; opal_list_append(pmap, &kv->super); /* app ldr */ app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx); kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_APPLDR); kv->type = OPAL_VPID; kv->data.name.vpid = app->first_rank; opal_list_append(pmap, &kv->super); /* app rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_APP_RANK); kv->type = OPAL_VPID; kv->data.name.vpid = pptr->app_rank; opal_list_append(pmap, &kv->super); /* app size */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_APP_SIZE); kv->type = OPAL_UINT32; kv->data.uint32 = app->num_procs; opal_list_append(info, &kv->super); } /* local rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_LOCAL_RANK); kv->type = OPAL_UINT16; kv->data.uint16 = pptr->local_rank; opal_list_append(pmap, &kv->super); /* node rank */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NODE_RANK); kv->type = OPAL_UINT16; kv->data.uint32 = pptr->node_rank; opal_list_append(pmap, &kv->super); /* node ID */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_NODEID); kv->type = OPAL_UINT32; kv->data.uint32 = pptr->node->index; opal_list_append(pmap, &kv->super); if (map->num_nodes < orte_hostname_cutoff) { kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_HOSTNAME); kv->type = OPAL_STRING; kv->data.string = strdup(pptr->node->name); opal_list_append(pmap, &kv->super); } } } /* mark the job as registered */ orte_set_attribute(&jdata->attributes, ORTE_JOB_NSPACE_REGISTERED, ORTE_ATTR_LOCAL, NULL, OPAL_BOOL); /* pass it down */ /* we are in an event, so no need to callback */ rc = opal_pmix.server_register_nspace(jdata->jobid, jdata->num_local_procs, info, NULL, NULL); OPAL_LIST_RELEASE(info); /* if the user has connected us to an external server, then we must * assume there is going to be some cross-mpirun exchange, and so * we protect against that situation by publishing the job info * for this job - this allows any subsequent "connect" to retrieve * the job info */ if (NULL != orte_data_server_uri) { opal_buffer_t buf; OBJ_CONSTRUCT(&buf, opal_buffer_t); if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &jdata, 1, ORTE_JOB))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return rc; } info = OBJ_NEW(opal_list_t); /* create a key-value with the key being the string jobid * and the value being the byte object */ kv = OBJ_NEW(opal_value_t); orte_util_convert_jobid_to_string(&kv->key, jdata->jobid); kv->type = OPAL_BYTE_OBJECT; opal_dss.unload(&buf, (void**)&kv->data.bo.bytes, &kv->data.bo.size); OBJ_DESTRUCT(&buf); opal_list_append(info, &kv->super); /* set the range to be session */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_RANGE); kv->type = OPAL_UINT; kv->data.uint = OPAL_PMIX_RANGE_SESSION; opal_list_append(info, &kv->super); /* set the persistence to be app */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_PERSISTENCE); kv->type = OPAL_INT; kv->data.integer = OPAL_PMIX_PERSIST_APP; opal_list_append(info, &kv->super); /* add our effective userid to the directives */ kv = OBJ_NEW(opal_value_t); kv->key = strdup(OPAL_PMIX_USERID); kv->type = OPAL_UINT32; kv->data.uint32 = geteuid(); opal_list_append(info, &kv->super); /* now publish it */ if (ORTE_SUCCESS != (rc = pmix_server_publish_fn(ORTE_PROC_MY_NAME, info, mycbfunc, info))) { ORTE_ERROR_LOG(rc); } } return rc; }