int main(void)
{
hwloc_topology_t topology;
int i;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
for(i=0; ; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, ancestor;
const char *value;
osdev = hwloc_intel_mic_get_device_osdev_by_index(topology, i);
if (!osdev)
break;
assert(osdev);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "mic", 3);
assert(!err);
assert(atoi(osdev->name+3) == (int) i);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "MIC");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "MICFamily");
printf("found MICFamily %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICSKU");
printf("found MICSKU %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICActiveCores");
printf("found MICActiveCores %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICMemorySize");
printf("found MICMemorySize %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_intel_mic_get_device_cpuset(topology, i, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
static void
pci_device_draw(hwloc_topology_t topology __hwloc_attribute_unused, struct draw_methods *methods, int logical, hwloc_obj_t level, void *output, unsigned depth, unsigned x, unsigned *retwidth, unsigned y, unsigned *retheight)
{
unsigned textwidth = gridsize;
unsigned textheight = (fontsize ? fontsize + gridsize : 0);
unsigned myheight = textheight;
unsigned mywidth = 0;
unsigned totwidth, totheight;
unsigned overlaidoffset = 0;
struct style style;
char text[64], _text[64];
const char *collapsestr = hwloc_obj_get_info_by_name(level, "lstopoCollapse");
unsigned collapse = collapsestr ? atoi(collapsestr) : 1;
int n;
DYNA_CHECK();
if (fontsize) {
if (collapse > 1) {
n = lstopo_obj_snprintf(_text, sizeof(_text), level, logical);
n = snprintf(text, sizeof(text), "%u x { %s }", collapse, _text);
} else {
n = lstopo_obj_snprintf(text, sizeof(text), level, logical);
}
textwidth = get_textwidth(output, methods, text, n, fontsize, gridsize);
}
if (collapse > 1) {
/* additional depths and height for overlaid boxes */
depth -= 2;
if (collapse > 2) {
overlaidoffset = gridsize;
} else {
overlaidoffset = gridsize/2;
}
textwidth += overlaidoffset;
textheight += overlaidoffset;
myheight = textheight;
}
RECURSE_RECT(level, &null_draw_methods, gridsize, gridsize);
lstopo_set_object_color(methods, topology, level, 0, &style);
if (collapse > 1) {
methods->box(output, style.bg.r, style.bg.g, style.bg.b, depth+2, x + overlaidoffset, *retwidth - overlaidoffset, y + overlaidoffset, *retheight - overlaidoffset);
if (collapse > 2)
methods->box(output, style.bg.r, style.bg.g, style.bg.b, depth+1, x + overlaidoffset/2, *retwidth - overlaidoffset, y + overlaidoffset/2, *retheight - overlaidoffset);
methods->box(output, style.bg.r, style.bg.g, style.bg.b, depth, x, *retwidth - overlaidoffset, y, *retheight - overlaidoffset);
} else {
methods->box(output, style.bg.r, style.bg.g, style.bg.b, depth, x, *retwidth, y, *retheight);
}
if (fontsize)
methods->text(output, style.t.r, style.t.g, style.t.b, fontsize, depth-1, x + gridsize, y + gridsize, text);
RECURSE_RECT(level, methods, gridsize, gridsize);
DYNA_SAVE();
}
void hwloc_init_cpuInfo(cpu_set_t cpuSet)
{
int i;
hwloc_obj_t obj;
hwloc_topology_init(&hwloc_topology);
hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_WHOLE_IO );
hwloc_topology_load(hwloc_topology);
obj = hwloc_get_obj_by_type(hwloc_topology, HWLOC_OBJ_SOCKET, 0);
cpuid_info.model = 0;
cpuid_info.family = 0;
cpuid_info.isIntel = 0;
cpuid_info.osname = malloc(MAX_MODEL_STRING_LENGTH * sizeof(char));
for(i=0;i<obj->infos_count;i++)
{
if (strcmp(obj->infos[i].name ,"CPUModelNumber") == 0)
{
cpuid_info.model = atoi(hwloc_obj_get_info_by_name(obj, "CPUModelNumber"));
}
else if (strcmp(obj->infos[i].name, "CPUFamilyNumber") == 0)
{
cpuid_info.family = atoi(hwloc_obj_get_info_by_name(obj, "CPUFamilyNumber"));
}
else if (strcmp(obj->infos[i].name, "CPUVendor") == 0 &&
strcmp(hwloc_obj_get_info_by_name(obj, "CPUVendor"), "GenuineIntel") == 0)
{
cpuid_info.isIntel = 1;
}
else if (strcmp(obj->infos[i].name ,"CPUModel") == 0)
{
strcpy(cpuid_info.osname, obj->infos[i].value);
}
}
cpuid_topology.numHWThreads = hwloc_get_nbobjs_by_type(hwloc_topology, HWLOC_OBJ_PU);
cpuid_info.stepping = get_stepping();
DEBUG_PRINT(DEBUGLEV_DEVELOP, HWLOC CpuInfo Family %d Model %d Stepping %d isIntel %d numHWThreads %d activeHWThreads %d,
cpuid_info.family,
cpuid_info.model,
cpuid_info.stepping,
cpuid_info.isIntel,
cpuid_topology.numHWThreads,
cpuid_topology.activeHWThreads)
return;
}
static inline const char *get_hn (hwloc_topology_t topo)
{
hwloc_obj_t obj;
const char *hn = NULL;
obj = hwloc_get_obj_by_type (topo, HWLOC_OBJ_MACHINE, 0);
if (obj)
hn = hwloc_obj_get_info_by_name (obj, "HostName");
return hn;
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t root;
const char *cluster_mode;
const char *memory_mode;
hwloc_topology_init(&topology);
hwloc_topology_load(topology);
root = hwloc_get_root_obj(topology);
cluster_mode = hwloc_obj_get_info_by_name(root, "ClusterMode");
memory_mode = hwloc_obj_get_info_by_name(root, "MemoryMode");
printf ("ClusterMode is '%s' MemoryMode is '%s'\n",
cluster_mode ? cluster_mode : "NULL",
memory_mode ? memory_mode : "NULL");
hwloc_topology_destroy(topology);
return 0;
}
static void assert_foo_bar(hwloc_topology_t topo, int setornot)
{
hwloc_obj_t root = hwloc_get_root_obj(topo);
const char *found = hwloc_obj_get_info_by_name(root, "Foo");
if (!setornot) {
assert(!found);
} else {
int diff;
assert(found);
diff = strcmp(found, "Bar");
assert(!diff);
}
}
static void
lstopo__prepare_custom_styles(struct lstopo_output *loutput, hwloc_obj_t obj)
{
struct lstopo_obj_userdata *lud = obj->userdata;
struct lstopo_style *s = &lud->style;
hwloc_obj_t child;
unsigned forcer, forceg, forceb;
const char *stylestr;
lud->style_set = 0;
stylestr = hwloc_obj_get_info_by_name(obj, "lstopoStyle");
if (stylestr) {
while (*stylestr != '\0') {
if (sscanf(stylestr, "%02x%02x%02x", &forcer, &forceg, &forceb) == 3
|| sscanf(stylestr, "Background=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->bg.r = forcer & 255;
s->bg.g = forceg & 255;
s->bg.b = forceb & 255;
lud->style_set |= LSTOPO_STYLE_BG;
loutput->methods->declare_color(loutput, &s->bg);
s->t.r = s->t.g = s->t.b = (s->bg.r + s->bg.g + s->bg.b < 0xff) ? 0xff : 0;
} else if (sscanf(stylestr, "Text=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->t.r = forcer & 255;
s->t.g = forceg & 255;
s->t.b = forceb & 255;
lud->style_set |= LSTOPO_STYLE_T;
loutput->methods->declare_color(loutput, &s->t);
} else if (sscanf(stylestr, "Text2=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->t2.r = forcer & 255;
s->t2.g = forceg & 255;
s->t2.b = forceb & 255;
lud->style_set |= LSTOPO_STYLE_T2;
loutput->methods->declare_color(loutput, &s->t2);
}
stylestr = strchr(stylestr, ';');
if (!stylestr)
break;
stylestr++;
}
}
for_each_child(child, obj)
lstopo__prepare_custom_styles(loutput, child);
for_each_memory_child(child, obj)
lstopo__prepare_custom_styles(loutput, child);
for_each_io_child(child, obj)
lstopo__prepare_custom_styles(loutput, child);
for_each_misc_child(child, obj)
lstopo__prepare_custom_styles(loutput, child);
}
/* next child, in all children list, ignoring PU if needed */
static hwloc_obj_t next_child(hwloc_topology_t topology, hwloc_obj_t parent, hwloc_obj_t prev)
{
hwloc_obj_t obj = prev;
again:
obj = hwloc_get_next_child(topology, parent, obj);
if (!obj)
return NULL;
if (obj->type == HWLOC_OBJ_PU && lstopo_ignore_pus)
goto again;
if (lstopo_collapse && obj->type == HWLOC_OBJ_PCI_DEVICE) {
const char *collapsestr = hwloc_obj_get_info_by_name(obj, "lstopoCollapse");
if (collapsestr && !strcmp(collapsestr, "0"))
goto again;
}
return obj;
}
/* Recursively output topology in a console fashion */
static void
output_topology (hwloc_topology_t topology, hwloc_obj_t l, hwloc_obj_t parent, FILE *output, int i, int logical, int verbose_mode)
{
hwloc_obj_t child;
int group_identical = (verbose_mode <= 1) && !lstopo_show_cpuset;
unsigned collapse = 1;
if (l->type == HWLOC_OBJ_PCI_DEVICE) {
const char *collapsestr = hwloc_obj_get_info_by_name(l, "lstopoCollapse");
if (collapsestr)
collapse = atoi(collapsestr);
if (!collapse)
return;
}
if (group_identical
&& parent && parent->arity == 1
&& l->cpuset && parent->cpuset && hwloc_bitmap_isequal(l->cpuset, parent->cpuset)) {
/* in non-verbose mode, merge objects with their parent is they are exactly identical */
fprintf(output, " + ");
} else {
if (parent)
fprintf(output, "\n");
indent (output, 2*i);
i++;
}
if (collapse > 1)
fprintf(output, "%u x { ", collapse);
output_console_obj(topology, l, output, logical, verbose_mode, collapse > 1);
if (collapse > 1)
fprintf(output, " }");
for(child = l->first_child; child; child = child->next_sibling)
if (child->type != HWLOC_OBJ_PU || !lstopo_ignore_pus)
output_topology (topology, child, l, output, i, logical, verbose_mode);
for(child = l->io_first_child; child; child = child->next_sibling)
output_topology (topology, child, l, output, i, logical, verbose_mode);
for(child = l->misc_first_child; child; child = child->next_sibling)
output_topology (topology, child, l, output, i, logical, verbose_mode);
}
/* count all children, ignoring PUs if needed */
static int count_children(struct lstopo_output *loutput, hwloc_obj_t obj)
{
unsigned total = obj->arity + obj->io_arity + obj->misc_arity;
if (loutput->ignore_pus) {
unsigned i;
for (i = 0; i < obj->arity; i++)
if (obj->children[i]->type == HWLOC_OBJ_PU)
total--;
}
if (loutput->collapse) {
hwloc_obj_t child;
for(child = obj->io_first_child; child; child = child->next_sibling) {
if (child->type == HWLOC_OBJ_PCI_DEVICE) {
const char *collapsestr = hwloc_obj_get_info_by_name(child, "lstopoCollapse");
if (collapsestr && !strcmp(collapsestr, "0"))
total--;
}
}
}
return total;
}
static int
lstopo_obj_snprintf(char *text, size_t textlen, hwloc_obj_t obj, int logical)
{
unsigned idx = logical ? obj->logical_index : obj->os_index;
const char *indexprefix = logical ? " L#" : " P#";
const char *value;
char typestr[32];
char indexstr[32]= "";
char attrstr[256];
char totmemstr[64] = "";
int attrlen;
/* For OSDev, Misc and Group, name replaces type+index+attrs */
if (obj->name && (obj->type == HWLOC_OBJ_OS_DEVICE || obj->type == HWLOC_OBJ_MISC || obj->type == HWLOC_OBJ_GROUP)) {
return snprintf(text, textlen, "%s", obj->name);
}
/* Type replaces the basic type name */
if ((value = hwloc_obj_get_info_by_name(obj, "Type")) != NULL) {
snprintf(typestr, sizeof(typestr), "%s", value);
} else {
hwloc_obj_type_snprintf(typestr, sizeof(typestr), obj, 0);
}
if (idx != (unsigned)-1 && obj->depth != 0
&& obj->type != HWLOC_OBJ_PCI_DEVICE
&& (obj->type != HWLOC_OBJ_BRIDGE || obj->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_HOST))
snprintf(indexstr, sizeof(indexstr), "%s%u", indexprefix, idx);
attrlen = hwloc_obj_attr_snprintf(attrstr, sizeof(attrstr), obj, " ", 0);
/* display the root total_memory if different from the local_memory (already shown) */
if (!obj->parent && obj->memory.total_memory > obj->memory.local_memory)
snprintf(totmemstr, sizeof(totmemstr), " (%lu%s total)",
(unsigned long) hwloc_memory_size_printf_value(obj->memory.total_memory, 0),
hwloc_memory_size_printf_unit(obj->memory.total_memory, 0));
if (attrlen > 0)
return snprintf(text, textlen, "%s%s (%s)%s", typestr, indexstr, attrstr, totmemstr);
else
return snprintf(text, textlen, "%s%s%s", typestr, indexstr, totmemstr);
}
int main(void)
{
hwloc_topology_t topology;
hwloc_obj_t obj;
unsigned n, i;
int devid, platformid;
const char *dev;
/* Allocate, initialize and load topology object. */
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
/* Find CUDA devices through the corresponding OS devices */
n = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
for (i = 0; i < n ; i++) {
const char *s;
obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_OS_DEVICE, i);
printf("%s:\n", obj->name);
s = hwloc_obj_get_info_by_name(obj, "Backend");
if (s && !strcmp(s, "CUDA")) {
/* This is a CUDA device */
assert(!strncmp(obj->name, "cuda", 4));
devid = atoi(obj->name + 4);
printf("CUDA device %d\n", devid);
s = hwloc_obj_get_info_by_name(obj, "GPUModel");
if (s)
printf("Model: %s\n", s);
s = hwloc_obj_get_info_by_name(obj, "CUDAGlobalMemorySize");
if (s)
printf("Memory: %s\n", s);
s = hwloc_obj_get_info_by_name(obj, "CUDAMultiProcessors");
if (s)
{
int mp = atoi(s);
s = hwloc_obj_get_info_by_name(obj, "CUDACoresPerMP");
if (s) {
int mp_cores = atoi(s);
printf("Cores: %d\n", mp * mp_cores);
}
}
}
if (s && !strcmp(s, "OpenCL")) {
/* This is an OpenCL device */
assert(!strncmp(obj->name, "opencl", 6));
platformid = atoi(obj->name + 6);
printf("OpenCL platform %d\n", platformid);
dev = strchr(obj->name + 6, 'd');
devid = atoi(dev + 1);
printf("OpenCL device %d\n", devid);
s = hwloc_obj_get_info_by_name(obj, "GPUModel");
if (s)
printf("Model: %s\n", s);
s = hwloc_obj_get_info_by_name(obj, "OpenCLGlobalMemorySize");
if (s)
printf("Memory: %s\n", s);
}
/* One can also use helpers from hwloc/cuda.h, hwloc/cudart.h,
* hwloc/opencl.h */
/* Find out cpuset this is connected to */
while (obj && (!obj->cpuset || hwloc_bitmap_iszero(obj->cpuset)))
obj = obj->parent;
if (obj) {
char *cpuset_string;
char name[16];
hwloc_obj_type_snprintf(name, sizeof(name), obj, 0);
hwloc_bitmap_asprintf(&cpuset_string, obj->cpuset);
printf("Location: %s P#%d\n", name, obj->os_index);
printf("Cpuset: %s\n", cpuset_string);
}
printf("\n");
}
/* Destroy topology object. */
hwloc_topology_destroy(topology);
return 0;
}
static void
os_device_draw(hwloc_topology_t topology __hwloc_attribute_unused, struct draw_methods *methods, int logical __hwloc_attribute_unused, hwloc_obj_t level, void *output, unsigned depth, unsigned x, unsigned *retwidth, unsigned y, unsigned *retheight)
{
unsigned textwidth = gridsize;
unsigned totheight = gridsize;
unsigned totwidth = gridsize;
struct style style;
char text[64];
int n;
unsigned nmorelines = 0, i;
char morelines[3][64];
if (fontsize) {
const char *coproctype;
if (HWLOC_OBJ_OSDEV_COPROC == level->attr->osdev.type
&& (coproctype = hwloc_obj_get_info_by_name(level, "CoProcType")) != NULL) {
if (!strcmp(coproctype, "CUDA")) {
const char *value, *value2, *value3;
value = hwloc_obj_get_info_by_name(level, "CUDAGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "CUDAL2CacheSize");
if (value) {
unsigned long long kb = strtoull(value, NULL, 10);
snprintf(morelines[nmorelines], sizeof(morelines[0]),
kb >= 10240 ? "L2 (%llu MB)" : "L2 (%llu kB)",
kb >= 10240 ? kb/1024 : kb);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "CUDAMultiProcessors");
value2 = hwloc_obj_get_info_by_name(level, "CUDACoresPerMP");
value3 = hwloc_obj_get_info_by_name(level, "CUDASharedMemorySizePerMP");
if (value && value2 && value3) {
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%s MP x (%s cores + %s kB)", value, value2, value3);
nmorelines++;
}
} else if (!strcmp(coproctype, "MIC")) {
const char *value;
value = hwloc_obj_get_info_by_name(level, "MICActiveCores");
if (value) {
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%s cores", value);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "MICMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
} else if (!strcmp(coproctype, "OpenCL")) {
const char *value;
value = hwloc_obj_get_info_by_name(level, "OpenCLComputeUnits");
if (value) {
unsigned long long cu = strtoull(value, NULL, 10);
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%llu compute units", cu);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "OpenCLGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
}
}
n = lstopo_obj_snprintf(text, sizeof(text), level, logical);
textwidth = get_textwidth(output, methods, text, n, fontsize, gridsize);
for(i=0; i<nmorelines; i++) {
int nn = strlen(morelines[i]);
int ntextwidth = get_textwidth(output, methods, morelines[i], nn, fontsize, gridsize);
if (ntextwidth > textwidth)
textwidth = ntextwidth;
}
totheight = gridsize + (fontsize + gridsize)*(nmorelines+1);
totwidth = gridsize + textwidth;
}
*retwidth = totwidth;
*retheight = totheight;
lstopo_set_object_color(methods, topology, level, 0, &style);
methods->box(output, style.bg.r, style.bg.g, style.bg.b, depth, x, *retwidth, y, *retheight);
if (fontsize) {
methods->text(output, style.t.r, style.t.g, style.t.b, fontsize, depth-1, x + gridsize, y + gridsize, text);
for(i=0; i<nmorelines; i++)
methods->text(output, style.t.r, style.t.g, style.t.b, fontsize, depth-1, x + gridsize, y + (i+2)*gridsize + (i+1)*fontsize, morelines[i]);
}
}
static void
prepare_more_text(struct lstopo_output *loutput, hwloc_obj_t obj)
{
struct lstopo_obj_userdata *lud = obj->userdata;
unsigned i;
if (!loutput->show_attrs[obj->type])
return;
if (HWLOC_OBJ_OS_DEVICE == obj->type) {
if (HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type && obj->subtype) {
/* Coprocessor */
if (!strcmp(obj->subtype, "CUDA")) {
/* CUDA */
const char *value, *value2, *value3;
value = hwloc_obj_get_info_by_name(obj, "CUDAGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
}
value = hwloc_obj_get_info_by_name(obj, "CUDAL2CacheSize");
if (value) {
unsigned long long kb = strtoull(value, NULL, 10);
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
kb >= 10240 ? "L2 (%llu MB)" : "L2 (%llu kB)",
kb >= 10240 ? kb/1024 : kb);
}
value = hwloc_obj_get_info_by_name(obj, "CUDAMultiProcessors");
value2 = hwloc_obj_get_info_by_name(obj, "CUDACoresPerMP");
value3 = hwloc_obj_get_info_by_name(obj, "CUDASharedMemorySizePerMP");
if (value && value2 && value3) {
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
"%s MP x (%s cores + %s kB)", value, value2, value3);
}
} else if (!strcmp(obj->subtype, "MIC")) {
/* MIC */
const char *value;
value = hwloc_obj_get_info_by_name(obj, "MICActiveCores");
if (value) {
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
"%s cores", value);
}
value = hwloc_obj_get_info_by_name(obj, "MICMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
}
} else if (!strcmp(obj->subtype, "OpenCL")) {
/* OpenCL */
const char *value;
value = hwloc_obj_get_info_by_name(obj, "OpenCLComputeUnits");
if (value) {
unsigned long long cu = strtoull(value, NULL, 10);
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
"%llu compute units", cu);
}
value = hwloc_obj_get_info_by_name(obj, "OpenCLGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
}
}
} else if (HWLOC_OBJ_OSDEV_BLOCK == obj->attr->osdev.type) {
/* Block */
const char *value;
value = hwloc_obj_get_info_by_name(obj, "Size");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(lud->text[lud->ntext++], sizeof(lud->text[0]),
mb >= 10485760 ? "%llu TB" : mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10485760 ? mb/1048576 : mb >= 10240 ? mb/1024 : mb);
}
}
}
for(i=1; i<lud->ntext; i++) {
unsigned nn = (unsigned)strlen(lud->text[i]);
unsigned ntextwidth = get_textwidth(loutput, lud->text[i], nn, loutput->fontsize);
if (ntextwidth > lud->textwidth)
lud->textwidth = ntextwidth;
}
}
void
output_draw(struct lstopo_output *loutput)
{
hwloc_topology_t topology = loutput->topology;
struct draw_methods *methods = loutput->methods;
int legend = loutput->legend;
unsigned gridsize = loutput->gridsize;
unsigned fontsize = loutput->fontsize;
hwloc_obj_t root = hwloc_get_root_obj(topology);
struct lstopo_obj_userdata *rlud = root->userdata;
unsigned depth = 100;
unsigned totwidth, totheight, offset, i;
time_t t;
char text[3][128];
unsigned ntext = 0;
char hostname[128] = "";
const char *forcedhostname = NULL;
unsigned long hostname_size = sizeof(hostname);
unsigned maxtextwidth = 0, textwidth;
if (legend && fontsize) {
forcedhostname = hwloc_obj_get_info_by_name(hwloc_get_root_obj(topology), "HostName");
if (!forcedhostname && hwloc_topology_is_thissystem(topology)) {
#if defined(HWLOC_WIN_SYS) && !defined(__CYGWIN__)
GetComputerName(hostname, &hostname_size);
#else
gethostname(hostname, hostname_size);
#endif
}
if (forcedhostname || *hostname) {
if (forcedhostname)
snprintf(text[ntext], sizeof(text[ntext]), "Host: %s", forcedhostname);
else
snprintf(text[ntext], sizeof(text[ntext]), "Host: %s", hostname);
textwidth = get_textwidth(loutput, text[ntext], (unsigned) strlen(text[ntext]), fontsize);
if (textwidth > maxtextwidth)
maxtextwidth = textwidth;
ntext++;
}
/* Display whether we're showing physical or logical IDs */
snprintf(text[ntext], sizeof(text[ntext]), "Indexes: %s", loutput->logical ? "logical" : "physical");
textwidth = get_textwidth(loutput, text[ntext], (unsigned) strlen(text[ntext]), fontsize);
if (textwidth > maxtextwidth)
maxtextwidth = textwidth;
ntext++;
/* Display timestamp */
t = time(NULL);
#ifdef HAVE_STRFTIME
{
struct tm *tmp;
tmp = localtime(&t);
strftime(text[ntext], sizeof(text[ntext]), "Date: %c", tmp);
}
#else /* HAVE_STRFTIME */
{
char *date;
int n;
date = ctime(&t);
n = strlen(date);
if (n && date[n-1] == '\n') {
date[n-1] = 0;
}
snprintf(text[ntext], sizeof(text[ntext]), "Date: %s", date);
}
#endif /* HAVE_STRFTIME */
textwidth = get_textwidth(loutput, text[ntext], (unsigned) strlen(text[ntext]), fontsize);
if (textwidth > maxtextwidth)
maxtextwidth = textwidth;
ntext++;
}
if (loutput->drawing == LSTOPO_DRAWING_PREPARE) {
/* compute root size, our size, and save it */
output_compute_pu_min_textwidth(loutput);
get_type_fun(root->type)(loutput, root, depth, 0, 0);
/* loutput width is max(root, legend) */
totwidth = rlud->width;
if (maxtextwidth + 2*gridsize > totwidth)
totwidth = maxtextwidth + 2*gridsize;
loutput->width = totwidth;
/* loutput height is sum(root, legend) */
totheight = rlud->height;
if (legend && fontsize)
totheight += gridsize + (ntext+loutput->legend_append_nr) * (gridsize+fontsize);
loutput->height = totheight;
} else { /* LSTOPO_DRAWING_DRAW */
/* restore our size that was computed during prepare */
totwidth = rlud->width;
totheight = rlud->height;
/* Draw root for real */
get_type_fun(root->type)(loutput, root, depth, 0, 0);
/* Draw legend */
if (legend && fontsize) {
offset = rlud->height + gridsize;
methods->box(loutput, &WHITE_COLOR, depth, 0, loutput->width, totheight, gridsize + (ntext+loutput->legend_append_nr) * (gridsize+fontsize));
for(i=0; i<ntext; i++, offset += gridsize + fontsize)
methods->text(loutput, &BLACK_COLOR, fontsize, depth, gridsize, offset, text[i]);
for(i=0; i<loutput->legend_append_nr; i++, offset += gridsize + fontsize)
methods->text(loutput, &BLACK_COLOR, fontsize, depth, gridsize, offset, loutput->legend_append[i]);
}
}
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
unsigned p, d;
osdev = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
/* we currently insert all OpenCL devices, except CPU devices */
if (!osdev) {
cl_device_type type;
clGetDeviceInfo(device_ids[j], CL_DEVICE_TYPE, sizeof(type), &type, NULL);
assert(type == CL_DEVICE_TYPE_CPU);
continue;
}
/* try to get it from PCI locality (only works with AMD extensions) */
osdev2 = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
if (osdev2) {
assert(osdev == osdev2);
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %u device %u\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %u device %u\n", cpuset_string, i, j);
free(cpuset_string);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = sscanf(osdev->name, "opencl%ud%u", &p, &d);
assert(err == 2);
assert(p == i);
assert(d == j);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = osdev->subtype;
assert(value);
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
static void
lstopo_set_object_color(struct draw_methods *methods,
hwloc_topology_t topology __hwloc_attribute_unused,
hwloc_obj_t obj, int arg, /* PU status (0=normal, 1=running, 2=forbidden)
* Machine status (0=normal, 1=displayed as a root/System) */
struct style *s)
{
unsigned forcer, forceg, forceb;
const char *style;
/* no need to deal with colors when computing max sizes */
if (methods == &getmax_draw_methods)
return;
memset(s, 0, sizeof(*s));
switch (obj->type) {
case HWLOC_OBJ_MACHINE:
if (arg == 0) {
s->bg.r = MACHINE_R_COLOR;
s->bg.g = MACHINE_G_COLOR;
s->bg.b = MACHINE_B_COLOR;
break;
}
assert(arg == 1); /* Machine printed as a System (when root) */
/* fallthrough */
case HWLOC_OBJ_SYSTEM:
s->bg.r = SYSTEM_R_COLOR;
s->bg.g = SYSTEM_G_COLOR;
s->bg.b = SYSTEM_B_COLOR;
break;
case HWLOC_OBJ_GROUP:
s->bg.r = MISC_R_COLOR;
s->bg.g = MISC_G_COLOR;
s->bg.b = MISC_B_COLOR;
break;
case HWLOC_OBJ_MISC:
s->bg.r = MISC_R_COLOR;
s->bg.g = MISC_G_COLOR;
s->bg.b = MISC_B_COLOR;
break;
case HWLOC_OBJ_NUMANODE:
s->bg.r = NODE_R_COLOR;
s->bg.g = NODE_G_COLOR;
s->bg.b = NODE_B_COLOR;
s->bg2.r = MEMORY_R_COLOR;
s->bg2.g = MEMORY_G_COLOR;
s->bg2.b = MEMORY_B_COLOR;
break;
case HWLOC_OBJ_PACKAGE:
s->bg.r = PACKAGE_R_COLOR;
s->bg.g = PACKAGE_G_COLOR;
s->bg.b = PACKAGE_B_COLOR;
break;
case HWLOC_OBJ_CORE:
s->bg.r = CORE_R_COLOR;
s->bg.g = CORE_G_COLOR;
s->bg.b = CORE_B_COLOR;
break;
case HWLOC_OBJ_L1CACHE:
case HWLOC_OBJ_L2CACHE:
case HWLOC_OBJ_L3CACHE:
case HWLOC_OBJ_L4CACHE:
case HWLOC_OBJ_L5CACHE:
case HWLOC_OBJ_L1ICACHE:
case HWLOC_OBJ_L2ICACHE:
case HWLOC_OBJ_L3ICACHE:
s->bg.r = CACHE_R_COLOR;
s->bg.g = CACHE_G_COLOR;
s->bg.b = CACHE_B_COLOR;
break;
case HWLOC_OBJ_PU:
switch (arg) {
case 0:
s->bg.r = THREAD_R_COLOR;
s->bg.g = THREAD_G_COLOR;
s->bg.b = THREAD_B_COLOR;
break;
case 1:
s->bg.r = RUNNING_R_COLOR;
s->bg.g = RUNNING_G_COLOR;
s->bg.b = RUNNING_B_COLOR;
break;
case 2:
s->bg.r = FORBIDDEN_R_COLOR;
s->bg.g = FORBIDDEN_G_COLOR;
s->bg.b = FORBIDDEN_B_COLOR;
break;
default:
assert(0);
}
break;
case HWLOC_OBJ_BRIDGE:
s->bg.r = BRIDGE_R_COLOR;
s->bg.g = BRIDGE_G_COLOR;
s->bg.b = BRIDGE_B_COLOR;
break;
case HWLOC_OBJ_PCI_DEVICE:
s->bg.r = PCI_DEVICE_R_COLOR;
s->bg.g = PCI_DEVICE_G_COLOR;
s->bg.b = PCI_DEVICE_B_COLOR;
break;
case HWLOC_OBJ_OS_DEVICE:
s->bg.r = OS_DEVICE_R_COLOR;
s->bg.g = OS_DEVICE_G_COLOR;
s->bg.b = OS_DEVICE_B_COLOR;
break;
default:
assert(0);
}
style = hwloc_obj_get_info_by_name(obj, "lstopoStyle");
if (style)
while (*style != '\0') {
if (sscanf(style, "%02x%02x%02x", &forcer, &forceg, &forceb) == 3
|| sscanf(style, "Background=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->bg.r = forcer & 255;
s->bg.g = forceg & 255;
s->bg.b = forceb & 255;
s->t.r = s->t.g = s->t.b = (s->bg.r + s->bg.g + s->bg.b < 0xff) ? 0xff : 0;
} else if (sscanf(style, "Background2=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->bg2.r = forcer & 255;
s->bg2.g = forceg & 255;
s->bg2.b = forceb & 255;
s->t2.r = s->t2.g = s->t2.b = (s->bg2.r + s->bg2.g + s->bg2.b < 0xff) ? 0xff : 0;
} else if (sscanf(style, "Text=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->t.r = forcer & 255;
s->t.g = forceg & 255;
s->t.b = forceb & 255;
} else if (sscanf(style, "Text2=#%02x%02x%02x", &forcer, &forceg, &forceb) == 3) {
s->t2.r = forcer & 255;
s->t2.g = forceg & 255;
s->t2.b = forceb & 255;
}
style = strchr(style, ';');
if (!style)
break;
style++;
}
}
static void
os_device_draw(struct lstopo_output *loutput, struct draw_methods *methods, hwloc_obj_t level, unsigned depth, unsigned x, unsigned *retwidth, unsigned y, unsigned *retheight)
{
hwloc_topology_t topology = loutput->topology;
int logical = loutput->logical;
unsigned gridsize = loutput->gridsize;
unsigned fontsize = loutput->fontsize;
unsigned myheight = 0, totheight;
unsigned mywidth = 0, totwidth;
unsigned textwidth = gridsize;
struct style style;
char text[64];
int n;
unsigned nmorelines = 0, i;
char morelines[3][64];
DYNA_CHECK();
if (fontsize) {
const char *coproctype = level->subtype;
if (HWLOC_OBJ_OSDEV_COPROC == level->attr->osdev.type && coproctype) {
if (!strcmp(coproctype, "CUDA")) {
const char *value, *value2, *value3;
value = hwloc_obj_get_info_by_name(level, "CUDAGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "CUDAL2CacheSize");
if (value) {
unsigned long long kb = strtoull(value, NULL, 10);
snprintf(morelines[nmorelines], sizeof(morelines[0]),
kb >= 10240 ? "L2 (%llu MB)" : "L2 (%llu kB)",
kb >= 10240 ? kb/1024 : kb);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "CUDAMultiProcessors");
value2 = hwloc_obj_get_info_by_name(level, "CUDACoresPerMP");
value3 = hwloc_obj_get_info_by_name(level, "CUDASharedMemorySizePerMP");
if (value && value2 && value3) {
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%s MP x (%s cores + %s kB)", value, value2, value3);
nmorelines++;
}
} else if (!strcmp(coproctype, "MIC")) {
const char *value;
value = hwloc_obj_get_info_by_name(level, "MICActiveCores");
if (value) {
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%s cores", value);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "MICMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
} else if (!strcmp(coproctype, "OpenCL")) {
const char *value;
value = hwloc_obj_get_info_by_name(level, "OpenCLComputeUnits");
if (value) {
unsigned long long cu = strtoull(value, NULL, 10);
snprintf(morelines[nmorelines], sizeof(morelines[0]), "%llu compute units", cu);
nmorelines++;
}
value = hwloc_obj_get_info_by_name(level, "OpenCLGlobalMemorySize");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
}
} else if (HWLOC_OBJ_OSDEV_BLOCK == level->attr->osdev.type) {
const char *value;
value = hwloc_obj_get_info_by_name(level, "Size");
if (value) {
unsigned long long mb = strtoull(value, NULL, 10) / 1024;
snprintf(morelines[nmorelines], sizeof(morelines[0]),
mb >= 10485760 ? "%llu TB" : mb >= 10240 ? "%llu GB" : "%llu MB",
mb >= 10485760 ? mb/1048576 : mb >= 10240 ? mb/1024 : mb);
nmorelines++;
}
}
n = lstopo_obj_snprintf(text, sizeof(text), level, logical);
textwidth = get_textwidth(loutput, methods, text, n, fontsize, gridsize);
for(i=0; i<nmorelines; i++) {
unsigned nn = (unsigned)strlen(morelines[i]);
unsigned ntextwidth = get_textwidth(loutput, methods, morelines[i], nn, fontsize, gridsize);
if (ntextwidth > textwidth)
textwidth = ntextwidth;
}
myheight = (fontsize + gridsize)*(nmorelines+1);
mywidth = 0;
}
RECURSE_RECT(level, &null_draw_methods, gridsize, gridsize);
lstopo_set_object_color(methods, topology, level, 0, &style);
methods->box(loutput, style.bg.r, style.bg.g, style.bg.b, depth, x, *retwidth, y, *retheight);
if (fontsize) {
methods->text(loutput, style.t.r, style.t.g, style.t.b, fontsize, depth-1, x + gridsize, y + gridsize, text);
for(i=0; i<nmorelines; i++)
methods->text(loutput, style.t.r, style.t.g, style.t.b, fontsize, depth-1, x + gridsize, y + (i+2)*gridsize + (i+1)*fontsize, morelines[i]);
}
RECURSE_RECT(level, methods, gridsize, gridsize);
DYNA_SAVE();
}
int main(void)
{
hwloc_topology_t topology;
int i;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_PCI_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_set_type_filter(topology, HWLOC_OBJ_OS_DEVICE, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
for(i=0; ; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, ancestor;
const char *value;
osdev = hwloc_intel_mic_get_device_osdev_by_index(topology, i);
if (!osdev)
break;
assert(osdev);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "mic", 3);
assert(!err);
assert(atoi(osdev->name+3) == (int) i);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "MIC");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "MICFamily");
printf("found MICFamily %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICSKU");
printf("found MICSKU %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICActiveCores");
printf("found MICActiveCores %s\n", value);
value = hwloc_obj_get_info_by_name(osdev, "MICMemorySize");
printf("found MICMemorySize %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_intel_mic_get_device_cpuset(topology, i, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
static int test(hwloc_topology_t orig, const char *callname) {
unsigned long forced_addr;
unsigned long fileoffset;
size_t shmem_length;
int synthetic_with_distances = (hwloc_obj_get_info_by_name(hwloc_get_root_obj(orig), "SyntheticDescription") != NULL);
char tmpname[] = "/tmp/hwloc_test_shmem.XXXXXX";
char cmd[512];
struct stat st;
int fd, err;
int ret = EXIT_SKIP;
printf("opening temporary file\n");
fd = mkstemp(tmpname);
if (fd < 0) {
perror("mkstemp");
goto out;
}
printf("opened %s\n", tmpname);
printf("exporting XML\n");
err = hwloc_topology_export_xml(orig, tmpname, 0);
assert(!err);
err = stat(tmpname, &st);
assert(!err);
printf("exported %lu bytes\n", (unsigned long) st.st_size);
fileoffset = st.st_size+1; /* skip a couple bytes to make sure the XML is don" */
fileoffset = (fileoffset + hwloc_getpagesize() - 1) &~(hwloc_getpagesize() - 1);
printf("will mmap at file offset %lu\n", fileoffset);
err = hwloc_shmem_topology_get_length(orig, &shmem_length, 0);
assert(!err);
printf("need mmap length %lu\n", (unsigned long) shmem_length);
#if SIZEOF_VOID_P == 8
forced_addr = 0x300000000000UL;
#else
forced_addr = 0xb0000000UL;
#endif
printf("write to shmem at address %lx in file %s offset %lu\n", forced_addr, tmpname, fileoffset);
err = hwloc_shmem_topology_write(orig, fd, fileoffset, (void*)(uintptr_t)forced_addr, shmem_length, 0);
if (err == -1 && errno == EBUSY) {
fprintf(stderr, "Failed to shmem write, requested mapping is busy\n");
goto out_with_fd;
}
assert(!err);
printf("wrote length %lu\n", (unsigned long) shmem_length);
printf("adopting locally\n");
ret = adopt(fd, fileoffset, forced_addr, shmem_length, synthetic_with_distances);
assert(ret == EXIT_SUCCESS || ret == EXIT_SKIP);
printf("adopting in other child process\n");
snprintf(cmd, sizeof(cmd), "%s %s %lu %lu %lu %d", callname, tmpname, fileoffset, forced_addr, (unsigned long) shmem_length, synthetic_with_distances);
printf("running command %s\n", cmd);
err = system(cmd);
assert(WIFEXITED(err));
printf("child process returned %d\n", WEXITSTATUS(err));
assert(WEXITSTATUS(err) == EXIT_SUCCESS || WEXITSTATUS(err) == EXIT_SKIP);
/* we caught errors above.
* return SKIP if both returned SKIP. otherwise SUCCESS
*/
if (WEXITSTATUS(err) == EXIT_SKIP && ret == EXIT_SKIP)
ret = EXIT_SKIP;
else
ret = EXIT_SUCCESS;
out_with_fd:
close(fd);
unlink(tmpname);
out:
return ret;
}
static const char *get_backend_name(hwloc_topology_t topo)
{
hwloc_obj_t root = hwloc_get_root_obj(topo);
return hwloc_obj_get_info_by_name(root, "Backend");
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
osdev = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
osdev2 = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
assert(osdev == osdev2);
if (!osdev) {
printf("no osdev for platform %d device %d\n", i, j);
continue;
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %d device %d\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %d device %d\n", cpuset_string, i, j);
free(cpuset_string);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "opencl", 6);
assert(!err);
assert(atoi(osdev->name+6) == (int) count);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "Name");
printf("found OSDev name %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
inline bool isCacheExclusive(hwloc_obj_t obj)
{
const char* value = hwloc_obj_get_info_by_name(obj, "Inclusive");
return value == nullptr || value[0] != '1';
}
static void
output_console_obj (hwloc_topology_t topology, hwloc_obj_t l, FILE *output, int logical, int verbose_mode, int collapsed)
{
unsigned idx = logical ? l->logical_index : l->os_index;
const char *value;
char pidxstr[16] = "P#[collapsed]";
char lidxstr[16] = "L#[collapsed]";
if (!collapsed || l->type != HWLOC_OBJ_PCI_DEVICE) {
snprintf(pidxstr, sizeof(pidxstr), "P#%u", l->os_index);
snprintf(lidxstr, sizeof(lidxstr), "L#%u", l->logical_index);
}
if (lstopo_show_cpuset < 2) {
char type[64], *attr, phys[32] = "";
int len;
value = hwloc_obj_get_info_by_name(l, "Type");
hwloc_obj_type_snprintf (type, sizeof(type), l, verbose_mode-1);
if (value)
fprintf(output, "%s(%s)", type, value);
else
fprintf(output, "%s", type);
if (l->depth != 0 && idx != (unsigned)-1
&& l->type != HWLOC_OBJ_MISC
&& l->type != HWLOC_OBJ_PCI_DEVICE
&& (l->type != HWLOC_OBJ_BRIDGE || l->attr->bridge.upstream_type == HWLOC_OBJ_BRIDGE_HOST))
fprintf(output, " %s", logical ? lidxstr : pidxstr);
if (l->name && (l->type == HWLOC_OBJ_MISC || l->type == HWLOC_OBJ_GROUP))
fprintf(output, " %s", l->name);
if (logical && l->os_index != (unsigned) -1 &&
(verbose_mode >= 2 || l->type == HWLOC_OBJ_PU || l->type == HWLOC_OBJ_NUMANODE))
snprintf(phys, sizeof(phys), "%s", pidxstr);
/* display attributes */
len = hwloc_obj_attr_snprintf (NULL, 0, l, " ", verbose_mode-1);
attr = malloc(len+1);
*attr = '\0';
hwloc_obj_attr_snprintf (attr, len+1, l, " ", verbose_mode-1);
if (*phys || *attr) {
const char *separator = *phys != '\0' && *attr!= '\0' ? " " : "";
fprintf(output, " (%s%s%s)",
phys, separator, attr);
}
free(attr);
/* display the root total_memory if not verbose (already shown)
* and different from the local_memory (already shown) */
if (verbose_mode == 1 && !l->parent && l->memory.total_memory > l->memory.local_memory)
fprintf(output, " (%lu%s total)",
(unsigned long) hwloc_memory_size_printf_value(l->memory.total_memory, 0),
hwloc_memory_size_printf_unit(l->memory.total_memory, 0));
/* append the name */
if (l->name && (l->type == HWLOC_OBJ_OS_DEVICE || verbose_mode >= 2)
&& l->type != HWLOC_OBJ_MISC && l->type != HWLOC_OBJ_GROUP)
fprintf(output, " \"%s\"", l->name);
}
if (!l->cpuset)
return;
if (lstopo_show_cpuset == 1)
fprintf(output, " cpuset=");
if (lstopo_show_cpuset) {
char *cpusetstr;
if (lstopo_show_taskset)
hwloc_bitmap_taskset_asprintf(&cpusetstr, l->cpuset);
else
hwloc_bitmap_asprintf(&cpusetstr, l->cpuset);
fprintf(output, "%s", cpusetstr);
free(cpusetstr);
}
/* annotate if the PU is forbidden/running */
if (l->type == HWLOC_OBJ_PU && verbose_mode >= 2) {
if (lstopo_pu_forbidden(l))
fprintf(output, " (forbidden)");
else if (lstopo_pu_running(topology, l))
fprintf(output, " (running)");
}
}
int main(void)
{
hwloc_topology_t topology;
cl_int clret;
cl_platform_id *platform_ids;
unsigned nrp, nrd, count, i, j;
int err;
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
clret = clGetPlatformIDs(0, NULL, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
platform_ids = malloc(nrp * sizeof(*platform_ids));
if (!platform_ids)
return 0;
clret = clGetPlatformIDs(nrp, platform_ids, &nrp);
if (CL_SUCCESS != clret || !nrp)
return 0;
count = 0;
for(i=0; i<nrp; i++) {
cl_device_id *device_ids;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, 0, NULL, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
device_ids = malloc(nrd * sizeof(*device_ids));
if (!device_ids)
continue;
clret = clGetDeviceIDs(platform_ids[i], CL_DEVICE_TYPE_ALL, nrd, device_ids, &nrd);
if (CL_SUCCESS != clret || !nrd)
continue;
for(j=0; j<nrd; j++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
unsigned p, d;
osdev = hwloc_opencl_get_device_osdev(topology, device_ids[j]);
osdev2 = hwloc_opencl_get_device_osdev_by_index(topology, i, j);
assert(osdev == osdev2);
if (!osdev) {
printf("no osdev for platform %u device %u\n", i, j);
continue;
}
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
set = hwloc_bitmap_alloc();
err = hwloc_opencl_get_device_cpuset(topology, device_ids[j], set);
if (err < 0) {
printf("no cpuset for platform %u device %u\n", i, j);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for platform %u device %u\n", cpuset_string, i, j);
free(cpuset_string);
if (hwloc_bitmap_isequal(hwloc_topology_get_complete_cpuset(topology), hwloc_topology_get_topology_cpuset(topology)))
/* only compare if the topology is complete, otherwise things can be significantly different */
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
}
hwloc_bitmap_free(set);
printf("found OSDev %s\n", osdev->name);
err = sscanf(osdev->name, "opencl%ud%u", &p, &d);
assert(err == 2);
assert(p == i);
assert(d == j);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "OpenCL");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "OpenCL");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
count++;
}
}
hwloc_topology_destroy(topology);
return 0;
}
static int setup_fork(orte_job_t *jdata,
orte_app_context_t *app)
{
int i;
char *param;
bool oversubscribed;
orte_node_t *node;
char **envcpy, **nps, **firstranks;
char *npstring, *firstrankstring;
char *num_app_ctx;
bool takeus = false;
/* see if we are included */
for (i=0; NULL != jdata->personality[i]; i++) {
if (0 == strcmp(jdata->personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* see if the mapper thinks we are oversubscribed */
oversubscribed = false;
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_OVERSUBSCRIBED)) {
oversubscribed = true;
}
/* setup base environment: copy the current environ and merge
in the app context environ */
if (NULL != app->env) {
/* manually free original context->env to avoid a memory leak */
char **tmp = app->env;
envcpy = opal_environ_merge(orte_launch_environ, app->env);
if (NULL != tmp) {
opal_argv_free(tmp);
}
} else {
envcpy = opal_argv_copy(orte_launch_environ);
}
app->env = envcpy;
/* special case handling for --prefix: this is somewhat icky,
but at least some users do this. :-\ It is possible that
when using --prefix, the user will also "-x PATH" and/or
"-x LD_LIBRARY_PATH", which would therefore clobber the
work that was done in the prior pls to ensure that we have
the prefix at the beginning of the PATH and
LD_LIBRARY_PATH. So examine the context->env and see if we
find PATH or LD_LIBRARY_PATH. If found, that means the
prior work was clobbered, and we need to re-prefix those
variables. */
param = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)¶m, OPAL_STRING);
for (i = 0; NULL != param && NULL != app->env && NULL != app->env[i]; ++i) {
char *newenv;
/* Reset PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
asprintf(&newenv, "%s/bin:%s", param, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
}
/* Reset LD_LIBRARY_PATH */
else if (0 == strncmp("LD_LIBRARY_PATH=", app->env[i], 16)) {
asprintf(&newenv, "%s/lib:%s", param, app->env[i] + 16);
opal_setenv("LD_LIBRARY_PATH", newenv, true, &app->env);
free(newenv);
}
}
if (NULL != param) {
free(param);
}
/* pass my contact info to the local proc so we can talk */
opal_setenv("OMPI_MCA_orte_local_daemon_uri", orte_process_info.my_daemon_uri, true, &app->env);
/* pass the hnp's contact info to the local proc in case it
* needs it
*/
if (NULL != orte_process_info.my_hnp_uri) {
opal_setenv("OMPI_MCA_orte_hnp_uri", orte_process_info.my_hnp_uri, true, &app->env);
}
/* setup yield schedule - do not override any user-supplied directive! */
if (oversubscribed) {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "1", false, &app->env);
} else {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "0", false, &app->env);
}
/* set the app_context number into the environment */
asprintf(¶m, "%ld", (long)app->idx);
opal_setenv("OMPI_MCA_orte_app_num", param, true, &app->env);
free(param);
/* although the total_slots_alloc is the universe size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here. Also required by the ompi_attributes code!
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->total_slots_alloc);
opal_setenv("OMPI_UNIVERSE_SIZE", param, true, &app->env);
free(param);
/* pass the number of nodes involved in this job */
asprintf(¶m, "%ld", (long)(jdata->map->num_nodes));
opal_setenv("OMPI_MCA_orte_num_nodes", param, true, &app->env);
free(param);
/* pass a param telling the child what type and model of cpu we are on,
* if we know it. If hwloc has the value, use what it knows. Otherwise,
* see if we were explicitly given it and use that value.
*/
hwloc_obj_t obj;
char *htmp;
if (NULL != opal_hwloc_topology) {
obj = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUType")) ||
NULL != (htmp = orte_local_cpu_type)) {
opal_setenv("OMPI_MCA_orte_cpu_type", htmp, true, &app->env);
}
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUModel")) ||
NULL != (htmp = orte_local_cpu_model)) {
opal_setenv("OMPI_MCA_orte_cpu_model", htmp, true, &app->env);
}
} else {
if (NULL != orte_local_cpu_type) {
opal_setenv("OMPI_MCA_orte_cpu_type", orte_local_cpu_type, true, &app->env);
}
if (NULL != orte_local_cpu_model) {
opal_setenv("OMPI_MCA_orte_cpu_model", orte_local_cpu_model, true, &app->env);
}
}
/* get shmem's best component name so we can provide a hint to the shmem
* framework. the idea here is to have someone figure out what component to
* select (via the shmem framework) and then have the rest of the
* components in shmem obey that decision. for more details take a look at
* the shmem framework in opal.
*/
if (NULL != (param = opal_shmem_base_best_runnable_component_name())) {
opal_setenv("OMPI_MCA_shmem_RUNTIME_QUERY_hint", param, true, &app->env);
free(param);
}
/* Set an info MCA param that tells the launched processes that
* any binding policy was applied by us (e.g., so that
* MPI_INIT doesn't try to bind itself)
*/
opal_setenv("OMPI_MCA_orte_bound_at_launch", "1", true, &app->env);
/* tell the ESS to avoid the singleton component - but don't override
* anything that may have been provided elsewhere
*/
opal_setenv("OMPI_MCA_ess", "^singleton", false, &app->env);
/* ensure that the spawned process ignores direct launch components,
* but do not overrride anything we were given */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", false, &app->env);
/* since we want to pass the name as separate components, make sure
* that the "name" environmental variable is cleared!
*/
opal_unsetenv("OMPI_MCA_orte_ess_name", &app->env);
asprintf(¶m, "%ld", (long)jdata->num_procs);
opal_setenv("OMPI_MCA_orte_ess_num_procs", param, true, &app->env);
/* although the num_procs is the comm_world size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
opal_setenv("OMPI_COMM_WORLD_SIZE", param, true, &app->env);
free(param);
/* users would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->num_local_procs);
opal_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, &app->env);
free(param);
/* forcibly set the local tmpdir base to match ours */
opal_setenv("OMPI_MCA_orte_tmpdir_base", orte_process_info.tmpdir_base, true, &app->env);
/* MPI-3 requires we provide some further info to the procs,
* so we pass them as envars to avoid introducing further
* ORTE calls in the MPI layer
*/
asprintf(&num_app_ctx, "%lu", (unsigned long)jdata->num_apps);
/* build some common envars we need to pass for MPI-3 compatibility */
nps = NULL;
firstranks = NULL;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
opal_argv_append_nosize(&nps, ORTE_VPID_PRINT(app->num_procs));
opal_argv_append_nosize(&firstranks, ORTE_VPID_PRINT(app->first_rank));
}
npstring = opal_argv_join(nps, ' ');
firstrankstring = opal_argv_join(firstranks, ' ');
opal_argv_free(nps);
opal_argv_free(firstranks);
/* add the MPI-3 envars */
opal_setenv("OMPI_NUM_APP_CTX", num_app_ctx, true, &app->env);
opal_setenv("OMPI_FIRST_RANKS", firstrankstring, true, &app->env);
opal_setenv("OMPI_APP_CTX_NUM_PROCS", npstring, true, &app->env);
free(num_app_ctx);
free(firstrankstring);
free(npstring);
return ORTE_SUCCESS;
}
int main(void)
{
hwloc_topology_t topology;
CUresult cres;
CUdevice device;
int count, i;
int err;
cres = cuInit(0);
if (cres != CUDA_SUCCESS) {
printf("cuInit failed %d\n", cres);
return 0;
}
cres = cuDeviceGetCount(&count);
if (cres != CUDA_SUCCESS) {
printf("cuDeviceGetCount failed %d\n", cres);
return 0;
}
printf("cuDeviceGetCount found %d devices\n", count);
hwloc_topology_init(&topology);
hwloc_topology_set_flags(topology, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
hwloc_topology_load(topology);
for(i=0; i<count; i++) {
hwloc_bitmap_t set;
hwloc_obj_t osdev, osdev2, ancestor;
const char *value;
cres = cuDeviceGet(&device, i);
if (cres != CUDA_SUCCESS) {
printf("failed to get device %d\n", i);
continue;
}
osdev = hwloc_cuda_get_device_osdev(topology, device);
assert(osdev);
osdev2 = hwloc_cuda_get_device_osdev_by_index(topology, i);
assert(osdev == osdev2);
ancestor = hwloc_get_non_io_ancestor_obj(topology, osdev);
printf("found OSDev %s\n", osdev->name);
err = strncmp(osdev->name, "cuda", 4);
assert(!err);
assert(atoi(osdev->name+4) == (int) i);
value = hwloc_obj_get_info_by_name(osdev, "Backend");
err = strcmp(value, "CUDA");
assert(!err);
assert(osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC);
value = hwloc_obj_get_info_by_name(osdev, "CoProcType");
err = strcmp(value, "CUDA");
assert(!err);
value = hwloc_obj_get_info_by_name(osdev, "GPUModel");
printf("found OSDev model %s\n", value);
set = hwloc_bitmap_alloc();
err = hwloc_cuda_get_device_cpuset(topology, device, set);
if (err < 0) {
printf("failed to get cpuset for device %d\n", i);
} else {
char *cpuset_string = NULL;
hwloc_bitmap_asprintf(&cpuset_string, set);
printf("got cpuset %s for device %d\n", cpuset_string, i);
assert(hwloc_bitmap_isequal(set, ancestor->cpuset));
free(cpuset_string);
}
hwloc_bitmap_free(set);
}
hwloc_topology_destroy(topology);
return 0;
}
