static int test(hwloc_topology_t orig, const char *callname) {
unsigned long forced_addr;
unsigned long fileoffset;
size_t shmem_length;
int synthetic_with_distances = (hwloc_obj_get_info_by_name(hwloc_get_root_obj(orig), "SyntheticDescription") != NULL);
char tmpname[] = "/tmp/hwloc_test_shmem.XXXXXX";
char cmd[512];
struct stat st;
int fd, err;
int ret = EXIT_SKIP;
printf("opening temporary file\n");
fd = mkstemp(tmpname);
if (fd < 0) {
perror("mkstemp");
goto out;
}
printf("opened %s\n", tmpname);
printf("exporting XML\n");
err = hwloc_topology_export_xml(orig, tmpname, 0);
assert(!err);
err = stat(tmpname, &st);
assert(!err);
printf("exported %lu bytes\n", (unsigned long) st.st_size);
fileoffset = st.st_size+1; /* skip a couple bytes to make sure the XML is don" */
fileoffset = (fileoffset + hwloc_getpagesize() - 1) &~(hwloc_getpagesize() - 1);
printf("will mmap at file offset %lu\n", fileoffset);
err = hwloc_shmem_topology_get_length(orig, &shmem_length, 0);
assert(!err);
printf("need mmap length %lu\n", (unsigned long) shmem_length);
#if SIZEOF_VOID_P == 8
forced_addr = 0x300000000000UL;
#else
forced_addr = 0xb0000000UL;
#endif
printf("write to shmem at address %lx in file %s offset %lu\n", forced_addr, tmpname, fileoffset);
err = hwloc_shmem_topology_write(orig, fd, fileoffset, (void*)(uintptr_t)forced_addr, shmem_length, 0);
if (err == -1 && errno == EBUSY) {
fprintf(stderr, "Failed to shmem write, requested mapping is busy\n");
goto out_with_fd;
}
assert(!err);
printf("wrote length %lu\n", (unsigned long) shmem_length);
printf("adopting locally\n");
ret = adopt(fd, fileoffset, forced_addr, shmem_length, synthetic_with_distances);
assert(ret == EXIT_SUCCESS || ret == EXIT_SKIP);
printf("adopting in other child process\n");
snprintf(cmd, sizeof(cmd), "%s %s %lu %lu %lu %d", callname, tmpname, fileoffset, forced_addr, (unsigned long) shmem_length, synthetic_with_distances);
printf("running command %s\n", cmd);
err = system(cmd);
assert(WIFEXITED(err));
printf("child process returned %d\n", WEXITSTATUS(err));
assert(WEXITSTATUS(err) == EXIT_SUCCESS || WEXITSTATUS(err) == EXIT_SKIP);
/* we caught errors above.
* return SKIP if both returned SKIP. otherwise SUCCESS
*/
if (WEXITSTATUS(err) == EXIT_SKIP && ret == EXIT_SKIP)
ret = EXIT_SKIP;
else
ret = EXIT_SUCCESS;
out_with_fd:
close(fd);
unlink(tmpname);
out:
return ret;
}
static int init(void)
{
#if HWLOC_API_VERSION >= 0x20000
int rc;
bool space_available = false;
uint64_t amount_space_avail = 0;
/* ensure we have the topology */
if (OPAL_SUCCESS != (rc = opal_hwloc_base_get_topology())) {
return rc;
}
if (VM_HOLE_NONE == mca_rtc_hwloc_component.kind) {
return ORTE_SUCCESS;
}
/* get the size of the topology shared memory segment */
if (0 != hwloc_shmem_topology_get_length(opal_hwloc_topology, &shmemsize, 0)) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc topology shmem not available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_SUCCESS;
}
if (ORTE_SUCCESS != (rc = find_hole(mca_rtc_hwloc_component.kind,
&shmemaddr, shmemsize))) {
/* we couldn't find a hole, so don't use the shmem support */
if (4 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
FILE *file = fopen("/proc/self/maps", "r");
if (file) {
char line[256];
opal_output(0, "%s Dumping /proc/self/maps",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
while (fgets(line, sizeof(line), file) != NULL) {
char *end = strchr(line, '\n');
if (end) {
*end = '\0';
}
opal_output(0, "%s", line);
}
fclose(file);
}
}
return ORTE_SUCCESS;
}
/* create the shmem file in our session dir so it
* will automatically get cleaned up */
asprintf(&shmemfile, "%s/hwloc.sm", orte_process_info.jobfam_session_dir);
/* let's make sure we have enough space for the backing file */
if (OPAL_SUCCESS != (rc = enough_space(shmemfile, shmemsize,
&amount_space_avail,
&space_available))) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s an error occurred while determining "
"whether or not %s could be created for topo shmem.",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), shmemfile);
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
if (!space_available) {
if (1 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
orte_show_help("help-orte-rtc-hwloc.txt", "target full", true,
shmemfile, orte_process_info.nodename,
(unsigned long)shmemsize,
(unsigned long long)amount_space_avail);
}
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
/* enough space is available, so create the segment */
if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) {
int err = errno;
if (1 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
orte_show_help("help-orte-rtc-hwloc.txt", "sys call fail", true,
orte_process_info.nodename,
"open(2)", "", strerror(err), err);
}
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
/* ensure nobody inherits this fd */
opal_fd_set_cloexec(shmemfd);
/* populate the shmem segment with the topology */
if (0 != (rc = hwloc_shmem_topology_write(opal_hwloc_topology, shmemfd, 0,
(void*)shmemaddr, shmemsize, 0))) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s an error occurred while writing topology to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), shmemfile);
unlink(shmemfile);
free(shmemfile);
shmemfile = NULL;
close(shmemfd);
shmemfd = -1;
return ORTE_SUCCESS;
}
#endif
return ORTE_SUCCESS;
}
