static int applicable(const S *ego, const problem *p_,
const planner *plnr, int *r)
{
const problem_mpi_transpose *p = (const problem_mpi_transpose *) p_;
int n_pes;
MPI_Comm_size(p->comm, &n_pes);
return (1
&& p->tblock * n_pes == p->ny
&& (!ego->preserve_input || (!NO_DESTROY_INPUTP(plnr)
&& p->I != p->O))
&& (*r = ego->radix(n_pes)) && *r < n_pes && *r > 1
&& enough_space(p->nx, p->ny, p->block, p->tblock, *r, n_pes)
&& (!CONSERVE_MEMORYP(plnr) || *r > 8
|| !X(toobig)((p->nx * (p->ny / n_pes) * p->vn) / *r))
&& (!NO_SLOWP(plnr) ||
(p->nx * (p->ny / n_pes) * p->vn) / n_pes <= SMALL_MESSAGE)
&& ONLY_TRANSPOSEDP(p->flags)
);
}
static int init(void)
{
#if HWLOC_API_VERSION >= 0x20000
int rc;
bool space_available = false;
uint64_t amount_space_avail = 0;
/* ensure we have the topology */
if (OPAL_SUCCESS != (rc = opal_hwloc_base_get_topology())) {
return rc;
}
if (VM_HOLE_NONE == mca_rtc_hwloc_component.kind) {
return ORTE_SUCCESS;
}
/* get the size of the topology shared memory segment */
if (0 != hwloc_shmem_topology_get_length(opal_hwloc_topology, &shmemsize, 0)) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s hwloc topology shmem not available",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_SUCCESS;
}
if (ORTE_SUCCESS != (rc = find_hole(mca_rtc_hwloc_component.kind,
&shmemaddr, shmemsize))) {
/* we couldn't find a hole, so don't use the shmem support */
if (4 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
FILE *file = fopen("/proc/self/maps", "r");
if (file) {
char line[256];
opal_output(0, "%s Dumping /proc/self/maps",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
while (fgets(line, sizeof(line), file) != NULL) {
char *end = strchr(line, '\n');
if (end) {
*end = '\0';
}
opal_output(0, "%s", line);
}
fclose(file);
}
}
return ORTE_SUCCESS;
}
/* create the shmem file in our session dir so it
* will automatically get cleaned up */
asprintf(&shmemfile, "%s/hwloc.sm", orte_process_info.jobfam_session_dir);
/* let's make sure we have enough space for the backing file */
if (OPAL_SUCCESS != (rc = enough_space(shmemfile, shmemsize,
&amount_space_avail,
&space_available))) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s an error occurred while determining "
"whether or not %s could be created for topo shmem.",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), shmemfile);
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
if (!space_available) {
if (1 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
orte_show_help("help-orte-rtc-hwloc.txt", "target full", true,
shmemfile, orte_process_info.nodename,
(unsigned long)shmemsize,
(unsigned long long)amount_space_avail);
}
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
/* enough space is available, so create the segment */
if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) {
int err = errno;
if (1 < opal_output_get_verbosity(orte_rtc_base_framework.framework_output)) {
orte_show_help("help-orte-rtc-hwloc.txt", "sys call fail", true,
orte_process_info.nodename,
"open(2)", "", strerror(err), err);
}
free(shmemfile);
shmemfile = NULL;
return ORTE_SUCCESS;
}
/* ensure nobody inherits this fd */
opal_fd_set_cloexec(shmemfd);
/* populate the shmem segment with the topology */
if (0 != (rc = hwloc_shmem_topology_write(opal_hwloc_topology, shmemfd, 0,
(void*)shmemaddr, shmemsize, 0))) {
opal_output_verbose(2, orte_rtc_base_framework.framework_output,
"%s an error occurred while writing topology to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), shmemfile);
unlink(shmemfile);
free(shmemfile);
shmemfile = NULL;
close(shmemfd);
shmemfd = -1;
return ORTE_SUCCESS;
}
#endif
return ORTE_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
segment_create(opal_shmem_ds_t *ds_buf,
const char *file_name,
size_t size)
{
int rc = OPAL_SUCCESS;
char *real_file_name = NULL;
pid_t my_pid = getpid();
bool space_available = false;
uint64_t amount_space_avail = 0;
/* the real size of the shared memory segment. this includes enough space
* to store our segment header.
*/
size_t real_size = size + sizeof(opal_shmem_seg_hdr_t);
opal_shmem_seg_hdr_t *seg_hdrp = MAP_FAILED;
/* init the contents of opal_shmem_ds_t */
shmem_ds_reset(ds_buf);
/* change the path of shmem mmap's backing store? */
if (0 != opal_shmem_mmap_relocate_backing_file) {
int err;
if (path_usable(opal_shmem_mmap_backing_file_base_dir, &err)) {
if (NULL ==
(real_file_name =
get_uniq_file_name(opal_shmem_mmap_backing_file_base_dir,
file_name))) {
/* out of resources */
return OPAL_ERROR;
}
}
/* a relocated backing store was requested, but the path specified
* cannot be used :-(. if the flag is negative, then warn and continue
* with the default path. otherwise, fail.
*/
else if (opal_shmem_mmap_relocate_backing_file < 0) {
opal_output(0, "shmem: mmap: WARNING: could not relocate "
"backing store to \"%s\" (%s). Continuing with "
"default path.\n",
opal_shmem_mmap_backing_file_base_dir, strerror(err));
}
/* must be positive, so fail */
else {
opal_output(0, "shmem: mmap: WARNING: could not relocate "
"backing store to \"%s\" (%s). Cannot continue with "
"shmem mmap.\n", opal_shmem_mmap_backing_file_base_dir,
strerror(err));
return OPAL_ERROR;
}
}
/* are we using the default path? */
if (NULL == real_file_name) {
/* use the path specified by the caller of this function */
if (NULL == (real_file_name = strdup(file_name))) {
/* out of resources */
return OPAL_ERROR;
}
}
OPAL_OUTPUT_VERBOSE(
(70, opal_shmem_base_framework.framework_output,
"%s: %s: backing store base directory: %s\n",
mca_shmem_mmap_component.super.base_version.mca_type_name,
mca_shmem_mmap_component.super.base_version.mca_component_name,
real_file_name)
);
/* determine whether the specified filename is on a network file system.
* this is an important check because if the backing store is located on
* a network filesystem, the user may see a shared memory performance hit.
*/
if (opal_shmem_mmap_nfs_warning && opal_path_nfs(real_file_name)) {
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
opal_show_help("help-opal-shmem-mmap.txt", "mmap on nfs", 1, hn,
real_file_name);
}
/* let's make sure we have enough space for the backing file */
if (OPAL_SUCCESS != (rc = enough_space(real_file_name,
real_size,
&amount_space_avail,
&space_available))) {
opal_output(0, "shmem: mmap: an error occurred while determining "
"whether or not %s could be created.", real_file_name);
/* rc is set */
goto out;
}
if (!space_available) {
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
rc = OPAL_ERR_OUT_OF_RESOURCE;
opal_show_help("help-opal-shmem-mmap.txt", "target full", 1,
real_file_name, hn, (unsigned long)real_size,
(unsigned long long)amount_space_avail);
goto out;
}
/* enough space is available, so create the segment */
if (-1 == (ds_buf->seg_id = open(real_file_name, O_CREAT | O_RDWR, 0600))) {
int err = errno;
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
"open(2)", "", strerror(err), err);
rc = OPAL_ERROR;
goto out;
}
/* size backing file - note the use of real_size here */
if (0 != ftruncate(ds_buf->seg_id, real_size)) {
int err = errno;
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
"ftruncate(2)", "", strerror(err), err);
rc = OPAL_ERROR;
goto out;
}
if (MAP_FAILED == (seg_hdrp = (opal_shmem_seg_hdr_t *)
mmap(NULL, real_size,
PROT_READ | PROT_WRITE, MAP_SHARED,
ds_buf->seg_id, 0))) {
int err = errno;
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
"mmap(2)", "", strerror(err), err);
rc = OPAL_ERROR;
goto out;
}
/* all is well */
else {
/* -- initialize the shared memory segment -- */
opal_atomic_rmb();
/* init segment lock */
opal_atomic_init(&seg_hdrp->lock, OPAL_ATOMIC_UNLOCKED);
/* i was the creator of this segment, so note that fact */
seg_hdrp->cpid = my_pid;
opal_atomic_wmb();
/* -- initialize the contents of opal_shmem_ds_t -- */
ds_buf->seg_cpid = my_pid;
ds_buf->seg_size = real_size;
ds_buf->seg_base_addr = (unsigned char *)seg_hdrp;
(void)strncpy(ds_buf->seg_name, real_file_name, OPAL_PATH_MAX - 1);
/* set "valid" bit because setment creation was successful */
OPAL_SHMEM_DS_SET_VALID(ds_buf);
OPAL_OUTPUT_VERBOSE(
(70, opal_shmem_base_framework.framework_output,
"%s: %s: create successful "
"(id: %d, size: %lu, name: %s)\n",
mca_shmem_mmap_component.super.base_version.mca_type_name,
mca_shmem_mmap_component.super.base_version.mca_component_name,
ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
);
}
out:
/* in this component, the id is the file descriptor returned by open. this
* check is here to see if it is safe to call close on the file descriptor.
* that is, we are making sure that our call to open was successful and
* we are not not in an error path.
*/
if (-1 != ds_buf->seg_id) {
if (0 != close(ds_buf->seg_id)) {
int err = errno;
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
"close(2)", "", strerror(err), err);
rc = OPAL_ERROR;
}
}
/* an error occured, so invalidate the shmem object and munmap if needed */
if (OPAL_SUCCESS != rc) {
if (MAP_FAILED != seg_hdrp) {
munmap((void *)seg_hdrp, real_size);
}
shmem_ds_reset(ds_buf);
}
/* safe to free now because its contents have already been copied */
if (NULL != real_file_name) {
free(real_file_name);
}
return rc;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
segment_create(opal_shmem_ds_t *ds_buf,
const char *file_name,
size_t size)
{
int rc = OPAL_SUCCESS;
pid_t my_pid = getpid();
char *temp1 = NULL, *temp2 = NULL;
bool space_available = false;
uint64_t amount_space_avail = 0;
/* the real size of the shared memory segment. this includes enough space
* to store our segment header.
*/
size_t real_size = size + sizeof(opal_shmem_seg_hdr_t);
opal_shmem_seg_hdr_t *seg_hdrp = NULL;
HANDLE hMapObject = INVALID_HANDLE_VALUE;
LPVOID lpvMem = NULL;
/* init the contents of opal_shmem_ds_t */
shmem_ds_reset(ds_buf);
/* On Windows the shared file will be created by the OS directly on the
* system ressources. Therefore, no file get involved in the operation.
* However, a unique key should be used as name for the shared memory object
* in order to allow all processes to access the same unique shared memory
* region. The key will be obtained from the original file_name by replacing
* all path separator occurences by '/' (as '\' is not allowed on the object
* name).
*/
temp1 = strdup(file_name);
if (NULL == temp1) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
temp2 = temp1;
while (NULL != (temp2 = strchr(temp2, OPAL_PATH_SEP[0])) ) {
*temp2 = '/';
}
/* let's make sure we have enough space for the backing file */
if (OPAL_SUCCESS != (rc = enough_space(temp1,
real_size,
&amount_space_avail,
&space_available))) {
opal_output(0, "shmem: windows: an error occurred while determining "
"whether or not %s could be created.", temp1);
/* rc is set */
free(temp1);
goto out;
}
if (!space_available) {
char hn[MAXHOSTNAMELEN];
gethostname(hn, MAXHOSTNAMELEN - 1);
hn[MAXHOSTNAMELEN - 1] = '\0';
rc = OPAL_ERR_OUT_OF_RESOURCE;
opal_show_help("help-opal-shmem-windows.txt", "target full", 1,
temp1, hn, (unsigned long)real_size,
(unsigned long long)amount_space_avail);
free(temp1);
goto out;
}
/* enough space is available, so create the segment */
/* use paging file */
hMapObject = CreateFileMapping(INVALID_HANDLE_VALUE,
/* no security attributes */
NULL,
/* read/write access */
PAGE_READWRITE,
/* size: high 32-bits */
0,
/* size: low 32-bits */
(DWORD)real_size,
/* name of map object */
temp1);
if (NULL == hMapObject) {
rc = OPAL_ERROR;
goto out;
}
/* Get a pointer to the file-mapped shared memory. */
lpvMem = MapViewOfFile(hMapObject, /* object to map view of */
FILE_MAP_WRITE, /* read/write access */
0, /* high offset: map from */
0, /* low offset: beginning */
0); /* default: map entire file */
if (NULL == lpvMem) {
rc = OPAL_ERROR;
goto out;
}
seg_hdrp = (opal_shmem_seg_hdr_t *)lpvMem;
/* all is well */
/* -- initialize the shared memory segment -- */
opal_atomic_rmb();
/* init segment lock */
opal_atomic_init(&seg_hdrp->lock, OPAL_ATOMIC_UNLOCKED);
/* i was the creator of this segment, so note that fact */
seg_hdrp->cpid = my_pid;
opal_atomic_wmb();
/* -- initialize the contents of opal_shmem_ds_t -- */
ds_buf->seg_cpid = my_pid;
ds_buf->seg_size = real_size;
ds_buf->seg_base_addr = (unsigned char *)seg_hdrp;
/* update path change in ds_buf */
memcpy(ds_buf->seg_name, temp1, OPAL_PATH_MAX);
/* relase the temporary file name */
free(temp1);
/* set "valid" bit because setment creation was successful */
OPAL_SHMEM_DS_SET_VALID(ds_buf);
OPAL_OUTPUT_VERBOSE(
(70, opal_shmem_base_output,
"%s: %s: create successful "
"(id: %d, size: %"PRIsize_t", name: %s)\n",
mca_shmem_windows_component.super.base_version.mca_type_name,
mca_shmem_windows_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->seg_size, ds_buf->seg_name)
);
out:
/* an error occured, so invalidate the shmem object and munmap if needed */
if (OPAL_SUCCESS != rc) {
if (NULL != seg_hdrp) {
UnmapViewOfFile((LPVOID)seg_hdrp);
}
shmem_ds_reset(ds_buf);
}
return rc;
}
