mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t *resources)
{
mca_common_sm_mpool_module_t *mpool_module;
mca_allocator_base_component_t* allocator_component;
/* Make a new mpool module */
mpool_module = (mca_common_sm_mpool_module_t *) malloc (sizeof (*mpool_module));
mca_common_sm_mpool_module_init(mpool_module);
/* set sm_size */
mpool_module->sm_size = resources->size;
allocator_component = mca_allocator_component_lookup(resources->allocator);
/* if specified allocator cannot be loaded - look for an alternative */
if (NULL == allocator_component) {
if (opal_list_get_size(&opal_allocator_base_framework.framework_components) == 0) {
mca_base_component_list_item_t *item =
(mca_base_component_list_item_t *)
opal_list_get_first(&opal_allocator_base_framework.framework_components);
allocator_component =
(mca_allocator_base_component_t *)item->cli_component;
opal_output(
0, "mca_common_sm_mpool_init: "
"unable to locate allocator: %s - using %s\n",
resources->allocator,
allocator_component->allocator_version.mca_component_name);
} else {
opal_output(0, "mca_common_sm_mpool_init: "
"unable to locate allocator: %s\n",
resources->allocator);
free(mpool_module);
return NULL;
}
}
mpool_module->mem_node = resources->mem_node;
if (NULL == (mpool_module->sm_common_module =
mca_common_sm_module_attach(&resources->bs_meta_buf,
sizeof(mca_common_sm_module_t), 8))) {
opal_output(0, "mca_common_sm_mpool_init: "
"unable to create shared memory mapping (%s)",
resources->bs_meta_buf.seg_name);
free(mpool_module);
return NULL;
}
/* setup allocator */
mpool_module->sm_allocator =
allocator_component->allocator_init (true, mca_common_sm_seg_alloc,
NULL, mpool_module->sm_common_module);
if (NULL == mpool_module->sm_allocator) {
opal_output(0, "mca_common_sm_mpool_init: unable to initialize allocator");
free(mpool_module);
return NULL;
}
return &mpool_module->super;
}
static int
sm_segment_attach(mca_btl_sm_component_t *comp_ptr)
{
int rc = OMPI_SUCCESS;
int fd = -1;
ssize_t bread = 0;
opal_shmem_ds_t *tmp_shmem_ds = calloc(1, sizeof(*tmp_shmem_ds));
if (NULL == tmp_shmem_ds) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (-1 == (fd = open(comp_ptr->sm_rndv_file_name, O_RDONLY))) {
int err = errno;
opal_show_help("help-mpi-btl-sm.txt", "sys call fail", true,
"open(2)", strerror(err), err);
rc = OMPI_ERR_IN_ERRNO;
goto out;
}
if ((ssize_t)sizeof(opal_shmem_ds_t) != (bread =
read(fd, tmp_shmem_ds, sizeof(opal_shmem_ds_t)))) {
opal_output(0, "sm_segment_attach: "
"Read inconsistency -- read: %lu, but expected: %lu!\n",
(unsigned long)bread,
(unsigned long)sizeof(opal_shmem_ds_t));
rc = OMPI_ERROR;
goto out;
}
if (NULL == (comp_ptr->sm_seg =
mca_common_sm_module_attach(tmp_shmem_ds,
sizeof(mca_common_sm_seg_header_t),
opal_cache_line_size))) {
/* don't have to detach here, because module_attach cleans up after
* itself on failure. */
opal_output(0, "sm_segment_attach: "
"mca_common_sm_module_attach failure!\n");
rc = OMPI_ERROR;
}
out:
if (-1 != fd) {
(void)close(fd);
}
if (tmp_shmem_ds) {
free(tmp_shmem_ds);
}
return rc;
}
static int bootstrap_comm(ompi_communicator_t *comm,
mca_coll_sm_module_t *module)
{
int i;
char *shortpath, *fullpath;
mca_coll_sm_component_t *c = &mca_coll_sm_component;
mca_coll_sm_comm_t *data = module->sm_comm_data;
int comm_size = ompi_comm_size(comm);
int num_segments = c->sm_comm_num_segments;
int num_in_use = c->sm_comm_num_in_use_flags;
int frag_size = c->sm_fragment_size;
int control_size = c->sm_control_size;
ompi_process_name_t *lowest_name = NULL;
size_t size;
ompi_proc_t *proc;
/* Make the rendezvous filename for this communicators shmem data
segment. The CID is not guaranteed to be unique among all
procs on this node, so also pair it with the PID of the proc
with the lowest ORTE name to form a unique filename. */
proc = ompi_group_peer_lookup(comm->c_local_group, 0);
lowest_name = OMPI_CAST_RTE_NAME(&proc->super.proc_name);
for (i = 1; i < comm_size; ++i) {
proc = ompi_group_peer_lookup(comm->c_local_group, i);
if (ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
OMPI_CAST_RTE_NAME(&proc->super.proc_name),
lowest_name) < 0) {
lowest_name = OMPI_CAST_RTE_NAME(&proc->super.proc_name);
}
}
asprintf(&shortpath, "coll-sm-cid-%d-name-%s.mmap", comm->c_contextid,
OMPI_NAME_PRINT(lowest_name));
if (NULL == shortpath) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): asprintf failed",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
fullpath = opal_os_path(false, ompi_process_info.job_session_dir,
shortpath, NULL);
free(shortpath);
if (NULL == fullpath) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): opal_os_path failed",
comm->c_contextid, comm->c_name);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Calculate how much space we need in the per-communicator shmem
data segment. There are several values to add:
- size of the barrier data (2 of these):
- fan-in data (num_procs * control_size)
- fan-out data (num_procs * control_size)
- size of the "in use" buffers:
- num_in_use_buffers * control_size
- size of the message fragment area (one for each segment):
- control (num_procs * control_size)
- fragment data (num_procs * (frag_size))
So it's:
barrier: 2 * control_size + 2 * control_size
in use: num_in_use * control_size
control: num_segments * (num_procs * control_size * 2 +
num_procs * control_size)
message: num_segments * (num_procs * frag_size)
*/
size = 4 * control_size +
(num_in_use * control_size) +
(num_segments * (comm_size * control_size * 2)) +
(num_segments * (comm_size * frag_size));
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): attaching to %" PRIsize_t " byte mmap: %s",
comm->c_contextid, comm->c_name, size, fullpath);
if (0 == ompi_comm_rank (comm)) {
data->sm_bootstrap_meta = mca_common_sm_module_create_and_attach (size, fullpath, sizeof(mca_common_sm_seg_header_t), 8);
if (NULL == data->sm_bootstrap_meta) {
opal_output_verbose(10, ompi_coll_base_framework.framework_output,
"coll:sm:enable:bootstrap comm (%d/%s): mca_common_sm_init_group failed",
comm->c_contextid, comm->c_name);
free(fullpath);
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (int i = 1 ; i < ompi_comm_size (comm) ; ++i) {
MCA_PML_CALL(send(&data->sm_bootstrap_meta->shmem_ds, sizeof (data->sm_bootstrap_meta->shmem_ds), MPI_BYTE,
i, MCA_COLL_BASE_TAG_BCAST, MCA_PML_BASE_SEND_STANDARD, comm));
}
} else {
opal_shmem_ds_t shmem_ds;
MCA_PML_CALL(recv(&shmem_ds, sizeof (shmem_ds), MPI_BYTE, 0, MCA_COLL_BASE_TAG_BCAST, comm, MPI_STATUS_IGNORE));
data->sm_bootstrap_meta = mca_common_sm_module_attach (&shmem_ds, sizeof(mca_common_sm_seg_header_t), 8);
}
/* All done */
free(fullpath);
return OMPI_SUCCESS;
}
static mca_mpool_base_module_t *
mca_mpool_sm_init(struct mca_mpool_base_resources_t *resources)
{
mca_mpool_sm_module_t *mpool_module;
mca_allocator_base_component_t* allocator_component;
ompi_proc_t **procs;
size_t num_all_procs, i, num_local_procs = 0;
/* README: this needs to change if procs in different jobs (even
* spawned ones) are to talk using shared memory */
if (NULL == (procs = ompi_proc_world(&num_all_procs))) {
/* out of resources, so just bail */
return NULL;
}
for (i = 0 ; i < num_all_procs ; ++i) {
if (OPAL_PROC_ON_LOCAL_NODE(procs[i]->proc_flags)) {
num_local_procs++;
}
}
/* Make a new mpool module */
mpool_module =
(mca_mpool_sm_module_t *)malloc(sizeof(mca_mpool_sm_module_t));
mca_mpool_sm_module_init(mpool_module);
/* set sm_size */
mpool_module->sm_size = resources->size;
/* clip at the min size */
if (mpool_module->sm_size < (long) ompi_mpool_sm_min_size) {
mpool_module->sm_size = (long) ompi_mpool_sm_min_size;
}
allocator_component = mca_allocator_component_lookup(
mca_mpool_sm_component.sm_allocator_name);
/* if specified allocator cannot be loaded - look for an alternative */
if (NULL == allocator_component) {
if (opal_list_get_size(&ompi_allocator_base_framework.framework_components) == 0) {
mca_base_component_list_item_t *item =
(mca_base_component_list_item_t *)
opal_list_get_first(&ompi_allocator_base_framework.framework_components);
allocator_component =
(mca_allocator_base_component_t *)item->cli_component;
opal_output(
0, "mca_mpool_sm_init: "
"unable to locate allocator: %s - using %s\n",
mca_mpool_sm_component.sm_allocator_name,
allocator_component->allocator_version.mca_component_name);
} else {
opal_output(0, "mca_mpool_sm_init: "
"unable to locate allocator: %s\n",
mca_mpool_sm_component.sm_allocator_name);
free(procs);
return NULL;
}
}
mpool_module->mem_node = resources->mem_node;
opal_output(mca_mpool_sm_component.verbose,
"mca_mpool_sm_init: shared memory size used: (%ld)",
mpool_module->sm_size);
if (NULL == (mpool_module->sm_common_module =
mca_common_sm_module_attach(&resources->bs_meta_buf,
sizeof(mca_common_sm_module_t), 8))) {
opal_output(mca_mpool_sm_component.verbose, "mca_mpool_sm_init: "
"unable to create shared memory mapping (%s)",
resources->bs_meta_buf.seg_name);
free(mpool_module);
free(procs);
return NULL;
}
free(procs);
/* setup allocator */
mpool_module->sm_allocator =
allocator_component->allocator_init(true,
mca_common_sm_seg_alloc,
NULL, &(mpool_module->super));
if (NULL == mpool_module->sm_allocator) {
opal_output(0, "mca_mpool_sm_init: unable to initialize allocator");
free(mpool_module);
return NULL;
}
return &mpool_module->super;
}
