/* WARNING: local_mapping and remote_mapping are stored in this
* function. The caller is responsible for their storage and will
* need to retain them till Comm_commit. */
int MPII_Comm_create_map(int local_n,
int remote_n,
int *local_mapping,
int *remote_mapping, MPIR_Comm * mapping_comm, MPIR_Comm * newcomm)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Comm_map_irregular(newcomm, mapping_comm, local_mapping,
local_n, MPIR_COMM_MAP_DIR__L2L, NULL);
if (mapping_comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) {
MPIR_Comm_map_irregular(newcomm, mapping_comm, remote_mapping,
remote_n, MPIR_COMM_MAP_DIR__R2R, NULL);
}
return mpi_errno;
}
int MPIR_Comm_create_map(int local_n,
int remote_n,
int *local_mapping,
int *remote_mapping,
MPID_Comm *mapping_comm,
MPID_Comm *newcomm)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Comm_map_irregular(newcomm, mapping_comm, local_mapping,
local_n, MPIR_COMM_MAP_DIR_L2L, NULL);
if (mapping_comm->comm_kind == MPID_INTERCOMM) {
MPIR_Comm_map_irregular(newcomm, mapping_comm, remote_mapping,
remote_n, MPIR_COMM_MAP_DIR_R2R, NULL);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Comm_copy(MPID_Comm * comm_ptr, int size, MPID_Comm ** outcomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIU_Context_id_t new_context_id, new_recvcontext_id;
MPID_Comm *newcomm_ptr = NULL;
MPIR_Comm_map_t *map;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_COPY);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_COPY);
/* Get a new context first. We need this to be collective over the
* input communicator */
/* If there is a context id cache in oldcomm, use it here. Otherwise,
* use the appropriate algorithm to get a new context id. Be careful
* of intercomms here */
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
mpi_errno = MPIR_Get_intercomm_contextid(comm_ptr, &new_context_id, &new_recvcontext_id);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
else {
mpi_errno = MPIR_Get_contextid_sparse(comm_ptr, &new_context_id, FALSE);
new_recvcontext_id = new_context_id;
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIU_Assert(new_context_id != 0);
}
/* This is the local size, not the remote size, in the case of
* an intercomm */
if (comm_ptr->rank >= size) {
*outcomm_ptr = 0;
/* always free the recvcontext ID, never the "send" ID */
MPIR_Free_contextid(new_recvcontext_id);
goto fn_exit;
}
/* We're left with the processes that will have a non-null communicator.
* Create the object, initialize the data, and return the result */
mpi_errno = MPIR_Comm_create(&newcomm_ptr);
if (mpi_errno)
goto fn_fail;
newcomm_ptr->context_id = new_context_id;
newcomm_ptr->recvcontext_id = new_recvcontext_id;
/* Save the kind of the communicator */
newcomm_ptr->comm_kind = comm_ptr->comm_kind;
newcomm_ptr->local_comm = 0;
/* There are two cases here - size is the same as the old communicator,
* or it is smaller. If the size is the same, we can just add a reference.
* Otherwise, we need to create a new network address mapping. Note that this is the
* test that matches the test on rank above. */
if (size == comm_ptr->local_size) {
/* Duplicate the network address mapping */
if (comm_ptr->comm_kind == MPID_INTRACOMM)
MPIR_Comm_map_dup(newcomm_ptr, comm_ptr, MPIR_COMM_MAP_DIR_L2L);
else
MPIR_Comm_map_dup(newcomm_ptr, comm_ptr, MPIR_COMM_MAP_DIR_R2R);
}
else {
int i;
if (comm_ptr->comm_kind == MPID_INTRACOMM)
MPIR_Comm_map_irregular(newcomm_ptr, comm_ptr, NULL, size, MPIR_COMM_MAP_DIR_L2L, &map);
else
MPIR_Comm_map_irregular(newcomm_ptr, comm_ptr, NULL, size, MPIR_COMM_MAP_DIR_R2R, &map);
for (i = 0; i < size; i++) {
/* For rank i in the new communicator, find the corresponding
* rank in the input communicator */
map->src_mapping[i] = i;
}
}
/* If it is an intercomm, duplicate the local network address references */
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
MPIR_Comm_map_dup(newcomm_ptr, comm_ptr, MPIR_COMM_MAP_DIR_L2L);
}
/* Set the sizes and ranks */
newcomm_ptr->rank = comm_ptr->rank;
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
newcomm_ptr->local_size = comm_ptr->local_size;
newcomm_ptr->remote_size = comm_ptr->remote_size;
newcomm_ptr->is_low_group = comm_ptr->is_low_group;
}
else {
newcomm_ptr->local_size = size;
newcomm_ptr->remote_size = size;
}
/* Inherit the error handler (if any) */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_COMM_MUTEX(comm_ptr));
newcomm_ptr->errhandler = comm_ptr->errhandler;
if (comm_ptr->errhandler) {
MPIR_Errhandler_add_ref(comm_ptr->errhandler);
}
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_COMM_MUTEX(comm_ptr));
/* FIXME do we want to copy coll_fns here? */
mpi_errno = MPIR_Comm_commit(newcomm_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Start with no attributes on this communicator */
newcomm_ptr->attributes = 0;
/* Copy over the info hints from the original communicator. */
mpi_errno = MPIR_Info_dup_impl(comm_ptr->info, &(newcomm_ptr->info));
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Comm_apply_hints(newcomm_ptr, newcomm_ptr->info);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
*outcomm_ptr = newcomm_ptr;
fn_fail:
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_COMM_COPY);
return mpi_errno;
}
int MPIR_Comm_commit(MPID_Comm * comm)
{
int mpi_errno = MPI_SUCCESS;
int num_local = -1, num_external = -1;
int local_rank = -1, external_rank = -1;
int *local_procs = NULL, *external_procs = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_COMMIT);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_COMMIT);
/* It's OK to relax these assertions, but we should do so very
* intentionally. For now this function is the only place that we create
* our hierarchy of communicators */
MPIU_Assert(comm->node_comm == NULL);
MPIU_Assert(comm->node_roots_comm == NULL);
mpi_errno = set_collops(comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_Comm_map_free(comm);
if (comm->comm_kind == MPID_INTRACOMM) {
mpi_errno = MPIU_Find_local_and_external(comm,
&num_local, &local_rank, &local_procs,
&num_external, &external_rank, &external_procs,
&comm->intranode_table, &comm->internode_table);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno) {
if (MPIR_Err_is_fatal(mpi_errno))
MPIR_ERR_POP(mpi_errno);
/* Non-fatal errors simply mean that this communicator will not have
* any node awareness. Node-aware collectives are an optimization. */
MPIU_DBG_MSG_P(COMM, VERBOSE, "MPIU_Find_local_and_external failed for comm_ptr=%p",
comm);
if (comm->intranode_table)
MPIU_Free(comm->intranode_table);
if (comm->internode_table)
MPIU_Free(comm->internode_table);
mpi_errno = MPI_SUCCESS;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
/* defensive checks */
MPIU_Assert(num_local > 0);
MPIU_Assert(num_local > 1 || external_rank >= 0);
MPIU_Assert(external_rank < 0 || external_procs != NULL);
/* if the node_roots_comm and comm would be the same size, then creating
* the second communicator is useless and wasteful. */
if (num_external == comm->remote_size) {
MPIU_Assert(num_local == 1);
goto fn_exit;
}
/* we don't need a local comm if this process is the only one on this node */
if (num_local > 1) {
mpi_errno = MPIR_Comm_create(&comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
comm->node_comm->context_id = comm->context_id + MPID_CONTEXT_INTRANODE_OFFSET;
comm->node_comm->recvcontext_id = comm->node_comm->context_id;
comm->node_comm->rank = local_rank;
comm->node_comm->comm_kind = MPID_INTRACOMM;
comm->node_comm->hierarchy_kind = MPID_HIERARCHY_NODE;
comm->node_comm->local_comm = NULL;
MPIU_DBG_MSG_D(CH3_OTHER, VERBOSE, "Create node_comm=%p\n", comm->node_comm);
comm->node_comm->local_size = num_local;
comm->node_comm->remote_size = num_local;
MPIR_Comm_map_irregular(comm->node_comm, comm, local_procs,
num_local, MPIR_COMM_MAP_DIR_L2L, NULL);
mpi_errno = set_collops(comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* don't call MPIR_Comm_commit here */
MPIR_Comm_map_free(comm->node_comm);
}
/* this process may not be a member of the node_roots_comm */
if (local_rank == 0) {
mpi_errno = MPIR_Comm_create(&comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
comm->node_roots_comm->context_id = comm->context_id + MPID_CONTEXT_INTERNODE_OFFSET;
comm->node_roots_comm->recvcontext_id = comm->node_roots_comm->context_id;
comm->node_roots_comm->rank = external_rank;
comm->node_roots_comm->comm_kind = MPID_INTRACOMM;
comm->node_roots_comm->hierarchy_kind = MPID_HIERARCHY_NODE_ROOTS;
comm->node_roots_comm->local_comm = NULL;
comm->node_roots_comm->local_size = num_external;
comm->node_roots_comm->remote_size = num_external;
MPIR_Comm_map_irregular(comm->node_roots_comm, comm,
external_procs, num_external, MPIR_COMM_MAP_DIR_L2L, NULL);
mpi_errno = set_collops(comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* don't call MPIR_Comm_commit here */
MPIR_Comm_map_free(comm->node_roots_comm);
}
comm->hierarchy_kind = MPID_HIERARCHY_PARENT;
}
fn_exit:
if (external_procs != NULL)
MPIU_Free(external_procs);
if (local_procs != NULL)
MPIU_Free(local_procs);
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_COMM_COMMIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
