int MPIR_Get_intercomm_contextid(MPID_Comm * comm_ptr, MPIU_Context_id_t * context_id,
MPIU_Context_id_t * recvcontext_id)
{
MPIU_Context_id_t mycontext_id, remote_context_id;
int mpi_errno = MPI_SUCCESS;
int tag = 31567; /* FIXME - we need an internal tag or
* communication channel. Can we use a different
* context instead?. Or can we use the tag
* provided in the intercomm routine? (not on a dup,
* but in that case it can use the collective context) */
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID);
if (!comm_ptr->local_comm) {
/* Manufacture the local communicator */
mpi_errno = MPIR_Setup_intercomm_localcomm(comm_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
mpi_errno = MPIR_Get_contextid_sparse(comm_ptr->local_comm, &mycontext_id, FALSE);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIU_Assert(mycontext_id != 0);
/* MPIC routine uses an internal context id. The local leads (process 0)
* exchange data */
remote_context_id = -1;
if (comm_ptr->rank == 0) {
mpi_errno = MPIC_Sendrecv(&mycontext_id, 1, MPIU_CONTEXT_ID_T_DATATYPE, 0, tag,
&remote_context_id, 1, MPIU_CONTEXT_ID_T_DATATYPE, 0, tag,
comm_ptr, MPI_STATUS_IGNORE, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
/* Make sure that all of the local processes now have this
* id */
mpi_errno = MPIR_Bcast_impl(&remote_context_id, 1, MPIU_CONTEXT_ID_T_DATATYPE,
0, comm_ptr->local_comm, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* The recvcontext_id must be the one that was allocated out of the local
* group, not the remote group. Otherwise we could end up posting two
* MPI_ANY_SOURCE,MPI_ANY_TAG recvs on the same context IDs even though we
* are attempting to post them for two separate communicators. */
*context_id = remote_context_id;
*recvcontext_id = mycontext_id;
fn_fail:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID);
return mpi_errno;
}
int MPIR_Get_intercomm_contextid_nonblock(MPID_Comm * comm_ptr, MPID_Comm * newcommp,
MPID_Request ** req)
{
int mpi_errno = MPI_SUCCESS;
int tag;
MPID_Sched_t s;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID_NONBLOCK);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID_NONBLOCK);
/* do as much local setup as possible */
if (!comm_ptr->local_comm) {
mpi_errno = MPIR_Setup_intercomm_localcomm(comm_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
/* now create a schedule */
mpi_errno = MPID_Sched_next_tag(comm_ptr, &tag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_Sched_create(&s);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* add some entries to it */
/* first get a context ID over the local comm */
mpi_errno =
sched_get_cid_nonblock(comm_ptr, newcommp, &newcommp->recvcontext_id, &newcommp->context_id,
s, MPID_INTERCOMM);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* finally, kick off the schedule and give the caller a request */
mpi_errno = MPID_Sched_start(&s, comm_ptr, tag, req);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
fn_fail:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_GET_INTERCOMM_CONTEXTID_NONBLOCK);
return mpi_errno;
}
int MPIR_Bcast_inter_MV2 (
void *buffer,
int count,
MPI_Datatype datatype,
int root,
MPID_Comm *comm_ptr,
int *errflag)
{
/* Intercommunicator broadcast.
Root sends to rank 0 in remote group. Remote group does local
intracommunicator broadcast.
*/
int rank, mpi_errno;
int mpi_errno_ret = MPI_SUCCESS;
MPI_Status status;
MPID_Comm *newcomm_ptr = NULL;
MPI_Comm comm;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_BCAST_INTER);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_BCAST_INTER);
comm = comm_ptr->handle;
if (root == MPI_PROC_NULL)
{
/* local processes other than root do nothing */
mpi_errno = MPI_SUCCESS;
}
else if (root == MPI_ROOT)
{
/* root sends to rank 0 on remote group and returns */
MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER( comm_ptr );
mpi_errno = MPIC_Send_ft(buffer, count, datatype, 0,
MPIR_BCAST_TAG, comm, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
MPIDU_ERR_CHECK_MULTIPLE_THREADS_EXIT( comm_ptr );
}
else
{
/* remote group. rank 0 on remote group receives from root */
rank = comm_ptr->rank;
if (rank == 0)
{
mpi_errno = MPIC_Recv_ft(buffer, count, datatype, root,
MPIR_BCAST_TAG, comm, &status, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
/* Get the local intracommunicator */
if (!comm_ptr->local_comm) {
MPIR_Setup_intercomm_localcomm( comm_ptr );
}
newcomm_ptr = comm_ptr->local_comm;
/* now do the usual broadcast on this intracommunicator
with rank 0 as root. */
mpi_errno = MPIR_Bcast_intra_MV2(buffer, count, datatype, 0, newcomm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_BCAST_INTER);
if (mpi_errno_ret)
mpi_errno = mpi_errno_ret;
else if (*errflag)
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**coll_fail");
return mpi_errno;
}
int MPIR_Allreduce_inter (
void *sendbuf,
void *recvbuf,
int count,
MPI_Datatype datatype,
MPI_Op op,
MPID_Comm *comm_ptr,
int *errflag )
{
/* Intercommunicator Allreduce.
We first do an intercommunicator reduce to rank 0 on left group,
then an intercommunicator reduce to rank 0 on right group, followed
by local intracommunicator broadcasts in each group.
We don't do local reduces first and then intercommunicator
broadcasts because it would require allocation of a temporary buffer.
*/
int rank, mpi_errno, root;
int mpi_errno_ret = MPI_SUCCESS;
MPID_Comm *newcomm_ptr = NULL;
rank = comm_ptr->rank;
/* first do a reduce from right group to rank 0 in left group,
then from left group to rank 0 in right group*/
if (comm_ptr->is_low_group) {
/* reduce from right group to rank 0*/
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, op,
root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
/* reduce to rank 0 of right group */
root = 0;
mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, op,
root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
else {
/* reduce to rank 0 of left group */
root = 0;
mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, op,
root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
/* reduce from right group to rank 0 */
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, op,
root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
/* Get the local intracommunicator */
if (!comm_ptr->local_comm)
MPIR_Setup_intercomm_localcomm( comm_ptr );
newcomm_ptr = comm_ptr->local_comm;
mpi_errno = MPIR_Bcast_impl(recvbuf, count, datatype, 0, newcomm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = TRUE;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
}
fn_exit:
if (mpi_errno_ret)
mpi_errno = mpi_errno_ret;
else if (*errflag)
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**coll_fail");
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Barrier_inter( MPID_Comm *comm_ptr, MPIR_Errflag_t *errflag )
{
int rank, mpi_errno = MPI_SUCCESS, root;
int mpi_errno_ret = MPI_SUCCESS;
int i = 0;
MPID_Comm *newcomm_ptr = NULL;
rank = comm_ptr->rank;
/* Get the local intracommunicator */
if (!comm_ptr->local_comm) {
mpi_errno = MPIR_Setup_intercomm_localcomm( comm_ptr );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
newcomm_ptr = comm_ptr->local_comm;
/* do a barrier on the local intracommunicator */
mpi_errno = MPIR_Barrier_intra(newcomm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = MPIR_ERR_GET_CLASS(mpi_errno);
MPIR_ERR_SET(mpi_errno, *errflag, "**fail");
MPIR_ERR_ADD(mpi_errno_ret, mpi_errno);
}
/* rank 0 on each group does an intercommunicator broadcast to the
remote group to indicate that all processes in the local group
have reached the barrier. We do a 1-byte bcast because a 0-byte
bcast will just return without doing anything. */
/* first broadcast from left to right group, then from right to
left group */
if (comm_ptr->is_low_group) {
/* bcast to right*/
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = MPIR_Bcast_inter(&i, 1, MPI_BYTE, root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = MPIR_ERR_GET_CLASS(mpi_errno);
MPIR_ERR_SET(mpi_errno, *errflag, "**fail");
MPIR_ERR_ADD(mpi_errno_ret, mpi_errno);
}
/* receive bcast from right */
root = 0;
mpi_errno = MPIR_Bcast_inter(&i, 1, MPI_BYTE, root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = MPIR_ERR_GET_CLASS(mpi_errno);
MPIR_ERR_SET(mpi_errno, *errflag, "**fail");
MPIR_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
else {
/* receive bcast from left */
root = 0;
mpi_errno = MPIR_Bcast_inter(&i, 1, MPI_BYTE, root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = MPIR_ERR_GET_CLASS(mpi_errno);
MPIR_ERR_SET(mpi_errno, *errflag, "**fail");
MPIR_ERR_ADD(mpi_errno_ret, mpi_errno);
}
/* bcast to left */
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = MPIR_Bcast_inter(&i, 1, MPI_BYTE, root, comm_ptr, errflag);
if (mpi_errno) {
/* for communication errors, just record the error but continue */
*errflag = MPIR_ERR_GET_CLASS(mpi_errno);
MPIR_ERR_SET(mpi_errno, *errflag, "**fail");
MPIR_ERR_ADD(mpi_errno_ret, mpi_errno);
}
}
fn_exit:
if (mpi_errno_ret)
mpi_errno = mpi_errno_ret;
else if (*errflag != MPIR_ERR_NONE)
MPIR_ERR_SET(mpi_errno, *errflag, "**coll_fail");
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Intercomm_merge_impl(MPID_Comm *comm_ptr, int high, MPID_Comm **new_intracomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
int local_high, remote_high, new_size;
MPIU_Context_id_t new_context_id;
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_INTERCOMM_MERGE_IMPL);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_INTERCOMM_MERGE_IMPL);
/* Make sure that we have a local intercommunicator */
if (!comm_ptr->local_comm) {
/* Manufacture the local communicator */
mpi_errno = MPIR_Setup_intercomm_localcomm( comm_ptr );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
/* Find the "high" value of the other group of processes. This
will be used to determine which group is ordered first in
the generated communicator. high is logical */
local_high = high;
if (comm_ptr->rank == 0) {
/* This routine allows use to use the collective communication
context rather than the point-to-point context. */
mpi_errno = MPIC_Sendrecv( &local_high, 1, MPI_INT, 0, 0,
&remote_high, 1, MPI_INT, 0, 0, comm_ptr,
MPI_STATUS_IGNORE, &errflag );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* If local_high and remote_high are the same, then order is arbitrary.
we use the gpids of the rank 0 member of the local and remote
groups to choose an order in this case. */
if (local_high == remote_high) {
MPID_Gpid ingpid, outgpid;
mpi_errno = MPID_GPID_Get( comm_ptr, 0, &ingpid );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIC_Sendrecv( &ingpid, sizeof(MPID_Gpid), MPI_BYTE, 0, 1,
&outgpid, sizeof(MPID_Gpid), MPI_BYTE, 0, 1, comm_ptr,
MPI_STATUS_IGNORE, &errflag );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* Note that the gpids cannot be the same because we are
starting from a valid intercomm */
int rc = memcmp(&ingpid,&outgpid,sizeof(MPID_Gpid));
if(rc < 0)
local_high = 1;
else if(rc > 0)
local_high = 0;
else
{
/* req#3930: The merge algorithm will deadlock if the gpids are inadvertently the
same due to implementation bugs in the MPID_GPID_Get() function */
MPIU_Assert(rc != 0);
}
}
}
/*
All processes in the local group now need to get the
value of local_high, which may have changed if both groups
of processes had the same value for high
*/
mpi_errno = MPIR_Bcast_impl( &local_high, 1, MPI_INT, 0, comm_ptr->local_comm, &errflag );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
mpi_errno = MPIR_Comm_create( new_intracomm_ptr );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
new_size = comm_ptr->local_size + comm_ptr->remote_size;
/* FIXME: For the intracomm, we need a consistent context id.
That means that one of the two groups needs to use
the recvcontext_id and the other must use the context_id */
if (local_high) {
(*new_intracomm_ptr)->context_id = comm_ptr->recvcontext_id + 2; /* See below */
}
else {
(*new_intracomm_ptr)->context_id = comm_ptr->context_id + 2; /* See below */
}
(*new_intracomm_ptr)->recvcontext_id = (*new_intracomm_ptr)->context_id;
(*new_intracomm_ptr)->remote_size = (*new_intracomm_ptr)->local_size = new_size;
(*new_intracomm_ptr)->rank = -1;
(*new_intracomm_ptr)->comm_kind = MPID_INTRACOMM;
/* Now we know which group comes first. Build the new mapping
from the existing comm */
mpi_errno = create_and_map(comm_ptr, local_high, (*new_intracomm_ptr));
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* We've setup a temporary context id, based on the context id
used by the intercomm. This allows us to perform the allreduce
operations within the context id algorithm, since we already
have a valid (almost - see comm_create_hook) communicator.
*/
mpi_errno = MPIR_Comm_commit((*new_intracomm_ptr));
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* printf( "About to get context id \n" ); fflush( stdout ); */
/* In the multi-threaded case, MPIR_Get_contextid_sparse assumes that the
calling routine already holds the single criticial section */
new_context_id = 0;
mpi_errno = MPIR_Get_contextid_sparse( (*new_intracomm_ptr), &new_context_id, FALSE );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIU_Assert(new_context_id != 0);
/* We release this communicator that was involved just to
* get valid context id and create true one
*/
mpi_errno = MPIR_Comm_release(*new_intracomm_ptr);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Comm_create( new_intracomm_ptr );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
(*new_intracomm_ptr)->remote_size = (*new_intracomm_ptr)->local_size = new_size;
(*new_intracomm_ptr)->rank = -1;
(*new_intracomm_ptr)->comm_kind = MPID_INTRACOMM;
(*new_intracomm_ptr)->context_id = new_context_id;
(*new_intracomm_ptr)->recvcontext_id = new_context_id;
mpi_errno = create_and_map(comm_ptr, local_high, (*new_intracomm_ptr));
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Comm_commit((*new_intracomm_ptr));
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_INTERCOMM_MERGE_IMPL);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPI_Intercomm_merge - Creates an intracommuncator from an intercommunicator
Input Parameters:
+ intercomm - Intercommunicator (handle)
- high - Used to order the groups within comm (logical)
when creating the new communicator. This is a boolean value; the group
that sets high true has its processes ordered `after` the group that sets
this value to false. If all processes in the intercommunicator provide
the same value, the choice of which group is ordered first is arbitrary.
Output Parameters:
. newintracomm - Created intracommunicator (handle)
Notes:
While all processes may provide the same value for the 'high' parameter,
this requires the MPI implementation to determine which group of
processes should be ranked first.
.N ThreadSafe
.N Fortran
Algorithm:
.Eb
.i Allocate contexts
.i Local and remote group leaders swap high values
.i Determine the high value.
.i Merge the two groups and make the intra-communicator
.Ee
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_EXHAUSTED
.seealso: MPI_Intercomm_create, MPI_Comm_free
@*/
int MPI_Intercomm_merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm)
{
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL;
MPID_Comm *new_intracomm_ptr;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_INTERCOMM_MERGE);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_INTERCOMM_MERGE);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(intercomm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr( intercomm, comm_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate comm_ptr */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno, FALSE );
/* If comm_ptr is not valid, it will be reset to null */
if (comm_ptr && comm_ptr->comm_kind != MPID_INTERCOMM) {
mpi_errno = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_COMM,
"**commnotinter", 0 );
}
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Make sure that we have a local intercommunicator */
if (!comm_ptr->local_comm) {
/* Manufacture the local communicator */
MPIR_Setup_intercomm_localcomm( comm_ptr );
}
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
int acthigh;
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
/* Check for consistent valus of high in each local group.
The Intel test suite checks for this; it is also an easy
error to make */
acthigh = high ? 1 : 0; /* Clamp high into 1 or 0 */
mpi_errno = MPIR_Allreduce_impl( MPI_IN_PLACE, &acthigh, 1, MPI_INT,
MPI_SUM, comm_ptr->local_comm, &errflag );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* acthigh must either == 0 or the size of the local comm */
if (acthigh != 0 && acthigh != comm_ptr->local_size) {
mpi_errno = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_ARG,
"**notsame",
"**notsame %s %s", "high",
"MPI_Intercomm_merge" );
goto fn_fail;
}
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPIR_Intercomm_merge_impl(comm_ptr, high, &new_intracomm_ptr);
if (mpi_errno) goto fn_fail;
MPIR_OBJ_PUBLISH_HANDLE(*newintracomm, new_intracomm_ptr->handle);
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_INTERCOMM_MERGE);
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_intercomm_merge",
"**mpi_intercomm_merge %C %d %p", intercomm, high, newintracomm);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIR_Ibarrier_inter(MPID_Comm *comm_ptr, MPID_Sched_t s)
{
int mpi_errno = MPI_SUCCESS;
int rank, root;
MPIR_SCHED_CHKPMEM_DECL(1);
char *buf = NULL;
MPIU_Assert(comm_ptr->comm_kind == MPID_INTERCOMM);
rank = comm_ptr->rank;
/* Get the local intracommunicator */
if (!comm_ptr->local_comm) {
mpi_errno = MPIR_Setup_intercomm_localcomm(comm_ptr);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
/* do a barrier on the local intracommunicator */
MPIU_Assert(comm_ptr->local_comm->coll_fns && comm_ptr->local_comm->coll_fns->Ibarrier_sched);
if(comm_ptr->local_size != 1) {
mpi_errno = comm_ptr->local_comm->coll_fns->Ibarrier_sched(comm_ptr->local_comm, s);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPID_SCHED_BARRIER(s);
}
/* rank 0 on each group does an intercommunicator broadcast to the
remote group to indicate that all processes in the local group
have reached the barrier. We do a 1-byte bcast because a 0-byte
bcast will just return without doing anything. */
MPIR_SCHED_CHKPMEM_MALLOC(buf, char *, 1, mpi_errno, "bcast buf");
buf[0] = 'D'; /* avoid valgrind warnings */
/* first broadcast from left to right group, then from right to
left group */
MPIU_Assert(comm_ptr->coll_fns && comm_ptr->coll_fns->Ibcast_sched);
if (comm_ptr->is_low_group) {
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = comm_ptr->coll_fns->Ibcast_sched(buf, 1, MPI_BYTE, root, comm_ptr, s);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPID_SCHED_BARRIER(s);
/* receive bcast from right */
root = 0;
mpi_errno = comm_ptr->coll_fns->Ibcast_sched(buf, 1, MPI_BYTE, root, comm_ptr, s);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
else {
/* receive bcast from left */
root = 0;
mpi_errno = comm_ptr->coll_fns->Ibcast_sched(buf, 1, MPI_BYTE, root, comm_ptr, s);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPID_SCHED_BARRIER(s);
/* bcast to left */
root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL;
mpi_errno = comm_ptr->coll_fns->Ibcast_sched(buf, 1, MPI_BYTE, root, comm_ptr, s);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
MPIR_SCHED_CHKPMEM_COMMIT(s);
fn_exit:
return mpi_errno;
fn_fail:
MPIR_SCHED_CHKPMEM_REAP(s);
goto fn_exit;
}
/* comm create impl for intercommunicators, assumes that the standard error
* checking has already taken place in the calling function */
PMPI_LOCAL int MPIR_Comm_create_inter(MPID_Comm *comm_ptr, MPID_Group *group_ptr,
MPID_Comm **newcomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIU_Context_id_t new_context_id;
int *mapping = NULL;
int *remote_mapping = NULL;
MPID_Comm *mapping_comm = NULL;
int remote_size = -1;
int rinfo[2];
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
MPIU_CHKLMEM_DECL(1);
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_CREATE_INTER);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_CREATE_INTER);
MPIU_Assert(comm_ptr->comm_kind == MPID_INTERCOMM);
/* Create a new communicator from the specified group members */
/* If there is a context id cache in oldcomm, use it here. Otherwise,
use the appropriate algorithm to get a new context id.
Creating the context id is collective over the *input* communicator,
so it must be created before we decide if this process is a
member of the group */
/* In the multi-threaded case, MPIR_Get_contextid_sparse assumes that the
calling routine already holds the single criticial section */
if (!comm_ptr->local_comm) {
MPIR_Setup_intercomm_localcomm( comm_ptr );
}
mpi_errno = MPIR_Get_contextid_sparse( comm_ptr->local_comm, &new_context_id, FALSE );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIU_Assert(new_context_id != 0);
MPIU_Assert(new_context_id != comm_ptr->recvcontext_id);
mpi_errno = MPIR_Comm_create_calculate_mapping(group_ptr, comm_ptr,
&mapping, &mapping_comm);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
*newcomm_ptr = NULL;
if (group_ptr->rank != MPI_UNDEFINED) {
/* Get the new communicator structure and context id */
mpi_errno = MPIR_Comm_create( newcomm_ptr );
if (mpi_errno) goto fn_fail;
(*newcomm_ptr)->recvcontext_id = new_context_id;
(*newcomm_ptr)->rank = group_ptr->rank;
(*newcomm_ptr)->comm_kind = comm_ptr->comm_kind;
/* Since the group has been provided, let the new communicator know
about the group */
(*newcomm_ptr)->local_comm = 0;
(*newcomm_ptr)->local_group = group_ptr;
MPIR_Group_add_ref( group_ptr );
(*newcomm_ptr)->local_size = group_ptr->size;
(*newcomm_ptr)->remote_group = 0;
(*newcomm_ptr)->is_low_group = comm_ptr->is_low_group;
}
/* There is an additional step. We must communicate the
information on the local context id and the group members,
given by the ranks so that the remote process can construct the
appropriate network address mapping.
First we exchange group sizes and context ids. Then the ranks
in the remote group, from which the remote network address
mapping can be constructed. We need to use the "collective"
context in the original intercommunicator */
if (comm_ptr->rank == 0) {
int info[2];
info[0] = new_context_id;
info[1] = group_ptr->size;
mpi_errno = MPIC_Sendrecv(info, 2, MPI_INT, 0, 0,
rinfo, 2, MPI_INT, 0, 0,
comm_ptr, MPI_STATUS_IGNORE, &errflag );
if (mpi_errno) { MPIR_ERR_POP( mpi_errno ); }
if (*newcomm_ptr != NULL) {
(*newcomm_ptr)->context_id = rinfo[0];
}
remote_size = rinfo[1];
MPIU_CHKLMEM_MALLOC(remote_mapping,int*,
remote_size*sizeof(int),
mpi_errno,"remote_mapping");
/* Populate and exchange the ranks */
mpi_errno = MPIC_Sendrecv( mapping, group_ptr->size, MPI_INT, 0, 0,
remote_mapping, remote_size, MPI_INT, 0, 0,
comm_ptr, MPI_STATUS_IGNORE, &errflag );
if (mpi_errno) { MPIR_ERR_POP( mpi_errno ); }
/* Broadcast to the other members of the local group */
mpi_errno = MPIR_Bcast_impl( rinfo, 2, MPI_INT, 0,
comm_ptr->local_comm, &errflag);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast_impl( remote_mapping, remote_size, MPI_INT, 0,
comm_ptr->local_comm, &errflag);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
}
int MPIR_Comm_split_impl(MPID_Comm *comm_ptr, int color, int key, MPID_Comm **newcomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Comm *local_comm_ptr;
splittype *table, *remotetable=0;
sorttype *keytable, *remotekeytable=0;
int rank, size, remote_size, i, new_size, new_remote_size,
first_entry = 0, first_remote_entry = 0, *last_ptr;
int in_newcomm; /* TRUE iff *newcomm should be populated */
MPIR_Context_id_t new_context_id, remote_context_id;
int errflag = FALSE;
MPIU_CHKLMEM_DECL(4);
rank = comm_ptr->rank;
size = comm_ptr->local_size;
remote_size = comm_ptr->remote_size;
/* Step 1: Find out what color and keys all of the processes have */
MPIU_CHKLMEM_MALLOC(table,splittype*,size*sizeof(splittype),mpi_errno,
"table");
table[rank].color = color;
table[rank].key = key;
/* Get the communicator to use in collectives on the local group of
processes */
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
if (!comm_ptr->local_comm) {
MPIR_Setup_intercomm_localcomm( comm_ptr );
}
local_comm_ptr = comm_ptr->local_comm;
}
else {
local_comm_ptr = comm_ptr;
}
/* Gather information on the local group of processes */
mpi_errno = MPIR_Allgather_impl( MPI_IN_PLACE, 2, MPI_INT, table, 2, MPI_INT, local_comm_ptr, &errflag );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* Step 2: How many processes have our same color? */
new_size = 0;
if (color != MPI_UNDEFINED) {
/* Also replace the color value with the index of the *next* value
in this set. The integer first_entry is the index of the
first element */
last_ptr = &first_entry;
for (i=0; i<size; i++) {
/* Replace color with the index in table of the next item
of the same color. We use this to efficiently populate
the keyval table */
if (table[i].color == color) {
new_size++;
*last_ptr = i;
last_ptr = &table[i].color;
}
}
}
/* We don't need to set the last value to -1 because we loop through
the list for only the known size of the group */
/* If we're an intercomm, we need to do the same thing for the remote
table, as we need to know the size of the remote group of the
same color before deciding to create the communicator */
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
splittype mypair;
/* For the remote group, the situation is more complicated.
We need to find the size of our "partner" group in the
remote comm. The easiest way (in terms of code) is for
every process to essentially repeat the operation for the
local group - perform an (intercommunicator) all gather
of the color and rank information for the remote group.
*/
MPIU_CHKLMEM_MALLOC(remotetable,splittype*,
remote_size*sizeof(splittype),mpi_errno,
"remotetable");
/* This is an intercommunicator allgather */
/* We must use a local splittype because we've already modified the
entries in table to indicate the location of the next rank of the
same color */
mypair.color = color;
mypair.key = key;
mpi_errno = MPIR_Allgather_impl( &mypair, 2, MPI_INT, remotetable, 2, MPI_INT,
comm_ptr, &errflag );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* Each process can now match its color with the entries in the table */
new_remote_size = 0;
last_ptr = &first_remote_entry;
for (i=0; i<remote_size; i++) {
/* Replace color with the index in table of the next item
of the same color. We use this to efficiently populate
the keyval table */
if (remotetable[i].color == color) {
new_remote_size++;
*last_ptr = i;
last_ptr = &remotetable[i].color;
}
}
/* Note that it might find that there a now processes in the remote
group with the same color. In that case, COMM_SPLIT will
return a null communicator */
}
