int MPIR_MAXLOC_check_dtype( MPI_Datatype type )
{
int mpi_errno = MPI_SUCCESS;
switch (type) {
/* first the C types */
case MPI_2INT:
case MPI_FLOAT_INT:
case MPI_LONG_INT:
case MPI_SHORT_INT:
case MPI_DOUBLE_INT:
#if defined(HAVE_LONG_DOUBLE)
case MPI_LONG_DOUBLE_INT:
#endif
/* now the Fortran types */
#ifdef HAVE_FORTRAN_BINDING
#ifndef HAVE_NO_FORTRAN_MPI_TYPES_IN_C
case MPI_2INTEGER:
case MPI_2REAL:
case MPI_2DOUBLE_PRECISION:
#endif
#endif
break;
default:
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OP, "**opundefined", "**opundefined %s", "MPI_MAXLOC");
}
return mpi_errno;
}
/* This dequeues req from the posted recv queue, set req's error code to comm_fail, and updates the req pointer.
Note that this creates a new error code if one hasn't already been created (i.e., if *error is MPI_SUCCESS). */
static inline void dequeue_and_set_error(MPID_Request **req, MPID_Request *prev_req, int *error, int rank)
{
MPID_Request *next = (*req)->dev.next;
if (*error == MPI_SUCCESS) {
if (rank == MPI_PROC_NULL)
MPIU_ERR_SET(*error, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
else
MPIU_ERR_SET1(*error, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail", "**comm_fail %d", rank);
}
/* remove from queue */
if (recvq_posted_head == *req)
recvq_posted_head = (*req)->dev.next;
else
prev_req->dev.next = (*req)->dev.next;
if (recvq_posted_tail == *req)
recvq_posted_tail = prev_req;
MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
/* set error and complete */
(*req)->status.MPI_ERROR = *error;
MPIDI_CH3U_Request_complete(*req);
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE,
(MPIU_DBG_FDEST, "set error of req %p (%#08x) to %#x and completing.",
*req, (*req)->handle, *error));
*req = next;
}
int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc);
MPIDI_STATE_DECL(MPID_STATE_MXM_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_TERMINATE);
if (vc->state != MPIDI_VC_STATE_CLOSED) {
/* VC is terminated as a result of a fault. Complete
* outstanding sends with an error and terminate connection
* immediately. */
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
}
else {
while (vc_area->pending_sends > 0)
MPID_nem_mxm_poll(FALSE);
}
mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MXM_VC_TERMINATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int vc_terminate(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
int req_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
MPIDI_STATE_DECL(MPID_STATE_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_VC_TERMINATE);
if (vc->state != MPIDI_VC_STATE_CLOSED) {
/* VC is terminated as a result of a fault. Complete
outstanding sends with an error and terminate
connection immediately. */
MPIU_ERR_SET1(req_errno, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
mpi_errno = MPID_nem_ptl_vc_terminated(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else if (vc_ptl->num_queued_sends == 0) {
mpi_errno = MPID_nem_ptl_vc_terminated(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
/* the send_queued function will call vc_terminated if vc->state is
CLOSED and the last queued send has been sent*/
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_VC_TERMINATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
void MPIR_MAXLOC(
void *invec,
void *inoutvec,
int *Len,
MPI_Datatype *type )
{
int mpi_errno = MPI_SUCCESS;
int i, len = *Len, flen;
flen = len * 2; /* used for Fortran types */
switch (*type) {
/* first the C types */
case MPI_2INT:
MPIR_MAXLOC_C_CASE(MPIR_2int_loctype);
case MPI_FLOAT_INT:
MPIR_MAXLOC_C_CASE(MPIR_floatint_loctype);
case MPI_LONG_INT:
MPIR_MAXLOC_C_CASE(MPIR_longint_loctype);
case MPI_SHORT_INT:
MPIR_MAXLOC_C_CASE(MPIR_shortint_loctype);
case MPI_DOUBLE_INT:
MPIR_MAXLOC_C_CASE(MPIR_doubleint_loctype);
#if defined(HAVE_LONG_DOUBLE)
case MPI_LONG_DOUBLE_INT:
MPIR_MAXLOC_C_CASE(MPIR_longdoubleint_loctype);
#endif
/* now the Fortran types */
#ifdef HAVE_FORTRAN_BINDING
#ifndef HAVE_NO_FORTRAN_MPI_TYPES_IN_C
case MPI_2INTEGER:
MPIR_MAXLOC_F_CASE(int);
case MPI_2REAL:
MPIR_MAXLOC_F_CASE(float);
case MPI_2DOUBLE_PRECISION:
MPIR_MAXLOC_F_CASE(double);
#endif
#endif
/* --BEGIN ERROR HANDLING-- */
default: {
MPIU_THREADPRIV_DECL;
MPIU_THREADPRIV_GET;
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OP, "**opundefined","**opundefined %s", "MPI_MAXLOC" );
MPIU_THREADPRIV_FIELD(op_errno) = mpi_errno;
break;
}
/* --END ERROR HANDLING-- */
}
}
int MPID_nem_scif_vc_terminate(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
int req_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE);
if (vc->state != MPIDI_VC_STATE_CLOSED) {
/* VC is terminated as a result of a fault. Complete
* outstanding sends with an error and terminate connection
* immediately. */
MPIU_ERR_SET1(req_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
mpi_errno = MPID_nem_scif_error_out_send_queue(vc, req_errno);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_scif_vc_terminated(vc);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
else {
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
vc_scif->terminate = 1;
/* VC is terminated as a result of the close protocol. Wait
* for sends to complete, then terminate. */
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
/* The sendq is empty, so we can immediately terminate the
* connection. */
mpi_errno = MPID_nem_scif_vc_terminated(vc);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
/* else: just return. We'll call vc_terminated() from the
* commrdy_handler once the sendq is empty. */
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* This dequeues req from the posted recv queue, set req's error code to comm_fail, and updates the req pointer.
Note that this creates a new error code if one hasn't already been created (i.e., if *error is MPI_SUCCESS). */
static inline void dequeue_and_set_error(MPID_Request **req, MPID_Request *prev_req, int *error, int rank)
{
MPID_Request *next = (*req)->dev.next;
if (*error == MPI_SUCCESS)
MPIU_ERR_SET1(*error, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", rank);
/* remove from queue */
if (recvq_posted_head == *req)
recvq_posted_head = (*req)->dev.next;
else
prev_req->dev.next = (*req)->dev.next;
if (recvq_posted_tail == *req)
recvq_posted_tail = prev_req;
/* set error and complete */
(*req)->status.MPI_ERROR = *error;
MPIDI_CH3U_Request_complete(*req);
*req = next;
}
/*@
MPI_Win_lock - Begin an RMA access epoch at the target process.
Input Parameters:
+ lock_type - Indicates whether other processes may access the target
window at the same time (if 'MPI_LOCK_SHARED') or not ('MPI_LOCK_EXCLUSIVE')
. rank - rank of locked window (nonnegative integer)
. assert - Used to optimize this call; zero may be used as a default.
See notes. (integer)
- win - window object (handle)
Notes:
The name of this routine is misleading. In particular, this
routine need not block, except when the target process is the calling
process.
Implementations may restrict the use of RMA communication that is
synchronized
by lock calls to windows in memory allocated by 'MPI_Alloc_mem'. Locks can
be used portably only in such memory.
The 'assert' argument is used to indicate special conditions for the
fence that an implementation may use to optimize the 'MPI_Win_lock'
operation. The value zero is always correct. Other assertion values
may be or''ed together. Assertions that are valid for 'MPI_Win_lock' are\:
. MPI_MODE_NOCHECK - no other process holds, or will attempt to acquire a
conflicting lock, while the caller holds the window lock. This is useful
when mutual exclusion is achieved by other means, but the coherence
operations that may be attached to the lock and unlock calls are still
required.
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_RANK
.N MPI_ERR_WIN
.N MPI_ERR_OTHER
@*/
int MPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win)
{
static const char FCNAME[] = "MPI_Win_lock";
int mpi_errno = MPI_SUCCESS;
MPID_Win *win_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_WIN_LOCK);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_WIN_LOCK);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_WIN(win, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Win_get_ptr( win, win_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPID_Comm *comm_ptr;
/* Validate win_ptr */
MPID_Win_valid_ptr( win_ptr, mpi_errno );
/* If win_ptr is not value, it will be reset to null */
if (mpi_errno) goto fn_fail;
if (assert != 0 && assert != MPI_MODE_NOCHECK) {
MPIU_ERR_SET1(mpi_errno,MPI_ERR_ARG,
"**lockassertval",
"**lockassertval %d", assert );
if (mpi_errno) goto fn_fail;
}
if (lock_type != MPI_LOCK_SHARED &&
lock_type != MPI_LOCK_EXCLUSIVE) {
MPIU_ERR_SET(mpi_errno,MPI_ERR_OTHER, "**locktype" );
}
if (win_ptr->lockRank != -1) {
MPIU_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**lockwhilelocked",
"**lockwhilelocked %d", win_ptr->lockRank );
}
comm_ptr = win_ptr->comm_ptr;
MPIR_ERRTEST_SEND_RANK(comm_ptr, rank, mpi_errno);
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPIU_RMA_CALL(win_ptr,
Win_lock(lock_type, rank, assert, win_ptr));
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* If the lock succeeded, remember which one with locked */
win_ptr->lockRank = rank;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_WIN_LOCK);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_win_lock",
"**mpi_win_lock %d %d %A %W", lock_type, rank, assert, win);
}
# endif
mpi_errno = MPIR_Err_return_win( win_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
MPID_Time_t timer_start;
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Store how much time is spent traversing the queue */
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (!MPIDI_CH3I_Comm_AS_enabled(comm)) {
MPIU_ERR_SET(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIR_T_INC(RECVQ_STATISTICS, posted_qlen);
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
/* If a match was not found, the timer was stopped after the traversal */
if (found)
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
int MPIDI_Isend_self(const void * buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset,
int type, MPID_Request ** request)
{
MPIDI_Message_match match;
MPID_Request * sreq;
MPID_Request * rreq;
MPIDI_VC_t * vc;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int found;
int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending message to self");
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, type);
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_SELF_MSG);
match.parts.rank = rank;
match.parts.tag = tag;
match.parts.context_id = comm->context_id + context_offset;
MPIU_THREAD_CS_ENTER(MSGQUEUE,);
rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&match, &found);
/* --BEGIN ERROR HANDLING-- */
if (rreq == NULL)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**nomem",
"**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
goto fn_exit;
}
/* --END ERROR HANDLING-- */
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIDI_Request_set_seqnum(rreq, seqnum);
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
if (found)
{
MPIDI_msg_sz_t data_sz;
/* we found a posted req, which we now own, so we can release the CS */
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,
"found posted receive request; copying data");
MPIDI_CH3U_Buffer_copy(buf, count, datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
rreq->status.count = (int)data_sz;
MPID_REQUEST_SET_COMPLETED(rreq);
MPID_Request_release(rreq);
/* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
MPIU_Object_set_ref(sreq, 1);
MPID_cc_set(&sreq->cc, 0);
}
else
{
if (type != MPIDI_REQUEST_TYPE_RSEND)
/*@
MPIDI_PG_Finalize - Finalize the process groups, including freeing all
process group structures
@*/
int MPIDI_PG_Finalize(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_PG_t *pg, *pgNext;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_FINALIZE);
/* Print the state of the process groups */
if (verbose) {
MPIU_PG_Printall( stdout );
}
/* FIXME - straighten out the use of PMI_Finalize - no use after
PG_Finalize */
if (pg_world->connData) {
#ifdef USE_PMI2_API
mpi_errno = PMI2_Finalize();
if (mpi_errno) MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|pmi_finalize");
#else
int rc;
rc = PMI_Finalize();
if (rc) {
MPIU_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**ch3|pmi_finalize",
"**ch3|pmi_finalize %d", rc);
}
#endif
}
/* Free the storage associated with the process groups */
pg = MPIDI_PG_list;
while (pg) {
pgNext = pg->next;
/* In finalize, we free all process group information, even if
the ref count is not zero. This can happen if the user
fails to use MPI_Comm_disconnect on communicators that
were created with the dynamic process routines.*/
/* XXX DJG FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(pg) == 0 || 1) {
if (pg == MPIDI_Process.my_pg)
MPIDI_Process.my_pg = NULL;
MPIU_Object_set_ref(pg, 0); /* satisfy assertions in PG_Destroy */
MPIDI_PG_Destroy( pg );
}
pg = pgNext;
}
/* If COMM_WORLD is still around (it normally should be),
try to free it here. The reason that we need to free it at this
point is that comm_world (and comm_self) still exist, and
hence the usual process to free the related VC structures will
not be invoked. */
if (MPIDI_Process.my_pg) {
MPIDI_PG_Destroy(MPIDI_Process.my_pg);
}
MPIDI_Process.my_pg = NULL;
/* ifdefing out this check because the list will not be NULL in
Ch3_finalize because
one additional reference is retained in MPIDI_Process.my_pg.
That reference is released
only after ch3_finalize returns. If I release it before ch3_finalize,
the ssm channel crashes. */
#if 0
if (MPIDI_PG_list != NULL)
{
/* --BEGIN ERROR HANDLING-- */
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_INTERN,
"**dev|pg_finalize|list_not_empty", NULL);
/* --END ERROR HANDLING-- */
}
#endif
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_FINALIZE);
return mpi_errno;
}
int MPIDI_PG_Destroy(MPIDI_PG_t * pg)
{
MPIDI_PG_t * pg_prev;
MPIDI_PG_t * pg_cur;
int i;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_DESTROY);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_DESTROY);
MPIU_Assert(MPIU_Object_get_ref(pg) == 0);
pg_prev = NULL;
pg_cur = MPIDI_PG_list;
while(pg_cur != NULL)
{
if (pg_cur == pg)
{
if (MPIDI_PG_iterator_next == pg)
{
MPIDI_PG_iterator_next = MPIDI_PG_iterator_next->next;
}
if (pg_prev == NULL)
MPIDI_PG_list = pg->next;
else
pg_prev->next = pg->next;
MPIU_DBG_MSG_FMT(CH3_DISCONNECT, VERBOSE, (MPIU_DBG_FDEST, "destroying pg=%p pg->id=%s", pg, (char *)pg->id));
for (i = 0; i < pg->size; ++i) {
/* FIXME it would be good if we could make this assertion.
Unfortunately, either:
1) We're not being disciplined and some caller of this
function doesn't bother to manage all the refcounts
because he thinks he knows better. Annoying, but not
strictly a bug.
(wdg - actually, that is a bug - managing the ref
counts IS required and missing one is a bug.)
2) There is a real bug lurking out there somewhere and we
just haven't hit it in the tests yet. */
/*MPIU_Assert(MPIU_Object_get_ref(pg->vct[i]) == 0);*/
MPIU_DBG_MSG_FMT(CH3_DISCONNECT, VERBOSE, (MPIU_DBG_FDEST, "about to free pg->vct=%p which contains vc=%p", pg->vct, &pg->vct[i]));
/* This used to be handled in MPID_VCRT_Release, but that was
not the right place to do this. The VC should only be freed
when the PG that it belongs to is freed, not just when the
VC's refcount drops to zero. [goodell@ 2008-06-13] */
/* In that case, the fact that the VC is in the PG should
increment the ref count - reflecting the fact that the
use in the PG constitutes a reference-count-incrementing
use. Alternately, if the PG is able to recreate a VC,
and can thus free unused (or idle) VCs, it should be allowed
to do so. [wdg 2008-08-31] */
mpi_errno = MPIU_CALL(MPIDI_CH3,VC_Destroy(&(pg->vct[i])));
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
MPIDI_PG_Destroy_fn(pg);
#if defined (_OSU_MVAPICH_)
for(i = 0; i < pg->size; i++) {
MPIDI_CH3I_Cleanup_after_connection(&pg->vct[i]);
}
#endif
MPIU_Free(pg->vct);
if (pg->connData) {
if (pg->freeConnInfo) {
(*pg->freeConnInfo)( pg );
}
else {
MPIU_Free(pg->connData);
}
}
mpi_errno = MPIU_CALL(MPIDI_CH3,PG_Destroy(pg));
MPIU_Free(pg);
goto fn_exit;
}
pg_prev = pg_cur;
pg_cur = pg_cur->next;
}
/* PG not found if we got here */
MPIU_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**dev|pg_not_found", "**dev|pg_not_found %p", pg);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_DESTROY);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_lmt_dma_progress(void)
{
int mpi_errno = MPI_SUCCESS;
struct lmt_dma_node *prev = NULL;
struct lmt_dma_node *free_me = NULL;
struct lmt_dma_node *cur = outstanding_head;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
/* Iterate over a linked-list of (req,status_idx)-tuples looking for
completed/failed requests. Currently knem only provides status to the
receiver, so all of these requests are recv requests. */
while (cur) {
switch (*cur->status_p) {
case KNEM_STATUS_SUCCESS:
{
/* complete the request if all data has been sent, remove it from the list */
int complete = 0;
mpi_errno = check_req_complete(cur->vc, cur->req, &complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
free_status_index(cur->status_p - knem_status);
if (complete) {
/* request was completed by the OnDataAvail fn */
MPID_nem_lmt_send_DONE(cur->vc, cur->req); /* tell the other side to complete its request */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
/* There is more data to send. We must inform the sender that we have
completely received the current batch and that the next batch should
be sent. */
MPID_nem_lmt_send_COOKIE(cur->vc, cur->req, NULL, 0);
}
/* Right now we always free the cur element, even if the
request is incomplete because it simplifies the logic. */
if (cur == outstanding_head) {
outstanding_head = cur->next;
prev = NULL;
free_me = cur;
cur = cur->next;
}
else {
prev->next = cur->next;
free_me = cur;
cur = cur->next;
}
if (free_me) MPIU_Free(free_me);
--MPID_nem_local_lmt_pending;
continue;
}
break;
case KNEM_STATUS_FAILED:
/* set the error status for the request, complete it then dequeue the entry */
cur->req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(cur->req->status.MPI_ERROR, MPI_ERR_OTHER, "**recv_status", "**recv_status %d", *cur->status_p);
MPIDI_CH3U_Request_complete(cur->req);
if (cur == outstanding_head) {
outstanding_head = cur->next;
prev = NULL;
free_me = cur;
cur = cur->next;
}
else {
prev->next = cur->next;
free_me = cur;
cur = cur->next;
}
if (free_me) MPIU_Free(free_me);
--MPID_nem_local_lmt_pending;
continue;
break;
case KNEM_STATUS_PENDING:
/* nothing to do here */
break;
default:
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_knem_status",
"**invalid_knem_status %d", *cur->status_p);
break;
}
prev = cur;
cur = cur->next;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
return mpi_errno;
fn_fail:
goto fn_exit;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (MPID_VCRT_Contains_failed_vc(comm->vcrt)) {
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
/*@
MPI_Win_lock_all - Begin an RMA access epoch at all processes on the given window.
Starts an RMA access epoch to all processes in win, with a lock type of
'MPI_Lock_shared'. During the epoch, the calling process can access the window
memory on all processes in win by using RMA operations. A window locked with
'MPI_Win_lock_all' must be unlocked with 'MPI_Win_unlock_all'. This routine is not
collective -- the ALL refers to a lock on all members of the group of the
window.
Input Parameters:
+ assert - Used to optimize this call; zero may be used as a default.
See notes. (integer)
- win - window object (handle)
Notes:
This call is not collective.
The 'assert' argument is used to indicate special conditions for the fence that
an implementation may use to optimize the 'MPI_Win_lock_all' operation. The
value zero is always correct. Other assertion values may be or''ed together.
Assertions that are valid for 'MPI_Win_lock_all' are\:
. 'MPI_MODE_NOCHECK' - No other process holds, or will attempt to acquire a
conflicting lock, while the caller holds the window lock. This is useful
when mutual exclusion is achieved by other means, but the coherence
operations that may be attached to the lock and unlock calls are still
required.
There may be additional overheads associated with using 'MPI_Win_lock' and
'MPI_Win_lock_all' concurrently on the same window. These overheads could be
avoided by specifying the assertion 'MPI_MODE_NOCHECK' when possible
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_RANK
.N MPI_ERR_WIN
.N MPI_ERR_OTHER
.seealso: MPI_Win_unlock_all
@*/
int MPI_Win_lock_all(int assert, MPI_Win win)
{
static const char FCNAME[] = "MPI_Win_lock_all";
int mpi_errno = MPI_SUCCESS;
MPID_Win *win_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_WIN_LOCK_ALL);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_WIN_LOCK_ALL);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_WIN(win, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Win_get_ptr( win, win_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate win_ptr */
MPID_Win_valid_ptr( win_ptr, mpi_errno );
if (mpi_errno) goto fn_fail;
if (assert != 0 && assert != MPI_MODE_NOCHECK) {
MPIU_ERR_SET1(mpi_errno,MPI_ERR_ARG,
"**lockassertval",
"**lockassertval %d", assert );
if (mpi_errno) goto fn_fail;
}
/* TODO: Validate that window is not already locked */
/* TODO: Validate that window is not already in active mode */
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPIU_RMA_CALL(win_ptr,
Win_lock_all(assert, win_ptr));
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_WIN_LOCK_ALL);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_win_lock_all",
"**mpi_win_lock_all %A %W", assert, win);
}
# endif
mpi_errno = MPIR_Err_return_win( win_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Win_free - Free an MPI RMA window
Input Parameter:
. win - window object (handle)
Notes:
If successfully freed, 'win' is set to 'MPI_WIN_NULL'.
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_WIN
.N MPI_ERR_OTHER
@*/
int MPI_Win_free(MPI_Win *win)
{
static const char FCNAME[] = "MPI_Win_free";
int mpi_errno = MPI_SUCCESS;
MPID_Win *win_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_WIN_FREE);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_RMA_FUNC_ENTER(MPID_STATE_MPI_WIN_FREE);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_WIN(*win, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Win_get_ptr( *win, win_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate win_ptr */
MPID_Win_valid_ptr( win_ptr, mpi_errno );
/* If win_ptr is not valid, it will be reset to null */
if (mpi_errno) goto fn_fail;
/* Check for unterminated lock epoch */
if (win_ptr->lockRank != -1) {
MPIU_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**winfreewhilelocked",
"**winfreewhilelocked %d", win_ptr->lockRank);
}
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
if (MPIR_Process.attr_free && win_ptr->attributes)
{
mpi_errno = MPIR_Process.attr_free( win_ptr->handle,
&win_ptr->attributes );
}
/*
* If the user attribute free function returns an error,
* then do not free the window
*/
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* We need to release the error handler */
/* no MPI_OBJ CS is needed here */
if (win_ptr->errhandler &&
! (HANDLE_GET_KIND(win_ptr->errhandler->handle) ==
HANDLE_KIND_BUILTIN) ) {
int in_use;
MPIR_Errhandler_release_ref( win_ptr->errhandler,&in_use);
if (!in_use) {
MPIU_Handle_obj_free( &MPID_Errhandler_mem, win_ptr->errhandler );
}
}
mpi_errno = MPIU_RMA_CALL(win_ptr,Win_free(&win_ptr));
*win = MPI_WIN_NULL;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_RMA_FUNC_EXIT(MPID_STATE_MPI_WIN_FREE);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_win_free", "**mpi_win_free %p", win);
}
# endif
mpi_errno = MPIR_Err_return_win( win_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
