static int MPIR_Bsend_check_active( void )
{
int mpi_errno = MPI_SUCCESS;
MPIR_Bsend_data_t *active = BsendBuffer.active, *next_active;
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Checking active starting at %p", active);
while (active) {
MPI_Request r = active->request->handle;
int flag;
next_active = active->next;
if (active->kind == IBSEND) {
/* We handle ibsend specially to allow for the user
to attempt and cancel the request. Also, to allow
for a cancel attempt (which must be attempted before
a successful test or wait), we only start
testing when the user has successfully released
the request (it is a grequest, the free call will do it) */
flag = 0;
/* XXX DJG FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(active->request) == 1) {
mpi_errno = MPIR_Test_impl(&r, &flag, MPI_STATUS_IGNORE );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
/* We need to invoke the progress engine in case we
need to advance other, incomplete communication. */
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
mpi_errno = MPID_Progress_test( );
MPID_Progress_end(&progress_state);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
} else {
mpi_errno = MPIR_Test_impl( &r, &flag, MPI_STATUS_IGNORE );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (flag) {
/* We're done. Remove this segment */
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Removing segment %p", active);
MPIR_Bsend_free_segment( active );
}
active = next_active;
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Next active is %p",active);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
PMPI_LOCAL int MPIR_Ibsend_free( void *extra )
{
ibsend_req_info *ibsend_info = (ibsend_req_info *)extra;
/* Release the MPID_Request (there is still another ref pending
within the bsendutil functions) */
/* XXX DJG FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(ibsend_info->req) > 1) {
int inuse;
/* Note that this should mean that the request was
cancelled (that would have decremented the ref count)
*/
MPIR_Request_release_ref( ibsend_info->req, &inuse );
}
MPIU_Free( ibsend_info );
return MPI_SUCCESS;
}
/*+
MPIU_Handle_obj_free - Free an object allocated with MPID_Handle_obj_new
Input Parameters:
+ objmem - Pointer to object block
- object - Object to delete
Notes:
This routine assumes that only a single thread calls it at a time; this
is true for the SINGLE_CS approach to thread safety
+*/
void MPIU_Handle_obj_free( MPIU_Object_alloc_t *objmem, void *object )
{
MPIU_Handle_common *obj = (MPIU_Handle_common *)object;
MPIU_THREAD_CS_ENTER(HANDLEALLOC,);
MPIU_DBG_MSG_FMT(HANDLE,TYPICAL,(MPIU_DBG_FDEST,
"Freeing object ptr %p (0x%08x kind=%s) refcount=%d",
(obj),
(obj)->handle,
MPIU_Handle_get_kind_str(HANDLE_GET_MPI_KIND((obj)->handle)),
MPIU_Object_get_ref(obj)));
MPIU_THREAD_MPI_OBJ_FINALIZE(obj);
#ifdef USE_MEMORY_TRACING
{
/* set the object memory to an invalid value (0xec), except for the handle field */
int tmp_handle;
tmp_handle = obj->handle;
memset(obj, 0xec, objmem->size);
obj->handle = tmp_handle;
}
#endif
MPL_VG_MEMPOOL_FREE(objmem, obj);
/* MEMPOOL_FREE marks the object NOACCESS, so we have to make the
* MPIU_Handle_common area that is used for internal book keeping
* addressable again. */
MPL_VG_MAKE_MEM_DEFINED(&obj->handle, sizeof(obj->handle));
MPL_VG_MAKE_MEM_UNDEFINED(&obj->next, sizeof(obj->next));
/* Necessary to prevent annotations from being misinterpreted. HB/HA arcs
* will be drawn between a req object in across a free/alloc boundary
* otherwise. Specifically, stores to obj->next when obj is actually an
* MPID_Request falsely look like a race to DRD and Helgrind because of the
* other lockfree synchronization used with requests. */
MPL_VG_ANNOTATE_NEW_MEMORY(obj, objmem->size);
obj->next = objmem->avail;
objmem->avail = obj;
MPIU_THREAD_CS_EXIT(HANDLEALLOC,);
}
int MPIDI_VCR_Dup(MPIDI_VCR orig_vcr, MPIDI_VCR * new_vcr)
{
MPIDI_STATE_DECL(MPID_STATE_MPID_VCR_DUP);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_VCR_DUP);
/* We are allowed to create a vc that belongs to no process group
as part of the initial connect/accept action, so in that case,
ignore the pg ref count update */
/* XXX DJG FIXME-MT should we be checking this? */
/* we probably need a test-and-incr operation or equivalent to avoid races */
if (MPIU_Object_get_ref(orig_vcr) == 0 && orig_vcr->pg) {
MPIDI_VC_add_ref( orig_vcr );
MPIDI_VC_add_ref( orig_vcr );
MPIDI_PG_add_ref( orig_vcr->pg );
}
else {
MPIDI_VC_add_ref(orig_vcr);
}
MPIU_DBG_MSG_FMT(REFCOUNT,TYPICAL,(MPIU_DBG_FDEST,"Incr VCR %p ref count",orig_vcr));
*new_vcr = orig_vcr;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_VCR_DUP);
return MPI_SUCCESS;
}
/*@
MPIDI_PG_Finalize - Finalize the process groups, including freeing all
process group structures
@*/
int MPIDI_PG_Finalize(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_PG_t *pg, *pgNext;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_FINALIZE);
/* Print the state of the process groups */
if (verbose) {
MPIU_PG_Printall( stdout );
}
/* FIXME - straighten out the use of PMI_Finalize - no use after
PG_Finalize */
if (pg_world->connData) {
#ifdef USE_PMI2_API
mpi_errno = PMI2_Finalize();
if (mpi_errno) MPIR_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|pmi_finalize");
#else
int rc;
rc = PMI_Finalize();
if (rc) {
MPIR_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**ch3|pmi_finalize",
"**ch3|pmi_finalize %d", rc);
}
#endif
}
/* Free the storage associated with the process groups */
pg = MPIDI_PG_list;
while (pg) {
pgNext = pg->next;
/* In finalize, we free all process group information, even if
the ref count is not zero. This can happen if the user
fails to use MPI_Comm_disconnect on communicators that
were created with the dynamic process routines.*/
/* XXX DJG FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(pg) == 0 || 1) {
if (pg == MPIDI_Process.my_pg)
MPIDI_Process.my_pg = NULL;
MPIU_Object_set_ref(pg, 0); /* satisfy assertions in PG_Destroy */
MPIDI_PG_Destroy( pg );
}
pg = pgNext;
}
/* If COMM_WORLD is still around (it normally should be),
try to free it here. The reason that we need to free it at this
point is that comm_world (and comm_self) still exist, and
hence the usual process to free the related VC structures will
not be invoked. */
if (MPIDI_Process.my_pg) {
MPIDI_PG_Destroy(MPIDI_Process.my_pg);
}
MPIDI_Process.my_pg = NULL;
/* ifdefing out this check because the list will not be NULL in
Ch3_finalize because
one additional reference is retained in MPIDI_Process.my_pg.
That reference is released
only after ch3_finalize returns. If I release it before ch3_finalize,
the ssm channel crashes. */
#if 0
if (MPIDI_PG_list != NULL)
{
/* --BEGIN ERROR HANDLING-- */
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_INTERN,
"**dev|pg_finalize|list_not_empty", NULL);
/* --END ERROR HANDLING-- */
}
#endif
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_FINALIZE);
return mpi_errno;
}
int MPIDI_PG_Destroy(MPIDI_PG_t * pg)
{
MPIDI_PG_t * pg_prev;
MPIDI_PG_t * pg_cur;
int i;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_DESTROY);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_DESTROY);
MPIU_Assert(MPIU_Object_get_ref(pg) == 0);
pg_prev = NULL;
pg_cur = MPIDI_PG_list;
while(pg_cur != NULL)
{
if (pg_cur == pg)
{
if (MPIDI_PG_iterator_next == pg)
{
MPIDI_PG_iterator_next = MPIDI_PG_iterator_next->next;
}
if (pg_prev == NULL)
MPIDI_PG_list = pg->next;
else
pg_prev->next = pg->next;
MPIU_DBG_MSG_FMT(CH3_DISCONNECT, VERBOSE, (MPIU_DBG_FDEST, "destroying pg=%p pg->id=%s", pg, (char *)pg->id));
for (i = 0; i < pg->size; ++i) {
/* FIXME it would be good if we could make this assertion.
Unfortunately, either:
1) We're not being disciplined and some caller of this
function doesn't bother to manage all the refcounts
because he thinks he knows better. Annoying, but not
strictly a bug.
(wdg - actually, that is a bug - managing the ref
counts IS required and missing one is a bug.)
2) There is a real bug lurking out there somewhere and we
just haven't hit it in the tests yet. */
/*MPIU_Assert(MPIU_Object_get_ref(pg->vct[i]) == 0);*/
MPIU_DBG_MSG_FMT(CH3_DISCONNECT, VERBOSE, (MPIU_DBG_FDEST, "about to free pg->vct=%p which contains vc=%p", pg->vct, &pg->vct[i]));
/* This used to be handled in MPIDI_VCRT_Release, but that was
not the right place to do this. The VC should only be freed
when the PG that it belongs to is freed, not just when the
VC's refcount drops to zero. [goodell@ 2008-06-13] */
/* In that case, the fact that the VC is in the PG should
increment the ref count - reflecting the fact that the
use in the PG constitutes a reference-count-incrementing
use. Alternately, if the PG is able to recreate a VC,
and can thus free unused (or idle) VCs, it should be allowed
to do so. [wdg 2008-08-31] */
mpi_errno = MPIDI_CH3_VC_Destroy(&(pg->vct[i]));
if (mpi_errno) { MPIR_ERR_POP(mpi_errno); }
}
MPIDI_PG_Destroy_fn(pg);
MPIU_Free(pg->vct);
if (pg->connData) {
if (pg->freeConnInfo) {
(*pg->freeConnInfo)( pg );
}
else {
MPIU_Free(pg->connData);
}
}
mpi_errno = MPIDI_CH3_PG_Destroy(pg);
MPIU_Free(pg);
goto fn_exit;
}
pg_prev = pg_cur;
pg_cur = pg_cur->next;
}
/* PG not found if we got here */
MPIR_ERR_SET1(mpi_errno,MPI_ERR_OTHER,
"**dev|pg_not_found", "**dev|pg_not_found %p", pg);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_DESTROY);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Comm_delete_internal(MPID_Comm * comm_ptr)
{
int in_use;
int mpi_errno = MPI_SUCCESS;
MPID_MPI_STATE_DECL(MPID_STATE_COMM_DELETE_INTERNAL);
MPID_MPI_FUNC_ENTER(MPID_STATE_COMM_DELETE_INTERNAL);
MPIU_Assert(MPIU_Object_get_ref(comm_ptr) == 0); /* sanity check */
/* Remove the attributes, executing the attribute delete routine.
* Do this only if the attribute functions are defined.
* This must be done first, because if freeing the attributes
* returns an error, the communicator is not freed */
if (MPIR_Process.attr_free && comm_ptr->attributes) {
/* Temporarily add a reference to this communicator because
* the attr_free code requires a valid communicator */
MPIU_Object_add_ref(comm_ptr);
mpi_errno = MPIR_Process.attr_free(comm_ptr->handle, &comm_ptr->attributes);
/* Release the temporary reference added before the call to
* attr_free */
MPIU_Object_release_ref(comm_ptr, &in_use);
}
/* If the attribute delete functions return failure, the
* communicator must not be freed. That is the reason for the
* test on mpi_errno here. */
if (mpi_errno == MPI_SUCCESS) {
/* If this communicator is our parent, and we're disconnecting
* from the parent, mark that fact */
if (MPIR_Process.comm_parent == comm_ptr)
MPIR_Process.comm_parent = NULL;
/* Notify the device that the communicator is about to be
* destroyed */
mpi_errno = MPID_Dev_comm_destroy_hook(comm_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Free info hints */
if (comm_ptr->info != NULL) {
MPIU_Info_free(comm_ptr->info);
}
/* release our reference to the collops structure, comes after the
* destroy_hook to allow the device to manage these vtables in a custom
* fashion */
if (comm_ptr->coll_fns && --comm_ptr->coll_fns->ref_count == 0) {
MPIU_Free(comm_ptr->coll_fns);
comm_ptr->coll_fns = NULL;
}
if (comm_ptr->comm_kind == MPID_INTERCOMM && comm_ptr->local_comm)
MPIR_Comm_release(comm_ptr->local_comm);
/* Free the local and remote groups, if they exist */
if (comm_ptr->local_group)
MPIR_Group_release(comm_ptr->local_group);
if (comm_ptr->remote_group)
MPIR_Group_release(comm_ptr->remote_group);
/* free the intra/inter-node communicators, if they exist */
if (comm_ptr->node_comm)
MPIR_Comm_release(comm_ptr->node_comm);
if (comm_ptr->node_roots_comm)
MPIR_Comm_release(comm_ptr->node_roots_comm);
if (comm_ptr->intranode_table != NULL)
MPIU_Free(comm_ptr->intranode_table);
if (comm_ptr->internode_table != NULL)
MPIU_Free(comm_ptr->internode_table);
/* Free the context value. This should come after freeing the
* intra/inter-node communicators since those free calls won't
* release this context ID and releasing this before then could lead
* to races once we make threading finer grained. */
/* This must be the recvcontext_id (i.e. not the (send)context_id)
* because in the case of intercommunicators the send context ID is
* allocated out of the remote group's bit vector, not ours. */
MPIR_Free_contextid(comm_ptr->recvcontext_id);
/* We need to release the error handler */
if (comm_ptr->errhandler &&
!(HANDLE_GET_KIND(comm_ptr->errhandler->handle) == HANDLE_KIND_BUILTIN)) {
int errhInuse;
MPIR_Errhandler_release_ref(comm_ptr->errhandler, &errhInuse);
if (!errhInuse) {
MPIU_Handle_obj_free(&MPID_Errhandler_mem, comm_ptr->errhandler);
}
}
/* Remove from the list of active communicators if
* we are supporting message-queue debugging. We make this
* conditional on having debugger support since the
* operation is not constant-time */
MPIR_COMML_FORGET(comm_ptr);
/* Check for predefined communicators - these should not
* be freed */
if (!(HANDLE_GET_KIND(comm_ptr->handle) == HANDLE_KIND_BUILTIN))
MPIU_Handle_obj_free(&MPID_Comm_mem, comm_ptr);
}
else {
/* If the user attribute free function returns an error,
* then do not free the communicator */
MPIR_Comm_add_ref(comm_ptr);
}
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_COMM_DELETE_INTERNAL);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPI_Comm_disconnect - Disconnect from a communicator
Input Parameter:
. comm - communicator (handle)
Notes:
This routine waits for all pending communication to complete, then frees the
communicator and sets 'comm' to 'MPI_COMM_NULL'. It may not be called
with 'MPI_COMM_WORLD' or 'MPI_COMM_SELF'.
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.seealso MPI_Comm_connect, MPI_Comm_join
@*/
int MPI_Comm_disconnect(MPI_Comm * comm)
{
static const char FCNAME[] = "MPI_Comm_disconnect";
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_COMM_DISCONNECT);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_COMM_DISCONNECT);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(*comm, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr( *comm, comm_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate comm_ptr */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno );
/* If comm_ptr is not valid, it will be reset to null */
if (mpi_errno)
{
goto fn_fail;
}
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
/*
* Since outstanding I/O bumps the reference count on the communicator,
* we wait until we hold the last reference count to
* ensure that all communication has completed. The reference count
* is 1 when the communicator is created, and it is incremented
* only for pending communication operations (and decremented when
* those complete).
*/
/* FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(comm_ptr) > 1)
{
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (MPIU_Object_get_ref(comm_ptr) > 1)
{
mpi_errno = MPID_Progress_wait(&progress_state);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPID_Progress_end(&progress_state);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
}
MPID_Progress_end(&progress_state);
}
mpi_errno = MPID_Comm_disconnect(comm_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
*comm = MPI_COMM_NULL;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_COMM_DISCONNECT);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_comm_disconnect",
"**mpi_comm_disconnect %C", *comm);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_VCRT_Release(struct MPIDI_VCRT *vcrt, int isDisconnect )
{
int in_use;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_VCRT_RELEASE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_VCRT_RELEASE);
MPIU_Object_release_ref(vcrt, &in_use);
MPIU_DBG_MSG_FMT(REFCOUNT,TYPICAL,(MPIU_DBG_FDEST, "Decr VCRT %p ref count",vcrt));
/* If this VC reference table is no longer in use, we can
decrement the reference count of each of the VCs. If the
count on the VCs goes to zero, then we can decrement the
ref count on the process group and so on.
*/
if (!in_use) {
int i, inuse;
for (i = 0; i < vcrt->size; i++)
{
MPIDI_VC_t * const vc = vcrt->vcr_table[i];
MPIDI_VC_release_ref(vc, &in_use);
/* Dynamic connections start with a refcount of 2 instead of 1.
* That way we can distinguish between an MPI_Free and an
* MPI_Comm_disconnect. */
/* XXX DJG FIXME-MT should we be checking this? */
/* probably not, need to do something like the following instead: */
#if 0
if (isDisconnect) {
MPIU_Assert(in_use);
/* FIXME this is still bogus, the VCRT may contain a mix of
* dynamic and non-dynamic VCs, so the ref_count isn't
* guaranteed to have started at 2. The best thing to do might
* be to avoid overloading the reference counting this way and
* use a separate check for dynamic VCs (another flag? compare
* PGs?) */
MPIU_Object_release_ref(vc, &in_use);
}
#endif
if (isDisconnect && MPIU_Object_get_ref(vc) == 1) {
MPIDI_VC_release_ref(vc, &in_use);
}
if (!in_use)
{
/* If the VC is myself then skip the close message */
if (vc->pg == MPIDI_Process.my_pg &&
vc->pg_rank == MPIDI_Process.my_pg_rank)
{
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0)
{
MPIDI_PG_Destroy(vc->pg);
}
continue;
}
/* FIXME: the correct test is ACTIVE or REMOTE_CLOSE */
/*if (vc->state != MPIDI_VC_STATE_INACTIVE) { */
if (vc->state == MPIDI_VC_STATE_ACTIVE ||
vc->state == MPIDI_VC_STATE_REMOTE_CLOSE)
{
MPIDI_CH3U_VC_SendClose( vc, i );
}
else
{
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0)
{
MPIDI_PG_Destroy(vc->pg);
}
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"vc=%p: not sending a close to %d, vc in state %s",
vc, i, MPIDI_VC_GetStateString(vc->state)));
}
/* NOTE: we used to * MPIDI_CH3_VC_Destroy(&(pg->vct[i])))
here but that is incorrect. According to the standard, it's
entirely possible (likely even) that this VC might still be
connected. VCs are now destroyed when the PG that "owns"
them is destroyed (see MPIDI_PG_Destroy). [goodell@ 2008-06-13] */
}
}
MPIU_Free(vcrt);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_VCRT_RELEASE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPIDI_CH3U_Handle_connection - handle connection event
Input Parameters:
+ vc - virtual connection
. event - connection event
NOTE:
This routine is used to transition the VC state.
The only events currently handled are TERMINATED events. This
routine should be called (with TERMINATED) whenever a connection is
terminated whether normally (in MPIDI_CH3_Connection_terminate() ),
or abnormally.
FIXME: Currently state transitions resulting from receiving CLOSE
packets are performed in MPIDI_CH3_PktHandler_Close(). Perhaps that
should move here.
@*/
int MPIDI_CH3U_Handle_connection(MPIDI_VC_t * vc, MPIDI_VC_Event_t event)
{
int inuse;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
switch (event)
{
case MPIDI_VC_EVENT_TERMINATED:
{
switch (vc->state)
{
case MPIDI_VC_STATE_CLOSED:
/* Normal termination. */
MPIDI_CHANGE_VC_STATE(vc, INACTIVE);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL,
"outstanding close operations = %d", MPIDI_Outstanding_close_ops);
if (MPIDI_Outstanding_close_ops == 0)
{
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
case MPIDI_VC_STATE_INACTIVE:
/* VC was terminated before it was activated.
This can happen if a failed process was
detected before the process used the VC. */
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL, "VC terminated before it was activated. We probably got a failed"
" process notification.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_ACTIVE:
case MPIDI_VC_STATE_REMOTE_CLOSE:
/* This is a premature termination. This process
has not started the close protocol. There may
be outstanding sends or receives on the local
side, remote side or both. */
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL, "Connection closed prematurely.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_LOCAL_CLOSE:
/* This is a premature termination. This process
has started the close protocol, but hasn't
received a CLOSE packet from the remote side.
This process may not have been able to send the
CLOSE ack=F packet to the remote side. There
may be outstanding sends or receives on the
local or remote sides. */
case MPIDI_VC_STATE_CLOSE_ACKED:
/* This is a premature termination. Both sides
have started the close protocol. This process
has received CLOSE ack=F, but not CLOSE ack=t.
This process may not have been able to send
CLOSE ack=T. There should not be any
outstanding sends or receives on either
side. */
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL, "Connection closed prematurely during close protocol. "
"Outstanding close operations = %d", MPIDI_Outstanding_close_ops);
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
if (MPIDI_Outstanding_close_ops == 0) {
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
default:
{
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL, "Unhandled connection state %d when closing connection",vc->state);
mpi_errno = MPIR_Err_create_code(
MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|unhandled_connection_state",
"**ch3|unhandled_connection_state %p %d", vc, vc->state);
goto fn_fail;
break;
}
}
/* FIXME: Decrement the reference count? Who increments? */
/* FIXME: The reference count is often already 0. But
not always */
/* MPIU_Object_set_ref(vc, 0); ??? */
/*
* FIXME: The VC used in connect accept has a NULL
* process group
*/
/* XXX DJG FIXME-MT should we be checking this ref_count? */
if (vc->pg != NULL && (MPIU_Object_get_ref(vc) == 0))
{
/* FIXME: Who increments the reference count that
this is decrementing? */
/* When the reference count for a vc becomes zero,
decrement the reference count
of the associated process group. */
/* FIXME: This should be done when the reference
count of the vc is first decremented */
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0) {
MPIDI_PG_Destroy(vc->pg);
}
}
break;
}
default:
{
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
return mpi_errno;
fn_fail:
goto fn_exit;
}
