int MPIDI_VCRT_Release(struct MPIDI_VCRT *vcrt, int isDisconnect )
{
int in_use;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_VCRT_RELEASE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_VCRT_RELEASE);
MPIU_Object_release_ref(vcrt, &in_use);
MPIU_DBG_MSG_FMT(REFCOUNT,TYPICAL,(MPIU_DBG_FDEST, "Decr VCRT %p ref count",vcrt));
/* If this VC reference table is no longer in use, we can
decrement the reference count of each of the VCs. If the
count on the VCs goes to zero, then we can decrement the
ref count on the process group and so on.
*/
if (!in_use) {
int i, inuse;
for (i = 0; i < vcrt->size; i++)
{
MPIDI_VC_t * const vc = vcrt->vcr_table[i];
MPIDI_VC_release_ref(vc, &in_use);
/* Dynamic connections start with a refcount of 2 instead of 1.
* That way we can distinguish between an MPI_Free and an
* MPI_Comm_disconnect. */
/* XXX DJG FIXME-MT should we be checking this? */
/* probably not, need to do something like the following instead: */
#if 0
if (isDisconnect) {
MPIU_Assert(in_use);
/* FIXME this is still bogus, the VCRT may contain a mix of
* dynamic and non-dynamic VCs, so the ref_count isn't
* guaranteed to have started at 2. The best thing to do might
* be to avoid overloading the reference counting this way and
* use a separate check for dynamic VCs (another flag? compare
* PGs?) */
MPIU_Object_release_ref(vc, &in_use);
}
#endif
if (isDisconnect && MPIU_Object_get_ref(vc) == 1) {
MPIDI_VC_release_ref(vc, &in_use);
}
if (!in_use)
{
/* If the VC is myself then skip the close message */
if (vc->pg == MPIDI_Process.my_pg &&
vc->pg_rank == MPIDI_Process.my_pg_rank)
{
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0)
{
MPIDI_PG_Destroy(vc->pg);
}
continue;
}
/* FIXME: the correct test is ACTIVE or REMOTE_CLOSE */
/*if (vc->state != MPIDI_VC_STATE_INACTIVE) { */
if (vc->state == MPIDI_VC_STATE_ACTIVE ||
vc->state == MPIDI_VC_STATE_REMOTE_CLOSE)
{
MPIDI_CH3U_VC_SendClose( vc, i );
}
else
{
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0)
{
MPIDI_PG_Destroy(vc->pg);
}
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"vc=%p: not sending a close to %d, vc in state %s",
vc, i, MPIDI_VC_GetStateString(vc->state)));
}
/* NOTE: we used to * MPIDI_CH3_VC_Destroy(&(pg->vct[i])))
here but that is incorrect. According to the standard, it's
entirely possible (likely even) that this VC might still be
connected. VCs are now destroyed when the PG that "owns"
them is destroyed (see MPIDI_PG_Destroy). [goodell@ 2008-06-13] */
}
}
MPIU_Free(vcrt);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_VCRT_RELEASE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPIDI_CH3U_Handle_connection - handle connection event
Input Parameters:
+ vc - virtual connection
. event - connection event
NOTE:
This routine is used to transition the VC state.
The only events currently handled are TERMINATED events. This
routine should be called (with TERMINATED) whenever a connection is
terminated whether normally (in MPIDI_CH3_Connection_terminate() ),
or abnormally.
FIXME: Currently state transitions resulting from receiving CLOSE
packets are performed in MPIDI_CH3_PktHandler_Close(). Perhaps that
should move here.
@*/
int MPIDI_CH3U_Handle_connection(MPIDI_VC_t * vc, MPIDI_VC_Event_t event)
{
int inuse;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
switch (event)
{
case MPIDI_VC_EVENT_TERMINATED:
{
switch (vc->state)
{
case MPIDI_VC_STATE_CLOSED:
/* Normal termination. */
MPIDI_CHANGE_VC_STATE(vc, INACTIVE);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,
"outstanding close operations = %d", MPIDI_Outstanding_close_ops);
if (MPIDI_Outstanding_close_ops == 0)
{
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
case MPIDI_VC_STATE_INACTIVE:
/* VC was terminated before it was activated.
This can happen if a failed process was
detected before the process used the VC. */
MPL_DBG_MSG(MPIDI_CH3_DBG_DISCONNECT,TYPICAL, "VC terminated before it was activated. We probably got a failed"
" process notification.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_ACTIVE:
case MPIDI_VC_STATE_REMOTE_CLOSE:
/* This is a premature termination. This process
has not started the close protocol. There may
be outstanding sends or receives on the local
side, remote side or both. */
MPL_DBG_MSG(MPIDI_CH3_DBG_DISCONNECT,TYPICAL, "Connection closed prematurely.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_LOCAL_CLOSE:
/* This is a premature termination. This process
has started the close protocol, but hasn't
received a CLOSE packet from the remote side.
This process may not have been able to send the
CLOSE ack=F packet to the remote side. There
may be outstanding sends or receives on the
local or remote sides. */
case MPIDI_VC_STATE_CLOSE_ACKED:
/* This is a premature termination. Both sides
have started the close protocol. This process
has received CLOSE ack=F, but not CLOSE ack=t.
This process may not have been able to send
CLOSE ack=T. There should not be any
outstanding sends or receives on either
side. */
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL, "Connection closed prematurely during close protocol. "
"Outstanding close operations = %d", MPIDI_Outstanding_close_ops);
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
if (MPIDI_Outstanding_close_ops == 0) {
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
default:
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL, "Unhandled connection state %d when closing connection",vc->state);
mpi_errno = MPIR_Err_create_code(
MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|unhandled_connection_state",
"**ch3|unhandled_connection_state %p %d", vc, vc->state);
goto fn_fail;
break;
}
}
/* FIXME: Decrement the reference count? Who increments? */
/* FIXME: The reference count is often already 0. But
not always */
/* MPIR_Object_set_ref(vc, 0); ??? */
/*
* FIXME: The VC used in connect accept has a NULL
* process group
*/
/* XXX DJG FIXME-MT should we be checking this ref_count? */
if (vc->pg != NULL && (MPIR_Object_get_ref(vc) == 0))
{
/* FIXME: Who increments the reference count that
this is decrementing? */
/* When the reference count for a vc becomes zero,
decrement the reference count
of the associated process group. */
/* FIXME: This should be done when the reference
count of the vc is first decremented */
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0) {
MPIDI_PG_Destroy(vc->pg);
}
}
break;
}
default:
{
break;
}
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
return mpi_errno;
fn_fail:
goto fn_exit;
}
