/**
* \brief Shut down the system
*
* At this time, no attempt is made to free memory being used for MPI structures.
* \return MPI_SUCCESS
*/
int MPID_Finalize()
{
pami_result_t rc;
int mpierrno = MPI_SUCCESS;
mpir_errflag_t errflag=MPIR_ERR_NONE;
MPIR_Barrier_impl(MPIR_Process.comm_world, &errflag);
#ifdef MPIDI_STATISTICS
if (MPIDI_Process.mp_statistics) {
MPIDI_print_statistics();
}
MPIDI_close_pe_extension();
#endif
#ifdef DYNAMIC_TASKING
mpidi_finalized = 1;
if(mpidi_dynamic_tasking) {
/* Tell the process group code that we're done with the process groups.
This will notify PMI (with PMI_Finalize) if necessary. It
also frees all PG structures, including the PG for COMM_WORLD, whose
pointer is also saved in MPIDI_Process.my_pg */
mpierrno = MPIDI_PG_Finalize();
if (mpierrno) {
TRACE_ERR("MPIDI_PG_Finalize returned with mpierrno=%d\n", mpierrno);
}
MPIDI_FreeParentPort();
}
if(_conn_info_list)
MPIU_Free(_conn_info_list);
MPIDI_free_all_tranid_node();
#endif
/* ------------------------- */
/* shutdown request queues */
/* ------------------------- */
MPIDI_Recvq_finalize();
PAMIX_Finalize(MPIDI_Client);
#ifdef MPID_NEEDS_ICOMM_WORLD
MPIR_Comm_release_always(MPIR_Process.icomm_world, 0);
#endif
MPIR_Comm_release_always(MPIR_Process.comm_self,0);
MPIR_Comm_release_always(MPIR_Process.comm_world,0);
rc = PAMI_Context_destroyv(MPIDI_Context, MPIDI_Process.avail_contexts);
MPID_assert_always(rc == PAMI_SUCCESS);
rc = PAMI_Client_destroy(&MPIDI_Client);
MPID_assert_always(rc == PAMI_SUCCESS);
#ifdef MPIDI_TRACE
{ int i;
for (i=0; i< MPIDI_Process.numTasks; i++) {
if (MPIDI_Trace_buf[i].R)
MPIU_Free(MPIDI_Trace_buf[i].R);
if (MPIDI_Trace_buf[i].PR)
MPIU_Free(MPIDI_Trace_buf[i].PR);
if (MPIDI_Trace_buf[i].S)
MPIU_Free(MPIDI_Trace_buf[i].S);
}
}
MPIU_Free(MPIDI_Trace_buf);
#endif
#ifdef OUT_OF_ORDER_HANDLING
MPIU_Free(MPIDI_In_cntr);
MPIU_Free(MPIDI_Out_cntr);
#endif
if (TOKEN_FLOW_CONTROL_ON)
{
#if TOKEN_FLOW_CONTROL
extern char *EagerLimit;
if (EagerLimit) MPIU_Free(EagerLimit);
MPIU_Free(MPIDI_Token_cntr);
MPIDI_close_mm();
#else
MPID_assert_always(0);
#endif
}
return MPI_SUCCESS;
}
int MPID_Finalize(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_FINALIZE);
/*
* Wait for all posted receives to complete. For now we are not doing
* this since it will cause invalid programs to hang.
* The side effect of not waiting is that posted any source receives
* may erroneous blow up.
*
* For now, we are placing a warning at the end of MPID_Finalize() to
* inform the user if any outstanding posted receives exist.
*/
/* FIXME: The correct action here is to begin a shutdown protocol
* that lets other processes know that this process is now
* in finalize.
*
* Note that only requests that have been freed with MPI_Request_free
* are valid at this point; other pending receives can be ignored
* since a valid program should wait or test for them before entering
* finalize.
*
* The easist fix is to allow an MPI_Barrier over comm_world (and
* any connected processes in the MPI-2 case). Once the barrier
* completes, all processes are in finalize and any remaining
* unmatched receives will never be matched (by a correct program;
* a program with a send in a separate thread that continues after
* some thread calls MPI_Finalize is erroneous).
*
* Avoiding the barrier is hard. Consider this sequence of steps:
* Send in-finalize message to all connected processes. Include
* information on whether there are pending receives.
* (Note that a posted receive with any source is a problem)
* (If there are many connections, then this may take longer than
* the barrier)
* Allow connection requests from anyone who has not previously
* connected only if there is an possible outstanding receive;
* reject others with a failure (causing the source process to
* fail).
* Respond to an in-finalize message with the number of posted receives
* remaining. If both processes have no remaining receives, they
* can both close the connection.
*
* Processes with no pending receives and no connections can exit,
* calling PMI_Finalize to let the process manager know that they
* are in a controlled exit.
*
* Processes that still have open connections must then try to contact
* the remaining processes.
*
* August 2010:
*
* The barrier has been removed so that finalize won't hang when
* another processes has died. This allows processes to finalize
* and exit while other processes are still executing. This has
* the following consequences:
*
* * If a process tries to send a message to a process that has
* exited before completing the matching receive, it will now
* get an error. This is an erroneous program.
*
* * If a process finalizes before completing a nonblocking
* send, the message might not be sent. Similarly, if it
* finalizes before completing all receives, the sender may
* get an error. These are erroneous programs.
*
* * A process may isend to another process that has already
* terminated, then cancel the send. The program is not
* erroneous in this case, but this will result in an error.
* This can be fixed by not returning an error until the app
* completes the send request. If the app cancels the
* request, we need to to search the pending send queue and
* cancel it, in which case an error shouldn't be generated.
*/
#ifdef MPID_NEEDS_ICOMM_WORLD
mpi_errno = MPIR_Comm_release_always(MPIR_Process.icomm_world, 0);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
#endif
mpi_errno = MPIR_Comm_release_always(MPIR_Process.comm_self, 0);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIR_Comm_release_always(MPIR_Process.comm_world, 0);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Re-enabling the close step because many tests are failing
* without it, particularly under gforker */
#if 1
/* FIXME: The close actions should use the same code as the other
connection close code */
mpi_errno = MPIDI_PG_Close_VCs();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/*
* Wait for all VCs to finish the close protocol
*/
mpi_errno = MPIDI_CH3U_VC_WaitForClose();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
#endif
/* Note that the CH3I_Progress_finalize call has been removed; the
CH3_Finalize routine should call it */
mpi_errno = MPIDI_CH3_Finalize();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* Tell the process group code that we're done with the process groups.
This will notify PMI (with PMI_Finalize) if necessary. It
also frees all PG structures, including the PG for COMM_WORLD, whose
pointer is also saved in MPIDI_Process.my_pg */
mpi_errno = MPIDI_PG_Finalize();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
#ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS
MPIDI_CH3_FreeParentPort();
#endif
/* Release any SRbuf pool storage */
if (MPIDI_CH3U_SRBuf_pool) {
MPIDI_CH3U_SRBuf_element_t *p, *pNext;
p = MPIDI_CH3U_SRBuf_pool;
while (p) {
pNext = p->next;
MPIU_Free(p);
p = pNext;
}
}
MPIDU_Ftb_finalize();
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_FINALIZE);
return mpi_errno;
fn_fail:
goto fn_exit;
}