static int ckpt_restart(void)
{
int mpi_errno = MPI_SUCCESS;
char *publish_bc_orig = NULL;
char *bc_val = NULL;
int val_max_sz;
int i;
MPIDI_STATE_DECL(MPID_STATE_CKPT_RESTART);
MPIDI_FUNC_ENTER(MPID_STATE_CKPT_RESTART);
/* First, clean up. We didn't shut anything down before the
checkpoint, so we need to go close and free any resources */
mpi_errno = MPID_nem_tcp_ckpt_cleanup();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_tcp_send_finalize();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_tcp_sm_finalize();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* Initialize the new business card */
mpi_errno = MPIDI_CH3I_BCInit(&bc_val, &val_max_sz);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
publish_bc_orig = bc_val;
/* Now we can restart */
mpi_errno = MPID_nem_tcp_init(MPIDI_Process.my_pg, MPIDI_Process.my_pg_rank, &bc_val, &val_max_sz);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* publish business card */
mpi_errno = MPIDI_PG_SetConnInfo(MPIDI_Process.my_pg_rank, (const char *)publish_bc_orig);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIU_Free(publish_bc_orig);
for (i = 0; i < MPIDI_Process.my_pg->size; ++i) {
MPIDI_VC_t *vc;
MPIDI_CH3I_VC *vc_ch;
if (i == MPIDI_Process.my_pg_rank)
continue;
MPIDI_PG_Get_vc(MPIDI_Process.my_pg, i, &vc);
vc_ch = &vc->ch;
if (!vc_ch->is_local) {
mpi_errno = vc_ch->ckpt_restart_vc(vc);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_CKPT_RESTART);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg_p, int pg_rank )
{
int mpi_errno = MPI_SUCCESS;
char *publish_bc_orig = NULL;
char *bc_val = NULL;
int val_max_remaining;
MPIDI_STATE_DECL(MPID_STATE_MPID_CH3_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_CH3_INIT);
mpi_errno = MPIDI_CH3I_Progress_init();
if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno);
/* Initialize the business card */
mpi_errno = MPIDI_CH3I_BCInit( &bc_val, &val_max_remaining );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
publish_bc_orig = bc_val;
/* initialize aspects specific to sockets */
mpi_errno = MPIDI_CH3U_Init_sock(has_parent, pg_p, pg_rank,
&bc_val, &val_max_remaining);
/* Set the connection information in our process group
(publish the business card ) */
MPIDI_PG_SetConnInfo( pg_rank, (const char *)publish_bc_orig );
/* Free the business card now that it is published
(note that publish_bc_orig is the head of bc_val ) */
MPIDI_CH3I_BCFree( publish_bc_orig );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_CH3_INIT);
return mpi_errno;
fn_fail:
if (publish_bc_orig != NULL) {
MPL_free(publish_bc_orig);
}
goto fn_exit;
}
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg, int pg_rank)
{
int mpi_errno = MPI_SUCCESS;
int pg_size, threshold, dpm = 0, p;
char *dpm_str, *value, *conn_info = NULL;
MPIDI_VC_t *vc;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_INIT);
if (MPIDI_CH3_Pkt_size_index[MPIDI_CH3_PKT_CLOSE] != sizeof (MPIDI_CH3_Pkt_close_t))
{
MPIU_ERR_SETFATALANDJUMP1(
mpi_errno,
MPI_ERR_OTHER,
"**fail",
"**fail %s",
"Failed sanity check! Packet size table mismatch");
}
pg_size = MPIDI_PG_Get_size(pg);
/*Determine to use which connection management*/
threshold = MPIDI_CH3I_CM_DEFAULT_ON_DEMAND_THRESHOLD;
/*check ON_DEMAND_THRESHOLD*/
value = getenv("MV2_ON_DEMAND_THRESHOLD");
if (value)
{
threshold = atoi(value);
}
dpm_str = getenv("MV2_SUPPORT_DPM");
if (dpm_str) {
dpm = !!atoi(dpm_str);
}
MPIDI_CH3I_Process.has_dpm = dpm;
if(MPIDI_CH3I_Process.has_dpm) {
setenv("MV2_ENABLE_AFFINITY", "0", 1);
}
#ifdef _ENABLE_XRC_
value = getenv ("MV2_USE_XRC");
if (value) {
USE_XRC = atoi(value);
if (USE_XRC) {
/* Enable on-demand */
threshold = 1;
}
}
#endif /* _ENABLE_XRC_ */
#ifdef _ENABLE_UD_
if ((value = getenv("MV2_HYBRID_ENABLE_THRESHOLD")) != NULL) {
rdma_hybrid_enable_threshold = atoi(value);
}
if ((value = getenv("MV2_USE_UD_HYBRID")) != NULL) {
rdma_enable_hybrid = atoi(value);
}
if (pg_size < rdma_hybrid_enable_threshold) {
rdma_enable_hybrid = 0;
}
#endif
if (pg_size > threshold || dpm
#ifdef _ENABLE_XRC_
|| USE_XRC
#endif /* _ENABLE_XRC_ */
#ifdef _ENABLE_UD_
|| rdma_enable_hybrid
#endif
)
{
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND;
MPIDI_CH3I_Process.num_conn = 0;
}
else
{
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_BASIC_ALL2ALL;
}
#if defined(RDMA_CM)
if (((value = getenv("MV2_USE_RDMA_CM")) != NULL
|| (value = getenv("MV2_USE_IWARP_MODE")) != NULL)
&& atoi(value) && ! dpm) {
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_RDMA_CM;
} else {
rdma_cm_get_hca_type(&MPIDI_CH3I_RDMA_Process);
}
#endif /* defined(RDMA_CM) */
MPIDI_PG_GetConnKVSname(&pg->ch.kvs_name);
#if defined(CKPT)
#if defined(RDMA_CM)
if (MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_RDMA_CM)
{
MPIU_Error_printf("Error: Checkpointing does not work with RDMA CM.\n"
"Please configure and compile MVAPICH2 with checkpointing disabled "
"or without support for RDMA CM.\n");
MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail");
}
#endif /* defined(RDMA_CM) */
// Always use CM_ON_DEMAND for Checkpoint/Restart and Migration
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND;
#endif /* defined(CKPT) */
#ifdef _ENABLE_UD_
if (rdma_enable_hybrid) {
MPIU_Assert(MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_ON_DEMAND);
}
#endif
/* save my vc_ptr for easy access */
MPIDI_PG_Get_vc(pg, pg_rank, &MPIDI_CH3I_Process.vc);
/* Initialize Progress Engine */
if ((mpi_errno = MPIDI_CH3I_Progress_init()))
{
MPIU_ERR_POP(mpi_errno);
}
/* Check for SMP only */
MPIDI_CH3I_set_smp_only();
if (!SMP_ONLY)
{
switch (MPIDI_CH3I_Process.cm_type)
{
/* allocate rmda memory and set up the queues */
case MPIDI_CH3I_CM_ON_DEMAND:
#if defined(RDMA_CM)
case MPIDI_CH3I_CM_RDMA_CM:
#endif /* defined(RDMA_CM) */
mpi_errno = MPIDI_CH3I_CM_Init(pg, pg_rank, &conn_info);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
break;
default:
/*call old init to setup all connections*/
if ((mpi_errno = MPIDI_CH3I_RDMA_init(pg, pg_rank)) != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
/* All vc should be connected */
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE;
}
break;
}
}
#if defined(CKPT)
#if defined(DISABLE_PTMALLOC)
MPIU_Error_printf("Error: Checkpointing does not work without registration "
"caching enabled.\nPlease configure and compile MVAPICH2 without checkpointing "
" or enable registration caching.\n");
MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail");
#endif /* defined(DISABLE_PTMALLOC) */
if ((mpi_errno = MPIDI_CH3I_CR_Init(pg, pg_rank, pg_size)))
{
MPIU_ERR_POP(mpi_errno);
}
#endif /* defined(CKPT) */
/* set connection info for dynamic process management */
if (conn_info && dpm) {
mpi_errno = MPIDI_PG_SetConnInfo(pg_rank, (const char *)conn_info);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
}
MPIU_Free(conn_info);
/* Initialize the smp channel */
if ((mpi_errno = MPIDI_CH3I_SMP_init(pg)))
{
MPIU_ERR_POP(mpi_errno);
}
if (SMP_INIT)
{
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
/* Mark the SMP VC as Idle */
if (vc->smp.local_nodes >= 0)
{
vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE;
if (SMP_ONLY) {
MPIDI_CH3I_SMP_Init_VC(vc);
}
#ifdef _ENABLE_XRC_
VC_XST_SET (vc, XF_SMP_VC);
#endif
}
}
} else {
extern int enable_shmem_collectives;
enable_shmem_collectives = SMP_INIT;
}
/* Set the eager max msg size now that we know SMP and RDMA are initialized.
* The max message size is also set during VC initialization, but the state
* of SMP is unknown at that time.
*/
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
vc->eager_max_msg_sz = MPIDI_CH3_EAGER_MAX_MSG_SIZE(vc);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
