int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg_p, int pg_rank ) { int mpi_errno = MPI_SUCCESS; char *publish_bc_orig = NULL; char *bc_val = NULL; int val_max_remaining; MPIDI_STATE_DECL(MPID_STATE_MPID_CH3_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MPID_CH3_INIT); mpi_errno = MPIDI_CH3I_Progress_init(); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); /* Initialize the business card */ mpi_errno = MPIDI_CH3I_BCInit( &bc_val, &val_max_remaining ); if (mpi_errno) MPIR_ERR_POP(mpi_errno); publish_bc_orig = bc_val; /* initialize aspects specific to sockets */ mpi_errno = MPIDI_CH3U_Init_sock(has_parent, pg_p, pg_rank, &bc_val, &val_max_remaining); /* Set the connection information in our process group (publish the business card ) */ MPIDI_PG_SetConnInfo( pg_rank, (const char *)publish_bc_orig ); /* Free the business card now that it is published (note that publish_bc_orig is the head of bc_val ) */ MPIDI_CH3I_BCFree( publish_bc_orig ); if (mpi_errno) MPIR_ERR_POP(mpi_errno); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_CH3_INIT); return mpi_errno; fn_fail: if (publish_bc_orig != NULL) { MPL_free(publish_bc_orig); } goto fn_exit; }
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t *pg_p, int pg_rank) { int mpi_errno = MPI_SUCCESS; int i; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_INIT); /* Override split_type */ MPID_Comm_fns = &comm_fns; mpi_errno = MPID_nem_init (pg_rank, pg_p, has_parent); if (mpi_errno) MPIR_ERR_POP (mpi_errno); nemesis_initialized = 1; MPIDI_CH3I_my_rank = pg_rank; MPIDI_CH3I_my_pg = pg_p; /* * Initialize Progress Engine */ mpi_errno = MPIDI_CH3I_Progress_init(); if (mpi_errno) MPIR_ERR_SETFATALANDJUMP (mpi_errno, MPI_ERR_OTHER, "**init_progress"); for (i = 0; i < pg_p->size; i++) { mpi_errno = MPIDI_CH3_VC_Init(&pg_p->vct[i]); if (mpi_errno) MPIR_ERR_POP(mpi_errno); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_INIT); return mpi_errno; fn_fail: goto fn_exit; }
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg, int pg_rank) { int mpi_errno = MPI_SUCCESS; int pg_size, threshold, dpm = 0, p; char *dpm_str, *value, *conn_info = NULL; MPIDI_VC_t *vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_INIT); if (MPIDI_CH3_Pkt_size_index[MPIDI_CH3_PKT_CLOSE] != sizeof (MPIDI_CH3_Pkt_close_t)) { MPIU_ERR_SETFATALANDJUMP1( mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s", "Failed sanity check! Packet size table mismatch"); } pg_size = MPIDI_PG_Get_size(pg); /*Determine to use which connection management*/ threshold = MPIDI_CH3I_CM_DEFAULT_ON_DEMAND_THRESHOLD; /*check ON_DEMAND_THRESHOLD*/ value = getenv("MV2_ON_DEMAND_THRESHOLD"); if (value) { threshold = atoi(value); } dpm_str = getenv("MV2_SUPPORT_DPM"); if (dpm_str) { dpm = !!atoi(dpm_str); } MPIDI_CH3I_Process.has_dpm = dpm; if(MPIDI_CH3I_Process.has_dpm) { setenv("MV2_ENABLE_AFFINITY", "0", 1); } #ifdef _ENABLE_XRC_ value = getenv ("MV2_USE_XRC"); if (value) { USE_XRC = atoi(value); if (USE_XRC) { /* Enable on-demand */ threshold = 1; } } #endif /* _ENABLE_XRC_ */ #ifdef _ENABLE_UD_ if ((value = getenv("MV2_HYBRID_ENABLE_THRESHOLD")) != NULL) { rdma_hybrid_enable_threshold = atoi(value); } if ((value = getenv("MV2_USE_UD_HYBRID")) != NULL) { rdma_enable_hybrid = atoi(value); } if (pg_size < rdma_hybrid_enable_threshold) { rdma_enable_hybrid = 0; } #endif if (pg_size > threshold || dpm #ifdef _ENABLE_XRC_ || USE_XRC #endif /* _ENABLE_XRC_ */ #ifdef _ENABLE_UD_ || rdma_enable_hybrid #endif ) { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND; MPIDI_CH3I_Process.num_conn = 0; } else { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_BASIC_ALL2ALL; } #if defined(RDMA_CM) if (((value = getenv("MV2_USE_RDMA_CM")) != NULL || (value = getenv("MV2_USE_IWARP_MODE")) != NULL) && atoi(value) && ! dpm) { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_RDMA_CM; } else { rdma_cm_get_hca_type(&MPIDI_CH3I_RDMA_Process); } #endif /* defined(RDMA_CM) */ MPIDI_PG_GetConnKVSname(&pg->ch.kvs_name); #if defined(CKPT) #if defined(RDMA_CM) if (MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_RDMA_CM) { MPIU_Error_printf("Error: Checkpointing does not work with RDMA CM.\n" "Please configure and compile MVAPICH2 with checkpointing disabled " "or without support for RDMA CM.\n"); MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail"); } #endif /* defined(RDMA_CM) */ // Always use CM_ON_DEMAND for Checkpoint/Restart and Migration MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND; #endif /* defined(CKPT) */ #ifdef _ENABLE_UD_ if (rdma_enable_hybrid) { MPIU_Assert(MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_ON_DEMAND); } #endif /* save my vc_ptr for easy access */ MPIDI_PG_Get_vc(pg, pg_rank, &MPIDI_CH3I_Process.vc); /* Initialize Progress Engine */ if ((mpi_errno = MPIDI_CH3I_Progress_init())) { MPIU_ERR_POP(mpi_errno); } /* Check for SMP only */ MPIDI_CH3I_set_smp_only(); if (!SMP_ONLY) { switch (MPIDI_CH3I_Process.cm_type) { /* allocate rmda memory and set up the queues */ case MPIDI_CH3I_CM_ON_DEMAND: #if defined(RDMA_CM) case MPIDI_CH3I_CM_RDMA_CM: #endif /* defined(RDMA_CM) */ mpi_errno = MPIDI_CH3I_CM_Init(pg, pg_rank, &conn_info); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } break; default: /*call old init to setup all connections*/ if ((mpi_errno = MPIDI_CH3I_RDMA_init(pg, pg_rank)) != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } /* All vc should be connected */ for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE; } break; } } #if defined(CKPT) #if defined(DISABLE_PTMALLOC) MPIU_Error_printf("Error: Checkpointing does not work without registration " "caching enabled.\nPlease configure and compile MVAPICH2 without checkpointing " " or enable registration caching.\n"); MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail"); #endif /* defined(DISABLE_PTMALLOC) */ if ((mpi_errno = MPIDI_CH3I_CR_Init(pg, pg_rank, pg_size))) { MPIU_ERR_POP(mpi_errno); } #endif /* defined(CKPT) */ /* set connection info for dynamic process management */ if (conn_info && dpm) { mpi_errno = MPIDI_PG_SetConnInfo(pg_rank, (const char *)conn_info); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } } MPIU_Free(conn_info); /* Initialize the smp channel */ if ((mpi_errno = MPIDI_CH3I_SMP_init(pg))) { MPIU_ERR_POP(mpi_errno); } if (SMP_INIT) { for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); /* Mark the SMP VC as Idle */ if (vc->smp.local_nodes >= 0) { vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE; if (SMP_ONLY) { MPIDI_CH3I_SMP_Init_VC(vc); } #ifdef _ENABLE_XRC_ VC_XST_SET (vc, XF_SMP_VC); #endif } } } else { extern int enable_shmem_collectives; enable_shmem_collectives = SMP_INIT; } /* Set the eager max msg size now that we know SMP and RDMA are initialized. * The max message size is also set during VC initialization, but the state * of SMP is unknown at that time. */ for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); vc->eager_max_msg_sz = MPIDI_CH3_EAGER_MAX_MSG_SIZE(vc); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_INIT); return mpi_errno; fn_fail: goto fn_exit; }