void MPID_nem_dbg_print_all_sendq(FILE *stream) { int i; MPIDI_PG_t *pg; MPIDI_VC_t *vc; MPIDI_PG_iterator iter; fprintf(stream, "========================================\n"); fprintf(stream, "MPI_COMM_WORLD ctx=%#x rank=%d\n", MPIR_Process.comm_world->context_id, MPIR_Process.comm_world->rank); fprintf(stream, "MPI_COMM_SELF ctx=%#x\n", MPIR_Process.comm_self->context_id); if (MPIR_Process.comm_parent) { fprintf(stream, "MPI_COMM_PARENT ctx=%#x recvctx=%#x\n", MPIR_Process.comm_self->context_id, MPIR_Process.comm_parent->recvcontext_id); } else { fprintf(stream, "MPI_COMM_PARENT (NULL)\n"); } MPIDI_PG_Get_iterator(&iter); while (MPIDI_PG_Has_next(&iter)) { MPIDI_PG_Get_next(&iter, &pg); fprintf(stream, "PG ptr=%p size=%d id=%s refcount=%d\n", pg, pg->size, (const char*)pg->id, MPIR_Object_get_ref(pg)); for (i = 0; i < MPIDI_PG_Get_size(pg); ++i) { MPIDI_PG_Get_vc(pg, i, &vc); MPID_nem_dbg_print_vc_sendq(stream, vc); } } fprintf(stream, "========================================\n"); }
int MPIDI_CH3U_Init_sock(int has_parent, MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p) { int mpi_errno = MPI_SUCCESS; int pg_size; int p; /* FIXME: Why are these unused? */ MPIU_UNREFERENCED_ARG(has_parent); MPIU_UNREFERENCED_ARG(pg_rank); /* * Initialize the VCs associated with this process group (and thus * MPI_COMM_WORLD) */ /* FIXME: Get the size from the process group */ pg_size = MPIDI_PG_Get_size(pg_p); /* FIXME: This should probably be the same as MPIDI_VC_InitSock. If not, why not? */ /* FIXME: Note that MPIDI_CH3_VC_Init sets state, sendq_head and tail. so this should be MPIDI_CH3_VC_Init( &pg_p->vct[p] ); followed by MPIDI_VC_InitSock( ditto ); In fact, there should be a single VC_Init call here */ /* FIXME: Why isn't this MPIDI_VC_Init( vc, NULL, 0 )? */ for (p = 0; p < pg_size; p++) { MPIDI_CH3I_VC *vcch = &pg_p->vct[p].ch; vcch->sendq_head = NULL; vcch->sendq_tail = NULL; vcch->state = MPIDI_CH3I_VC_STATE_UNCONNECTED; vcch->sock = MPIDU_SOCK_INVALID_SOCK; vcch->conn = NULL; } mpi_errno = MPIDI_CH3U_Get_business_card_sock(pg_rank, bc_val_p, val_max_sz_p); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**init_buscard"); } fn_exit: return mpi_errno; fn_fail: /* FIXME: This doesn't belong here, since the pg is not created in this routine */ /* --BEGIN ERROR HANDLING-- */ if (pg_p != NULL) { MPIDI_PG_Destroy(pg_p); } goto fn_exit; /* --END ERROR HANDLING-- */ }
int MPID_nem_tcp_get_vc_from_conninfo (char *pg_id, int pg_rank, struct MPIDI_VC **vc) { int mpi_errno = MPI_SUCCESS; MPIDI_PG_t *pg; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO); MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "pg_id=%s pg_rank=%d", pg_id, pg_rank)); mpi_errno = MPIDI_PG_Find (pg_id, &pg); if (mpi_errno) MPIU_ERR_POP (mpi_errno); MPIU_ERR_CHKINTERNAL(pg == NULL, mpi_errno, "invalid PG"); MPIU_ERR_CHKINTERNAL(pg_rank < 0 || pg_rank > MPIDI_PG_Get_size (pg), mpi_errno, "invalid pg_rank"); MPIDI_PG_Get_vc_set_active (pg, pg_rank, vc); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO); return mpi_errno; fn_fail: goto fn_exit; }
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg, int pg_rank) { int mpi_errno = MPI_SUCCESS; int pg_size, threshold, dpm = 0, p; char *dpm_str, *value, *conn_info = NULL; MPIDI_VC_t *vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_INIT); if (MPIDI_CH3_Pkt_size_index[MPIDI_CH3_PKT_CLOSE] != sizeof (MPIDI_CH3_Pkt_close_t)) { MPIU_ERR_SETFATALANDJUMP1( mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s", "Failed sanity check! Packet size table mismatch"); } pg_size = MPIDI_PG_Get_size(pg); /*Determine to use which connection management*/ threshold = MPIDI_CH3I_CM_DEFAULT_ON_DEMAND_THRESHOLD; /*check ON_DEMAND_THRESHOLD*/ value = getenv("MV2_ON_DEMAND_THRESHOLD"); if (value) { threshold = atoi(value); } dpm_str = getenv("MV2_SUPPORT_DPM"); if (dpm_str) { dpm = !!atoi(dpm_str); } MPIDI_CH3I_Process.has_dpm = dpm; if(MPIDI_CH3I_Process.has_dpm) { setenv("MV2_ENABLE_AFFINITY", "0", 1); } #ifdef _ENABLE_XRC_ value = getenv ("MV2_USE_XRC"); if (value) { USE_XRC = atoi(value); if (USE_XRC) { /* Enable on-demand */ threshold = 1; } } #endif /* _ENABLE_XRC_ */ #ifdef _ENABLE_UD_ if ((value = getenv("MV2_HYBRID_ENABLE_THRESHOLD")) != NULL) { rdma_hybrid_enable_threshold = atoi(value); } if ((value = getenv("MV2_USE_UD_HYBRID")) != NULL) { rdma_enable_hybrid = atoi(value); } if (pg_size < rdma_hybrid_enable_threshold) { rdma_enable_hybrid = 0; } #endif if (pg_size > threshold || dpm #ifdef _ENABLE_XRC_ || USE_XRC #endif /* _ENABLE_XRC_ */ #ifdef _ENABLE_UD_ || rdma_enable_hybrid #endif ) { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND; MPIDI_CH3I_Process.num_conn = 0; } else { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_BASIC_ALL2ALL; } #if defined(RDMA_CM) if (((value = getenv("MV2_USE_RDMA_CM")) != NULL || (value = getenv("MV2_USE_IWARP_MODE")) != NULL) && atoi(value) && ! dpm) { MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_RDMA_CM; } else { rdma_cm_get_hca_type(&MPIDI_CH3I_RDMA_Process); } #endif /* defined(RDMA_CM) */ MPIDI_PG_GetConnKVSname(&pg->ch.kvs_name); #if defined(CKPT) #if defined(RDMA_CM) if (MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_RDMA_CM) { MPIU_Error_printf("Error: Checkpointing does not work with RDMA CM.\n" "Please configure and compile MVAPICH2 with checkpointing disabled " "or without support for RDMA CM.\n"); MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail"); } #endif /* defined(RDMA_CM) */ // Always use CM_ON_DEMAND for Checkpoint/Restart and Migration MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND; #endif /* defined(CKPT) */ #ifdef _ENABLE_UD_ if (rdma_enable_hybrid) { MPIU_Assert(MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_ON_DEMAND); } #endif /* save my vc_ptr for easy access */ MPIDI_PG_Get_vc(pg, pg_rank, &MPIDI_CH3I_Process.vc); /* Initialize Progress Engine */ if ((mpi_errno = MPIDI_CH3I_Progress_init())) { MPIU_ERR_POP(mpi_errno); } /* Check for SMP only */ MPIDI_CH3I_set_smp_only(); if (!SMP_ONLY) { switch (MPIDI_CH3I_Process.cm_type) { /* allocate rmda memory and set up the queues */ case MPIDI_CH3I_CM_ON_DEMAND: #if defined(RDMA_CM) case MPIDI_CH3I_CM_RDMA_CM: #endif /* defined(RDMA_CM) */ mpi_errno = MPIDI_CH3I_CM_Init(pg, pg_rank, &conn_info); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } break; default: /*call old init to setup all connections*/ if ((mpi_errno = MPIDI_CH3I_RDMA_init(pg, pg_rank)) != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } /* All vc should be connected */ for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE; } break; } } #if defined(CKPT) #if defined(DISABLE_PTMALLOC) MPIU_Error_printf("Error: Checkpointing does not work without registration " "caching enabled.\nPlease configure and compile MVAPICH2 without checkpointing " " or enable registration caching.\n"); MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail"); #endif /* defined(DISABLE_PTMALLOC) */ if ((mpi_errno = MPIDI_CH3I_CR_Init(pg, pg_rank, pg_size))) { MPIU_ERR_POP(mpi_errno); } #endif /* defined(CKPT) */ /* set connection info for dynamic process management */ if (conn_info && dpm) { mpi_errno = MPIDI_PG_SetConnInfo(pg_rank, (const char *)conn_info); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } } MPIU_Free(conn_info); /* Initialize the smp channel */ if ((mpi_errno = MPIDI_CH3I_SMP_init(pg))) { MPIU_ERR_POP(mpi_errno); } if (SMP_INIT) { for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); /* Mark the SMP VC as Idle */ if (vc->smp.local_nodes >= 0) { vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE; if (SMP_ONLY) { MPIDI_CH3I_SMP_Init_VC(vc); } #ifdef _ENABLE_XRC_ VC_XST_SET (vc, XF_SMP_VC); #endif } } } else { extern int enable_shmem_collectives; enable_shmem_collectives = SMP_INIT; } /* Set the eager max msg size now that we know SMP and RDMA are initialized. * The max message size is also set during VC initialization, but the state * of SMP is unknown at that time. */ for (p = 0; p < pg_size; ++p) { MPIDI_PG_Get_vc(pg, p, &vc); vc->eager_max_msg_sz = MPIDI_CH3_EAGER_MAX_MSG_SIZE(vc); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_INIT); return mpi_errno; fn_fail: goto fn_exit; }