void MPID_nem_dbg_print_all_sendq(FILE *stream)
{
int i;
MPIDI_PG_t *pg;
MPIDI_VC_t *vc;
MPIDI_PG_iterator iter;
fprintf(stream, "========================================\n");
fprintf(stream, "MPI_COMM_WORLD ctx=%#x rank=%d\n", MPIR_Process.comm_world->context_id, MPIR_Process.comm_world->rank);
fprintf(stream, "MPI_COMM_SELF ctx=%#x\n", MPIR_Process.comm_self->context_id);
if (MPIR_Process.comm_parent) {
fprintf(stream, "MPI_COMM_PARENT ctx=%#x recvctx=%#x\n",
MPIR_Process.comm_self->context_id,
MPIR_Process.comm_parent->recvcontext_id);
}
else {
fprintf(stream, "MPI_COMM_PARENT (NULL)\n");
}
MPIDI_PG_Get_iterator(&iter);
while (MPIDI_PG_Has_next(&iter)) {
MPIDI_PG_Get_next(&iter, &pg);
fprintf(stream, "PG ptr=%p size=%d id=%s refcount=%d\n", pg, pg->size, (const char*)pg->id, MPIR_Object_get_ref(pg));
for (i = 0; i < MPIDI_PG_Get_size(pg); ++i) {
MPIDI_PG_Get_vc(pg, i, &vc);
MPID_nem_dbg_print_vc_sendq(stream, vc);
}
}
fprintf(stream, "========================================\n");
}
int MPIDI_CH3U_Init_sock(int has_parent, MPIDI_PG_t *pg_p, int pg_rank,
char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
int pg_size;
int p;
/* FIXME: Why are these unused? */
MPIU_UNREFERENCED_ARG(has_parent);
MPIU_UNREFERENCED_ARG(pg_rank);
/*
* Initialize the VCs associated with this process group (and thus
* MPI_COMM_WORLD)
*/
/* FIXME: Get the size from the process group */
pg_size = MPIDI_PG_Get_size(pg_p);
/* FIXME: This should probably be the same as MPIDI_VC_InitSock. If
not, why not? */
/* FIXME: Note that MPIDI_CH3_VC_Init sets state, sendq_head and tail.
so this should be MPIDI_CH3_VC_Init( &pg_p->vct[p] );
followed by MPIDI_VC_InitSock( ditto );
In fact, there should be a single VC_Init call here */
/* FIXME: Why isn't this MPIDI_VC_Init( vc, NULL, 0 )? */
for (p = 0; p < pg_size; p++)
{
MPIDI_CH3I_VC *vcch = &pg_p->vct[p].ch;
vcch->sendq_head = NULL;
vcch->sendq_tail = NULL;
vcch->state = MPIDI_CH3I_VC_STATE_UNCONNECTED;
vcch->sock = MPIDU_SOCK_INVALID_SOCK;
vcch->conn = NULL;
}
mpi_errno = MPIDI_CH3U_Get_business_card_sock(pg_rank,
bc_val_p, val_max_sz_p);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**init_buscard");
}
fn_exit:
return mpi_errno;
fn_fail:
/* FIXME: This doesn't belong here, since the pg is not created in
this routine */
/* --BEGIN ERROR HANDLING-- */
if (pg_p != NULL)
{
MPIDI_PG_Destroy(pg_p);
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_tcp_get_vc_from_conninfo (char *pg_id, int pg_rank, struct MPIDI_VC **vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_PG_t *pg;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "pg_id=%s pg_rank=%d", pg_id, pg_rank));
mpi_errno = MPIDI_PG_Find (pg_id, &pg);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
MPIU_ERR_CHKINTERNAL(pg == NULL, mpi_errno, "invalid PG");
MPIU_ERR_CHKINTERNAL(pg_rank < 0 || pg_rank > MPIDI_PG_Get_size (pg), mpi_errno, "invalid pg_rank");
MPIDI_PG_Get_vc_set_active (pg, pg_rank, vc);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg, int pg_rank)
{
int mpi_errno = MPI_SUCCESS;
int pg_size, threshold, dpm = 0, p;
char *dpm_str, *value, *conn_info = NULL;
MPIDI_VC_t *vc;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_INIT);
if (MPIDI_CH3_Pkt_size_index[MPIDI_CH3_PKT_CLOSE] != sizeof (MPIDI_CH3_Pkt_close_t))
{
MPIU_ERR_SETFATALANDJUMP1(
mpi_errno,
MPI_ERR_OTHER,
"**fail",
"**fail %s",
"Failed sanity check! Packet size table mismatch");
}
pg_size = MPIDI_PG_Get_size(pg);
/*Determine to use which connection management*/
threshold = MPIDI_CH3I_CM_DEFAULT_ON_DEMAND_THRESHOLD;
/*check ON_DEMAND_THRESHOLD*/
value = getenv("MV2_ON_DEMAND_THRESHOLD");
if (value)
{
threshold = atoi(value);
}
dpm_str = getenv("MV2_SUPPORT_DPM");
if (dpm_str) {
dpm = !!atoi(dpm_str);
}
MPIDI_CH3I_Process.has_dpm = dpm;
if(MPIDI_CH3I_Process.has_dpm) {
setenv("MV2_ENABLE_AFFINITY", "0", 1);
}
#ifdef _ENABLE_XRC_
value = getenv ("MV2_USE_XRC");
if (value) {
USE_XRC = atoi(value);
if (USE_XRC) {
/* Enable on-demand */
threshold = 1;
}
}
#endif /* _ENABLE_XRC_ */
#ifdef _ENABLE_UD_
if ((value = getenv("MV2_HYBRID_ENABLE_THRESHOLD")) != NULL) {
rdma_hybrid_enable_threshold = atoi(value);
}
if ((value = getenv("MV2_USE_UD_HYBRID")) != NULL) {
rdma_enable_hybrid = atoi(value);
}
if (pg_size < rdma_hybrid_enable_threshold) {
rdma_enable_hybrid = 0;
}
#endif
if (pg_size > threshold || dpm
#ifdef _ENABLE_XRC_
|| USE_XRC
#endif /* _ENABLE_XRC_ */
#ifdef _ENABLE_UD_
|| rdma_enable_hybrid
#endif
)
{
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND;
MPIDI_CH3I_Process.num_conn = 0;
}
else
{
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_BASIC_ALL2ALL;
}
#if defined(RDMA_CM)
if (((value = getenv("MV2_USE_RDMA_CM")) != NULL
|| (value = getenv("MV2_USE_IWARP_MODE")) != NULL)
&& atoi(value) && ! dpm) {
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_RDMA_CM;
} else {
rdma_cm_get_hca_type(&MPIDI_CH3I_RDMA_Process);
}
#endif /* defined(RDMA_CM) */
MPIDI_PG_GetConnKVSname(&pg->ch.kvs_name);
#if defined(CKPT)
#if defined(RDMA_CM)
if (MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_RDMA_CM)
{
MPIU_Error_printf("Error: Checkpointing does not work with RDMA CM.\n"
"Please configure and compile MVAPICH2 with checkpointing disabled "
"or without support for RDMA CM.\n");
MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail");
}
#endif /* defined(RDMA_CM) */
// Always use CM_ON_DEMAND for Checkpoint/Restart and Migration
MPIDI_CH3I_Process.cm_type = MPIDI_CH3I_CM_ON_DEMAND;
#endif /* defined(CKPT) */
#ifdef _ENABLE_UD_
if (rdma_enable_hybrid) {
MPIU_Assert(MPIDI_CH3I_Process.cm_type == MPIDI_CH3I_CM_ON_DEMAND);
}
#endif
/* save my vc_ptr for easy access */
MPIDI_PG_Get_vc(pg, pg_rank, &MPIDI_CH3I_Process.vc);
/* Initialize Progress Engine */
if ((mpi_errno = MPIDI_CH3I_Progress_init()))
{
MPIU_ERR_POP(mpi_errno);
}
/* Check for SMP only */
MPIDI_CH3I_set_smp_only();
if (!SMP_ONLY)
{
switch (MPIDI_CH3I_Process.cm_type)
{
/* allocate rmda memory and set up the queues */
case MPIDI_CH3I_CM_ON_DEMAND:
#if defined(RDMA_CM)
case MPIDI_CH3I_CM_RDMA_CM:
#endif /* defined(RDMA_CM) */
mpi_errno = MPIDI_CH3I_CM_Init(pg, pg_rank, &conn_info);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
break;
default:
/*call old init to setup all connections*/
if ((mpi_errno = MPIDI_CH3I_RDMA_init(pg, pg_rank)) != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
/* All vc should be connected */
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE;
}
break;
}
}
#if defined(CKPT)
#if defined(DISABLE_PTMALLOC)
MPIU_Error_printf("Error: Checkpointing does not work without registration "
"caching enabled.\nPlease configure and compile MVAPICH2 without checkpointing "
" or enable registration caching.\n");
MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**fail");
#endif /* defined(DISABLE_PTMALLOC) */
if ((mpi_errno = MPIDI_CH3I_CR_Init(pg, pg_rank, pg_size)))
{
MPIU_ERR_POP(mpi_errno);
}
#endif /* defined(CKPT) */
/* set connection info for dynamic process management */
if (conn_info && dpm) {
mpi_errno = MPIDI_PG_SetConnInfo(pg_rank, (const char *)conn_info);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
}
MPIU_Free(conn_info);
/* Initialize the smp channel */
if ((mpi_errno = MPIDI_CH3I_SMP_init(pg)))
{
MPIU_ERR_POP(mpi_errno);
}
if (SMP_INIT)
{
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
/* Mark the SMP VC as Idle */
if (vc->smp.local_nodes >= 0)
{
vc->ch.state = MPIDI_CH3I_VC_STATE_IDLE;
if (SMP_ONLY) {
MPIDI_CH3I_SMP_Init_VC(vc);
}
#ifdef _ENABLE_XRC_
VC_XST_SET (vc, XF_SMP_VC);
#endif
}
}
} else {
extern int enable_shmem_collectives;
enable_shmem_collectives = SMP_INIT;
}
/* Set the eager max msg size now that we know SMP and RDMA are initialized.
* The max message size is also set during VC initialization, but the state
* of SMP is unknown at that time.
*/
for (p = 0; p < pg_size; ++p)
{
MPIDI_PG_Get_vc(pg, p, &vc);
vc->eager_max_msg_sz = MPIDI_CH3_EAGER_MAX_MSG_SIZE(vc);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
