/** * \brief Initialize the DCMF ARMCI resources */ int ARMCIX_Init () { DCMF_CriticalSection_enter(0); DCMF_Messager_initialize (); ARMCIX_DCMF_Connection_initialize (); /* Determine request pool defaults */ int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000; ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX); int ARMCIX_DCMF_REQUESTPOOL_INC = 0; ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC); ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC); ARMCIX_DCMF_Get_register (); ARMCIX_DCMF_Put_register (__connection); ARMCIX_DCMF_Acc_register (__connection); ARMCIX_DCMF_Fence_register (__connection); ARMCIX_DCMF_Rmw_register (); /* Determine interrupt mode */ int interrupts = 1; ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts); ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts); DCMF_Configure_t config; memset (&config, 0x00, sizeof(DCMF_Configure_t)); config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON; DCMF_Messager_configure (&config, &config); DCMF_Messager_configure (NULL, &config); //ARMCIX_DCMF_request_print ("after armcix_init"); DCMF_CriticalSection_exit(0); return 0; }
/** \brief Checks the Environment variables at initialization and stores the results */ void MPIDI_Env_setup() { MPIDO_Embedded_Info_Set * properties = &MPIDI_CollectiveProtocols.properties; int dval = 0; char *envopts; /* unset all properties of a mpidi_coll_protocol by default */ MPIDO_INFO_ZERO(properties); /* Set the verbose level */ dval = 0; ENV_Int(getenv("DCMF_VERBOSE"), &dval); MPIDI_Process.verbose = dval; /* Enable the statistics */ dval = 0; ENV_Bool(getenv("DCMF_STATISTICS"), &dval); MPIDI_Process.statistics = dval; /* Determine eager limit */ dval = 1200; ENV_Int(getenv("DCMF_RVZ"), &dval); ENV_Int(getenv("DCMF_RZV"), &dval); ENV_Int(getenv("DCMF_EAGER"), &dval); MPIDI_Process.eager_limit = dval; /* Determine number of outstanding bcast requets (for async protocols) */ dval = 32; ENV_Int(getenv("DCMF_NUMREQUESTS"), &dval); MPIDI_CollectiveProtocols.numrequests = dval; dval = 0; /* Default is not to use optimized rendezvous. */ ENV_Int(getenv("DCMF_OPTRVZ"), &dval); ENV_Int(getenv("DCMF_OPTRZV"), &dval); MPIDI_Process.optrzv_limit = MPIDI_Process.eager_limit + dval; /* Determine interrupt mode */ dval = 0; ENV_Bool(getenv("DCMF_INTERRUPT"), &dval); ENV_Bool(getenv("DCMF_INTERRUPTS"), &dval); MPIDI_Process.use_interrupts = dval; /* Enable Sender-side-matching of messages */ dval = 0; ENV_Bool(getenv("DCMF_SENDER_SIDE_MATCHING"), &dval); ENV_Bool(getenv("DCMF_SSM"), &dval); MPIDI_Process.use_ssm = dval; /* Set the status of the optimized topology functions */ dval = 1; ENV_Bool(getenv("DCMF_TOPOLOGY"), &dval); MPIDI_Process.optimized.topology = dval; /* Set the status of the optimized collectives */ MPIDI_Process.optimized.collectives = 1; MPIDI_Process.optimized.tree = 1; envopts = getenv("DCMF_COLLECTIVE"); if(envopts == NULL) envopts = getenv("DCMF_COLLECTIVES"); if(envopts != NULL) { if(strncasecmp(envopts, "NOT", 3) == 0) /* turn off just tree */ { MPIDI_Process.optimized.collectives = 1; MPIDI_Process.optimized.tree = 0; } else if(atoi(envopts)==0) { MPIDI_Process.optimized.collectives = 0; MPIDI_Process.optimized.tree = 0; } else { MPIDI_Process.optimized.collectives = 1; MPIDI_Process.optimized.tree = 1; } } dval = 1000; ENV_Int(getenv("DCMF_RMA_PENDING"), &dval); MPIDI_Process.rma_pending = dval; /* Initialize selection variables */ /* turn these off in MPIDI_Coll_register if registration fails * or based on env vars (below) */ if(!MPIDI_Process.optimized.collectives) MPIDO_INFO_SET(properties, MPIDO_USE_MPICH); if(!MPIDI_Process.optimized.tree) MPIDO_INFO_SET(properties, MPIDO_USE_NOTREE_OPT_COLLECTIVES); /* default setting of flags for the mpidi_coll_protocol object */ MPIDO_MSET_INFO(properties, MPIDO_USE_GI_BARRIER, MPIDO_USE_CCMI_GI_BARRIER, MPIDO_USE_RECT_BARRIER, MPIDO_USE_BINOM_BARRIER, MPIDO_USE_LOCKBOX_LBARRIER, MPIDO_USE_BINOM_LBARRIER, MPIDO_USE_RECT_DPUT_BCAST, MPIDO_USE_RECT_SINGLETH_BCAST, MPIDO_USE_BINOM_SINGLETH_BCAST, MPIDO_USE_RECT_BCAST, MPIDO_USE_ARECT_BCAST, MPIDO_USE_BINOM_BCAST, MPIDO_USE_ABINOM_BCAST, MPIDO_USE_SCATTER_GATHER_BCAST, MPIDO_USE_TREE_BCAST, MPIDO_USE_CCMI_TREE_BCAST, MPIDO_USE_CCMI_TREE_DPUT_BCAST, MPIDO_USE_STORAGE_ALLREDUCE, MPIDO_USE_RECT_ALLREDUCE, MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE, MPIDO_USE_RECTRING_ALLREDUCE, MPIDO_USE_BINOM_ALLREDUCE, MPIDO_USE_ARECT_ALLREDUCE, MPIDO_USE_ARECTRING_ALLREDUCE, MPIDO_USE_ABINOM_ALLREDUCE, MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE, MPIDO_USE_TREE_ALLREDUCE, MPIDO_USE_PIPELINED_TREE_ALLREDUCE, MPIDO_USE_CCMI_TREE_ALLREDUCE, MPIDO_USE_STORAGE_REDUCE, MPIDO_USE_RECT_REDUCE, MPIDO_USE_RECTRING_REDUCE, MPIDO_USE_BINOM_REDUCE, MPIDO_USE_TREE_REDUCE, MPIDO_USE_ALLREDUCE_ALLGATHER, MPIDO_USE_BCAST_ALLGATHER, MPIDO_USE_ABCAST_ALLGATHER, MPIDO_USE_ALLTOALL_ALLGATHER, MPIDO_USE_PREALLREDUCE_ALLGATHER, MPIDO_USE_ARECT_BCAST_ALLGATHER, MPIDO_USE_ALLREDUCE_ALLGATHERV, MPIDO_USE_BCAST_ALLGATHERV, MPIDO_USE_ABCAST_ALLGATHERV, MPIDO_USE_ALLTOALL_ALLGATHERV, MPIDO_USE_PREALLREDUCE_ALLGATHERV, MPIDO_USE_ARECT_BCAST_ALLGATHERV, MPIDO_USE_ALLTOALL_SCATTERV, MPIDO_USE_PREALLREDUCE_SCATTERV, MPIDO_USE_BCAST_SCATTER, MPIDO_USE_REDUCE_GATHER, MPIDO_USE_REDUCESCATTER, MPIDO_USE_TORUS_ALLTOALL, MPIDO_USE_TORUS_ALLTOALLV, MPIDO_USE_TORUS_ALLTOALLW, MPIDO_USE_PREMALLOC_ALLTOALL, MPIDO_END_ARGS); MPIDI_CollectiveProtocols.bcast_asynccutoff = 8192; MPIDI_CollectiveProtocols.allreduce_asynccutoff = 131072; MPIDI_CollectiveProtocols.numcolors = 0; dval = 8192; ENV_Int(getenv("DCMF_ASYNCCUTOFF"), &dval); MPIDI_CollectiveProtocols.bcast_asynccutoff = dval; envopts = getenv("DCMF_SCATTER"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* mpich */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_SCATTER); MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTER); } } envopts = getenv("DCMF_SCATTERV"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* mpich */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_SCATTERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_SCATTERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTERV); } else if(strncasecmp(envopts, "A", 1) == 0) /* alltoall */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTERV); MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_SCATTERV); } else if(strncasecmp(envopts, "B", 1) == 0) /* bcast */ { MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_SCATTERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_SCATTERV); } } envopts = getenv("DCMF_GATHER"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_GATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_REDUCE_GATHER); } } envopts = getenv("DCMF_REDUCESCATTER"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* mpich */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCESCATTER); MPIDO_INFO_UNSET(properties, MPIDO_USE_REDUCESCATTER); } } envopts = getenv("DCMF_BCAST"); if(envopts != NULL) { MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_DPUT_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_SINGLETH_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_DPUT_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_SINGLETH_BCAST); MPIDO_INFO_UNSET(properties, MPIDO_USE_SCATTER_GATHER_BCAST); MPIDO_INFO_SET(properties, MPIDO_BCAST_ENVVAR); if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BCAST); } else if(strncasecmp(envopts, "AR", 2) == 0) /* async rectangle */ { MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST); } else if(strncasecmp(envopts, "AB", 2) == 0) /* async binomials */ { MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST); } else if(strncasecmp(envopts, "R", 1) == 0) /* Rectangle */ { MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST); } else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */ { MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST); } else if(strncasecmp(envopts, "T", 1) == 0) /* Global Tree */ { MPIDO_INFO_SET(properties, MPIDO_USE_TREE_BCAST); } else if(strncasecmp(envopts, "D", 1) == 0) /* Direct put */ { MPIDO_INFO_SET(properties, MPIDO_USE_RECT_DPUT_BCAST); } else if(strncasecmp(envopts, "SB", 2) == 0) /* Single thread, binomial */ { MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_SINGLETH_BCAST); } else if(strncasecmp(envopts, "SR", 2) == 0) /* Single thread, rect */ { MPIDO_INFO_SET(properties, MPIDO_USE_RECT_SINGLETH_BCAST); } else if(strncasecmp(envopts, "SG", 2) == 0) /* Scatter/gather via pt2pt */ { MPIDO_INFO_SET(properties, MPIDO_USE_SCATTER_GATHER_BCAST); } else if(strncasecmp(envopts, "CD", 2) == 0) /* CCMI Tree dput*/ { MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_DPUT_BCAST); } else if(strncasecmp(envopts, "C", 1) == 0) /* CCMI Tree */ { MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_BCAST); } else { fprintf(stderr, "Valid bcasts are: M, AR, AB, R, B, T, C, CD, D, SR, SB, and SG. Using MPICH\n"); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BCAST); } } envopts = getenv("DCMF_NUMCOLORS"); if(envopts != NULL) { int colors = atoi(envopts); if(colors < 0 || colors > 3) fprintf(stderr,"Invalid DCMF_NUMCOLORS option\n"); else MPIDI_CollectiveProtocols.numcolors = colors; } envopts = getenv("DCMF_SAFEALLGATHER"); if(envopts != NULL) { if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1) MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_ALLGATHER); } envopts = getenv("DCMF_SAFEALLGATHERV"); if(envopts != NULL) { if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1) MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_ALLGATHERV); } envopts = getenv("DCMF_SAFESCATTERV"); if(envopts != NULL) { if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1) MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_SCATTERV); } envopts = getenv("DCMF_ALLTOALL"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALL); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALL); } else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */ ; /* This is on by default in MPIDI_Coll_register */ /* MPIDI_CollectiveProtocols.alltoall.usetorus = 1; */ else fprintf(stderr,"Invalid DCMF_ALLTOALL option\n"); } envopts = getenv("DCMF_ALLTOALL_PREMALLOC"); if(envopts == NULL) envopts = getenv("DCMF_ALLTOALLV_PREMALLOC"); if(envopts == NULL) envopts = getenv("DCMF_ALLTOALLW_PREMALLOC"); if(envopts != NULL) { /* Do not reuse the malloc'd storage */ if(strncasecmp(envopts, "N", 1) == 0) { MPIDO_INFO_UNSET(properties, MPIDO_USE_PREMALLOC_ALLTOALL); } else if(strncasecmp(envopts, "Y", 1) == 0) /* defaults to Y */ { MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_ALLTOALL); } else fprintf(stderr,"Invalid DCMF_ALLTOALL(VW)_PREMALLOC option\n"); } envopts = getenv("DCMF_ALLTOALLV"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALLV); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALLV); } else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */ ; /* This is on by default in MPIDI_Coll_register */ /* MPIDI_CollectiveProtocols.alltoallv.usetorus = 1;*/ else fprintf(stderr,"Invalid DCMF_ALLTOALLV option\n"); } envopts = getenv("DCMF_ALLTOALLW"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALLW); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALLW); } else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */ ; /* This is on by default in MPIDI_Coll_register */ /* MPIDI_CollectiveProtocols.alltoallw.usetorus = 1;*/ else fprintf(stderr,"Invalid DCMF_ALLTOALLW option\n"); } envopts = getenv("DCMF_ALLGATHER"); if(envopts != NULL) { /* The user specified something, so set the env var, then * turn off all options and pick the right one. */ MPIDO_INFO_SET(properties, MPIDO_ALLGATHER_ENVVAR); MPIDO_INFO_UNSET(properties, MPIDO_USE_ABCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_ALLGATHER); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_ALLGATHER); if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHER); } else if(strncasecmp(envopts, "ALLR", 4) == 0) /* ALLREDUCE */ { MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_ALLGATHER); } /* Bcast is never used in the glue, just async bcasts */ else if(strncasecmp(envopts, "B", 1) == 0) /* BCAST */ { MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_ALLGATHER); MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST_ALLGATHER); MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHER); } else if(strncasecmp(envopts, "ALLT", 4) == 0) /* ALLTOALL */ { MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_ALLGATHER); } else if(strncasecmp(envopts, "AS", 2) == 0) /*Async bcast */ { MPIDO_INFO_SET(properties, MPIDO_USE_ABCAST_ALLGATHER); MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHER); MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHER); } else { fprintf(stderr,"DCMF_ALLGATHER options: ALLR, BCAST, ALLT, AS, M\n"); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHER); } } envopts = getenv("DCMF_ALLGATHERV"); if(envopts != NULL) { MPIDO_INFO_UNSET(properties, MPIDO_USE_ABCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_ALLGATHERV); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_ALLGATHERV); MPIDO_INFO_SET(properties, MPIDO_ALLGATHERV_ENVVAR); if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHERV); } else if(strncasecmp(envopts, "ALLR", 4) == 0) /* ALLREDUCE */ { MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_ALLGATHERV); } /* Bcast is never used in the glue, just async bcast now */ else if(strncasecmp(envopts, "B", 1) == 0) /* BCAST */ { MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_ALLGATHERV); MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST_ALLGATHERV); MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHERV); } else if(strncasecmp(envopts, "ALLT", 4) == 0) /* ALLTOALL */ { MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_ALLGATHERV); } else if(strncasecmp(envopts, "AS", 2) == 0) /*Async bcast */ { MPIDO_INFO_SET(properties, MPIDO_USE_ABCAST_ALLGATHERV); MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHERV); MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHERV); } else { fprintf(stderr,"DCMF_ALLGATHERV options: ALLR, BCAST, ALLT, AS, M\n"); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHERV); } } envopts = getenv("DCMF_ALLREDUCE"); if(envopts != NULL) { /* The user has specified something. Turn off all options for now */ /* Then, turn on the specific option they want. If they specify an*/ /* invalid option, use MPICH and let them know */ MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECTRING_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECTRING_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_SHORT_ASYNC_BINOM_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_PIPELINED_TREE_ALLREDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE); MPIDO_INFO_SET(properties, MPIDO_ALLREDUCE_ENVVAR); if(strncasecmp(envopts, "ARI", 3) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_ARECTRING_ALLREDUCE); else if(strncasecmp(envopts, "AR", 2) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_ALLREDUCE); else if(strncasecmp(envopts, "SR", 2) == 0) /* Short async rect */ MPIDO_INFO_SET(properties, MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE); else if(strncasecmp(envopts, "SB", 2) == 0) /* Short async binom */ MPIDO_INFO_SET(properties, MPIDO_USE_SHORT_ASYNC_BINOM_ALLREDUCE); else if(strncasecmp(envopts, "AB", 2) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_ALLREDUCE); else if(strncasecmp(envopts, "RI", 2) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_RECTRING_ALLREDUCE); else if(strncasecmp(envopts, "R", 1) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_RECT_ALLREDUCE); else if(strncasecmp(envopts, "C", 1) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_ALLREDUCE); else if(strncasecmp(envopts, "B", 1) == 0) MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_ALLREDUCE); else if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLREDUCE); } else if(strncasecmp(envopts, "T", 1) == 0) /* Tree */ MPIDO_INFO_SET(properties, MPIDO_USE_TREE_ALLREDUCE); else if(strncasecmp(envopts, "P", 1) == 0) /* CCMI Pipelined Tree */ MPIDO_INFO_SET(properties, MPIDO_USE_PIPELINED_TREE_ALLREDUCE); else if(strncasecmp(envopts, "D", 1) == 0) /* Rect dput */ MPIDO_INFO_SET(properties, MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE); else { fprintf(stderr, "Invalid DCMF_ALLREDUCE option - %s. Using mpich\n", envopts); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLREDUCE); } } envopts = getenv("DCMF_ALLREDUCE_REUSE_STORAGE"); if(envopts != NULL) { if(strncasecmp(envopts, "N", 1) == 0) /* Do not reuse the malloc'd storage */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_STORAGE_ALLREDUCE); /* fprintf(stderr, "N allreduce.reusestorage %X\n", 0); */ } else if (strncasecmp(envopts, "Y", 1) == 0); /* defaults to Y */ else fprintf(stderr,"Invalid DCMF_ALLREDUCE_REUSE_STORAGE option\n"); } envopts = getenv("DCMF_REDUCE_PREMALLOC"); if(envopts != NULL) { if(strncasecmp(envopts, "N", 1) == 0) { MPIDO_INFO_UNSET(properties, MPIDO_USE_PREMALLOC_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_REDUCE); } else if(strncasecmp(envopts, "Y", 1) == 0) /* defaults to Y */ { MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_REDUCE); MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_REDUCE); } else fprintf(stderr,"Invalid DCMF_REDUCE_PREMALLOC option\n"); } envopts = getenv("DCMF_REDUCE"); if(envopts != NULL) { MPIDO_INFO_SET(properties, MPIDO_REDUCE_ENVVAR); MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECTRING_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_REDUCE); MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_REDUCE); if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCE); } else if(strncasecmp(envopts, "RI", 2) == 0) /* Rectangle Ring*/ { MPIDO_INFO_SET(properties, MPIDO_USE_RECTRING_REDUCE); } else if(strncasecmp(envopts, "A", 1) == 0) /* Allreduce */ { MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_REDUCE); MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_REDUCE); } else if(strncasecmp(envopts, "R", 1) == 0) /* Rectangle */ { MPIDO_INFO_SET(properties, MPIDO_USE_RECT_REDUCE); } else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */ { MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_REDUCE); } else if(strncasecmp(envopts, "T", 1) == 0) /* Tree */ { MPIDO_INFO_SET(properties, MPIDO_USE_TREE_REDUCE); } else if(strncasecmp(envopts, "C", 1) == 0) /* CCMI Tree */ { MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_REDUCE); } else { fprintf(stderr,"Invalid DCMF_REDUCE option. Defaulting to MPICH\n"); MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCE); } } envopts = getenv("DCMF_REDUCE_REUSE_STORAGE"); if(envopts != NULL) { if(strncasecmp(envopts, "N", 1) == 0) /* Do not reuse the malloc'd storage */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_STORAGE_REDUCE); /*fprintf(stderr, "N protocol.reusestorage %X\n", 0);*/ } else if(strncasecmp(envopts, "Y", 1) == 0); /* defaults to Y */ else fprintf(stderr,"Invalid DCMF_REDUCE_REUSE_STORAGE option\n"); } envopts = getenv("DCMF_BARRIER"); if(envopts != NULL) { if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */ { MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BARRIER); /* still need to register a barrier for DCMF collectives */ MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BARRIER); /* MPIDI_Coll_register changes this state for us */ /* MPIDI_CollectiveProtocols.barrier.usegi = 1; */ } else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER); } else if(strncasecmp(envopts, "G", 1) == 0) /* GI */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER); } else if(strncasecmp(envopts, "R", 1) == 0) /* Rect */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER); } else if(strncasecmp(envopts, "C", 1) == 0) /* GI */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER); MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER); } else fprintf(stderr,"Invalid DCMF_BARRIER option\n"); } envopts = getenv("DCMF_LOCALBARRIER"); if(envopts != NULL) { if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_LOCKBOX_LBARRIER); } else if(strncasecmp(envopts, "L", 1) == 0) /* Lockbox */ { MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_LBARRIER); } else fprintf(stderr,"Invalid DCMF_LOCALBARRIER option\n"); } /* star-mpi is off by default unless user requests it */ dval = 0; ENV_Int(getenv("DCMF_STAR"), &dval); if (dval > 0 ) { STAR_info.enabled = dval; /* initialize the repositories of STAR */ STAR_InitRepositories(); } dval = 0; ENV_Int(getenv("DCMF_STAR_NUM_INVOCS"), &dval); if (dval > 0 ) STAR_info.invocs_per_algorithm = dval; dval = 0; ENV_Int(getenv("DCMF_STAR_TRACEBACK_LEVEL"), &dval); if (dval > 0) STAR_info.traceback_levels = dval; dval = 0; ENV_Int(getenv("DCMF_STAR_CHECK_CALLSITE"), &dval); if (dval > 0) STAR_info.agree_on_callsite = dval; ENV_Int(getenv("DCMF_STAR_VERBOSE"), &dval); if (STAR_info.enabled && dval > 0) STAR_info.debug = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_ALLTOALL_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.alltoall_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_ALLGATHER_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.allgather_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_ALLGATHERV_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.allgatherv_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_ALLREDUCE_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.allreduce_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_REDUCE_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.reduce_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_BCAST_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.bcast_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_SCATTER_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.scatter_threshold = dval; dval = -1; ENV_Int(getenv("DCMF_STAR_GATHER_THRESHOLD"), &dval); if (dval >= 0 ) STAR_info.gather_threshold = dval; /* end of star stuff */ }
/** * \brief Initialize the DCMF ARMCI resources */ int ARMCIX_Init () { DCMF_CriticalSection_enter(0); DCMF_Messager_initialize (); ARMCIX_DCMF_Connection_initialize (); /* Determine request pool defaults */ int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000; ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX); int ARMCIX_DCMF_REQUESTPOOL_INC = 0; ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC); ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC); ARMCIX_DCMF_Get_register (); ARMCIX_DCMF_Put_register (__connection); ARMCIX_DCMF_Acc_register (__connection); ARMCIX_DCMF_Fence_register (__connection); ARMCIX_DCMF_Rmw_register (); /* Initializer helper thread or configure interrupt mode */ int interrupts = 0; ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts); ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts); //fprintf(stdout,"interrupts = %d\n",interrupts); /*if (interrupts==1){ if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts ON\n"); } else { if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts OFF\n"); }*/ if (interrupts==0) { int ret = pthread_create(&armcix_advance_thread, NULL, armcix_advance, NULL); if ( ret != 0 ) { if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create failed\n"); armcix_advance_active = 0; } else { if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create succeeded\n"); armcix_advance_active = 1; } } DCMF_Configure_t config; memset (&config, 0x00, sizeof(DCMF_Configure_t)); config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON; DCMF_Messager_configure (&config, &config); DCMF_Messager_configure (NULL, &config); //ARMCIX_DCMF_request_print ("after armcix_init"); DCMF_CriticalSection_exit(0); return 0; }