/**
* \brief Initialize the DCMF ARMCI resources
*/
int ARMCIX_Init ()
{
DCMF_CriticalSection_enter(0);
DCMF_Messager_initialize ();
ARMCIX_DCMF_Connection_initialize ();
/* Determine request pool defaults */
int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX);
int ARMCIX_DCMF_REQUESTPOOL_INC = 0;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_Get_register ();
ARMCIX_DCMF_Put_register (__connection);
ARMCIX_DCMF_Acc_register (__connection);
ARMCIX_DCMF_Fence_register (__connection);
ARMCIX_DCMF_Rmw_register ();
/* Determine interrupt mode */
int interrupts = 1;
ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts);
ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts);
DCMF_Configure_t config;
memset (&config, 0x00, sizeof(DCMF_Configure_t));
config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON;
DCMF_Messager_configure (&config, &config);
DCMF_Messager_configure (NULL, &config);
//ARMCIX_DCMF_request_print ("after armcix_init");
DCMF_CriticalSection_exit(0);
return 0;
}
/** \brief Checks the Environment variables at initialization and stores the results */
void
MPIDI_Env_setup()
{
MPIDO_Embedded_Info_Set * properties = &MPIDI_CollectiveProtocols.properties;
int dval = 0;
char *envopts;
/* unset all properties of a mpidi_coll_protocol by default */
MPIDO_INFO_ZERO(properties);
/* Set the verbose level */
dval = 0;
ENV_Int(getenv("DCMF_VERBOSE"), &dval);
MPIDI_Process.verbose = dval;
/* Enable the statistics */
dval = 0;
ENV_Bool(getenv("DCMF_STATISTICS"), &dval);
MPIDI_Process.statistics = dval;
/* Determine eager limit */
dval = 1200;
ENV_Int(getenv("DCMF_RVZ"), &dval);
ENV_Int(getenv("DCMF_RZV"), &dval);
ENV_Int(getenv("DCMF_EAGER"), &dval);
MPIDI_Process.eager_limit = dval;
/* Determine number of outstanding bcast requets (for async protocols) */
dval = 32;
ENV_Int(getenv("DCMF_NUMREQUESTS"), &dval);
MPIDI_CollectiveProtocols.numrequests = dval;
dval = 0; /* Default is not to use optimized rendezvous. */
ENV_Int(getenv("DCMF_OPTRVZ"), &dval);
ENV_Int(getenv("DCMF_OPTRZV"), &dval);
MPIDI_Process.optrzv_limit = MPIDI_Process.eager_limit + dval;
/* Determine interrupt mode */
dval = 0;
ENV_Bool(getenv("DCMF_INTERRUPT"), &dval);
ENV_Bool(getenv("DCMF_INTERRUPTS"), &dval);
MPIDI_Process.use_interrupts = dval;
/* Enable Sender-side-matching of messages */
dval = 0;
ENV_Bool(getenv("DCMF_SENDER_SIDE_MATCHING"), &dval);
ENV_Bool(getenv("DCMF_SSM"), &dval);
MPIDI_Process.use_ssm = dval;
/* Set the status of the optimized topology functions */
dval = 1;
ENV_Bool(getenv("DCMF_TOPOLOGY"), &dval);
MPIDI_Process.optimized.topology = dval;
/* Set the status of the optimized collectives */
MPIDI_Process.optimized.collectives = 1;
MPIDI_Process.optimized.tree = 1;
envopts = getenv("DCMF_COLLECTIVE");
if(envopts == NULL)
envopts = getenv("DCMF_COLLECTIVES");
if(envopts != NULL)
{
if(strncasecmp(envopts, "NOT", 3) == 0) /* turn off just tree */
{
MPIDI_Process.optimized.collectives = 1;
MPIDI_Process.optimized.tree = 0;
}
else if(atoi(envopts)==0)
{
MPIDI_Process.optimized.collectives = 0;
MPIDI_Process.optimized.tree = 0;
}
else
{
MPIDI_Process.optimized.collectives = 1;
MPIDI_Process.optimized.tree = 1;
}
}
dval = 1000;
ENV_Int(getenv("DCMF_RMA_PENDING"), &dval);
MPIDI_Process.rma_pending = dval;
/* Initialize selection variables */
/* turn these off in MPIDI_Coll_register if registration fails
* or based on env vars (below) */
if(!MPIDI_Process.optimized.collectives)
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH);
if(!MPIDI_Process.optimized.tree)
MPIDO_INFO_SET(properties, MPIDO_USE_NOTREE_OPT_COLLECTIVES);
/* default setting of flags for the mpidi_coll_protocol object */
MPIDO_MSET_INFO(properties,
MPIDO_USE_GI_BARRIER,
MPIDO_USE_CCMI_GI_BARRIER,
MPIDO_USE_RECT_BARRIER,
MPIDO_USE_BINOM_BARRIER,
MPIDO_USE_LOCKBOX_LBARRIER,
MPIDO_USE_BINOM_LBARRIER,
MPIDO_USE_RECT_DPUT_BCAST,
MPIDO_USE_RECT_SINGLETH_BCAST,
MPIDO_USE_BINOM_SINGLETH_BCAST,
MPIDO_USE_RECT_BCAST,
MPIDO_USE_ARECT_BCAST,
MPIDO_USE_BINOM_BCAST,
MPIDO_USE_ABINOM_BCAST,
MPIDO_USE_SCATTER_GATHER_BCAST,
MPIDO_USE_TREE_BCAST,
MPIDO_USE_CCMI_TREE_BCAST,
MPIDO_USE_CCMI_TREE_DPUT_BCAST,
MPIDO_USE_STORAGE_ALLREDUCE,
MPIDO_USE_RECT_ALLREDUCE,
MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE,
MPIDO_USE_RECTRING_ALLREDUCE,
MPIDO_USE_BINOM_ALLREDUCE,
MPIDO_USE_ARECT_ALLREDUCE,
MPIDO_USE_ARECTRING_ALLREDUCE,
MPIDO_USE_ABINOM_ALLREDUCE,
MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE,
MPIDO_USE_TREE_ALLREDUCE,
MPIDO_USE_PIPELINED_TREE_ALLREDUCE,
MPIDO_USE_CCMI_TREE_ALLREDUCE,
MPIDO_USE_STORAGE_REDUCE,
MPIDO_USE_RECT_REDUCE,
MPIDO_USE_RECTRING_REDUCE,
MPIDO_USE_BINOM_REDUCE,
MPIDO_USE_TREE_REDUCE,
MPIDO_USE_ALLREDUCE_ALLGATHER,
MPIDO_USE_BCAST_ALLGATHER,
MPIDO_USE_ABCAST_ALLGATHER,
MPIDO_USE_ALLTOALL_ALLGATHER,
MPIDO_USE_PREALLREDUCE_ALLGATHER,
MPIDO_USE_ARECT_BCAST_ALLGATHER,
MPIDO_USE_ALLREDUCE_ALLGATHERV,
MPIDO_USE_BCAST_ALLGATHERV,
MPIDO_USE_ABCAST_ALLGATHERV,
MPIDO_USE_ALLTOALL_ALLGATHERV,
MPIDO_USE_PREALLREDUCE_ALLGATHERV,
MPIDO_USE_ARECT_BCAST_ALLGATHERV,
MPIDO_USE_ALLTOALL_SCATTERV,
MPIDO_USE_PREALLREDUCE_SCATTERV,
MPIDO_USE_BCAST_SCATTER,
MPIDO_USE_REDUCE_GATHER,
MPIDO_USE_REDUCESCATTER,
MPIDO_USE_TORUS_ALLTOALL,
MPIDO_USE_TORUS_ALLTOALLV,
MPIDO_USE_TORUS_ALLTOALLW,
MPIDO_USE_PREMALLOC_ALLTOALL,
MPIDO_END_ARGS);
MPIDI_CollectiveProtocols.bcast_asynccutoff = 8192;
MPIDI_CollectiveProtocols.allreduce_asynccutoff = 131072;
MPIDI_CollectiveProtocols.numcolors = 0;
dval = 8192;
ENV_Int(getenv("DCMF_ASYNCCUTOFF"), &dval);
MPIDI_CollectiveProtocols.bcast_asynccutoff = dval;
envopts = getenv("DCMF_SCATTER");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* mpich */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_SCATTER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTER);
}
}
envopts = getenv("DCMF_SCATTERV");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* mpich */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_SCATTERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_SCATTERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTERV);
}
else if(strncasecmp(envopts, "A", 1) == 0) /* alltoall */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_SCATTERV);
MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_SCATTERV);
}
else if(strncasecmp(envopts, "B", 1) == 0) /* bcast */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_SCATTERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_SCATTERV);
}
}
envopts = getenv("DCMF_GATHER");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0)
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_GATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_REDUCE_GATHER);
}
}
envopts = getenv("DCMF_REDUCESCATTER");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* mpich */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCESCATTER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_REDUCESCATTER);
}
}
envopts = getenv("DCMF_BCAST");
if(envopts != NULL)
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_DPUT_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_SINGLETH_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_DPUT_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_SINGLETH_BCAST);
MPIDO_INFO_UNSET(properties, MPIDO_USE_SCATTER_GATHER_BCAST);
MPIDO_INFO_SET(properties, MPIDO_BCAST_ENVVAR);
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BCAST);
}
else if(strncasecmp(envopts, "AR", 2) == 0) /* async rectangle */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST);
}
else if(strncasecmp(envopts, "AB", 2) == 0) /* async binomials */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST);
}
else if(strncasecmp(envopts, "R", 1) == 0) /* Rectangle */
{
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST);
}
else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Global Tree */
{
MPIDO_INFO_SET(properties, MPIDO_USE_TREE_BCAST);
}
else if(strncasecmp(envopts, "D", 1) == 0) /* Direct put */
{
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_DPUT_BCAST);
}
else if(strncasecmp(envopts, "SB", 2) == 0) /* Single thread, binomial */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_SINGLETH_BCAST);
}
else if(strncasecmp(envopts, "SR", 2) == 0) /* Single thread, rect */
{
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_SINGLETH_BCAST);
}
else if(strncasecmp(envopts, "SG", 2) == 0) /* Scatter/gather via pt2pt */
{
MPIDO_INFO_SET(properties, MPIDO_USE_SCATTER_GATHER_BCAST);
}
else if(strncasecmp(envopts, "CD", 2) == 0) /* CCMI Tree dput*/
{
MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_DPUT_BCAST);
}
else if(strncasecmp(envopts, "C", 1) == 0) /* CCMI Tree */
{
MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_BCAST);
}
else
{
fprintf(stderr,
"Valid bcasts are: M, AR, AB, R, B, T, C, CD, D, SR, SB, and SG. Using MPICH\n");
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BCAST);
}
}
envopts = getenv("DCMF_NUMCOLORS");
if(envopts != NULL)
{
int colors = atoi(envopts);
if(colors < 0 || colors > 3)
fprintf(stderr,"Invalid DCMF_NUMCOLORS option\n");
else
MPIDI_CollectiveProtocols.numcolors = colors;
}
envopts = getenv("DCMF_SAFEALLGATHER");
if(envopts != NULL)
{
if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1)
MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_ALLGATHER);
}
envopts = getenv("DCMF_SAFEALLGATHERV");
if(envopts != NULL)
{
if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1)
MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_ALLGATHERV);
}
envopts = getenv("DCMF_SAFESCATTERV");
if(envopts != NULL)
{
if((strncasecmp(envopts, "Y", 1) == 0) || atoi(envopts)==1)
MPIDO_INFO_UNSET(properties, MPIDO_USE_PREALLREDUCE_SCATTERV);
}
envopts = getenv("DCMF_ALLTOALL");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALL);
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALL);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */
;
/* This is on by default in MPIDI_Coll_register */
/* MPIDI_CollectiveProtocols.alltoall.usetorus = 1; */
else
fprintf(stderr,"Invalid DCMF_ALLTOALL option\n");
}
envopts = getenv("DCMF_ALLTOALL_PREMALLOC");
if(envopts == NULL)
envopts = getenv("DCMF_ALLTOALLV_PREMALLOC");
if(envopts == NULL)
envopts = getenv("DCMF_ALLTOALLW_PREMALLOC");
if(envopts != NULL)
{ /* Do not reuse the malloc'd storage */
if(strncasecmp(envopts, "N", 1) == 0)
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_PREMALLOC_ALLTOALL);
}
else if(strncasecmp(envopts, "Y", 1) == 0) /* defaults to Y */
{
MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_ALLTOALL);
}
else
fprintf(stderr,"Invalid DCMF_ALLTOALL(VW)_PREMALLOC option\n");
}
envopts = getenv("DCMF_ALLTOALLV");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALLV);
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALLV);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */
;
/* This is on by default in MPIDI_Coll_register */
/* MPIDI_CollectiveProtocols.alltoallv.usetorus = 1;*/
else
fprintf(stderr,"Invalid DCMF_ALLTOALLV option\n");
}
envopts = getenv("DCMF_ALLTOALLW");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_TORUS_ALLTOALLW);
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLTOALLW);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Torus */
;
/* This is on by default in MPIDI_Coll_register */
/* MPIDI_CollectiveProtocols.alltoallw.usetorus = 1;*/
else
fprintf(stderr,"Invalid DCMF_ALLTOALLW option\n");
}
envopts = getenv("DCMF_ALLGATHER");
if(envopts != NULL)
{
/* The user specified something, so set the env var, then
* turn off all options and pick the right one. */
MPIDO_INFO_SET(properties, MPIDO_ALLGATHER_ENVVAR);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_ALLGATHER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_ALLGATHER);
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHER);
}
else if(strncasecmp(envopts, "ALLR", 4) == 0) /* ALLREDUCE */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_ALLGATHER);
}
/* Bcast is never used in the glue, just async bcasts */
else if(strncasecmp(envopts, "B", 1) == 0) /* BCAST */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_ALLGATHER);
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST_ALLGATHER);
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHER);
}
else if(strncasecmp(envopts, "ALLT", 4) == 0) /* ALLTOALL */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_ALLGATHER);
}
else if(strncasecmp(envopts, "AS", 2) == 0) /*Async bcast */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ABCAST_ALLGATHER);
MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHER);
MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHER);
}
else
{
fprintf(stderr,"DCMF_ALLGATHER options: ALLR, BCAST, ALLT, AS, M\n");
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHER);
}
}
envopts = getenv("DCMF_ALLGATHERV");
if(envopts != NULL)
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLTOALL_ALLGATHERV);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_ALLGATHERV);
MPIDO_INFO_SET(properties, MPIDO_ALLGATHERV_ENVVAR);
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHERV);
}
else if(strncasecmp(envopts, "ALLR", 4) == 0) /* ALLREDUCE */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_ALLGATHERV);
}
/* Bcast is never used in the glue, just async bcast now */
else if(strncasecmp(envopts, "B", 1) == 0) /* BCAST */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BCAST_ALLGATHERV);
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_BCAST_ALLGATHERV);
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BCAST_ALLGATHERV);
}
else if(strncasecmp(envopts, "ALLT", 4) == 0) /* ALLTOALL */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ALLTOALL_ALLGATHERV);
}
else if(strncasecmp(envopts, "AS", 2) == 0) /*Async bcast */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ABCAST_ALLGATHERV);
MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_BCAST_ALLGATHERV);
MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_BCAST_ALLGATHERV);
}
else
{
fprintf(stderr,"DCMF_ALLGATHERV options: ALLR, BCAST, ALLT, AS, M\n");
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLGATHERV);
}
}
envopts = getenv("DCMF_ALLREDUCE");
if(envopts != NULL)
{
/* The user has specified something. Turn off all options for now */
/* Then, turn on the specific option they want. If they specify an*/
/* invalid option, use MPICH and let them know */
MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECT_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ABINOM_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ARECTRING_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECTRING_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_SHORT_ASYNC_BINOM_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_PIPELINED_TREE_ALLREDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE);
MPIDO_INFO_SET(properties, MPIDO_ALLREDUCE_ENVVAR);
if(strncasecmp(envopts, "ARI", 3) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_ARECTRING_ALLREDUCE);
else if(strncasecmp(envopts, "AR", 2) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_ARECT_ALLREDUCE);
else if(strncasecmp(envopts, "SR", 2) == 0) /* Short async rect */
MPIDO_INFO_SET(properties, MPIDO_USE_SHORT_ASYNC_RECT_ALLREDUCE);
else if(strncasecmp(envopts, "SB", 2) == 0) /* Short async binom */
MPIDO_INFO_SET(properties, MPIDO_USE_SHORT_ASYNC_BINOM_ALLREDUCE);
else if(strncasecmp(envopts, "AB", 2) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_ABINOM_ALLREDUCE);
else if(strncasecmp(envopts, "RI", 2) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_RECTRING_ALLREDUCE);
else if(strncasecmp(envopts, "R", 1) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_ALLREDUCE);
else if(strncasecmp(envopts, "C", 1) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_ALLREDUCE);
else if(strncasecmp(envopts, "B", 1) == 0)
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_ALLREDUCE);
else if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLREDUCE);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Tree */
MPIDO_INFO_SET(properties, MPIDO_USE_TREE_ALLREDUCE);
else if(strncasecmp(envopts, "P", 1) == 0) /* CCMI Pipelined Tree */
MPIDO_INFO_SET(properties, MPIDO_USE_PIPELINED_TREE_ALLREDUCE);
else if(strncasecmp(envopts, "D", 1) == 0) /* Rect dput */
MPIDO_INFO_SET(properties, MPIDO_USE_RRING_DPUT_SINGLETH_ALLREDUCE);
else
{
fprintf(stderr,
"Invalid DCMF_ALLREDUCE option - %s. Using mpich\n", envopts);
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_ALLREDUCE);
}
}
envopts = getenv("DCMF_ALLREDUCE_REUSE_STORAGE");
if(envopts != NULL)
{
if(strncasecmp(envopts, "N", 1) == 0) /* Do not reuse the malloc'd storage */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_STORAGE_ALLREDUCE);
/* fprintf(stderr, "N allreduce.reusestorage %X\n", 0); */
}
else if (strncasecmp(envopts, "Y", 1) == 0); /* defaults to Y */
else
fprintf(stderr,"Invalid DCMF_ALLREDUCE_REUSE_STORAGE option\n");
}
envopts = getenv("DCMF_REDUCE_PREMALLOC");
if(envopts != NULL)
{
if(strncasecmp(envopts, "N", 1) == 0)
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_PREMALLOC_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_REDUCE);
}
else if(strncasecmp(envopts, "Y", 1) == 0) /* defaults to Y */
{
MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_REDUCE);
MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_REDUCE);
}
else
fprintf(stderr,"Invalid DCMF_REDUCE_PREMALLOC option\n");
}
envopts = getenv("DCMF_REDUCE");
if(envopts != NULL)
{
MPIDO_INFO_SET(properties, MPIDO_REDUCE_ENVVAR);
MPIDO_INFO_UNSET(properties, MPIDO_USE_ALLREDUCE_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_TREE_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECTRING_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_REDUCE);
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_TREE_REDUCE);
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCE);
}
else if(strncasecmp(envopts, "RI", 2) == 0) /* Rectangle Ring*/
{
MPIDO_INFO_SET(properties, MPIDO_USE_RECTRING_REDUCE);
}
else if(strncasecmp(envopts, "A", 1) == 0) /* Allreduce */
{
MPIDO_INFO_SET(properties, MPIDO_USE_ALLREDUCE_REDUCE);
MPIDO_INFO_SET(properties, MPIDO_USE_PREMALLOC_REDUCE);
}
else if(strncasecmp(envopts, "R", 1) == 0) /* Rectangle */
{
MPIDO_INFO_SET(properties, MPIDO_USE_RECT_REDUCE);
}
else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */
{
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_REDUCE);
}
else if(strncasecmp(envopts, "T", 1) == 0) /* Tree */
{
MPIDO_INFO_SET(properties, MPIDO_USE_TREE_REDUCE);
}
else if(strncasecmp(envopts, "C", 1) == 0) /* CCMI Tree */
{
MPIDO_INFO_SET(properties, MPIDO_USE_CCMI_TREE_REDUCE);
}
else
{
fprintf(stderr,"Invalid DCMF_REDUCE option. Defaulting to MPICH\n");
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_REDUCE);
}
}
envopts = getenv("DCMF_REDUCE_REUSE_STORAGE");
if(envopts != NULL)
{
if(strncasecmp(envopts, "N", 1) == 0) /* Do not reuse the malloc'd storage */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_STORAGE_REDUCE);
/*fprintf(stderr, "N protocol.reusestorage %X\n", 0);*/
}
else if(strncasecmp(envopts, "Y", 1) == 0); /* defaults to Y */
else
fprintf(stderr,"Invalid DCMF_REDUCE_REUSE_STORAGE option\n");
}
envopts = getenv("DCMF_BARRIER");
if(envopts != NULL)
{
if(strncasecmp(envopts, "M", 1) == 0) /* MPICH */
{
MPIDO_INFO_SET(properties, MPIDO_USE_MPICH_BARRIER);
/* still need to register a barrier for DCMF collectives */
MPIDO_INFO_SET(properties, MPIDO_USE_BINOM_BARRIER);
/* MPIDI_Coll_register changes this state for us */
/* MPIDI_CollectiveProtocols.barrier.usegi = 1; */
}
else if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER);
}
else if(strncasecmp(envopts, "G", 1) == 0) /* GI */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER);
}
else if(strncasecmp(envopts, "R", 1) == 0) /* Rect */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_CCMI_GI_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER);
}
else if(strncasecmp(envopts, "C", 1) == 0) /* GI */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_RECT_BARRIER);
MPIDO_INFO_UNSET(properties, MPIDO_USE_GI_BARRIER);
}
else
fprintf(stderr,"Invalid DCMF_BARRIER option\n");
}
envopts = getenv("DCMF_LOCALBARRIER");
if(envopts != NULL)
{
if(strncasecmp(envopts, "B", 1) == 0) /* Binomial */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_LOCKBOX_LBARRIER);
}
else if(strncasecmp(envopts, "L", 1) == 0) /* Lockbox */
{
MPIDO_INFO_UNSET(properties, MPIDO_USE_BINOM_LBARRIER);
}
else
fprintf(stderr,"Invalid DCMF_LOCALBARRIER option\n");
}
/* star-mpi is off by default unless user requests it */
dval = 0;
ENV_Int(getenv("DCMF_STAR"), &dval);
if (dval > 0 )
{
STAR_info.enabled = dval;
/* initialize the repositories of STAR */
STAR_InitRepositories();
}
dval = 0;
ENV_Int(getenv("DCMF_STAR_NUM_INVOCS"), &dval);
if (dval > 0 )
STAR_info.invocs_per_algorithm = dval;
dval = 0;
ENV_Int(getenv("DCMF_STAR_TRACEBACK_LEVEL"), &dval);
if (dval > 0)
STAR_info.traceback_levels = dval;
dval = 0;
ENV_Int(getenv("DCMF_STAR_CHECK_CALLSITE"), &dval);
if (dval > 0)
STAR_info.agree_on_callsite = dval;
ENV_Int(getenv("DCMF_STAR_VERBOSE"), &dval);
if (STAR_info.enabled && dval > 0)
STAR_info.debug = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_ALLTOALL_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.alltoall_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_ALLGATHER_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.allgather_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_ALLGATHERV_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.allgatherv_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_ALLREDUCE_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.allreduce_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_REDUCE_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.reduce_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_BCAST_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.bcast_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_SCATTER_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.scatter_threshold = dval;
dval = -1;
ENV_Int(getenv("DCMF_STAR_GATHER_THRESHOLD"), &dval);
if (dval >= 0 )
STAR_info.gather_threshold = dval;
/* end of star stuff */
}
/**
* \brief Initialize the DCMF ARMCI resources
*/
int ARMCIX_Init ()
{
DCMF_CriticalSection_enter(0);
DCMF_Messager_initialize ();
ARMCIX_DCMF_Connection_initialize ();
/* Determine request pool defaults */
int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX);
int ARMCIX_DCMF_REQUESTPOOL_INC = 0;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_Get_register ();
ARMCIX_DCMF_Put_register (__connection);
ARMCIX_DCMF_Acc_register (__connection);
ARMCIX_DCMF_Fence_register (__connection);
ARMCIX_DCMF_Rmw_register ();
/* Initializer helper thread or configure interrupt mode */
int interrupts = 0;
ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts);
ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts);
//fprintf(stdout,"interrupts = %d\n",interrupts);
/*if (interrupts==1){
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts ON\n");
} else {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts OFF\n");
}*/
if (interrupts==0) {
int ret = pthread_create(&armcix_advance_thread, NULL, armcix_advance, NULL);
if ( ret != 0 ) {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create failed\n");
armcix_advance_active = 0;
} else {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create succeeded\n");
armcix_advance_active = 1;
}
}
DCMF_Configure_t config;
memset (&config, 0x00, sizeof(DCMF_Configure_t));
config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON;
DCMF_Messager_configure (&config, &config);
DCMF_Messager_configure (NULL, &config);
//ARMCIX_DCMF_request_print ("after armcix_init");
DCMF_CriticalSection_exit(0);
return 0;
}
