static int init_mx( MPIDI_PG_t *pg_p )
{
mx_endpoint_addr_t local_endpoint_addr;
mx_return_t ret;
mx_param_t param;
int mpi_errno = MPI_SUCCESS;
int r;
r = MPL_putenv("MX_DISABLE_SHARED=1");
MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
r = MPL_putenv("MX_DISABLE_SELF=1");
MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
ret = mx_init();
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_init", "**mx_init %s", mx_strerror (ret));
mx_set_error_handler(MX_ERRORS_RETURN);
/*
ret = mx_get_info(NULL, MX_NIC_COUNT, NULL, 0, &nic_count, sizeof(int));
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
count = ++nic_count;
mx_nics = (uint64_t *)MPIU_Malloc(count*sizeof(uint64_t));
ret = mx_get_info(NULL, MX_NIC_IDS, NULL, 0, mx_nics, count*sizeof(uint64_t));
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
do{
ret = mx_nic_id_to_board_number(mx_nics[index],&mx_board_num);
index++;
}while(ret != MX_SUCCESS);
*/
#ifndef USE_CTXT_AS_MARK
param.key = MX_PARAM_CONTEXT_ID;
param.val.context_id.bits = NEM_MX_MATCHING_BITS - SHIFT_TYPE;
param.val.context_id.shift = SHIFT_TYPE;
ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,¶m,1,&MPID_nem_mx_local_endpoint);
#else
ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,NULL,0,&MPID_nem_mx_local_endpoint);
#endif
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_open_endpoint", "**mx_open_endpoint %s", mx_strerror (ret));
ret = mx_get_endpoint_addr(MPID_nem_mx_local_endpoint,&local_endpoint_addr);
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_endpoint_addr", "**mx_get_endpoint_addr %s", mx_strerror (ret));
ret = mx_decompose_endpoint_addr(local_endpoint_addr,&MPID_nem_mx_local_nic_id,&MPID_nem_mx_local_endpoint_id);
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_decompose_endpoint_addr", "**mx_decompose_endpoint_addr %s", mx_strerror (ret));
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
HYD_status HYDU_putenv(struct HYD_env *env, HYD_env_overwrite_t overwrite)
{
char *tmp[HYD_NUM_TMP_STRINGS], *str;
int i;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
/* If the overwrite flag is false, just exit */
if (MPL_env2str(env->env_name, (const char **) &str) && overwrite == HYD_ENV_OVERWRITE_FALSE)
goto fn_exit;
i = 0;
tmp[i++] = MPL_strdup(env->env_name);
tmp[i++] = MPL_strdup("=");
tmp[i++] = env->env_value ? MPL_strdup(env->env_value) : MPL_strdup("");
tmp[i++] = NULL;
status = HYDU_str_alloc_and_join(tmp, &str);
HYDU_ERR_POP(status, "unable to join strings\n");
MPL_putenv(str);
for (i = 0; tmp[i]; i++)
MPL_free(tmp[i]);
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
static int restore_env(pid_t parent_pid, int rank)
{
FILE *f;
char env_filename[MAX_STR_LEN];
char var_val[MAX_STR_LEN];
int ret;
MPIU_Snprintf(env_filename, MAX_STR_LEN, "/tmp/hydra-env-file-%d:%d", parent_pid, rank);
f = fopen(env_filename, "r");
CHECK_ERR(!f, MPIU_Strerror (errno));
ret = unlink(env_filename);
CHECK_ERR(ret, MPIU_Strerror (errno));
while (fgets(var_val, MAX_STR_LEN, f)) {
size_t len = strlen(var_val);
/* remove newline */
if (var_val[len-1] == '\n')
var_val[len-1] = '\0';
ret = MPL_putenv(MPIU_Strdup(var_val));
CHECK_ERR(ret != 0, MPIU_Strerror (errno));
}
ret = fclose(f);
CHECK_ERR(ret, MPIU_Strerror (errno));
return 0;
}
static HYD_status global_env_fn(char *arg, char ***argv)
{
int i, count;
char *str;
HYD_status status = HYD_SUCCESS;
count = atoi(**argv);
for (i = 0; i < count; i++) {
(*argv)++;
str = **argv;
/* Environment variables are quoted; remove them */
if (*str == '\'') {
str++;
str[strlen(str) - 1] = 0;
}
if (!strcmp(arg, "global-inherited-env")) {
HYDU_append_env_str_to_list(str, &HYD_pmcd_pmip.user_global.global_env.inherited);
/* Make sure to let proxy aware of HYDRA related variables */
if (!strncmp(str, "HYDRA_", 6)) {
MPL_putenv(str);
}
} else if (!strcmp(arg, "global-system-env"))
HYDU_append_env_str_to_list(str, &HYD_pmcd_pmip.user_global.global_env.system);
else if (!strcmp(arg, "global-user-env"))
HYDU_append_env_str_to_list(str, &HYD_pmcd_pmip.user_global.global_env.user);
}
(*argv)++;
HYDU_FUNC_EXIT();
return status;
}
int MPID_nem_mxm_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p)
{
int r;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_INIT);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_INIT);
/* first make sure that our private fields in the vc and req fit into the area provided */
MPIR_Assert(sizeof(MPID_nem_mxm_vc_area) <= MPIDI_NEM_VC_NETMOD_AREA_LEN);
MPIR_Assert(sizeof(MPID_nem_mxm_req_area) <= MPIDI_NEM_REQ_NETMOD_AREA_LEN);
/* mpich-specific initialization of mxm */
/* check if the user is not trying to override the tls setting
* before resetting it */
if (getenv("MXM_TLS") == NULL) {
r = MPL_putenv("MXM_TLS=rc,dc,ud");
MPIR_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
}
/* [PB @ 2014-10-06] If hugepage support is not enabled, we force
* memory allocation to go through mmap. This is mainly to
* workaround issues in MXM with Ubuntu where the detection has
* some issues (either because of bugs on the platform or within
* MXM) causing errors. This can probably be deleted eventually
* when this issue is resolved. */
if (MPIR_CVAR_NEMESIS_MXM_HUGEPAGE == 0) {
if (getenv("MXM_MEM_ALLOC") == NULL) {
r = MPL_putenv("MXM_MEM_ALLOC=mmap,libc,sysv");
MPIR_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
}
}
mpi_errno = _mxm_init(pg_rank, pg_p->size);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_mxm_get_business_card(pg_rank, bc_val_p, val_max_sz_p);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno =
MPIDI_CH3I_Register_anysource_notification(MPID_nem_mxm_anysource_posted,
MPID_nem_mxm_anysource_matched);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_register_initcomp_cb(_mxm_post_init);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3U_Comm_register_create_hook(_mxm_add_comm, NULL);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3U_Comm_register_destroy_hook(_mxm_del_comm, NULL);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIDI_Anysource_improbe_fn = MPID_nem_mxm_anysource_improbe;
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
