int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided)
{
int mpi_errno = MPI_SUCCESS;
int has_args;
int has_env;
int thread_provided;
int exit_init_cs_on_failure = 0;
/* For any code in the device that wants to check for runtime
decisions on the value of isThreaded, set a provisional
value here. We could let the MPID_Init routine override this */
#ifdef HAVE_RUNTIME_THREADCHECK
MPIR_ThreadInfo.isThreaded = required == MPI_THREAD_MULTIPLE;
#endif
MPIU_THREAD_CS_INIT;
/* FIXME: Move to os-dependent interface? */
#ifdef HAVE_WINDOWS_H
/* prevent the process from bringing up an error message window if mpich
asserts */
_CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE );
_CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDERR );
_CrtSetReportHook2(_CRT_RPTHOOK_INSTALL, assert_hook);
#ifdef _WIN64
{
/* FIXME: (Windows) This severly degrades performance but fixes alignment
issues with the datatype code. */
/* Prevent misaligned faults on Win64 machines */
UINT mode, old_mode;
old_mode = SetErrorMode(SEM_NOALIGNMENTFAULTEXCEPT);
mode = old_mode | SEM_NOALIGNMENTFAULTEXCEPT;
SetErrorMode(mode);
}
#endif
#endif
/* We need this inorder to implement IS_THREAD_MAIN */
# if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED) && defined(MPICH_IS_THREADED)
{
MPID_Thread_self(&MPIR_ThreadInfo.master_thread);
}
# endif
#ifdef HAVE_ERROR_CHECKING
/* Because the PARAM system has not been initialized, temporarily
uncondtionally enable error checks. Once the PARAM system is
initialized, this may be reset */
MPIR_Process.do_error_checks = 1;
#else
MPIR_Process.do_error_checks = 0;
#endif
/* Initialize necessary subsystems and setup the predefined attribute
values. Subsystems may change these values. */
MPIR_Process.attrs.appnum = -1;
MPIR_Process.attrs.host = 0;
MPIR_Process.attrs.io = 0;
MPIR_Process.attrs.lastusedcode = MPI_ERR_LASTCODE;
MPIR_Process.attrs.tag_ub = 0;
MPIR_Process.attrs.universe = MPIR_UNIVERSE_SIZE_NOT_SET;
MPIR_Process.attrs.wtime_is_global = 0;
/* Set the functions used to duplicate attributes. These are
when the first corresponding keyval is created */
MPIR_Process.attr_dup = 0;
MPIR_Process.attr_free = 0;
#ifdef HAVE_CXX_BINDING
/* Set the functions used to call functions in the C++ binding
for reductions and attribute operations. These are null
until a C++ operation is defined. This allows the C code
that implements these operations to not invoke a C++ code
directly, which may force the inclusion of symbols known only
to the C++ compiler (e.g., under more non-GNU compilers, including
Solaris and IRIX). */
MPIR_Process.cxx_call_op_fn = 0;
#endif
/* This allows the device to select an alternative function for
dimsCreate */
MPIR_Process.dimsCreate = 0;
/* "Allocate" from the reserved space for builtin communicators and
(partially) initialize predefined communicators. comm_parent is
intially NULL and will be allocated by the device if the process group
was started using one of the MPI_Comm_spawn functions. */
MPIR_Process.comm_world = MPID_Comm_builtin + 0;
MPIR_Comm_init(MPIR_Process.comm_world);
MPIR_Process.comm_world->handle = MPI_COMM_WORLD;
MPIR_Process.comm_world->context_id = 0 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_world->recvcontext_id = 0 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_world->comm_kind = MPID_INTRACOMM;
/* This initialization of the comm name could be done only when
comm_get_name is called */
MPIU_Strncpy(MPIR_Process.comm_world->name, "MPI_COMM_WORLD",
MPI_MAX_OBJECT_NAME);
MPIR_Process.comm_self = MPID_Comm_builtin + 1;
MPIR_Comm_init(MPIR_Process.comm_self);
MPIR_Process.comm_self->handle = MPI_COMM_SELF;
MPIR_Process.comm_self->context_id = 1 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_self->recvcontext_id = 1 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_self->comm_kind = MPID_INTRACOMM;
MPIU_Strncpy(MPIR_Process.comm_self->name, "MPI_COMM_SELF",
MPI_MAX_OBJECT_NAME);
#ifdef MPID_NEEDS_ICOMM_WORLD
MPIR_Process.icomm_world = MPID_Comm_builtin + 2;
MPIR_Comm_init(MPIR_Process.icomm_world);
MPIR_Process.icomm_world->handle = MPIR_ICOMM_WORLD;
MPIR_Process.icomm_world->context_id = 2 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.icomm_world->recvcontext_id= 2 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.icomm_world->comm_kind = MPID_INTRACOMM;
MPIU_Strncpy(MPIR_Process.icomm_world->name, "MPI_ICOMM_WORLD",
MPI_MAX_OBJECT_NAME);
/* Note that these communicators are not ready for use - MPID_Init
will setup self and world, and icomm_world if it desires it. */
#endif
MPIR_Process.comm_parent = NULL;
/* Setup the initial communicator list in case we have
enabled the debugger message-queue interface */
MPIR_COMML_REMEMBER( MPIR_Process.comm_world );
MPIR_COMML_REMEMBER( MPIR_Process.comm_self );
/* Call any and all MPID_Init type functions */
MPIR_Err_init();
MPIR_Datatype_init();
MPIR_Group_init();
/* MPIU_Timer_pre_init(); */
mpi_errno = MPIR_Param_init_params();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Wait for debugger to attach if requested. */
if (MPIR_PARAM_DEBUG_HOLD) {
volatile int hold = 1;
while (hold)
#ifdef HAVE_USLEEP
usleep(100);
#endif
;
}
#if HAVE_ERROR_CHECKING == MPID_ERROR_LEVEL_RUNTIME
MPIR_Process.do_error_checks = MPIR_PARAM_ERROR_CHECKING;
#endif
/* define MPI as initialized so that we can use MPI functions within
MPID_Init if necessary */
MPIR_Process.initialized = MPICH_WITHIN_MPI;
/* We can't acquire any critical sections until this point. Any
* earlier the basic data structures haven't been initialized */
MPIU_THREAD_CS_ENTER(INIT,required);
exit_init_cs_on_failure = 1;
mpi_errno = MPID_Init(argc, argv, required, &thread_provided,
&has_args, &has_env);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Capture the level of thread support provided */
MPIR_ThreadInfo.thread_provided = thread_provided;
if (provided) *provided = thread_provided;
#ifdef HAVE_RUNTIME_THREADCHECK
MPIR_ThreadInfo.isThreaded = (thread_provided == MPI_THREAD_MULTIPLE);
#endif
/* FIXME: Define these in the interface. Does Timer init belong here? */
MPIU_dbg_init(MPIR_Process.comm_world->rank);
MPIU_Timer_init(MPIR_Process.comm_world->rank,
MPIR_Process.comm_world->local_size);
#ifdef USE_MEMORY_TRACING
MPIU_trinit( MPIR_Process.comm_world->rank );
/* Indicate that we are near the end of the init step; memory
allocated already will have an id of zero; this helps
separate memory leaks in the initialization code from
leaks in the "active" code */
/* Uncomment this code to leave out any of the MPID_Init/etc
memory allocations from the memory leak testing */
/* MPIU_trid( 1 ); */
#endif
#ifdef USE_DBG_LOGGING
MPIU_DBG_Init( argc, argv, has_args, has_env,
MPIR_Process.comm_world->rank );
#endif
/* Initialize the C versions of the Fortran link-time constants.
We now initialize the Fortran symbols from within the Fortran
interface in the routine that first needs the symbols.
This fixes a problem with symbols added by a Fortran compiler that
are not part of the C runtime environment (the Portland group
compilers would do this)
*/
#if defined(HAVE_FORTRAN_BINDING) && defined(HAVE_MPI_F_INIT_WORKS_WITH_C)
mpirinitf_();
#endif
/* FIXME: Does this need to come before the call to MPID_InitComplete?
For some debugger support, MPIR_WaitForDebugger may want to use
MPI communication routines to collect information for the debugger */
#ifdef HAVE_DEBUGGER_SUPPORT
MPIR_WaitForDebugger();
#endif
/* Let the device know that the rest of the init process is completed */
if (mpi_errno == MPI_SUCCESS)
mpi_errno = MPID_InitCompleted();
#if defined(_OSU_MVAPICH_) || defined(_OSU_PSM_)
if (is_shmem_collectives_enabled()){
if (check_split_comm(pthread_self())){
int my_id, size;
PMPI_Comm_rank(MPI_COMM_WORLD, &my_id);
PMPI_Comm_size(MPI_COMM_WORLD, &size);
disable_split_comm(pthread_self());
create_2level_comm(MPI_COMM_WORLD, size, my_id);
enable_split_comm(pthread_self());
}
}
#endif /* defined(_OSU_MVAPICH_) || defined(_OSU_PSM_) */
fn_exit:
MPIU_THREAD_CS_EXIT(INIT,required);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
/* signal to error handling routines that core services are unavailable */
MPIR_Process.initialized = MPICH_PRE_INIT;
if (exit_init_cs_on_failure) {
MPIU_THREAD_CS_EXIT(INIT,required);
}
MPIU_THREAD_CS_FINALIZE;
return mpi_errno;
/* --END ERROR HANDLING-- */
}
int MPID_Init(int *argc, char ***argv,
int threadlevel_requested, int *threadlevel_provided,
int *has_args, int *has_env)
{
int mpi_errno = MPI_SUCCESS;
int pg_rank, pg_size, pg_id_sz;
int appnum = -1;
/* int universe_size; */
int has_parent;
pscom_socket_t *socket;
pscom_err_t rc;
char *pg_id_name;
char *parent_port;
/* Call any and all MPID_Init type functions */
MPIR_Err_init();
MPIR_Datatype_init();
MPIR_Group_init();
mpid_debug_init();
assert(PSCOM_ANYPORT == -1); /* all codeplaces which depends on it are marked with: "assert(PSP_ANYPORT == -1);" */
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_INIT);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_INIT);
PMICALL(PMI_Init(&has_parent));
PMICALL(PMI_Get_rank(&pg_rank));
PMICALL(PMI_Get_size(&pg_size));
PMICALL(PMI_Get_appnum(&appnum));
*has_args = 1;
*has_env = 1;
/* without PMI_Get_universe_size() we see pmi error:
'[unset]: write_line error; fd=-1' in PMI_KVS_Get()! */
/* PMICALL(PMI_Get_universe_size(&universe_size)); */
if (pg_rank < 0) pg_rank = 0;
if (pg_size <= 0) pg_size = 1;
if (
#ifndef MPICH_IS_THREADED
1
#else
threadlevel_requested < MPI_THREAD_MULTIPLE
#endif
) {
rc = pscom_init(PSCOM_VERSION);
if (rc != PSCOM_SUCCESS) {
fprintf(stderr, "pscom_init(0x%04x) failed : %s\n",
PSCOM_VERSION,
pscom_err_str(rc));
exit(1);
}
} else {
rc = pscom_init_thread(PSCOM_VERSION);
if (rc != PSCOM_SUCCESS) {
fprintf(stderr, "pscom_init_thread(0x%04x) failed : %s\n",
PSCOM_VERSION,
pscom_err_str(rc));
exit(1);
}
}
/* Initialize the switches */
pscom_env_get_uint(&MPIDI_Process.env.enable_collectives, "PSP_COLLECTIVES");
#ifdef PSCOM_HAS_ON_DEMAND_CONNECTIONS
/* if (pg_size > 32) MPIDI_Process.env.enable_ondemand = 1; */
pscom_env_get_uint(&MPIDI_Process.env.enable_ondemand, "PSP_ONDEMAND");
#else
MPIDI_Process.env.enable_ondemand = 0;
#endif
/* enable_ondemand_spawn defaults to enable_ondemand */
MPIDI_Process.env.enable_ondemand_spawn = MPIDI_Process.env.enable_ondemand;
pscom_env_get_uint(&MPIDI_Process.env.enable_ondemand_spawn, "PSP_ONDEMAND_SPAWN");
/* take SMP-related locality information into account (e.g., for MPI_Win_allocate_shared) */
pscom_env_get_uint(&MPIDI_Process.env.enable_smp_awareness, "PSP_SMP_AWARENESS");
/* take MSA-related topology information into account */
pscom_env_get_uint(&MPIDI_Process.env.enable_msa_awareness, "PSP_MSA_AWARENESS");
if(MPIDI_Process.env.enable_msa_awareness) {
pscom_env_get_uint(&MPIDI_Process.msa_module_id, "PSP_MSA_MODULE_ID");
}
#ifdef MPID_PSP_TOPOLOGY_AWARE_COLLOPS
/* use hierarchy-aware collectives on SMP level */
pscom_env_get_uint(&MPIDI_Process.env.enable_smp_aware_collops, "PSP_SMP_AWARE_COLLOPS");
/* use hierarchy-aware collectives on MSA level (disables SMP-aware collops / FIX ME!) */
pscom_env_get_uint(&MPIDI_Process.env.enable_msa_aware_collops, "PSP_MSA_AWARE_COLLOPS");
if(MPIDI_Process.env.enable_msa_aware_collops) MPIDI_Process.env.enable_smp_aware_collops = 0;
#endif
#ifdef MPID_PSP_CREATE_HISTOGRAM
/* collect statistics information and print them at the end of a run */
pscom_env_get_uint(&MPIDI_Process.env.enable_histogram, "PSP_HISTOGRAM");
pscom_env_get_uint(&MPIDI_Process.histo.max_size, "PSP_HISTOGRAM_MAX");
pscom_env_get_uint(&MPIDI_Process.histo.min_size, "PSP_HISTOGRAM_MIN");
pscom_env_get_uint(&MPIDI_Process.histo.step_width, "PSP_HISTOGRAM_SHIFT");
#endif
/*
pscom_env_get_uint(&mpir_allgather_short_msg, "PSP_ALLGATHER_SHORT_MSG");
pscom_env_get_uint(&mpir_allgather_long_msg, "PSP_ALLGATHER_LONG_MSG");
pscom_env_get_uint(&mpir_allreduce_short_msg, "PSP_ALLREDUCE_SHORT_MSG");
pscom_env_get_uint(&mpir_alltoall_short_msg, "PSP_ALLTOALL_SHORT_MSG");
pscom_env_get_uint(&mpir_alltoall_medium_msg, "PSP_ALLTOALL_MEDIUM_MSG");
pscom_env_get_uint(&mpir_alltoall_throttle, "PSP_ALLTOALL_THROTTLE");
pscom_env_get_uint(&mpir_bcast_short_msg, "PSP_BCAST_SHORT_MSG");
pscom_env_get_uint(&mpir_bcast_long_msg, "PSP_BCAST_LONG_MSG");
pscom_env_get_uint(&mpir_bcast_min_procs, "PSP_BCAST_MIN_PROCS");
pscom_env_get_uint(&mpir_gather_short_msg, "PSP_GATHER_SHORT_MSG");
pscom_env_get_uint(&mpir_gather_vsmall_msg, "PSP_GATHER_VSMALL_MSG");
pscom_env_get_uint(&mpir_redscat_commutative_long_msg, "PSP_REDSCAT_COMMUTATIVE_LONG_MSG");
pscom_env_get_uint(&mpir_redscat_noncommutative_short_msg, "PSP_REDSCAT_NONCOMMUTATIVE_SHORT_MSG");
pscom_env_get_uint(&mpir_reduce_short_msg, "PSP_REDUCE_SHORT_MSG");
pscom_env_get_uint(&mpir_scatter_short_msg, "PSP_SCATTER_SHORT_MSG");
*/
socket = pscom_open_socket(0, 0);
if (!MPIDI_Process.env.enable_ondemand) {
socket->ops.con_accept = mpid_con_accept;
}
{
char name[10];
snprintf(name, sizeof(name), "r%07u", (unsigned)pg_rank);
pscom_socket_set_name(socket, name);
}
rc = pscom_listen(socket, PSCOM_ANYPORT);
if (rc != PSCOM_SUCCESS) { PRINTERROR("pscom_listen(PSCOM_ANYPORT)"); goto fn_fail; }
/* Note that if pmi is not availble, the value of MPI_APPNUM is not set */
/* if (appnum != -1) {*/
MPIR_Process.attrs.appnum = appnum;
/* }*/
#if 0
// see mpiimpl.h:
// typedef struct PreDefined_attrs {
// int appnum; /* Application number provided by mpiexec (MPI-2) */
// int host; /* host */
// int io; /* standard io allowed */
// int lastusedcode; /* last used error code (MPI-2) */
// int tag_ub; /* Maximum message tag */
// int universe; /* Universe size from mpiexec (MPI-2) */
// int wtime_is_global; /* Wtime is global over processes in COMM_WORLD */
// } PreDefined_attrs;
#endif
MPIR_Process.attrs.tag_ub = MPIDI_TAG_UB;
/* obtain the id of the process group */
PMICALL(PMI_KVS_Get_name_length_max(&pg_id_sz));
pg_id_name = MPL_malloc(pg_id_sz + 1, MPL_MEM_STRINGS);
if (!pg_id_name) { PRINTERROR("MPL_malloc()"); goto fn_fail; }
PMICALL(PMI_KVS_Get_my_name(pg_id_name, pg_id_sz));
/* safe */
/* MPIDI_Process.socket = socket; */
MPIDI_Process.my_pg_rank = pg_rank;
MPIDI_Process.my_pg_size = pg_size;
MPIDI_Process.pg_id_name = pg_id_name;
if (!MPIDI_Process.env.enable_ondemand) {
/* Create and establish all connections */
if (InitPortConnections(socket) != MPI_SUCCESS) goto fn_fail;
} else {
/* Create all connections as "on demand" connections. */
if (InitPscomConnections(socket) != MPI_SUCCESS) goto fn_fail;
}
#ifdef MPID_PSP_TOPOLOGY_AWARE_COLLOPS
{
int grank;
int my_node_id = -1;
int remote_node_id = -1;
int* node_id_table;
if(MPIDI_Process.env.enable_msa_awareness && MPIDI_Process.env.enable_msa_aware_collops) {
my_node_id = MPIDI_Process.msa_module_id;
assert(my_node_id > -1);
} else if(MPIDI_Process.env.enable_smp_awareness && MPIDI_Process.env.enable_smp_aware_collops) {
if (!MPIDI_Process.env.enable_ondemand) {
/* In the PSP_ONDEMAND=0 case, we can just check the pscom connection types: */
for (grank = 0; grank < pg_size; grank++) {
pscom_connection_t *con = grank2con_get(grank);
if( (con->type == PSCOM_CON_TYPE_SHM) || (pg_rank == grank) ) {
my_node_id = grank;
break;
}
}
} else {
/* In the PSP_ONDEMAND=1 case, we have to use a hash of the host name: */
my_node_id = MPID_PSP_get_host_hash();
if(my_node_id < 0) my_node_id *= -1;
}
assert(my_node_id > -1);
} else {
/* No hierarchy-awareness requested */
assert(my_node_id == -1);
}
if(my_node_id > -1) {
node_id_table = MPL_malloc(pg_size * sizeof(int), MPL_MEM_OBJECT);
if(pg_rank != 0) {
/* gather: */
pscom_connection_t *con = grank2con_get(0);
assert(con);
pscom_send(con, NULL, 0, &my_node_id, sizeof(int));
/* bcast: */
rc = pscom_recv_from(con, NULL, 0, node_id_table, pg_size*sizeof(int));
assert(rc == PSCOM_SUCCESS);
} else {
/* gather: */
node_id_table[0] = my_node_id;
for(grank=1; grank < pg_size; grank++) {
pscom_connection_t *con = grank2con_get(grank);
assert(con);
rc = pscom_recv_from(con, NULL, 0, &remote_node_id, sizeof(int));
assert(rc == PSCOM_SUCCESS);
node_id_table[grank] = remote_node_id;
}
/* bcast: */
for(grank=1; grank < pg_size; grank++) {
pscom_connection_t *con = grank2con_get(grank);
pscom_send(con, NULL, 0, node_id_table, pg_size*sizeof(int));
}
}
MPIDI_Process.node_id_table = node_id_table;
} else {
/* No hierarchy-awareness requested */
assert(MPIDI_Process.node_id_table == NULL);
}
}
#endif
/*
* Initialize the MPI_COMM_WORLD object
*/
{
MPIR_Comm * comm;
int grank;
MPIDI_PG_t * pg_ptr;
int pg_id_num;
MPIDI_VCRT_t * vcrt;
comm = MPIR_Process.comm_world;
comm->rank = pg_rank;
comm->remote_size = pg_size;
comm->local_size = pg_size;
comm->pscom_socket = socket;
vcrt = MPIDI_VCRT_Create(comm->remote_size);
assert(vcrt);
MPID_PSP_comm_set_vcrt(comm, vcrt);
MPIDI_PG_Convert_id(pg_id_name, &pg_id_num);
MPIDI_PG_Create(pg_size, pg_id_num, &pg_ptr);
assert(pg_ptr == MPIDI_Process.my_pg);
for (grank = 0; grank < pg_size; grank++) {
/* MPIR_CheckDisjointLpids() in mpi/comm/intercomm_create.c expect
lpid to be smaller than 4096!!!
Else you will see an "Fatal error in MPI_Intercomm_create"
*/
pscom_connection_t *con = grank2con_get(grank);
pg_ptr->vcr[grank] = MPIDI_VC_Create(pg_ptr, grank, con, grank);
comm->vcr[grank] = MPIDI_VC_Dup(pg_ptr->vcr[grank]);
}
mpi_errno = MPIR_Comm_commit(comm);
assert(mpi_errno == MPI_SUCCESS);
}
/*
* Initialize the MPI_COMM_SELF object
*/
{
MPIR_Comm * comm;
MPIDI_VCRT_t * vcrt;
comm = MPIR_Process.comm_self;
comm->rank = 0;
comm->remote_size = 1;
comm->local_size = 1;
comm->pscom_socket = socket;
vcrt = MPIDI_VCRT_Create(comm->remote_size);
assert(vcrt);
MPID_PSP_comm_set_vcrt(comm, vcrt);
comm->vcr[0] = MPIDI_VC_Dup(MPIR_Process.comm_world->vcr[pg_rank]);
mpi_errno = MPIR_Comm_commit(comm);
assert(mpi_errno == MPI_SUCCESS);
}
/* ToDo: move MPID_enable_receive_dispach to bg thread */
MPID_enable_receive_dispach(socket);
if (threadlevel_provided) {
*threadlevel_provided = (MPICH_THREAD_LEVEL < threadlevel_requested) ?
MPICH_THREAD_LEVEL : threadlevel_requested;
}
if (has_parent) {
MPIR_Comm * comm;
mpi_errno = MPID_PSP_GetParentPort(&parent_port);
assert(mpi_errno == MPI_SUCCESS);
/*
printf("%s:%u:%s Child with Parent: %s\n", __FILE__, __LINE__, __func__, parent_port);
*/
mpi_errno = MPID_Comm_connect(parent_port, NULL, 0,
MPIR_Process.comm_world, &comm);
if (mpi_errno != MPI_SUCCESS) {
fprintf(stderr, "MPI_Comm_connect(parent) failed!\n");
goto fn_fail;
}
assert(comm != NULL);
MPL_strncpy(comm->name, "MPI_COMM_PARENT", MPI_MAX_OBJECT_NAME);
MPIR_Process.comm_parent = comm;
}
MPID_PSP_shm_rma_init();
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_INIT);
return mpi_errno;
/* --- */
fn_fail:
/* A failing MPI_Init() did'nt call the MPI error handler, which
mostly calls abort(). This cause MPI_Init() to return the mpi_errno,
which nobody check, causing segfaultm double frees and so on. To
prevent strange error messages, we now call _exit(1) here.
*/
_exit(1);
}
int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided)
{
int mpi_errno = MPI_SUCCESS;
int has_args;
int has_env;
int thread_provided;
int exit_init_cs_on_failure = 0;
MPIR_Info *info_ptr;
/* For any code in the device that wants to check for runtime
decisions on the value of isThreaded, set a provisional
value here. We could let the MPID_Init routine override this */
#if defined MPICH_IS_THREADED
MPIR_ThreadInfo.isThreaded = required == MPI_THREAD_MULTIPLE;
#endif /* MPICH_IS_THREADED */
#if defined(MPICH_IS_THREADED)
mpi_errno = thread_cs_init();
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
#endif
/* FIXME: Move to os-dependent interface? */
#ifdef HAVE_WINDOWS_H
/* prevent the process from bringing up an error message window if mpich
asserts */
_CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE );
_CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDERR );
_CrtSetReportHook2(_CRT_RPTHOOK_INSTALL, assert_hook);
#ifdef _WIN64
{
/* FIXME: (Windows) This severly degrades performance but fixes alignment
issues with the datatype code. */
/* Prevent misaligned faults on Win64 machines */
UINT mode, old_mode;
old_mode = SetErrorMode(SEM_NOALIGNMENTFAULTEXCEPT);
mode = old_mode | SEM_NOALIGNMENTFAULTEXCEPT;
SetErrorMode(mode);
}
#endif
#endif
/* We need this inorder to implement IS_THREAD_MAIN */
# if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED) && defined(MPICH_IS_THREADED)
{
MPID_Thread_self(&MPIR_ThreadInfo.master_thread);
}
# endif
#ifdef HAVE_ERROR_CHECKING
/* Because the PARAM system has not been initialized, temporarily
uncondtionally enable error checks. Once the PARAM system is
initialized, this may be reset */
MPIR_Process.do_error_checks = 1;
#else
MPIR_Process.do_error_checks = 0;
#endif
/* Initialize necessary subsystems and setup the predefined attribute
values. Subsystems may change these values. */
MPIR_Process.attrs.appnum = -1;
MPIR_Process.attrs.host = MPI_PROC_NULL;
MPIR_Process.attrs.io = MPI_PROC_NULL;
MPIR_Process.attrs.lastusedcode = MPI_ERR_LASTCODE;
MPIR_Process.attrs.tag_ub = 0;
MPIR_Process.attrs.universe = MPIR_UNIVERSE_SIZE_NOT_SET;
MPIR_Process.attrs.wtime_is_global = 0;
/* Set the functions used to duplicate attributes. These are
when the first corresponding keyval is created */
MPIR_Process.attr_dup = 0;
MPIR_Process.attr_free = 0;
#ifdef HAVE_CXX_BINDING
/* Set the functions used to call functions in the C++ binding
for reductions and attribute operations. These are null
until a C++ operation is defined. This allows the C code
that implements these operations to not invoke a C++ code
directly, which may force the inclusion of symbols known only
to the C++ compiler (e.g., under more non-GNU compilers, including
Solaris and IRIX). */
MPIR_Process.cxx_call_op_fn = 0;
#endif
#ifdef HAVE_F08_BINDING
MPIR_C_MPI_STATUS_IGNORE = MPI_STATUS_IGNORE;
MPIR_C_MPI_STATUSES_IGNORE = MPI_STATUSES_IGNORE;
MPIR_C_MPI_ARGV_NULL = MPI_ARGV_NULL;
MPIR_C_MPI_ARGVS_NULL = MPI_ARGVS_NULL;
MPIR_C_MPI_UNWEIGHTED = MPI_UNWEIGHTED;
MPIR_C_MPI_WEIGHTS_EMPTY = MPI_WEIGHTS_EMPTY;
MPIR_C_MPI_ERRCODES_IGNORE = MPI_ERRCODES_IGNORE;
#endif
/* This allows the device to select an alternative function for
dimsCreate */
MPIR_Process.dimsCreate = 0;
/* "Allocate" from the reserved space for builtin communicators and
(partially) initialize predefined communicators. comm_parent is
intially NULL and will be allocated by the device if the process group
was started using one of the MPI_Comm_spawn functions. */
MPIR_Process.comm_world = MPIR_Comm_builtin + 0;
MPII_Comm_init(MPIR_Process.comm_world);
MPIR_Process.comm_world->handle = MPI_COMM_WORLD;
MPIR_Process.comm_world->context_id = 0 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_world->recvcontext_id = 0 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_world->comm_kind = MPIR_COMM_KIND__INTRACOMM;
/* This initialization of the comm name could be done only when
comm_get_name is called */
MPL_strncpy(MPIR_Process.comm_world->name, "MPI_COMM_WORLD",
MPI_MAX_OBJECT_NAME);
MPIR_Process.comm_self = MPIR_Comm_builtin + 1;
MPII_Comm_init(MPIR_Process.comm_self);
MPIR_Process.comm_self->handle = MPI_COMM_SELF;
MPIR_Process.comm_self->context_id = 1 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_self->recvcontext_id = 1 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_self->comm_kind = MPIR_COMM_KIND__INTRACOMM;
MPL_strncpy(MPIR_Process.comm_self->name, "MPI_COMM_SELF",
MPI_MAX_OBJECT_NAME);
#ifdef MPID_NEEDS_ICOMM_WORLD
MPIR_Process.icomm_world = MPIR_Comm_builtin + 2;
MPII_Comm_init(MPIR_Process.icomm_world);
MPIR_Process.icomm_world->handle = MPIR_ICOMM_WORLD;
MPIR_Process.icomm_world->context_id = 2 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.icomm_world->recvcontext_id= 2 << MPIR_CONTEXT_PREFIX_SHIFT;
MPIR_Process.icomm_world->comm_kind = MPIR_COMM_KIND__INTRACOMM;
MPL_strncpy(MPIR_Process.icomm_world->name, "MPI_ICOMM_WORLD",
MPI_MAX_OBJECT_NAME);
/* Note that these communicators are not ready for use - MPID_Init
will setup self and world, and icomm_world if it desires it. */
#endif
MPIR_Process.comm_parent = NULL;
/* Setup the initial communicator list in case we have
enabled the debugger message-queue interface */
MPII_COMML_REMEMBER( MPIR_Process.comm_world );
MPII_COMML_REMEMBER( MPIR_Process.comm_self );
/* Call any and all MPID_Init type functions */
MPIR_Err_init();
MPIR_Datatype_init();
MPIR_Group_init();
/* MPIU_Timer_pre_init(); */
/* Wait for debugger to attach if requested. */
if (MPIR_CVAR_DEBUG_HOLD) {
volatile int hold = 1;
while (hold)
#ifdef HAVE_USLEEP
usleep(100);
#endif
;
}
#if defined(HAVE_ERROR_CHECKING) && (HAVE_ERROR_CHECKING == MPID_ERROR_LEVEL_RUNTIME)
MPIR_Process.do_error_checks = MPIR_CVAR_ERROR_CHECKING;
#endif
/* define MPI as initialized so that we can use MPI functions within
MPID_Init if necessary */
OPA_store_int(&MPIR_Process.mpich_state, MPICH_MPI_STATE__IN_INIT);
/* We can't acquire any critical sections until this point. Any
* earlier the basic data structures haven't been initialized */
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
exit_init_cs_on_failure = 1;
/* create MPI_INFO_NULL object */
/* FIXME: Currently this info object is empty, we need to add data to this
as defined by the standard. */
info_ptr = MPIR_Info_builtin + 1;
info_ptr->handle = MPI_INFO_ENV;
MPIR_Object_set_ref(info_ptr, 1);
info_ptr->next = NULL;
info_ptr->key = NULL;
info_ptr->value = NULL;
mpi_errno = MPID_Init(argc, argv, required, &thread_provided,
&has_args, &has_env);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* Assert: tag_ub should be a power of 2 minus 1 */
MPIR_Assert(((unsigned)MPIR_Process.attrs.tag_ub & ((unsigned)MPIR_Process.attrs.tag_ub + 1)) == 0);
/* Set aside tag space for tagged collectives and failure notification */
#ifdef HAVE_TAG_ERROR_BITS
MPIR_Process.attrs.tag_ub >>= 3;
#else
MPIR_Process.attrs.tag_ub >>= 1;
#endif
/* The bit for error checking is set in a macro in mpiimpl.h for
* performance reasons. */
MPIR_Process.tagged_coll_mask = MPIR_Process.attrs.tag_ub + 1;
/* Assert: tag_ub is at least the minimum asked for in the MPI spec */
MPIR_Assert( MPIR_Process.attrs.tag_ub >= 32767 );
/* Capture the level of thread support provided */
MPIR_ThreadInfo.thread_provided = thread_provided;
if (provided) *provided = thread_provided;
#if defined MPICH_IS_THREADED
MPIR_ThreadInfo.isThreaded = (thread_provided == MPI_THREAD_MULTIPLE);
#endif /* MPICH_IS_THREADED */
/* FIXME: Define these in the interface. Does Timer init belong here? */
MPII_Timer_init(MPIR_Process.comm_world->rank,
MPIR_Process.comm_world->local_size);
#ifdef USE_MEMORY_TRACING
#ifdef MPICH_IS_THREADED
MPL_trinit( MPIR_Process.comm_world->rank, MPIR_ThreadInfo.isThreaded );
#else
MPL_trinit( MPIR_Process.comm_world->rank, 0 );
#endif
/* Indicate that we are near the end of the init step; memory
allocated already will have an id of zero; this helps
separate memory leaks in the initialization code from
leaks in the "active" code */
#endif
#ifdef MPL_USE_DBG_LOGGING
/* FIXME: This is a hack to handle the common case of two worlds.
* If the parent comm is not NULL, we always give the world number
* as "1" (false). */
#ifdef MPICH_IS_THREADED
MPL_dbg_init( argc, argv, has_args, has_env,
MPIR_Process.comm_parent != NULL, MPIR_Process.comm_world->rank,
MPIR_ThreadInfo.isThreaded );
#else
MPL_dbg_init( argc, argv, has_args, has_env,
MPIR_Process.comm_parent != NULL, MPIR_Process.comm_world->rank,
0 );
#endif
MPIR_DBG_INIT = MPL_dbg_class_alloc("INIT", "init");
MPIR_DBG_PT2PT = MPL_dbg_class_alloc("PT2PT", "pt2pt");
MPIR_DBG_THREAD = MPL_dbg_class_alloc("THREAD", "thread");
MPIR_DBG_DATATYPE = MPL_dbg_class_alloc("DATATYPE", "datatype");
MPIR_DBG_HANDLE = MPL_dbg_class_alloc("HANDLE", "handle");
MPIR_DBG_COMM = MPL_dbg_class_alloc("COMM", "comm");
MPIR_DBG_BSEND = MPL_dbg_class_alloc("BSEND", "bsend");
MPIR_DBG_ERRHAND = MPL_dbg_class_alloc("ERRHAND", "errhand");
MPIR_DBG_OTHER = MPL_dbg_class_alloc("OTHER", "other");
MPIR_DBG_REQUEST = MPL_dbg_class_alloc("REQUEST", "request");
MPIR_DBG_ASSERT = MPL_dbg_class_alloc("ASSERT", "assert");
MPIR_DBG_STRING = MPL_dbg_class_alloc("STRING", "string");
#endif
/* Initialize the C versions of the Fortran link-time constants.
We now initialize the Fortran symbols from within the Fortran
interface in the routine that first needs the symbols.
This fixes a problem with symbols added by a Fortran compiler that
are not part of the C runtime environment (the Portland group
compilers would do this)
*/
#if defined(HAVE_FORTRAN_BINDING) && defined(HAVE_MPI_F_INIT_WORKS_WITH_C)
mpirinitf_();
#endif
/* FIXME: Does this need to come before the call to MPID_InitComplete?
For some debugger support, MPII_Wait_for_debugger may want to use
MPI communication routines to collect information for the debugger */
#ifdef HAVE_DEBUGGER_SUPPORT
MPII_Wait_for_debugger();
#endif
/* Let the device know that the rest of the init process is completed */
if (mpi_errno == MPI_SUCCESS)
mpi_errno = MPID_InitCompleted();
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
/* Make fields of MPIR_Process global visible and set mpich_state
atomically so that MPI_Initialized() etc. are thread safe */
OPA_write_barrier();
OPA_store_int(&MPIR_Process.mpich_state, MPICH_MPI_STATE__POST_INIT);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
/* signal to error handling routines that core services are unavailable */
OPA_store_int(&MPIR_Process.mpich_state, MPICH_MPI_STATE__PRE_INIT);
if (exit_init_cs_on_failure) {
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
}
#if defined(MPICH_IS_THREADED)
MPIR_Thread_CS_Finalize();
#endif
return mpi_errno;
/* --END ERROR HANDLING-- */
}