void shmem_init(void)
{
int err = OSHMEM_SUCCESS;
int provided;
int required = SHMEM_THREAD_SINGLE;
if (oshmem_shmem_initialized) {
/*
* SPEC: If start_pes() is called multiple times, subsequent calls have no effect.
*/
return;
}
shmem_api_logger_output = opal_output_open(NULL);
opal_output_set_verbosity(shmem_api_logger_output,
oshmem_shmem_api_verbose);
err = oshmem_shmem_init(0, NULL, required, &provided);
if (OSHMEM_SUCCESS != err) {
/* since spec does not propagete error to user we can only abort */
SHMEM_API_ERROR("SHMEM failed to initialize - aborting");
oshmem_shmem_abort(-1);
}
OPAL_CR_INIT_LIBRARY();
}
int oshmem_shmem_preconnect_all(void)
{
int rc = OSHMEM_SUCCESS;
/* force qp creation and rkey exchange for memheap. Does not force exchange of static vars */
if (oshmem_preconnect_all) {
long val;
int nproc;
int my_pe;
int i;
val = 0xdeadbeaf;
if (!preconnect_value) {
rc =
MCA_MEMHEAP_CALL(private_alloc(sizeof(long), (void **)&preconnect_value));
}
if (!preconnect_value || (rc != OSHMEM_SUCCESS)) {
SHMEM_API_ERROR("shmem_preconnect_all failed");
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
nproc = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
for (i = 0; i < nproc; i++) {
shmem_long_p(preconnect_value, val, (my_pe + i) % nproc);
}
shmem_barrier_all();
SHMEM_API_VERBOSE(5, "Preconnected all PEs");
}
return OSHMEM_SUCCESS;
}
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
if (!oshmem_shmem_initialized) {
if (!ompi_mpi_initialized && !ompi_mpi_finalized) {
ret = ompi_mpi_init(argc, argv, requested, provided);
}
MPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
if (OSHMEM_SUCCESS == ret) {
ret = _shmem_init(argc, argv, requested, provided);
}
if (OSHMEM_SUCCESS == ret) {
oshmem_shmem_initialized = true;
if (OSHMEM_SUCCESS != shmem_lock_init()) {
SHMEM_API_ERROR( "shmem_lock_init() failed");
return OSHMEM_ERROR;
}
/* this is a collective op, implies barrier */
MCA_MEMHEAP_CALL(get_all_mkeys());
oshmem_shmem_preconnect_all();
#if OSHMEM_OPAL_THREAD_ENABLE
pthread_t thread_id;
int perr;
perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
if (perr != 0)
{
SHMEM_API_ERROR("cannot creat opal thread for SHMEM");
return OSHMEM_ERROR;
}
#endif
}
}
#ifdef SIGUSR1
signal(SIGUSR1,sighandler__SIGUSR1);
signal(SIGTERM,sighandler__SIGTERM);
#endif
return ret;
}
int oshmem_shmem_preconnect_all(void)
{
int mca_value = 0;
int rc = OSHMEM_SUCCESS;
(void) mca_base_var_register("oshmem",
"runtime",
NULL,
"preconnect_all",
"Whether to force SHMEM processes to fully "
"wire-up the connections between SHMEM "
"processes during "
"initialization (vs. making connections lazily -- "
"upon the first SHMEM traffic between each "
"process peer pair)",
MCA_BASE_VAR_TYPE_INT,
NULL,
0,
MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_value);
/* force qp creation and rkey exchange for memheap. Does not force exchange of static vars */
if (mca_value) {
long val;
int nproc = 0;
int i;
val = 0xdeadbeaf;
if (!preconnect_value) {
rc =
MCA_MEMHEAP_CALL(private_alloc(sizeof(long), (void **)&preconnect_value));
}
if (!preconnect_value || (rc != OSHMEM_SUCCESS)) {
SHMEM_API_ERROR("shmem_preconnect_all failed");
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
nproc = _num_pes();
for (i = 0; i < nproc; i++) {
shmem_long_p(preconnect_value, val, i);
}
shmem_fence();
shmem_barrier_all();
SHMEM_API_VERBOSE(5, "Preconnected all PEs");
}
return OSHMEM_SUCCESS;
}
static inline void _shmem_init(int required, int *provided)
{
int err = OSHMEM_SUCCESS;
if (oshmem_shmem_initialized) {
/*
* SPEC: If start_pes() is called multiple times, subsequent calls have no effect.
*/
return;
}
err = oshmem_shmem_init(0, NULL, required, provided);
if (OSHMEM_SUCCESS != err) {
/* since spec does not propagete error to user we can only abort */
SHMEM_API_ERROR("SHMEM failed to initialize - aborting");
oshmem_shmem_abort(-1);
}
OPAL_CR_INIT_LIBRARY();
#if HAVE_ON_EXIT
on_exit(shmem_onexit, NULL);
#endif
}
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
OMPI_TIMING_INIT(32);
if (!oshmem_shmem_initialized) {
ret = ompi_mpi_init(argc, argv, requested, provided, true);
OMPI_TIMING_NEXT("ompi_mpi_init");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
PMPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
OMPI_TIMING_NEXT("PMPI_Comm_dup");
SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc);
ret = _shmem_init(argc, argv, requested, provided);
OMPI_TIMING_NEXT("_shmem_init");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_mpi_comm_query");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_enable");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_base_select");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
oshmem_shmem_initialized = true;
if (OSHMEM_SUCCESS != shmem_lock_init()) {
SHMEM_API_ERROR( "shmem_lock_init() failed");
return OSHMEM_ERROR;
}
OMPI_TIMING_NEXT("shmem_lock_init");
/* this is a collective op, implies barrier */
MCA_MEMHEAP_CALL(get_all_mkeys());
OMPI_TIMING_NEXT("get_all_mkeys()");
oshmem_shmem_preconnect_all();
OMPI_TIMING_NEXT("shmem_preconnect_all");
#if OSHMEM_OPAL_THREAD_ENABLE
pthread_t thread_id;
int perr;
perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
if (0 != perr) {
SHMEM_API_ERROR("cannot create opal thread for SHMEM");
return OSHMEM_ERROR;
}
#endif
OMPI_TIMING_NEXT("THREAD_ENABLE");
}
#ifdef SIGUSR1
signal(SIGUSR1,sighandler__SIGUSR1);
signal(SIGTERM,sighandler__SIGTERM);
#endif
OMPI_TIMING_OUT;
OMPI_TIMING_FINALIZE;
return ret;
}
