void shmem_init(void) { int err = OSHMEM_SUCCESS; int provided; int required = SHMEM_THREAD_SINGLE; if (oshmem_shmem_initialized) { /* * SPEC: If start_pes() is called multiple times, subsequent calls have no effect. */ return; } shmem_api_logger_output = opal_output_open(NULL); opal_output_set_verbosity(shmem_api_logger_output, oshmem_shmem_api_verbose); err = oshmem_shmem_init(0, NULL, required, &provided); if (OSHMEM_SUCCESS != err) { /* since spec does not propagete error to user we can only abort */ SHMEM_API_ERROR("SHMEM failed to initialize - aborting"); oshmem_shmem_abort(-1); } OPAL_CR_INIT_LIBRARY(); }
int oshmem_shmem_preconnect_all(void) { int rc = OSHMEM_SUCCESS; /* force qp creation and rkey exchange for memheap. Does not force exchange of static vars */ if (oshmem_preconnect_all) { long val; int nproc; int my_pe; int i; val = 0xdeadbeaf; if (!preconnect_value) { rc = MCA_MEMHEAP_CALL(private_alloc(sizeof(long), (void **)&preconnect_value)); } if (!preconnect_value || (rc != OSHMEM_SUCCESS)) { SHMEM_API_ERROR("shmem_preconnect_all failed"); return OSHMEM_ERR_OUT_OF_RESOURCE; } nproc = oshmem_num_procs(); my_pe = oshmem_my_proc_id(); for (i = 0; i < nproc; i++) { shmem_long_p(preconnect_value, val, (my_pe + i) % nproc); } shmem_barrier_all(); SHMEM_API_VERBOSE(5, "Preconnected all PEs"); } return OSHMEM_SUCCESS; }
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided) { int ret = OSHMEM_SUCCESS; if (!oshmem_shmem_initialized) { if (!ompi_mpi_initialized && !ompi_mpi_finalized) { ret = ompi_mpi_init(argc, argv, requested, provided); } MPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world); if (OSHMEM_SUCCESS == ret) { ret = _shmem_init(argc, argv, requested, provided); } if (OSHMEM_SUCCESS == ret) { oshmem_shmem_initialized = true; if (OSHMEM_SUCCESS != shmem_lock_init()) { SHMEM_API_ERROR( "shmem_lock_init() failed"); return OSHMEM_ERROR; } /* this is a collective op, implies barrier */ MCA_MEMHEAP_CALL(get_all_mkeys()); oshmem_shmem_preconnect_all(); #if OSHMEM_OPAL_THREAD_ENABLE pthread_t thread_id; int perr; perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL); if (perr != 0) { SHMEM_API_ERROR("cannot creat opal thread for SHMEM"); return OSHMEM_ERROR; } #endif } } #ifdef SIGUSR1 signal(SIGUSR1,sighandler__SIGUSR1); signal(SIGTERM,sighandler__SIGTERM); #endif return ret; }
int oshmem_shmem_preconnect_all(void) { int mca_value = 0; int rc = OSHMEM_SUCCESS; (void) mca_base_var_register("oshmem", "runtime", NULL, "preconnect_all", "Whether to force SHMEM processes to fully " "wire-up the connections between SHMEM " "processes during " "initialization (vs. making connections lazily -- " "upon the first SHMEM traffic between each " "process peer pair)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_value); /* force qp creation and rkey exchange for memheap. Does not force exchange of static vars */ if (mca_value) { long val; int nproc = 0; int i; val = 0xdeadbeaf; if (!preconnect_value) { rc = MCA_MEMHEAP_CALL(private_alloc(sizeof(long), (void **)&preconnect_value)); } if (!preconnect_value || (rc != OSHMEM_SUCCESS)) { SHMEM_API_ERROR("shmem_preconnect_all failed"); return OSHMEM_ERR_OUT_OF_RESOURCE; } nproc = _num_pes(); for (i = 0; i < nproc; i++) { shmem_long_p(preconnect_value, val, i); } shmem_fence(); shmem_barrier_all(); SHMEM_API_VERBOSE(5, "Preconnected all PEs"); } return OSHMEM_SUCCESS; }
static inline void _shmem_init(int required, int *provided) { int err = OSHMEM_SUCCESS; if (oshmem_shmem_initialized) { /* * SPEC: If start_pes() is called multiple times, subsequent calls have no effect. */ return; } err = oshmem_shmem_init(0, NULL, required, provided); if (OSHMEM_SUCCESS != err) { /* since spec does not propagete error to user we can only abort */ SHMEM_API_ERROR("SHMEM failed to initialize - aborting"); oshmem_shmem_abort(-1); } OPAL_CR_INIT_LIBRARY(); #if HAVE_ON_EXIT on_exit(shmem_onexit, NULL); #endif }
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided) { int ret = OSHMEM_SUCCESS; OMPI_TIMING_INIT(32); if (!oshmem_shmem_initialized) { ret = ompi_mpi_init(argc, argv, requested, provided, true); OMPI_TIMING_NEXT("ompi_mpi_init"); if (OSHMEM_SUCCESS != ret) { return ret; } PMPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world); OMPI_TIMING_NEXT("PMPI_Comm_dup"); SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc); ret = _shmem_init(argc, argv, requested, provided); OMPI_TIMING_NEXT("_shmem_init"); OMPI_TIMING_IMPORT_OPAL("mca_scoll_mpi_comm_query"); OMPI_TIMING_IMPORT_OPAL("mca_scoll_enable"); OMPI_TIMING_IMPORT_OPAL("mca_scoll_base_select"); if (OSHMEM_SUCCESS != ret) { return ret; } oshmem_shmem_initialized = true; if (OSHMEM_SUCCESS != shmem_lock_init()) { SHMEM_API_ERROR( "shmem_lock_init() failed"); return OSHMEM_ERROR; } OMPI_TIMING_NEXT("shmem_lock_init"); /* this is a collective op, implies barrier */ MCA_MEMHEAP_CALL(get_all_mkeys()); OMPI_TIMING_NEXT("get_all_mkeys()"); oshmem_shmem_preconnect_all(); OMPI_TIMING_NEXT("shmem_preconnect_all"); #if OSHMEM_OPAL_THREAD_ENABLE pthread_t thread_id; int perr; perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL); if (0 != perr) { SHMEM_API_ERROR("cannot create opal thread for SHMEM"); return OSHMEM_ERROR; } #endif OMPI_TIMING_NEXT("THREAD_ENABLE"); } #ifdef SIGUSR1 signal(SIGUSR1,sighandler__SIGUSR1); signal(SIGTERM,sighandler__SIGTERM); #endif OMPI_TIMING_OUT; OMPI_TIMING_FINALIZE; return ret; }