int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
if (!oshmem_shmem_initialized) {
if (!ompi_mpi_initialized && !ompi_mpi_finalized) {
ret = ompi_mpi_init(argc, argv, requested, provided);
}
MPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
if (OSHMEM_SUCCESS == ret) {
ret = _shmem_init(argc, argv, requested, provided);
}
if (OSHMEM_SUCCESS == ret) {
oshmem_shmem_initialized = true;
if (OSHMEM_SUCCESS != shmem_lock_init()) {
SHMEM_API_ERROR( "shmem_lock_init() failed");
return OSHMEM_ERROR;
}
/* this is a collective op, implies barrier */
MCA_MEMHEAP_CALL(get_all_mkeys());
oshmem_shmem_preconnect_all();
#if OSHMEM_OPAL_THREAD_ENABLE
pthread_t thread_id;
int perr;
perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
if (perr != 0)
{
SHMEM_API_ERROR("cannot creat opal thread for SHMEM");
return OSHMEM_ERROR;
}
#endif
}
}
#ifdef SIGUSR1
signal(SIGUSR1,sighandler__SIGUSR1);
signal(SIGTERM,sighandler__SIGTERM);
#endif
return ret;
}
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
OMPI_TIMING_INIT(32);
if (!oshmem_shmem_initialized) {
ret = ompi_mpi_init(argc, argv, requested, provided, true);
OMPI_TIMING_NEXT("ompi_mpi_init");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
PMPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
OMPI_TIMING_NEXT("PMPI_Comm_dup");
SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc);
ret = _shmem_init(argc, argv, requested, provided);
OMPI_TIMING_NEXT("_shmem_init");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_mpi_comm_query");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_enable");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_base_select");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
oshmem_shmem_initialized = true;
if (OSHMEM_SUCCESS != shmem_lock_init()) {
SHMEM_API_ERROR( "shmem_lock_init() failed");
return OSHMEM_ERROR;
}
OMPI_TIMING_NEXT("shmem_lock_init");
/* this is a collective op, implies barrier */
MCA_MEMHEAP_CALL(get_all_mkeys());
OMPI_TIMING_NEXT("get_all_mkeys()");
oshmem_shmem_preconnect_all();
OMPI_TIMING_NEXT("shmem_preconnect_all");
#if OSHMEM_OPAL_THREAD_ENABLE
pthread_t thread_id;
int perr;
perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
if (0 != perr) {
SHMEM_API_ERROR("cannot create opal thread for SHMEM");
return OSHMEM_ERROR;
}
#endif
OMPI_TIMING_NEXT("THREAD_ENABLE");
}
#ifdef SIGUSR1
signal(SIGUSR1,sighandler__SIGUSR1);
signal(SIGTERM,sighandler__SIGTERM);
#endif
OMPI_TIMING_OUT;
OMPI_TIMING_FINALIZE;
return ret;
}
static int __shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
char *error = NULL;
if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) {
error = "oshmem_proc_init() failed";
goto error;
}
/* We need to do this anyway.
* This place requires to be reviewed and more elegant way is expected
*/
ompi_proc_local_proc = (ompi_proc_t*) oshmem_proc_local_proc;
if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_init())) {
error = "oshmem_group_cache_list_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
error = "oshmem_op_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_spml_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_scoll_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret = mca_spml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_spml_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret =
mca_scoll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_scoll_base_find_available() failed";
goto error;
}
/* Initialize each SHMEM handle subsystem */
/* Initialize requests */
if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) {
error = "oshmem_request_init() failed";
goto error;
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OSHMEM_SUCCESS != (ret = oshmem_proc_set_arch())) {
error = "oshmem_proc_set_arch failed";
goto error;
}
/* start SPML/BTL's */
ret = MCA_SPML_CALL(enable(true));
if (OSHMEM_SUCCESS != ret) {
error = "SPML control failed";
goto error;
}
/* There is issue with call add_proc twice so
* we need to use btl info got from PML add_procs() before call of SPML add_procs()
*/
{
ompi_proc_t** procs = NULL;
size_t nprocs = 0;
procs = ompi_proc_world(&nprocs);
while (nprocs--) {
oshmem_group_all->proc_array[nprocs]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] =
procs[nprocs]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
}
free(procs);
}
ret =
MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
if (OSHMEM_SUCCESS != ret) {
error = "SPML add procs failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_memheap_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) {
error = "mca_select_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_atomic_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS
!= (ret =
mca_atomic_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_atomic_base_find_available() failed";
goto error;
}
/* This call should be done after memheap initialization */
if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) {
error = "mca_scoll_enable() failed";
goto error;
}
if (OSHMEM_SUCCESS != shmem_lock_init()) {
error = "shmem_lock_init() failed";
goto error;
}
error:
if (ret != OSHMEM_SUCCESS) {
const char *err_msg = opal_strerror(ret);
orte_show_help("help-shmem-runtime.txt",
"shmem_init:startup:internal-failure",
true,
"SHMEM_INIT",
"SHMEM_INIT",
error,
err_msg,
ret);
return ret;
}
return ret;
}