/* in some cases, all PE procs are required to do a modex so they * can (at the least) exchange their architecture. Since we cannot * know in advance if this was required, we provide a separate function * to set the arch (instead of doing it inside of oshmem_proc_init) that * can be called after the modex completes in oshmem_shmem_init. Thus, we * know that - regardless of how the arch is known, whether via modex * or dropped in from a local daemon - the arch can be set correctly * at this time */ int oshmem_proc_set_arch(void) { oshmem_proc_t *proc = NULL; opal_list_item_t *item = NULL; int ret = OSHMEM_SUCCESS; OPAL_THREAD_LOCK(&oshmem_proc_lock); for (item = opal_list_get_first(&oshmem_proc_list); item != opal_list_get_end(&oshmem_proc_list); item = opal_list_get_next(item)) { proc = (oshmem_proc_t*) item; if (OSHMEM_PROC_VPID(proc) != ORTE_PROC_MY_NAME->vpid) { /* if arch is different than mine, create a new convertor for this proc */ if (proc->super.proc_arch != opal_local_arch) { #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT OBJ_RELEASE(proc->super.proc_convertor); proc->super.proc_convertor = opal_convertor_create(proc->super.proc_arch, 0); #else orte_show_help("help-shmem-runtime.txt", "heterogeneous-support-unavailable", true, orte_process_info.nodename, proc->super.proc_hostname == NULL ? "<hostname unavailable>" : proc->super.proc_hostname); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return OSHMEM_ERR_NOT_SUPPORTED; #endif } } } /* Set predefined groups */ ret = oshmem_proc_group_init(); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return ret; }
static int _shmem_init(int argc, char **argv, int requested, int *provided) { int ret = OSHMEM_SUCCESS; char *error = NULL; oshmem_mpi_thread_requested = requested; oshmem_mpi_thread_provided = requested; /* Register the OSHMEM layer's MCA parameters */ if (OSHMEM_SUCCESS != (ret = oshmem_shmem_register_params())) { error = "oshmem_info_register: oshmem_register_params failed"; goto error; } /* Setting verbosity for macros like SHMEM_API_VERBOSE, SHMEM_API_ERROR. * We need to set it right after registering mca verbosity variables */ shmem_api_logger_output = opal_output_open(NULL); opal_output_set_verbosity(shmem_api_logger_output, oshmem_shmem_api_verbose); /* initialize info */ if (OSHMEM_SUCCESS != (ret = oshmem_info_init())) { error = "oshmem_info_init() failed"; goto error; } /* initialize proc */ if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) { error = "oshmem_proc_init() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) { error = "oshmem_op_init() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) { error = "mca_spml_base_open() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) { error = "mca_scoll_base_open() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_spml_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1))) { error = "mca_spml_base_select() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_scoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) { error = "mca_scoll_base_find_available() failed"; goto error; } /* Initialize each SHMEM handle subsystem */ /* Initialize requests */ if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) { error = "oshmem_request_init() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_init())) { error = "oshmem_proc_group_init() failed"; goto error; } /* start SPML/BTL's */ ret = MCA_SPML_CALL(enable(true)); if (OSHMEM_SUCCESS != ret) { error = "SPML control failed"; goto error; } ret = MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count)); if (OSHMEM_SUCCESS != ret) { error = "SPML add procs failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_sshmem_base_framework, MCA_BASE_OPEN_DEFAULT))) { error = "mca_sshmem_base_open() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_sshmem_base_select())) { error = "mca_sshmem_base_select() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) { error = "mca_memheap_base_open() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) { error = "mca_memheap_base_select() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) { error = "mca_atomic_base_open() failed"; goto error; } if (OSHMEM_SUCCESS != (ret = mca_atomic_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) { error = "mca_atomic_base_find_available() failed"; goto error; } /* This call should be done after memheap initialization */ if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) { error = "mca_scoll_enable() failed"; goto error; } (*provided) = oshmem_mpi_thread_provided; oshmem_mpi_thread_multiple = (oshmem_mpi_thread_provided == SHMEM_THREAD_MULTIPLE) ? true : false; error: if (ret != OSHMEM_SUCCESS) { const char *err_msg = opal_strerror(ret); opal_show_help("help-shmem-runtime.txt", "shmem_init:startup:internal-failure", true, "SHMEM_INIT", "SHMEM_INIT", error, err_msg, ret); return ret; } return ret; }