/* in some cases, all PE procs are required to do a modex so they
* can (at the least) exchange their architecture. Since we cannot
* know in advance if this was required, we provide a separate function
* to set the arch (instead of doing it inside of oshmem_proc_init) that
* can be called after the modex completes in oshmem_shmem_init. Thus, we
* know that - regardless of how the arch is known, whether via modex
* or dropped in from a local daemon - the arch can be set correctly
* at this time
*/
int oshmem_proc_set_arch(void)
{
oshmem_proc_t *proc = NULL;
opal_list_item_t *item = NULL;
int ret = OSHMEM_SUCCESS;
OPAL_THREAD_LOCK(&oshmem_proc_lock);
for (item = opal_list_get_first(&oshmem_proc_list);
item != opal_list_get_end(&oshmem_proc_list);
item = opal_list_get_next(item)) {
proc = (oshmem_proc_t*) item;
if (OSHMEM_PROC_VPID(proc) != ORTE_PROC_MY_NAME->vpid) {
/* if arch is different than mine, create a new convertor for this proc */
if (proc->super.proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(proc->super.proc_convertor);
proc->super.proc_convertor = opal_convertor_create(proc->super.proc_arch, 0);
#else
orte_show_help("help-shmem-runtime.txt",
"heterogeneous-support-unavailable",
true,
orte_process_info.nodename,
proc->super.proc_hostname == NULL ?
"<hostname unavailable>" :
proc->super.proc_hostname);
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return OSHMEM_ERR_NOT_SUPPORTED;
#endif
}
}
}
/* Set predefined groups */
ret = oshmem_proc_group_init();
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return ret;
}
static int _shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
char *error = NULL;
oshmem_mpi_thread_requested = requested;
oshmem_mpi_thread_provided = requested;
/* Register the OSHMEM layer's MCA parameters */
if (OSHMEM_SUCCESS != (ret = oshmem_shmem_register_params())) {
error = "oshmem_info_register: oshmem_register_params failed";
goto error;
}
/* Setting verbosity for macros like SHMEM_API_VERBOSE, SHMEM_API_ERROR.
* We need to set it right after registering mca verbosity variables
*/
shmem_api_logger_output = opal_output_open(NULL);
opal_output_set_verbosity(shmem_api_logger_output,
oshmem_shmem_api_verbose);
/* initialize info */
if (OSHMEM_SUCCESS != (ret = oshmem_info_init())) {
error = "oshmem_info_init() failed";
goto error;
}
/* initialize proc */
if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) {
error = "oshmem_proc_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
error = "oshmem_op_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_spml_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_scoll_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_spml_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_spml_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_scoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_scoll_base_find_available() failed";
goto error;
}
/* Initialize each SHMEM handle subsystem */
/* Initialize requests */
if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) {
error = "oshmem_request_init() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_init())) {
error = "oshmem_proc_group_init() failed";
goto error;
}
/* start SPML/BTL's */
ret = MCA_SPML_CALL(enable(true));
if (OSHMEM_SUCCESS != ret) {
error = "SPML control failed";
goto error;
}
ret =
MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
if (OSHMEM_SUCCESS != ret) {
error = "SPML add procs failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_sshmem_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_sshmem_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_sshmem_base_select())) {
error = "mca_sshmem_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_memheap_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) {
error = "mca_memheap_base_select() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) {
error = "mca_atomic_base_open() failed";
goto error;
}
if (OSHMEM_SUCCESS != (ret = mca_atomic_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
error = "mca_atomic_base_find_available() failed";
goto error;
}
/* This call should be done after memheap initialization */
if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) {
error = "mca_scoll_enable() failed";
goto error;
}
(*provided) = oshmem_mpi_thread_provided;
oshmem_mpi_thread_multiple = (oshmem_mpi_thread_provided == SHMEM_THREAD_MULTIPLE) ? true : false;
error: if (ret != OSHMEM_SUCCESS) {
const char *err_msg = opal_strerror(ret);
opal_show_help("help-shmem-runtime.txt",
"shmem_init:startup:internal-failure",
true,
"SHMEM_INIT",
"SHMEM_INIT",
error,
err_msg,
ret);
return ret;
}
return ret;
}
