int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs) { size_t i, n; int my_rank = oshmem_my_proc_id(); oshmem_shmem_barrier(); #if MXM_API >= MXM_VERSION(2,0) if (mca_spml_ikrit.bulk_disconnect) { mxm_ep_powerdown(mca_spml_ikrit.mxm_ep); } #endif while (NULL != opal_list_remove_first(&mca_spml_ikrit.active_peers)) { }; OBJ_DESTRUCT(&mca_spml_ikrit.active_peers); for (n = 0; n < nprocs; n++) { i = (my_rank + n) % nprocs; if (mca_spml_ikrit.mxm_peers[i]->mxm_conn) { mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i]->mxm_conn); } if (mca_spml_ikrit.hw_rdma_channel && mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn) { mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn); } destroy_ptl_idx(i); if (mca_spml_ikrit.mxm_peers[i]) { OBJ_RELEASE(mca_spml_ikrit.mxm_peers[i]); } } if (mca_spml_ikrit.mxm_peers) free(mca_spml_ikrit.mxm_peers); return OSHMEM_SUCCESS; }
int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs) { size_t i, n; int my_rank = oshmem_my_proc_id(); oshmem_shmem_barrier(); if (mca_spml_ikrit.bulk_disconnect) { mxm_ep_powerdown(mca_spml_ikrit.mxm_ep); } while (NULL != opal_list_remove_first(&mca_spml_ikrit.active_peers)) { }; OBJ_DESTRUCT(&mca_spml_ikrit.active_peers); for (n = 0; n < nprocs; n++) { i = (my_rank + n) % nprocs; mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i].mxm_conn); if (mca_spml_ikrit.hw_rdma_channel) { assert(mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn != mca_spml_ikrit.mxm_peers[i].mxm_conn); mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn); } mxm_peer_destruct(&mca_spml_ikrit.mxm_peers[i]); } free(mca_spml_ikrit.mxm_peers); return OSHMEM_SUCCESS; }
int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs) { int my_rank = oshmem_my_proc_id(); size_t num_reqs, max_reqs; void *dreq, **dreqs; ucp_ep_h ep; size_t i, n; oshmem_shmem_barrier(); if (!mca_spml_ucx.ucp_peers) { return OSHMEM_SUCCESS; } max_reqs = mca_spml_ucx.num_disconnect; if (max_reqs > nprocs) { max_reqs = nprocs; } dreqs = malloc(sizeof(*dreqs) * max_reqs); if (dreqs == NULL) { return OMPI_ERR_OUT_OF_RESOURCE; } num_reqs = 0; for (i = 0; i < nprocs; ++i) { n = (i + my_rank) % nprocs; ep = mca_spml_ucx.ucp_peers[n].ucp_conn; if (ep == NULL) { continue; } SPML_VERBOSE(10, "disconnecting from peer %d", n); dreq = ucp_disconnect_nb(ep); if (dreq != NULL) { if (UCS_PTR_IS_ERR(dreq)) { SPML_ERROR("ucp_disconnect_nb(%d) failed: %s", n, ucs_status_string(UCS_PTR_STATUS(dreq))); } else { dreqs[num_reqs++] = dreq; } } mca_spml_ucx.ucp_peers[n].ucp_conn = NULL; if ((int)num_reqs >= mca_spml_ucx.num_disconnect) { mca_spml_ucx_waitall(dreqs, &num_reqs); } } mca_spml_ucx_waitall(dreqs, &num_reqs); free(dreqs); opal_pmix.fence(NULL, 0); free(mca_spml_ucx.ucp_peers); return OSHMEM_SUCCESS; }
int mca_spml_ucx_del_procs(oshmem_proc_t** procs, size_t nprocs) { size_t i, n; int my_rank = oshmem_my_proc_id(); oshmem_shmem_barrier(); if (!mca_spml_ucx.ucp_peers) { return OSHMEM_SUCCESS; } for (n = 0; n < nprocs; n++) { i = (my_rank + n) % nprocs; if (mca_spml_ucx.ucp_peers[i].ucp_conn) { ucp_ep_destroy(mca_spml_ucx.ucp_peers[i].ucp_conn); } } free(mca_spml_ucx.ucp_peers); return OSHMEM_SUCCESS; }
int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs) { opal_common_ucx_del_proc_t *del_procs; size_t i; int ret; oshmem_shmem_barrier(); if (!mca_spml_ucx_ctx_default.ucp_peers) { return OSHMEM_SUCCESS; } del_procs = malloc(sizeof(*del_procs) * nprocs); if (del_procs == NULL) { return OMPI_ERR_OUT_OF_RESOURCE; } for (i = 0; i < nprocs; ++i) { del_procs[i].ep = mca_spml_ucx_ctx_default.ucp_peers[i].ucp_conn; del_procs[i].vpid = i; /* mark peer as disconnected */ mca_spml_ucx_ctx_default.ucp_peers[i].ucp_conn = NULL; } ret = opal_common_ucx_del_procs(del_procs, nprocs, oshmem_my_proc_id(), mca_spml_ucx.num_disconnect, mca_spml_ucx_ctx_default.ucp_worker); free(del_procs); free(mca_spml_ucx.remote_addrs_tbl); free(mca_spml_ucx_ctx_default.ucp_peers); mca_spml_ucx_ctx_default.ucp_peers = NULL; opal_common_ucx_mca_proc_added(); return ret; }
void mca_memheap_modex_recv_all(void) { int i; int j; int nprocs, my_pe; opal_buffer_t *msg = NULL; void *send_buffer = NULL; char *rcv_buffer = NULL; int size; int *rcv_size = NULL; int *rcv_n_transports = NULL; int *rcv_offsets = NULL; int rc = OSHMEM_SUCCESS; size_t buffer_size; if (!mca_memheap_base_key_exchange) { oshmem_shmem_barrier(); return; } nprocs = oshmem_num_procs(); my_pe = oshmem_my_proc_id(); /* buffer allocation for num_transports * message sizes and offsets */ rcv_size = (int *)malloc(nprocs * sizeof(int)); if (NULL == rcv_size) { MEMHEAP_ERROR("failed to get rcv_size buffer"); rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto exit_fatal; } rcv_offsets = (int *)malloc(nprocs * sizeof(int)); if (NULL == rcv_offsets) { MEMHEAP_ERROR("failed to get rcv_offsets buffer"); rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto exit_fatal; } rcv_n_transports = (int *)malloc(nprocs * sizeof(int)); if (NULL == rcv_offsets) { MEMHEAP_ERROR("failed to get rcv_offsets buffer"); rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto exit_fatal; } /* serialize our own mkeys */ msg = OBJ_NEW(opal_buffer_t); if (NULL == msg) { MEMHEAP_ERROR("failed to get msg buffer"); rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto exit_fatal; } for (j = 0; j < memheap_map->n_segments; j++) { pack_local_mkeys(msg, 0, j, 1); } /* we assume here that int32_t returned by opal_dss.unload * is equal to size of int we use for MPI_Allgather, MPI_Allgatherv */ assert(sizeof(int32_t) == sizeof(int)); /* Do allgather */ opal_dss.unload(msg, &send_buffer, &size); MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %d segments", size, memheap_map->n_segments); /* we need to send num_transports and message sizes separately * since message sizes depend on types of btl used */ rc = oshmem_shmem_allgather(&memheap_map->num_transports, rcv_n_transports, sizeof(int)); if (MPI_SUCCESS != rc) { MEMHEAP_ERROR("allgather failed"); goto exit_fatal; } rc = oshmem_shmem_allgather(&size, rcv_size, sizeof(int)); if (MPI_SUCCESS != rc) { MEMHEAP_ERROR("allgather failed"); goto exit_fatal; } /* calculating offsets (displacements) for allgatherv */ rcv_offsets[0] = 0; for (i = 1; i < nprocs; i++) { rcv_offsets[i] = rcv_offsets[i - 1] + rcv_size[i - 1]; } buffer_size = rcv_offsets[nprocs - 1] + rcv_size[nprocs - 1]; rcv_buffer = malloc (buffer_size); if (NULL == rcv_buffer) { MEMHEAP_ERROR("failed to allocate recieve buffer"); rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto exit_fatal; } rc = oshmem_shmem_allgatherv(send_buffer, rcv_buffer, size, rcv_size, rcv_offsets); if (MPI_SUCCESS != rc) { free (rcv_buffer); MEMHEAP_ERROR("allgatherv failed"); goto exit_fatal; } opal_dss.load(msg, rcv_buffer, buffer_size); /* deserialize mkeys */ OPAL_THREAD_LOCK(&memheap_oob.lck); for (i = 0; i < nprocs; i++) { if (i == my_pe) { continue; } msg->unpack_ptr = (void *)((intptr_t) msg->base_ptr + rcv_offsets[i]); for (j = 0; j < memheap_map->n_segments; j++) { map_segment_t *s; s = &memheap_map->mem_segs[j]; if (NULL != s->mkeys_cache[i]) { MEMHEAP_VERBOSE(10, "PE%d: segment%d already exists, mkey will be replaced", i, j); } else { s->mkeys_cache[i] = (sshmem_mkey_t *) calloc(rcv_n_transports[i], sizeof(sshmem_mkey_t)); if (NULL == s->mkeys_cache[i]) { MEMHEAP_ERROR("PE%d: segment%d: Failed to allocate mkeys cache entry", i, j); oshmem_shmem_abort(-1); } } memheap_oob.mkeys = s->mkeys_cache[i]; unpack_remote_mkeys(msg, i); } } OPAL_THREAD_UNLOCK(&memheap_oob.lck); exit_fatal: if (rcv_size) { free(rcv_size); } if (rcv_offsets) { free(rcv_offsets); } if (rcv_n_transports) { free(rcv_n_transports); } if (send_buffer) { free(send_buffer); } if (msg) { OBJ_RELEASE(msg); } /* This function requires abort in any error case */ if (OSHMEM_SUCCESS != rc) { oshmem_shmem_abort(rc); } }
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs) { spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL; spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL; spml_ikrit_mxm_ep_conn_info_t my_ep_info; size_t mxm_addr_len = MXM_MAX_ADDR_LEN; mxm_error_t err; size_t i, n; int rc = OSHMEM_ERROR; ompi_proc_t *proc_self; int my_rank = oshmem_my_proc_id(); OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t); /* Allocate connection requests */ ep_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs); if (NULL == ep_info) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } if (mca_spml_ikrit.hw_rdma_channel) { ep_hw_rdma_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs); if (NULL == ep_hw_rdma_info) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } } mca_spml_ikrit.mxm_peers = (mxm_peer_t *) calloc(nprocs , sizeof(mxm_peer_t)); if (NULL == mca_spml_ikrit.mxm_peers) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } memset(&my_ep_info, 0, sizeof(my_ep_info)); if (mca_spml_ikrit.hw_rdma_channel) { err = mxm_ep_get_address(mca_spml_ikrit.mxm_hw_rdma_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len); if (MXM_OK != err) { orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true, mxm_error_string(err)); rc = OSHMEM_ERROR; goto bail; } oshmem_shmem_allgather(&my_ep_info, ep_hw_rdma_info, sizeof(spml_ikrit_mxm_ep_conn_info_t)); } err = mxm_ep_get_address(mca_spml_ikrit.mxm_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len); if (MXM_OK != err) { orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true, mxm_error_string(err)); rc = OSHMEM_ERROR; goto bail; } oshmem_shmem_allgather(&my_ep_info, ep_info, sizeof(spml_ikrit_mxm_ep_conn_info_t)); opal_progress_register(spml_ikrit_progress); /* Get the EP connection requests for all the processes from modex */ for (n = 0; n < nprocs; ++n) { /* mxm 2.0 keeps its connections on a list. Make sure * that list have different order on every rank */ i = (my_rank + n) % nprocs; mxm_peer_construct(&mca_spml_ikrit.mxm_peers[i]); err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, ep_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i].mxm_conn); if (MXM_OK != err) { SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err)); goto bail; } mxm_conn_ctx_set(mca_spml_ikrit.mxm_peers[i].mxm_conn, &mca_spml_ikrit.mxm_peers[i]); if (mca_spml_ikrit.hw_rdma_channel) { err = mxm_ep_connect(mca_spml_ikrit.mxm_hw_rdma_ep, ep_hw_rdma_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn); if (MXM_OK != err) { SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err)); goto bail; } } else { mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn = mca_spml_ikrit.mxm_peers[i].mxm_conn; } } if (ep_info) free(ep_info); if (ep_hw_rdma_info) free(ep_hw_rdma_info); if (mca_spml_ikrit.bulk_connect) { /* Need a barrier to ensure remote peers already created connection */ oshmem_shmem_barrier(); mxm_ep_wireup(mca_spml_ikrit.mxm_ep); } proc_self = oshmem_proc_group_find(oshmem_group_all, my_rank); /* identify local processes and change transport to SHM */ for (i = 0; i < nprocs; i++) { if (procs[i]->super.proc_name.jobid != proc_self->super.proc_name.jobid || !OPAL_PROC_ON_LOCAL_NODE(procs[i]->super.proc_flags)) { continue; } if (procs[i] == proc_self) continue; /* use zcopy for put/get via sysv shared memory with fallback to RDMA */ mca_spml_ikrit.mxm_peers[i].ptl_id = MXM_PTL_SHM; } SPML_VERBOSE(50, "*** ADDED PROCS ***"); return OSHMEM_SUCCESS; bail: if (ep_info) free(ep_info); if (ep_hw_rdma_info) free(ep_hw_rdma_info); SPML_ERROR("add procs FAILED rc=%d", rc); return rc; }
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs) { spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL; spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL; spml_ikrit_mxm_ep_conn_info_t my_ep_info = {{0}}; #if MXM_API < MXM_VERSION(2,0) mxm_conn_req_t *conn_reqs; int timeout; #else size_t mxm_addr_len = MXM_MAX_ADDR_LEN; #endif mxm_error_t err; size_t i, n; int rc = OSHMEM_ERROR; ompi_proc_t *proc_self; int my_rank = oshmem_my_proc_id(); OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t); /* Allocate connection requests */ #if MXM_API < MXM_VERSION(2,0) conn_reqs = malloc(nprocs * sizeof(mxm_conn_req_t)); if (NULL == conn_reqs) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } memset(conn_reqs, 0x0, sizeof(mxm_conn_req_t)); #endif ep_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs); if (NULL == ep_info) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } if (mca_spml_ikrit.hw_rdma_channel) { ep_hw_rdma_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs); if (NULL == ep_hw_rdma_info) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } } mca_spml_ikrit.mxm_peers = (mxm_peer_t **) malloc(nprocs * sizeof(*(mca_spml_ikrit.mxm_peers))); if (NULL == mca_spml_ikrit.mxm_peers) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } #if MXM_API < MXM_VERSION(2,0) if (OSHMEM_SUCCESS != spml_ikrit_get_ep_address(&my_ep_info, MXM_PTL_SELF)) { rc = OSHMEM_ERROR; goto bail; } if (OSHMEM_SUCCESS != spml_ikrit_get_ep_address(&my_ep_info, MXM_PTL_RDMA)) { rc = OSHMEM_ERROR; goto bail; } #else if (mca_spml_ikrit.hw_rdma_channel) { err = mxm_ep_get_address(mca_spml_ikrit.mxm_hw_rdma_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len); if (MXM_OK != err) { orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true, mxm_error_string(err)); rc = OSHMEM_ERROR; goto bail; } oshmem_shmem_allgather(&my_ep_info, ep_hw_rdma_info, sizeof(spml_ikrit_mxm_ep_conn_info_t)); } err = mxm_ep_get_address(mca_spml_ikrit.mxm_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len); if (MXM_OK != err) { orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true, mxm_error_string(err)); rc = OSHMEM_ERROR; goto bail; } #endif oshmem_shmem_allgather(&my_ep_info, ep_info, sizeof(spml_ikrit_mxm_ep_conn_info_t)); opal_progress_register(spml_ikrit_progress); /* Get the EP connection requests for all the processes from modex */ for (n = 0; n < nprocs; ++n) { /* mxm 2.0 keeps its connections on a list. Make sure * that list have different order on every rank */ i = (my_rank + n) % nprocs; mca_spml_ikrit.mxm_peers[i] = OBJ_NEW(mxm_peer_t); if (NULL == mca_spml_ikrit.mxm_peers[i]) { rc = OSHMEM_ERR_OUT_OF_RESOURCE; goto bail; } mca_spml_ikrit.mxm_peers[i]->pe = i; #if MXM_API < MXM_VERSION(2,0) conn_reqs[i].ptl_addr[MXM_PTL_SELF] = (struct sockaddr *) &ep_info[i].addr.ptl_addr[MXM_PTL_SELF]; conn_reqs[i].ptl_addr[MXM_PTL_SHM] = NULL; conn_reqs[i].ptl_addr[MXM_PTL_RDMA] = (struct sockaddr *) &ep_info[i].addr.ptl_addr[MXM_PTL_RDMA]; #else err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, ep_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i]->mxm_conn); if (MXM_OK != err) { SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err)); goto bail; } if (OSHMEM_SUCCESS != create_ptl_idx(i)) goto bail; mxm_conn_ctx_set(mca_spml_ikrit.mxm_peers[i]->mxm_conn, mca_spml_ikrit.mxm_peers[i]); if (mca_spml_ikrit.hw_rdma_channel) { err = mxm_ep_connect(mca_spml_ikrit.mxm_hw_rdma_ep, ep_hw_rdma_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn); if (MXM_OK != err) { SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err)); goto bail; } } else { mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn = mca_spml_ikrit.mxm_peers[i]->mxm_conn; } #endif } #if MXM_API < MXM_VERSION(2,0) /* Connect to remote peers */ if (mxm_get_version() < MXM_VERSION(1,5)) { timeout = 1000; } else { timeout = -1; } err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, conn_reqs, nprocs, timeout); if (MXM_OK != err) { SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err)); for (i = 0; i < nprocs; ++i) { if (MXM_OK != conn_reqs[i].error) { SPML_ERROR("MXM EP connect to %s error: %s\n", procs[i]->proc_hostname, mxm_error_string(conn_reqs[i].error)); } } rc = OSHMEM_ERR_CONNECTION_FAILED; goto bail; } /* Save returned connections */ for (i = 0; i < nprocs; ++i) { mca_spml_ikrit.mxm_peers[i]->mxm_conn = conn_reqs[i].conn; if (OSHMEM_SUCCESS != create_ptl_idx(i)) { rc = OSHMEM_ERR_CONNECTION_FAILED; goto bail; } mxm_conn_ctx_set(conn_reqs[i].conn, mca_spml_ikrit.mxm_peers[i]); } if (conn_reqs) free(conn_reqs); #endif if (ep_info) free(ep_info); if (ep_hw_rdma_info) free(ep_hw_rdma_info); #if MXM_API >= MXM_VERSION(2,0) if (mca_spml_ikrit.bulk_connect) { /* Need a barrier to ensure remote peers already created connection */ oshmem_shmem_barrier(); mxm_ep_wireup(mca_spml_ikrit.mxm_ep); } #endif proc_self = oshmem_proc_group_find(oshmem_group_all, my_rank); /* identify local processes and change transport to SHM */ for (i = 0; i < nprocs; i++) { if (procs[i]->super.proc_name.jobid != proc_self->super.proc_name.jobid || !OPAL_PROC_ON_LOCAL_NODE(procs[i]->super.proc_flags)) { continue; } if (procs[i] == proc_self) continue; /* use zcopy for put/get via sysv shared memory */ OSHMEM_PROC_DATA(procs[i])->transport_ids[0] = MXM_PTL_SHM; OSHMEM_PROC_DATA(procs[i])->transport_ids[1] = MXM_PTL_RDMA; OSHMEM_PROC_DATA(procs[i])->num_transports = 2; } SPML_VERBOSE(50, "*** ADDED PROCS ***"); return OSHMEM_SUCCESS; bail: #if MXM_API < MXM_VERSION(2,0) if (conn_reqs) free(conn_reqs); #endif if (ep_info) free(ep_info); if (ep_hw_rdma_info) free(ep_hw_rdma_info); SPML_ERROR("add procs FAILED rc=%d", rc); return rc; }
static int _shmem_finalize(void) { int ret = OSHMEM_SUCCESS; shmem_barrier_all(); shmem_lock_finalize(); /* Finalize preconnect framework */ if (OSHMEM_SUCCESS != (ret = oshmem_shmem_preconnect_all_finalize())) { return ret; } /* free requests */ if (OSHMEM_SUCCESS != (ret = oshmem_request_finalize())) { return ret; } /* must free cached groups before we kill collectives */ if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_free())) { return ret; } /* We need to call mca_scoll_base_group_unselect explicitly for each group * that are not freed by oshmem_group_cache_list_free. We can only release its collectives at this point */ mca_scoll_base_group_unselect(oshmem_group_all); mca_scoll_base_group_unselect(oshmem_group_self); /* Close down MCA modules */ if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_atomic_base_framework) ) ) { return ret; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_scoll_base_framework) ) ) { return ret; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_memheap_base_framework) ) ) { return ret; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_sshmem_base_framework) ) ) { return ret; } if (OSHMEM_SUCCESS != (ret = MCA_SPML_CALL(del_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count)))) { return ret; } oshmem_shmem_barrier(); /* free spml resource */ if (OSHMEM_SUCCESS != (ret = mca_spml_base_finalize())) { return ret; } if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_spml_base_framework) ) ) { return ret; } /* free op resources */ if (OSHMEM_SUCCESS != (ret = oshmem_op_finalize())) { return ret; } /* free proc_group resources */ if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_finalize())) { return ret; } /* free proc resources */ if (OSHMEM_SUCCESS != (ret = oshmem_proc_finalize())) { return ret; } /* free info resources */ if (OSHMEM_SUCCESS != (ret = oshmem_info_finalize())) { return ret; } return ret; }