int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
{
size_t i, n;
int my_rank = oshmem_my_proc_id();
oshmem_shmem_barrier();
#if MXM_API >= MXM_VERSION(2,0)
if (mca_spml_ikrit.bulk_disconnect) {
mxm_ep_powerdown(mca_spml_ikrit.mxm_ep);
}
#endif
while (NULL != opal_list_remove_first(&mca_spml_ikrit.active_peers)) {
};
OBJ_DESTRUCT(&mca_spml_ikrit.active_peers);
for (n = 0; n < nprocs; n++) {
i = (my_rank + n) % nprocs;
if (mca_spml_ikrit.mxm_peers[i]->mxm_conn) {
mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i]->mxm_conn);
}
if (mca_spml_ikrit.hw_rdma_channel && mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn) {
mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn);
}
destroy_ptl_idx(i);
if (mca_spml_ikrit.mxm_peers[i]) {
OBJ_RELEASE(mca_spml_ikrit.mxm_peers[i]);
}
}
if (mca_spml_ikrit.mxm_peers)
free(mca_spml_ikrit.mxm_peers);
return OSHMEM_SUCCESS;
}
int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t i, n;
int my_rank = oshmem_my_proc_id();
oshmem_shmem_barrier();
if (mca_spml_ikrit.bulk_disconnect) {
mxm_ep_powerdown(mca_spml_ikrit.mxm_ep);
}
while (NULL != opal_list_remove_first(&mca_spml_ikrit.active_peers)) {
};
OBJ_DESTRUCT(&mca_spml_ikrit.active_peers);
for (n = 0; n < nprocs; n++) {
i = (my_rank + n) % nprocs;
mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i].mxm_conn);
if (mca_spml_ikrit.hw_rdma_channel) {
assert(mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn != mca_spml_ikrit.mxm_peers[i].mxm_conn);
mxm_ep_disconnect(mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn);
}
mxm_peer_destruct(&mca_spml_ikrit.mxm_peers[i]);
}
free(mca_spml_ikrit.mxm_peers);
return OSHMEM_SUCCESS;
}
int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
{
int my_rank = oshmem_my_proc_id();
size_t num_reqs, max_reqs;
void *dreq, **dreqs;
ucp_ep_h ep;
size_t i, n;
oshmem_shmem_barrier();
if (!mca_spml_ucx.ucp_peers) {
return OSHMEM_SUCCESS;
}
max_reqs = mca_spml_ucx.num_disconnect;
if (max_reqs > nprocs) {
max_reqs = nprocs;
}
dreqs = malloc(sizeof(*dreqs) * max_reqs);
if (dreqs == NULL) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
num_reqs = 0;
for (i = 0; i < nprocs; ++i) {
n = (i + my_rank) % nprocs;
ep = mca_spml_ucx.ucp_peers[n].ucp_conn;
if (ep == NULL) {
continue;
}
SPML_VERBOSE(10, "disconnecting from peer %d", n);
dreq = ucp_disconnect_nb(ep);
if (dreq != NULL) {
if (UCS_PTR_IS_ERR(dreq)) {
SPML_ERROR("ucp_disconnect_nb(%d) failed: %s", n,
ucs_status_string(UCS_PTR_STATUS(dreq)));
} else {
dreqs[num_reqs++] = dreq;
}
}
mca_spml_ucx.ucp_peers[n].ucp_conn = NULL;
if ((int)num_reqs >= mca_spml_ucx.num_disconnect) {
mca_spml_ucx_waitall(dreqs, &num_reqs);
}
}
mca_spml_ucx_waitall(dreqs, &num_reqs);
free(dreqs);
opal_pmix.fence(NULL, 0);
free(mca_spml_ucx.ucp_peers);
return OSHMEM_SUCCESS;
}
int mca_spml_ucx_del_procs(oshmem_proc_t** procs, size_t nprocs)
{
size_t i, n;
int my_rank = oshmem_my_proc_id();
oshmem_shmem_barrier();
if (!mca_spml_ucx.ucp_peers) {
return OSHMEM_SUCCESS;
}
for (n = 0; n < nprocs; n++) {
i = (my_rank + n) % nprocs;
if (mca_spml_ucx.ucp_peers[i].ucp_conn) {
ucp_ep_destroy(mca_spml_ucx.ucp_peers[i].ucp_conn);
}
}
free(mca_spml_ucx.ucp_peers);
return OSHMEM_SUCCESS;
}
int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
{
opal_common_ucx_del_proc_t *del_procs;
size_t i;
int ret;
oshmem_shmem_barrier();
if (!mca_spml_ucx_ctx_default.ucp_peers) {
return OSHMEM_SUCCESS;
}
del_procs = malloc(sizeof(*del_procs) * nprocs);
if (del_procs == NULL) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < nprocs; ++i) {
del_procs[i].ep = mca_spml_ucx_ctx_default.ucp_peers[i].ucp_conn;
del_procs[i].vpid = i;
/* mark peer as disconnected */
mca_spml_ucx_ctx_default.ucp_peers[i].ucp_conn = NULL;
}
ret = opal_common_ucx_del_procs(del_procs, nprocs, oshmem_my_proc_id(),
mca_spml_ucx.num_disconnect,
mca_spml_ucx_ctx_default.ucp_worker);
free(del_procs);
free(mca_spml_ucx.remote_addrs_tbl);
free(mca_spml_ucx_ctx_default.ucp_peers);
mca_spml_ucx_ctx_default.ucp_peers = NULL;
opal_common_ucx_mca_proc_added();
return ret;
}
void mca_memheap_modex_recv_all(void)
{
int i;
int j;
int nprocs, my_pe;
opal_buffer_t *msg = NULL;
void *send_buffer = NULL;
char *rcv_buffer = NULL;
int size;
int *rcv_size = NULL;
int *rcv_n_transports = NULL;
int *rcv_offsets = NULL;
int rc = OSHMEM_SUCCESS;
size_t buffer_size;
if (!mca_memheap_base_key_exchange) {
oshmem_shmem_barrier();
return;
}
nprocs = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
/* buffer allocation for num_transports
* message sizes and offsets */
rcv_size = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_size) {
MEMHEAP_ERROR("failed to get rcv_size buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rcv_offsets = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_offsets) {
MEMHEAP_ERROR("failed to get rcv_offsets buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rcv_n_transports = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_offsets) {
MEMHEAP_ERROR("failed to get rcv_offsets buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
/* serialize our own mkeys */
msg = OBJ_NEW(opal_buffer_t);
if (NULL == msg) {
MEMHEAP_ERROR("failed to get msg buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
for (j = 0; j < memheap_map->n_segments; j++) {
pack_local_mkeys(msg, 0, j, 1);
}
/* we assume here that int32_t returned by opal_dss.unload
* is equal to size of int we use for MPI_Allgather, MPI_Allgatherv */
assert(sizeof(int32_t) == sizeof(int));
/* Do allgather */
opal_dss.unload(msg, &send_buffer, &size);
MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %d segments", size, memheap_map->n_segments);
/* we need to send num_transports and message sizes separately
* since message sizes depend on types of btl used */
rc = oshmem_shmem_allgather(&memheap_map->num_transports, rcv_n_transports, sizeof(int));
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("allgather failed");
goto exit_fatal;
}
rc = oshmem_shmem_allgather(&size, rcv_size, sizeof(int));
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("allgather failed");
goto exit_fatal;
}
/* calculating offsets (displacements) for allgatherv */
rcv_offsets[0] = 0;
for (i = 1; i < nprocs; i++) {
rcv_offsets[i] = rcv_offsets[i - 1] + rcv_size[i - 1];
}
buffer_size = rcv_offsets[nprocs - 1] + rcv_size[nprocs - 1];
rcv_buffer = malloc (buffer_size);
if (NULL == rcv_buffer) {
MEMHEAP_ERROR("failed to allocate recieve buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rc = oshmem_shmem_allgatherv(send_buffer, rcv_buffer, size, rcv_size, rcv_offsets);
if (MPI_SUCCESS != rc) {
free (rcv_buffer);
MEMHEAP_ERROR("allgatherv failed");
goto exit_fatal;
}
opal_dss.load(msg, rcv_buffer, buffer_size);
/* deserialize mkeys */
OPAL_THREAD_LOCK(&memheap_oob.lck);
for (i = 0; i < nprocs; i++) {
if (i == my_pe) {
continue;
}
msg->unpack_ptr = (void *)((intptr_t) msg->base_ptr + rcv_offsets[i]);
for (j = 0; j < memheap_map->n_segments; j++) {
map_segment_t *s;
s = &memheap_map->mem_segs[j];
if (NULL != s->mkeys_cache[i]) {
MEMHEAP_VERBOSE(10, "PE%d: segment%d already exists, mkey will be replaced", i, j);
} else {
s->mkeys_cache[i] = (sshmem_mkey_t *) calloc(rcv_n_transports[i],
sizeof(sshmem_mkey_t));
if (NULL == s->mkeys_cache[i]) {
MEMHEAP_ERROR("PE%d: segment%d: Failed to allocate mkeys cache entry", i, j);
oshmem_shmem_abort(-1);
}
}
memheap_oob.mkeys = s->mkeys_cache[i];
unpack_remote_mkeys(msg, i);
}
}
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
exit_fatal:
if (rcv_size) {
free(rcv_size);
}
if (rcv_offsets) {
free(rcv_offsets);
}
if (rcv_n_transports) {
free(rcv_n_transports);
}
if (send_buffer) {
free(send_buffer);
}
if (msg) {
OBJ_RELEASE(msg);
}
/* This function requires abort in any error case */
if (OSHMEM_SUCCESS != rc) {
oshmem_shmem_abort(rc);
}
}
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs)
{
spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL;
spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL;
spml_ikrit_mxm_ep_conn_info_t my_ep_info;
size_t mxm_addr_len = MXM_MAX_ADDR_LEN;
mxm_error_t err;
size_t i, n;
int rc = OSHMEM_ERROR;
ompi_proc_t *proc_self;
int my_rank = oshmem_my_proc_id();
OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t);
/* Allocate connection requests */
ep_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs);
if (NULL == ep_info) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
if (mca_spml_ikrit.hw_rdma_channel) {
ep_hw_rdma_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs);
if (NULL == ep_hw_rdma_info) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
}
mca_spml_ikrit.mxm_peers = (mxm_peer_t *) calloc(nprocs , sizeof(mxm_peer_t));
if (NULL == mca_spml_ikrit.mxm_peers) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
memset(&my_ep_info, 0, sizeof(my_ep_info));
if (mca_spml_ikrit.hw_rdma_channel) {
err = mxm_ep_get_address(mca_spml_ikrit.mxm_hw_rdma_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len);
if (MXM_OK != err) {
orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true,
mxm_error_string(err));
rc = OSHMEM_ERROR;
goto bail;
}
oshmem_shmem_allgather(&my_ep_info, ep_hw_rdma_info,
sizeof(spml_ikrit_mxm_ep_conn_info_t));
}
err = mxm_ep_get_address(mca_spml_ikrit.mxm_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len);
if (MXM_OK != err) {
orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true,
mxm_error_string(err));
rc = OSHMEM_ERROR;
goto bail;
}
oshmem_shmem_allgather(&my_ep_info, ep_info,
sizeof(spml_ikrit_mxm_ep_conn_info_t));
opal_progress_register(spml_ikrit_progress);
/* Get the EP connection requests for all the processes from modex */
for (n = 0; n < nprocs; ++n) {
/* mxm 2.0 keeps its connections on a list. Make sure
* that list have different order on every rank */
i = (my_rank + n) % nprocs;
mxm_peer_construct(&mca_spml_ikrit.mxm_peers[i]);
err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, ep_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i].mxm_conn);
if (MXM_OK != err) {
SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
goto bail;
}
mxm_conn_ctx_set(mca_spml_ikrit.mxm_peers[i].mxm_conn, &mca_spml_ikrit.mxm_peers[i]);
if (mca_spml_ikrit.hw_rdma_channel) {
err = mxm_ep_connect(mca_spml_ikrit.mxm_hw_rdma_ep, ep_hw_rdma_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn);
if (MXM_OK != err) {
SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
goto bail;
}
} else {
mca_spml_ikrit.mxm_peers[i].mxm_hw_rdma_conn = mca_spml_ikrit.mxm_peers[i].mxm_conn;
}
}
if (ep_info)
free(ep_info);
if (ep_hw_rdma_info)
free(ep_hw_rdma_info);
if (mca_spml_ikrit.bulk_connect) {
/* Need a barrier to ensure remote peers already created connection */
oshmem_shmem_barrier();
mxm_ep_wireup(mca_spml_ikrit.mxm_ep);
}
proc_self = oshmem_proc_group_find(oshmem_group_all, my_rank);
/* identify local processes and change transport to SHM */
for (i = 0; i < nprocs; i++) {
if (procs[i]->super.proc_name.jobid != proc_self->super.proc_name.jobid ||
!OPAL_PROC_ON_LOCAL_NODE(procs[i]->super.proc_flags)) {
continue;
}
if (procs[i] == proc_self)
continue;
/* use zcopy for put/get via sysv shared memory with fallback to RDMA */
mca_spml_ikrit.mxm_peers[i].ptl_id = MXM_PTL_SHM;
}
SPML_VERBOSE(50, "*** ADDED PROCS ***");
return OSHMEM_SUCCESS;
bail:
if (ep_info)
free(ep_info);
if (ep_hw_rdma_info)
free(ep_hw_rdma_info);
SPML_ERROR("add procs FAILED rc=%d", rc);
return rc;
}
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs)
{
spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL;
spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL;
spml_ikrit_mxm_ep_conn_info_t my_ep_info = {{0}};
#if MXM_API < MXM_VERSION(2,0)
mxm_conn_req_t *conn_reqs;
int timeout;
#else
size_t mxm_addr_len = MXM_MAX_ADDR_LEN;
#endif
mxm_error_t err;
size_t i, n;
int rc = OSHMEM_ERROR;
ompi_proc_t *proc_self;
int my_rank = oshmem_my_proc_id();
OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t);
/* Allocate connection requests */
#if MXM_API < MXM_VERSION(2,0)
conn_reqs = malloc(nprocs * sizeof(mxm_conn_req_t));
if (NULL == conn_reqs) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
memset(conn_reqs, 0x0, sizeof(mxm_conn_req_t));
#endif
ep_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs);
if (NULL == ep_info) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
if (mca_spml_ikrit.hw_rdma_channel) {
ep_hw_rdma_info = calloc(sizeof(spml_ikrit_mxm_ep_conn_info_t), nprocs);
if (NULL == ep_hw_rdma_info) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
}
mca_spml_ikrit.mxm_peers = (mxm_peer_t **) malloc(nprocs
* sizeof(*(mca_spml_ikrit.mxm_peers)));
if (NULL == mca_spml_ikrit.mxm_peers) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
#if MXM_API < MXM_VERSION(2,0)
if (OSHMEM_SUCCESS
!= spml_ikrit_get_ep_address(&my_ep_info, MXM_PTL_SELF)) {
rc = OSHMEM_ERROR;
goto bail;
}
if (OSHMEM_SUCCESS
!= spml_ikrit_get_ep_address(&my_ep_info, MXM_PTL_RDMA)) {
rc = OSHMEM_ERROR;
goto bail;
}
#else
if (mca_spml_ikrit.hw_rdma_channel) {
err = mxm_ep_get_address(mca_spml_ikrit.mxm_hw_rdma_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len);
if (MXM_OK != err) {
orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true,
mxm_error_string(err));
rc = OSHMEM_ERROR;
goto bail;
}
oshmem_shmem_allgather(&my_ep_info, ep_hw_rdma_info,
sizeof(spml_ikrit_mxm_ep_conn_info_t));
}
err = mxm_ep_get_address(mca_spml_ikrit.mxm_ep, &my_ep_info.addr.ep_addr, &mxm_addr_len);
if (MXM_OK != err) {
orte_show_help("help-oshmem-spml-ikrit.txt", "unable to get endpoint address", true,
mxm_error_string(err));
rc = OSHMEM_ERROR;
goto bail;
}
#endif
oshmem_shmem_allgather(&my_ep_info, ep_info,
sizeof(spml_ikrit_mxm_ep_conn_info_t));
opal_progress_register(spml_ikrit_progress);
/* Get the EP connection requests for all the processes from modex */
for (n = 0; n < nprocs; ++n) {
/* mxm 2.0 keeps its connections on a list. Make sure
* that list have different order on every rank */
i = (my_rank + n) % nprocs;
mca_spml_ikrit.mxm_peers[i] = OBJ_NEW(mxm_peer_t);
if (NULL == mca_spml_ikrit.mxm_peers[i]) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto bail;
}
mca_spml_ikrit.mxm_peers[i]->pe = i;
#if MXM_API < MXM_VERSION(2,0)
conn_reqs[i].ptl_addr[MXM_PTL_SELF] =
(struct sockaddr *) &ep_info[i].addr.ptl_addr[MXM_PTL_SELF];
conn_reqs[i].ptl_addr[MXM_PTL_SHM] = NULL;
conn_reqs[i].ptl_addr[MXM_PTL_RDMA] =
(struct sockaddr *) &ep_info[i].addr.ptl_addr[MXM_PTL_RDMA];
#else
err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, ep_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i]->mxm_conn);
if (MXM_OK != err) {
SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
goto bail;
}
if (OSHMEM_SUCCESS != create_ptl_idx(i))
goto bail;
mxm_conn_ctx_set(mca_spml_ikrit.mxm_peers[i]->mxm_conn, mca_spml_ikrit.mxm_peers[i]);
if (mca_spml_ikrit.hw_rdma_channel) {
err = mxm_ep_connect(mca_spml_ikrit.mxm_hw_rdma_ep, ep_hw_rdma_info[i].addr.ep_addr, &mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn);
if (MXM_OK != err) {
SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
goto bail;
}
} else {
mca_spml_ikrit.mxm_peers[i]->mxm_hw_rdma_conn = mca_spml_ikrit.mxm_peers[i]->mxm_conn;
}
#endif
}
#if MXM_API < MXM_VERSION(2,0)
/* Connect to remote peers */
if (mxm_get_version() < MXM_VERSION(1,5)) {
timeout = 1000;
} else {
timeout = -1;
}
err = mxm_ep_connect(mca_spml_ikrit.mxm_ep, conn_reqs, nprocs, timeout);
if (MXM_OK != err) {
SPML_ERROR("MXM returned connect error: %s\n", mxm_error_string(err));
for (i = 0; i < nprocs; ++i) {
if (MXM_OK != conn_reqs[i].error) {
SPML_ERROR("MXM EP connect to %s error: %s\n",
procs[i]->proc_hostname, mxm_error_string(conn_reqs[i].error));
}
}
rc = OSHMEM_ERR_CONNECTION_FAILED;
goto bail;
}
/* Save returned connections */
for (i = 0; i < nprocs; ++i) {
mca_spml_ikrit.mxm_peers[i]->mxm_conn = conn_reqs[i].conn;
if (OSHMEM_SUCCESS != create_ptl_idx(i)) {
rc = OSHMEM_ERR_CONNECTION_FAILED;
goto bail;
}
mxm_conn_ctx_set(conn_reqs[i].conn, mca_spml_ikrit.mxm_peers[i]);
}
if (conn_reqs)
free(conn_reqs);
#endif
if (ep_info)
free(ep_info);
if (ep_hw_rdma_info)
free(ep_hw_rdma_info);
#if MXM_API >= MXM_VERSION(2,0)
if (mca_spml_ikrit.bulk_connect) {
/* Need a barrier to ensure remote peers already created connection */
oshmem_shmem_barrier();
mxm_ep_wireup(mca_spml_ikrit.mxm_ep);
}
#endif
proc_self = oshmem_proc_group_find(oshmem_group_all, my_rank);
/* identify local processes and change transport to SHM */
for (i = 0; i < nprocs; i++) {
if (procs[i]->super.proc_name.jobid != proc_self->super.proc_name.jobid ||
!OPAL_PROC_ON_LOCAL_NODE(procs[i]->super.proc_flags)) {
continue;
}
if (procs[i] == proc_self)
continue;
/* use zcopy for put/get via sysv shared memory */
OSHMEM_PROC_DATA(procs[i])->transport_ids[0] = MXM_PTL_SHM;
OSHMEM_PROC_DATA(procs[i])->transport_ids[1] = MXM_PTL_RDMA;
OSHMEM_PROC_DATA(procs[i])->num_transports = 2;
}
SPML_VERBOSE(50, "*** ADDED PROCS ***");
return OSHMEM_SUCCESS;
bail:
#if MXM_API < MXM_VERSION(2,0)
if (conn_reqs)
free(conn_reqs);
#endif
if (ep_info)
free(ep_info);
if (ep_hw_rdma_info)
free(ep_hw_rdma_info);
SPML_ERROR("add procs FAILED rc=%d", rc);
return rc;
}
static int _shmem_finalize(void)
{
int ret = OSHMEM_SUCCESS;
shmem_barrier_all();
shmem_lock_finalize();
/* Finalize preconnect framework */
if (OSHMEM_SUCCESS != (ret = oshmem_shmem_preconnect_all_finalize())) {
return ret;
}
/* free requests */
if (OSHMEM_SUCCESS != (ret = oshmem_request_finalize())) {
return ret;
}
/* must free cached groups before we kill collectives */
if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_free())) {
return ret;
}
/* We need to call mca_scoll_base_group_unselect explicitly for each group
* that are not freed by oshmem_group_cache_list_free. We can only release its collectives at this point */
mca_scoll_base_group_unselect(oshmem_group_all);
mca_scoll_base_group_unselect(oshmem_group_self);
/* Close down MCA modules */
if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_atomic_base_framework) ) ) {
return ret;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_scoll_base_framework) ) ) {
return ret;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_memheap_base_framework) ) ) {
return ret;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_sshmem_base_framework) ) ) {
return ret;
}
if (OSHMEM_SUCCESS
!= (ret =
MCA_SPML_CALL(del_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count)))) {
return ret;
}
oshmem_shmem_barrier();
/* free spml resource */
if (OSHMEM_SUCCESS != (ret = mca_spml_base_finalize())) {
return ret;
}
if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_spml_base_framework) ) ) {
return ret;
}
/* free op resources */
if (OSHMEM_SUCCESS != (ret = oshmem_op_finalize())) {
return ret;
}
/* free proc_group resources */
if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_finalize())) {
return ret;
}
/* free proc resources */
if (OSHMEM_SUCCESS != (ret = oshmem_proc_finalize())) {
return ret;
}
/* free info resources */
if (OSHMEM_SUCCESS != (ret = oshmem_info_finalize())) {
return ret;
}
return ret;
}
