/* TODO: move func into common place, use it with rkey exchng too */
static int oshmem_shmem_xchng(
void *local_data, int local_size, int nprocs,
void **rdata_p, int **roffsets_p, int **rsizes_p)
{
int *rcv_sizes = NULL;
int *rcv_offsets = NULL;
void *rcv_buf = NULL;
int rc;
int i;
/* do llgatherv */
rcv_offsets = malloc(nprocs * sizeof(*rcv_offsets));
if (NULL == rcv_offsets) {
goto err;
}
/* todo: move into separate function. do allgatherv */
rcv_sizes = malloc(nprocs * sizeof(*rcv_sizes));
if (NULL == rcv_sizes) {
goto err;
}
rc = oshmem_shmem_allgather(&local_size, rcv_sizes, sizeof(int));
if (MPI_SUCCESS != rc) {
goto err;
}
/* calculate displacements */
rcv_offsets[0] = 0;
for (i = 1; i < nprocs; i++) {
rcv_offsets[i] = rcv_offsets[i - 1] + rcv_sizes[i - 1];
}
rcv_buf = malloc(rcv_offsets[nprocs - 1] + rcv_sizes[nprocs - 1]);
if (NULL == rcv_buf) {
goto err;
}
rc = oshmem_shmem_allgatherv(local_data, rcv_buf, local_size, rcv_sizes, rcv_offsets);
if (MPI_SUCCESS != rc) {
goto err;
}
*rdata_p = rcv_buf;
*roffsets_p = rcv_offsets;
*rsizes_p = rcv_sizes;
return OSHMEM_SUCCESS;
err:
if (rcv_buf)
free(rcv_buf);
if (rcv_offsets)
free(rcv_offsets);
if (rcv_sizes)
free(rcv_sizes);
return OSHMEM_ERROR;
}
void mca_memheap_modex_recv_all(void)
{
int i;
int j;
int nprocs, my_pe;
opal_buffer_t *msg = NULL;
void *send_buffer = NULL;
char *rcv_buffer = NULL;
int size;
int *rcv_size = NULL;
int *rcv_n_transports = NULL;
int *rcv_offsets = NULL;
int rc = OSHMEM_SUCCESS;
size_t buffer_size;
if (!mca_memheap_base_key_exchange) {
oshmem_shmem_barrier();
return;
}
nprocs = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
/* buffer allocation for num_transports
* message sizes and offsets */
rcv_size = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_size) {
MEMHEAP_ERROR("failed to get rcv_size buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rcv_offsets = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_offsets) {
MEMHEAP_ERROR("failed to get rcv_offsets buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rcv_n_transports = (int *)malloc(nprocs * sizeof(int));
if (NULL == rcv_offsets) {
MEMHEAP_ERROR("failed to get rcv_offsets buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
/* serialize our own mkeys */
msg = OBJ_NEW(opal_buffer_t);
if (NULL == msg) {
MEMHEAP_ERROR("failed to get msg buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
for (j = 0; j < memheap_map->n_segments; j++) {
pack_local_mkeys(msg, 0, j, 1);
}
/* we assume here that int32_t returned by opal_dss.unload
* is equal to size of int we use for MPI_Allgather, MPI_Allgatherv */
assert(sizeof(int32_t) == sizeof(int));
/* Do allgather */
opal_dss.unload(msg, &send_buffer, &size);
MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %d segments", size, memheap_map->n_segments);
/* we need to send num_transports and message sizes separately
* since message sizes depend on types of btl used */
rc = oshmem_shmem_allgather(&memheap_map->num_transports, rcv_n_transports, sizeof(int));
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("allgather failed");
goto exit_fatal;
}
rc = oshmem_shmem_allgather(&size, rcv_size, sizeof(int));
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("allgather failed");
goto exit_fatal;
}
/* calculating offsets (displacements) for allgatherv */
rcv_offsets[0] = 0;
for (i = 1; i < nprocs; i++) {
rcv_offsets[i] = rcv_offsets[i - 1] + rcv_size[i - 1];
}
buffer_size = rcv_offsets[nprocs - 1] + rcv_size[nprocs - 1];
rcv_buffer = malloc (buffer_size);
if (NULL == rcv_buffer) {
MEMHEAP_ERROR("failed to allocate recieve buffer");
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
goto exit_fatal;
}
rc = oshmem_shmem_allgatherv(send_buffer, rcv_buffer, size, rcv_size, rcv_offsets);
if (MPI_SUCCESS != rc) {
free (rcv_buffer);
MEMHEAP_ERROR("allgatherv failed");
goto exit_fatal;
}
opal_dss.load(msg, rcv_buffer, buffer_size);
/* deserialize mkeys */
OPAL_THREAD_LOCK(&memheap_oob.lck);
for (i = 0; i < nprocs; i++) {
if (i == my_pe) {
continue;
}
msg->unpack_ptr = (void *)((intptr_t) msg->base_ptr + rcv_offsets[i]);
for (j = 0; j < memheap_map->n_segments; j++) {
map_segment_t *s;
s = &memheap_map->mem_segs[j];
if (NULL != s->mkeys_cache[i]) {
MEMHEAP_VERBOSE(10, "PE%d: segment%d already exists, mkey will be replaced", i, j);
} else {
s->mkeys_cache[i] = (sshmem_mkey_t *) calloc(rcv_n_transports[i],
sizeof(sshmem_mkey_t));
if (NULL == s->mkeys_cache[i]) {
MEMHEAP_ERROR("PE%d: segment%d: Failed to allocate mkeys cache entry", i, j);
oshmem_shmem_abort(-1);
}
}
memheap_oob.mkeys = s->mkeys_cache[i];
unpack_remote_mkeys(msg, i);
}
}
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
exit_fatal:
if (rcv_size) {
free(rcv_size);
}
if (rcv_offsets) {
free(rcv_offsets);
}
if (rcv_n_transports) {
free(rcv_n_transports);
}
if (send_buffer) {
free(send_buffer);
}
if (msg) {
OBJ_RELEASE(msg);
}
/* This function requires abort in any error case */
if (OSHMEM_SUCCESS != rc) {
oshmem_shmem_abort(rc);
}
}
