/** Create ARMCI mutexes. Collective.
*
* @param[in] count Number of mutexes to create on the calling process
*/
int ARMCI_Create_mutexes(int count) {
if (armci_mutex_hdl != NULL)
ARMCII_Error("attempted to create ARMCI mutexes multiple times");
armci_mutex_hdl = ARMCIX_Create_mutexes_hdl(count, &ARMCI_GROUP_WORLD);
if (armci_mutex_hdl != NULL)
return 0;
else
return 1;
}
/** Create a distributed shared memory region. Collective on ARMCI group.
*
* @param[in] local_size Size of the local slice of the memory region.
* @param[out] base_ptrs Array of base pointers for each process in group.
* @param[in] group Group on which to perform allocation.
* @return Pointer to the memory region object.
*/
gmr_t *gmr_create(gmr_size_t local_size, void **base_ptrs, ARMCI_Group *group) {
int i;
gmr_size_t aggregate_size;
int alloc_me, alloc_nproc;
int world_me, world_nproc;
MPI_Group world_group, alloc_group;
gmr_t *mreg;
gmr_slice_t *alloc_slices, gmr_slice;
ARMCII_Assert(local_size >= 0);
ARMCII_Assert(group != NULL);
MPI_Comm_rank(group->comm, &alloc_me);
MPI_Comm_size(group->comm, &alloc_nproc);
MPI_Comm_rank(ARMCI_GROUP_WORLD.comm, &world_me);
MPI_Comm_size(ARMCI_GROUP_WORLD.comm, &world_nproc);
mreg = malloc(sizeof(gmr_t));
ARMCII_Assert(mreg != NULL);
mreg->slices = malloc(sizeof(gmr_slice_t)*world_nproc);
ARMCII_Assert(mreg->slices != NULL);
alloc_slices = malloc(sizeof(gmr_slice_t)*alloc_nproc);
ARMCII_Assert(alloc_slices != NULL);
mreg->group = *group; /* NOTE: I think it is invalid in GA/ARMCI to
free a group before its allocations. If
this is not the case, then assignment here
is incorrect and this should really
duplicated the group (communicator). */
mreg->nslices = world_nproc;
mreg->access_mode = ARMCIX_MODE_ALL;
mreg->lock_state = GMR_LOCK_UNLOCKED;
mreg->dla_lock_count = 0;
mreg->prev = NULL;
mreg->next = NULL;
/* Allocate my slice of the GMR */
alloc_slices[alloc_me].size = local_size;
if (local_size == 0) {
alloc_slices[alloc_me].base = NULL;
} else {
MPI_Alloc_mem(local_size, MPI_INFO_NULL, &(alloc_slices[alloc_me].base));
ARMCII_Assert(alloc_slices[alloc_me].base != NULL);
}
/* Debugging: Zero out shared memory if enabled */
if (ARMCII_GLOBAL_STATE.debug_alloc && local_size > 0) {
ARMCII_Assert(alloc_slices[alloc_me].base != NULL);
ARMCII_Bzero(alloc_slices[alloc_me].base, local_size);
}
/* All-to-all on <base, size> to build up slices vector */
gmr_slice = alloc_slices[alloc_me];
MPI_Allgather( &gmr_slice, sizeof(gmr_slice_t), MPI_BYTE,
alloc_slices, sizeof(gmr_slice_t), MPI_BYTE, group->comm);
/* Check for a global size 0 allocation */
for (i = aggregate_size = 0; i < alloc_nproc; i++) {
aggregate_size += alloc_slices[i].size;
}
/* Everyone asked for 0 bytes, return a NULL vector */
if (aggregate_size == 0) {
free(alloc_slices);
free(mreg->slices);
free(mreg);
for (i = 0; i < alloc_nproc; i++)
base_ptrs[i] = NULL;
return NULL;
}
MPI_Win_create(alloc_slices[alloc_me].base, (MPI_Aint) local_size, 1, MPI_INFO_NULL, group->comm, &mreg->window);
/* Populate the base pointers array */
for (i = 0; i < alloc_nproc; i++)
base_ptrs[i] = alloc_slices[i].base;
/* We have to do lookup on global ranks, so shovel the contents of
alloc_slices into the mreg->slices array which is indexed by global rank. */
memset(mreg->slices, 0, sizeof(gmr_slice_t)*world_nproc);
MPI_Comm_group(ARMCI_GROUP_WORLD.comm, &world_group);
MPI_Comm_group(group->comm, &alloc_group);
for (i = 0; i < alloc_nproc; i++) {
int world_rank;
MPI_Group_translate_ranks(alloc_group, 1, &i, world_group, &world_rank);
mreg->slices[world_rank] = alloc_slices[i];
}
free(alloc_slices);
MPI_Group_free(&world_group);
MPI_Group_free(&alloc_group);
/* Create the RMW mutex: Keeps RMW operations atomic wrt each other */
mreg->rmw_mutex = ARMCIX_Create_mutexes_hdl(1, group);
/* Append the new region onto the region list */
if (gmr_list == NULL) {
gmr_list = mreg;
} else {
gmr_t *parent = gmr_list;
while (parent->next != NULL)
parent = parent->next;
parent->next = mreg;
mreg->prev = parent;
}
return mreg;
}
