/*\ ************** Begin Group Collective Memory Allocation ******************
* returns array of pointers to blocks of memory allocated by everybody
* Note: as the same shared memory region can be mapped at different locations
* in each process address space, the array might hold different values
* on every process. However, the addresses are legitimate
* and can be used in the ARMCI data transfer operations.
* ptr_arr[nproc]
\*/
int ARMCI_Malloc_group(void *ptr_arr[], armci_size_t bytes,
ARMCI_Group *group)
{
void *ptr;
int grp_me, grp_nproc;
ARMCI_PR_DBG("enter",0);
ARMCI_Group_size(group, &grp_nproc);
ARMCI_Group_rank(group, &grp_me);
if(DEBUG_)fprintf(stderr,"%d (grp_id=%d) bytes in armci_malloc_group %d\n",
armci_me, grp_me, (int)bytes);
ARMCI_PR_DBG("exit",0);
return(0);
}
void test_one_group(ARMCI_Group *group, int *pid_list) {
int grp_me, grp_size;
int i,j,src_proc,dst_proc;
double *ddst_put[MAXPROC];
double dsrc[ELEMS];
int elems[2] = {MAXPROC,ELEMS};
int value = -1, bytes, world_me;
MP_MYID(&world_me);
ARMCI_Group_rank(group, &grp_me);
ARMCI_Group_size(group, &grp_size);
if(grp_me==0) printf("GROUP SIZE = %d\n", grp_size);
printf("%d:group rank = %d\n", me, grp_me);
src_proc = 0; dst_proc = grp_size-1;
bytes = ELEMS*sizeof(double);
ARMCI_Malloc_group((void **)ddst_put, bytes, group);
for(i=0; i<ELEMS; i++) dsrc[i]=i*1.001*(grp_me+1);
for(i=0; i<ELEMS; i++) ddst_put[grp_me][i]=-1.0;
armci_msg_group_barrier(group);
if(grp_me==src_proc) {
/* NOTE: make sure to specify absolute ids in ARMCI calls */
ARMCI_Put(dsrc, &ddst_put[dst_proc][0], bytes,
ARMCI_Absolute_id(group,dst_proc));
}
armci_msg_group_barrier(group);
/* NOTE: make sure to specify absolute ids in ARMCI calls */
ARMCI_Fence(ARMCI_Absolute_id(group,dst_proc));
sleep(1);
/* Verify*/
if(grp_me==dst_proc) {
for(j=0; j<ELEMS; j++) {
if(ARMCI_ABS(ddst_put[grp_me][j]-j*1.001*(src_proc+1)) > 0.1) {
printf("\t%d: ddst_put[%d][%d] = %lf and expected value is %lf\n",
me, grp_me, j, ddst_put[grp_me][j], j*1.001*(src_proc+1));
ARMCI_Error("groups: armci put failed...1", 0);
}
}
printf("\n%d(%d): Test O.K. Verified\n", dst_proc, world_me);
}
armci_msg_group_barrier(group);
ARMCI_Free_group(ddst_put[grp_me], group);
}
void armci_altix_shm_malloc_group(void *ptr_arr[], armci_size_t bytes,
ARMCI_Group *group) {
long size=bytes;
void *ptr;
int i,grp_me, grp_nproc;
armci_grp_attr_t *grp_attr=ARMCI_Group_getattr(group);
ARMCI_PR_DBG("enter",0);
ARMCI_Group_size(group, &grp_nproc);
ARMCI_Group_rank(group, &grp_me);
armci_msg_group_lgop(&size,1,"max",group);
ptr=kr_malloc((size_t)size, &altix_ctx_shmem_grp);
if(size!=0 && ptr==NULL)
armci_die("armci_altix_shm_malloc_group(): malloc failed for groups. Increase _SHMMAX_ALTIX_GRP", armci_me);
bzero(ptr_arr,(grp_nproc)*sizeof(void*));
ptr_arr[grp_me] = ptr;
for(i=0; i< grp_nproc; i++) if(i!=grp_me) ptr_arr[i]=shmem_ptr(ptr,ARMCI_Absolute_id(group, i));
ARMCI_PR_DBG("exit",0);
}
int ARMCI_Free_group(void *ptr, ARMCI_Group *group)
{
int grp_me, grp_nproc, grp_master, grp_clus_me;
armci_grp_attr_t *grp_attr=ARMCI_Group_getattr(group);
ARMCI_PR_DBG("enter",0);
if(!ptr)return 1;
ARMCI_Group_size(group, &grp_nproc);
ARMCI_Group_rank(group, &grp_me);
if(grp_me == MPI_UNDEFINED) { /* check if the process is in this group */
armci_die("armci_malloc_group: process is not a member in this group",
armci_me);
}
/* get the group cluster info */
grp_clus_me = grp_attr->grp_clus_me;
grp_master = grp_attr->grp_clus_info[grp_clus_me].master;
ARMCI_PR_DBG("exit",0);
return 0;
}
int ARMCI_Uses_shm_grp(ARMCI_Group *group)
{
int uses=0, grp_me, grp_nproc, grp_nclus;
ARMCI_PR_DBG("enter",0);
armci_grp_attr_t *grp_attr=ARMCI_Group_getattr(group);
ARMCI_Group_size(group, &grp_nproc);
ARMCI_Group_rank(group, &grp_me);
grp_nclus = grp_attr->grp_nclus;
#if (defined(SYSV) || defined(WIN32) || defined(MMAP) ||defined(HITACHI)) \
&& !defined(NO_SHM)
# ifdef RMA_NEEDS_SHMEM
if(grp_nproc >1) uses= 1; /* always unless serial mode */
# else
if(grp_nproc != grp_nclus)uses= 1; /* only when > 1 node used */
# endif
#endif
if(DEBUG_) fprintf(stderr,"%d (grp_id=%d):uses shmem %d\n",armci_me, grp_me, uses);
ARMCI_PR_DBG("exit",0);
return uses;
}
int main(int argc, char **argv) {
int me, nproc;
int i, *procs;
ARMCI_Group g_world, g_odd, g_even;
MPI_Init(&argc, &argv);
ARMCI_Init();
MPI_Comm_rank(MPI_COMM_WORLD, &me);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
procs = malloc(sizeof(int) * ( nproc/2 + (nproc % 2 ? 1 : 0 )));
if (me == 0) printf("ARMCI Group test starting on %d procs\n", nproc);
ARMCI_Group_get_world(&g_world);
if (me == 0) printf(" + Creating odd group\n");
for (i = 1; i < nproc; i += 2) {
procs[i/2] = i;
}
ARMCI_Group_create_child(i/2, procs, &g_odd, &g_world);
if (me == 0) printf(" + Creating even group\n");
for (i = 0; i < nproc; i += 2) {
procs[i/2] = i;
}
ARMCI_Group_create_child(i/2, procs, &g_even, &g_world);
/***********************************************************************/
{
int grp_me, grp_nproc;
double t_abs_to_grp, t_grp_to_abs;
const int iter = 1000000;
if (me == 0) {
ARMCI_Group_rank(&g_even, &grp_me);
ARMCI_Group_size(&g_even, &grp_nproc);
t_abs_to_grp = MPI_Wtime();
for (i = 0; i < iter; i++)
ARMCII_Translate_absolute_to_group(&g_even, (grp_me+1) % grp_nproc);
t_abs_to_grp = MPI_Wtime() - t_abs_to_grp;
t_grp_to_abs = MPI_Wtime();
for (i = 0; i < iter; i++)
ARMCI_Absolute_id(&g_even, (grp_me+1) % grp_nproc);
t_grp_to_abs = MPI_Wtime() - t_grp_to_abs;
printf("t_abs_to_grp = %f us, t_grp_to_abs = %f us\n", t_abs_to_grp/iter * 1.0e6, t_grp_to_abs/iter * 1.0e6);
}
ARMCI_Barrier();
}
/***********************************************************************/
if (me == 0) printf(" + Freeing groups\n");
if (me % 2 > 0)
ARMCI_Group_free(&g_odd);
else
ARMCI_Group_free(&g_even);
free(procs);
ARMCI_Finalize();
MPI_Finalize();
return 0;
}
/*
Create a child group for to the given group.
@param n IN #procs in this group (<= that in group_parent)
@param pid_list IN The list of proc ids (w.r.t. group_parent)
@param group_out OUT Handle to store the created group
@param group_parent IN Parent group
*/
void ARMCI_Group_create_child(int n, int *pid_list, ARMCI_Group *group_out,
ARMCI_Group *grp_parent) {
int i,grp_me;
ARMCI_iGroup *igroup = (ARMCI_iGroup *)group_out;
#ifdef ARMCI_GROUP
armci_grp_attr_t *grp_attr = &igroup->grp_attr;
int world_me, parent_grp_me;
#else
int rv;
ARMCI_iGroup *igroup_parent = (ARMCI_iGroup *)grp_parent;
MPI_Group *group_parent;
MPI_Comm *comm_parent;
#endif
#ifdef ARMCI_GROUP
ARMCI_Group_rank(grp_parent, &parent_grp_me);
for(i=0; i<n; i++) {
if(pid_list[i] == parent_grp_me) {
break;
}
}
if(i==n) {
/*this initialization is used in group free*/
grp_attr->nproc=0;
grp_attr->proc_list = NULL;
return; /*not in group to be created*/
}
#endif
for(i=0; i<n-1;i++) {
if(pid_list[i] > pid_list[i+1]){
armci_die("ARMCI_Group_create: Process ids are not sorted ",armci_me);
break;
}
}
#ifdef ARMCI_GROUP
grp_attr->grp_clus_info = NULL;
grp_attr->nproc = n;
grp_attr->proc_list = (int *)malloc(n*sizeof(int));
assert(grp_attr->proc_list!=NULL);
for(i=0; i<n; i++) {
grp_attr->proc_list[i] = ARMCI_Absolute_id(grp_parent,pid_list[i]);
}
/* MPI_Comm_rank(MPI_COMM_WORLD, &world_me); */
world_me = armci_msg_me();
grp_attr->grp_me = grp_me = MPI_UNDEFINED;
for(i=0; i<n; i++) {
if(igroup->grp_attr.proc_list[i] == world_me) {
grp_attr->grp_me = grp_me = i;
break;
}
}
if(grp_me != MPI_UNDEFINED) armci_cache_attr(group_out);
armci_msg_group_barrier(group_out);
#else
/* NOTE: default group is the parent group */
group_parent = &(igroup_parent->igroup);
comm_parent = &(igroup_parent->icomm);
rv=MPI_Group_incl(*group_parent, n, pid_list, &(igroup->igroup));
if(rv != MPI_SUCCESS) armci_die("MPI_Group_incl: Failed ",armci_me);
rv = MPI_Comm_create(*comm_parent, (MPI_Group)(igroup->igroup),
(MPI_Comm*)&(igroup->icomm));
if(rv != MPI_SUCCESS) armci_die("MPI_Comm_create: Failed ",armci_me);
/* processes belong to this group should cache attributes */
MPI_Group_rank((MPI_Group)(igroup->igroup), &grp_me);
igroup->grp_attr.grp_clus_info=NULL;
if(grp_me != MPI_UNDEFINED) armci_cache_attr(group_out);
#endif
}
/**
* Group cluster information "grp_clus_info" (similar to armci_clus_info)
*/
static void group_process_list(ARMCI_Group *group,
armci_grp_attr_t *grp_attr) {
ARMCI_iGroup *igroup = (ARMCI_iGroup *)group;
#ifndef ARMCI_GROUP
ARMCI_Comm comm = igroup->icomm;
#endif
int grp_me, grp_nproc, grp_nclus, grp_clus_me;
armci_clus_t *grp_clus_info=NULL;
#ifdef CLUSTER
int i, len, root=0;
#endif
#ifndef ARMCI_GROUP
if(comm==MPI_COMM_NULL || igroup->igroup==MPI_GROUP_NULL)
armci_die("group_process_list: NULL COMMUNICATOR",0);
#endif
ARMCI_Group_rank(group, &grp_me);
ARMCI_Group_size(group, &grp_nproc);
#ifdef CLUSTER
#ifdef ARMCI_GROUP
/*all processes construct the clus_info structure in parallel*/
grp_clus_info = group_construct_clusinfo(&grp_nclus, igroup);
#else
/* process 0 gets group cluster information: grp_nclus, grp_clus_info */
if(grp_me == 0) {
grp_clus_info = group_construct_clusinfo(&grp_nclus, igroup);
}
/* process 0 broadcasts group cluster information */
len = sizeof(int);
ARMCI_Bcast_(&grp_nclus, len, root, comm);
if(grp_me){
/* allocate memory */
grp_clus_info = (armci_clus_t*)malloc(grp_nclus*sizeof(armci_clus_t));
if(!armci_clus_info)armci_die("malloc failed for clusinfo",armci_nclus);
}
len = sizeof(armci_clus_t)*grp_nclus;
ARMCI_Bcast_(grp_clus_info, len, root, comm);
#endif
/* determine current group cluster node id by comparing me to master */
grp_clus_me = grp_nclus-1;
for(i =0; i< grp_nclus-1; i++) {
if(grp_me < grp_clus_info[i+1].master){
grp_clus_me=i;
break;
}
}
#else
grp_clus_me = 0;
grp_nclus = 1;
grp_clus_info = (armci_clus_t*)malloc(grp_nclus*sizeof(armci_clus_t));
if(!grp_clus_info)armci_die("malloc failed for clusinfo",grp_nclus);
strcpy(grp_clus_info[0].hostname, armci_clus_info[0].hostname);
grp_clus_info[0].master=0;
grp_clus_info[0].nslave=grp_nproc;
#endif
#ifdef ARMCI_GROUP
/*Set in ARMCI_Group_create. ARMCI_Group_rank is used before
setting this field. So moving it there in the generic
implementation.*/
#else
grp_attr->grp_me = grp_me;
#endif
grp_attr->grp_clus_info = grp_clus_info;
grp_attr->grp_nclus = grp_nclus;
grp_attr->grp_clus_me = grp_clus_me;
}
