void DDI_Comm_destroy(int commid) {
const DDI_Comm *comm = (const DDI_Comm *) Comm_find(commid);
DDI_Comm *curr_comm = &gv(ddi_base_comm);
DDI_Comm *prev_comm = NULL;
/*
DDI_Data *curr_data = &gv(ddi_base_data);
DDI_Data *prev_data = NULL;
*/
Comm_sync(124,comm);
if(commid == DDI_COMM_WORLD) {
fprintf(stdout,"%s: Cannot destroy DDI_COMM_WORLD.\n",DDI_Id());
Fatal_error(911);
}
while(curr_comm->next && curr_comm->id != commid) {
prev_comm = curr_comm;
curr_comm = (DDI_Comm *) curr_comm->next;
}
if(curr_comm->id != commid) {
fprintf(stdout,"%s: Error in DDI_Comm_destroy - Comm not found.\n",DDI_Id());
Fatal_error(911);
}
/*
while(curr_data->next && curr_data->id != curr_comm->data_id) {
prev_data = curr_data;
curr_data = (DDI_Data *) curr_data->next;
}
if(curr_data->id != curr_comm->data_id) {
fprintf(stdout,"%s: Error in DDI_Comm_destroy - Data not found.\n",DDI_Id());
Fatal_error(911);
}
* ----------------------------------------------------------------------- *\
Delete item from DDI_Data linked-list.
\* ----------------------------------------------------------------------- *
if(curr_comm->me_local == 0) shmdt((void *) curr_data->sync_array);
prev_data->next = curr_data->next;
free(curr_data);
*/
/* ----------------------------------------------------------------------- *\
Delete item from DDI_Comm linked-list.
\* ----------------------------------------------------------------------- */
free(curr_comm->smp_pid);
free(curr_comm->local_nid);
free(curr_comm->global_pid);
free(curr_comm->global_nid);
free(curr_comm->global_dsid);
free(curr_comm->node_master);
prev_comm->next = curr_comm->next;
free(curr_comm);
}
/* -------------------------------------------------------------------- *\
DDI_Create_custom(idim,jdim,jcols,handle)
=========================================
[IN] idim - Number of rows in the array to be created.
[IN] jdim - Number of columns in the array to be created.
[IN] jcols - Array holding the number of columns to be given to
- each processor when creating the distributed array.
[OUT] handle - Handle given to the newly created array.
Creates a distributed array where the user can customize how the
array is distributed across the processors.
\* -------------------------------------------------------------------- */
void DDI_Create_custom(int idim,int jdim,int *jcols,int *handle) {
int i,np,me,nn,my;
int inode;
DDI_INT64 totwrds;
DDI_INT64 longrows,longcols,longslice,longnd,long2g;
# ifndef USE_SYSV
int remote_id;
# endif
DDI_Patch patch;
const DDI_Comm *comm = (const DDI_Comm *) Comm_find(DDI_WORKING_COMM);
np = comm->np;
me = comm->me;
nn = comm->nn;
my = comm->my;
Comm_sync(3001,comm);
/* find an unused handle */
for (i=0; i<gv(nxtdda); ++i) {
if (gv(ddacomm)[i] == DDI_COMM_NULL) break;
}
if (i==gv(nxtdda)) ++gv(nxtdda);
*handle = i;
# ifndef USE_SYSV
remote_id = my;
# endif
DEBUG_ROOT(LVL2,(stdout," DDI: Entering DDI_Create_custom.\n"))
DEBUG_ROOT(LVL2,(stdout," DDI: Creating Array [%i] - %ix%i=%i.\n",*handle,idim,jdim,idim*jdim))
DEBUG_OUT(LVL3,(stdout,"%s: Entering DDI_Create_custom.\n",DDI_Id()))
# ifdef DS_SIGNAL
if(comm->me_local == 1) {
signal(SIGALRM,DS_Thread_main);
}
# endif
if(me == 0) {
if(gv(dda_output)) {
longrows = idim;
longcols = jdim;
totwrds = longrows*longcols;
fprintf(stdout," DDI: Creating Array [%i] - %i x %i = %li words.\n",
*handle,idim,jdim,totwrds);
fflush(stdout);
}
}
/*
Make sure each slice of the distributed array will be under 2 GWords.
Even on 64-bit hardware, most counting in this program is done
with 32-bit data types, meaning we can't count higher than 2**31-1.
If on 32-bit hardware, the 'long' data types here will be 32-bits,
and so we'll see crazy products, including less than zero.
In present form, nothing will be trapped here on a 32 bit machine!
*/
longrows = idim;
longcols = jdim;
totwrds = longrows*longcols;
/* Total distributed array over 2 Gwords is OK, but each */
/* slice (MEMDDI per data server) must be under 2 GWords. */
/* TCP/IP has gv(nd)=-1 (uninitialized) */
/* Cray on one node has gv(nd)=0 since no d.s. exists. */
# if defined DDI_MPI
longnd = gv(nd);
if (longnd <= 0) longnd=1;
# endif
# if defined DDI_SOC
longnd = np;
# endif
longslice = totwrds/longnd;
/* next is largest signed 32 bit integer, stored as 64 bit quantity */
long2g = 2147483643;
if (longslice > long2g)
{
fprintf(stdout,"\n");
fprintf(stdout," DDI: trouble creating distributed array!\n");
fprintf(stdout," Current number of data servers is %li\n",longnd);
fprintf(stdout," so each data server's slice of array");
fprintf(stdout," [%i] is %li words\n",*handle,longslice);
fprintf(stdout,"\n");
fprintf(stdout," Add more processors so required total array");
fprintf(stdout," size %li words\n",totwrds);
fprintf(stdout," divided by no. of processors (data servers)");
fprintf(stdout," is less than 2 Gwords= %li\n",long2g);
fprintf(stdout," For example, %li or more data servers...\n",
1+totwrds/long2g);
fprintf(stdout,"\n");
fflush(stdout);
Fatal_error(911);
}
/* ------------------------------------ *\
Ensure 'jcols' is properly formatted
\* ------------------------------------ */
for(i=0; i<np; i++) {
if(jcols[i] < 0 && me == 0) {
fprintf(stdout," Error in argument 3 of DDI_Create_custom: Values must be >= 0.\n");
Fatal_error(911);
}
if(i > 0)
if(jcols[i] < jcols[i-1]) {
fprintf(stdout," Error in argument 3 of DDI_Create_custom: Values must increase monotonically.\n");
Fatal_error(911);
}
}
/* ----------------------------------------------------------------- *\
Check to ensure the maximum number of arrays hasn't been reached.
\* ----------------------------------------------------------------- */
if( gv(nxtdda) == MAX_DD_ARRAYS ) {
if(me == 0) {
fprintf(stderr," DDI Error: The maximum number of distributed arrays [%i] has been reached.\n",MAX_DD_ARRAYS);
fprintf(stderr," Information: The maximum number of distributed arrays is a DDI compile-time option.\n");
}
Fatal_error(911);
}
gv(nrow)[*handle] = idim;
gv(ncol)[*handle] = jdim;
gv(ddacomm)[*handle]=gv(ddi_working_comm);
/* ---------------------------------------------------- *\
Generate Column Mapping by Compute Process & by Node
\* ---------------------------------------------------- */
for(i=0,inode=-1; i<np; i++) {
gv(pcmap)[*handle][i] = jcols[i];
/* if(inode == gv(ddiprocs)[i].node) continue; */
if(inode == comm->local_nid[i]) continue;
gv(ncmap)[*handle][++inode] = gv(pcmap)[*handle][i];
}
gv(pcmap)[*handle][np] = jdim;
gv(ncmap)[*handle][nn] = jdim;
/* -------------------------- *\
Get local patch dimensions
\* -------------------------- */
DDI_DistribP(*handle,me,&patch);
/* ----------------------------- *\
Create Distributed Data Array
\* ----------------------------- */
patch.handle = *handle;
# if defined WINTEL
patch.oper = DDI_CREATE_OP;
# else
patch.oper = DDI_CREATE;
# endif
patch.size = jdim;
# if defined USE_SYSV || defined DDI_ARMCI || defined DDI_MPI2
DDI_Index_create(&patch);
# else
DDI_Send_request(&patch,&remote_id,NULL);
# endif
/* ----------------------------- *\
Synchronize Compute Processes
\* ----------------------------- */
Comm_sync(3002,comm);
DEBUG_OUT(LVL3,(stdout,"%s: Leaving DDI_Create_custom.\n",DDI_Id()))
}
void Comm_create(int np,int *ids, int ngroups, int mygroup, int comm_id, int *new_comm_id) {
int i,ip,in,ismp,nn,nid,np_local,tmp;
int err;
size_t size;
const DDI_Comm *cur_comm = (DDI_Comm *) Comm_find(comm_id);
const DDI_Comm *comm_world = &gv(ddi_base_comm);
DDI_Comm *new_comm = (DDI_Comm *) Malloc(sizeof(DDI_Comm));
DDI_Comm *end_comm = (DDI_Comm *) Comm_find_end();
DEBUG_ROOT(LVL1,(stdout," DDI: Entering DDI_Create_comm.\n"))
DEBUG_OUT(LVL2,(stdout,"%s: Entering DDI_Create_comm.\n",DDI_Id()))
Comm_sync(123,cur_comm);
/* ------------------------------- *\
Add new_comm to the linked list
\* ------------------------------- */
new_comm->next = NULL;
end_comm->next = (void *) new_comm;
/* new_data->next = NULL; */
/* end_data->next = (void *) new_data; */
new_comm->ngroups = ngroups;
new_comm->mygroup = mygroup;
new_comm->id = *new_comm_id = gv(ddi_comm_id)++;
new_comm->local_nid = (int *) Malloc(np*sizeof(int));
new_comm->global_pid = (int *) Malloc(np*sizeof(int));
new_comm->global_nid = (int *) Malloc(np*sizeof(int));
i = 0;
if(np > 1) {
do {
if(ids[i] > cur_comm->np) {
fprintf(stdout,"%s: Invalid id list in DDI_Comm_create.\n",DDI_Id());
Fatal_error(911);
}
if(ids[i+1] < ids[i]) {
tmp = ids[i];
ids[i] = ids[i+1];
ids[i+1] = tmp;
if(i) i--; i--;
}
} while(++i < np-1);
}
Comm_sync(126,cur_comm);
nn = -1;
nid = -1;
np_local = 0;
for(i=0; i<np; i++) {
new_comm->global_pid[i] = cur_comm->global_pid[ids[i]];
new_comm->global_nid[i] = cur_comm->global_nid[ids[i]];
fflush(stdout);
if(new_comm->global_nid[i] != nid) {
nid = new_comm->global_nid[i];
nn++;
}
new_comm->local_nid[i] = nn;
if(nid == comm_world->my) np_local++;
}
nn++;
/*
fprintf(stdout,"%s: new_comm->nn = %i.\n",DDI_Id(),nn);
fprintf(stdout,"%s: new_comm->np_local = %i.\n",DDI_Id(),np_local);
*/
Comm_sync(127,cur_comm);
DEBUG_ROOT(LVL5,(stdout," comm_create - global_pid/global_nid formed.\n"))
new_comm->smp_pid = (int *) Malloc(np_local*sizeof(int));
new_comm->node_master = (int *) Malloc(nn*sizeof(int));
new_comm->global_dsid = (int *) Malloc(nn*sizeof(int));
for(ip=0,in=-1,ismp=0,nid=-1; ip<np; ip++) {
if(new_comm->global_nid[ip] != nid) {
in++;
nid = new_comm->global_nid[ip];
new_comm->global_dsid[in] = comm_world->global_dsid[nid];
new_comm->node_master[in] = new_comm->global_pid[ip];
}
if(new_comm->global_pid[ip] == comm_world->me) {
new_comm->me = ip;
new_comm->my = in;
}
if(nid == comm_world->my) {
if(new_comm->global_pid[ip] == comm_world->me) new_comm->me_local = ismp;
new_comm->smp_pid[ismp++] = new_comm->global_pid[ip];
}
}
new_comm->nn = nn;
new_comm->np = np;
new_comm->np_local = ismp;
DEBUG_OUT(LVL5,(stdout,"%s: np=%i, nn=%i, np_smp=%i, me=%i, my=%i, me_smp=%i.\n",
DDI_Id(),new_comm->np,new_comm->nn,new_comm->np_local,
new_comm->me,new_comm->my,new_comm->me_local));
# if defined DDI_MPI
new_comm->world_comm = cur_comm->world_comm;
MPI_Comm_split(cur_comm->smp_comm,new_comm->global_pid[0],new_comm->me_local,&new_comm->smp_comm);
MPI_Comm_split(cur_comm->compute_comm,new_comm->global_pid[0],new_comm->me,&new_comm->compute_comm);
MPI_Comm_split(new_comm->compute_comm,new_comm->me_local,new_comm->my,&new_comm->node_comm);
# endif
DEBUG_OUT(LVL3,(stdout,"%s: Exiting DDI_Comm_create.\n",DDI_Id()))
}
