/* -------------------------------------------------------------------- *\
DDI_Create_custom(idim,jdim,jcols,handle)
=========================================
[IN] idim - Number of rows in the array to be created.
[IN] jdim - Number of columns in the array to be created.
[IN] jcols - Array holding the number of columns to be given to
- each processor when creating the distributed array.
[OUT] handle - Handle given to the newly created array.
Creates a distributed array where the user can customize how the
array is distributed across the processors.
\* -------------------------------------------------------------------- */
void DDI_Create_custom(int idim,int jdim,int *jcols,int *handle) {
int i,np,me,nn,my;
int inode;
DDI_INT64 totwrds;
DDI_INT64 longrows,longcols,longslice,longnd,long2g;
# ifndef USE_SYSV
int remote_id;
# endif
DDI_Patch patch;
const DDI_Comm *comm = (const DDI_Comm *) Comm_find(DDI_WORKING_COMM);
np = comm->np;
me = comm->me;
nn = comm->nn;
my = comm->my;
Comm_sync(3001,comm);
/* find an unused handle */
for (i=0; i<gv(nxtdda); ++i) {
if (gv(ddacomm)[i] == DDI_COMM_NULL) break;
}
if (i==gv(nxtdda)) ++gv(nxtdda);
*handle = i;
# ifndef USE_SYSV
remote_id = my;
# endif
DEBUG_ROOT(LVL2,(stdout," DDI: Entering DDI_Create_custom.\n"))
DEBUG_ROOT(LVL2,(stdout," DDI: Creating Array [%i] - %ix%i=%i.\n",*handle,idim,jdim,idim*jdim))
DEBUG_OUT(LVL3,(stdout,"%s: Entering DDI_Create_custom.\n",DDI_Id()))
# ifdef DS_SIGNAL
if(comm->me_local == 1) {
signal(SIGALRM,DS_Thread_main);
}
# endif
if(me == 0) {
if(gv(dda_output)) {
longrows = idim;
longcols = jdim;
totwrds = longrows*longcols;
fprintf(stdout," DDI: Creating Array [%i] - %i x %i = %li words.\n",
*handle,idim,jdim,totwrds);
fflush(stdout);
}
}
/*
Make sure each slice of the distributed array will be under 2 GWords.
Even on 64-bit hardware, most counting in this program is done
with 32-bit data types, meaning we can't count higher than 2**31-1.
If on 32-bit hardware, the 'long' data types here will be 32-bits,
and so we'll see crazy products, including less than zero.
In present form, nothing will be trapped here on a 32 bit machine!
*/
longrows = idim;
longcols = jdim;
totwrds = longrows*longcols;
/* Total distributed array over 2 Gwords is OK, but each */
/* slice (MEMDDI per data server) must be under 2 GWords. */
/* TCP/IP has gv(nd)=-1 (uninitialized) */
/* Cray on one node has gv(nd)=0 since no d.s. exists. */
# if defined DDI_MPI
longnd = gv(nd);
if (longnd <= 0) longnd=1;
# endif
# if defined DDI_SOC
longnd = np;
# endif
longslice = totwrds/longnd;
/* next is largest signed 32 bit integer, stored as 64 bit quantity */
long2g = 2147483643;
if (longslice > long2g)
{
fprintf(stdout,"\n");
fprintf(stdout," DDI: trouble creating distributed array!\n");
fprintf(stdout," Current number of data servers is %li\n",longnd);
fprintf(stdout," so each data server's slice of array");
fprintf(stdout," [%i] is %li words\n",*handle,longslice);
fprintf(stdout,"\n");
fprintf(stdout," Add more processors so required total array");
fprintf(stdout," size %li words\n",totwrds);
fprintf(stdout," divided by no. of processors (data servers)");
fprintf(stdout," is less than 2 Gwords= %li\n",long2g);
fprintf(stdout," For example, %li or more data servers...\n",
1+totwrds/long2g);
fprintf(stdout,"\n");
fflush(stdout);
Fatal_error(911);
}
/* ------------------------------------ *\
Ensure 'jcols' is properly formatted
\* ------------------------------------ */
for(i=0; i<np; i++) {
if(jcols[i] < 0 && me == 0) {
fprintf(stdout," Error in argument 3 of DDI_Create_custom: Values must be >= 0.\n");
Fatal_error(911);
}
if(i > 0)
if(jcols[i] < jcols[i-1]) {
fprintf(stdout," Error in argument 3 of DDI_Create_custom: Values must increase monotonically.\n");
Fatal_error(911);
}
}
/* ----------------------------------------------------------------- *\
Check to ensure the maximum number of arrays hasn't been reached.
\* ----------------------------------------------------------------- */
if( gv(nxtdda) == MAX_DD_ARRAYS ) {
if(me == 0) {
fprintf(stderr," DDI Error: The maximum number of distributed arrays [%i] has been reached.\n",MAX_DD_ARRAYS);
fprintf(stderr," Information: The maximum number of distributed arrays is a DDI compile-time option.\n");
}
Fatal_error(911);
}
gv(nrow)[*handle] = idim;
gv(ncol)[*handle] = jdim;
gv(ddacomm)[*handle]=gv(ddi_working_comm);
/* ---------------------------------------------------- *\
Generate Column Mapping by Compute Process & by Node
\* ---------------------------------------------------- */
for(i=0,inode=-1; i<np; i++) {
gv(pcmap)[*handle][i] = jcols[i];
/* if(inode == gv(ddiprocs)[i].node) continue; */
if(inode == comm->local_nid[i]) continue;
gv(ncmap)[*handle][++inode] = gv(pcmap)[*handle][i];
}
gv(pcmap)[*handle][np] = jdim;
gv(ncmap)[*handle][nn] = jdim;
/* -------------------------- *\
Get local patch dimensions
\* -------------------------- */
DDI_DistribP(*handle,me,&patch);
/* ----------------------------- *\
Create Distributed Data Array
\* ----------------------------- */
patch.handle = *handle;
# if defined WINTEL
patch.oper = DDI_CREATE_OP;
# else
patch.oper = DDI_CREATE;
# endif
patch.size = jdim;
# if defined USE_SYSV || defined DDI_ARMCI || defined DDI_MPI2
DDI_Index_create(&patch);
# else
DDI_Send_request(&patch,&remote_id,NULL);
# endif
/* ----------------------------- *\
Synchronize Compute Processes
\* ----------------------------- */
Comm_sync(3002,comm);
DEBUG_OUT(LVL3,(stdout,"%s: Leaving DDI_Create_custom.\n",DDI_Id()))
}
/* -------------------------------------------------------------------- *\
DDI_Server()
============
Called by DDI processes that specialize to become data servers.
\* -------------------------------------------------------------------- */
void DDI_Server() {
/* --------------- *\
Local Variables
\* --------------- */
int from;
char ack=57;
char server=1;
DDI_Patch *msg;
DDI_Patch patch;
size_t counter_value = 0;
const DDI_Comm *comm = (const DDI_Comm *) Comm_find(DDI_COMM_WORLD);
# ifdef CRAY_MPI
int i;
int nfinished = 0;
int last = -1;
int size = sizeof(DDI_Patch);
index_ds = (int *) Malloc(comm->np*sizeof(int));
p = (DDI_Patch *) Malloc(comm->np*sizeof(DDI_Patch));
s = (MPI_Status *) Malloc(comm->np*sizeof(MPI_Status));
r = (MPI_Request *) Malloc(comm->np*sizeof(MPI_Request));
/* ----------------------------------------------------------- *\
Post IRecvs for remote data requests from all the processes
\* ----------------------------------------------------------- */
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Posting MPI_IRecvs for data requests.\n",DDI_Id()))
for(i=0; i<comm->np; i++) {
MPI_Irecv(&p[i],size,MPI_BYTE,i,37,comm->world_comm,&r[i]);
}
NRequests = comm->np;
# endif
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Starting DDI data server.\n",DDI_Id()))
/* -------------------- *\
DDI Data Server Loop
\* -------------------- */
do {
# ifdef CRAY_MPI
MPI_Testsome(NRequests,r,&nfinished,index_ds,s);
for(i=0; i<nfinished; i++) {
msg = &p[index_ds[i]];
from = s[i].MPI_SOURCE;
# else
DDI_Recv_request(&patch,&from);
msg = (DDI_Patch *) &patch;
# endif
switch(msg->oper) {
case DDI_DEBUGFLAG:
DebugOutput(msg->handle);
break;
case DDI_MEMORY:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_MEMORY request.\n",DDI_Id()))
DDI_Memory_server(msg->size);
Comm_send(&ack,1,from,comm);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) DDI_MEMORY requested completed.\n",DDI_Id()))
break;
case DDI_CREATE:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_CREATE[%i] request.\n",DDI_Id(),msg->handle))
DDI_Index_create(msg);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Array[%i] created successfully.\n",DDI_Id(),msg->handle))
break;
case DDI_DESTROY:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_DESTROY[%i] request.\n",DDI_Id(),msg->handle))
DDI_Index_destroy(msg);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Array[%i] destroyed successfully.\n",DDI_Id(),msg->handle))
break;
case DDI_ZERO:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_ZERO request from %i.\n",DDI_Id(),from))
DDI_Array_zero(msg->handle);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_ZERO request from %i.\n",DDI_Id(),from))
break;
case DDI_GET:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_GET request from %i.\n",DDI_Id(),from))
DDI_Get_server(msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_GET request from %i.\n",DDI_Id(),from))
break;
case DDI_PUT:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_PUT request from %i.\n",DDI_Id(),from))
DDI_Put_server(msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_PUT request from %i.\n",DDI_Id(),from))
break;
case DDI_ACC:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_ACC request from %i.\n",DDI_Id(),from))
DDI_Acc_server(msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_ACC request from %i.\n",DDI_Id(),from))
break;
case DDI_GETACC:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_GETACC request from %i.\n",DDI_Id(),from))
DDI_GetAcc_server(msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_GETACC request from %i.\n",DDI_Id(),from))
break;
case DDI_DLBRESET:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_DLBRESET request from %i.\n",DDI_Id(),from))
DDI_DLBReset_local();
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_DLBRESET request from %i.\n",DDI_Id(),from))
break;
break;
case DDI_DLBNEXT:
DEBUG_OUT(LVL2,(stdout,"%s: (DS) Received DDI_DLBNEXT request from %i.\n",DDI_Id(),from))
DDI_DLBNext_local(&counter_value);
Comm_send(&counter_value,sizeof(size_t),from,comm);
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Finished DDI_DLBNEXT request from %i.\n",DDI_Id(),from))
break;
case DDI_GDLBRESET:
DDI_GDLBReset_local();
break;
case DDI_GDLBNEXT:
DDI_GDLBNext_local(&counter_value);
DDI_Send(&counter_value,sizeof(size_t),from);
break;
case DDI_QUIT: /* Quit server loop, synchronize, and exit */
DEBUG_OUT(LVL3,(stdout,"%s: (DS) Received DDI_QUIT request\n",DDI_Id()))
/* if(me == np) DB_Close(); */
DDI_Memory_finalize();
Comm_send(&ack,1,from,comm);
server=0;
break;
/* --------------------------------------------- *\
Clean-up distributed-memory & check for leaks
\* --------------------------------------------- */
/*
case DB_CREATE_ENTRY:
DEBUG_OUT(LVL3,(stdout,"%s: Recieved DB_CREATE_ENTRY request.\n",DDI_Id()))
if(me != np) {
fprintf(stdout,"%s: recieved DB request but is not master data server.\n",DDI_Id());
Fatal_error(911);
}
DB_Create_server(&msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: Finished DB_CREATE_ENTRY request.\n",DDI_Id()))
break;
case DB_READ_ENTRY:
DEBUG_OUT(LVL3,(stdout,"%s: Recieved DB_READ_ENTRY request.\n",DDI_Id()))
if(me != np) {
fprintf(stdout,"%s: recieved DB request but is not master data server.\n",DDI_Id());
Fatal_error(911);
}
DB_Read_server(&msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: Finished DB_READ_ENTRY request.\n",DDI_Id()))
break;
case DB_WRITE_ENTRY:
DEBUG_OUT(LVL3,(stdout,"%s: Recieved DB_WRITE_ENTRY request.\n",DDI_Id()))
if(me != np) {
fprintf(stdout,"%s: recieved DB request but is not master data server.\n",DDI_Id());
Fatal_error(911);
}
DB_Write_server(&msg,from);
DEBUG_OUT(LVL3,(stdout,"%s: Finished DB_WRITE_ENTRY request.\n",DDI_Id()))
break;
*/
}
# ifdef CRAY_MPI
/* ----------------------------------------------------- *\
Repost the asynchronus IRecv for remote data requests
\* ----------------------------------------------------- */
MPI_Irecv(&p[index_ds[i]],size,MPI_BYTE,from,37,comm->world_comm,&r[index_ds[i]]);
}
# endif
} while(server);
/* -------------------------------------------------------------------------- *\
If using MPI and not TCP socekts -- cancel/release the persistent receives
\* -------------------------------------------------------------------------- */
# if defined DDI_MPI && !defined DDI_SOC
/* Working on this bit */
# endif
/* ------------------------------- *\
Finalize communication and exit
\* ------------------------------- */
DDI_Finalize();
exit(0);
}
