/*@@
@routine PUGH_SetupPGH
@date Fri Feb 21 10:21:36 1997
@author Gabrielle Allen, Tom Goodale, Paul Walker
@desc
Sets up the PGH distribution based on processor number and
problem size.
This includes setting an optimal domain decomposition for each
dimension, and setting ghosts and overlaps.
@enddesc
@var callerid
@vdesc back reference to the calling cGH structure
@vtype void *
@vio in
@endvar
@var dim
@vdesc number of dimensions to set up
@vtype int
@vio in
@endvar
@var nsize
@vdesc number of grid points in every dimension
@vtype int *
@vio in
@endvar
@var nghostzones
@vdesc number of ghostzones in every dimension
@vtype int *
@vio in
@endvar
@var staggertype
@vdesc staggering type for variables on this pGH
@vtype int
@vio in
@endvar
@var perme
@vdesc periodic boundary flags for every dimension
@vtype int *
@vio in
@endvar
@returntype pGH *
@returndesc
the pointer to the new pGH structure
@endreturndesc
@@*/
pGH *PUGH_SetupPGH (void *callerid,
int dim,
int *nsize,
int *nghostzones,
int staggertype,
int *perme)
{
DECLARE_CCTK_PARAMETERS
pGH *pughGH;
int i, idim;
int *nprocs;
/* Allocate for myself */
pughGH = (pGH *) malloc (sizeof (pGH));
/* Set an identifier for my parent */
pughGH->callerid = callerid;
pughGH->GFExtras = (pGExtras **) malloc (dim * sizeof (pGExtras *));
pughGH->Connectivity = (pConnectivity **) malloc (dim*sizeof(pConnectivity*));
/* Set the total number of processors */
Setup_nProcs (pughGH, dim);
/* Set up connectivity and extras for each dimension */
for (idim = 1; idim <= dim; idim++)
{
nprocs = (int *) malloc (idim * sizeof (int));
PUGH_SetupDefaultTopology (idim, nprocs);
/* Check that there are enough grid points in this dimension
* to make parallelising it worthwhile
*/
for (i = 0; i < idim; i++)
{
if ((! nprocs[i]) && (abs (nsize[i]) <= 2 * nghostzones[i] + 1))
{
nprocs[i] = 1;
}
}
pughGH->Connectivity[idim-1] = PUGH_SetupConnectivity (idim, pughGH->nprocs,
nprocs, perme);
free(nprocs);
pughGH->GFExtras[idim-1] = PUGH_SetupPGExtras (idim, perme, staggertype,
nsize, nghostzones,
pughGH->nprocs,
pughGH->Connectivity[idim-1]->nprocs,
pughGH->myproc);
}
/* create the timer for communication time */
pughGH->comm_time = timer_output ? CCTK_TimerCreateI () : -1;
pughGH->active = 1;
pughGH->commmodel = PUGH_ALLOCATEDBUFFERS;
pughGH->identity_string = NULL;
pughGH->level = 0;
pughGH->mglevel = 0;
pughGH->convlevel = 0;
pughGH->forceSync = 0;
pughGH->nvariables = 0;
pughGH->narrays = 0;
pughGH->variables = NULL;
USE_CCTK_PARAMETERS; return (pughGH);
}
/*@@
@routine PUGH_SetupGAGroup
@date January 19 2000
@author Gabrielle Allen
@desc
Set up a group of grid array variables (CCTK_GF or CCTK_ARRAY
types) on a pGH.
@enddesc
@calls CCTK_VarTypeSize
PUGH_SetupConnectivity
PUGH_SetupPGExtras
PUGH_SetupGArrayGroupComm
PUGH_SetupGArray
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var nsize
@vdesc size dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var ghostsize
@vdesc ghostsize dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var gtype
@vdesc group type
@vtype int
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var dim
@vdesc number of dimensions for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var staggercode
@vdesc stagger code for variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
PUGH_SUCCESS (0) if successful, or
PUGH_ERRORMEMORY (negative) if memory allocation failed
@endreturndesc
@@*/
static int PUGH_SetupGAGroup (pGH *newGH,
int *nsize,
int *ghostsize,
int gtype,
int vtype,
int dim,
int n_variables,
int staggercode,
int n_timelevels,
int vectorgroup)
{
int i;
int variable;
int var_size;
int retval;
int level;
void ***temp;
pConnectivity *connectivity;
pGExtras *extras;
pComm *groupcomm;
int *perme;
int *nprocs;
retval = PUGH_SUCCESS;
var_size = CCTK_VarTypeSize (vtype);
if (gtype == CCTK_ARRAY)
{
/* Arrays can't (yet) have periodicity and manual setup,
so initialize them to zero */
perme = (int *) calloc (dim, sizeof (int));
nprocs = (int *) calloc (dim, sizeof (int));
if (! (perme && nprocs))
{
CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
}
/* Check that there are enough grid points in this dimension
* to make parallelising it worthwhile
*/
for (i = 0 ; i < dim; i++)
{
if (! nprocs[i] && abs (nsize[i]) <= 2*ghostsize[i] + 1)
{
nprocs[i] = 1;
}
}
connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);
extras = PUGH_SetupPGExtras (dim,
perme,
staggercode,
nsize,
ghostsize,
newGH->nprocs,
connectivity->nprocs,
newGH->myproc);
free (nprocs);
free (perme);
}
else
{
/* this is for CCTK_GF variables */
connectivity = newGH->Connectivity[dim-1];
extras = newGH->GFExtras[dim-1];
}
/* Set up the communication buffer used for all variables within this group.
Note: only with allocated buffers we can have group communication. */
if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
groupcomm = PUGH_SetupGArrayGroupComm (newGH,
dim,
newGH->nvariables,
n_variables,
0,
vtype,
extras);
}
else
{
groupcomm = NULL;
}
temp = (void ***) realloc (newGH->variables, (newGH->nvariables+n_variables) *
sizeof (void **));
if(temp)
{
newGH->variables = temp;
for (variable = 0; variable < n_variables; variable++)
{
newGH->variables[newGH->nvariables] = (void **) malloc (n_timelevels *
sizeof (void *));
if (newGH->variables[newGH->nvariables])
{
for (level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
PUGH_SetupGArray (newGH,
extras,
connectivity,
groupcomm,
CCTK_VarName (newGH->nvariables),
newGH->nvariables,
newGH->narrays,
var_size,
vtype,
staggercode,
vectorgroup ? n_variables : 1,
variable,
variable > 0 ? newGH->variables[newGH->nvariables-variable][level] : NULL);
newGH->narrays++;
}
newGH->nvariables++;
}
else
{
retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
if (retval)
{
CCTK_WARN (1, "Memory allocation error in PUGH_SetupGAGroup");
}
return (retval);
}
