int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
{
int group; /* group index */
cGroup pgroup; /* group information */
int rc; /* return code */
#ifdef DEBUG_PUGH
printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
fflush(stdout);
#endif
/* get the group info from its index */
group = CCTK_GroupIndex(groupname);
CCTK_GroupData(group, &pgroup);
if (pgroup.grouptype == CCTK_SCALAR)
{
rc = 1;
}
else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
CCTK_FirstVarIndexI(group),
PUGH_ALLCOMM);
}
else
{
CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
rc = 0;
}
return (rc);
}
/*@@
@routine PUGH_SetupGroup
@date Mon Feb 8 19:37:55 1999
@author Tom Goodale
@desc
Sets up a group on a pGH
@enddesc
@calls PUGH_SetupScalarGroup
PUGH_SetupGAGroup
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var nsize
@vdesc size dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var ghostsize
@vdesc ghostsize dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var gtype
@vdesc group type
@vtype int
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var dim
@vdesc number of dimensions for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var staggercode
@vdesc stagger code for variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
return code of @seeroutine PUGH_SetupScalarGroup or
@seeroutine PUGH_SetupGAGroup, or
PUGH_ERROR if invalid group type was passed in
@endreturndesc
@@*/
int PUGH_SetupGroup (pGH *newGH,
int *nsize,
int *nghostsize,
int gtype,
int vtype,
int dim,
int n_variables,
int staggercode,
int n_timelevels,
int vectorgroup)
{
int retval;
if (gtype == CCTK_SCALAR)
{
retval = PUGH_SetupScalarGroup (newGH, vtype, n_variables, n_timelevels, vectorgroup);
}
else if (gtype == CCTK_ARRAY || gtype == CCTK_GF)
{
retval = PUGH_SetupGAGroup (newGH, nsize, nghostsize, gtype, vtype, dim,
n_variables, staggercode, n_timelevels,vectorgroup);
}
else
{
CCTK_WARN (0, "Unknown group type in PUGH_SetupGroup");
retval = PUGH_ERROR;
}
return (retval);
}
/*@@
@routine PUGH_ArrayGroupSize
@author Gabrielle Allen
@date 11 Apr 1999
@desc
Returns size of arrays in a group, in a given direction.
Either group index or its name must be given,
the name takes precedence.
@enddesc
@calls CCTK_GroupIndex
CCTK_FirstVarIndexI
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype const cGH *
@vio in
@endvar
@var dir
@vdesc direction to return size of
@vtype int
@vio in
@endvar
@var group
@vdesc index of group
@vtype int
@vio in
@endvar
@var groupname
@vdesc name of group
@vtype const char *
@vio in
@endvar
@returntype const int *
@returndesc
pointer to the size variable for the given direction, or
NULL if given direction or group index/name are invalid
@endreturndesc
@@*/
const int *PUGH_ArrayGroupSize (const cGH *GH,
int dir,
int group,
const char *groupname)
{
int first_var;
pGA *GA;
const int *retval;
if (groupname)
{
group = CCTK_GroupIndex (groupname);
}
first_var = CCTK_FirstVarIndexI (group);
if (first_var >= 0)
{
/* get pointer to pGA for a timelevel of first variable in this group */
GA = (pGA *) PUGH_pGH (GH)->variables[first_var][0];
if (GA->storage == PUGH_STORAGE)
{
if (dir >= 0 && dir < GA->extras->dim)
{
retval = &GA->extras->lnsize[dir];
}
else
{
CCTK_WARN (1, "Wrong value for dir");
retval = NULL;
}
}
else
{
retval = &_cctk_one;
}
}
else
{
if (groupname)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group name '%s'",
groupname);
}
else
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group ID %d",
group);
}
retval = NULL;
}
return (retval);
}
int PUGH_SyncGroup(cGH *GH, const char *groupname)
{
cGroup pgroup; /* group information */
int group; /* group index */
int rc; /* return code */
#ifdef DEBUG_PUGH
printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
fflush (stdout);
#endif
/* get the group info from its index */
group = CCTK_GroupIndex(groupname);
if (group < 0)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_SyncGroup: Unknown group: %s", groupname);
rc = -1;
}
else
{
CCTK_GroupData(group, &pgroup);
if (pgroup.grouptype == CCTK_SCALAR)
{
rc = 0;
CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_SyncGroup: Synchronising scalar group: %s",groupname);
}
else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
}
else
{
CCTK_WARN(1, "PUGH_SyncGroup: Unknown group type");
rc = 0;
}
}
return (rc);
}
int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
{
int group; /* group index */
cGroup pgroup; /* pointer to group information */
int rc; /* return code */
pGH *pughGH;
int var;
#ifdef DEBUG_PUGH
printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
fflush(stdout);
#endif
/* get the group info from its index */
group = CCTK_GroupIndex(groupname);
CCTK_GroupData(group, &pgroup);
if (pgroup.grouptype == CCTK_SCALAR)
{
rc = 1;
}
else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
pughGH=PUGH_pGH(GH);
var = CCTK_FirstVarIndexI(group);
/* FIXME: workaround. This one is really bad ! */
rc = PUGH_DisableGArrayGroupComm(pughGH, var,(((pGA ***)pughGH->variables)[var][0])->groupcomm);
}
else
{
CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
rc = 0;
}
return (rc);
}
void Time_Simple(CCTK_ARGUMENTS)
{
DECLARE_CCTK_PARAMETERS
DECLARE_CCTK_ARGUMENTS
CCTK_REAL min_spacing=0;
/* Calculate the minimum grid spacing */
if (cctk_dim>=1)
{
min_spacing = cctk_delta_space[0];
}
if (cctk_dim>=2)
{
min_spacing = (min_spacing<cctk_delta_space[1] ?
min_spacing : cctk_delta_space[1]);
}
if (cctk_dim>=3)
{
min_spacing = (min_spacing<cctk_delta_space[2] ?
min_spacing : cctk_delta_space[2]);
}
if (cctk_dim>=4)
{
CCTK_WARN(0,"Time Step not defined for greater than 4 dimensions");
}
/* Calculate the timestep */
cctkGH->cctk_delta_time = dtfac*min_spacing;
USE_CCTK_PARAMETERS; USE_CCTK_CARGUMENTS }
/*@@
@routine PUGH_GenerateTopology
@date Fri Nov 5 11:31:21 1999
@author Tom Goodale
@desc
Generate the appropriate processor topology for this processor
decomposition.
@enddesc
@calls
@calledby
@history
@hdate Tue Jan 30 17:04:50 2001 @hauthor Tom Goodale
@hdesc Added call to integer root function and qsort to
avoid problems with real to integer conversions and
demonstrable failure of the algorithm when dealing
with large prime factors.
@endhistory
@@*/
int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs)
{
int i;
int used_procs;
int free_procs;
int retval;
int free_dims;
used_procs = 0;
free_procs = total_procs;
retval = 0;
free_dims = dim;
for(i=0; i < dim; i++)
{
if((nprocs[i])>0)
{
free_dims--;
if(used_procs)
{
used_procs *= nprocs[i];
}
else
{
used_procs = nprocs[i];
}
if (total_procs%used_procs)
{
CCTK_WARN(0, "Inconsistent PUGH topology");
retval = 1;
}
else
{
free_procs = total_procs/used_procs;
}
}
}
/* Ok calculate topology if necessary */
if(free_dims && ! retval)
{
/* This algorithm gives the most number of processors
* in the highest dimension.
*/
int *working;
int root;
int place;
root = free_dims;
working = (int *)calloc(free_dims,sizeof(int));
#ifdef DEBUG_PUGH
printf("Processor topology for dimension %d\n",dim);
#endif
for(i = 0; i < free_dims ; i++)
{
working[i] = PUGH_IntegerRoot(free_procs, root);
while(free_procs % working[i]) working[i]--;
#ifdef DEBUG_PUGH
printf(" working[%d] = %d\n",i,working[i]);
#endif
free_procs /= working[i];
root--;
}
/* The above doesn't necessarily sort them properly
* e.g. if one of the factors is a prime then the
* above will sort the 1 before the prime.
*/
qsort(working,free_dims,sizeof(int),IntSort);
for(i = 0,place=0; i < dim ; i++)
{
if(nprocs[i] <= 0)
{
nprocs[i] = working[place];
place++;
}
#ifdef DEBUG_PUGH
printf(" nprocs[%d] = %d\n",i,nprocs[i]);
#endif
}
free(working);
}
return retval;
}
CCTK_INT Hyperslab_DefineGlobalMappingByIndex (
const cGH *GH,
CCTK_INT vindex,
CCTK_INT dim,
const CCTK_INT *direction /* vdim*dim */,
const CCTK_INT *origin /* vdim */,
const CCTK_INT *extent /* dim */,
const CCTK_INT *downsample /* dim */,
CCTK_INT table_handle,
CCTK_INT target_proc,
t_hslabConversionFn conversion_fn,
CCTK_INT *hsize /* dim */)
{
unsigned int vdim, hdim, num_dirs;
int retval;
int stagger_index;
int myproc;
int npoints;
hslab_mapping_t *mapping;
const char *error_msg;
const pGH *pughGH; /* pointer to the current pGH */
const pGA *GA; /* the variable's GA structure from PUGH */
cGroup vinfo;
/* PUGHSlab doesn't use table information */
if (table_handle >= 0)
{
CCTK_WARN (1, "Hyperslab_DefineGlobalMappingByIndex: table information is "
"ignored");
}
/* check parameter consistency */
retval = 0;
error_msg = NULL;
if (CCTK_GroupData (CCTK_GroupIndexFromVarI (vindex), &vinfo) < 0)
{
error_msg = "invalid variable index given";
retval = -1;
}
else if (vinfo.grouptype != CCTK_GF && vinfo.grouptype != CCTK_ARRAY)
{
error_msg = "invalid variable group type given "
"(not a CCTK_GF or CCTK_ARRAY type)";
retval = -2;
}
else if (dim < 0 || dim > vinfo.dim)
{
error_msg = "invalid hyperslab dimension given";
retval = -2;
}
else if (! direction || ! origin || ! extent || ! hsize)
{
error_msg = "NULL pointer(s) passed for direction/origin/extent/hsize "
"parameters";
retval = -3;
}
else if (target_proc >= CCTK_nProcs (GH))
{
error_msg = "invalid target procesor ID given";
retval = -4;
}
else if ((pughGH = (const pGH *) PUGH_pGH (GH)) == NULL)
{
error_msg = "no PUGH GH extension registered (PUGH not activated ?)";
retval = -4;
}
else
{
for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
retval |= origin[vdim] < 0;
if (vdim < (unsigned int) dim)
{
retval |= extent[vdim] <= 0;
if (downsample)
{
retval |= downsample[vdim] <= 0;
}
}
}
if (retval)
{
error_msg = "invalid hyperslab origin/extent/downsample vectors given";
retval = -5;
}
}
if (! retval)
{
mapping = (hslab_mapping_t *) malloc (sizeof (hslab_mapping_t));
if (mapping)
{
mapping->vectors = (int *) malloc ((6*vinfo.dim + 2*dim) * sizeof (int));
}
if (mapping == NULL || mapping->vectors == NULL)
{
if (mapping)
{
free (mapping);
}
error_msg = "couldn't allocate hyperslab mapping structure";
retval = -6;
}
}
/* return in case of errors */
if (retval)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: %s", error_msg);
return (retval);
}
mapping->hdim = (unsigned int) dim;
mapping->vinfo = vinfo;
mapping->target_proc = target_proc;
mapping->conversion_fn = conversion_fn;
/* assign memory for the other vectors */
mapping->local_startpoint = mapping->vectors + 0*vinfo.dim;
mapping->local_endpoint = mapping->vectors + 1*vinfo.dim;
mapping->global_startpoint = mapping->vectors + 2*vinfo.dim;
mapping->global_endpoint = mapping->vectors + 3*vinfo.dim;
mapping->do_dir = mapping->vectors + 4*vinfo.dim;
mapping->downsample = mapping->vectors + 5*vinfo.dim;
mapping->local_hsize = mapping->vectors + 6*vinfo.dim + 0*dim;
mapping->global_hsize = mapping->vectors + 6*vinfo.dim + 1*dim;
/* check direction vectors */
for (hdim = 0; hdim < mapping->hdim; hdim++)
{
num_dirs = 0;
for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
if (direction[hdim*vinfo.dim + vdim])
{
num_dirs++;
}
}
if (num_dirs == 0)
{
free (mapping->vectors); free (mapping);
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: %d-direction vector "
"is a null vector", hdim);
return (-7);
}
mapping->is_diagonal_in_3D = num_dirs == 3 && mapping->hdim == 1;
if (num_dirs != 1 && ! mapping->is_diagonal_in_3D)
{
free (mapping->vectors); free (mapping);
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: %d-direction vector "
"isn't axis-orthogonal", hdim);
return (-7);
}
}
/* diagonals can be extracted from non-staggered 3D variables only */
if (mapping->is_diagonal_in_3D && vinfo.stagtype != 0)
{
free (mapping->vectors); free (mapping);
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: diagonals can be "
"extracted from non-staggered 3D variables only");
return (-7);
}
for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
mapping->do_dir[vdim] = 0;
for (hdim = 0; hdim < mapping->hdim; hdim++)
{
if (direction[hdim*vinfo.dim + vdim])
{
mapping->do_dir[vdim]++;
}
}
if (mapping->do_dir[vdim] > 1)
{
free (mapping->vectors); free (mapping);
CCTK_WARN (1, "Hyperslab_DefineGlobalMappingByIndex: duplicate direction "
"vectors given");
return (-8);
}
}
/* get the pGH pointer and the variable's GA structure */
GA = (const pGA *) pughGH->variables[vindex][0];
myproc = CCTK_MyProc (GH);
/* check extent */
for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
if (mapping->do_dir[vdim] && hdim < mapping->hdim)
{
if (origin[vdim] + extent[hdim] > GA->extras->nsize[vdim])
{
free (mapping->vectors); free (mapping);
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: extent in "
"%d-direction exceeds grid size", hdim);
return (-8);
}
hdim++;
}
else if (mapping->is_diagonal_in_3D &&
origin[vdim] + extent[0] > GA->extras->nsize[vdim])
{
free (mapping->vectors); free (mapping);
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"Hyperslab_DefineGlobalMappingByIndex: extent in "
"%d-direction exceeds grid size", vdim);
return (-8);
}
}
/* now fill out the hyperslab mapping structure */
for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
mapping->downsample[vdim] = 1;
if (mapping->do_dir[vdim] && hdim < mapping->hdim)
{
if (downsample)
{
mapping->downsample[vdim] = downsample[hdim];
}
mapping->global_hsize[hdim] = extent[hdim] / mapping->downsample[vdim];
if (extent[hdim] % mapping->downsample[vdim])
{
mapping->global_hsize[hdim]++;
}
/* subtract ghostzones for periodic BC */
if (GA->connectivity->perme[vdim])
{
mapping->global_hsize[hdim] -= 2 * GA->extras->nghostzones[vdim];
}
hdim++;
}
else if (mapping->is_diagonal_in_3D)
{
mapping->totals = extent[0] / mapping->downsample[0];
if (extent[0] % mapping->downsample[0])
{
mapping->totals++;
}
/* subtract ghostzones for periodic BC */
if (GA->connectivity->perme[vdim])
{
mapping->totals -= 2 * GA->extras->nghostzones[vdim];
}
if ((unsigned int) mapping->global_hsize[0] > mapping->totals)
{
mapping->global_hsize[0] = mapping->totals;
}
}
}
/* check whether the full local data patch was requested as hyperslab */
mapping->is_full_hyperslab = IsFullHyperslab (GA, origin, extent, mapping);
if (mapping->is_full_hyperslab)
{
memset (mapping->local_startpoint, 0, vinfo.dim * sizeof (int));
memcpy (mapping->local_endpoint, GA->extras->lnsize, vinfo.dim*sizeof(int));
mapping->totals = GA->extras->npoints;
}
else if (mapping->is_diagonal_in_3D)
{
/* just initialize the downsample and global_startpoint vectors */
for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
mapping->downsample[vdim] = mapping->downsample[0];
mapping->global_startpoint[vdim] = origin[vdim];
}
}
else
{
/* compute the global endpoint */
for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
mapping->global_endpoint[vdim] = origin[vdim] +
(mapping->do_dir[vdim] ? extent[hdim++] : 1);
}
/* compute this processor's global startpoint from the global ranges */
for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
stagger_index = CCTK_StaggerDirIndex (vdim, vinfo.stagtype);
if (origin[vdim] < MY_GLOBAL_EP (GA->extras, myproc, stagger_index, vdim))
{
mapping->global_startpoint[vdim] = origin[vdim];
if (origin[vdim] < MY_GLOBAL_SP (GA->extras, myproc,stagger_index,vdim))
{
npoints = (MY_GLOBAL_SP (GA->extras, myproc, stagger_index, vdim)
- origin[vdim]) / mapping->downsample[vdim];
if ((MY_GLOBAL_SP (GA->extras, myproc, stagger_index, vdim)
- origin[vdim]) % mapping->downsample[vdim])
{
npoints++;
}
mapping->global_startpoint[vdim] += npoints*mapping->downsample[vdim];
}
}
else
{
mapping->global_startpoint[vdim] = -1;
}
}
/* compute the local start- and endpoint from the global ranges */
mapping->totals = 1;
for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
{
stagger_index = CCTK_StaggerDirIndex (vdim, vinfo.stagtype);
if (mapping->global_startpoint[vdim] >= 0 &&
mapping->global_startpoint[vdim] < MY_GLOBAL_EP (GA->extras, myproc,
stagger_index,vdim))
{
mapping->local_startpoint[vdim] = mapping->global_startpoint[vdim] -
GA->extras->lb[myproc][vdim];
}
else
{
mapping->local_startpoint[vdim] = -1;
}
if (mapping->global_endpoint[vdim] > MY_GLOBAL_SP (GA->extras, myproc,
stagger_index, vdim))
{
mapping->local_endpoint[vdim] =
MIN (MY_LOCAL_EP (GA->extras, stagger_index, vdim),
mapping->global_endpoint[vdim] -
GA->extras->lb[myproc][vdim]);
}
else
{
mapping->local_endpoint[vdim] = -1;
}
#ifdef DEBUG
printf ("direction %d: global ranges [%d, %d), local ranges[%d, %d)\n",
vdim,
mapping->global_startpoint[vdim], mapping->global_endpoint[vdim],
mapping->local_startpoint[vdim], mapping->local_endpoint[vdim]);
#endif
if (mapping->local_endpoint[vdim] < 0 ||
mapping->local_startpoint[vdim] < 0)
{
mapping->totals = 0;
mapping->local_endpoint[vdim] = mapping->local_startpoint[vdim];
}
if (mapping->do_dir[vdim])
{
/* compute the local size in each hyperslab dimension */
mapping->local_hsize[hdim] = (mapping->local_endpoint[vdim] -
mapping->local_startpoint[vdim]) /
mapping->downsample[vdim];
if ((mapping->local_endpoint[vdim] - mapping->local_startpoint[vdim]) %
mapping->downsample[vdim])
{
mapping->local_hsize[hdim]++;
}
mapping->totals *= mapping->local_hsize[hdim];
hdim++;
}
}
} /* end of else branch for 'if (mapping->is_full_hyperslab)' */
#ifdef DEBUG
printf ("total number of hyperslab data points: %d\n", mapping->totals);
#endif
#if 0
if (mapping->totals > 0)
{
/* if requested, compute the offsets into the global hyperslab */
if (hoffset_global)
{
for (i = hdim = 0; i < vinfo.dim; i++)
{
if (mapping->do_dir[i])
{
if (mapping->is_full_hyperslab)
{
hoffset_global[hdim] = GA->extras->lb[myproc][i];
}
else
{
hoffset_global[hdim] = (mapping->global_startpoint[i] -
origin[i]) / mapping->downsample[i];
if (GA->connectivity->perme[i])
{
hoffset_global[hdim] -= GA->extras->nghostzones[i];
}
}
#ifdef DEBUG
printf ("hoffset_global, hsize in direction %d: %d, %d\n",
hdim, hoffset_global[hdim], mapping->local_hsize[hdim]);
#endif
hdim++;
}
}
}
}
#endif
/* add this mapping to the mapping list */
if (mapping_list)
{
mapping_list->prev = mapping;
}
mapping->prev = NULL;
mapping->next = mapping_list;
mapping_list = mapping;
mapping->handle = nmapping_list++;
/* set the global hsize in the return arguments */
if (hsize)
{
for (hdim = 0; hdim < mapping->hdim; hdim++)
{
hsize[hdim] = mapping->global_hsize[hdim];
}
}
return (mapping->handle);
}
/*@@
@routine Setup_nProcs
@date Tue Apr 18 15:21:42 2000
@author Tom Goodale
@desc
Setup PUGH_COMM_WORLD communicator and make sure that CCTK_INT
and CCTK_REAL sizes are the same on all processors.
Define MPI datatypes for CCTK_REAL and CCTK_COMPLEX variables.
@enddesc
@var pughGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var dim
@vdesc dimension of processor topology to set up
@vtype int
@vio in
@endvar
@returntype int
@returndesc
0 for success
@endreturndesc
@@*/
static int Setup_nProcs (pGH *pughGH, int dim)
{
#ifdef CCTK_MPI
CCTK_REAL4 sum_sizes [2], compiled_sizes [2];
/* Set up my communicator. This would allow us to run pugh on
a subset of processors at a later date if, for instance, we
were using panda or what not.
*/
CACTUS_MPI_ERROR (MPI_Comm_dup (MPI_COMM_WORLD, &pughGH->PUGH_COMM_WORLD));
CACTUS_MPI_ERROR (MPI_Comm_size (pughGH->PUGH_COMM_WORLD, &pughGH->nprocs));
CACTUS_MPI_ERROR (MPI_Comm_rank (pughGH->PUGH_COMM_WORLD, &pughGH->myproc));
/* check that all executables uses the same integer and fp precision
within a metacomputing environment
NOTE: We cannot use CCTK_INT4s in MPI_Allreduce() since
(although they have the same size) they might be correspond to
different MPI datatypes.
CCTK_REAL4 should always refer to MPI_FLOAT. */
compiled_sizes [0] = sizeof (CCTK_INT);
compiled_sizes [1] = sizeof (CCTK_REAL);
CACTUS_MPI_ERROR (MPI_Allreduce (compiled_sizes, sum_sizes, 2, PUGH_MPI_REAL4,
MPI_SUM, pughGH->PUGH_COMM_WORLD));
if (compiled_sizes [0] * pughGH->nprocs != sum_sizes [0])
{
CCTK_WARN (0, "Cannot run executables with different precision for "
"CCTK_INTs within a metacomputing environment !\n"
"Please configure with unique CCTK_INTEGER_PRECISION !");
}
if (compiled_sizes [1] * pughGH->nprocs != sum_sizes [1])
{
CCTK_WARN (0, "Cannot run executables with different precision for "
"CCTK_REALs within a metacomputing environment !\n"
"Please configure with unique CCTK_REAL_PRECISION !");
}
/* define the complex datatype as a concatanation of 2 PUGH_MPI_REALs */
CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL,
&pughGH->PUGH_mpi_complex));
CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex));
/* dito for fixed-precision reals */
#ifdef CCTK_REAL4
CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL4,
&pughGH->PUGH_mpi_complex8));
CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex8));
#endif
#ifdef CCTK_REAL8
CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL8,
&pughGH->PUGH_mpi_complex16));
CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex16));
#endif
#ifdef CCTK_REAL16
CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL16,
&pughGH->PUGH_mpi_complex32));
CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex32));
#endif
#else
pughGH->nprocs = 1;
pughGH->myproc = 0;
#endif
pughGH->dim = dim;
return (0);
}
/*@@
@routine PUGH_EnableGArrayGroupStorage
@author Tom Goodale
@date 30 Mar 1999
@desc
Enables storage for a set of variables
@enddesc
@calls PUGH_EnableGArrayDataStorage
@var pughGH
@vdesc Pointer to PUGH GH extensions
@vtype pGH *
@vio in
@endvar
@var first_var
@vdesc index of the first variable to enable storage for
@vtype int
@vio in
@endvar
@var n_variables
@vdesc total number of variables to enable storage for
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc total number of timelevels to enable storage for
@vtype int
@vio in
@endvar
@returntype int
@returndesc
1 if storage was already enabled
0 if storage was enabled (for at least one variable)
-1 if there was an inconsistency in storage allocation
@endreturndesc
@@*/
static int PUGH_EnableGArrayGroupStorage (pGH *pughGH,
int first_var,
int n_variables,
int n_timelevels)
{
DECLARE_CCTK_PARAMETERS
int nstorage; /* Number of Arrays for which storage was set */
int nnostorage; /* Number of Arrays for which no storage was set */
int retval;
int var;
pGA *GA;
int level;
nstorage = nnostorage = 0;
for (var = first_var; var < first_var + n_variables; var++)
{
for (level = 0; level < n_timelevels; level++)
{
GA = (pGA *) pughGH->variables[var][level];
if (! GA->storage)
{
#ifdef DEBUG_PUGH
printf (" PUGH_EnableGArrayGroupStorage: request for var '%s' "
"timelevel %d\n", GA->name, level);
fflush (stdout);
#endif
PUGH_EnableGArrayDataStorage (GA,
pughGH->myproc,
initialize_memory,
padding_active,
padding_cacheline_bits,
padding_size,
padding_address_spacing);
((cGH *) pughGH->callerid)->data[var][level] = GA->data;
nnostorage++;
}
else
{
((cGH *) pughGH->callerid)->data[var][level] = GA->data;
nstorage++;
}
}
}
if (nstorage > 0 && nnostorage > 0)
{
CCTK_WARN (0, "Group storage violation in PUGH_EnableGArrayGroupStorage");
retval = -1;
}
else
{
retval = nstorage > 0 ? 1 : 0;
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_DisableGroupStorage
@author Tom Goodale
@date 30 Mar 1999
@desc
Disables storage for all variables in the group
indicated by groupname.
@enddesc
@calls CCTK_GroupIndex
CCTK_GroupData
CCTK_FirstVarIndexI
PUGH_DisableGArrayGroupStorage
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype cGH *
@vio in
@endvar
@var groupname
@vdesc name of the group to enable storage for
@vtype const char *
@vio in
@endvar
@returntype int
@returndesc
1 if storage for given group was disabled
-1 if group type is invalid
@endreturndesc
@@*/
int PUGH_DisableGroupStorage (cGH *GH, const char *groupname)
{
DECLARE_CCTK_PARAMETERS
int group; /* group index */
cGroup pgroup; /* group information */
int vtypesize, retval;
pGA ***variables;
int first_var, var, level;
int unchanged; /* count how many aren't toggled */
char *temp;
#ifdef DEBUG_PUGH
printf (" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
fflush (stdout);
#endif
group = CCTK_GroupIndex (groupname);
CCTK_GroupData (group, &pgroup);
/* get global index of first variable in group */
first_var = CCTK_FirstVarIndexI (group);
variables = (pGA ***) PUGH_pGH (GH)->variables;
/* get the group info from its index */
unchanged = 0;
retval = 1;
if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
for (var = first_var; var < first_var+pgroup.numvars; var++)
{
for (level = 0; level < pgroup.numtimelevels; level++)
{
unchanged += PUGH_DisableGArrayDataStorage (variables[var][level]);
}
}
}
else if (pgroup.grouptype == CCTK_SCALAR)
{
vtypesize = CCTK_VarTypeSize (pgroup.vartype);
for (var = first_var; var < first_var+pgroup.numvars; var++)
{
for (level = 0; level < pgroup.numtimelevels; level++)
{
temp = (char *) variables[var][level];
if (temp[vtypesize] == PUGH_STORAGE)
{
temp[vtypesize] = PUGH_NOSTORAGE;
}
else
{
unchanged++;
}
}
}
}
else
{
CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
retval = -1;
}
/* Report on memory usage */
if (!CCTK_Equals(storage_verbose,"no") && retval >= 0)
{
if (unchanged == 0)
{
/* Memory toggled */
if (pgroup.grouptype == CCTK_GF)
{
totalnumberGF -= pgroup.numvars;
}
else if (pgroup.grouptype == CCTK_ARRAY)
{
totalnumberGA -= pgroup.numvars;
}
totalstorage -= (variables[first_var][0]->extras->npoints *
variables[first_var][0]->varsize *
pgroup.numtimelevels * pgroup.numvars) /
(float) (1024 * 1024);
if (CCTK_Equals(storage_verbose,"yes"))
{
CCTK_VInfo (CCTK_THORNSTRING, "Switched memory off for group '%s'"
" [GFs: %d GAs: %d Total Size: %6.2fMB]",
groupname, totalnumberGF, totalnumberGA, totalstorage);
}
}
else if (unchanged == pgroup.numvars)
{
/* Memory already off */
if (CCTK_Equals(storage_verbose,"yes"))
{
CCTK_VInfo (CCTK_THORNSTRING, "Memory already off for group '%s'", groupname);
}
}
else
{
CCTK_WARN (1, "PUGH_DisableGroupStorage: Inconsistency in group memory assignment");
}
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_EnableGroupStorage
@author Tom Goodale
@date 30 Mar 1999
@desc
Enables storage for all variables in the group
indicated by groupname.
@enddesc
@calls CCTK_GroupIndex
CCTK_GroupData
PUGH_EnableScalarGroupStorage
PUGH_EnableGArrayGroupStorage
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype cGH *
@vio in
@endvar
@var groupname
@vdesc name of the group to enable storage for
@vtype const char *
@vio in
@endvar
@returntype int
@returndesc
return code of @seeroutine PUGH_EnableScalarGroupStorage or
@seeroutine PUGH_EnableGArrayGroupStorage: <BR>
1 if storage was already enabled, or <BR>
0 if storage was successfully enabled <BR>
-1 if group type is not one of the above <BR>
-2 if an invalid GH pointer was given <BR>
-3 if invalid groupname was given
@endreturndesc
@@*/
int PUGH_EnableGroupStorage (cGH *GH, const char *groupname)
{
DECLARE_CCTK_PARAMETERS
int group; /* group index */
int first_var; /* first variable's index */
cGroup pgroup; /* group information */
int retval;
pGA *GA;
pGH *pughGH;
#ifdef DEBUG_PUGH
printf (" PUGH_EnableGroupStorage: request for group '%s'\n", groupname);
fflush (stdout);
#endif
pughGH = PUGH_pGH (GH);
group = CCTK_GroupIndex (groupname);
if (pughGH && group >= 0)
{
first_var = CCTK_FirstVarIndexI (group);
/* get the group info from its index */
CCTK_GroupData (group, &pgroup);
if (pgroup.grouptype == CCTK_SCALAR)
{
retval = PUGH_EnableScalarGroupStorage (pughGH,
first_var,
pgroup.numvars,
pgroup.numtimelevels);
}
else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
retval = PUGH_EnableGArrayGroupStorage (pughGH,
first_var,
pgroup.numvars,
pgroup.numtimelevels);
if (!CCTK_Equals(storage_verbose,"no") && retval == 0)
{
/* get GA pointer of first var in group */
GA = (pGA *) pughGH->variables[first_var][0];
if (pgroup.grouptype == CCTK_GF)
{
totalnumberGF += pgroup.numvars * pgroup.numtimelevels;
}
else
{
totalnumberGA += pgroup.numvars * pgroup.numtimelevels;
}
totalstorage += (GA->extras->npoints * GA->varsize *
pgroup.numtimelevels * pgroup.numvars) /
(float) (1024*1024);
if (totalstorage > maxstorage)
{
numberGF = totalnumberGF;
numberGA = totalnumberGA;
maxstorage = totalstorage;
}
/* Report on memory usage */
if (CCTK_Equals(storage_verbose,"yes"))
{
CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'"
" [GFs: %d GAs: %d Total Size: %6.2fMB]",
groupname, totalnumberGF, totalnumberGA, totalstorage);
}
}
}
else
{
CCTK_WARN (1, "PUGH_EnableGroupStorage: Unknown group type");
retval = -1;
}
}
else
{
if (! pughGH)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_EnableGroupStorage: Error with cctkGH pointer "
"for group %s", groupname);
retval = -2;
}
else
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_EnableGroupStorage: Invalid group %s", groupname);
retval = -3;
}
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_QueryGroupStorage
@author Gabrielle Allen
@date 13 Apr 1999
@desc
Checks if group has storage assigned.
Either group index or its name must be given,
the name takes precedence.
@enddesc
@calls CCTK_GroupIndex
CCTK_FirstVarIndexI
CCTK_GroupTypeI
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype const cGH *
@vio in
@endvar
@var group
@vdesc index of group
@vtype int
@vio in
@endvar
@var groupname
@vdesc name of the group to be queried
@vtype const char *
@vio in
@endvar
@returntype int
@returndesc
1 if storage for this group is assigned
0 if storage for this group is not assigned
-1 if given group index/name is invalid
@endreturndesc
@@*/
int PUGH_QueryGroupStorage (const cGH *GH, int group, const char *groupname)
{
int first_var;
int storage;
int grouptype;
int vtypesize;
int retval;
pGH *pughGH;
retval = -1;
if (groupname)
{
group = CCTK_GroupIndex (groupname);
}
/* get first variable in group */
first_var = CCTK_FirstVarIndexI (group);
if (first_var >= 0)
{
pughGH = PUGH_pGH (GH);
grouptype = CCTK_GroupTypeI (group);
if (grouptype == CCTK_SCALAR)
{
vtypesize = CCTK_VarTypeSize (CCTK_VarTypeI (first_var));
storage = ((char *) pughGH->variables[first_var][0])[vtypesize];
}
else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
{
storage = ((pGA *) pughGH->variables[first_var][0])->storage;
}
else
{
storage = -1;
CCTK_WARN (1, "Unknown group type in PUGH_QueryGroupStorage");
}
if (storage == PUGH_STORAGE)
{
retval = 1;
}
else if (storage == PUGH_NOSTORAGE)
{
retval = 0;
}
else
{
CCTK_WARN (1, "Inconsistency in PUGH_QueryGroupStorage");
}
}
else
{
if (groupname)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group name '%s'",
groupname);
}
else
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group ID %d",
group);
}
}
return (retval);
}
/*@@
@routine PUGH_EnableComm
@date Sun Jan 23 12:46:23 2000
@author Gabrielle Allen
@desc
This sets the docomm[2*dim] array of the GA based
on the setting of the comm flag and allocates the comm buffers.
@enddesc
@history
@date Mon Jun 05 2000 @author Thomas Radke
Moved buffer allocation from PUGH_EnableGArrayDataStorage
@endhistory
@@*/
static int PUGH_EnableComm(pGH *pughGH,
pComm *comm,
int commflag)
{
int retval; /* return value */
#ifdef CCTK_MPI
pGA *GA; /* GA structure the pComm belongs to */
int i, idir; /* loopers */
int dir; /* direction */
int sz; /* buffer size */
#endif
retval = 1;
#ifdef CCTK_MPI
/* check whether this comm buffer is already set up */
if (comm->commflag == commflag)
{
return (retval);
}
/* get the GA structure the comm structure belongs to
For a group comm structure this GA is the first one within the group. */
GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
if (comm->commflag == PUGH_NOCOMM)
{
#ifdef DEBUG_PUGH
printf (" PUGH_EnableComm: allocating comm buffer for %d vars starting "
"with %d '%s'\n", comm->n_vars, comm->first_var, GA->name);
fflush (stdout);
#endif
/* allocate memory for communication buffers: 2 faces per direction */
for (i = 0; i < 2 * GA->extras->dim; i++)
{
if (GA->connectivity->neighbours[pughGH->myproc][i] >= 0)
{
dir = i/2;
/* no ghostzones -> no comm buffers */
sz = comm->n_vars * GA->extras->nghostzones[dir];
if (sz > 0)
{
for (idir = 0; idir < GA->extras->dim; idir++)
{
if (idir != dir)
{
sz *= GA->extras->lnsize[idir];
}
}
comm->buffer_sz[i] = sz;
comm->send_buffer[i] = malloc(sz * GA->varsize);
comm->recv_buffer[i] = malloc(sz * GA->varsize);
if (! (comm->send_buffer[i] && comm->recv_buffer[i]))
{
for (; i >=0 ; i--)
{
if (comm->send_buffer[i])
{
free(comm->send_buffer[i]);
}
if (comm->recv_buffer[i])
{
free(comm->recv_buffer[i]);
}
comm->buffer_sz[i] = 0;
}
CCTK_WARN(1, "Out of memory !");
retval = -1;
break;
}
}
else
{
comm->buffer_sz[i] = 0;
comm->send_buffer[i] = NULL;
comm->recv_buffer[i] = NULL;
}
}
else /* no neighbor -> no comm buffers */
{
comm->buffer_sz[i] = 0;
comm->send_buffer[i] = NULL;
comm->recv_buffer[i] = NULL;
}
}
}
/* set up comm flags for each face */
if (retval >= 0)
{
/* Copy commflag */
comm->commflag = commflag;
/* First set all communcation off */
for (idir = 0; idir < 2 * GA->extras->dim; idir++)
comm->docomm[idir] = 0;
if (commflag == PUGH_ALLCOMM)
{
for (idir = 0; idir < 2 * GA->extras->dim; idir++)
{
comm->docomm[idir] = comm->buffer_sz[idir] > 0;
}
}
else if (commflag == PUGH_PLUSFACESCOMM)
{
for (idir = 0; idir < GA->extras->dim; idir++)
{
comm->docomm[2*idir+1] = comm->buffer_sz[2*idir+1] > 0;
}
}
else if (commflag == PUGH_MINUSFACESCOMM)
{
for (idir = 0; idir < GA->extras->dim; idir++)
{
comm->docomm[2*idir] = comm->buffer_sz[2*idir] > 0;
}
}
else
{
for (idir = 0; idir < GA->extras->dim; idir++)
{
if (commflag == PUGH_COMM(idir))
{
comm->docomm[2*idir] = comm->buffer_sz[2*idir] > 0;
comm->docomm[2*idir+1] = comm->buffer_sz[2*idir+1] > 0;
}
}
}
/* FIXME Add back the check that you have a valid COMM model: Gab */
/* Handle nsize = 1 type cases. This is only important for one
processor MPI periodic boundaries */
for (idir = 0; idir < GA->extras->dim; idir++)
{
if (GA->extras->nsize[idir] == 1)
{
comm->docomm[2*idir] = 0;
comm->docomm[2*idir+1] = 0;
}
}
}
#else
/* get rid of compiler warning about unused parameters */
pughGH = pughGH;
comm = comm;
commflag = commflag;
#endif /* CCTK_MPI */
return retval;
}
/*@@
@routine PUGH_SetupScalarGroup
@date Wed Feb 17 04:45:49 1999
@author Tom Goodale
@desc
Set up a group of scalar variables on a pGH.
For efficiency reasons, PUGH allocates storage for scalars
only once when the group is created.
The current state of storage allocation (which is toggled by
Enable/DisableGroupStorage) is stored in a byte-sized flag
immediately after the scalar data.
@enddesc
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
PUGH_SUCCESS (0) if successful, or
PUGH_ERRORMEMORY (negative) if memory allocation failed
@endreturndesc
@@*/
static int PUGH_SetupScalarGroup (pGH *newGH,
int vtype,
int n_variables,
int n_timelevels,
int vectorgroup)
{
DECLARE_CCTK_PARAMETERS
int variable, level, vtypesize, retval;
void *temp;
retval = 0; /* PUGH_SUCCESS */
temp = realloc (newGH->variables,
(newGH->nvariables + n_variables) * sizeof (void **));
if (temp)
{
newGH->variables = (void ***) temp;
vtypesize = CCTK_VarTypeSize (vtype);
for (variable = 0; variable < n_variables && retval == 0; variable++)
{
temp = malloc (n_timelevels * sizeof (void *));
if (temp)
{
newGH->variables[newGH->nvariables + variable] = (void **) temp;
for (level = 0; level < n_timelevels; level++)
{
/* allocate one more byte for the query_storage flag */
temp = malloc (vtypesize + 1);
if (temp)
{
newGH->variables[newGH->nvariables + variable][level] = temp;
/* reset the query_storage flag */
((char *) temp)[vtypesize] = PUGH_NOSTORAGE;
}
else
{
retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
if (! retval)
{
newGH->nvariables += n_variables;
}
else
{
CCTK_WARN (1, "Memory allocation error in PUGH_SetupScalarGroup");
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_SetupGAGroup
@date January 19 2000
@author Gabrielle Allen
@desc
Set up a group of grid array variables (CCTK_GF or CCTK_ARRAY
types) on a pGH.
@enddesc
@calls CCTK_VarTypeSize
PUGH_SetupConnectivity
PUGH_SetupPGExtras
PUGH_SetupGArrayGroupComm
PUGH_SetupGArray
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var nsize
@vdesc size dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var ghostsize
@vdesc ghostsize dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var gtype
@vdesc group type
@vtype int
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var dim
@vdesc number of dimensions for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var staggercode
@vdesc stagger code for variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
PUGH_SUCCESS (0) if successful, or
PUGH_ERRORMEMORY (negative) if memory allocation failed
@endreturndesc
@@*/
static int PUGH_SetupGAGroup (pGH *newGH,
int *nsize,
int *ghostsize,
int gtype,
int vtype,
int dim,
int n_variables,
int staggercode,
int n_timelevels,
int vectorgroup)
{
int i;
int variable;
int var_size;
int retval;
int level;
void ***temp;
pConnectivity *connectivity;
pGExtras *extras;
pComm *groupcomm;
int *perme;
int *nprocs;
retval = PUGH_SUCCESS;
var_size = CCTK_VarTypeSize (vtype);
if (gtype == CCTK_ARRAY)
{
/* Arrays can't (yet) have periodicity and manual setup,
so initialize them to zero */
perme = (int *) calloc (dim, sizeof (int));
nprocs = (int *) calloc (dim, sizeof (int));
if (! (perme && nprocs))
{
CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
}
/* Check that there are enough grid points in this dimension
* to make parallelising it worthwhile
*/
for (i = 0 ; i < dim; i++)
{
if (! nprocs[i] && abs (nsize[i]) <= 2*ghostsize[i] + 1)
{
nprocs[i] = 1;
}
}
connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);
extras = PUGH_SetupPGExtras (dim,
perme,
staggercode,
nsize,
ghostsize,
newGH->nprocs,
connectivity->nprocs,
newGH->myproc);
free (nprocs);
free (perme);
}
else
{
/* this is for CCTK_GF variables */
connectivity = newGH->Connectivity[dim-1];
extras = newGH->GFExtras[dim-1];
}
/* Set up the communication buffer used for all variables within this group.
Note: only with allocated buffers we can have group communication. */
if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
groupcomm = PUGH_SetupGArrayGroupComm (newGH,
dim,
newGH->nvariables,
n_variables,
0,
vtype,
extras);
}
else
{
groupcomm = NULL;
}
temp = (void ***) realloc (newGH->variables, (newGH->nvariables+n_variables) *
sizeof (void **));
if(temp)
{
newGH->variables = temp;
for (variable = 0; variable < n_variables; variable++)
{
newGH->variables[newGH->nvariables] = (void **) malloc (n_timelevels *
sizeof (void *));
if (newGH->variables[newGH->nvariables])
{
for (level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
PUGH_SetupGArray (newGH,
extras,
connectivity,
groupcomm,
CCTK_VarName (newGH->nvariables),
newGH->nvariables,
newGH->narrays,
var_size,
vtype,
staggercode,
vectorgroup ? n_variables : 1,
variable,
variable > 0 ? newGH->variables[newGH->nvariables-variable][level] : NULL);
newGH->narrays++;
}
newGH->nvariables++;
}
else
{
retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
if (retval)
{
CCTK_WARN (1, "Memory allocation error in PUGH_SetupGAGroup");
}
return (retval);
}