/*@@
@routine PUGH_EnableScalarGroupStorage
@author Thomas Radke
@date Thu 30 Aug 2001
@desc
Enables storage for a group of CCTK_SCALAR variables
For efficiency reasons, PUGH allocates storage for scalars
only once when the group is created.
The current state of storage allocation (which is toggled by
Enable/DisableGroupStorage) is stored in a byte-sized flag
immediately after the scalar data.
@enddesc
@var pughGH
@vdesc Pointer to PUGH GH extensions
@vtype pGH *
@vio in
@endvar
@var first_var
@vdesc index of the first variable to enable storage for
@vtype int
@vio in
@endvar
@var n_variables
@vdesc total number of variables to enable storage for
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc total number of timelevels to enable storage for
@vtype int
@vio in
@endvar
@returntype int
@returndesc
1 if storage was already enabled before
0 if storage was successfully enabled
@endreturndesc
@@*/
static int PUGH_EnableScalarGroupStorage (pGH *pughGH,
int first_var,
int n_variables,
int n_timelevels)
{
DECLARE_CCTK_PARAMETERS
int vtype, vtypesize, variable, level, retval;
void *temp;
vtype = CCTK_VarTypeI (first_var);
vtypesize = CCTK_VarTypeSize (vtype);
temp = pughGH->variables[first_var][0];
retval = ((char *) temp)[vtypesize] == PUGH_STORAGE;
/* don't assign storage if was already switched on */
if (! retval)
{
for (variable = 0; variable < n_variables; variable++)
{
for (level = 0; level < n_timelevels; level++)
{
temp = pughGH->variables[variable+first_var][level];
/* raise the query_storage flag */
((char *) temp)[vtypesize] = PUGH_STORAGE;
/* initialize memory if desired */
if (! CCTK_Equals (initialize_memory, "none"))
{
PUGH_InitializeMemory (initialize_memory, vtype, vtypesize, temp);
}
}
}
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_DisableGroupStorage
@author Tom Goodale
@date 30 Mar 1999
@desc
Disables storage for all variables in the group
indicated by groupname.
@enddesc
@calls CCTK_GroupIndex
CCTK_GroupData
CCTK_FirstVarIndexI
PUGH_DisableGArrayGroupStorage
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype cGH *
@vio in
@endvar
@var groupname
@vdesc name of the group to enable storage for
@vtype const char *
@vio in
@endvar
@returntype int
@returndesc
1 if storage for given group was disabled
-1 if group type is invalid
@endreturndesc
@@*/
int PUGH_DisableGroupStorage (cGH *GH, const char *groupname)
{
DECLARE_CCTK_PARAMETERS
int group; /* group index */
cGroup pgroup; /* group information */
int vtypesize, retval;
pGA ***variables;
int first_var, var, level;
int unchanged; /* count how many aren't toggled */
char *temp;
#ifdef DEBUG_PUGH
printf (" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
fflush (stdout);
#endif
group = CCTK_GroupIndex (groupname);
CCTK_GroupData (group, &pgroup);
/* get global index of first variable in group */
first_var = CCTK_FirstVarIndexI (group);
variables = (pGA ***) PUGH_pGH (GH)->variables;
/* get the group info from its index */
unchanged = 0;
retval = 1;
if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
for (var = first_var; var < first_var+pgroup.numvars; var++)
{
for (level = 0; level < pgroup.numtimelevels; level++)
{
unchanged += PUGH_DisableGArrayDataStorage (variables[var][level]);
}
}
}
else if (pgroup.grouptype == CCTK_SCALAR)
{
vtypesize = CCTK_VarTypeSize (pgroup.vartype);
for (var = first_var; var < first_var+pgroup.numvars; var++)
{
for (level = 0; level < pgroup.numtimelevels; level++)
{
temp = (char *) variables[var][level];
if (temp[vtypesize] == PUGH_STORAGE)
{
temp[vtypesize] = PUGH_NOSTORAGE;
}
else
{
unchanged++;
}
}
}
}
else
{
CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
retval = -1;
}
/* Report on memory usage */
if (!CCTK_Equals(storage_verbose,"no") && retval >= 0)
{
if (unchanged == 0)
{
/* Memory toggled */
if (pgroup.grouptype == CCTK_GF)
{
totalnumberGF -= pgroup.numvars;
}
else if (pgroup.grouptype == CCTK_ARRAY)
{
totalnumberGA -= pgroup.numvars;
}
totalstorage -= (variables[first_var][0]->extras->npoints *
variables[first_var][0]->varsize *
pgroup.numtimelevels * pgroup.numvars) /
(float) (1024 * 1024);
if (CCTK_Equals(storage_verbose,"yes"))
{
CCTK_VInfo (CCTK_THORNSTRING, "Switched memory off for group '%s'"
" [GFs: %d GAs: %d Total Size: %6.2fMB]",
groupname, totalnumberGF, totalnumberGA, totalstorage);
}
}
else if (unchanged == pgroup.numvars)
{
/* Memory already off */
if (CCTK_Equals(storage_verbose,"yes"))
{
CCTK_VInfo (CCTK_THORNSTRING, "Memory already off for group '%s'", groupname);
}
}
else
{
CCTK_WARN (1, "PUGH_DisableGroupStorage: Inconsistency in group memory assignment");
}
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_SetupScalarGroup
@date Wed Feb 17 04:45:49 1999
@author Tom Goodale
@desc
Set up a group of scalar variables on a pGH.
For efficiency reasons, PUGH allocates storage for scalars
only once when the group is created.
The current state of storage allocation (which is toggled by
Enable/DisableGroupStorage) is stored in a byte-sized flag
immediately after the scalar data.
@enddesc
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
PUGH_SUCCESS (0) if successful, or
PUGH_ERRORMEMORY (negative) if memory allocation failed
@endreturndesc
@@*/
static int PUGH_SetupScalarGroup (pGH *newGH,
int vtype,
int n_variables,
int n_timelevels,
int vectorgroup)
{
DECLARE_CCTK_PARAMETERS
int variable, level, vtypesize, retval;
void *temp;
retval = 0; /* PUGH_SUCCESS */
temp = realloc (newGH->variables,
(newGH->nvariables + n_variables) * sizeof (void **));
if (temp)
{
newGH->variables = (void ***) temp;
vtypesize = CCTK_VarTypeSize (vtype);
for (variable = 0; variable < n_variables && retval == 0; variable++)
{
temp = malloc (n_timelevels * sizeof (void *));
if (temp)
{
newGH->variables[newGH->nvariables + variable] = (void **) temp;
for (level = 0; level < n_timelevels; level++)
{
/* allocate one more byte for the query_storage flag */
temp = malloc (vtypesize + 1);
if (temp)
{
newGH->variables[newGH->nvariables + variable][level] = temp;
/* reset the query_storage flag */
((char *) temp)[vtypesize] = PUGH_NOSTORAGE;
}
else
{
retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
if (! retval)
{
newGH->nvariables += n_variables;
}
else
{
CCTK_WARN (1, "Memory allocation error in PUGH_SetupScalarGroup");
}
USE_CCTK_PARAMETERS; return (retval);
}
/*@@
@routine PUGH_QueryGroupStorage
@author Gabrielle Allen
@date 13 Apr 1999
@desc
Checks if group has storage assigned.
Either group index or its name must be given,
the name takes precedence.
@enddesc
@calls CCTK_GroupIndex
CCTK_FirstVarIndexI
CCTK_GroupTypeI
@var GH
@vdesc Pointer to CCTK grid hierarchy
@vtype const cGH *
@vio in
@endvar
@var group
@vdesc index of group
@vtype int
@vio in
@endvar
@var groupname
@vdesc name of the group to be queried
@vtype const char *
@vio in
@endvar
@returntype int
@returndesc
1 if storage for this group is assigned
0 if storage for this group is not assigned
-1 if given group index/name is invalid
@endreturndesc
@@*/
int PUGH_QueryGroupStorage (const cGH *GH, int group, const char *groupname)
{
int first_var;
int storage;
int grouptype;
int vtypesize;
int retval;
pGH *pughGH;
retval = -1;
if (groupname)
{
group = CCTK_GroupIndex (groupname);
}
/* get first variable in group */
first_var = CCTK_FirstVarIndexI (group);
if (first_var >= 0)
{
pughGH = PUGH_pGH (GH);
grouptype = CCTK_GroupTypeI (group);
if (grouptype == CCTK_SCALAR)
{
vtypesize = CCTK_VarTypeSize (CCTK_VarTypeI (first_var));
storage = ((char *) pughGH->variables[first_var][0])[vtypesize];
}
else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
{
storage = ((pGA *) pughGH->variables[first_var][0])->storage;
}
else
{
storage = -1;
CCTK_WARN (1, "Unknown group type in PUGH_QueryGroupStorage");
}
if (storage == PUGH_STORAGE)
{
retval = 1;
}
else if (storage == PUGH_NOSTORAGE)
{
retval = 0;
}
else
{
CCTK_WARN (1, "Inconsistency in PUGH_QueryGroupStorage");
}
}
else
{
if (groupname)
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group name '%s'",
groupname);
}
else
{
CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
"PUGH_ArrayGroupSize: Invalid group ID %d",
group);
}
}
return (retval);
}
/*@@
@routine PUGH_SetupGAGroup
@date January 19 2000
@author Gabrielle Allen
@desc
Set up a group of grid array variables (CCTK_GF or CCTK_ARRAY
types) on a pGH.
@enddesc
@calls CCTK_VarTypeSize
PUGH_SetupConnectivity
PUGH_SetupPGExtras
PUGH_SetupGArrayGroupComm
PUGH_SetupGArray
@var newGH
@vdesc Pointer to PUGH grid hierarchy
@vtype pGH *
@vio in
@endvar
@var nsize
@vdesc size dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var ghostsize
@vdesc ghostsize dimensions for variables in this group
@vtype int *
@vio in
@endvar
@var gtype
@vdesc group type
@vtype int
@vio in
@endvar
@var vtype
@vdesc CCTK data type for variables in this group
@vtype int
@vio in
@endvar
@var dim
@vdesc number of dimensions for variables in this group
@vtype int
@vio in
@endvar
@var n_variables
@vdesc number of variables in this group
@vtype int
@vio in
@endvar
@var staggercode
@vdesc stagger code for variables in this group
@vtype int
@vio in
@endvar
@var n_timelevels
@vdesc number of timelevels in this group
@vtype int
@vio in
@endvar
@var vectorgroup
@vdesc is this a vector group ?
@vtype int
@vio in
@endvar
@returntype int
@returndesc
PUGH_SUCCESS (0) if successful, or
PUGH_ERRORMEMORY (negative) if memory allocation failed
@endreturndesc
@@*/
static int PUGH_SetupGAGroup (pGH *newGH,
int *nsize,
int *ghostsize,
int gtype,
int vtype,
int dim,
int n_variables,
int staggercode,
int n_timelevels,
int vectorgroup)
{
int i;
int variable;
int var_size;
int retval;
int level;
void ***temp;
pConnectivity *connectivity;
pGExtras *extras;
pComm *groupcomm;
int *perme;
int *nprocs;
retval = PUGH_SUCCESS;
var_size = CCTK_VarTypeSize (vtype);
if (gtype == CCTK_ARRAY)
{
/* Arrays can't (yet) have periodicity and manual setup,
so initialize them to zero */
perme = (int *) calloc (dim, sizeof (int));
nprocs = (int *) calloc (dim, sizeof (int));
if (! (perme && nprocs))
{
CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
}
/* Check that there are enough grid points in this dimension
* to make parallelising it worthwhile
*/
for (i = 0 ; i < dim; i++)
{
if (! nprocs[i] && abs (nsize[i]) <= 2*ghostsize[i] + 1)
{
nprocs[i] = 1;
}
}
connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);
extras = PUGH_SetupPGExtras (dim,
perme,
staggercode,
nsize,
ghostsize,
newGH->nprocs,
connectivity->nprocs,
newGH->myproc);
free (nprocs);
free (perme);
}
else
{
/* this is for CCTK_GF variables */
connectivity = newGH->Connectivity[dim-1];
extras = newGH->GFExtras[dim-1];
}
/* Set up the communication buffer used for all variables within this group.
Note: only with allocated buffers we can have group communication. */
if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
groupcomm = PUGH_SetupGArrayGroupComm (newGH,
dim,
newGH->nvariables,
n_variables,
0,
vtype,
extras);
}
else
{
groupcomm = NULL;
}
temp = (void ***) realloc (newGH->variables, (newGH->nvariables+n_variables) *
sizeof (void **));
if(temp)
{
newGH->variables = temp;
for (variable = 0; variable < n_variables; variable++)
{
newGH->variables[newGH->nvariables] = (void **) malloc (n_timelevels *
sizeof (void *));
if (newGH->variables[newGH->nvariables])
{
for (level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
PUGH_SetupGArray (newGH,
extras,
connectivity,
groupcomm,
CCTK_VarName (newGH->nvariables),
newGH->nvariables,
newGH->narrays,
var_size,
vtype,
staggercode,
vectorgroup ? n_variables : 1,
variable,
variable > 0 ? newGH->variables[newGH->nvariables-variable][level] : NULL);
newGH->narrays++;
}
newGH->nvariables++;
}
else
{
retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
retval = PUGH_ERRORMEMORY;
}
if (retval)
{
CCTK_WARN (1, "Memory allocation error in PUGH_SetupGAGroup");
}
return (retval);
}
