Exemple #1
0
int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
{
  int group;               /* group index */
  cGroup pgroup;           /* group information */
  int rc;                  /* return code */

#ifdef DEBUG_PUGH
  printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
  fflush(stdout);
#endif

  /* get the group info from its index */
  group = CCTK_GroupIndex(groupname);
  CCTK_GroupData(group, &pgroup);

  if (pgroup.grouptype == CCTK_SCALAR)
  {
    rc = 1;
  }
  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
  {
    rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
                                    CCTK_FirstVarIndexI(group),
                                    PUGH_ALLCOMM);
  }
  else
  {
    CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
    rc = 0;
  }

  return (rc);
}
Exemple #2
0
 /*@@
   @routine    PUGH_SetupGroup
   @date       Mon Feb  8 19:37:55 1999
   @author     Tom Goodale
   @desc
               Sets up a group on a pGH
   @enddesc
   @calls      PUGH_SetupScalarGroup
               PUGH_SetupGAGroup

   @var        newGH
   @vdesc      Pointer to PUGH grid hierarchy
   @vtype      pGH *
   @vio        in
   @endvar
   @var        nsize
   @vdesc      size dimensions for variables in this group
   @vtype      int *
   @vio        in
   @endvar
   @var        ghostsize
   @vdesc      ghostsize dimensions for variables in this group
   @vtype      int *
   @vio        in
   @endvar
   @var        gtype
   @vdesc      group type
   @vtype      int
   @vio        in
   @endvar
   @var        vtype
   @vdesc      CCTK data type for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        dim
   @vdesc      number of dimensions for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_variables
   @vdesc      number of variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        staggercode
   @vdesc      stagger code for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_timelevels
   @vdesc      number of timelevels in this group
   @vtype      int
   @vio        in
   @endvar
   @var        vectorgroup
   @vdesc      is this a vector group ?
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               return code of @seeroutine PUGH_SetupScalarGroup or
               @seeroutine PUGH_SetupGAGroup, or
               PUGH_ERROR if invalid group type was passed in
   @endreturndesc
@@*/
int PUGH_SetupGroup (pGH *newGH,
                     int *nsize,
                     int *nghostsize,
                     int  gtype,
                     int  vtype,
                     int  dim,
                     int  n_variables,
                     int  staggercode,
                     int  n_timelevels,
                     int  vectorgroup)
{
  int retval;

  if (gtype == CCTK_SCALAR)
  {
    retval = PUGH_SetupScalarGroup (newGH, vtype, n_variables, n_timelevels, vectorgroup);
  }
  else if (gtype == CCTK_ARRAY || gtype == CCTK_GF)
  {
    retval = PUGH_SetupGAGroup (newGH, nsize, nghostsize, gtype, vtype, dim,
                                n_variables, staggercode, n_timelevels,vectorgroup);
  }
  else
  {
    CCTK_WARN (0, "Unknown group type in PUGH_SetupGroup");
    retval = PUGH_ERROR;
  }

  return (retval);
}
Exemple #3
0
 /*@@
   @routine    PUGH_ArrayGroupSize
   @author     Gabrielle Allen
   @date       11 Apr 1999
   @desc
               Returns size of arrays in a group, in a given direction.
               Either group index or its name must be given,
               the name takes precedence.
   @enddesc
   @calls      CCTK_GroupIndex
               CCTK_FirstVarIndexI


   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        dir
   @vdesc      direction to return size of
   @vtype      int
   @vio        in
   @endvar
   @var        group
   @vdesc      index of group
   @vtype      int
   @vio        in
   @endvar
   @var        groupname
   @vdesc      name of group
   @vtype      const char *
   @vio        in
   @endvar

   @returntype const int *
   @returndesc
               pointer to the size variable for the given direction, or
               NULL if given direction or group index/name are invalid
   @endreturndesc
@@*/
const int *PUGH_ArrayGroupSize (const cGH *GH,
                                int dir,
                                int group,
                                const char *groupname)
{
  int first_var;
  pGA *GA;
  const int *retval;


  if (groupname)
  {
    group = CCTK_GroupIndex (groupname);
  }

  first_var = CCTK_FirstVarIndexI (group);
  if (first_var >= 0)
  {
    /* get pointer to pGA for a timelevel of first variable in this group */
    GA = (pGA *) PUGH_pGH (GH)->variables[first_var][0];

    if (GA->storage == PUGH_STORAGE)
    {
      if (dir >= 0 && dir < GA->extras->dim)
      {
        retval = &GA->extras->lnsize[dir];
      }
      else
      {
        CCTK_WARN (1, "Wrong value for dir");
        retval = NULL;
      }
    }
    else
    {
      retval = &_cctk_one;
    }
  }
  else
  {
    if (groupname)
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_ArrayGroupSize: Invalid group name '%s'",
                  groupname);
    }
    else
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_ArrayGroupSize: Invalid group ID %d",
                  group);
    }

    retval = NULL;
  }

  return (retval);
}
Exemple #4
0
int PUGH_SyncGroup(cGH *GH, const char *groupname)
{
  cGroup pgroup;           /* group information */
  int group;               /* group index */
  int rc;                  /* return code */

#ifdef DEBUG_PUGH
  printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
  fflush (stdout);
#endif

  /* get the group info from its index */
  group = CCTK_GroupIndex(groupname);
  if (group < 0)
  {
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "PUGH_SyncGroup: Unknown group: %s", groupname);
    rc = -1;
  }
  else
  {
    CCTK_GroupData(group, &pgroup);

    if (pgroup.grouptype == CCTK_SCALAR)
    {
      rc = 0;
      CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
		 "PUGH_SyncGroup: Synchronising scalar group: %s",groupname);
    }
    else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
    {
      rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
    }
    else
    {
      CCTK_WARN(1, "PUGH_SyncGroup: Unknown group type");
      rc = 0;
    }
  }

  return (rc);
}
Exemple #5
0
int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
{
  int group;               /* group index */
  cGroup pgroup;           /* pointer to group information */
  int rc;                  /* return code */

  pGH *pughGH;
  int var;

#ifdef DEBUG_PUGH
  printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
  fflush(stdout);
#endif

  /* get the group info from its index */
  group = CCTK_GroupIndex(groupname);
  CCTK_GroupData(group, &pgroup);

  if (pgroup.grouptype == CCTK_SCALAR)
  {
    rc = 1;
  }
  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
  {
    pughGH=PUGH_pGH(GH);
    var = CCTK_FirstVarIndexI(group);

    /* FIXME:  workaround.  This one is really bad ! */
    rc = PUGH_DisableGArrayGroupComm(pughGH, var,(((pGA ***)pughGH->variables)[var][0])->groupcomm);
  }
  else
  {
    CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
    rc = 0;
  }

  return (rc);
}
Exemple #6
0
void Time_Simple(CCTK_ARGUMENTS)
{
  DECLARE_CCTK_PARAMETERS
  DECLARE_CCTK_ARGUMENTS

  CCTK_REAL min_spacing=0;

  /* Calculate the minimum grid spacing */
  if (cctk_dim>=1)
  {
    min_spacing = cctk_delta_space[0];
  }

  if (cctk_dim>=2)
  {
    min_spacing = (min_spacing<cctk_delta_space[1] ?
	min_spacing : cctk_delta_space[1]);
  }
  
  if (cctk_dim>=3)
  {
    min_spacing = (min_spacing<cctk_delta_space[2] ?
	min_spacing : cctk_delta_space[2]);
  }

  if (cctk_dim>=4)
  {
    CCTK_WARN(0,"Time Step not defined for greater than 4 dimensions");
  }

  /* Calculate the timestep */
  cctkGH->cctk_delta_time = dtfac*min_spacing;

  
  

  USE_CCTK_PARAMETERS;   USE_CCTK_CARGUMENTS }
Exemple #7
0
 /*@@
   @routine    PUGH_GenerateTopology
   @date       Fri Nov  5 11:31:21 1999
   @author     Tom Goodale
   @desc 
   Generate the appropriate processor topology for this processor
   decomposition.
   @enddesc 
   @calls     
   @calledby   
   @history 
   @hdate Tue Jan 30 17:04:50 2001 @hauthor Tom Goodale
   @hdesc Added call to integer root function and qsort to
          avoid problems with real to integer conversions and
          demonstrable failure of the algorithm when dealing
          with large prime factors. 
   @endhistory 

@@*/
int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs)
{
  int i;
  int used_procs;
  int free_procs;
  int retval;
  int free_dims;

  used_procs = 0;
  free_procs = total_procs;

  retval = 0;

  free_dims = dim;

  for(i=0; i < dim; i++)
  {
    if((nprocs[i])>0)
    {
      free_dims--;
      if(used_procs) 
      {
        used_procs *= nprocs[i];
      }
      else
      {
        used_procs = nprocs[i];
      }
      if (total_procs%used_procs)
      {
        CCTK_WARN(0, "Inconsistent PUGH topology");
        retval = 1;
      }
      else
      {
        free_procs = total_procs/used_procs;
      }
    }
  }

  /* Ok calculate topology if necessary */
  if(free_dims && ! retval)
  {
    /* This algorithm gives the most number of processors
     * in the highest dimension.
     */

    int *working;
    int root;
    int place;

    root = free_dims;
    working = (int *)calloc(free_dims,sizeof(int));
#ifdef DEBUG_PUGH
    printf("Processor topology for dimension %d\n",dim);
#endif


    for(i = 0; i < free_dims  ; i++)
    {
      working[i] = PUGH_IntegerRoot(free_procs, root);
      
      while(free_procs % working[i]) working[i]--;

#ifdef DEBUG_PUGH
      printf(" working[%d] = %d\n",i,working[i]);
#endif
      free_procs /= working[i];
      root--;
    }

    
    /* The above doesn't necessarily sort them properly 
     * e.g. if one of the factors is a prime then the
     * above will sort the 1 before the prime.
     */
    qsort(working,free_dims,sizeof(int),IntSort);

    for(i = 0,place=0; i < dim ; i++)
    {
      if(nprocs[i] <= 0)
      {
        nprocs[i] = working[place];
        place++;
      }

#ifdef DEBUG_PUGH
      printf(" nprocs[%d] = %d\n",i,nprocs[i]);
#endif
    }

    free(working);
  }

  return retval;
}
Exemple #8
0
CCTK_INT Hyperslab_DefineGlobalMappingByIndex (
           const cGH *GH,
           CCTK_INT vindex,
           CCTK_INT dim,
           const CCTK_INT *direction  /* vdim*dim */,
           const CCTK_INT *origin     /* vdim */,
           const CCTK_INT *extent     /* dim */,
           const CCTK_INT *downsample /* dim */,
           CCTK_INT table_handle,
           CCTK_INT target_proc,
           t_hslabConversionFn conversion_fn,
           CCTK_INT *hsize            /* dim */)
{
  unsigned int vdim, hdim, num_dirs;
  int retval;
  int stagger_index;
  int myproc;
  int npoints;
  hslab_mapping_t *mapping;
  const char *error_msg;
  const pGH *pughGH;              /* pointer to the current pGH */
  const pGA *GA;                  /* the variable's GA structure from PUGH */
  cGroup vinfo;


  /* PUGHSlab doesn't use table information */
  if (table_handle >= 0)
  {
    CCTK_WARN (1, "Hyperslab_DefineGlobalMappingByIndex: table information is "
                  "ignored");
  }

  /* check parameter consistency */
  retval = 0;
  error_msg = NULL;
  if (CCTK_GroupData (CCTK_GroupIndexFromVarI (vindex), &vinfo) < 0)
  {
    error_msg = "invalid variable index given";
    retval = -1;
  }
  else if (vinfo.grouptype != CCTK_GF && vinfo.grouptype != CCTK_ARRAY)
  {
    error_msg = "invalid variable group type given "
                "(not a CCTK_GF or CCTK_ARRAY type)";
    retval = -2;
  }
  else if (dim < 0 || dim > vinfo.dim)
  {
    error_msg = "invalid hyperslab dimension given";
    retval = -2;
  }
  else if (! direction || ! origin || ! extent || ! hsize)
  {
    error_msg = "NULL pointer(s) passed for direction/origin/extent/hsize "
                "parameters";
    retval = -3;
  }
  else if (target_proc >= CCTK_nProcs (GH))
  {
    error_msg = "invalid target procesor ID given";
    retval = -4;
  }
  else if ((pughGH = (const pGH *) PUGH_pGH (GH)) == NULL)
  {
    error_msg = "no PUGH GH extension registered (PUGH not activated ?)";
    retval = -4;
  }
  else
  {
    for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      retval |= origin[vdim] < 0;
      if (vdim < (unsigned int) dim)
      {
        retval |= extent[vdim] <= 0;
        if (downsample)
        {
          retval |= downsample[vdim] <= 0;
        }
      }
    }
    if (retval)
    {
      error_msg = "invalid hyperslab origin/extent/downsample vectors given";
      retval = -5;
    }
  }
  if (! retval)
  {
    mapping = (hslab_mapping_t *) malloc (sizeof (hslab_mapping_t));
    if (mapping)
    {
      mapping->vectors = (int *) malloc ((6*vinfo.dim + 2*dim) * sizeof (int));
    }
    if (mapping == NULL || mapping->vectors == NULL)
    {
      if (mapping)
      {
        free (mapping);
      }
      error_msg = "couldn't allocate hyperslab mapping structure";
      retval = -6;
    }
  }

  /* return in case of errors */
  if (retval)
  {
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Hyperslab_DefineGlobalMappingByIndex: %s", error_msg);
    return (retval);
  }

  mapping->hdim = (unsigned int) dim;
  mapping->vinfo = vinfo;
  mapping->target_proc = target_proc;
  mapping->conversion_fn = conversion_fn;

  /* assign memory for the other vectors */
  mapping->local_startpoint  = mapping->vectors + 0*vinfo.dim;
  mapping->local_endpoint    = mapping->vectors + 1*vinfo.dim;
  mapping->global_startpoint = mapping->vectors + 2*vinfo.dim;
  mapping->global_endpoint   = mapping->vectors + 3*vinfo.dim;
  mapping->do_dir            = mapping->vectors + 4*vinfo.dim;
  mapping->downsample        = mapping->vectors + 5*vinfo.dim;
  mapping->local_hsize       = mapping->vectors + 6*vinfo.dim + 0*dim;
  mapping->global_hsize      = mapping->vectors + 6*vinfo.dim + 1*dim;

  /* check direction vectors */
  for (hdim = 0; hdim < mapping->hdim; hdim++)
  {
    num_dirs = 0;
    for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      if (direction[hdim*vinfo.dim + vdim])
      {
        num_dirs++;
      }
    }
    if (num_dirs == 0)
    {
      free (mapping->vectors); free (mapping);
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Hyperslab_DefineGlobalMappingByIndex: %d-direction vector "
                  "is a null vector", hdim);
      return (-7);
    }

    mapping->is_diagonal_in_3D = num_dirs == 3 && mapping->hdim == 1;
    if (num_dirs != 1 && ! mapping->is_diagonal_in_3D)
    {
      free (mapping->vectors); free (mapping);
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Hyperslab_DefineGlobalMappingByIndex: %d-direction vector "
                  "isn't axis-orthogonal", hdim);
      return (-7);
    }
  }

  /* diagonals can be extracted from non-staggered 3D variables only */ 
  if (mapping->is_diagonal_in_3D && vinfo.stagtype != 0)
  {
    free (mapping->vectors); free (mapping);
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Hyperslab_DefineGlobalMappingByIndex: diagonals can be "
                "extracted from non-staggered 3D variables only");
    return (-7);
  }

  for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
  {
    mapping->do_dir[vdim] = 0;
    for (hdim = 0; hdim < mapping->hdim; hdim++)
    {
      if (direction[hdim*vinfo.dim + vdim])
      {
        mapping->do_dir[vdim]++;
      }
    }
    if (mapping->do_dir[vdim] > 1)
    {
      free (mapping->vectors); free (mapping);
      CCTK_WARN (1, "Hyperslab_DefineGlobalMappingByIndex: duplicate direction "
                    "vectors given");
      return (-8);
    }
  }

  /* get the pGH pointer and the variable's GA structure */
  GA     = (const pGA *) pughGH->variables[vindex][0];
  myproc = CCTK_MyProc (GH);

  /* check extent */
  for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
  {
    if (mapping->do_dir[vdim] && hdim < mapping->hdim)
    {
      if (origin[vdim] + extent[hdim] > GA->extras->nsize[vdim])
      {
        free (mapping->vectors); free (mapping);
        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                    "Hyperslab_DefineGlobalMappingByIndex: extent in "
                    "%d-direction exceeds grid size", hdim);
        return (-8);
      }
      hdim++;
    }
    else if (mapping->is_diagonal_in_3D &&
             origin[vdim] + extent[0] > GA->extras->nsize[vdim])
    {
      free (mapping->vectors); free (mapping);
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Hyperslab_DefineGlobalMappingByIndex: extent in "
                  "%d-direction exceeds grid size", vdim);
      return (-8);
    }
  }

  /* now fill out the hyperslab mapping structure */
  for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
  {
    mapping->downsample[vdim] = 1;

    if (mapping->do_dir[vdim] && hdim < mapping->hdim)
    {
      if (downsample)
      {
        mapping->downsample[vdim] = downsample[hdim];
      }
      mapping->global_hsize[hdim] = extent[hdim] / mapping->downsample[vdim];
      if (extent[hdim] % mapping->downsample[vdim])
      {
        mapping->global_hsize[hdim]++;
      }
      /* subtract ghostzones for periodic BC */
      if (GA->connectivity->perme[vdim])
      {
        mapping->global_hsize[hdim] -= 2 * GA->extras->nghostzones[vdim];
      }
      hdim++;
    }
    else if (mapping->is_diagonal_in_3D)
    {
      mapping->totals = extent[0] / mapping->downsample[0];
      if (extent[0] % mapping->downsample[0])
      {
        mapping->totals++;
      }
      /* subtract ghostzones for periodic BC */
      if (GA->connectivity->perme[vdim])
      {
        mapping->totals -= 2 * GA->extras->nghostzones[vdim];
      }
      if ((unsigned int) mapping->global_hsize[0] > mapping->totals)
      {
        mapping->global_hsize[0] = mapping->totals;
      }
    }
  }

  /* check whether the full local data patch was requested as hyperslab */
  mapping->is_full_hyperslab = IsFullHyperslab (GA, origin, extent, mapping);
  if (mapping->is_full_hyperslab)
  {
    memset (mapping->local_startpoint, 0, vinfo.dim * sizeof (int));
    memcpy (mapping->local_endpoint, GA->extras->lnsize, vinfo.dim*sizeof(int));
    mapping->totals = GA->extras->npoints;
  }
  else if (mapping->is_diagonal_in_3D)
  {
    /* just initialize the downsample and global_startpoint vectors */
    for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      mapping->downsample[vdim] = mapping->downsample[0];
      mapping->global_startpoint[vdim] = origin[vdim];
    }
  }
  else
  {
    /* compute the global endpoint */
    for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      mapping->global_endpoint[vdim] = origin[vdim] +
                                   (mapping->do_dir[vdim] ? extent[hdim++] : 1);
    }

    /* compute this processor's global startpoint from the global ranges */
    for (vdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      stagger_index = CCTK_StaggerDirIndex (vdim, vinfo.stagtype);

      if (origin[vdim] < MY_GLOBAL_EP (GA->extras, myproc, stagger_index, vdim))
      {
        mapping->global_startpoint[vdim] = origin[vdim];
        if (origin[vdim] < MY_GLOBAL_SP (GA->extras, myproc,stagger_index,vdim))
        {
          npoints = (MY_GLOBAL_SP (GA->extras, myproc, stagger_index, vdim)
                     - origin[vdim]) / mapping->downsample[vdim];
          if ((MY_GLOBAL_SP (GA->extras, myproc, stagger_index, vdim)
               - origin[vdim]) % mapping->downsample[vdim])
          {
            npoints++;
          }
          mapping->global_startpoint[vdim] += npoints*mapping->downsample[vdim];
        }
      }
      else
      {
        mapping->global_startpoint[vdim] = -1;
      }
    }

    /* compute the local start- and endpoint from the global ranges */
    mapping->totals = 1;
    for (vdim = hdim = 0; vdim < (unsigned int) vinfo.dim; vdim++)
    {
      stagger_index = CCTK_StaggerDirIndex (vdim, vinfo.stagtype);

      if (mapping->global_startpoint[vdim] >= 0 &&
          mapping->global_startpoint[vdim] <  MY_GLOBAL_EP (GA->extras, myproc,
                                                            stagger_index,vdim))
      {
        mapping->local_startpoint[vdim] = mapping->global_startpoint[vdim] -
                                          GA->extras->lb[myproc][vdim];
      }
      else
      {
        mapping->local_startpoint[vdim] = -1;
      }

      if (mapping->global_endpoint[vdim] > MY_GLOBAL_SP (GA->extras, myproc,
                                                         stagger_index, vdim))
      {
        mapping->local_endpoint[vdim] =
          MIN (MY_LOCAL_EP (GA->extras, stagger_index, vdim),
                            mapping->global_endpoint[vdim] -
                            GA->extras->lb[myproc][vdim]);
      }
      else
      {
        mapping->local_endpoint[vdim] = -1;
      }

#ifdef DEBUG
      printf ("direction %d: global ranges [%d, %d), local ranges[%d, %d)\n",
              vdim,
              mapping->global_startpoint[vdim], mapping->global_endpoint[vdim],
              mapping->local_startpoint[vdim], mapping->local_endpoint[vdim]);
#endif

      if (mapping->local_endpoint[vdim] < 0 ||
          mapping->local_startpoint[vdim] < 0)
      {
        mapping->totals = 0;
        mapping->local_endpoint[vdim] = mapping->local_startpoint[vdim];
      }

      if (mapping->do_dir[vdim])
      {
        /* compute the local size in each hyperslab dimension */
        mapping->local_hsize[hdim] = (mapping->local_endpoint[vdim] -
                                   mapping->local_startpoint[vdim]) /
                                  mapping->downsample[vdim];
        if ((mapping->local_endpoint[vdim] - mapping->local_startpoint[vdim]) %
            mapping->downsample[vdim])
        {
          mapping->local_hsize[hdim]++;
        }
        mapping->totals *= mapping->local_hsize[hdim];
        hdim++;
      }
    }
  } /* end of else branch for 'if (mapping->is_full_hyperslab)' */

#ifdef DEBUG
  printf ("total number of hyperslab data points: %d\n", mapping->totals);
#endif

#if 0
  if (mapping->totals > 0)
  {
    /* if requested, compute the offsets into the global hyperslab */
    if (hoffset_global)
    {
      for (i = hdim = 0; i < vinfo.dim; i++)
      {
        if (mapping->do_dir[i])
        {
          if (mapping->is_full_hyperslab)
          {
            hoffset_global[hdim] = GA->extras->lb[myproc][i];
          }
          else
          {
            hoffset_global[hdim] = (mapping->global_startpoint[i] -
                                    origin[i]) / mapping->downsample[i];
            if (GA->connectivity->perme[i])
            {
              hoffset_global[hdim] -= GA->extras->nghostzones[i];
            }
          }
#ifdef DEBUG
          printf ("hoffset_global, hsize in direction %d: %d, %d\n",
                  hdim, hoffset_global[hdim], mapping->local_hsize[hdim]);
#endif
          hdim++;
        }
      }
    }
  }
#endif

  /* add this mapping to the mapping list */
  if (mapping_list)
  {
    mapping_list->prev = mapping;
  }
  mapping->prev = NULL;
  mapping->next = mapping_list;
  mapping_list = mapping;

  mapping->handle = nmapping_list++;

  /* set the global hsize in the return arguments */
  if (hsize)
  {
    for (hdim = 0; hdim < mapping->hdim; hdim++)
    {
      hsize[hdim] = mapping->global_hsize[hdim];
    }
  }

  return (mapping->handle);
}
Exemple #9
0
 /*@@
   @routine    Setup_nProcs
   @date       Tue Apr 18 15:21:42 2000
   @author     Tom Goodale
   @desc
               Setup PUGH_COMM_WORLD communicator and make sure that CCTK_INT
               and CCTK_REAL sizes are the same on all processors.
               Define MPI datatypes for CCTK_REAL and CCTK_COMPLEX variables.
   @enddesc

   @var        pughGH
   @vdesc      Pointer to PUGH grid hierarchy
   @vtype      pGH *
   @vio        in
   @endvar
   @var        dim
   @vdesc      dimension of processor topology to set up
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               0 for success
   @endreturndesc
@@*/
static int Setup_nProcs (pGH *pughGH, int dim)
{
#ifdef CCTK_MPI
  CCTK_REAL4 sum_sizes [2], compiled_sizes [2];


  /* Set up my communicator. This would allow us to run pugh on
     a subset of processors at a later date if, for instance, we
     were using panda or what not.
   */
  CACTUS_MPI_ERROR (MPI_Comm_dup (MPI_COMM_WORLD, &pughGH->PUGH_COMM_WORLD));

  CACTUS_MPI_ERROR (MPI_Comm_size (pughGH->PUGH_COMM_WORLD, &pughGH->nprocs));
  CACTUS_MPI_ERROR (MPI_Comm_rank (pughGH->PUGH_COMM_WORLD, &pughGH->myproc));

  /* check that all executables uses the same integer and fp precision
     within a metacomputing environment
     NOTE: We cannot use CCTK_INT4s in MPI_Allreduce() since
           (although they have the same size) they might be correspond to
           different MPI datatypes.
           CCTK_REAL4 should always refer to MPI_FLOAT. */
  compiled_sizes [0] = sizeof (CCTK_INT);
  compiled_sizes [1] = sizeof (CCTK_REAL);
  CACTUS_MPI_ERROR (MPI_Allreduce (compiled_sizes, sum_sizes, 2, PUGH_MPI_REAL4,
                                   MPI_SUM, pughGH->PUGH_COMM_WORLD));
  if (compiled_sizes [0] * pughGH->nprocs != sum_sizes [0])
  {
    CCTK_WARN (0, "Cannot run executables with different precision for "
                  "CCTK_INTs within a metacomputing environment !\n"
                  "Please configure with unique CCTK_INTEGER_PRECISION !");
  }

  if (compiled_sizes [1] * pughGH->nprocs != sum_sizes [1])
  {
    CCTK_WARN (0, "Cannot run executables with different precision for "
                  "CCTK_REALs within a metacomputing environment !\n"
                  "Please configure with unique CCTK_REAL_PRECISION !");
  }

  /* define the complex datatype as a concatanation of 2 PUGH_MPI_REALs */
  CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL,
                                         &pughGH->PUGH_mpi_complex));
  CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex));

  /* dito for fixed-precision reals */
#ifdef CCTK_REAL4
  CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL4,
                                         &pughGH->PUGH_mpi_complex8));
  CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex8));
#endif
#ifdef CCTK_REAL8
  CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL8,
                                         &pughGH->PUGH_mpi_complex16));
  CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex16));
#endif
#ifdef CCTK_REAL16
  CACTUS_MPI_ERROR (MPI_Type_contiguous (2, PUGH_MPI_REAL16,
                                         &pughGH->PUGH_mpi_complex32));
  CACTUS_MPI_ERROR (MPI_Type_commit (&pughGH->PUGH_mpi_complex32));
#endif

#else
  pughGH->nprocs = 1;
  pughGH->myproc = 0;
#endif

  pughGH->dim = dim;

  return (0);
}
Exemple #10
0
 /*@@
   @routine    PUGH_EnableGArrayGroupStorage
   @author     Tom Goodale
   @date       30 Mar 1999
   @desc
               Enables storage for a set of variables
   @enddesc
   @calls      PUGH_EnableGArrayDataStorage

   @var        pughGH
   @vdesc      Pointer to PUGH GH extensions
   @vtype      pGH *
   @vio        in
   @endvar
   @var        first_var
   @vdesc      index of the first variable to enable storage for
   @vtype      int
   @vio        in
   @endvar
   @var        n_variables
   @vdesc      total number of variables to enable storage for
   @vtype      int
   @vio        in
   @endvar
   @var        n_timelevels
   @vdesc      total number of timelevels to enable storage for
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
                1 if storage was already enabled
                0 if storage was enabled (for at least one variable)
               -1 if there was an inconsistency in storage allocation
   @endreturndesc
@@*/
static int PUGH_EnableGArrayGroupStorage (pGH *pughGH,
                                          int first_var,
                                          int n_variables,
                                          int n_timelevels)
{
  DECLARE_CCTK_PARAMETERS
  int nstorage;   /* Number of Arrays for which storage was set */
  int nnostorage; /* Number of Arrays for which no storage was set */
  int retval;
  int var;
  pGA *GA;
  int level;


  nstorage = nnostorage = 0;

  for (var = first_var; var < first_var + n_variables; var++)
  {
    for (level = 0; level < n_timelevels; level++)
    {
      GA = (pGA *) pughGH->variables[var][level];

      if (! GA->storage)
      {
#ifdef DEBUG_PUGH
        printf (" PUGH_EnableGArrayGroupStorage: request for var '%s' "
                "timelevel %d\n", GA->name, level);
        fflush (stdout);
#endif
        PUGH_EnableGArrayDataStorage (GA,
                                      pughGH->myproc,
                                      initialize_memory,
                                      padding_active,
                                      padding_cacheline_bits,
                                      padding_size,
                                      padding_address_spacing);

        ((cGH *) pughGH->callerid)->data[var][level] = GA->data;

        nnostorage++;
      }
      else
      {
        ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
        nstorage++;
      }
    }
  }

  if (nstorage > 0 && nnostorage > 0)
  {
    CCTK_WARN (0, "Group storage violation in PUGH_EnableGArrayGroupStorage");
    retval = -1;
  }
  else
  {
    retval = nstorage > 0 ? 1 : 0;
  }

  USE_CCTK_PARAMETERS;   return (retval);
}
Exemple #11
0
 /*@@
   @routine    PUGH_DisableGroupStorage
   @author     Tom Goodale
   @date       30 Mar 1999
   @desc
               Disables storage for all variables in the group
               indicated by groupname.
   @enddesc
   @calls      CCTK_GroupIndex
               CCTK_GroupData
               CCTK_FirstVarIndexI
               PUGH_DisableGArrayGroupStorage

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH *
   @vio        in
   @endvar
   @var        groupname
   @vdesc      name of the group to enable storage for
   @vtype      const char *
   @vio        in
   @endvar

   @returntype int
   @returndesc
                1 if storage for given group was disabled
               -1 if group type is invalid
   @endreturndesc
@@*/
int PUGH_DisableGroupStorage (cGH *GH, const char *groupname)
 {
  DECLARE_CCTK_PARAMETERS
  int group;               /* group index */
  cGroup pgroup;           /* group information */
  int vtypesize, retval;
  pGA ***variables;
  int first_var, var, level;
  int unchanged;           /* count how many aren't toggled */
  char *temp;


#ifdef DEBUG_PUGH
  printf (" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
  fflush (stdout);
#endif

  group = CCTK_GroupIndex (groupname);
  CCTK_GroupData (group, &pgroup);

  /* get global index of first variable in group */
  first_var = CCTK_FirstVarIndexI (group);

  variables = (pGA ***) PUGH_pGH (GH)->variables;

  /* get the group info from its index */
  unchanged = 0;

  retval = 1;
  if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
  {
    for (var = first_var; var < first_var+pgroup.numvars; var++)
    {
      for (level = 0; level < pgroup.numtimelevels; level++)
      {
        unchanged += PUGH_DisableGArrayDataStorage (variables[var][level]);
      }
    }
  }
  else if (pgroup.grouptype == CCTK_SCALAR)
  {
    vtypesize = CCTK_VarTypeSize (pgroup.vartype);
    for (var = first_var; var < first_var+pgroup.numvars; var++)
    {
      for (level = 0; level < pgroup.numtimelevels; level++)
      {
        temp = (char *) variables[var][level];
        if (temp[vtypesize] == PUGH_STORAGE)
        {
          temp[vtypesize] = PUGH_NOSTORAGE;
        }
        else
        {
          unchanged++;
        }
      }
    }
  }
  else
  {
    CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
    retval = -1;
  }

  /* Report on memory usage */
  if (!CCTK_Equals(storage_verbose,"no") && retval >= 0)
  {
    if (unchanged == 0)
    {

      /* Memory toggled */
      if (pgroup.grouptype == CCTK_GF)
      {
        totalnumberGF  -= pgroup.numvars;
      }
      else if (pgroup.grouptype == CCTK_ARRAY)
      {
        totalnumberGA  -= pgroup.numvars;
      }
      totalstorage -= (variables[first_var][0]->extras->npoints *
                       variables[first_var][0]->varsize *
                       pgroup.numtimelevels * pgroup.numvars) /
                      (float) (1024 * 1024);
      if (CCTK_Equals(storage_verbose,"yes"))
      {
        CCTK_VInfo (CCTK_THORNSTRING, "Switched memory off for group '%s'"
                    "  [GFs: %d GAs: %d Total Size: %6.2fMB]",
                    groupname, totalnumberGF, totalnumberGA, totalstorage);
      }
    }
    else if (unchanged == pgroup.numvars)
    {
      /* Memory already off */
      if (CCTK_Equals(storage_verbose,"yes"))
      {
        CCTK_VInfo (CCTK_THORNSTRING, "Memory already off for group '%s'", groupname);
      }
    }
    else
    {
      CCTK_WARN (1, "PUGH_DisableGroupStorage: Inconsistency in group memory assignment");
    }
  }

  USE_CCTK_PARAMETERS;   return (retval);
}
Exemple #12
0
 /*@@
   @routine    PUGH_EnableGroupStorage
   @author     Tom Goodale
   @date       30 Mar 1999
   @desc
               Enables storage for all variables in the group
               indicated by groupname.
   @enddesc
   @calls      CCTK_GroupIndex
               CCTK_GroupData
               PUGH_EnableScalarGroupStorage
               PUGH_EnableGArrayGroupStorage

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH *
   @vio        in
   @endvar
   @var        groupname
   @vdesc      name of the group to enable storage for
   @vtype      const char *
   @vio        in
   @endvar

   @returntype int
   @returndesc
                return code of @seeroutine PUGH_EnableScalarGroupStorage or
                @seeroutine PUGH_EnableGArrayGroupStorage: <BR>
                1 if storage was already enabled, or <BR>
                0 if storage was successfully enabled <BR>
               -1 if group type is not one of the above <BR>
               -2 if an invalid GH pointer was given <BR>
               -3 if invalid groupname was given
   @endreturndesc
@@*/
int PUGH_EnableGroupStorage (cGH *GH, const char *groupname)
{
  DECLARE_CCTK_PARAMETERS
  int group;               /* group index */
  int first_var;           /* first variable's index */
  cGroup pgroup;           /* group information */
  int retval;
  pGA *GA;
  pGH *pughGH;


#ifdef DEBUG_PUGH
  printf (" PUGH_EnableGroupStorage: request for group '%s'\n", groupname);
  fflush (stdout);
#endif

  pughGH = PUGH_pGH (GH);
  group = CCTK_GroupIndex (groupname);

  if (pughGH && group >= 0)
  {
    first_var = CCTK_FirstVarIndexI (group);

    /* get the group info from its index */
    CCTK_GroupData (group, &pgroup);

    if (pgroup.grouptype == CCTK_SCALAR)
    {
      retval = PUGH_EnableScalarGroupStorage (pughGH,
                                              first_var,
                                              pgroup.numvars,
                                              pgroup.numtimelevels);
    }
    else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
    {
      retval = PUGH_EnableGArrayGroupStorage (pughGH,
                                              first_var,
                                              pgroup.numvars,
                                              pgroup.numtimelevels);
      if (!CCTK_Equals(storage_verbose,"no") && retval == 0)
      {
        /* get GA pointer of first var in group */
        GA = (pGA *) pughGH->variables[first_var][0];
        if (pgroup.grouptype == CCTK_GF)
        {
          totalnumberGF  += pgroup.numvars * pgroup.numtimelevels;
        }
        else
        {
          totalnumberGA  += pgroup.numvars * pgroup.numtimelevels;
        }
        totalstorage += (GA->extras->npoints * GA->varsize *
                         pgroup.numtimelevels * pgroup.numvars) /
                        (float) (1024*1024);
        if (totalstorage > maxstorage)
        {
          numberGF = totalnumberGF;
          numberGA = totalnumberGA;
          maxstorage = totalstorage;
        }

        /* Report on memory usage */
        if (CCTK_Equals(storage_verbose,"yes"))
        {
          CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'"
                      "  [GFs: %d GAs: %d Total Size: %6.2fMB]",
                      groupname, totalnumberGF, totalnumberGA, totalstorage);
        }
      }

    }
    else
    {
      CCTK_WARN (1, "PUGH_EnableGroupStorage: Unknown group type");
      retval = -1;
    }

  }
  else
  {
    if (! pughGH)
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_EnableGroupStorage: Error with cctkGH pointer "
                  "for group %s", groupname);
      retval = -2;
    }
    else
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_EnableGroupStorage: Invalid group %s", groupname);
      retval = -3;
    }
  }

  USE_CCTK_PARAMETERS;   return (retval);
}
Exemple #13
0
 /*@@
   @routine    PUGH_QueryGroupStorage
   @author     Gabrielle Allen
   @date       13 Apr 1999
   @desc
               Checks if group has storage assigned.
               Either group index or its name must be given,
               the name takes precedence.
   @enddesc
   @calls      CCTK_GroupIndex
               CCTK_FirstVarIndexI
               CCTK_GroupTypeI

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        group
   @vdesc      index of group
   @vtype      int
   @vio        in
   @endvar
   @var        groupname
   @vdesc      name of the group to be queried
   @vtype      const char *
   @vio        in
   @endvar

   @returntype int
   @returndesc
                1 if storage for this group is assigned
                0 if storage for this group is not assigned
               -1 if given group index/name is invalid
   @endreturndesc
@@*/
int PUGH_QueryGroupStorage (const cGH *GH, int group, const char *groupname)
{
  int first_var;
  int storage;
  int grouptype;
  int vtypesize;
  int retval;
  pGH *pughGH;


  retval = -1;

  if (groupname)
  {
    group = CCTK_GroupIndex (groupname);
  }

  /* get first variable in group */
  first_var = CCTK_FirstVarIndexI (group);
  if (first_var >= 0)
  {
    pughGH = PUGH_pGH (GH);
    grouptype = CCTK_GroupTypeI (group);
    if (grouptype == CCTK_SCALAR)
    {
      vtypesize = CCTK_VarTypeSize (CCTK_VarTypeI (first_var));
      storage = ((char *) pughGH->variables[first_var][0])[vtypesize];
    }
    else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
    {
      storage = ((pGA *) pughGH->variables[first_var][0])->storage;
    }
    else
    {
      storage = -1;
      CCTK_WARN (1, "Unknown group type in PUGH_QueryGroupStorage");
    }

    if (storage == PUGH_STORAGE)
    {
      retval = 1;
    }
    else if (storage == PUGH_NOSTORAGE)
    {
      retval = 0;
    }
    else
    {
      CCTK_WARN (1, "Inconsistency in PUGH_QueryGroupStorage");
    }
  }
  else
  {
    if (groupname)
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_ArrayGroupSize: Invalid group name '%s'",
                  groupname);
    }
    else
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "PUGH_ArrayGroupSize: Invalid group ID %d",
                  group);
    }
  }

  return (retval);
}
Exemple #14
0
 /*@@
   @routine    PUGH_EnableComm
   @date       Sun Jan 23 12:46:23 2000
   @author     Gabrielle Allen
   @desc
   This sets the docomm[2*dim] array of the GA based
   on the setting of the comm flag and allocates the comm buffers.
   @enddesc
   @history
               @date Mon Jun 05 2000 @author Thomas Radke
               Moved buffer allocation from PUGH_EnableGArrayDataStorage
   @endhistory
@@*/
static int PUGH_EnableComm(pGH *pughGH,
                           pComm *comm,
                           int commflag)
{
  int retval;              /* return value */
#ifdef CCTK_MPI
  pGA *GA;                 /* GA structure the pComm belongs to */
  int i, idir;             /* loopers */
  int dir;                 /* direction */
  int sz;                  /* buffer size */
#endif


  retval = 1;

#ifdef CCTK_MPI
  /* check whether this comm buffer is already set up */
  if (comm->commflag == commflag)
  {
    return (retval);
  }

  /* get the GA structure the comm structure belongs to
     For a group comm structure this GA is the first one within the group. */
  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];

  if (comm->commflag == PUGH_NOCOMM)
  {

#ifdef DEBUG_PUGH
    printf (" PUGH_EnableComm: allocating comm buffer for %d vars starting "
            "with %d '%s'\n", comm->n_vars, comm->first_var, GA->name);
    fflush (stdout);
#endif

    /* allocate memory for communication buffers: 2 faces per direction */
    for (i = 0; i < 2 * GA->extras->dim; i++)
    {
      if (GA->connectivity->neighbours[pughGH->myproc][i] >= 0)
      {
        dir = i/2;

        /* no ghostzones -> no comm buffers */
        sz = comm->n_vars * GA->extras->nghostzones[dir];
        if (sz > 0)
        {
          for (idir = 0; idir < GA->extras->dim; idir++)
          {
            if (idir != dir)
            {
              sz *= GA->extras->lnsize[idir];
            }
          }
          comm->buffer_sz[i]   = sz;
          comm->send_buffer[i] = malloc(sz * GA->varsize);
          comm->recv_buffer[i] = malloc(sz * GA->varsize);

          if (! (comm->send_buffer[i] && comm->recv_buffer[i]))
          {
            for (; i >=0 ; i--)
            {
              if (comm->send_buffer[i])
              {
                free(comm->send_buffer[i]);
              }
              if (comm->recv_buffer[i])
              {
                free(comm->recv_buffer[i]);
              }
              comm->buffer_sz[i] = 0;
            }

            CCTK_WARN(1, "Out of memory !");
            retval = -1;
            break;
          }
        }
        else
        {
          comm->buffer_sz[i]   = 0;
          comm->send_buffer[i] = NULL;
          comm->recv_buffer[i] = NULL;
        }
      }
      else               /* no neighbor -> no comm buffers */
      {
        comm->buffer_sz[i]   = 0;
        comm->send_buffer[i] = NULL;
        comm->recv_buffer[i] = NULL;
      }
    }
  }

  /* set up comm flags for each face */
  if (retval >= 0)
  {
    /* Copy commflag */
    comm->commflag = commflag;

    /* First set all communcation off */
    for (idir = 0; idir < 2 * GA->extras->dim; idir++)
      comm->docomm[idir] = 0;

    if (commflag == PUGH_ALLCOMM)
    {
      for (idir = 0; idir < 2 * GA->extras->dim; idir++)
      {
        comm->docomm[idir] = comm->buffer_sz[idir] > 0;
      }
    }
    else if (commflag == PUGH_PLUSFACESCOMM)
    {
      for (idir = 0; idir < GA->extras->dim; idir++)
      {
        comm->docomm[2*idir+1] = comm->buffer_sz[2*idir+1] > 0;
      }
    }
    else if (commflag == PUGH_MINUSFACESCOMM)
    {
      for (idir = 0; idir < GA->extras->dim; idir++)
      {
        comm->docomm[2*idir] = comm->buffer_sz[2*idir] > 0;
      }
    }
    else
    {
      for (idir = 0; idir < GA->extras->dim; idir++)
      {
        if (commflag == PUGH_COMM(idir))
        {
          comm->docomm[2*idir] = comm->buffer_sz[2*idir] > 0;
          comm->docomm[2*idir+1] = comm->buffer_sz[2*idir+1] > 0;
        }
      }
    }

    /* FIXME Add back the check that you have a valid COMM model: Gab */

    /* Handle nsize = 1 type cases. This is only important for one
       processor MPI periodic boundaries */

    for (idir = 0; idir < GA->extras->dim; idir++)
    {
      if (GA->extras->nsize[idir] == 1)
      {
        comm->docomm[2*idir] = 0;
        comm->docomm[2*idir+1] = 0;
      }
    }
  }

#else

  /* get rid of compiler warning about unused parameters */
  pughGH = pughGH;
  comm = comm;
  commflag = commflag;

#endif   /* CCTK_MPI */

  return retval;
}
Exemple #15
0
 /*@@
   @routine    PUGH_SetupScalarGroup
   @date       Wed Feb 17 04:45:49 1999
   @author     Tom Goodale
   @desc
               Set up a group of scalar variables on a pGH.
               For efficiency reasons, PUGH allocates storage for scalars
               only once when the group is created.
               The current state of storage allocation (which is toggled by
               Enable/DisableGroupStorage) is stored in a byte-sized flag
               immediately after the scalar data.
   @enddesc

   @var        newGH
   @vdesc      Pointer to PUGH grid hierarchy
   @vtype      pGH *
   @vio        in
   @endvar
   @var        vtype
   @vdesc      CCTK data type for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_variables
   @vdesc      number of variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_timelevels
   @vdesc      number of timelevels in this group
   @vtype      int
   @vio        in
   @endvar
   @var        vectorgroup
   @vdesc      is this a vector group ?
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               PUGH_SUCCESS (0) if successful, or
               PUGH_ERRORMEMORY (negative) if memory allocation failed
   @endreturndesc
@@*/
static int PUGH_SetupScalarGroup (pGH *newGH,
                                  int vtype,
                                  int n_variables,
                                  int n_timelevels,
                                  int vectorgroup)
{
  DECLARE_CCTK_PARAMETERS
  int variable, level, vtypesize, retval;
  void *temp;


  retval = 0;  /* PUGH_SUCCESS */

  temp = realloc (newGH->variables,
                  (newGH->nvariables + n_variables) * sizeof (void **));
  if (temp)
  {
    newGH->variables = (void ***) temp;
    vtypesize = CCTK_VarTypeSize (vtype);

    for (variable = 0; variable < n_variables && retval == 0; variable++)
    {
      temp = malloc (n_timelevels * sizeof (void *));
      if (temp)
      {
        newGH->variables[newGH->nvariables + variable] = (void **) temp;
        for (level = 0; level < n_timelevels; level++)
        {
          /* allocate one more byte for the query_storage flag */
          temp = malloc (vtypesize + 1);
          if (temp)
          {
            newGH->variables[newGH->nvariables + variable][level] = temp;

            /* reset the query_storage flag */
            ((char *) temp)[vtypesize] = PUGH_NOSTORAGE;
          }
          else
          {
            retval = PUGH_ERRORMEMORY;
            break;
          }
        }
      }
      else
      {
        retval = PUGH_ERRORMEMORY;
      }
    }
  }
  else
  {
    retval = PUGH_ERRORMEMORY;
  }

  if (! retval)
  {
    newGH->nvariables += n_variables;
  }
  else
  {
    CCTK_WARN (1, "Memory allocation error in PUGH_SetupScalarGroup");
  }

  USE_CCTK_PARAMETERS;   return (retval);
}
Exemple #16
0
 /*@@
   @routine    PUGH_SetupGAGroup
   @date       January 19 2000
   @author     Gabrielle Allen
   @desc
               Set up a group of grid array variables (CCTK_GF or CCTK_ARRAY
               types) on a pGH.
   @enddesc
   @calls      CCTK_VarTypeSize
               PUGH_SetupConnectivity
               PUGH_SetupPGExtras
               PUGH_SetupGArrayGroupComm
               PUGH_SetupGArray

   @var        newGH
   @vdesc      Pointer to PUGH grid hierarchy
   @vtype      pGH *
   @vio        in
   @endvar
   @var        nsize
   @vdesc      size dimensions for variables in this group
   @vtype      int *
   @vio        in
   @endvar
   @var        ghostsize
   @vdesc      ghostsize dimensions for variables in this group
   @vtype      int *
   @vio        in
   @endvar
   @var        gtype
   @vdesc      group type
   @vtype      int
   @vio        in
   @endvar
   @var        vtype
   @vdesc      CCTK data type for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        dim
   @vdesc      number of dimensions for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_variables
   @vdesc      number of variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        staggercode
   @vdesc      stagger code for variables in this group
   @vtype      int
   @vio        in
   @endvar
   @var        n_timelevels
   @vdesc      number of timelevels in this group
   @vtype      int
   @vio        in
   @endvar
   @var        vectorgroup
   @vdesc      is this a vector group ?
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               PUGH_SUCCESS (0) if successful, or
               PUGH_ERRORMEMORY (negative) if memory allocation failed
   @endreturndesc
@@*/
static int PUGH_SetupGAGroup (pGH *newGH,
                              int *nsize,
                              int *ghostsize,
                              int  gtype,
                              int  vtype,
                              int  dim,
                              int  n_variables,
                              int  staggercode,
                              int  n_timelevels,
                              int  vectorgroup)
{
  int i;
  int           variable;
  int           var_size;
  int           retval;
  int           level;
  void       ***temp;
  pConnectivity *connectivity;
  pGExtras      *extras;
  pComm         *groupcomm;
  int *perme;
  int *nprocs;


  retval = PUGH_SUCCESS;
  var_size = CCTK_VarTypeSize (vtype);

  if (gtype == CCTK_ARRAY)
  {
    /* Arrays can't (yet) have periodicity and manual setup,
       so initialize them to zero */
    perme = (int *) calloc (dim, sizeof (int));
    nprocs = (int *) calloc (dim, sizeof (int));
    if (! (perme && nprocs))
    {
      CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
    }

    /* Check that there are enough grid points in this dimension
     * to make parallelising it worthwhile
     */
    for (i = 0 ; i < dim; i++)
    {
      if (! nprocs[i] && abs (nsize[i]) <= 2*ghostsize[i] + 1)
      {
        nprocs[i] = 1;
      }
    }

    connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);

    extras = PUGH_SetupPGExtras (dim,
                                 perme,
                                 staggercode,
                                 nsize,
                                 ghostsize,
                                 newGH->nprocs,
                                 connectivity->nprocs,
                                 newGH->myproc);

    free (nprocs);
    free (perme);
  }
  else
  {
    /* this is for CCTK_GF variables */
    connectivity = newGH->Connectivity[dim-1];
    extras = newGH->GFExtras[dim-1];
  }

  /* Set up the communication buffer used for all variables within this group.
     Note: only with allocated buffers we can have group communication. */
  if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
  {
    groupcomm = PUGH_SetupGArrayGroupComm (newGH,
                                           dim,
                                           newGH->nvariables,
                                           n_variables,
                                           0,
                                           vtype,
                                           extras);
  }
  else
  {
    groupcomm = NULL;
  }

  temp = (void ***) realloc (newGH->variables, (newGH->nvariables+n_variables) *
                             sizeof (void **));

  if(temp)
  {
    newGH->variables = temp;

    for (variable = 0; variable < n_variables; variable++)
    {
      newGH->variables[newGH->nvariables] = (void **) malloc (n_timelevels *
                                                              sizeof (void *));

      if (newGH->variables[newGH->nvariables])
      {
        for (level = 0; level < n_timelevels; level++)
        {
          newGH->variables[newGH->nvariables][level] =
            PUGH_SetupGArray (newGH,
                              extras,
                              connectivity,
                              groupcomm,
                              CCTK_VarName (newGH->nvariables),
                              newGH->nvariables,
                              newGH->narrays,
                              var_size,
                              vtype,
                              staggercode,
                              vectorgroup ? n_variables : 1,
                              variable,
                              variable > 0 ? newGH->variables[newGH->nvariables-variable][level] : NULL);
          newGH->narrays++;
        }
        newGH->nvariables++;
      }
      else
      {
        retval = PUGH_ERRORMEMORY;
        break;
      }
    }
  }
  else
  {
    retval = PUGH_ERRORMEMORY;
  }

  if (retval)
  {
    CCTK_WARN (1, "Memory allocation error in PUGH_SetupGAGroup");
  }

  return (retval);
}