Exemplo n.º 1
0
int pp_initialise_records(PPrec **recs, int nrec, PPlist *heaplist) {
  int rec;
  PPrec *recp;
  PPhdr *hdrp;

  for (rec=0; rec < nrec ; rec++) {

    recp = recs[rec];
    hdrp = &recp->hdr;

    recp->disambig_index = -1;
    recp->supervar_index = -1;

    /* store level info */
    CKP(  recp->lev = pp_malloc(sizeof(PPlevel), heaplist)  );
    CKI(  pp_lev_set(recp->lev, hdrp)  );

    /* store time info */
    CKP(  recp->time = pp_malloc(sizeof(PPtime), heaplist)  );
    CKI(  pp_time_set(recp->time, hdrp)  );
    recp->mean_period = pp_mean_period(recp->time);
  }
  return 0;

  ERRBLKI("pp_initialise_records");
}
Exemplo n.º 2
0
PProtmap *pp_get_rotmap(  const PPhdr *hdr, PPlist *rotmaplist, PPlist *heaplist )
{

  PProtmap *rotmap;

  if (pp_is_rotated_grid(hdr)) {
    
    CKP(  rotmap = pp_malloc( sizeof(PProtmap), heaplist)  );

    rotmap->pole_lon = hdr->BPLON;
    rotmap->pole_lat = hdr->BPLAT;
    rotmap->truepole_gridlon = 0.;
    rotmap->map_var = NULL;  /* we'll set this properly later */

    CKI(  pp_list_add_or_find( rotmaplist, &rotmap, pp_compare_rotmaps, 0, 
			       pp_free, NULL, heaplist )  );
    
    return rotmap;

  } else {
    return NON_ROTATED;
  }

  ERRBLKP("pp_get_rotmap");
}
Exemplo n.º 3
0
CuFile *pp_create_file(const char *controlpath){

  CuFile *file;
  PPfile *ppfile;
  
  /* get file structure */
  file = CuCreateFile(CuPP);
  if (file == (CuFile*)0) goto err1;

  /* allocate internal structure - hang it off cu_file structure */
  ppfile=pp_malloc(sizeof(PPfile), NULL);
  if (ppfile==NULL) goto err2;
  
  file->internp = ppfile;
  ppfile->fh = NULL;
  strncpy(file->controlpath, controlpath, CU_MAX_PATH);
  ppfile->landmask = NULL;
  ppfile->store_raw_headers = 0;
  ppfile->basetime = 0;

  /* initialise heap list */
  ppfile->heaplist = pp_list_new(NULL);
  if (ppfile->heaplist == NULL) goto err3;

  return file;

 err3:
  pp_free(ppfile,NULL);
 err2:
  CuDeleteFile(file->id);
 err1:
  pp_error("pp_create_file");
  return NULL;
}
Exemplo n.º 4
0
/*
 * es_ht_make_table() - Make a new table of the indicated size.
 * return : HASH_TAB *
 * maxsym(in): The number of hash slots in the table.
 * hash_function(in): The hash function used to map keys to slots.
 * cmp_function(in): The comparison function used to compare two entries.
 */
HASH_TAB *
es_ht_make_table (unsigned maxsym, HT_HASH_FN hash_function, HT_CMP_FN cmp_function)
{
  HASH_TAB_IMPL *p;
  int n;

  if (maxsym == 0)
    {
      maxsym = 127;
    }

  n = sizeof (HASH_TAB_IMPL) + (maxsym * sizeof (BUCKET *));
  p = (HASH_TAB_IMPL *) pp_malloc (n);
  memset ((char *) p, 0, n);

  p->ifs.free_table = es_ht_free_table;
  p->ifs.add_symbol = es_ht_add_symbol;
  p->ifs.remove_symbol = es_ht_remove_symbol;
  p->ifs.find_symbol = es_ht_find_symbol;
  p->ifs.next_symbol = es_ht_next_symbol;
  p->ifs.print_table = es_ht_print_table;
  p->ifs.get_symbol_count = es_ht_get_symbol_count;

  p->size = maxsym;
  p->numsyms = 0;
  p->hash = hash_function;
  p->cmp = cmp_function;

  return (HASH_TAB *) p;
}
Exemplo n.º 5
0
static void
grow_ptr_vec (PTR_VEC * vec)
{
  int new_max_elems;
  void **new_elems;

  new_max_elems = vec->max_elems + vec->chunk_size;
  if (vec->elems == vec->inline_elems)
    {
      new_elems =
	(void **) pp_malloc (sizeof (vec->elems[0]) * new_max_elems);
      memcpy ((char *) new_elems, (char *) vec->elems,
	      sizeof (vec->elems[0]) * vec->n_elems);
    }
  else
    {
      new_elems = (void **)
	realloc (vec->elems, sizeof (vec->elems[0]) * new_max_elems);
    }

  if (new_elems == NULL)
    {
      esql_yyverror (pp_get_msg (EX_MISC_SET, MSG_OUT_OF_MEMORY));
      exit (1);
    }

  vec->elems = new_elems;
  vec->max_elems = new_max_elems;
}
Exemplo n.º 6
0
PPdata *pp_data_new(CuType type, int n, PPlist *heaplist)
{
    int size;
    PPdata *data;

    size = cutypelen(type);

    CKP(   data=pp_malloc(sizeof(PPdata),heaplist)   );
    CKP(   data->values=pp_malloc(n*size,heaplist)   );

    data->type = type;
    data->n = n;

    return data;

    ERRBLKP("pp_data_new");
}
Exemplo n.º 7
0
/*
 * es_ht_alloc_new_symbol() - Allocate space for a new symbol.
 * return : pointer to the user space
 * size(in): size of symbol
 */
void *
es_ht_alloc_new_symbol (int size)
{
  BUCKET *sym = (BUCKET *) pp_malloc (size + sizeof (BUCKET));
  if (sym == NULL)
    {
      return NULL;
    }

  memset (sym, 0, size + sizeof (BUCKET));
  return (void *) (sym + 1);
}
Exemplo n.º 8
0
void *pp_read_header(const PPfile *ppfile, PPlist *heaplist)
{
  void *hdr;
  /* reads a PP header -- file must be positioned at start of header,
   *  (after any fortran record length integer) */
  
  CKP(   hdr=pp_malloc(n_hdr * wordsize,heaplist)   );
  ERRIF(   pp_read_words(hdr,n_int_hdr,convert_int,ppfile)   !=n_int_hdr);

  ERRIF(   pp_read_words((char *)hdr + n_int_hdr * wordsize,
		      n_real_hdr,convert_real,ppfile)   !=n_real_hdr);
  
  return hdr; /* success */

  ERRBLKP("pp_read_header");    
}
Exemplo n.º 9
0
/*
 * pp_strdup() - Safe strdup. prints a message and exits if no memory.
 * return : char *
 * str: string to be copied
 */
char *
pp_strdup (const char *str)
{
  long n;
  char *tmp;

  if (str == NULL)
    {
      return NULL;
    }
  /* includes the null terminator of 'str' */
  n = strlen (str) + 1;
  tmp = pp_malloc (n);
  memcpy (tmp, str, n);
  return tmp;
}
Exemplo n.º 10
0
int pp_get_new_fvar(int recno, PPrec *recp, PPfile *ppfile, PPlist *rotmaps, 
		    PPfieldvar **fvarp, 
		    PPgenaxis **xaxisp, PPgenaxis **yaxisp, PPgenaxis **zaxisp, PPgenaxis **taxisp, 
		    int *landmask_used_p, 
		    PPlist *heaplist)
{
  int landmask_used = 0;
  PPhdr *hdrp;
  PPfieldvar *fvar;
  PPgenaxis *xaxis, *yaxis, *zaxis, *taxis; /* JAK 2005-01-05 */

  hdrp = &recp->hdr;

  /* get PPvar structure, and initialise certain structure members for tidiness */
  CKP(   fvar = pp_malloc(sizeof(PPfieldvar), heaplist)   );
  CKP(   fvar->axes = pp_list_new(heaplist)  );  /* JAK 2005-01-05 */
  fvar->firstrecno = recno;
  fvar->firstrec = recp;

  if (pp_get_var_compression(hdrp) == 2) {
    /* land/sea mask compression: for horiz axes use those of LSM */
    xaxis = ppfile->landmask->xaxis;
    yaxis = ppfile->landmask->yaxis;
    landmask_used = 1;
  } else {
    CKI(  pp_set_horizontal_axes(recp, ppfile, &xaxis, &yaxis, rotmaps, heaplist)  );
  }

  CKP(   zaxis=pp_genaxis_new(zaxis_type, zdir, heaplist)   );
  CKI(  pp_zaxis_set(zaxis, hdrp)  );
  
  CKP(   taxis=pp_genaxis_new(taxis_type, tdir, heaplist)   );
  CKI(  pp_taxis_set(taxis, hdrp, ppfile)  );
  
  /* return to caller */
  *fvarp = fvar;
  *xaxisp = xaxis;
  *yaxisp = yaxis;
  *zaxisp = zaxis;
  *taxisp = taxis;
  *landmask_used_p = landmask_used;

  return 0;
  ERRBLKI("pp_get_new_fvar");
}
Exemplo n.º 11
0
int pp_store_raw_header(PPhdr *hdrp, const void *hdr, PPlist *heaplist)
{
  const Fint *ihdr;
  const Freal *rhdr;
  PPrawhdr *rawhdr;


  ihdr = (Fint*) hdr;
  rhdr = (Freal*) (ihdr + n_int_hdr);

  CKP(  rawhdr = pp_malloc(sizeof(PPrawhdr), heaplist)  );
  memcpy(rawhdr->ihdr, ihdr, bytes_int_hdr);
  memcpy(rawhdr->rhdr, rhdr, bytes_real_hdr);
  hdrp->rawhdr = rawhdr;

  return 0;
  ERRBLKI("pp_store_raw_header");
}
Exemplo n.º 12
0
char *pp_t_units(const PPtaxis *taxis, PPlist *heaplist)
{
    char *units;

    const int string_length = 30;
    const char *fmt = "days since %04d-%02d-%02d %02d:%02d:%02d";

    const PPdate *orig;

    orig = &taxis->time_orig;

    CKP(   units=pp_malloc(string_length+1,heaplist)    );

    snprintf(units,string_length+1,fmt,
             orig->year, orig->month, orig->day,
             orig->hour, orig->minute, orig->second);

    return units;

    ERRBLKP("pp_t_units");
}
Exemplo n.º 13
0
int pp_set_landmask(int nrec, PPrec **recs, PPfile *ppfile, PPlist *rotmaps, PPlist *heaplist)
{
  /* 
   *  find the land mask variable (if present) and if so, then store it in ppfile->landmask
   *  (also adding any associated rotated grid mappings to rotmaps)
   */
  int rec;
  PPrec *recp;
  PPhdr *hdrp;
  PPgenaxis *xaxis, *yaxis;
  PPlandmask *landmask;

  for (rec = 0; rec < nrec ; rec++) {

    recp = recs[rec];
    hdrp = &recp->hdr;    

    if (pp_var_is_land_mask(hdrp)) {
      
      CKP(   landmask = pp_malloc(sizeof(PPlandmask), heaplist)   );

      CKI(  pp_set_horizontal_axes(recp, ppfile, &xaxis, &yaxis, rotmaps, heaplist)  );

      CKP(   landmask->data = pp_data_new(inttype, pp_genaxis_len(xaxis) * pp_genaxis_len(yaxis), heaplist)   ); /* JAK 2005-01-05 */

      /* read in land mask data values */

      landmask->xaxis = xaxis;
      landmask->yaxis = yaxis;
      CKP(   landmask->data->values = pp_read_data_record(recp, ppfile, heaplist)   );

      ppfile->landmask = landmask;

    }	
  }
  return 0;
  ERRBLKI("pp_set_landmask");
}
Exemplo n.º 14
0
/* 
 * Allocate CuVar array and pre-fill certain elements
 */
CuVar *pp_init_cu_vars(CuFile *file, int nvars, PPlist *heaplist)
{
  CuVar *cuvars, *var;  
  PPvar *ppvar;
  int varid;

  CKP(   cuvars = CuCreateVars(file, nvars)   );

  for (varid = 0; varid < nvars; varid++) {
    var = &cuvars[varid];
    var->datatype = realtype;
    CKP(   var->internp = pp_malloc(sizeof(PPvar), heaplist)   );
    ppvar = (PPvar*)var->internp;
    ppvar->firstrecno = -1;
    ppvar->lastrecno = -1;
    ppvar->data = NULL;
    CKP(   ppvar->atts = pp_list_new(heaplist)   );
  }

  return cuvars;

  ERRBLKP("pp_init_cu_vars");
}
Exemplo n.º 15
0
PProtgrid *pp_get_rotgrid(  PPgenaxis *xaxis, PPgenaxis *yaxis, PPlist *rotgridlist, PPlist *heaplist )
{
  PProtmap *rotmap, *rotmap_y;
  PProtgrid *rotgrid;

  rotmap   = pp_genaxis_rotmap(xaxis);
  rotmap_y = pp_genaxis_rotmap(yaxis);
  ERRIF ( rotmap_y != rotmap );

  if (rotmap == NON_ROTATED)
    return NON_ROTATED_GRID;

  CKP(  rotgrid = pp_malloc(  sizeof(PProtgrid), heaplist)  );
  rotgrid->rotmap = rotmap;
  rotgrid->xaxis = xaxis;
  rotgrid->yaxis = yaxis;

  CKI(  pp_list_add_or_find( rotgridlist, &rotgrid, pp_compare_rotgrids, 0, 
			     pp_free, NULL, heaplist )  );  

  return rotgrid;

  ERRBLKP("pp_get_rotgrid");
}
Exemplo n.º 16
0
/*
 * es_ht_print_table() - This function prints the table with given
 *    print function.
 * return : 0 if a sorted table can't be printed because of insufficient
 *    memory, else return 1 if the table was printed.
 * table(in): The hash table to be printed.
 * print(in): The function used to print a symbol.
 * param(in): Parameter passed to the print function.
 * sort(in): TRUE if the table should be sorted.
 *
 * note : The print function is called with two arguments:
 *       (*print)(sym, param)
 *       'sym' is a pointer to a BUCKET user area, and 'param' is the
 *       third argument to ptab().
 */
static int
es_ht_print_table (HASH_TAB * table, void (*print) (), void *param, int sort)
{
  BUCKET **outtab, **outp, *sym, **symtab;
  int i;
  HASH_TAB_IMPL *tbl = (HASH_TAB_IMPL *) table;

  if (tbl == NULL || tbl->size == 0)	/* Table is empty */
    {
      return 1;
    }

  if (!sort)
    {
      for (symtab = tbl->table, i = tbl->size; --i >= 0; ++symtab)
	{
	  /* 
	   * Print all symbols in the current chain.  The +1 in the
	   * print call adjusts the bucket pointer to point to the user
	   * area of the bucket.
	   */
	  for (sym = *symtab; sym; sym = sym->next)
	    {
	      (*print) (sym + 1, param);
	    }
	}
    }
  else
    {
      /* 
       * Allocate enough memory for 'outtab', an array of pointers to
       * BUCKETs, and initialize it.  'outtab' is different from the
       * actual hash table in that every 'outtab' element points to a
       * single BUCKET structure, rather than to a linked list of them.
       *
       * Go ahead and just use malloc() here; it's not a terrible
       * problem if we can't get enough memory.
       */
      outtab = (BUCKET **) pp_malloc (tbl->numsyms * sizeof (BUCKET *));
      outp = outtab;

      for (symtab = tbl->table, i = tbl->size; --i >= 0; ++symtab)
	{
	  for (sym = *symtab; sym; sym = sym->next)
	    {
	      if (outp > outtab + tbl->numsyms)
		{
		  es_write_log ("es_ht_print_table", pp_get_msg (EX_HASH_SET, MSG_TABLE_OVERFLOW));
		  free_and_init (outtab);
		  return 0;
		}
	      *outp++ = sym;
	    }
	}

      /* 
       * Sort 'outtab' and then print it.
       */
      es_User_cmp = tbl->cmp;
      qsort (outtab, tbl->numsyms, sizeof (BUCKET *), es_symcmp);

      for (outp = outtab, i = tbl->numsyms; --i >= 0; ++outp)
	{
	  (*print) ((*outp) + 1, param);
	}

      free_and_init (outtab);
    }

  return 1;
}
Exemplo n.º 17
0
int pp_data_read(const CuFile *file, const CuVar *var, const long start[], const long count[], void *values)
{
  const PPfile *ppfile;
  PPlist *heaplist;
  const PPvar *ppvar;
  int startrec, endrec, nrec, recno;
  int cx,cy,sx,sy,iy;
  const void *src;
  void *data, *ptr, *dest;
  int nx,ny, ndim;
  int idim;
  int *indices, *size;
  int carryout;
  int destoffset;

  ndim=var->ndims;

  ERRIF(ndim < 2);
    
  ppfile = file->internp;
  ppvar = var->internp;
  heaplist = ppfile->heaplist;

  startrec = ppvar->firstrecno;
  endrec = ppvar->lastrecno;
  nrec = endrec - startrec + 1;

  CKP(   size=pp_malloc(ndim*sizeof(int),heaplist)   );
  /* last two items in indices aren't actually used but define for completeness */
  CKP(   indices=pp_malloc(ndim*sizeof(int),heaplist)   );

  for (idim=0; idim<ndim; idim++) {
    size[idim] = file->dims[var->dims[idim]].len;
    indices[idim]=0;

    if (start[idim]<0 || count[idim]<0 || start[idim]+count[idim]>size[idim])
      return CU_EINVALCOORDS;    

  }

  /* some constants for use later */
  nx=size[ndim-1];
  ny=size[ndim-2];
  cx=count[ndim-1];
  cy=count[ndim-2];
  sx=start[ndim-1];
  sy=start[ndim-2];
  
  /*
   * JAK this needs moving to pp_process
   *  if (nrec != nz * nt) {
   * CuError(CU_EINVALCOORDS,"refusing to read variable which has missing combinations of z,t");
   * return CU_EINVALCOORDS;
   *}
  */ 
  /* we can now assume that the records loop over correct times and levels
   * (loop over time is the more slowly varying dimension because that's
   * how we sorted them)
   */

  carryout=0;
  while (!carryout) {

    recno=0;
    destoffset=0; 

    for (idim=0; idim < ndim-2 ; idim++) { /* treat inner 2 dim as record dims */
      recno *= size[idim];
      destoffset *= count[idim];

      recno += indices[idim]+start[idim];
      destoffset += indices[idim];
      
    }

    recno=startrec+recno;
    ptr=(char*)values+destoffset*wordsize*cx*cy;

    CKP(   data=pp_read_data_record(ppfile->recs[recno],ppfile,heaplist)   );
    
    for (iy=0; iy<cy; iy++) {
      src = (char*) data + ((sy+iy)*nx + sx) * wordsize;
      dest = (char*) ptr + (iy*cx) * wordsize;	
      memcpy(dest,src,cx*wordsize);
    }

    CKI(  pp_free(data,heaplist)  );
    /* increment indices */
    for (idim=ndim-2-1 ; idim>=0; idim--) {
      indices[idim]++;
      if (indices[idim]==count[idim])
	indices[idim]=0; /* carry, so keep looping */
      else
	break; /* no internal carry */
    }
    if (idim<0)
      carryout=1;
  }
  
  return CU_SUCCESS;

  ERRBLK("pp_data_read",CU_SERROR);
}
Exemplo n.º 18
0
int pp_calc_rot_grid(PProtgrid *rotgrid, PPdata **lons_return, PPdata **lats_return, PPlist *heaplist)
{

  int nx, ny, i, j;
  int offset, offset1;
  PPgenaxis *xaxis;
  PPgenaxis *yaxis;

  PPdata *londata, *latdata, *rlondata, *rlatdata;

  Freal *lons, *lats, *rlons, *rlats; /* "r" stands for rotated */

  double latpole_rad, coslatpole, sinlatpole, cosrlat, sinrlat;
  double *cosdrlon, *sindrlon;
  double rlonN, lonpole, drlon_rad, dlon_rad, rlat_rad, lon;
  double cycdx, sinlat;
  

  const double dtor = M_PI / 180.;

  CKP(rotgrid);

  xaxis = rotgrid->xaxis;
  yaxis = rotgrid->yaxis;

  nx = pp_genaxis_len(xaxis);
  ny = pp_genaxis_len(yaxis);

  /* get input, output and workspace arrays */

  CKP(   rlondata = pp_genaxis_to_values(xaxis,0,heaplist)  );
  ERRIF(rlondata->type != realtype);

  CKP(   rlatdata = pp_genaxis_to_values(yaxis,0,heaplist)  );
  ERRIF(rlatdata->type != realtype);

  CKP(   londata = pp_data_new(realtype,nx*ny,heaplist)   );
  CKP(   latdata = pp_data_new(realtype,nx*ny,heaplist)   );
  CKP(   cosdrlon = pp_malloc(nx*sizeof(double),heaplist)  );
  CKP(   sindrlon = pp_malloc(nx*sizeof(double),heaplist)  );
  
  /* some pointers for convenience (and speed?) */
  rlons = (Freal*) rlondata->values;
  rlats = (Freal*) rlatdata->values;
  lons = londata->values; 
  lats = latdata->values;

  latpole_rad = rotgrid->rotmap->pole_lat * dtor;
  coslatpole = cos(latpole_rad);
  sinlatpole = sin(latpole_rad);
  
  rlonN = rotgrid->rotmap->truepole_gridlon;
  lonpole = rotgrid->rotmap->pole_lon;

  for (i=0; i<nx; i++) {

    drlon_rad = (rlons[i] - rlonN) * dtor;
    cosdrlon[i] = cos(drlon_rad);
    sindrlon[i] = sin(drlon_rad);
  }

  for (j=0; j<ny; j++) {

    rlat_rad = rlats[j] * dtor;
    cosrlat = cos(rlat_rad);
    sinrlat = sin(rlat_rad);

    offset1 = j*nx;

    for (i=0; i<nx; i++) {

      offset = offset1 + i;
      
      cycdx = cosrlat * cosdrlon[i];

      dlon_rad = atan2( -cosrlat*sindrlon[i], sinrlat*coslatpole - cycdx*sinlatpole );

      lon = (dlon_rad/dtor + lonpole);

      /* put in range 0 <= lon < 360
       * NOTE: This code previously put in range -180 to 180.
       *       The actual code was the following:
       *           lon -= lon_modulo * floor(lon / lon_modulo + 0.5);
       *       This was changed because the subsetting functions in CDAT
       *       didn't like the negative longitudes.
       */
      lon -= lon_modulo * floor(lon / lon_modulo);

      sinlat = cycdx * coslatpole + sinrlat * sinlatpole;
      if (sinlat > 1.)
	sinlat = 1.;
      else if (sinlat < -1.)
	sinlat = -1.;

      lons[offset] = lon;
      lats[offset] = asin(sinlat) / dtor;

    }
  }
  

  /* free workspace arrays */
  CKI(   pp_free(rlondata, heaplist)  );
  CKI(   pp_free(rlatdata, heaplist)  );
  CKI(   pp_free(cosdrlon, heaplist)  );
  CKI(   pp_free(sindrlon, heaplist)  );

  /* return pointers */
  if (lons_return != NULL)
    *lons_return = londata;

  if (lats_return != NULL)
    *lats_return = latdata;

  return 0;
  ERRBLKI("pp_calc_rot_grid");
}
Exemplo n.º 19
0
int pp_data_copy(const CuFile *file, const CuVar *var, const long start[], const long count[], void *values) {

  const PPfile *ppfile;
  PPlist *heaplist;
  const PPvar *ppvar;
  const PPdata *data;
  const void *src;
  int ndim,idim;
  int *indices, *size;
  int srcoffset, destoffset;
  int carryout;
  int is_int, is_real;

  ppfile = file->internp;
  ppvar = var->internp;
  heaplist = ppfile->heaplist;
  data = ppvar->data;

  CKP(   src=data->values    );

  ndim = var->ndims;

  if (ndim > 0) {
    CKP(   size=pp_malloc(ndim*sizeof(int),heaplist)   );
    CKP(   indices=pp_malloc(ndim*sizeof(int),heaplist)   );
  } else {
    size = indices = NULL;
    /* won't use these NULL values - only to suppress compiler warnings about uninitialised */
  }

  for (idim=0; idim<ndim; idim++) {
    size[idim] = file->dims[var->dims[idim]].len;
    indices[idim]=0;

    if (start[idim]<0 || count[idim]<0 || start[idim]+count[idim]>size[idim])
      return CU_EINVALCOORDS;    

  }
  carryout=0;

  is_int = (data->type == inttype);
  is_real = (data->type == realtype);
  ERRIF(!is_int && !is_real);

  while (!carryout) {
    /* indices loop from 0 to count-1 in each dimension: keep looping until
     * carry-out from most slowly varying dimension
     */

    /* locate hyperslab element within source and destination 1d arrays */
    srcoffset=0;
    destoffset=0;
    for (idim=0; idim < ndim ; idim++) {
      srcoffset *= size[idim];      
      destoffset *= count[idim];

      srcoffset += indices[idim]+start[idim];
      destoffset += indices[idim];
    }

    /* copy data */
    if (is_int)
      *((Fint *)values + destoffset) = *((Fint *)src + srcoffset);
    else if (is_real)
      *((Freal *)values + destoffset) = *((Freal *)src + srcoffset);    

    /* increment indices */
    for (idim=ndim-1 ; idim>=0; idim--) {
      indices[idim]++;
      if (indices[idim]==count[idim])
	indices[idim]=0; /* carry, so keep looping */
      else
	break; /* no internal carry */
    }
    if (idim<0)
      carryout=1;
  }
  
  if (ndim > 0) {
    CKI(  pp_free(size,heaplist)  );
    CKI(  pp_free(indices,heaplist)  );
  }

  return CU_SUCCESS;    

  ERRBLK("pp_data_copy",CU_SERROR);
}
Exemplo n.º 20
0
int pp_read_all_headers(CuFile *file)
{
  FILE *fh;
  int rec, nrec, recsize, filerec, nlrec;
  void *hdr;
  PPfile *ppfile;
  PPrec **recs,*recp;
  PPlist *heaplist;

  Fint start_lookup, nlookup1, nlookup2, lbbegin, dataset_type, start_data;
  size_t hdr_start, hdr_size, lbbegin_offset, datapos;

  int *valid;
  PPhdr *hdrp;  
  int fieldsfile;


  ppfile=file->internp;
  fh=ppfile->fh;
  heaplist=ppfile->heaplist;

  switch(ppfile->type) {
  case pp_type:
    
    fseek(fh,0,SEEK_SET);
    /* count the PP records in the file */
    for (nrec=0; (recsize=pp_skip_fortran_record(ppfile)) != -1; nrec++) {
      ERRIF(recsize==-2);
      if (recsize != n_hdr * ppfile->wordsize) {
	CuError(CU_EOPEN,"Opening PP file %s: unsupported header length: %d words",
		file->controlpath, recsize / ppfile->wordsize);
	ERR;
      }
      ERRIF(   pp_skip_fortran_record(ppfile)   <0); /* skip the data record */
    }
    
    /* now rewind, and read in all the PP header data */
    fseek(fh,0,SEEK_SET);
    
    ppfile->nrec=nrec;
    CKP(   recs=pp_malloc(nrec*sizeof(PPrec*),heaplist)   );
    ppfile->recs = recs;
    
    for (rec=0; rec<nrec; rec++){
      
      /* fill in the record information - for each record, read the header
       * record into heap memory, copy out of it the elements we want to store
       * then free the heap memory.
       */
      
      /* just skip the fortran integers - we've already tested header length */
      CKI(   pp_skip_word(ppfile)   );
      CKP(   hdr=pp_read_header(ppfile,heaplist)   );
      CKI(   pp_skip_word(ppfile)   );
      
      CKP(   recp=pp_malloc(sizeof(PPrec),heaplist)   );
      recs[rec]=recp;
      hdrp=&recp->hdr;
      
      pp_store_header(hdrp, hdr);

      if (ppfile->store_raw_headers) {
	CKI(  pp_store_raw_header(hdrp, hdr, heaplist)  );
      }
      else {
	hdrp->rawhdr = NULL;
      }

      recp->recno = rec;
      
      /* skip data record but store length */
      recp->datapos = ftell(fh) + ppfile->wordsize;
      recp->disklen = pp_skip_fortran_record(ppfile) / ppfile->wordsize;
      
      /* work out datalen */
      pp_evaluate_lengths(hdrp, ppfile, &recp->datalen, NULL);      
      
      CKI(  pp_free(hdr,heaplist)  );
    }
    break;
  case um_type:

    /* pick out certain information from the fixed length header */
    
    CKI(   fseek(fh,4*ppfile->wordsize,SEEK_SET)  );
    ERRIF(   pp_read_words(&dataset_type, 1,  convert_int, ppfile)   !=1);

    CKI(   fseek(fh,149*ppfile->wordsize,SEEK_SET)  );
    ERRIF(   pp_read_words(&start_lookup, 1,  convert_int, ppfile)   !=1);
    ERRIF(   pp_read_words(&nlookup1, 1,  convert_int, ppfile)   !=1);
    ERRIF(   pp_read_words(&nlookup2, 1,  convert_int, ppfile)   !=1);
    
    CKI(   fseek(fh,159*ppfile->wordsize,SEEK_SET)  );
    ERRIF(   pp_read_words(&start_data, 1,  convert_int, ppfile)   !=1);

    /* fieldsfiles includes ancillary files and initial dumps */
    fieldsfile = (dataset_type == 1 || dataset_type == 3 || dataset_type == 4);

    /* (first dim of lookup documented as being 64 or 128, so 
     * allow header longer than n_hdr (64) -- discarding excess -- but not shorter)
     */
    if (nlookup1 < n_hdr) {
      CuError(CU_EOPEN,"Opening UM file %s: unsupported header length: %d words",
		file->controlpath, nlookup1);
      ERR;
    }

    /* count the valid records in the file */
    /* loop over all records and pick out the valid ones - test for LBBEGIN != -99 */
    nrec=0;
    hdr_start = (start_lookup - 1) * ppfile->wordsize;
    hdr_size = nlookup1 * ppfile->wordsize;
    lbbegin_offset = 28 * ppfile->wordsize;

    CKP(   valid = pp_malloc(nlookup2 * sizeof(int),heaplist)   );

    for (filerec=0; filerec<nlookup2; filerec++) {
      valid[filerec]=0;
      CKI(   fseek(fh, hdr_start + filerec * hdr_size + lbbegin_offset, SEEK_SET)   );
      ERRIF(   pp_read_words(&lbbegin, 1,  convert_int, ppfile)   !=1);

      if (lbbegin != -99) {
	/* valid record */
	valid[filerec]=1;
	nrec++;
      } else {
#ifdef BREAKATFIRSTINVALID
        break;
#endif
	valid[filerec]=0;
      }
    }

    /* now read in all the PP header data */
    ppfile->nrec=nrec;
    CKP(   recs=pp_malloc(nrec*sizeof(PPrec*),heaplist)   );
    ppfile->recs = recs;


    rec=0;  /* valid record number, as opposed to  
	     * filerec which is total record number */

    datapos = (start_data-1) * ppfile->wordsize;
#ifdef BREAKATFIRSTINVALID
      nlrec=nrec;
#else
      nlrec=nlookup2;
#endif
    for (filerec=0; filerec<nlrec ; filerec++) {
      if (valid[filerec]) {

	/* seek to correct position, read in header into tmp dynamic array,
	 * store wanted elements in record structure, free tmp array
	 */
	CKI(   fseek(fh, hdr_start + filerec*hdr_size, SEEK_SET)   );	
	CKP(   hdr=pp_read_header(ppfile,heaplist)   );
	
	CKP(   recp=pp_malloc(sizeof(PPrec),heaplist)   );
	recs[rec]=recp;
	hdrp=&recp->hdr;
      	pp_store_header(hdrp,hdr);
	/* Set this for UM fieldsfile, else testing an uninitialised variable 
           in pp_var_get_extra_atts later. */
	hdrp->rawhdr = NULL;
	CKI(  pp_free(hdr,heaplist)  );

	/* work out datalen and disklen */
	pp_evaluate_lengths(hdrp, ppfile, &recp->datalen, &recp->disklen);
	/* use LBBEGIN if it is set - this will not work if LBBEGIN refers to 
	 * start record rather than start address
	 */
	if (hdrp->LBBEGIN != 0) {
	  /* Extra cast to uint handles files with LBBEGIN up to 2^32 */
	  recp->datapos=(size_t)((uint)hdrp->LBBEGIN)*ppfile->wordsize; 
	} else {
	  recp->datapos = datapos;
	}
	/* If LBNREC and LBBEGIN are both non-zero and it's not a FIELDSfile,
	 *   the file has well-formed records.  In that case, 
	 *   LBBEGIN should be correct, so do an assertion
	 */
	if (!fieldsfile && hdrp->LBNREC != 0 && hdrp->LBBEGIN != 0) {
	  if (recp->datapos != hdrp->LBBEGIN * ppfile->wordsize) {
	    
	    CuError(CU_EOPEN,"start of data record mismatch: %d %d",
		    recp->datapos, hdrp->LBBEGIN * ppfile->wordsize);
	    ERR;
	  }
	}

	datapos += recp->disklen * ppfile->wordsize;
	rec++;
      }
    }
    

    CKI(  pp_free(valid,heaplist)  );

    break;
  default:
    pp_switch_bug("pp_read_all_headers");
    ERR;
  }
  
  return 0;
  
  ERRBLKI("pp_read_all_headers");
}
Exemplo n.º 21
0
void *pp_read_data_record(const PPrec *rec, const PPfile *ppfile, PPlist *heaplist)
{
  size_t bytes, packed_bytes, nread;
  FILE *fh;
  PPconvert conv;
  void *data, *data_expanded, *packed_data;
  CuType vartype;
  const PPhdr *hdrp;
  int pack;
  int nint;
  int ipt, npoint, npoint_used;
  Fint valid_landmask_value, *landmask_vals;
  Freal mdi;

  const void *fill_ptr;
  const char *srcptr;  /* points to compressed data; this could be of type Fint or Freal;
			* use char* instead of void* to allow ptr arithmetic
			*/
  char *destptr; /* uncompressed data */

  CKP(ppfile);
  CKP(rec);
  CKP(heaplist);
  fh = ppfile->fh;

  fseek(fh, rec->datapos, SEEK_SET);
  hdrp=&rec->hdr;

  pack = pp_get_var_packing(hdrp);
  vartype=pp_get_var_type(hdrp);

  if (pack==0) {
   /* unpacked data -- read and convert according to type */
    
    bytes = rec->datalen * wordsize;
    CKP(   data = pp_malloc(bytes,heaplist)   );
    
    if (vartype==inttype)
      conv = convert_int;
    else if (vartype==realtype)
      conv = convert_real;
    else {
      conv=convert_none;
      pp_switch_bug("pp_read_data_record"); ERR;
    }
    nread = pp_read_words(data, rec->datalen, conv, ppfile);
    ERRIF(nread != rec->datalen);
  }
  else {

    /* PACKING IN USE */

    /* first allocate array and read in packed data */

    packed_bytes = rec->disklen * ppfile->wordsize;
    CKP(   packed_data = pp_malloc(packed_bytes,heaplist)   );

    nread = pp_read_words(packed_data, rec->disklen, convert_none, ppfile);
    ERRIF(nread != rec->disklen);

    /* and allocate array for unpacked data*/
    bytes = rec->datalen * wordsize;
    CKP(   data = pp_malloc(bytes,heaplist)   );

    /* NOW UNPACK ACCORDING TO PACKING TYPE: */

    switch(pack) {

    case 1:

      /* WGDOS */

      /* for this case we will use unwgdos routine */

      /* unwgdos routine wants to know number of native integers in input.
       * input type might not be native int, so calculate:
       */
      nint = rec->disklen * ppfile->wordsize / sizeof(int);
      mdi = *(Freal*)pp_get_var_fill_value(hdrp);

      CKI(   pp_unwgdos_wrap(packed_data, nint, data, rec->datalen, mdi, heaplist)   );
      
      break;
      
    
    case 2:
      /* CRAY 32-bit method */
      
      if (vartype != realtype) {
	CuError(CU_EINTERN,"Cray 32-bit unpacking supported only for REAL type data");
	ERR;
      }
      
      /* 
       * in the event of a 64-bit file (which it probably is, else 32-bit packing is 
       * redundant), and if we're on a little-endian machine, the file was written on
       * a cray, so the 64-bit byte swapping (whether done by cdunifpp or previously)
       * will have had the side-effect of swapping pairs of 32-bit words and we need
       * to swap them back again.
       *
       * NB  LITTLE_ENDIAN_MACHINE  is defined (if at all) in cdunifpp.h
       */
      
#ifdef LITTLE_ENDIAN_MACHINE
      if (ppfile->wordsize == 8)
	pp_swap32couplets(packed_data,packed_bytes);
#endif
      
      for (ipt=0; ipt < rec->datalen ; ipt++)
	*(((Freal*) (data)) + ipt) = *(((Freal4*) (packed_data)) + ipt);
    
      break;
    
    case 3:
      CuError(CU_EINTERN,"GRIB unpacking not supported");
      ERR;
      /* break; */

    default:
      pp_switch_bug("pp_read_data_record"); ERR;
    }

    /* Okay - data unpacked - free up packed data */
    CKI(  pp_free(packed_data,heaplist)  );
  }


  /* if land or sea mask compression, then allocate another array, and
   * copy the relevant data across, filling the gaps with missing data
   */

  if (pp_get_var_compression(hdrp) == 2) {
    
    npoint = pp_genaxis_len(ppfile->landmask->xaxis) * pp_genaxis_len(ppfile->landmask->yaxis);
    bytes = npoint * wordsize;
    CKP(   data_expanded = pp_malloc(bytes,heaplist)   );
    
    switch ((hdrp->LBPACK/100)%10) {
    case 1:
      /* land mask compression */
      valid_landmask_value = 1;
      break;
    case 2:
      /* sea mask compression */
      valid_landmask_value = 0;
      break;
    default:
      pp_switch_bug("pp_read_data_record"); ERR;
    }

    landmask_vals = ppfile->landmask->data->values;
    srcptr = data;
    destptr = data_expanded;

    CKP(   fill_ptr = pp_get_var_fill_value(hdrp)   );

    npoint_used = 0;

    for (ipt = 0; ipt < npoint; ipt++) {
      if (abs(landmask_vals[ipt]) == valid_landmask_value) {

	if (npoint_used >= rec->datalen) {
	  CuError(CU_EINTERN,"Uncompressing tried to use more compressed data than available");
	  ERR;
	}

	memcpy(destptr,srcptr,wordsize);
	srcptr += wordsize;
	npoint_used++;
      }
      else {
	memcpy(destptr,fill_ptr,wordsize);
      }
      destptr += wordsize;
    }

    if (npoint_used != rec->datalen) {
      CuError(CU_EINTERN,"Uncompressing did not use all the compressed data");
      ERR;
    }


    CKI(  pp_free(data,heaplist)  );
    data = data_expanded;
  }

  return data;

  ERRBLKP("pp_read_data_record");
}
Exemplo n.º 22
0
int pp_process(CuFile *file)
{
  int rec, nrec, at_start_rec, at_end_rec;
  PPfile *ppfile;
  PPfieldvar *fvar;
  PPrec *recp;
  PPhdr *hdrp;
  PPgenaxis *xaxis, *yaxis, *zaxis, *taxis; /* JAK 2005-01-05 */
  PPlist *heaplist;
  PPlist *fieldvars;
  PPlisthandle handle, thandle;
  PPlist *gatts, *catts;

  int ndims, dimid;
  int idim; /* dim number of given type */
  int have_time_mean, tmdimid; /* dimensions used for meaning (CF cell methods) */
  CuDim *cudims,*dim;
  PPdim *ppdim;

  int nvars, varid, cvarid;
  int ncvars; /* coord vars */
  int nfvars; /* field vars */  
  int nvrec;  
  PPrec **recs, **vrecs;
  CuVar *cuvars, *var;  
  PPvar *ppvar;
  PPlist *atts;
  PPlist *axislist;
  PPlist *xaxes, *yaxes, *taxes, *zaxes; /* JAK 2005-01-05 */
  PPgenaxis *axis;  /*JAK 2005-01-10 */
  int have_hybrid_sigmap;
  PPaxistype axistype;
  
  int rotmapid, rotgridid;
  PPlist *rotgrids, *rotmaps;
  PProtmap *rotmap;
  PProtgrid *rotgrid;
  CuVar *lonvar, *latvar;  
  PPvar *lonppvar, *latppvar;

  PPlandmask *landmask;

  char *varnamea, *varnameb;

  char dimnamestem[CU_MAX_NAME], units[CU_MAX_NAME];
  char formulaterms[MAX_ATT_LEN+1];

  int dont_free_horiz_axes;

  int added;

  int zindex,tindex,svindex;

  /* ------------------------------------------------------ */  
  /* initialisation constants which matter */
  ncvars = 0;
  have_hybrid_sigmap = 0;
  have_time_mean = 0;
  svindex = 1;

  /* initialisation constants just to avoid compiler warnings
   * (rather than get accustomed to ignoring warnings)
   * but flow logic should mean that these vars do actually get
   * initialised elsewhere before use
   */
  at_end_rec=0;
  xaxis=yaxis=NULL;
  fvar=NULL;
  zaxis=NULL;
  taxis=NULL;
  axislist=NULL;
  tmdimid=-1;
  dont_free_horiz_axes=0;
  /* ------------------------------------------------------ */

  ppfile = file->internp;

  heaplist=ppfile->heaplist;

  nrec = ppfile->nrec;  
  recs = ppfile->recs;

  /* initialise elements in the records before sorting */
  CKI(   pp_initialise_records(recs, nrec, heaplist)   );

  /* sort the records */
  qsort(recs, nrec, sizeof(PPrec*), pp_compare_records);  

  /* now sort out the list of variables and dimensions */

  CKP(   fieldvars=pp_list_new(heaplist)   );
  CKP(   xaxes=pp_list_new(heaplist)   );
  CKP(   yaxes=pp_list_new(heaplist)   );
  CKP(   zaxes=pp_list_new(heaplist)   );
  CKP(   taxes=pp_list_new(heaplist)   );
  CKP(   rotmaps=pp_list_new(heaplist)  );
  CKP(   rotgrids=pp_list_new(heaplist)  );

  /* before main loop over records, look for land mask */
  for (rec=0; rec<nrec ; rec++) {

    recp = recs[rec];
    hdrp = &recp->hdr;    

    if (pp_var_is_land_mask(hdrp)) {
      
      CKP(   landmask = pp_malloc(sizeof(PPlandmask),heaplist)   );

      CKI(  pp_set_horizontal_axes(recp,ppfile,&xaxis,&yaxis,rotmaps,heaplist)  );

      CKP(   landmask->data = pp_data_new(inttype,pp_genaxis_len(xaxis) * pp_genaxis_len(yaxis),heaplist)   ); /* JAK 2005-01-05 */

      /* read in land mask data values */

      landmask->xaxis = xaxis;
      landmask->yaxis = yaxis;
      CKP(   landmask->data->values=pp_read_data_record(recp,ppfile,heaplist)   );

      ppfile->landmask = landmask;

    }	
  }

  /* ====== START LOOP OVER RECORDS ====== */

  for (rec=0; rec<nrec ; rec++) {

    recp = recs[rec];
    hdrp = &recp->hdr;

    /* we are at start record of a variable at the very start, or if at we were at the
     * end record last time
     */
    at_start_rec = ( rec == 0 || at_end_rec );

    /* we are at end record of a variable at the very end, or if the header shows a 
     * difference from the next record which constitutes a different variable
     */
    at_end_rec = ( rec == nrec-1 ||
		   pp_records_from_different_vars(recs[rec+1],recp));
    
    /*------------------------------*/
    /* allow for variables which are unsupported for some reason */

    if (at_start_rec)
      if (pp_test_skip_var(hdrp, ppfile->landmask))
	continue;
      
    /* -------

    if (at_start_rec) 
      puts("++++++ START OF VARIABLE +++++++++");
    printf("processing record %d / %d\n",rec,nrec);
    pp_dump_header(hdrp);

    ------ */


    if (at_start_rec) {
      /* ====== THINGS DONE ONLY AT START RECORD OF EACH VARIABLE ====== */

      /* get PPvar structure, and initialise certain structure members for tidiness */
      CKP(   fvar=pp_malloc(sizeof(PPfieldvar), heaplist)   );
      CKP(   fvar->axes=pp_list_new(heaplist)  );  /* JAK 2005-01-05 */
      fvar->firstrecno = rec;
      fvar->firstrec = recp;

      if (pp_get_var_compression(hdrp) == 2) {
	/* land/sea mask compression: for horiz axes use those of LSM */
	xaxis = ppfile->landmask->xaxis;
	yaxis = ppfile->landmask->yaxis;
	dont_free_horiz_axes = 1;
      } else {

	CKI(  pp_set_horizontal_axes(recp,ppfile,&xaxis,&yaxis,rotmaps,heaplist)  );

	dont_free_horiz_axes = 0;
      }

      CKP(   zaxis=pp_genaxis_new(zaxis_type,zdir,heaplist)   );
      CKI(  pp_zaxis_set(zaxis,hdrp)  );

      CKP(   taxis=pp_genaxis_new(taxis_type,tdir,heaplist)   );
      CKI(  pp_taxis_set(taxis,hdrp)  );

    }

    /* construct pp_lev struct, and add it to the z axis if not already present
     * (could already be present if field has multiple times on each level)
     */

    /* ====== THINGS DONE FOR EVERY PP RECORD ====== */

    CKI(  pp_zaxis_add(zaxis, recp->lev, &zindex, heaplist)  );
    recp->zindex = zindex;

    CKI(  pp_taxis_add(taxis, recp->time, &tindex, heaplist)  );
    recp->tindex = tindex;

    /* ===================================================== */
    if (at_end_rec) {
     /* ====== THINGS DONE ONLY AT END RECORD OF EACH VARIABLE ====== */

      fvar->lastrecno = rec;
      nvrec = fvar->lastrecno - fvar->firstrecno + 1;
      vrecs = recs + fvar->firstrecno;

      /* now if the axes are not regular, free the axes, split the variable into a number of variables and try again... */
      if (pp_set_disambig_index(zaxis, taxis, vrecs, nvrec, svindex)) {
	
	/* increment the supervar index, used later to show the connection between
	 *  the separate variables into which this one will be split
	 */
	svindex++;

	/* now re-sort this part of the record list, now that we have set the disambig index */
	qsort(vrecs, nvrec, sizeof(PPrec*), pp_compare_records);  

	/* now go back to the start record of the variable; set to one less because it
	 * will get incremented in the "for" loop reinitialisation
	 */
	rec = fvar->firstrecno - 1;

	/* and free the stuff assoc with the var we won't be using */
	if (!dont_free_horiz_axes) {
	  CKI(  pp_genaxis_free(xaxis,heaplist)  );
	  CKI(  pp_genaxis_free(yaxis,heaplist)  );
	}
	CKI(  pp_genaxis_free(zaxis,heaplist)  );
	CKI(  pp_genaxis_free(taxis,heaplist)  );
	CKI(  pp_free(fvar,heaplist)  );

	continue;
      }

      /*------------------------------------------------------------*/

      /*
       * For each axis, see if it matches an axis which already exists from a previous
       * variable.
       *
       * If so, then free the structure and point to the existing occurrence instead.
       *
       * If not, then add to the list.
       */

      /* x */
      CKI(  added = pp_list_add_or_find(xaxes, &xaxis, pp_genaxis_compare, 0, 
					(dont_free_horiz_axes ? NULL : (free_func) pp_genaxis_free),
					NULL, heaplist)  );
      if (added)
	ncvars++;

      /* y */
      CKI(  added = pp_list_add_or_find(yaxes, &yaxis, pp_genaxis_compare, 0, 
					(dont_free_horiz_axes ? NULL : (free_func) pp_genaxis_free),
					NULL, heaplist)  );
      if (added)
	ncvars++;

      /* z */
      CKI(  added = pp_list_add_or_find(zaxes, &zaxis, pp_genaxis_compare, 0, 
					(free_func) pp_genaxis_free,
					NULL, heaplist)  );
      if (added) {
	ncvars++;
	if (pp_zaxis_lev_type(zaxis) == hybrid_sigmap_lev_type) {
	  /* two more coord vars (a and b coeffs) */
	  ncvars+=2;
	  have_hybrid_sigmap=1;
	}
	if (pp_zaxis_lev_type(zaxis) == hybrid_height_lev_type) {
	  /* two more coord vars (a and b coeffs) */
	  ncvars+=2;
	}
      }

      /* t */
      CKI(  added = pp_list_add_or_find(taxes, &taxis, pp_genaxis_compare, 0, 
					(free_func) pp_genaxis_free,
					NULL, heaplist)  );
      if (added) {
	ncvars++;
	if (pp_taxis_is_time_mean(taxis)) {
	  /* need to make sure we have the mean dim (size 2),
	   * also one more coordinate var
	   */	  
	  have_time_mean=1;
	  ncvars++;
	}
      }

      /* associate var with these axes */
      CKI(   pp_list_add(fvar->axes,xaxis,heaplist)   );
      CKI(   pp_list_add(fvar->axes,yaxis,heaplist)   );
      CKI(   pp_list_add(fvar->axes,zaxis,heaplist)   );
      CKI(   pp_list_add(fvar->axes,taxis,heaplist)   );
      
      /* get the rotated grid, if any 
       * (NB this is done *after* the pp_list_add_or_find stuff above, because 
       * otherwise the axis pointers could get orphaned if the axes are found to 
       * be duplicates)
       */
      CKP(  fvar->rotgrid = pp_get_rotgrid(xaxis,yaxis,rotgrids,heaplist)  );

      /* add the variable */
      CKI(   pp_list_add(fieldvars, fvar, heaplist)   );
      
      /* ===================================================== */
    }
  }

    
  /*  ====================================================================
   *  Having completed the loop over records, we now know the number of
   *  dimensions and variables, so we can finally do the relevant calls
   *  to allocate these arrays and populate them usefully.
   *  ====================================================================
   */
  /* FIRST ALLOCATE THE ARRAYS, and initialise some values */
  nfvars = pp_list_size(fieldvars);

  if (nfvars <= 0) {
    CuError(CU_EOPEN,"No valid fields in file\n");
    ERR; /* not the most elegant dealing with this error - ideally would free this file */
  }
  ndims = pp_list_size(xaxes) + pp_list_size(yaxes) + pp_list_size(zaxes) + pp_list_size(taxes);

  if (have_time_mean){    
    tmdimid=ndims;
    ndims++;
  }
  if (have_hybrid_sigmap) {
    /* will need a scalar variable called "p0" */
   ncvars++;
 }

  CKP(   cudims = CuCreateDims(file,ndims)   );

  /* need a grid_mapping variable for every rotation mapping,
   * and need lon and lat variables for every rotated grid
   */
  ncvars += pp_list_size(rotmaps) + 2*pp_list_size(rotgrids);

  nvars = ncvars + nfvars;

  CKP(   cuvars = CuCreateVars(file,nvars)   );

  for (dimid=0; dimid<ndims; dimid++) {
    dim=&cudims[dimid];
    dim->var = (CuVar*)0;
    dim->coord = (CuVar*)0;
    dim->datatype = realtype;
    dim->dimtype = CuGlobalDim;

    /* uncomment if internal structure is to be used
     *   CKP(   dim->internp = pp_malloc(sizeof(PPdim), heaplist)   );
     *   ppdim=(PPdim*)dim->internp;
     */
  }
  for (varid=0; varid<nvars; varid++) {
    var=&cuvars[varid];
    var->datatype = realtype;
    CKP(   var->internp = pp_malloc(sizeof(PPvar), heaplist)   );
    ppvar=(PPvar*)var->internp;
    ppvar->firstrecno=-1;
    ppvar->lastrecno=-1;
    ppvar->data=NULL;
    CKP(   ppvar->atts = pp_list_new(heaplist)   );
  }


  /* 
   * NOW POPULATE THE STRUCTURES
   *
   * The procedure will be to loop over all the axes, adding dimensions and
   * variables associated with those axes.
   *
   * Having done that, any dimensions not associated with axes will be added,
   * and then the field variables will be added.
   */

  dimid=0;
  varid=0;

  for (axistype=0; axistype<num_axistype; axistype++) {
    switch(axistype){
    case xaxistype: axislist=xaxes; break;
    case yaxistype: axislist=yaxes; break;
    case zaxistype: axislist=zaxes; break;
    case taxistype: axislist=taxes; break;
    default: pp_switch_bug("cdunifpp_process");
    }
    pp_list_startwalk(axislist,&handle);
    idim=0;

    while ((axis=pp_list_walk(&handle,0))!=NULL) {
      dim=&cudims[dimid];
      var=&cuvars[varid];
      ppdim=(PPdim*) dim->internp;
      ppvar=(PPvar*) var->internp;
      
      dim->coord = var;

      axis->dimid=dimid;
      dim->len=pp_genaxis_len(axis);
      CKP(  ppvar->data=pp_genaxis_getCF(axis,dimnamestem,units,ppvar->atts,heaplist)  );
      sprintf(dim->name,dimnamestem,idim);
      if (units != NULL) {
	strncpy(dim->units,units,CU_MAX_NAME);
	dim->units[CU_MAX_NAME]='\0';
      }
      strncpy(var->name,dim->name,CU_MAX_NAME);
      var->name[CU_MAX_NAME]='\0';
      var->ndims=1;
      var->dims[0] = dimid;
      varid++;
      /* now add certain variables for hybrid_sigmap z axis */
      if (axistype == zaxistype && pp_zaxis_lev_type(axis) == hybrid_sigmap_lev_type) {
      
	catts=ppvar->atts; /* attribute list for the main coord var */

	/* Hybrid sigma-p A coefficient */
	var=&cuvars[varid];
	ppvar=(PPvar*) var->internp;
	sprintf(var->name,"z%d_hybrid_sigmap_acoeff",idim);
	varnamea=var->name;
	CKP(   ppvar->data = pp_genaxis_to_values(axis,hybrid_sigmap_a_type,heaplist)   );
	CKI(   pp_add_string_att(ppvar->atts,"units","Pa",heaplist)   );
	CKI(   pp_add_string_att(ppvar->atts,"long_name",
				 "atmospheric hybrid sigma-pressure 'A' coefficient",heaplist)   );
	var->ndims=1;
	var->dims[0] = dimid;
	varid++;

	/* Hybrid sigma-p B coefficient */
	var=&cuvars[varid];
	ppvar=(PPvar*) var->internp;
	sprintf(var->name,"z%d_hybrid_sigmap_bcoeff",idim);
	varnameb=var->name;
	CKP(   ppvar->data = pp_genaxis_to_values(axis,hybrid_sigmap_b_type,heaplist)   );
	CKI(   pp_add_string_att(ppvar->atts,"long_name",
				 "atmospheric hybrid sigma-pressure 'B' coefficient",heaplist)   );
	var->ndims=1;
	var->dims[0] = dimid;
	varid++;
	
	snprintf(formulaterms,MAX_ATT_LEN,"ap: %s b: %s ps: ps p0: p0",varnamea,varnameb);

	CKI(   pp_add_string_att(catts,"formula_terms",formulaterms,heaplist)   );
	CKI(   pp_add_string_att(catts,"standard_name","atmosphere_hybrid_sigma_pressure_coordinate",heaplist)   );

	CKI(   pp_add_string_att(catts,"comments",
				 "The \"ps\" term in formula_terms is set to \"ps\" variable. "
				 "This variable may or may not be provided.",heaplist)   );
      }

      /* now add certain variables for hybrid_height z axis */
      if (axistype == zaxistype && pp_zaxis_lev_type(axis) == hybrid_height_lev_type) {
      
	catts=ppvar->atts; /* attribute list for the main coord var */

	/* Hybrid height A coefficient */
	var=&cuvars[varid];
	ppvar=(PPvar*) var->internp;
	sprintf(var->name,"z%d_hybrid_height_acoeff",idim);
	varnamea=var->name;
	CKP(   ppvar->data = pp_genaxis_to_values(axis,hybrid_height_a_type,heaplist)   );
	CKI(   pp_add_string_att(ppvar->atts,"units","m",heaplist)   );
	var->ndims=1;
	var->dims[0] = dimid;
	varid++;

	/* Hybrid height B coefficient */
	var=&cuvars[varid];
	ppvar=(PPvar*) var->internp;
	sprintf(var->name,"z%d_hybrid_height_bcoeff",idim);
	varnameb=var->name;
	CKP(   ppvar->data = pp_genaxis_to_values(axis,hybrid_height_b_type,heaplist)   );
	var->ndims=1;
	var->dims[0] = dimid;
	varid++;
	
	snprintf(formulaterms,MAX_ATT_LEN,"a: %s b: %s orog: orography",varnamea,varnameb);

	CKI(   pp_add_string_att(catts,"formula_terms",formulaterms,heaplist)   );
	CKI(   pp_add_string_att(catts,"standard_name","atmosphere_hybrid_sigma_pressure_coordinate",heaplist)   );

	CKI(   pp_add_string_att(catts,"comments",
				 "The \"orog\" term in formula_terms is set to \"orography\" variable. "
				 "This variable may or may not be provided.",heaplist)   );
      }


      /* add the boundary variable for time mean */
      if (axistype == taxistype && pp_taxis_is_time_mean(axis)) {
	catts=ppvar->atts; /* attribute list for the main coord var */

	var=&cuvars[varid];
	ppvar=(PPvar*) var->internp;
	sprintf(var->name,"time_bnd%d",idim);
	CKP(   ppvar->data = pp_taxis_to_boundary_values(axis->axis,heaplist)   );
	var->ndims=2;
	var->dims[0]=dimid;
	var->dims[1]=tmdimid;
	
	CKI(   pp_add_string_att(catts,"bounds",var->name,heaplist)   );

	varid++;
      }
      dimid++;
      idim++;      
    } /* end loop over axes of given */
  } /* end loop over axis types */

  /* add nv dimension if we had time mean */
  if (have_time_mean) {

    dim=&cudims[dimid];

    strcpy(dim->name,"nv");
    dim->len=2;
    
    /* Should have tmdimid=dimid, but actually we already set it above (it evaluates to ndims)
     * as we needed it before we got here.  So just do a check here.
     */
    if ( tmdimid != dimid ) {
      pp_error_mesg("cdunifpp_process","ID wrong for 'nv' dimension?");
      ERR;
    }

    dimid++;
  }

  /* add p0 variable if we had hybrid_sigmap coords */
  if (have_hybrid_sigmap) {
     
    var=&cuvars[varid];
    ppvar=(PPvar*) var->internp;
    sprintf(var->name,"p0");
    var->ndims=0;
    CKI(   pp_add_string_att(ppvar->atts,"long_name",
			     "reference pressure value for atmospheric hybrid sigma-pressure coordinates",
			     heaplist)   );
    /* single value consisting of p0 */
    CKP(   ppvar->data = pp_data_new(realtype,1,heaplist)   );
    ((Freal*)(ppvar->data->values))[0]=reference_pressure;
    varid++;
  }

  /* add any rotated_pole variables */
  rotmapid=0;
  pp_list_startwalk(rotmaps,&handle);
  while ((rotmap=pp_list_walk(&handle,0))!=NULL) {
    
    var=&cuvars[varid];
    ppvar=(PPvar*) var->internp;
    sprintf(var->name,"rotated_pole%d",rotmapid);

    strncpy(rotmap->name,var->name,CU_MAX_NAME);
    rotmap->name[CU_MAX_NAME]='\0';

    /* single value of arbitrary type; set as integer = 0 */
    var->datatype=inttype;
    var->ndims=0;
    CKP(   ppvar->data = pp_data_new(inttype,1,heaplist)   );
    ((Freal*)(ppvar->data->values))[0]=0;

    /* and add attributes */
    catts=ppvar->atts;
    CKI(  pp_add_string_att(catts,"grid_mapping_name","rotated_latitude_longitude",heaplist)  );
    CKI(  pp_add_att(catts,"grid_north_pole_longitude",realtype,1,&rotmap->pole_lon,heaplist)  );
    CKI(  pp_add_att(catts,"grid_north_pole_latitude",realtype,1,&rotmap->pole_lat,heaplist)  );
    CKI(  pp_add_att(catts,"north_pole_grid_longitude",realtype,1,&rotmap->truepole_gridlon,heaplist)  );

    rotmapid++;
    varid++;
  }

  /* and add any lon, lat variables for rotated grids */
  rotgridid=0;
  pp_list_startwalk(rotgrids,&handle);
  while ((rotgrid=pp_list_walk(&handle,0))!=NULL) {

    lonvar=&cuvars[varid];
    lonppvar=(PPvar*) lonvar->internp;
    varid++;

    latvar=&cuvars[varid];
    latppvar=(PPvar*) latvar->internp;
    varid++;

    xaxis = rotgrid->xaxis;
    yaxis = rotgrid->yaxis;
      
    sprintf(lonvar->name,"true_lon%d",rotgridid);
    sprintf(latvar->name,"true_lat%d",rotgridid);

    sprintf(rotgrid->coords,"%s %s",lonvar->name,latvar->name);

    lonvar->ndims=2;
    latvar->ndims=2;

    lonvar->dims[0]=yaxis->dimid;
    latvar->dims[0]=yaxis->dimid;

    lonvar->dims[1]=xaxis->dimid;
    latvar->dims[1]=xaxis->dimid;

    CKI(   pp_calc_rot_grid(rotgrid,&lonppvar->data,&latppvar->data,heaplist)   );
    CKI(  pp_add_string_att(lonppvar->atts,"long_name","longitude",heaplist)  );
    CKI(  pp_add_string_att(latppvar->atts,"long_name","latitude",heaplist)  );

    CKI(  pp_add_string_att(lonppvar->atts,"standard_name","longitude",heaplist)  );
    CKI(  pp_add_string_att(latppvar->atts,"standard_name","latitude",heaplist)  );

    CKI(  pp_add_string_att(lonppvar->atts,"units","degrees_east",heaplist)  );
    CKI(  pp_add_string_att(latppvar->atts,"units","degrees_north",heaplist)  );

    CKI(  pp_add_att(lonppvar->atts,"modulo",realtype,1,&lon_modulo,heaplist)  );

    rotgridid++;
  }

  /* sanity check - the variable ID for the next variable to be added should 
   * now match the number of coordinate variables
   */
  if ( varid != ncvars ) {
    pp_error_mesg("cdunifpp_process","wrong number of coord vars?");
    ERR;
  }
  

  /* add all the attributes for coord variables
   * (didn't do inside the loop because more complicated
   * for hybrid z coords / t mean)
   */
  for (cvarid=0; cvarid<ncvars; cvarid++) {
    var=&cuvars[cvarid];
    ppvar=(PPvar*) var->internp;
    CKI(   pp_copy_and_free_atts(file,var,ppvar->atts,heaplist)   );
  }

  

  /*========================================================
   * Okay we've done all the variables related to dimensions
   * Add the field variables.
   *========================================================
   */
  pp_list_startwalk(fieldvars,&handle);
  while ((fvar=pp_list_walk(&handle,0))!=NULL) {
    var=&cuvars[varid];
    ppvar=(PPvar*) var->internp;
    atts = ppvar->atts;
    hdrp = &fvar->firstrec->hdr;

    CKI(   pp_var_lookup(hdrp, &fvar->stashmeta)   );

    CKI(   pp_get_var_name(varid, fvar->stashmeta.shortname, cuvars)   );
    
    var->ndims=4; /* rpw axeslist len */

    /*
     *  Axes in fvar->axes list are fastest varying first (lon,lat,lev,time)
     *  But require them in netCDF-like order (time,lev,lat,lon), so
     *  reverse the order while copying into var->dims.
     */
    idim=var->ndims;
    pp_list_startwalk(fvar->axes,&thandle);
    while ((axis=pp_list_walk(&thandle,0)) != NULL) {
      var->dims[--idim] = axis->dimid;
    }

    var->datatype = pp_get_var_type(hdrp);

    ppvar->firstrecno = fvar->firstrecno;
    ppvar->lastrecno = fvar->lastrecno;

    CKI(   pp_var_get_extra_atts(var, fvar, cudims, atts, heaplist)   );
   
    CKI(   pp_copy_and_free_atts(file, var, atts, heaplist)   );

    varid++;
  }

  /* sanity check - the variable ID for the next variable to be added (if there was one,
   * which there isn't), should now match the total number of variables
   */
  if ( varid != nvars ) {
    pp_error_mesg("cdunifpp_process","wrong number of vars?");
    ERR;
  }

  
  /* set numbers in file structure */
  file->ndims=ndims;
  file->nvars=nvars;
  file->recdim=-1;

  /* set global attributes */
  CKP(   gatts = pp_get_global_attributes(file->controlpath, ppfile, heaplist)   );
  CKI(   pp_copy_and_free_atts(file,NULL,gatts,heaplist)   );

  /*========================================================
   * All done and ready for dimget / varget.
   *========================================================
   */

  /* free what memory we can */
  CKI(   pp_free_tmp_vars(xaxes, yaxes, zaxes, taxes, fieldvars, heaplist)   );

  return 0;

  ERRBLKI("pp_process");
}