int main(int argc, char **argv) {
  MPI_Offset i, j, k;
  int status;
  int ncid;
  int dimid1, dimid2, dimid3, udimid;
  int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1];
  MPI_Offset square_start[2], cube_start[3] = {0, 0, 0};
  MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50};
  MPI_Offset xytime_start[3] = {0, 0, 0};
  MPI_Offset xytime_count[3] = {100, 50, 50};
  MPI_Offset time_start[1], time_count[1] = {25};
  int square_id, cube_id, xytime_id, time_id;
  static char title[] = "example netCDF dataset";
  static char description[] = "2-D integer array";
  int data[100][50][50], buffer[100];
  int rank;
  int nprocs;
  MPI_Comm comm = MPI_COMM_WORLD;
  double TotalWriteTime;
  params opts;

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);

  if (rank == 0) 
	  fprintf(stderr, "Testing write ... \n");
  parse_write_args(argc, argv, rank, &opts);

  MPI_Barrier(MPI_COMM_WORLD);
  TotalWriteTime = MPI_Wtime();

  /**********  START OF NETCDF ACCESS **************/

  /**
   * Create the dataset
   *   File name: "testwrite.nc"
   *   Dataset API: Collective
   */

  status = ncmpi_create(comm, opts.outfname, NC_CLOBBER|NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid);
  if (status != NC_NOERR) handle_error(status);


  /**
   * Create a global attribute:
   *    :title = "example netCDF dataset";
   */
  sprintf(title, "%s:%d of %d", title, rank, nprocs);
  printf("title:%s\n", title);
  status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title",
                          strlen(title), title);
  if (status != NC_NOERR) handle_error(status);
  
   
  /**
   * Add 4 pre-defined dimensions:
   *   x = 100, y = 100, z = 100, time = NC_UNLIMITED
   */
  status = ncmpi_def_dim(ncid, "x", 100L, &dimid1);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_def_dim(ncid, "y", 100L, &dimid2);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_def_dim(ncid, "z", 100L, &dimid3);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid);
  if (status != NC_NOERR) handle_error(status);

  /**
   * Define the dimensionality and then add 4 variables:
   *    square(x, y), cube(x,y,z), time(time), xytime(time, x, y)  
   */

  square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1;
  square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2;
  cube_dim[2] = dimid3;
  xytime_dim[0] = udimid;
  time_dim[0] = udimid;
  status = ncmpi_def_var (ncid, "square", NC_INT, 2, square_dim, &square_id);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_def_var (ncid, "cube", NC_INT, 3, cube_dim, &cube_id);
  if (status != NC_NOERR) handle_error(status);
 // status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id);
  status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_def_var (ncid, "xytime", NC_INT, 3, xytime_dim, &xytime_id);
  if (status != NC_NOERR) handle_error(status);

  /**
   * Add an attribute for variable: 
   *    square: decsription = "2-D integer array"
   */

  status = ncmpi_put_att_text (ncid, square_id, "description",
                          strlen(description), description);
  if (status != NC_NOERR) handle_error(status);

  /**
   * End Define Mode (switch to data mode)
   *   Dataset API: Collective
   */
  
  status = ncmpi_enddef(ncid);
  if (status != NC_NOERR){  
	handle_error(status);
  	status = ncmpi_close(ncid);
 	if (status != NC_NOERR) handle_error(status);
	if (rank == 0) {
	  fprintf(stderr, "Fatal Error: file header is inconsistent!\n");
	}
 }
  /**
   * Data Partition (Assume 4 processors):
   *   square: 2-D, (Block, Block), 50*50 from 100*100 
   *   cube:   3-D, (*, Block, Block), 100*50*50 from 100*100*100
   *   xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
   *   time:   1-D, Block-wise, 25 from 100
   */
  else {
	  square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50;
	  square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50;
	  time_start[0] = (rank%4) * 25;


  /**
   * Packing data in the buffer 
   */

  /* Data for variable: time */
	  for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ )
	    buffer[i - time_start[0]] = i;   

  /* Data for variable: square, cube and xytime */
	  for ( i = 0; i < 100; i++ )
	    for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ )
	      for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ )
	        data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k;

  /**
   * Write data into variables: square, cube, time and xytime  
   *   Access Method: subarray
   *   Data Mode API: collective
   */ 
  
	  status = ncmpi_put_vara_int_all(ncid, square_id,
                    square_start, square_count,
                    &data[0][0][0]);
	  if (status != NC_NOERR) handle_error(status);
	  status = ncmpi_put_vara_int_all(ncid, cube_id,
                    cube_start, cube_count,
                    &data[0][0][0]);
	  if (status != NC_NOERR) handle_error(status);
	  status = ncmpi_put_vara_int_all(ncid, time_id,
                    time_start, time_count,
                    (void *)buffer);
	  if (status != NC_NOERR) handle_error(status);
	  status = ncmpi_put_vara_int_all(ncid, xytime_id,
                    xytime_start, xytime_count,
                    &data[0][0][0]);
	  if (status != NC_NOERR) handle_error(status);

/*
status = ncmpi_sync(ncid);
if (status != NC_NOERR) handle_error(status); 
status = ncmpi_redef(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_del_att(ncid, square_id, "description");
if (status != NC_NOERR) handle_error(status); 
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
*/

  /**
   * Close the dataset
   *   Dataset API:  collective
   */

	  status = ncmpi_close(ncid);
	  if (status != NC_NOERR) handle_error(status);
  /*******************  END OF NETCDF ACCESS  ****************/

	MPI_Barrier(MPI_COMM_WORLD); 
	TotalWriteTime = MPI_Wtime() - TotalWriteTime;

	if (rank == 0) {
	  fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname);
	  fprintf(stderr, "Total Write Time = %10.8f\n", TotalWriteTime);
	}
  }
  MPI_Finalize();
  return 0;
}
Esempio n. 2
0
int test(char* fname, int enable_log) {
    int buffer[MAXPROCESSES];
    MPI_Offset start[MAXPROCESSES][2], count[MAXPROCESSES][2];
    MPI_Offset *sp[MAXPROCESSES], *cp[MAXPROCESSES];
    MPI_Offset stride[2];

    int i, j, ret;
    int NProc, MyRank, NP;      // Total process; Rank
    int fid;        // Data set ID
    int did[2];     // IDs of dimension
    int vid[4];        // IDs for variables
    int dims[2];
    char tmp[1024];
    MPI_Info Info;

    MPI_Comm_size(MPI_COMM_WORLD, &NP);
    MPI_Comm_rank(MPI_COMM_WORLD, &MyRank);

    if (NP == 1) {    // Act if there is WIDTH processes for easy debugging. Most debugger supports only single proccesses.
        NProc = SINGLEPROCNP;
        MyRank = SINGLEPROCRANK;
    }
    else{
        NProc = NP;
    }

    if (MyRank < MAXPROCESSES) {
        // Ensure each process have a independent buffer directory

        MPI_Info_create(&Info);
        if (enable_log) {
            MPI_Info_set(Info, "pnetcdf_log", "enable");
        }

        // Create new cdf file
        ret = ncmpi_create(MPI_COMM_WORLD, fname, NC_CLOBBER, Info, &fid);
        if (ret != NC_NOERR) {
            printf("Error create file\n");
            goto ERROR;
        }
        ret = ncmpi_set_fill(fid, NC_FILL, NULL);
        if (ret != NC_NOERR) {
            printf("Error set fill\n");
            goto ERROR;
        }
        ret = ncmpi_def_dim(fid, "X", NProc, did);  // X
        if (ret != NC_NOERR) {
            printf("Error def dim X\n");
            goto ERROR;
        }
        ret = ncmpi_def_dim(fid, "Y", NProc * 4, did + 1);    // Y
        if (ret != NC_NOERR) {
            printf("Error def dim Y\n");
            goto ERROR;
        }
        ret = ncmpi_def_var(fid, "M0", NC_INT, 2, did, vid + 0);
        if (ret != NC_NOERR) {
            printf("Error def var M0\n");
            goto ERROR;
        }
        ret = ncmpi_def_var(fid, "M1", NC_INT, 2, did, vid + 1);
        if (ret != NC_NOERR) {
            printf("Error def var M1\n");
            goto ERROR;
        }
        ret = ncmpi_def_var(fid, "M2", NC_INT, 2, did, vid + 2);
        if (ret != NC_NOERR) {
            printf("Error def var M2\n");
            goto ERROR;
        }
        ret = ncmpi_def_var(fid, "M3", NC_INT, 2, did, vid + 3);
        if (ret != NC_NOERR) {
            printf("Error def var M3\n");
            goto ERROR;
        }
        ret = ncmpi_enddef(fid);
        if (ret != NC_NOERR) {
            printf("Error enddef\n");
            goto ERROR;
        }

        // We all write rank from now on
        for (i = 0; i < NProc; i++) {
            buffer[i] = MyRank;
        }

        // put_var1
        for (i = 0; i < 4; i++) {
            for (j = 0; j < NProc; j++) {
                start[0][0] = MyRank;
                start[0][1] = i * NProc + j;
                ret = ncmpi_put_var1_int_all(fid, vid[i], start[0], buffer);
                if (ret != NC_NOERR) {
                    printf("Error put_var1\n");
                    goto ERROR;
                }
            }
        }

        // put_vara
        for (i = 0; i < 4; i++) {
            start[0][0] = 0;
            start[0][1] = ((i + 1) % 4) * NProc + MyRank;
            count[0][0] = NProc;
            count[0][1] = 1;
            ret = ncmpi_put_vara_int_all(fid, vid[i], start[0], count[0], buffer);
            if (ret != NC_NOERR) {
                printf("Error put_vara\n");
                goto ERROR;
            }
        }

        // put_vars
        for (i = 0; i < 4; i++) {
            start[0][0] = MyRank;
            start[0][1] = ((i + 2) % 4) * NProc + (MyRank % 2);
            count[0][0] = 1;
            count[0][1] = NProc / 2;
            stride[0] = 1;
            stride[1] = 2;
            ret = ncmpi_put_vars_int_all(fid, vid[i], start[0], count[0], stride, buffer);
            if (ret != NC_NOERR) {
                printf("Error put_vars\n");
                goto ERROR;
            }
        }

        // put_varn
        for (j = 0; j < 4; j++) {
            for (i = 0; i < NProc; i++) {
                count[i][0] = 1;
                count[i][1] = 1;
                start[i][0] = (MyRank + i) % NProc;
                start[i][1] = i + ((j + 3) % 4) * NProc;
                sp[i] = (MPI_Offset*)start[i];
                cp[i] = (MPI_Offset*)count[i];
            }
            ret = ncmpi_put_varn_int_all(fid, vid[j], NProc, sp, cp, buffer);
            if (ret != NC_NOERR) {
                printf("Error put_varn\n");
                goto ERROR;
            }
        }

        // Commit log into cdf file

        ret = ncmpi_close(fid);       // Close file
        if (ret != NC_NOERR) {
            printf("Error close");
            goto ERROR;
        }
    }

ERROR:;
    return 0;
}
Esempio n. 3
0
// Traj_NcEnsemble::writeArray() // TODO RemdValues
int Traj_NcEnsemble::writeArray(int set, FramePtrArray const& Farray) {
# ifdef HAS_PNETCDF
  MPI_Offset pstart_[4];
  MPI_Offset pcount_[4];
# define start_ pstart_
# define count_ pcount_
# endif
  start_[0] = ncframe_; // Frame
  start_[2] = 0;        // Atoms
  start_[3] = 0;        // XYZ
  count_[0] = 1; // Frame
  count_[1] = 1; // Ensemble
  count_[3] = 3; // XYZ
  for (int member = ensembleStart_; member != ensembleEnd_; member++) {
    //rprintf("DEBUG: Writing set %i, member %i\n", set+1, member); 
#   ifdef MPI
    Frame* frm = Farray[0];
#   else
    Frame* frm = Farray[member];
#   endif
    start_[1] = member;   // Ensemble
    count_[2] = Ncatom(); // Atoms
    // Write Coords
    //DebugIndices(); // DEBUG
    DoubleToFloat(Coord_, frm->xAddress());
#   ifdef HAS_PNETCDF
    if (ncmpi_put_vara_float_all(ncid_, coordVID_, start_, count_, Coord_))
#   else
    if (NC::CheckErr(nc_put_vara_float(ncid_, coordVID_, start_, count_, Coord_)))
#   endif
    {
      mprinterr("Error: Netcdf Writing coords frame %i\n", set+1);
      return 1;
    }
    // Write velocity.
    if (velocityVID_ != -1) {
      DoubleToFloat(Coord_, frm->vAddress());
#     ifdef HAS_PNETCDF
      if (ncmpi_put_vara_float_all(ncid_, velocityVID_, start_, count_, Coord_))
#     else
      if (NC::CheckErr(nc_put_vara_float(ncid_, velocityVID_, start_, count_, Coord_)) )
#     endif
      {
        mprinterr("Error: Netcdf writing velocity frame %i\n", set+1);
        return 1;
      }
    }
    // Write box
    if (cellLengthVID_ != -1) {
      count_[2] = 3;
#     ifdef HAS_PNETCDF
      if (ncmpi_put_vara_double_all(ncid_,cellLengthVID_,start_,count_,frm->bAddress()))
#     else
      if (NC::CheckErr(nc_put_vara_double(ncid_,cellLengthVID_,start_,count_,frm->bAddress())) )
#     endif
      {
        mprinterr("Error: Writing cell lengths frame %i.\n", set+1);
        return 1;
      }
#     ifdef HAS_PNETCDF
      if (ncmpi_put_vara_double_all(ncid_,cellAngleVID_,start_,count_,frm->bAddress()+3))
#     else
      if (NC::CheckErr(nc_put_vara_double(ncid_,cellAngleVID_,start_,count_,frm->bAddress()+3)))
#     endif
      {
        mprinterr("Error: Writing cell angles frame %i.\n", set+1);
        return 1;
      }
    }
    // Write temperature
    if (TempVID_!=-1) {
#     ifdef HAS_PNETCDF
      if (ncmpi_put_vara_double_all(ncid_,TempVID_,start_,count_,frm->tAddress()))
#     else
      if (NC::CheckErr(nc_put_vara_double(ncid_,TempVID_,start_,count_,frm->tAddress())))
#     endif
      {
        mprinterr("Error: Writing temperature frame %i.\n", set+1);
        return 1;
      }
    }
    // Write indices
    if (indicesVID_ != -1) {
      count_[2] = remd_dimension_;
#     ifdef HAS_PNETCDF
      if (ncmpi_put_vara_int_all(ncid_,indicesVID_,start_,count_,frm->iAddress()))
#     else
      if (NC::CheckErr(nc_put_vara_int(ncid_,indicesVID_,start_,count_,frm->iAddress())))
#     endif
      {
        mprinterr("Error: Writing indices frame %i.\n", set+1);
        return 1;
      }
    }
  }
# ifdef HAS_PNETCDF
  //ncmpi_sync(ncid_);
# else
  nc_sync(ncid_); // Necessary after every write??
# endif
  ++ncframe_;
# ifdef HAS_PNETCDF
  // DEBUG
# undef start_
# undef count_
# endif
  return 0;
}
Esempio n. 4
0
int main(int argc, char **argv) {

  int i, j;
  int status;
  int ncid1, ncid2;
  int ndims, nvars, ngatts, unlimdimid;
  char name[NC_MAX_NAME];
  nc_type type, vartypes[NC_MAX_VARS];
  MPI_Offset attlen;
  MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS];
  void *valuep;
  int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS];
  int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */
  int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS];
  int isRecvar;
  params opts;

  int rank;
  int nprocs;
  MPI_Comm comm = MPI_COMM_WORLD;
  

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);

  if (rank == 0)
	  fprintf(stderr, "Testing read ... ");
  parse_read_args(argc, argv, rank, &opts);

  /**********  START OF NETCDF ACCESS **************/


  /* Read a netCDF file and write it out to another file */

  /**
   * Open the input dataset - ncid1:
   *   File name: "../data/test_float.nc"
   *   Dataset API: Collective
   * And create the output dataset - ncid2:
   *   File name: "testread.nc"
   *   Dataset API: Collective
   */

  status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1);
  if (status != NC_NOERR) handle_error(status);

  status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2);
  if (status != NC_NOERR) handle_error(status);


  /**
   * Inquire the dataset definitions of input dataset AND
   * Add dataset definitions for output dataset.
   */

  status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid);
  if (status != NC_NOERR) handle_error(status);


  /* Inquire global attributes, assume CHAR attributes. */

  for (i = 0; i < ngatts; i++) {
    status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name);
    if (status != NC_NOERR) handle_error(status);
    status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen);
    if (status != NC_NOERR) handle_error(status);
    switch (type) {
      case NC_CHAR: 
	valuep = (void *)malloc(attlen * sizeof(char));
	status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep);
	if (status != NC_NOERR) handle_error(status);
	status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep);
	if (status != NC_NOERR) handle_error(status);
	free(valuep);
        break;
      case NC_SHORT:
        valuep = (void *)malloc(attlen * sizeof(short));
        status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      case NC_INT:
        valuep = (void *)malloc(attlen * sizeof(int));
        status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      case NC_FLOAT:
        valuep = (void *)malloc(attlen * sizeof(float));
        status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      case NC_DOUBLE:
        valuep = (void *)malloc(attlen * sizeof(double));
        status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      default:
	;
	/* TODO: handle unexpected types */
    }
  }

  /* Inquire dimension */

  for (i = 0; i < ndims; i++) {
    status = ncmpi_inq_dim(ncid1, i, name, &dimlen);
    if (status != NC_NOERR) handle_error(status);
    if (i == unlimdimid)
      dimlen = NC_UNLIMITED;
    status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i);
    if (status != NC_NOERR) handle_error(status);
  }

  /* Inquire variables */

  for (i = 0; i < nvars; i++) {
    status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i);
    if (status != NC_NOERR) handle_error(status);

    status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i);
    if (status != NC_NOERR) handle_error(status);

    /* var attributes, assume CHAR attributes */

    for (j = 0; j < varnatts[i]; j++) {
      status = ncmpi_inq_attname(ncid1, i, j, name);
      if (status != NC_NOERR) handle_error(status);
      status = ncmpi_inq_att (ncid1, i, name, &type, &attlen);
      if (status != NC_NOERR) handle_error(status);
      switch (type) {
        case NC_CHAR: 
	  valuep = (void *)malloc(attlen * sizeof(char));
	  status = ncmpi_get_att_text(ncid1, i, name, valuep);
	  if (status != NC_NOERR) handle_error(status);
	  status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep);
	  if (status != NC_NOERR) handle_error(status);
	  free(valuep);
          break;
        case NC_SHORT:
          valuep = (void *)malloc(attlen * sizeof(short));
          status = ncmpi_get_att_short(ncid1, i, name, valuep);
          if (status != NC_NOERR) handle_error(status);
          status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep);
          if (status != NC_NOERR) handle_error(status);
          free(valuep);
          break;
        case NC_INT:
          valuep = (void *)malloc(attlen * sizeof(int));
          status = ncmpi_get_att_int(ncid1, i, name, valuep);
          if (status != NC_NOERR) handle_error(status);
          status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep);
          if (status != NC_NOERR) handle_error(status);
          free(valuep);
          break;
        case NC_FLOAT:
          valuep = (void *)malloc(attlen * sizeof(float));
          status = ncmpi_get_att_float(ncid1, i, name, valuep);
          if (status != NC_NOERR) handle_error(status);
          status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep);
          if (status != NC_NOERR) handle_error(status);
          free(valuep);
          break;
        case NC_DOUBLE:
          valuep = (void *)malloc(attlen * sizeof(double));
          status = ncmpi_get_att_double(ncid1, i, name, valuep);
          if (status != NC_NOERR) handle_error(status);
          status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep);
          if (status != NC_NOERR) handle_error(status);
          free(valuep);
          break;
	default:
	  ; /* TODO: handle unexpected types */
      }
    }
  }

  /**
   * End Define Mode (switch to data mode) for output dataset
   *   Dataset API: Collective
   */

  status = ncmpi_enddef(ncid2);
  if (status != NC_NOERR) handle_error(status);

  /**
   * Read data of variables from input dataset 
   * (ONLY DEAL WITH: NC_INT, NC_FLOAT, NC_DOUBLE for now)
   * Write the data out to the corresponding variables in the output dataset
   *
   *  Data Partition (Assume 4 processors):
   *   square: 2-D, (Block, *), 25*100 from 100*100
   *   cube:   3-D, (Block, *, *), 25*100*100 from 100*100*100
   *   xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100
   *   time:   1-D, Block-wise, 25 from 100
   *
   *  Data Mode API: collective
   */

  for (i = 0; i < NC_MAX_VAR_DIMS; i++)
    start[i] = 0;
  for (i = 0; i < nvars; i++) {
    isRecvar = 0;
    varsize = 1;
    for (j = 0; j < varndims[i]; j++) {
      status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j);
      if (status != NC_NOERR) handle_error(status);
      if (j == 0) {
        shape[j] /= nprocs;
	start[j] = shape[j] * rank;
      }
      varsize *= shape[j];
      if (vardims[i][j] == unlimdimid)
	isRecvar = 1;
    }
    switch (vartypes[i]) {
      case NC_CHAR: 
        break;
      case NC_SHORT:
        valuep = (void *)malloc(varsize * sizeof(short));
        status = ncmpi_get_vara_short_all(ncid1, i, start, shape, (short *)valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_vara_short_all(ncid2, varids[i],
                                     start, shape, (short *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      case NC_INT:
	valuep = (void *)malloc(varsize * sizeof(int));
        status = ncmpi_get_vara_int_all(ncid1, i, start, shape, (int *)valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_vara_int_all(ncid2, varids[i],
                                     start, shape, (int *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
	break;
      case NC_FLOAT:
        valuep = (void *)malloc(varsize * sizeof(float));
        status = ncmpi_get_vara_float_all(ncid1, i, start, shape, (float *)valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_vara_float_all(ncid2, varids[i],
                                     start, shape, (float *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      case NC_DOUBLE:
        valuep = (void *)malloc(varsize * sizeof(double));
        status = ncmpi_get_vara_double_all(ncid1, i, start, shape, (double *)valuep);
        if (status != NC_NOERR) handle_error(status);
        status = ncmpi_put_vara_double_all(ncid2, varids[i],
                                     start, shape, (double *)valuep);
        if (status != NC_NOERR) handle_error(status);
        free(valuep);
        break;
      default:
	; /* TODO: handle unexpected types */
    }
  }

  /**
   * Close the datasets
   *   Dataset API:  collective
   */

  status = ncmpi_close(ncid1);
  if (status != NC_NOERR) handle_error(status);
  status = ncmpi_close(ncid2);
  if (status != NC_NOERR) handle_error(status);

  /*******************  END OF NETCDF ACCESS  ****************/

if (rank == 0)
  fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname);

  MPI_Finalize();
  return 0;
}
Esempio n. 5
0
/* write out variable's data from in-memory structure */
void
load_netcdf(void *rec_start)
{
    int i, idim;
    int stat = NC_NOERR;
    MPI_Offset *start, *count;
    char *charvalp = NULL;
    short *shortvalp = NULL;
    int *intvalp = NULL;
    float *floatvalp = NULL;
    double *doublevalp = NULL;
    unsigned char *ubytevalp = NULL;
    unsigned short *ushortvalp = NULL;
    unsigned int *uintvalp = NULL;
    long long *int64valp = NULL;
    unsigned long long *uint64valp = NULL;
    MPI_Offset total_size;

    /* load values into variable */

    switch (vars[varnum].type) {
      case NC_CHAR:
      case NC_BYTE:
	charvalp = (char *) rec_start;
	break;
      case NC_SHORT:
	shortvalp = (short *) rec_start;
	break;
      case NC_INT:
	intvalp = (int *) rec_start;
	break;
      case NC_FLOAT:
	floatvalp = (float *) rec_start;
	break;
      case NC_DOUBLE:
	doublevalp = (double *) rec_start;
	break;
      case NC_UBYTE:
	ubytevalp = (unsigned char *) rec_start;
	break;
      case NC_USHORT:
	ushortvalp = (unsigned short *) rec_start;
	break;
      case NC_UINT:
	uintvalp = (unsigned int *) rec_start;
	break;
      case NC_INT64:
	int64valp = (long long *) rec_start;
	break;
      case NC_UINT64:
	uint64valp = (unsigned long long *) rec_start;
	break;
      default:
	derror("Unhandled type %d\n", vars[varnum].type);
	break;
    }

    start = (MPI_Offset*) malloc(vars[varnum].ndims * 2 * sizeof(MPI_Offset));
    count = start + vars[varnum].ndims;

    if (vars[varnum].ndims > 0) {
	/* initialize start to upper left corner (0,0,0,...) */
	start[0] = 0;
	if (vars[varnum].dims[0] == rec_dim) {
	    count[0] = vars[varnum].nrecs;
	}
	else {
	    count[0] = dims[vars[varnum].dims[0]].size;
	}
    }

    for (idim = 1; idim < vars[varnum].ndims; idim++) {
	start[idim] = 0;
	count[idim] = dims[vars[varnum].dims[idim]].size;
    }

    total_size = nctypesize(vars[varnum].type);
    for (idim=0; idim<vars[varnum].ndims; idim++)
        total_size *= count[idim];

    /* If the total put size is more than 2GB, then put one subarray at a time.
     * Here the subarray is from 1, 2, ... ndims, except 0.
     * This is not a perfect solution. To be improved.
     */
    if (total_size > INT_MAX) {
        MPI_Offset nchunks=count[0];
        MPI_Offset subarray_nelems=1;
        for (idim=1; idim<vars[varnum].ndims; idim++)
            subarray_nelems *= count[idim];

        count[0] = 1;
        switch (vars[varnum].type) {
            case NC_BYTE:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_schar_all(ncid, varnum, start, count, (signed char *)charvalp);
                     check_err(stat, "ncmpi_put_vara_schar_all", __func__, __LINE__, __FILE__);
                     charvalp += subarray_nelems;
                 }
                 break;
            case NC_CHAR:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_text_all(ncid, varnum, start, count, charvalp);
                     check_err(stat, "ncmpi_put_vara_text_all", __func__, __LINE__, __FILE__);
                     charvalp += subarray_nelems;
                 }
                 break;
            case NC_SHORT:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_short_all(ncid, varnum, start, count, shortvalp);
                     check_err(stat, "ncmpi_put_vara_short_all", __func__, __LINE__, __FILE__);
                     shortvalp += subarray_nelems;
                 }
                 break;
            case NC_INT:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_int_all(ncid, varnum, start, count, intvalp);
                     check_err(stat, "ncmpi_put_vara_int_all", __func__, __LINE__, __FILE__);
                     intvalp += subarray_nelems;
                 }
                 break;
            case NC_FLOAT:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_float_all(ncid, varnum, start, count, floatvalp);
                     check_err(stat, "ncmpi_put_vara_float_all", __func__, __LINE__, __FILE__);
                     floatvalp += subarray_nelems;
                 }
                 break;
            case NC_DOUBLE:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_double_all(ncid, varnum, start, count, doublevalp);
                     check_err(stat, "ncmpi_put_vara_double_all", __func__, __LINE__, __FILE__);
                     doublevalp += subarray_nelems;
                 }
                 break;
            case NC_UBYTE:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_uchar_all(ncid, varnum, start, count, ubytevalp);
                     check_err(stat, "ncmpi_put_vara_uchar_all", __func__, __LINE__, __FILE__);
                     ubytevalp += subarray_nelems;
                 }
                 break;
            case NC_USHORT:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_ushort_all(ncid, varnum, start, count, ushortvalp);
                     check_err(stat, "ncmpi_put_vara_ushort_all", __func__, __LINE__, __FILE__);
                     ushortvalp += subarray_nelems;
                 }
                 break;
            case NC_UINT:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_uint_all(ncid, varnum, start, count, uintvalp);
                     check_err(stat, "ncmpi_put_vara_uint_all", __func__, __LINE__, __FILE__);
                     uintvalp += subarray_nelems;
                 }
                 break;
            case NC_INT64:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_longlong_all(ncid, varnum, start, count, int64valp);
                     check_err(stat, "ncmpi_put_vara_longlong_all", __func__, __LINE__, __FILE__);
                     int64valp += subarray_nelems;
                 }
                 break;
            case NC_UINT64:
                 for (i=0; i<nchunks; i++) {
                     start[0] = i;
                     stat = ncmpi_put_vara_ulonglong_all(ncid, varnum, start, count, uint64valp);
                     check_err(stat, "ncmpi_put_vara_ulonglong_all", __func__, __LINE__, __FILE__);
                     uint64valp += subarray_nelems;
                 }
                 break;
            default:
                     derror("Unhandled type %d\n", vars[varnum].type);
                     break;
        }
    }
    else {
        switch (vars[varnum].type) {
            case NC_BYTE:
                stat = ncmpi_put_vara_schar_all(ncid, varnum, start, count, (signed char *)charvalp);
                check_err(stat, "ncmpi_put_vara_schar_all", __func__, __LINE__, __FILE__);
                break;
            case NC_CHAR:
                stat = ncmpi_put_vara_text_all(ncid, varnum, start, count, charvalp);
                check_err(stat, "ncmpi_put_vara_text_all", __func__, __LINE__, __FILE__);
                break;
            case NC_SHORT:
                stat = ncmpi_put_vara_short_all(ncid, varnum, start, count, shortvalp);
                check_err(stat, "ncmpi_put_vara_short_all", __func__, __LINE__, __FILE__);
                break;
            case NC_INT:
                stat = ncmpi_put_vara_int_all(ncid, varnum, start, count, intvalp);
                check_err(stat, "ncmpi_put_vara_int_all", __func__, __LINE__, __FILE__);
                break;
            case NC_FLOAT:
                stat = ncmpi_put_vara_float_all(ncid, varnum, start, count, floatvalp);
                check_err(stat, "ncmpi_put_vara_float_all", __func__, __LINE__, __FILE__);
                break;
            case NC_DOUBLE:
                stat = ncmpi_put_vara_double_all(ncid, varnum, start, count, doublevalp);
                check_err(stat, "ncmpi_put_vara_double_all", __func__, __LINE__, __FILE__);
                break;
            case NC_UBYTE:
                stat = ncmpi_put_vara_uchar_all(ncid, varnum, start, count, ubytevalp);
                check_err(stat, "ncmpi_put_vara_uchar_all", __func__, __LINE__, __FILE__);
                break;
            case NC_USHORT:
                stat = ncmpi_put_vara_ushort_all(ncid, varnum, start, count, ushortvalp);
                check_err(stat, "ncmpi_put_vara_ushort_all", __func__, __LINE__, __FILE__);
                break;
            case NC_UINT:
                stat = ncmpi_put_vara_uint_all(ncid, varnum, start, count, uintvalp);
                check_err(stat, "ncmpi_put_vara_uint_all", __func__, __LINE__, __FILE__);
                break;
            case NC_INT64:
                stat = ncmpi_put_vara_longlong_all(ncid, varnum, start, count, int64valp);
                check_err(stat, "ncmpi_put_vara_longlong_all", __func__, __LINE__, __FILE__);
                break;
            case NC_UINT64:
                stat = ncmpi_put_vara_ulonglong_all(ncid, varnum, start, count, uint64valp);
                check_err(stat, "ncmpi_put_vara_ulonglong_all", __func__, __LINE__, __FILE__);
                break;
            default:
                derror("Unhandled type %d\n", vars[varnum].type);
                break;
        }
    }
    free(start);
}
int main(int argc, char **argv) {
    int ret, ncfile, nprocs, rank, dimid, varid1, varid2, ndims=1;
    MPI_Offset start, count=1;
    char buf[13] = "Hello World\n";
    int *data;

    MPI_Init(&argc, &argv);

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    if (argc != 2) {
        if (rank == 0) printf("Usage: %s filename\n", argv[0]);
        MPI_Finalize();
        exit(-1);
    }

    if (rank == 0) {
        ret = ncmpi_create(MPI_COMM_SELF, argv[1],
                           NC_CLOBBER|NC_64BIT_OFFSET, MPI_INFO_NULL, &ncfile);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_def_dim(ncfile, "d1", nprocs, &dimid);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_def_var(ncfile, "v1", NC_INT, ndims, &dimid, &varid1);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_def_var(ncfile, "v2", NC_INT, ndims, &dimid, &varid2);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_put_att_text(ncfile, NC_GLOBAL, "string", 13, buf);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);
        
        ret = ncmpi_enddef(ncfile);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        /* first reason this approach is not scalable:  need to allocate
        * enough memory to hold data from all processors */
        data = calloc(nprocs, sizeof(int));
    }

    /* second reason this approch is not scalable: sending to rank 0
     * introduces a serialization point, even if using an optimized
     * collective routine */
    MPI_Gather(&rank, 1, MPI_INT, data, 1, MPI_INT, 0, MPI_COMM_WORLD);

    if (rank == 0) {
        /* and lastly, the third reason this approach is not scalable: I/O
         * happens from a single processor.  This approach can be ok if the
         * amount of data is quite small, but almost always the underlying
         * MPI-IO library can do a better job */
        start=0, count=nprocs;
        ret = ncmpi_put_vara_int_all(ncfile, varid1, &start, &count, data);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_put_vara_int_all(ncfile, varid2, &start, &count, data);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);

        ret = ncmpi_close(ncfile);
        if (ret != NC_NOERR) handle_error(ret, __LINE__);
    }

    MPI_Finalize();
    return 0;
}
int main(int argc, char **argv)
{
    char filename[256];
    int err, nerrs=0, ncid, dimid[NDIMS], varid[5], ndims=NDIMS;
    int i, j, k, nprocs, rank, req, *buf;
    MPI_Offset start[NDIMS] = {0};
    MPI_Offset count[NDIMS] = {0};
    MPI_Offset stride[NDIMS] = {0};

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    if (argc > 2) {
        if (!rank) printf("Usage: %s [filename]\n",argv[0]);
        MPI_Finalize();
        return 1;
    }
    if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
    else           strcpy(filename, "testfile.nc");

    if (rank == 0) {
        char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
        sprintf(cmd_str, "*** TESTING C   %s for NULL stride ", basename(argv[0]));
        printf("%-66s ------ ", cmd_str); fflush(stdout);
        free(cmd_str);
    }

    err = ncmpi_create(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid);
    CHECK_ERR

    err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
    CHECK_ERR

    err = ncmpi_def_dim(ncid, "X", nprocs*NX, &dimid[1]);
    CHECK_ERR

    err = ncmpi_def_var(ncid, "v0", NC_INT, ndims, dimid, &varid[0]);
    CHECK_ERR

    err = ncmpi_def_var(ncid, "v1", NC_INT, ndims, dimid, &varid[1]);
    CHECK_ERR

    err = ncmpi_def_var(ncid, "v2", NC_INT, ndims, dimid, &varid[2]);
    CHECK_ERR

    err = ncmpi_def_var(ncid, "v3", NC_INT, ndims, dimid, &varid[3]);
    CHECK_ERR

    err = ncmpi_def_var(ncid, "v4", NC_INT, ndims, dimid, &varid[4]);
    CHECK_ERR

    err = ncmpi_enddef(ncid);
    CHECK_ERR

    start[0] = 0;
    start[1] = rank*NX;
    count[0] = NY;
    count[1] = NX;
    buf = (int*) malloc((size_t)NY * NX * sizeof(int));
    for (i=0; i<NY*NX; i++) buf[i] = rank+10;

    err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf);
    CHECK_ERR
    CHECK_PUT_BUF

    err = ncmpi_put_vars_int_all(ncid, varid[1], start, count, NULL, buf);
    CHECK_ERR
    CHECK_PUT_BUF

    start[0] = 0;
    start[1] = rank;
    count[0] = NY;
    count[1] = NX;
    stride[0] = 1;
    stride[1] = nprocs;
    err = ncmpi_put_vars_int_all(ncid, varid[2], start, count, stride, buf);
    CHECK_ERR
    CHECK_PUT_BUF

    /* test bput_vars */
    err = ncmpi_buffer_attach(ncid, NY*NX*sizeof(int));
    CHECK_ERR

    start[0] = 0;
    start[1] = rank*NX;
    count[0] = NY;
    count[1] = NX;
    err = ncmpi_bput_vars_int(ncid, varid[3], start, count, NULL, buf, &req);
    CHECK_ERR

    err = ncmpi_wait_all(ncid, 1, &req, NULL);
    CHECK_ERR
    CHECK_PUT_BUF

    start[0] = 0;
    start[1] = rank;
    count[0] = NY;
    count[1] = NX;
    stride[0] = 1;
    stride[1] = nprocs;
    err = ncmpi_bput_vars_int(ncid, varid[4], start, count, stride, buf, &req);
    CHECK_ERR

    err = ncmpi_wait_all(ncid, 1, &req, NULL);
    CHECK_ERR
    CHECK_PUT_BUF
    free(buf);

    err = ncmpi_buffer_detach(ncid);
    CHECK_ERR

    buf = (int*) malloc((size_t)NY * NX * nprocs * sizeof(int));
    memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
    err = ncmpi_get_var_int_all(ncid, varid[0], buf);
    CHECK_ERR

    /* check read buffer contents */
    /*  v0 =
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13 ;
     */
    for (i=0; i<NY; i++) {
        for (j=0; j<nprocs; j++) {
            for (k=0; k<NX; k++) {
                if (buf[i*nprocs*NX+j*NX+k] != j+10) {
                    printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
                           __LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
                    nerrs++;
                }
            }
        }
    }

    memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
    err = ncmpi_get_var_int_all(ncid, varid[1], buf);
    CHECK_ERR

    /* check read buffer contents */
    /*  v1 =
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13 ;
     */
    for (i=0; i<NY; i++) {
        for (j=0; j<nprocs; j++) {
            for (k=0; k<NX; k++) {
                if (buf[i*nprocs*NX+j*NX+k] != j+10) {
                    printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
                           __LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
                    nerrs++;
                }
            }
        }
    }

    memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
    err = ncmpi_get_var_int_all(ncid, varid[2], buf);
    CHECK_ERR

    /* check read buffer contents */
    /*  v2 =
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13 ;
     */
    for (i=0; i<NY; i++) {
        for (k=0; k<NX; k++) {
            for (j=0; j<nprocs; j++) {
                if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
                    printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
                           __LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
                    nerrs++;
                }
            }
        }
    }

    memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
    err = ncmpi_get_var_int_all(ncid, varid[3], buf);
    CHECK_ERR

    /* check read buffer contents */
    /*  v3 =
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13,
     *    10, 10, 11, 11, 12, 12, 13, 13 ;
     */
    for (i=0; i<NY; i++) {
        for (j=0; j<nprocs; j++) {
            for (k=0; k<NX; k++) {
                if (buf[i*nprocs*NX+j*NX+k] != j+10) {
                    printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
                           __LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
                    nerrs++;
                }
            }
        }
    }

    memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
    err = ncmpi_get_var_int_all(ncid, varid[4], buf);
    CHECK_ERR

    /* check read buffer contents */
    /*  v4 =
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13,
     *    10, 11, 12, 13, 10, 11, 12, 13 ;
     */
    for (i=0; i<NY; i++) {
        for (k=0; k<NX; k++) {
            for (j=0; j<nprocs; j++) {
                if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
                    printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
                           __LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
                    nerrs++;
                }
            }
        }
    }

    err = ncmpi_close(ncid);
    CHECK_ERR

    free(buf);

    /* check if PnetCDF freed all internal malloc */
    MPI_Offset malloc_size, sum_size;
    err = ncmpi_inq_malloc_size(&malloc_size);
    if (err == NC_NOERR) {
        MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
        if (rank == 0 && sum_size > 0)
            printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
                   sum_size);
        if (malloc_size > 0) ncmpi_inq_malloc_list();
    }

    MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
    if (rank == 0) {
        if (nerrs) printf(FAIL_STR,nerrs);
        else       printf(PASS_STR);
    }

    MPI_Finalize();
    return (nerrs > 0);
}
Esempio n. 8
0
int main(int argc, char** argv)
{
    extern int optind;
    char filename[256];
    int i, j, rank, nprocs, verbose=1, err, nerrs=0;
    int ncid, cmode, varid, dimid[2], buf[NY][NX];
    char str_att[128];
    float float_att[100];
    MPI_Offset  global_ny, global_nx;
    MPI_Offset start[2], count[2];

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    /* get command-line arguments */
    while ((i = getopt(argc, argv, "hq")) != EOF)
        switch(i) {
            case 'q': verbose = 0;
                      break;
            case 'h':
            default:  if (rank==0) usage(argv[0]);
                      MPI_Finalize();
                      return 1;
        }
    if (argv[optind] == NULL) strcpy(filename, "testfile.nc");
    else                      snprintf(filename, 256, "%s", argv[optind]);

    MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);

    if (verbose && rank == 0) printf("%s: example of using put_vara APIs\n",__FILE__);

    /* create a new file for writing ----------------------------------------*/
    cmode = NC_CLOBBER | NC_64BIT_DATA;
    err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid);
    ERR

    /* the global array is NY * (NX * nprocs) */
    global_ny = NY;
    global_nx = NX * nprocs;

    for (i=0; i<NY; i++)
        for (j=0; j<NX; j++)
             buf[i][j] = rank;

    /* add a global attribute: a time stamp at rank 0 */
    time_t ltime = time(NULL); /* get the current calendar time */
    asctime_r(localtime(&ltime), str_att);

    /* make sure the time string are consistent among all processes */
    MPI_Bcast(str_att, strlen(str_att), MPI_CHAR, 0, MPI_COMM_WORLD);

    err = ncmpi_put_att_text(ncid, NC_GLOBAL, "history", strlen(str_att),
                             &str_att[0]);
    ERR

    /* define dimensions x and y */
    err = ncmpi_def_dim(ncid, "Y", global_ny, &dimid[0]);
    ERR
    err = ncmpi_def_dim(ncid, "X", global_nx, &dimid[1]);
    ERR

    /* define a 2D variable of integer type */
    err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid);
    ERR

    /* add attributes to the variable */
    strcpy(str_att, "example attribute of type text.");
    err = ncmpi_put_att_text(ncid, varid, "str_att_name", strlen(str_att),
                             &str_att[0]);
    ERR

    for (i=0; i<8; i++) float_att[i] = i;
    err = ncmpi_put_att_float(ncid, varid, "float_att_name", NC_FLOAT, 8,
                              &float_att[0]);
    ERR
    long long int64_att=10000000000LL;
    err = ncmpi_put_att_longlong(ncid, varid, "int64_att_name", NC_INT64, 1,
                              &int64_att);
    ERR

    /* do not forget to exit define mode */
    err = ncmpi_enddef(ncid);
    ERR

    /* now we are in data mode */
    start[0] = 0;
    start[1] = NX * rank;
    count[0] = NY;
    count[1] = NX;

    err = ncmpi_put_vara_int_all(ncid, varid, start, count, &buf[0][0]);
    ERR

    err = ncmpi_close(ncid);
    ERR

    /* check if there is any PnetCDF internal malloc residue */
    MPI_Offset malloc_size, sum_size;
    err = ncmpi_inq_malloc_size(&malloc_size);
    if (err == NC_NOERR) {
        MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
        if (rank == 0 && sum_size > 0)
            printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
                   sum_size);
    }

    MPI_Finalize();
    return (nerrs > 0);
}
Esempio n. 9
0
/*----< main() >------------------------------------------------------------*/
int main(int argc, char **argv)
{
    int i, j, err, nerrs=0, rank, nprocs;
    int ncid, dimid[2], varid, req, status;

    MPI_Offset start[2], count[2], stride[2], imap[2];
    int   var[6][4];
    float k, rh[4][6];
    signed char  varT[4][6];
    char filename[256];

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    if (argc > 2) {
        if (!rank) printf("Usage: %s [filename]\n",argv[0]);
        MPI_Finalize();
        return 1;
    }
    if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
    else           strcpy(filename, "testfile.nc");

    if (rank == 0) {
        char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
        sprintf(cmd_str, "*** TESTING C   %s for get/put varm ", basename(argv[0]));
        printf("%-66s ------ ", cmd_str); fflush(stdout);
        free(cmd_str);
    }

#ifdef DEBUG
    if (nprocs > 1 && rank == 0)
        printf("Warning: %s is designed to run on 1 process\n", argv[0]);
#endif

    err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA,
                       MPI_INFO_NULL, &ncid); CHECK_ERR

    /* define a variable of a 6 x 4 integer array in the nc file */
    err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR
    err = ncmpi_def_dim(ncid, "X", 4, &dimid[1]); CHECK_ERR
    err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR
    err = ncmpi_enddef(ncid); CHECK_ERR

    /* create a 6 x 4 integer variable in the file with contents:
           0,  1,  2,  3,
           4,  5,  6,  7,
           8,  9, 10, 11,
          12, 13, 14, 15,
          16, 17, 18, 19,
          20, 21, 22, 23
     */
    for (j=0; j<6; j++) for (i=0; i<4; i++) var[j][i] = j*4+i;

    start[0] = 0; start[1] = 0;
    count[0] = 6; count[1] = 4;
    if (rank > 0) count[0] = count[1] = 0;
    err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR

    if (nprocs > 1) MPI_Barrier(MPI_COMM_WORLD);

    err = ncmpi_close(ncid); CHECK_ERR

    err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR

    err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR

    /* read the variable back in the matrix transposed way, rh is 4 x 6 */
     count[0] = 6;  count[1] = 4;
    stride[0] = 1; stride[1] = 1;
      imap[0] = 1;   imap[1] = 6;   /* would be {4, 1} if not transposing */

    for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;

    err = ncmpi_iget_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0], &req); CHECK_ERR

    err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
    err = status; CHECK_ERR

    /* check the contents of read */
    k = 0.0;
    for (i=0; i<6; i++) {
        for (j=0; j<4; j++) {
            if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
                printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
                __LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
                nerrs++;
                break;
            }
            k += 1.0;
        }
    }
#ifdef PRINT_ON_SCREEN
    /* print the contents of read */
    for (j=0; j<4; j++) {
        printf("[%2d]: ",j);
        for (i=0; i<6; i++) {
            printf("%5.1f",rh[j][i]);
        }
        printf("\n");
    }
#endif
    /* the stdout should be:
           [ 0]:   0.0  4.0  8.0 12.0 16.0 20.0
           [ 1]:   1.0  5.0  9.0 13.0 17.0 21.0
           [ 2]:   2.0  6.0 10.0 14.0 18.0 22.0
           [ 3]:   3.0  7.0 11.0 15.0 19.0 23.0
     */

    for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;

    err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); CHECK_ERR

    /* check the contents of read */
    k = 0.0;
    for (i=0; i<6; i++) {
        for (j=0; j<4; j++) {
            if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
                printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
                __LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
                nerrs++;
                break;
            }
            k += 1.0;
        }
    }
#ifdef PRINT_ON_SCREEN
    /* print the contents of read */
    for (j=0; j<4; j++) {
        printf("[%2d]: ",j);
        for (i=0; i<6; i++) {
            printf("%5.1f",rh[j][i]);
        }
        printf("\n");
    }
#endif
    /* the stdout should be:
           [ 0]:   0.0  4.0  8.0 12.0 16.0 20.0
           [ 1]:   1.0  5.0  9.0 13.0 17.0 21.0
           [ 2]:   2.0  6.0 10.0 14.0 18.0 22.0
           [ 3]:   3.0  7.0 11.0 15.0 19.0 23.0
     */


    err = ncmpi_close(ncid); CHECK_ERR

    err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR

    err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR

    /* testing get_varm(), first zero-out the variable in the file */
    memset(&var[0][0], 0, 6*4*sizeof(int));
    start[0] = 0; start[1] = 0;
    count[0] = 6; count[1] = 4;
    if (rank > 0) count[0] = count[1] = 0;
    err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR

    /* set the contents of the write buffer varT, a 4 x 6 char array
          50, 51, 52, 53, 54, 55,
          56, 57, 58, 59, 60, 61,
          62, 63, 64, 65, 66, 67,
          68, 69, 70, 71, 72, 73
     */
    for (j=0; j<4; j++) for (i=0; i<6; i++) varT[j][i] = j*6+i + 50;

    /* write varT to the NC variable in the matrix transposed way */
    start[0]  = 0; start[1]  = 0;
    count[0]  = 6; count[1]  = 4;
    stride[0] = 1; stride[1] = 1;
    imap[0]   = 1; imap[1]   = 6;   /* would be {4, 1} if not transposing */
    if (rank > 0) count[0] = count[1] = 0;

    err = ncmpi_iput_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0], &req); CHECK_ERR

    err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
    err = status; CHECK_ERR

    /* the output from command "ncmpidump -v var test.nc" should be:
           var =
            50, 56, 62, 68,
            51, 57, 63, 69,
            52, 58, 64, 70,
            53, 59, 65, 71,
            54, 60, 66, 72,
            55, 61, 67, 73 ;
     */

    /* check if the contents of write buffer have been altered */
    for (j=0; j<4; j++) {
        for (i=0; i<6; i++) {
            if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
                /* this error is a pnetcdf internal error, if occurs */
                printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
                __LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
                nerrs++;
                break;
            }
        }
    }
    err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); CHECK_ERR

    /* check if the contents of write buffer have been altered */
    for (j=0; j<4; j++) {
        for (i=0; i<6; i++) {
            if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
                /* this error is a pnetcdf internal error, if occurs */
                printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
                __LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
                nerrs++;
                break;
            }
        }
    }

    err = ncmpi_close(ncid); CHECK_ERR

    /* check if PnetCDF freed all internal malloc */
    MPI_Offset malloc_size, sum_size;
    err = ncmpi_inq_malloc_size(&malloc_size);
    if (err == NC_NOERR) {
        MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
        if (rank == 0 && sum_size > 0)
            printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
                   sum_size);
        if (malloc_size > 0) ncmpi_inq_malloc_list();
    }

    MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
    if (rank == 0) {
        if (nerrs) printf(FAIL_STR,nerrs);
        else       printf(PASS_STR);
    }

    MPI_Finalize();
    return (nerrs > 0);
}
Esempio n. 10
0
/*
  writes output in pnetcdf format

  nblocks: local number of blocks
  vblocks: pointer to array of vblocks
  out_file: output file name
  comm: MPI communicator
*/
void pnetcdf_write(int nblocks, struct vblock_t *vblocks,
       char *out_file, MPI_Comm comm) {

#ifdef USEPNETCDF
  int err;
  int ncid, cmode, varids[23], dimids[8], dimids_2D[2];
  MPI_Offset start[2], count[2];

  MPI_Offset quants[NUM_QUANTS]; /* quantities per block */
  MPI_Offset proc_quants[NUM_QUANTS]; /* quantities per process */
  MPI_Offset tot_quants[NUM_QUANTS]; /* total quantities all global blocks */
  MPI_Offset block_ofsts[NUM_QUANTS]; /* starting offsets for each block */

  /* init */
  int i;
  for (i = 0; i < NUM_QUANTS; i++) {
    quants[i] = 0;
    proc_quants[i] = 0;
    tot_quants[i] = 0;
    block_ofsts[i] = 0;
  }

  /* sum quantities over local blocks */
  int b;
  for (b = 0; b < nblocks; b++) {
    proc_quants[NUM_VERTS] += vblocks[b].num_verts;
    proc_quants[NUM_COMP_CELLS] += vblocks[b].num_complete_cells;
    proc_quants[NUM_CELL_FACES] += vblocks[b].tot_num_cell_faces;
    proc_quants[NUM_FACE_VERTS] += vblocks[b].tot_num_face_verts;
    proc_quants[NUM_ORIG_PARTS] += vblocks[b].num_orig_particles;
    proc_quants[NUM_NEIGHBORS] += DIY_Num_neighbors(0, b);
  }
  proc_quants[NUM_BLOCKS] = nblocks;

  /* sum per process values to be global ones */
  MPI_Allreduce(proc_quants, tot_quants, NUM_QUANTS, MPI_OFFSET, MPI_SUM, comm);

  /* prefix sum proc offsets */
  MPI_Exscan(proc_quants, &block_ofsts, NUM_QUANTS, MPI_OFFSET, MPI_SUM, comm);

  /* create a new file for writing */
  cmode = NC_CLOBBER | NC_64BIT_DATA;
  err = ncmpi_create(comm, out_file, cmode, MPI_INFO_NULL, &ncid); ERR;

  /* define dimensions */
  err = ncmpi_def_dim(ncid, "num_g_blocks", tot_quants[NUM_BLOCKS],
          &dimids[0]); ERR;
  err = ncmpi_def_dim(ncid, "XYZ", 3, &dimids[1]); ERR;
  err = ncmpi_def_dim(ncid, "num_g_verts", tot_quants[NUM_VERTS],
          &dimids[2]); ERR;
  err = ncmpi_def_dim(ncid, "num_g_complete_cells", tot_quants[NUM_COMP_CELLS],
          &dimids[3]); ERR;
  err = ncmpi_def_dim(ncid, "tot_num_g_cell_faces", tot_quants[NUM_CELL_FACES],
          &dimids[4]); ERR;
  err = ncmpi_def_dim(ncid, "tot_num_g_face_verts", tot_quants[NUM_FACE_VERTS],
          &dimids[5]); ERR;
  err = ncmpi_def_dim(ncid, "num_g_orig_particles", tot_quants[NUM_ORIG_PARTS],
          &dimids[6]); ERR;
  err = ncmpi_def_dim(ncid, "num_g_neighbors", tot_quants[NUM_NEIGHBORS],
          &dimids[7]); ERR;

  /* define variables */
  err = ncmpi_def_var(ncid, "num_verts", NC_INT, 1, &dimids[0],
          &varids[0]); ERR;
  err = ncmpi_def_var(ncid, "num_complete_cells", NC_INT, 1, &dimids[0],
          &varids[1]); ERR;
  err = ncmpi_def_var(ncid, "tot_num_cell_faces", NC_INT, 1, &dimids[0],
          &varids[2]); ERR;
  err = ncmpi_def_var(ncid, "tot_num_face_verts", NC_INT, 1, &dimids[0],
          &varids[3]); ERR;
  err = ncmpi_def_var(ncid, "num_orig_particles", NC_INT, 1, &dimids[0],
          &varids[4]); ERR;

  /* block offsets */
  err = ncmpi_def_var(ncid, "block_off_num_verts", NC_INT64, 1, &dimids[0],
          &varids[5]); ERR;
  err = ncmpi_def_var(ncid, "block_off_num_complete_cells", NC_INT64, 1,
          &dimids[0], &varids[6]); ERR;
  err = ncmpi_def_var(ncid, "block_off_tot_num_cell_faces", NC_INT64, 1,
          &dimids[0], &varids[7]); ERR;
  err = ncmpi_def_var(ncid, "block_off_tot_num_face_verts", NC_INT64, 1,
          &dimids[0], &varids[8]); ERR;
  err = ncmpi_def_var(ncid, "block_off_num_orig_particles", NC_INT64, 1,
          &dimids[0], &varids[9]); ERR;

  dimids_2D[0] = dimids[0];
  dimids_2D[1] = dimids[1];
  err = ncmpi_def_var(ncid, "mins", NC_FLOAT, 2, dimids_2D, &varids[11]); ERR;
  err = ncmpi_def_var(ncid, "maxs", NC_FLOAT, 2, dimids_2D, &varids[12]); ERR;

  dimids_2D[0] = dimids[2];
  dimids_2D[1] = dimids[1];
  err = ncmpi_def_var(ncid, "save_verts", NC_FLOAT, 2, dimids_2D,
          &varids[13]); ERR;
  dimids_2D[0] = dimids[6];
  dimids_2D[1] = dimids[1];
  err = ncmpi_def_var(ncid, "sites", NC_FLOAT, 2, dimids_2D,
          &varids[14]); ERR;
  err = ncmpi_def_var(ncid, "complete_cells", NC_INT, 1, &dimids[3],
          &varids[15]); ERR;
  err = ncmpi_def_var(ncid, "areas", NC_FLOAT, 1, &dimids[3],
          &varids[16]); ERR;
  err = ncmpi_def_var(ncid, "vols", NC_FLOAT, 1, &dimids[3], &varids[17]); ERR;
  err = ncmpi_def_var(ncid, "num_cell_faces", NC_INT, 1, &dimids[3],
          &varids[18]); ERR;
  err = ncmpi_def_var(ncid, "num_face_verts", NC_INT, 1, &dimids[4],
          &varids[19]); ERR;
  err = ncmpi_def_var(ncid, "face_verts", NC_INT, 1, &dimids[5],
          &varids[20]); ERR;
  err = ncmpi_def_var(ncid, "neighbors", NC_INT, 1, &dimids[7],
          &varids[21]); ERR;
  err = ncmpi_def_var(ncid, "g_block_ids", NC_INT, 1, &dimids[0],
          &varids[22]); ERR;

  /* exit define mode */
  err = ncmpi_enddef(ncid); ERR;

  /* write all variables.
     to improve: we can try nonblocking I/O to aggregate small requests */

  for (b = 0; b < nblocks; b++) {

    struct vblock_t *v = &vblocks[b];

    /* quantities */
    start[0] = block_ofsts[NUM_BLOCKS];
    count[0] = 1;
    err = ncmpi_put_vara_int_all(ncid, varids[0], start, count,
         &v->num_verts); ERR;
    err = ncmpi_put_vara_int_all(ncid, varids[1], start, count,
         &v->num_complete_cells); ERR;
    err = ncmpi_put_vara_int_all(ncid, varids[2], start, count,
         &v->tot_num_cell_faces); ERR;
    err = ncmpi_put_vara_int_all(ncid, varids[3], start, count,
         &v->tot_num_face_verts); ERR;
    err = ncmpi_put_vara_int_all(ncid, varids[4], start, count,
         &v->num_orig_particles); ERR;

    /* block offsets */
    err = ncmpi_put_vara_longlong_all(ncid, varids[5], start, count,
              &block_ofsts[NUM_VERTS]); ERR;
    err = ncmpi_put_vara_longlong_all(ncid, varids[6], start, count,
              &block_ofsts[NUM_COMP_CELLS]); ERR;
    err = ncmpi_put_vara_longlong_all(ncid, varids[7], start, count,
              &block_ofsts[NUM_CELL_FACES]); ERR;
    err = ncmpi_put_vara_longlong_all(ncid, varids[8], start, count,
              &block_ofsts[NUM_FACE_VERTS]); ERR;
    err = ncmpi_put_vara_longlong_all(ncid, varids[9], start, count,
              &block_ofsts[NUM_ORIG_PARTS]); ERR;

    /* block bounds */
    start[0] = block_ofsts[NUM_BLOCKS];
    count[0] = 1;
    start[1] = 0;
    count[1] = 3;
    err = ncmpi_put_vara_float_all(ncid, varids[11], start, count,
           v->mins); ERR;
    err = ncmpi_put_vara_float_all(ncid, varids[12], start, count,
           v->maxs); ERR;

    /* save_verts */
    start[0] = block_ofsts[NUM_VERTS];
    start[1] = 0;
    count[0] = v->num_verts;
    count[1] = 3;
    err = ncmpi_put_vara_float_all(ncid, varids[13], start, count,
           v->save_verts); ERR;

    /* sites */
    start[0] = block_ofsts[NUM_ORIG_PARTS];
    start[1] = 0;
    count[0] = v->num_orig_particles;
    count[1] = 3;
    err = ncmpi_put_vara_float_all(ncid, varids[14], start, count,
           v->sites); ERR;

    /* complete cells */
    start[0] = block_ofsts[NUM_COMP_CELLS];
    count[0] = v->num_complete_cells;
    err = ncmpi_put_vara_int_all(ncid, varids[15], start, count,
         v->complete_cells); ERR;

    /* areas */
    start[0] = block_ofsts[NUM_COMP_CELLS];
    count[0] = v->num_complete_cells;
    err = ncmpi_put_vara_float_all(ncid, varids[16], start, count,
           v->areas); ERR;

    /* volumes */
    start[0] = block_ofsts[NUM_COMP_CELLS];
    count[0] = v->num_complete_cells;
    err = ncmpi_put_vara_float_all(ncid, varids[17], start, count,
           v->vols); ERR;

    /* num_cell_faces */
    start[0] = block_ofsts[NUM_COMP_CELLS];
    count[0] = v->num_complete_cells;
    err = ncmpi_put_vara_int_all(ncid, varids[18], start, count,
         v->num_cell_faces); ERR;

    /* num_face_verts */
    start[0] = block_ofsts[NUM_CELL_FACES];
    count[0] = v->tot_num_cell_faces;
    err = ncmpi_put_vara_int_all(ncid, varids[19], start, count,
         v->num_face_verts); ERR;

    /* face verts */
    start[0] = block_ofsts[NUM_FACE_VERTS];
    count[0] = v->tot_num_face_verts;
    err = ncmpi_put_vara_int_all(ncid, varids[20], start, count,
         v->face_verts); ERR;

    /* neighbors */
    int *neighbors = (int*)malloc(DIY_Num_neighbors(0, b) * sizeof(int));
    int num_neighbors = DIY_Get_neighbors(0, b, neighbors);
    start[0] = block_ofsts[NUM_NEIGHBORS];
    count[0] = num_neighbors;
    err = ncmpi_put_vara_int_all(ncid, varids[21], start, count, neighbors);
    ERR;

    /* gids */
    int gid = DIY_Gid(0, b);
    start[0] = block_ofsts[NUM_BLOCKS];
    count[0] = 1;
    err = ncmpi_put_vara_int_all(ncid, varids[22], start, count,
         &gid); ERR;

    /* update block offsets */
    block_ofsts[NUM_VERTS] += v->num_verts;
    block_ofsts[NUM_COMP_CELLS] += v->num_complete_cells;
    block_ofsts[NUM_CELL_FACES] += v->tot_num_cell_faces;
    block_ofsts[NUM_FACE_VERTS] += v->tot_num_face_verts;
    block_ofsts[NUM_ORIG_PARTS] += v->num_orig_particles;
    block_ofsts[NUM_NEIGHBORS] += num_neighbors;
    block_ofsts[NUM_BLOCKS]++;

    /* debug */
/*     fprintf(stderr, "gid = %d num_verts = %d num_complete_cells = %d " */
/* 	    "tot_num_cell_faces = %d tot_num_face_verts = %d " */
/* 	    "num_orig_particles = %d\n", */
/* 	    gid, v->num_verts, v->num_complete_cells, v->tot_num_cell_faces, */
/* 	    v->tot_num_face_verts, v->num_orig_particles); */

  }

  err = ncmpi_close(ncid); ERR;
#endif

}
Esempio n. 11
0
int main(int argc, char** argv)
{
    extern int optind;
    char filename[256];
    int i, j, verbose=1, rank, nprocs, err, nerrs=0;
    int myNX, G_NX, myOff, num_reqs;
    int ncid, cmode, varid, dimid[2], *reqs, *sts, **buf;
    MPI_Offset start[2], count[2];
    MPI_Info info;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);

    /* get command-line arguments */
    while ((i = getopt(argc, argv, "hq")) != EOF)
        switch(i) {
            case 'q': verbose = 0;
                      break;
            case 'h':
            default:  if (rank==0) usage(argv[0]);
                      MPI_Finalize();
                      return 1;
        }
    if (argv[optind] == NULL) strcpy(filename, "testfile.nc");
    else                      snprintf(filename, 256, "%s", argv[optind]);

    /* set an MPI-IO hint to disable file offset alignment for fixed-size
     * variables */
    MPI_Info_create(&info);
    MPI_Info_set(info, "nc_var_align_size", "1");

    cmode = NC_CLOBBER | NC_64BIT_DATA;
    err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
    ERR

    MPI_Info_free(&info);

    /* the global array is NY * (NX * nprocs) */
    G_NX  = NX * nprocs;
    myOff = NX * rank;
    myNX  = NX;
    if (verbose) printf("%2d: myOff=%3d myNX=%3d\n",rank,myOff,myNX);

    err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
    ERR
    err = ncmpi_def_dim(ncid, "X", G_NX, &dimid[1]);
    ERR
    err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid);
    ERR
    err = ncmpi_enddef(ncid);
    ERR

    /* First, fill the entire array with zeros, using a blocking I/O.
       Every process writes a subarray of size NY * myNX */
    buf    = (int**) malloc(myNX * sizeof(int*));
    buf[0] = (int*)  calloc(NY * myNX, sizeof(int));
    start[0] = 0;   start[1] = myOff;
    count[0] = NY;  count[1] = myNX;
    err = ncmpi_put_vara_int_all(ncid, varid, start, count, buf[0]);
    free(buf[0]);

    /* initialize the buffer with rank ID. Also make the case interesting,
       by allocating buffers separately */
    for (i=0; i<myNX; i++) {
        buf[i] = (int*) malloc(NY * sizeof(int));
        for (j=0; j<NY; j++) buf[i][j] = rank;
    }

    reqs = (int*) malloc(myNX * sizeof(int));
    sts  = (int*) malloc(myNX * sizeof(int));

    /* each proc writes myNX single columns of the 2D array */
    start[0]  = 0;   start[1] = rank;
    count[0]  = NY;  count[1] = 1;
    if (verbose)
        printf("%2d: start=%3lld %3lld count=%3lld %3lld\n",
               rank, start[0],start[1], count[0],count[1]);

    num_reqs = 0;
    for (i=0; i<myNX; i++) {
        err = ncmpi_iput_vara_int(ncid, varid, start, count, buf[i],
                                  &reqs[num_reqs++]);
        ERR
        start[1] += nprocs;
    }
    err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
    ERR

    /* check status of all requests */
    for (i=0; i<num_reqs; i++)
        if (sts[i] != NC_NOERR)
            printf("Error at line %d in %s: nonblocking write fails on request %d (%s)\n",
                   __LINE__,__FILE__,i, ncmpi_strerror(sts[i]));

    err = ncmpi_close(ncid); ERR

    /* read back using the same access pattern */
    err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); ERR

    err = ncmpi_inq_varid(ncid, "var", &varid); ERR

    for (i=0; i<myNX; i++)
        for (j=0; j<NY; j++) buf[i][j] = -1;

    /* each proc reads myNX single columns of the 2D array */
    start[0]  = 0;   start[1] = rank;
    count[0]  = NY;  count[1] = 1;

    num_reqs = 0;
    for (i=0; i<myNX; i++) {
        err = ncmpi_iget_vara_int(ncid, varid, start, count, buf[i],
                                  &reqs[num_reqs++]);
        ERR
        start[1] += nprocs;
    }
    err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
    ERR

    /* check status of all requests */
    for (i=0; i<num_reqs; i++)
        if (sts[i] != NC_NOERR)
            printf("Error at line %d in %s: nonblocking write fails on request %d (%s)\n",
                   __LINE__,__FILE__,i, ncmpi_strerror(sts[i]));

    for (i=0; i<myNX; i++) {
        for (j=0; j<NY; j++)
            if (buf[i][j] != rank)
                printf("Error at line %d in %s: expect buf[%d][%d]=%d but got %d\n",
                __LINE__,__FILE__,i,j,rank,buf[i][j]);
    }

    err = ncmpi_close(ncid);
    ERR

    free(sts);
    free(reqs);
    for (i=0; i<myNX; i++) free(buf[i]);
    free(buf);

    /* check if there is any PnetCDF internal malloc residue */
    MPI_Offset malloc_size, sum_size;
    err = ncmpi_inq_malloc_size(&malloc_size);
    if (err == NC_NOERR) {
        MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
        if (rank == 0 && sum_size > 0)
            printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
                   sum_size);
    }

    MPI_Finalize();
    return (nerrs > 0);
}
Esempio n. 12
0
/*---< pnetcdf_write() >-----------------------------------------------------*/
int pnetcdf_write(char      *filename,     /* input file name */
                  int        save_in_new_file,  /* 0 or 1 */
                  int        numClusters,  /* no. clusters */
                  int        numObjs,      /* no. data objects */
                  int        numCoords,    /* no. coordinates (local) */
                  float    **clusters,     /* [numClusters][numCoords] centers*/
                  int       *membership,   /* [numObjs] */
                  int        totalNumObjs, /* total no. data objects */
                  MPI_Comm   comm,
                  int        verbose)
{
    int   rank, nproc, divd, rem;
    char  outFileName[1024];
    int        ncid, dimids[2], dim_num_obj, clusters_varid, membership_varid, retval;
    MPI_Offset start, count;

    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &nproc);

    /* output: the coordinates of the cluster centres ----------------------*/
    /* only proc 0 matters this, because clusters[] are the same across all proc */

    /* to save the results in a new netCDF file */
    if (save_in_new_file) {
        if (strcasecmp(filename+strlen(filename)-3, ".nc") == 0) {
            strcpy(outFileName, filename);
            outFileName[strlen(filename)-3] = '\0';
            strcat(outFileName, ".cluster_centres.nc");
        }
        else
            sprintf(outFileName, "%s.cluster_centres.nc", filename);

        if (rank == 0 && verbose) {
            printf("Writing coordinates of K=%d cluster centers to file \"%s\"\n",
                   numClusters, outFileName);

            printf("Writing membership of N=%d data objects to file \"%s\"\n",
                   totalNumObjs, outFileName);
        }

        /* Create the file. The NC_CLOBBER parameter tells netCDF to
         * overwrite this file, if it already exists.*/
        if ((retval = ncmpi_create(comm, outFileName, NC_CLOBBER | NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid)))
            ERR2(retval);

        /* Define the dimensions. NetCDF will hand back an ID for each. */
        if ((retval = ncmpi_def_dim(ncid, "num_clusters", numClusters, &dimids[0])))
            ERR2(retval);

        if ((retval = ncmpi_def_dim(ncid, "num_coordinates", numCoords, &dimids[1])))
            ERR2(retval);

        if ((retval = ncmpi_def_dim(ncid, "num_elements", totalNumObjs, &dim_num_obj)))
            ERR2(retval);

        /* Define the clusters variable. The type of the variable in this case is
         * NC_FLOAT (4-byte float). */
        if ((retval = ncmpi_def_var(ncid, "clusters", NC_FLOAT, 2, dimids, &clusters_varid)))
            ERR2(retval);

        /* Define the membership variable. The type of the variable in this case is
         * NC_INT (4-byte integer). */
        if ((retval = ncmpi_def_var(ncid, "membership", NC_INT, 1, &dim_num_obj, &membership_varid)))
            ERR2(retval);

        /* End define mode. This tells netCDF we are done defining
         * metadata. */
        if ((retval = ncmpi_enddef(ncid)))
            ERR2(retval);
    }
    else { /* add new variables into existing netCDF file */
    }

    /* write cluster centers */
    if ((retval = ncmpi_put_var_float_all(ncid, clusters_varid, *clusters)))
        ERR2(retval);

    /* write membership variable */
    divd  = totalNumObjs / nproc;
    rem   = totalNumObjs % nproc;
    start = (rank < rem) ? rank*(divd+1) : rank*divd + rem;
    count = numObjs;
    if (_debug) printf("%2d: start=%lld count=%lld\n",rank,start,count);

    if ((retval = ncmpi_put_vara_int_all(ncid, membership_varid, &start, &count, membership)))
        ERR2(retval);

   /* Close the file. This frees up any internal netCDF resources
    * associated with the file, and flushes any buffers. */
   if ((retval = ncmpi_close(ncid)))
      ERR2(retval);

    return 1;
}