int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0;
int ncid, varid, dimid[2], req, st;
MPI_Offset start[2], count[2], stride[2];
unsigned char buffer[NY][NX];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for ncmpi_end_indep_data ", basename(argv[0]));
printf("%-66s ------ ",cmd_str);
free(cmd_str);
}
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA,
MPI_INFO_NULL, &ncid);
CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var", NC_UBYTE, NDIMS, dimid, &varid); CHECK_ERR
err = ncmpi_enddef(ncid); CHECK_ERR
for (i=0; i<NY; i++) for (j=0; j<NX; j++) buffer[i][j] = rank+10;
start[0] = 0; start[1] = NX*rank;
count[0] = NY/2; count[1] = NX/2;
stride[0] = 2; stride[1] = 2;
err = ncmpi_buffer_attach(ncid, NY*NX); CHECK_ERR
err = ncmpi_begin_indep_data(ncid); CHECK_ERR
err = ncmpi_bput_vars_uchar(ncid, varid, start, count, stride,
&buffer[0][0], &req);
CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_end_indep_data(ncid); CHECK_ERR
/* calling wait API after exiting independent data mode on purpose */
err = ncmpi_wait_all(ncid, 1, &req, &st); CHECK_ERR
err = st; CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_buffer_detach(ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0) {
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
ncmpi_inq_malloc_list();
}
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char** argv)
{
extern int optind;
char *filename="testfile.nc", exec[128];
int i, j, k, n, rank, nprocs, verbose=1, err;
int ncid, cmode, varid[4], dimid[2], nreqs, reqs[4], sts[4];
unsigned int *buffer[4];
int num_segs[4], req_lens[4];
MPI_Offset ***starts, ***counts;
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
strcpy(exec, argv[0]);
/* get command-line arguments */
while ((i = getopt(argc, argv, "hq")) != EOF)
switch(i) {
case 'q': verbose = 0;
break;
case 'h':
default: if (rank==0) usage(argv[0]);
MPI_Finalize();
return 0;
}
argc -= optind;
argv += optind;
if (argc == 1) filename = argv[0]; /* optional argument */
if (nprocs != 4 && rank == 0 && verbose)
printf("Warning: %s is intended to run on 4 processes\n",exec);
/* set an MPI-IO hint to disable file offset alignment for fix-sized
* variables */
MPI_Info_create(&info);
MPI_Info_set(info, "nc_var_align_size", "1");
/* create a new file for writing ----------------------------------------*/
cmode = NC_CLOBBER | NC_64BIT_DATA;
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
ERR
MPI_Info_free(&info);
/* create a global array of size NY * NX */
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); ERR
err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); ERR
err = ncmpi_def_var(ncid, "var0", NC_UINT, NDIMS, dimid, &varid[0]); ERR
err = ncmpi_def_var(ncid, "var1", NC_UINT, NDIMS, dimid, &varid[1]); ERR
err = ncmpi_def_var(ncid, "var2", NC_UINT, NDIMS, dimid, &varid[2]); ERR
err = ncmpi_def_var(ncid, "var3", NC_UINT, NDIMS, dimid, &varid[3]); ERR
err = ncmpi_enddef(ncid); ERR
/* allocate space for starts and counts */
starts = calloc_3D(4, 6, NDIMS);
counts = calloc_3D(4, 6, NDIMS);
n = rank % 4;
num_segs[n] = 4; /* number of segments for this request */
starts[n][0][0]=0; starts[n][0][1]=5; counts[n][0][0]=1; counts[n][0][1]=2;
starts[n][1][0]=1; starts[n][1][1]=0; counts[n][1][0]=1; counts[n][1][1]=1;
starts[n][2][0]=2; starts[n][2][1]=6; counts[n][2][0]=1; counts[n][2][1]=2;
starts[n][3][0]=3; starts[n][3][1]=0; counts[n][3][0]=1; counts[n][3][1]=3;
/* starts[n][][] n_counts[n][][] indicate the following: ("-" means skip)
- - - - - X X - - -
X - - - - - - - - -
- - - - - - X X - -
X X X - - - - - - -
*/
n = (rank+1) % 4;
num_segs[n] = 6; /* number of segments for this request */
starts[n][0][0]=0; starts[n][0][1]=3; counts[n][0][0]=1; counts[n][0][1]=2;
starts[n][1][0]=0; starts[n][1][1]=8; counts[n][1][0]=1; counts[n][1][1]=2;
starts[n][2][0]=1; starts[n][2][1]=5; counts[n][2][0]=1; counts[n][2][1]=2;
starts[n][3][0]=2; starts[n][3][1]=0; counts[n][3][0]=1; counts[n][3][1]=2;
starts[n][4][0]=2; starts[n][4][1]=8; counts[n][4][0]=1; counts[n][4][1]=2;
starts[n][5][0]=3; starts[n][5][1]=4; counts[n][5][0]=1; counts[n][5][1]=3;
/* starts[n][][] counts[n][][] indicate the following pattern.
- - - X X - - - X X
- - - - - X X - - -
X X - - - - - - X X
- - - - X X X - - -
*/
n = (rank+2) % 4;
num_segs[n] = 5; /* number of segments for this request */
starts[n][0][0]=0; starts[n][0][1]=7; counts[n][0][0]=1; counts[n][0][1]=1;
starts[n][1][0]=1; starts[n][1][1]=1; counts[n][1][0]=1; counts[n][1][1]=3;
starts[n][2][0]=1; starts[n][2][1]=7; counts[n][2][0]=1; counts[n][2][1]=3;
starts[n][3][0]=2; starts[n][3][1]=2; counts[n][3][0]=1; counts[n][3][1]=1;
starts[n][4][0]=3; starts[n][4][1]=3; counts[n][4][0]=1; counts[n][4][1]=1;
/* starts[n][][] counts[n][][] indicate the following pattern.
- - - - - - - X - -
- X X X - - - X X X
- - X - - - - - - -
- - - X - - - - - -
*/
n = (rank+3) % 4;
num_segs[n] = 4; /* number of segments for this request */
starts[n][0][0]=0; starts[n][0][1]=0; counts[n][0][0]=1; counts[n][0][1]=3;
starts[n][1][0]=1; starts[n][1][1]=4; counts[n][1][0]=1; counts[n][1][1]=1;
starts[n][2][0]=2; starts[n][2][1]=3; counts[n][2][0]=1; counts[n][2][1]=3;
starts[n][3][0]=3; starts[n][3][1]=7; counts[n][3][0]=1; counts[n][3][1]=3;
/*starts[n][][] counts[n][][] indicate the following pattern.
X X X - - - - - - -
- - - - X - - - - -
- - - X X X - - - -
- - - - - - - X X X
*/
/* only rank 0, 1, 2, and 3 do I/O:
* each of ranks 0 to 3 write 4 nonblocking requests */
nreqs = 4;
if (rank >= 4) nreqs = 0;
/* bufsize must be max of data type converted before and after */
MPI_Offset bufsize = 0;
/* calculate length of each varn request, number of segments in each
* varn request, and allocate write buffer */
for (i=0; i<nreqs; i++) {
req_lens[i] = 0; /* total length this request */
for (j=0; j<num_segs[i]; j++) {
MPI_Offset req_len=1;
for (k=0; k<NDIMS; k++)
req_len *= counts[i][j][k];
req_lens[i] += req_len;
}
if (verbose) printf("req_lens[%d]=%d\n",i,req_lens[i]);
/* allocate I/O buffer and initialize its contents */
buffer[i] = (unsigned int*) malloc(req_lens[i] * sizeof(unsigned int));
for (j=0; j<req_lens[i]; j++) buffer[i][j] = rank;
bufsize += req_lens[i];
}
bufsize *= sizeof(unsigned int);
if (verbose) printf("%d: Attach buffer size %lld\n", rank, bufsize);
/* give PnetCDF a space to buffer the nonblocking requests */
if (bufsize > 0) {
err = ncmpi_buffer_attach(ncid, bufsize); ERR
}
int main(int argc, char** argv)
{
char filename[256];
int i, j, k, rank, nprocs, verbose=0, err, nerrs=0;
int ncid, cmode, varid[4], dimid[2], nreqs, reqs[4], sts[4];
unsigned int *buffer[4];
int num_segs[4] = {4, 6, 5, 4};
int req_lens[4], my_nsegs[4];
MPI_Offset **starts[4], **counts[4];
MPI_Offset n_starts[4][6][2] = {{{0,5}, {1,0}, {2,6}, {3,0}, {0,0}, {0,0}},
{{0,3}, {0,8}, {1,5}, {2,0}, {2,8}, {3,4}},
{{0,7}, {1,1}, {1,7}, {2,2}, {3,3}, {0,0}},
{{0,0}, {1,4}, {2,3}, {3,7}, {0,0}, {0,0}}};
MPI_Offset n_counts[4][6][2] = {{{1,2}, {1,1}, {1,2}, {1,3}, {0,0}, {0,0}},
{{1,2}, {1,2}, {1,2}, {1,2}, {1,2}, {1,3}},
{{1,1}, {1,3}, {1,3}, {1,1}, {1,1}, {0,0}},
{{1,3}, {1,1}, {1,3}, {1,3}, {0,0}, {0,0}}};
/* n_starts[0][][] n_counts[0][][] indicate the following: ("-" means skip)
- - - - - X X - - -
X - - - - - - - - -
- - - - - - X X - -
X X X - - - - - - -
n_starts[1][][] n_counts[1][][] indicate the following pattern.
- - - X X - - - X X
- - - - - X X - - -
X X - - - - - - X X
- - - - X X X - - -
n_starts[2][][] n_counts[2][][] indicate the following pattern.
- - - - - - - X - -
- X X X - - - X X X
- - X - - - - - - -
- - - X - - - - - -
n_starts[3][][] n_counts[3][][] indicate the following pattern.
X X X - - - - - - -
- - - - X - - - - -
- - - X X X - - - -
- - - - - - - X X X
*/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 0;
}
strcpy(filename, "testfile.nc");
if (argc == 2) strcpy(filename, argv[1]);
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char cmd_str[256];
sprintf(cmd_str, "*** TESTING C %s for bput_varn_uint ", argv[0]);
printf("%-66s ------ ", cmd_str);
}
if (verbose && nprocs != 4 && rank == 0)
printf("Warning: %s is intended to run on 4 processes\n",argv[0]);
/* create a new file for writing ----------------------------------------*/
cmode = NC_CLOBBER | NC_64BIT_DATA;
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid);
ERR
/* create a global array of size NY * NX */
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); ERR
err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); ERR
err = ncmpi_def_var(ncid, "var0", NC_UINT, NDIMS, dimid, &varid[0]); ERR
err = ncmpi_def_var(ncid, "var1", NC_UINT, NDIMS, dimid, &varid[1]); ERR
err = ncmpi_def_var(ncid, "var2", NC_UINT, NDIMS, dimid, &varid[2]); ERR
err = ncmpi_def_var(ncid, "var3", NC_UINT, NDIMS, dimid, &varid[3]); ERR
err = ncmpi_enddef(ncid); ERR
/* allocate space for starts and counts */
starts[0] = (MPI_Offset**) malloc(4 * 6 * sizeof(MPI_Offset*));
counts[0] = (MPI_Offset**) malloc(4 * 6 * sizeof(MPI_Offset*));
starts[0][0] = (MPI_Offset*) calloc(4 * 6 * NDIMS, sizeof(MPI_Offset));
counts[0][0] = (MPI_Offset*) calloc(4 * 6 * NDIMS, sizeof(MPI_Offset));
for (i=1; i<4; i++) {
starts[i] = starts[i-1] + 6;
counts[i] = counts[i-1] + 6;
starts[i][0] = starts[i-1][0] + 6 * NDIMS;
counts[i][0] = counts[i-1][0] + 6 * NDIMS;
}
for (i=0; i<4; i++) {
for (j=1; j<6; j++) {
starts[i][j] = starts[i][j-1] + NDIMS;
counts[i][j] = counts[i][j-1] + NDIMS;
}
}
/* set values for starts and counts */
for (i=0; i<4; i++) {
int n = (i + rank) % 4;
my_nsegs[i] = num_segs[n]; /* number of segments for this request */
for (j=0; j<6; j++) {
for (k=0; k<NDIMS; k++) {
starts[i][j][k] = n_starts[n][j][k];
counts[i][j][k] = n_counts[n][j][k];
}
}
}
/* test error code: NC_ENULLABUF */
err = ncmpi_bput_varn_uint(ncid, varid[0], 1, NULL, NULL, NULL, &reqs[0]);
if (err != NC_ENULLABUF) {
printf("Error at line %d: expecting error code NC_ENULLABUF but got %s\n",
__LINE__, nc_err_code_name(err));
nerrs++;
}
/* only rank 0, 1, 2, and 3 do I/O:
* each of ranks 0 to 3 write 4 nonblocking requests */
nreqs = 4;
if (rank >= 4) nreqs = 0;
/* bufsize must be max of data type converted before and after */
MPI_Offset bufsize = 0;
/* calculate length of each varn request and allocate write buffer */
for (i=0; i<nreqs; i++) {
req_lens[i] = 0; /* total length this request */
for (j=0; j<my_nsegs[i]; j++) {
MPI_Offset req_len=1;
for (k=0; k<NDIMS; k++)
req_len *= counts[i][j][k];
req_lens[i] += req_len;
}
if (verbose) printf("req_lens[%d]=%d\n",i,req_lens[i]);
/* allocate I/O buffer and initialize its contents */
buffer[i] = (unsigned int*) malloc(req_lens[i] * sizeof(unsigned int));
for (j=0; j<req_lens[i]; j++) buffer[i][j] = rank;
bufsize += req_lens[i];
}
bufsize *= sizeof(unsigned int);
/* give PnetCDF a space to buffer the nonblocking requests */
if (bufsize > 0) {
err = ncmpi_buffer_attach(ncid, bufsize); ERR
}
int main(int argc, char **argv)
{
char filename[256];
int err, nerrs=0, ncid, dimid[NDIMS], varid[5], ndims=NDIMS;
int i, j, k, nprocs, rank, req, *buf;
MPI_Offset start[NDIMS] = {0};
MPI_Offset count[NDIMS] = {0};
MPI_Offset stride[NDIMS] = {0};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for NULL stride ", basename(argv[0]));
printf("%-66s ------ ", cmd_str); fflush(stdout);
free(cmd_str);
}
err = ncmpi_create(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid);
CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
CHECK_ERR
err = ncmpi_def_dim(ncid, "X", nprocs*NX, &dimid[1]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v0", NC_INT, ndims, dimid, &varid[0]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v1", NC_INT, ndims, dimid, &varid[1]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v2", NC_INT, ndims, dimid, &varid[2]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v3", NC_INT, ndims, dimid, &varid[3]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v4", NC_INT, ndims, dimid, &varid[4]);
CHECK_ERR
err = ncmpi_enddef(ncid);
CHECK_ERR
start[0] = 0;
start[1] = rank*NX;
count[0] = NY;
count[1] = NX;
buf = (int*) malloc((size_t)NY * NX * sizeof(int));
for (i=0; i<NY*NX; i++) buf[i] = rank+10;
err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf);
CHECK_ERR
CHECK_PUT_BUF
err = ncmpi_put_vars_int_all(ncid, varid[1], start, count, NULL, buf);
CHECK_ERR
CHECK_PUT_BUF
start[0] = 0;
start[1] = rank;
count[0] = NY;
count[1] = NX;
stride[0] = 1;
stride[1] = nprocs;
err = ncmpi_put_vars_int_all(ncid, varid[2], start, count, stride, buf);
CHECK_ERR
CHECK_PUT_BUF
/* test bput_vars */
err = ncmpi_buffer_attach(ncid, NY*NX*sizeof(int));
CHECK_ERR
start[0] = 0;
start[1] = rank*NX;
count[0] = NY;
count[1] = NX;
err = ncmpi_bput_vars_int(ncid, varid[3], start, count, NULL, buf, &req);
CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, NULL);
CHECK_ERR
CHECK_PUT_BUF
start[0] = 0;
start[1] = rank;
count[0] = NY;
count[1] = NX;
stride[0] = 1;
stride[1] = nprocs;
err = ncmpi_bput_vars_int(ncid, varid[4], start, count, stride, buf, &req);
CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, NULL);
CHECK_ERR
CHECK_PUT_BUF
free(buf);
err = ncmpi_buffer_detach(ncid);
CHECK_ERR
buf = (int*) malloc((size_t)NY * NX * nprocs * sizeof(int));
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[0], buf);
CHECK_ERR
/* check read buffer contents */
/* v0 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[1], buf);
CHECK_ERR
/* check read buffer contents */
/* v1 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[2], buf);
CHECK_ERR
/* check read buffer contents */
/* v2 =
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13 ;
*/
for (i=0; i<NY; i++) {
for (k=0; k<NX; k++) {
for (j=0; j<nprocs; j++) {
if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[3], buf);
CHECK_ERR
/* check read buffer contents */
/* v3 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[4], buf);
CHECK_ERR
/* check read buffer contents */
/* v4 =
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13 ;
*/
for (i=0; i<NY; i++) {
for (k=0; k<NX; k++) {
for (j=0; j<nprocs; j++) {
if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
nerrs++;
}
}
}
}
err = ncmpi_close(ncid);
CHECK_ERR
free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
if (malloc_size > 0) ncmpi_inq_malloc_list();
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
file = (file_desc_t *) malloc(sizeof(file_desc_t));
file->next = NULL;
file->iosystem = ios;
file->iotype = *iotype;
file->buffer.validvars=0;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.vid=NULL;
file->buffer.ioid=-1;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
msg = PIO_MSG_CREATE_FILE;
file->mode = mode;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
// The 64 bit options are not compatable with hdf5 format files
// printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO);
file->mode = file->mode | NC_MPIIO | NC_NETCDF4;
//printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4);
ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh));
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_create(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
if(ierr == PIO_NOERR){
if(ios->io_rank==0){
printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT);
}
int oldfill;
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
// ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill);
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
msg = PIO_MSG_OPEN_FILE;
ios = pio_get_iosystem_from_id(iosysid);
if(ios==NULL){
printf("bad iosysid %d\n",iosysid);
return PIO_EBADID;
}
file = (file_desc_t *) malloc(sizeof(*file));
if(file==NULL){
return PIO_ENOMEM;
}
file->iotype = *iotype;
file->next = NULL;
file->iosystem = ios;
file->mode = mode;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
file->buffer.validvars=0;
file->buffer.vid=NULL;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
#ifdef _MPISERIAL
ierr = nc_open(filename, file->mode, &(file->fh));
#else
file->mode = file->mode | NC_MPIIO;
ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh));
#endif
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
// *** Note the INTENTIONAL FALLTHROUGH ***
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
// This should only be done with a file opened to append
if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){
if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT);
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
break;
}
// If we failed to open a file due to an incompatible type of NetCDF, try it
// once with just plain old basic NetCDF
#ifdef _NETCDF
if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n");
// reset ierr on all tasks
ierr = PIO_NOERR;
// reset file markers for NETCDF on all tasks
file->iotype = PIO_IOTYPE_NETCDF;
// open netcdf file serially on main task
if(ios->io_rank==0){
ierr = nc_open(filename, file->mode, &(file->fh)); }
}
#endif
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr==PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
