int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0;
int ncid, varid, dimid[2], req, st;
MPI_Offset start[2], count[2], stride[2];
unsigned char buffer[NY][NX];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for ncmpi_end_indep_data ", basename(argv[0]));
printf("%-66s ------ ",cmd_str);
free(cmd_str);
}
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA,
MPI_INFO_NULL, &ncid);
CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var", NC_UBYTE, NDIMS, dimid, &varid); CHECK_ERR
err = ncmpi_enddef(ncid); CHECK_ERR
for (i=0; i<NY; i++) for (j=0; j<NX; j++) buffer[i][j] = rank+10;
start[0] = 0; start[1] = NX*rank;
count[0] = NY/2; count[1] = NX/2;
stride[0] = 2; stride[1] = 2;
err = ncmpi_buffer_attach(ncid, NY*NX); CHECK_ERR
err = ncmpi_begin_indep_data(ncid); CHECK_ERR
err = ncmpi_bput_vars_uchar(ncid, varid, start, count, stride,
&buffer[0][0], &req);
CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_end_indep_data(ncid); CHECK_ERR
/* calling wait API after exiting independent data mode on purpose */
err = ncmpi_wait_all(ncid, 1, &req, &st); CHECK_ERR
err = st; CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_buffer_detach(ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0) {
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
ncmpi_inq_malloc_list();
}
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char **argv) {
int i, j, rank, nprocs, ret;
int ncfile, ndims, nvars, ngatts, unlimited;
int var_ndims, var_natts;;
MPI_Offset *dim_sizes, var_size;
MPI_Offset *start, *count;
int *requests, *statuses;
char varname[NC_MAX_NAME+1];
int dimids[NC_MAX_VAR_DIMS];
nc_type type;
int **data;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc != 2) {
if (rank == 0) printf("Usage: %s filename\n", argv[0]);
MPI_Finalize();
exit(-1);
}
ret = ncmpi_open(MPI_COMM_WORLD, argv[1], NC_NOWRITE, MPI_INFO_NULL,
&ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* reader knows nothing about dataset, but we can interrogate with query
* routines: ncmpi_inq tells us how many of each kind of "thing"
* (dimension, variable, attribute) we will find in the file */
/* no commnunication needed after ncmpi_open: all processors have a cached
* veiw of the metadata once ncmpi_open returns */
ret = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* we do not really need the name of the dimension or the variable for
* reading in this example. we could, in a different example, take the
* name of a variable on the command line and read just that one */
dim_sizes = calloc(ndims, sizeof(MPI_Offset));
/* netcdf dimension identifiers are allocated sequentially starting
* at zero; same for variable identifiers */
for(i=0; i<ndims; i++) {
ret = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) );
if (ret != NC_NOERR) handle_error(ret, __LINE__);
}
requests = calloc(nvars, sizeof(int));
statuses = calloc(nvars, sizeof(int));
data = malloc(nvars*sizeof(int*));
for(i=0; i<nvars; i++) {
/* much less coordination in this case compared to rank 0 doing all
* the i/o: everyone already has the necessary information */
ret = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids,
&var_natts);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
start = calloc(var_ndims, sizeof(MPI_Offset));
count = calloc(var_ndims, sizeof(MPI_Offset));
/* we will simply decompose along one dimension. Generally the
* application has some algorithim for domain decomposistion. Note
* that data decomposistion can have an impact on i/o performance.
* Often it's best just to do what is natural for the application,
* but something to consider if performance is not what was
* expected/desired */
start[0] = (dim_sizes[dimids[0]]/nprocs)*rank;
count[0] = (dim_sizes[dimids[0]]/nprocs);
var_size = count[0];
for (j=1; j<var_ndims; j++) {
start[j] = 0;
count[j] = dim_sizes[dimids[j]];
var_size *= count[j];
}
switch(type) {
case NC_INT:
data[i] = calloc(var_size, sizeof(int));
/* as with the writes, this call is independent: we
* will do any coordination (if desired) in a
* subsequent ncmpi_wait_all() call */
ret = ncmpi_iget_vara(ncfile, i, start, count, data[i],
var_size, MPI_INT, &(requests[i]));
if (ret != NC_NOERR) handle_error(ret, __LINE__);
break;
default:
/* we can do this for all the known netcdf types but this
* example is already getting too long */
fprintf(stderr, "unsupported NetCDF type \n");
}
free(start);
free(count);
}
ret = ncmpi_wait_all(ncfile, nvars, requests, statuses);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* now that the ncmpi_wait_all has returned, the caller can do stuff with
* the buffers passed in to the non-blocking operations. The buffer resue
* rules are similar to MPI non-blocking messages */
for (i=0; i<nvars; i++) {
if (data[i] != NULL) free(data[i]);
}
free(data);
free(dim_sizes);
free(requests);
free(statuses);
ret = ncmpi_close(ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
MPI_Finalize();
return 0;
}
int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0, expected;
int ncid, cmode, varid[2], dimid[2], req[4], st[4], *buf;
int *buf0, *buf1, *buf2;
size_t len;
MPI_Offset start[2], count[2];
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* this program is intended to run on one process */
if (rank) goto fn_exit;
/* get command-line arguments */
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for writing interleaved fileviews ", basename(argv[0]));
printf("%-66s ------ ", cmd_str);
free(cmd_str);
}
MPI_Info_create(&info);
MPI_Info_set(info, "romio_cb_write", "disable");
MPI_Info_set(info, "ind_wr_buffer_size", "8");
/* these 2 hints are required to cause a core dump if r1758 fix is not
* presented */
/* create a new file for writing ----------------------------------------*/
cmode = NC_CLOBBER | NC_64BIT_DATA;
err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR
MPI_Info_free(&info);
/* define dimensions Y and X */
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR
/* define 2D variables of integer type */
err = ncmpi_def_var(ncid, "var0", NC_INT, 2, dimid, &varid[0]); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 2, dimid, &varid[1]); CHECK_ERR
/* enable fill mode */
err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR
/* do not forget to exit define mode */
err = ncmpi_enddef(ncid); CHECK_ERR
/* now we are in data mode */
buf = (int*) malloc(NY*NX * sizeof(int));
/* fill the entire variable var0 with -1s */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_put_var_int_all(ncid, varid[0], buf); CHECK_ERR
/* write 8 x 2 elements so this only interleaves the next two
* iput requests */
start[0] = 0; start[1] = 3;
count[0] = 8; count[1] = 2;
len = (size_t)(count[0] * count[1]);
buf0 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf0[i] = 50+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf0, &req[0]);
CHECK_ERR
/* write 1 x 3 elements */
start[0] = 1; start[1] = 8;
count[0] = 1; count[1] = 5;
len = (size_t)(count[0] * count[1]);
buf1 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf1[i] = 60+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf1, &req[1]);
CHECK_ERR
/* write 1 x 3 elements */
start[0] = 3; start[1] = 7;
count[0] = 1; count[1] = 5;
len = (size_t)(count[0] * count[1]);
buf2 = (int*) malloc(len * sizeof(int));
for (i=0; i<len; i++) buf2[i] = 70+i;
err = ncmpi_iput_vara_int(ncid, varid[0], start, count, buf2, &req[2]);
CHECK_ERR
err = ncmpi_wait_all(ncid, 3, req, st); CHECK_ERR
free(buf0); free(buf1); free(buf2);
/* fill the entire variable var1 with -1s */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_put_var_int_all(ncid, varid[1], buf); CHECK_ERR
/* write 8 x 2 elements so this only interleaves the next two iput
* requests */
start[0] = 0; start[1] = 3;
count[0] = 8; count[1] = 2;
len = (size_t)(count[0] * count[1]);
buf0 = (int*) malloc(len * sizeof(int));
for (i=0; i<count[0]*count[1]; i++) buf0[i] = 50+i;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf0, &req[0]);
CHECK_ERR
/* rearrange buffer contents, as buf is 2D */
for (i=0; i<5; i++) buf[i] = 10 + i;
for (i=5; i<10; i++) buf[i] = 10 + i + 5;
for (i=10; i<15; i++) buf[i] = 10 + i + 10;
start[0] = 6; start[1] = 7;
count[0] = 3; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf, &req[1]);
CHECK_ERR
for (i=15; i<20; i++) buf[i] = 10 + i - 10;
for (i=20; i<25; i++) buf[i] = 10 + i - 5;
start[0] = 6; start[1] = 12;
count[0] = 2; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+15, &req[2]);
CHECK_ERR
for (i=25; i<30; i++) buf[i] = 10 + i;
start[0] = 8; start[1] = 12;
count[0] = 1; count[1] = 5;
err = ncmpi_iput_vara_int(ncid, varid[1], start, count, buf+25, &req[3]);
CHECK_ERR
err = ncmpi_wait_all(ncid, 4, req, st); CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<16; i++) CHECK_CONTENTS(buf0, 50 + i)
for (i=0; i<5; i++) CHECK_CONTENTS(buf, 10 + i)
for (i=5; i<10; i++) CHECK_CONTENTS(buf, 10 + i + 5)
for (i=10; i<15; i++) CHECK_CONTENTS(buf, 10 + i + 10)
for (i=15; i<20; i++) CHECK_CONTENTS(buf, 10 + i - 10)
for (i=20; i<25; i++) CHECK_CONTENTS(buf, 10 + i - 5)
for (i=25; i<30; i++) CHECK_CONTENTS(buf, 10 + i)
err = ncmpi_close(ncid); CHECK_ERR
free(buf0);
/* open the same file and read back for validate */
err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL,
&ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var0", &varid[0]); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var1", &varid[1]); CHECK_ERR
/* read the entire array */
for (i=0; i<NY*NX; i++) buf[i] = -1;
err = ncmpi_get_var_int_all(ncid, varid[0], buf); CHECK_ERR
/* check if the contents of buf are expected */
expected = 50;
for (j=0; j<8; j++) {
for (i=3; i<5; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
expected = 60;
j = 1;
for (i=8; i<13; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
expected = 70;
j = 3;
for (i=7; i<12; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
/* initialize the contents of the array to a different value */
for (i=0; i<NY*NX; i++) buf[i] = -1;
/* read the entire array */
err = ncmpi_get_var_int_all(ncid, varid[1], buf); CHECK_ERR
/* check if the contents of buf are expected */
expected = 10;
for (j=6; j<9; j++) {
for (i=7; i<17; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d]=%d, should be %d\n",
rank, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
expected = 50;
for (j=0; j<8; j++) {
for (i=3; i<5; i++) {
if (buf[j*NX+i] != expected) {
printf("%d: Unexpected read buf[%d][%d]=%d, should be %d\n",
rank, j, i, buf[j*NX+i], expected);
nerrs++;
}
expected++;
}
}
err = ncmpi_close(ncid); CHECK_ERR
free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR && malloc_size > 0) {
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", malloc_size);
ncmpi_inq_malloc_list();
}
fn_exit:
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char **argv)
{
char filename[256];
int err, nerrs=0, ncid, dimid[NDIMS], varid[5], ndims=NDIMS;
int i, j, k, nprocs, rank, req, *buf;
MPI_Offset start[NDIMS] = {0};
MPI_Offset count[NDIMS] = {0};
MPI_Offset stride[NDIMS] = {0};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for NULL stride ", basename(argv[0]));
printf("%-66s ------ ", cmd_str); fflush(stdout);
free(cmd_str);
}
err = ncmpi_create(MPI_COMM_WORLD, filename, 0, MPI_INFO_NULL, &ncid);
CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
CHECK_ERR
err = ncmpi_def_dim(ncid, "X", nprocs*NX, &dimid[1]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v0", NC_INT, ndims, dimid, &varid[0]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v1", NC_INT, ndims, dimid, &varid[1]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v2", NC_INT, ndims, dimid, &varid[2]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v3", NC_INT, ndims, dimid, &varid[3]);
CHECK_ERR
err = ncmpi_def_var(ncid, "v4", NC_INT, ndims, dimid, &varid[4]);
CHECK_ERR
err = ncmpi_enddef(ncid);
CHECK_ERR
start[0] = 0;
start[1] = rank*NX;
count[0] = NY;
count[1] = NX;
buf = (int*) malloc((size_t)NY * NX * sizeof(int));
for (i=0; i<NY*NX; i++) buf[i] = rank+10;
err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf);
CHECK_ERR
CHECK_PUT_BUF
err = ncmpi_put_vars_int_all(ncid, varid[1], start, count, NULL, buf);
CHECK_ERR
CHECK_PUT_BUF
start[0] = 0;
start[1] = rank;
count[0] = NY;
count[1] = NX;
stride[0] = 1;
stride[1] = nprocs;
err = ncmpi_put_vars_int_all(ncid, varid[2], start, count, stride, buf);
CHECK_ERR
CHECK_PUT_BUF
/* test bput_vars */
err = ncmpi_buffer_attach(ncid, NY*NX*sizeof(int));
CHECK_ERR
start[0] = 0;
start[1] = rank*NX;
count[0] = NY;
count[1] = NX;
err = ncmpi_bput_vars_int(ncid, varid[3], start, count, NULL, buf, &req);
CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, NULL);
CHECK_ERR
CHECK_PUT_BUF
start[0] = 0;
start[1] = rank;
count[0] = NY;
count[1] = NX;
stride[0] = 1;
stride[1] = nprocs;
err = ncmpi_bput_vars_int(ncid, varid[4], start, count, stride, buf, &req);
CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, NULL);
CHECK_ERR
CHECK_PUT_BUF
free(buf);
err = ncmpi_buffer_detach(ncid);
CHECK_ERR
buf = (int*) malloc((size_t)NY * NX * nprocs * sizeof(int));
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[0], buf);
CHECK_ERR
/* check read buffer contents */
/* v0 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[1], buf);
CHECK_ERR
/* check read buffer contents */
/* v1 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[2], buf);
CHECK_ERR
/* check read buffer contents */
/* v2 =
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13 ;
*/
for (i=0; i<NY; i++) {
for (k=0; k<NX; k++) {
for (j=0; j<nprocs; j++) {
if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[3], buf);
CHECK_ERR
/* check read buffer contents */
/* v3 =
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13,
* 10, 10, 11, 11, 12, 12, 13, 13 ;
*/
for (i=0; i<NY; i++) {
for (j=0; j<nprocs; j++) {
for (k=0; k<NX; k++) {
if (buf[i*nprocs*NX+j*NX+k] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+j*NX+k, j+10, buf[i*nprocs*NX+j*NX+k]);
nerrs++;
}
}
}
}
memset(buf, 0, (size_t)NY * NX * nprocs * sizeof(int));
err = ncmpi_get_var_int_all(ncid, varid[4], buf);
CHECK_ERR
/* check read buffer contents */
/* v4 =
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13,
* 10, 11, 12, 13, 10, 11, 12, 13 ;
*/
for (i=0; i<NY; i++) {
for (k=0; k<NX; k++) {
for (j=0; j<nprocs; j++) {
if (buf[i*nprocs*NX+k*nprocs+j] != j+10) {
printf("Error at line %d in %s: expected buffer[%d]=%d but got %d\n",
__LINE__,__FILE__,i*nprocs*NX+k*nprocs+j, j+10, buf[i*nprocs*NX+k*nprocs+j]);
nerrs++;
}
}
}
}
err = ncmpi_close(ncid);
CHECK_ERR
free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
if (malloc_size > 0) ncmpi_inq_malloc_list();
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
/*----< main() >------------------------------------------------------------*/
int main(int argc, char **argv)
{
int i, j, err, nerrs=0, rank, nprocs;
int ncid, dimid[2], varid, req, status;
MPI_Offset start[2], count[2], stride[2], imap[2];
int var[6][4];
float k, rh[4][6];
signed char varT[4][6];
char filename[256];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for get/put varm ", basename(argv[0]));
printf("%-66s ------ ", cmd_str); fflush(stdout);
free(cmd_str);
}
#ifdef DEBUG
if (nprocs > 1 && rank == 0)
printf("Warning: %s is designed to run on 1 process\n", argv[0]);
#endif
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA,
MPI_INFO_NULL, &ncid); CHECK_ERR
/* define a variable of a 6 x 4 integer array in the nc file */
err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 4, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR
err = ncmpi_enddef(ncid); CHECK_ERR
/* create a 6 x 4 integer variable in the file with contents:
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23
*/
for (j=0; j<6; j++) for (i=0; i<4; i++) var[j][i] = j*4+i;
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR
if (nprocs > 1) MPI_Barrier(MPI_COMM_WORLD);
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR
/* read the variable back in the matrix transposed way, rh is 4 x 6 */
count[0] = 6; count[1] = 4;
stride[0] = 1; stride[1] = 1;
imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */
for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;
err = ncmpi_iget_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0], &req); CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
err = status; CHECK_ERR
/* check the contents of read */
k = 0.0;
for (i=0; i<6; i++) {
for (j=0; j<4; j++) {
if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
__LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
nerrs++;
break;
}
k += 1.0;
}
}
#ifdef PRINT_ON_SCREEN
/* print the contents of read */
for (j=0; j<4; j++) {
printf("[%2d]: ",j);
for (i=0; i<6; i++) {
printf("%5.1f",rh[j][i]);
}
printf("\n");
}
#endif
/* the stdout should be:
[ 0]: 0.0 4.0 8.0 12.0 16.0 20.0
[ 1]: 1.0 5.0 9.0 13.0 17.0 21.0
[ 2]: 2.0 6.0 10.0 14.0 18.0 22.0
[ 3]: 3.0 7.0 11.0 15.0 19.0 23.0
*/
for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0;
err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); CHECK_ERR
/* check the contents of read */
k = 0.0;
for (i=0; i<6; i++) {
for (j=0; j<4; j++) {
if (rh[j][i] != k) {
#ifdef PRINT_ERR_ON_SCREEN
printf("Error at line %d in %s: expecting rh[%d][%d]=%f but got %f\n",
__LINE__,__FILE__,j,i,k,rh[j][i]);
#endif
nerrs++;
break;
}
k += 1.0;
}
}
#ifdef PRINT_ON_SCREEN
/* print the contents of read */
for (j=0; j<4; j++) {
printf("[%2d]: ",j);
for (i=0; i<6; i++) {
printf("%5.1f",rh[j][i]);
}
printf("\n");
}
#endif
/* the stdout should be:
[ 0]: 0.0 4.0 8.0 12.0 16.0 20.0
[ 1]: 1.0 5.0 9.0 13.0 17.0 21.0
[ 2]: 2.0 6.0 10.0 14.0 18.0 22.0
[ 3]: 3.0 7.0 11.0 15.0 19.0 23.0
*/
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR
err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR
/* testing get_varm(), first zero-out the variable in the file */
memset(&var[0][0], 0, 6*4*sizeof(int));
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR
/* set the contents of the write buffer varT, a 4 x 6 char array
50, 51, 52, 53, 54, 55,
56, 57, 58, 59, 60, 61,
62, 63, 64, 65, 66, 67,
68, 69, 70, 71, 72, 73
*/
for (j=0; j<4; j++) for (i=0; i<6; i++) varT[j][i] = j*6+i + 50;
/* write varT to the NC variable in the matrix transposed way */
start[0] = 0; start[1] = 0;
count[0] = 6; count[1] = 4;
stride[0] = 1; stride[1] = 1;
imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */
if (rank > 0) count[0] = count[1] = 0;
err = ncmpi_iput_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0], &req); CHECK_ERR
err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR
err = status; CHECK_ERR
/* the output from command "ncmpidump -v var test.nc" should be:
var =
50, 56, 62, 68,
51, 57, 63, 69,
52, 58, 64, 70,
53, 59, 65, 71,
54, 60, 66, 72,
55, 61, 67, 73 ;
*/
/* check if the contents of write buffer have been altered */
for (j=0; j<4; j++) {
for (i=0; i<6; i++) {
if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
/* this error is a pnetcdf internal error, if occurs */
printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
__LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
nerrs++;
break;
}
}
}
err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); CHECK_ERR
/* check if the contents of write buffer have been altered */
for (j=0; j<4; j++) {
for (i=0; i<6; i++) {
if (varT[j][i] != j*6+i + 50) {
#ifdef PRINT_ERR_ON_SCREEN
/* this error is a pnetcdf internal error, if occurs */
printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n",
__LINE__,__FILE__,j,i,j*6+i + 50,varT[j][i]);
#endif
nerrs++;
break;
}
}
}
err = ncmpi_close(ncid); CHECK_ERR
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
if (malloc_size > 0) ncmpi_inq_malloc_list();
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char** argv)
{
extern int optind;
char filename[256];
int i, j, verbose=1, rank, nprocs, err, nerrs=0;
int myNX, G_NX, myOff, num_reqs;
int ncid, cmode, varid, dimid[2], *reqs, *sts, **buf;
MPI_Offset start[2], count[2];
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* get command-line arguments */
while ((i = getopt(argc, argv, "hq")) != EOF)
switch(i) {
case 'q': verbose = 0;
break;
case 'h':
default: if (rank==0) usage(argv[0]);
MPI_Finalize();
return 1;
}
if (argv[optind] == NULL) strcpy(filename, "testfile.nc");
else snprintf(filename, 256, "%s", argv[optind]);
/* set an MPI-IO hint to disable file offset alignment for fixed-size
* variables */
MPI_Info_create(&info);
MPI_Info_set(info, "nc_var_align_size", "1");
cmode = NC_CLOBBER | NC_64BIT_DATA;
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
ERR
MPI_Info_free(&info);
/* the global array is NY * (NX * nprocs) */
G_NX = NX * nprocs;
myOff = NX * rank;
myNX = NX;
if (verbose) printf("%2d: myOff=%3d myNX=%3d\n",rank,myOff,myNX);
err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
ERR
err = ncmpi_def_dim(ncid, "X", G_NX, &dimid[1]);
ERR
err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid);
ERR
err = ncmpi_enddef(ncid);
ERR
/* First, fill the entire array with zeros, using a blocking I/O.
Every process writes a subarray of size NY * myNX */
buf = (int**) malloc(myNX * sizeof(int*));
buf[0] = (int*) calloc(NY * myNX, sizeof(int));
start[0] = 0; start[1] = myOff;
count[0] = NY; count[1] = myNX;
err = ncmpi_put_vara_int_all(ncid, varid, start, count, buf[0]);
free(buf[0]);
/* initialize the buffer with rank ID. Also make the case interesting,
by allocating buffers separately */
for (i=0; i<myNX; i++) {
buf[i] = (int*) malloc(NY * sizeof(int));
for (j=0; j<NY; j++) buf[i][j] = rank;
}
reqs = (int*) malloc(myNX * sizeof(int));
sts = (int*) malloc(myNX * sizeof(int));
/* each proc writes myNX single columns of the 2D array */
start[0] = 0; start[1] = rank;
count[0] = NY; count[1] = 1;
if (verbose)
printf("%2d: start=%3lld %3lld count=%3lld %3lld\n",
rank, start[0],start[1], count[0],count[1]);
num_reqs = 0;
for (i=0; i<myNX; i++) {
err = ncmpi_iput_vara_int(ncid, varid, start, count, buf[i],
&reqs[num_reqs++]);
ERR
start[1] += nprocs;
}
err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
ERR
/* check status of all requests */
for (i=0; i<num_reqs; i++)
if (sts[i] != NC_NOERR)
printf("Error at line %d in %s: nonblocking write fails on request %d (%s)\n",
__LINE__,__FILE__,i, ncmpi_strerror(sts[i]));
err = ncmpi_close(ncid); ERR
/* read back using the same access pattern */
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); ERR
err = ncmpi_inq_varid(ncid, "var", &varid); ERR
for (i=0; i<myNX; i++)
for (j=0; j<NY; j++) buf[i][j] = -1;
/* each proc reads myNX single columns of the 2D array */
start[0] = 0; start[1] = rank;
count[0] = NY; count[1] = 1;
num_reqs = 0;
for (i=0; i<myNX; i++) {
err = ncmpi_iget_vara_int(ncid, varid, start, count, buf[i],
&reqs[num_reqs++]);
ERR
start[1] += nprocs;
}
err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
ERR
/* check status of all requests */
for (i=0; i<num_reqs; i++)
if (sts[i] != NC_NOERR)
printf("Error at line %d in %s: nonblocking write fails on request %d (%s)\n",
__LINE__,__FILE__,i, ncmpi_strerror(sts[i]));
for (i=0; i<myNX; i++) {
for (j=0; j<NY; j++)
if (buf[i][j] != rank)
printf("Error at line %d in %s: expect buf[%d][%d]=%d but got %d\n",
__LINE__,__FILE__,i,j,rank,buf[i][j]);
}
err = ncmpi_close(ncid);
ERR
free(sts);
free(reqs);
for (i=0; i<myNX; i++) free(buf[i]);
free(buf);
/* check if there is any PnetCDF internal malloc residue */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Finalize();
return (nerrs > 0);
}
/*----< ncmpii_mgetput_varm() >-----------------------------------------------*/
static int
ncmpii_mgetput_varm(int ncid,
int num,
int varids[], /* [num] */
MPI_Offset* const starts[], /* [num] */
MPI_Offset* const counts[], /* [num] */
MPI_Offset* const strides[], /* [num] */
MPI_Offset* const imaps[], /* [num] */
void *bufs[], /* [num] */
MPI_Offset bufcounts[], /* [num] */
MPI_Datatype datatypes[], /* [num] */
int rw_flag, /* WRITE_REQ or READ_REQ */
int io_method) /* COLL_IO or INDEP_IO */
{
int i, status=NC_NOERR, *req_ids=NULL, *statuses=NULL;
NC *ncp=NULL;
CHECK_NCID
if (rw_flag == WRITE_REQ)
CHECK_WRITE_PERMISSION
if (NC_indef(ncp)) return NC_EINDEFINE;
/* check to see that the desired MPI file handle is opened */
if (io_method == COLL_IO)
CHECK_COLLECTIVE_FH
else
CHECK_INDEP_FH
if (num > 0) {
req_ids = (int*) NCI_Malloc(2 * num * sizeof(int));
statuses = req_ids + num;
}
/* for each request call ncmpi_igetput_varm() */
for (i=0; i<num; i++) {
NC_var *varp;
MPI_Offset *start, *count;
CHECK_VARID(varids[i], varp)
if (starts == NULL) { /* var */
GET_FULL_DIMENSIONS
status = ncmpii_igetput_varm(ncp, varp, start, count, NULL,
NULL, bufs[i], bufcounts[i],
datatypes[i], &req_ids[i], rw_flag, 0);
if (varp->ndims > 0) NCI_Free(start);
} else if (counts == NULL) { /* var1 */
GET_FULL_DIMENSIONS
GET_ONE_COUNT
status = ncmpii_igetput_varm(ncp, varp, starts[i], count, NULL,
NULL, bufs[i], bufcounts[i],
datatypes[i], &req_ids[i], rw_flag, 0);
if (varp->ndims > 0) NCI_Free(count);
} else if (strides == NULL) { /* vara */
status = ncmpii_igetput_varm(ncp, varp, starts[i], counts[i], NULL,
NULL, bufs[i], bufcounts[i],
datatypes[i], &req_ids[i], rw_flag, 0);
} else if (imaps == NULL) { /* vars */
status = ncmpii_igetput_varm(ncp, varp, starts[i], counts[i],
strides[i], NULL, bufs[i], bufcounts[i],
datatypes[i], &req_ids[i], rw_flag, 0);
} else { /* varm */
status = ncmpii_igetput_varm(ncp, varp, starts[i], counts[i],
strides[i], imaps[i], bufs[i],
bufcounts[i], datatypes[i],
&req_ids[i], rw_flag, 0);
}
}
if (status != NC_NOERR)
return status;
if (io_method == COLL_IO)
status = ncmpi_wait_all(ncid, num, req_ids, statuses);
else
status = ncmpi_wait(ncid, num, req_ids, statuses);
if (status != NC_NOERR)
return status;
if (num > 0)
NCI_Free(req_ids);
for (i=0; i<num; i++)
if (statuses[i] != NC_NOERR)
return statuses[i];
return NC_NOERR;
}
