/*
* Close a file through the NCMPI interface.
*/
static void NCMPI_Close(void *fd, IOR_param_t * param)
{
if (param->collective == FALSE) {
NCMPI_CHECK(ncmpi_end_indep_data(*(int *)fd),
"cannot disable independent data mode");
}
NCMPI_CHECK(ncmpi_close(*(int *)fd), "cannot close file");
free(fd);
}
//----------------------------------------------------------------
// Return a negative value when failed, otherwise return 0
int read_var_from_netcdf(int file_id, const char *var_name, struct Type type)
{
int TIMES=1;
int LATS=local_box_size[1];
int LONS=local_box_size[0];
assert(var_name != 0);
assert(var_data != 0);
int varidp,ndims,nvars,ngatts,unlimited;
nc_type xtypep;
//int dataset_id = H5Dopen2(file_id, var_name, H5P_DEFAULT);
int dataset_id = ncmpi_inq_varid(file_id, var_name, &varidp);
MPI_Offset start[]={local_box_offset[2],local_box_offset[1],local_box_offset[0]};
MPI_Offset count[]={TIMES,LATS,LONS};
dataset_id = ncmpi_inq_vartype(file_id,varidp, &xtypep);
// if (dataset_id !=0)
// terminate_with_error_msg("ERROR: Failed to open NetCDF dataset for variable %s\n", var_name);
int read_error = 0;
if (type.atomic_type == DOUBLE)
read_error = ncmpi_get_vara_double(file_id, varidp, start, count, var_data);
else if (type.atomic_type == FLOAT){
ncmpi_begin_indep_data(file_id);
read_error = ncmpi_get_vara_float(file_id, varidp, start, count, (float *)var_data);
ncmpi_end_indep_data(file_id);
if (read_error != NC_NOERR)
terminate_with_error_msg("ERROR: Can not read the data for the variable %s \n", var_name);
}
else if (type.atomic_type == INT)
read_error = ncmpi_get_vara_int(file_id, varidp, start, count, var_data);
// else if (type.atomic_type == UINT)
// read_error = ncmpi_get_vara_uint(file_id, varidp, start, count, var_data);
// else if (type.atomic_type == CHAR)
// read_error = ncmpi_get_vara_char(file_id, varidp, start, count, var_data);
else if (type.atomic_type == UCHAR)
read_error = ncmpi_get_vara_uchar(file_id, varidp, start, count, var_data);
else
terminate_with_error_msg("ERROR: Unsupported type. Type = %d\n", type.atomic_type);
if (read_error < 0)
return -1;
return 0;
}
void BIL_Pio_read_nc_blocks(MPI_Comm all_readers_comm, MPI_Comm io_comm,
int num_blocks, BIL_Block* blocks) {
int i;
for (i = 0; i < num_blocks; i++) {
int fp;
BIL_Timing_fopen_start(all_readers_comm);
assert(ncmpi_open(io_comm, blocks[i].file_name, NC_NOWRITE,
BIL->io_hints, &fp) == NC_NOERR);
BIL_Timing_fopen_stop(all_readers_comm);
ncmpi_begin_indep_data(fp);
// Find the id, type, and size of the variable.
int var_id;
assert(ncmpi_inq_varid(fp, blocks[i].var_name, &var_id) == NC_NOERR);
nc_type var_type;
assert(ncmpi_inq_vartype(fp, var_id, &var_type) == NC_NOERR);
// Create extra variables specifically for the netCDF API.
MPI_Offset nc_dim_starts[BIL_MAX_NUM_DIMS];
MPI_Offset nc_dim_sizes[BIL_MAX_NUM_DIMS];
int j;
for (j = 0; j < blocks[i].num_dims; j++) {
nc_dim_starts[j] = blocks[i].starts[j];
nc_dim_sizes[j] = blocks[i].sizes[j];
}
MPI_Datatype nc_var_type;
BIL_Pio_nc_to_mpi_type(var_type, &nc_var_type, &(blocks[i].var_size));
// Allocate room for data and read it independently.
blocks[i].data = BIL_Misc_malloc(blocks[i].total_size * blocks[i].var_size);
BIL_Timing_io_start(all_readers_comm);
assert(ncmpi_get_vara(fp, var_id, nc_dim_starts, nc_dim_sizes,
blocks[i].data, blocks[i].total_size,
nc_var_type) == NC_NOERR);
BIL_Timing_io_stop(all_readers_comm,
blocks[i].total_size * blocks[i].var_size);
// Clean up.
ncmpi_end_indep_data(fp);
ncmpi_close(fp);
}
}
void
ITLRandomField::_AddTimeStamp(
int iTimeStamp
)
{
// ADD-BY-LEETEN 09/01/2011-BEGIN
if( iNcId >= 0 )
{
// write the time stamp
#ifndef WITH_PNETCDF // ADD-BY-LEETEN 08/12/2011
size_t uStart = viTimeStamps.size();
size_t uCount = 1;
ASSERT_NETCDF(nc_put_vara_int(
iNcId,
iNcTimeVarId,
&uStart,
&uCount,
&iTimeStamp));
// ADD-BY-LEETEN 08/12/2011-BEGIN
#else // #ifndef WITH_PNETCDF
MPI_Offset uStart = viTimeStamps.size();
MPI_Offset uCount = 1;
ASSERT_NETCDF(ncmpi_begin_indep_data(iNcId));
if( 0 == iRank )
ASSERT_NETCDF(ncmpi_put_vara_int(
iNcId,
iNcTimeVarId,
&uStart,
&uCount,
&iTimeStamp));
ASSERT_NETCDF(ncmpi_end_indep_data(iNcId));
#endif // #ifndef WITH_PNETCDF
// ADD-BY-LEETEN 08/12/2011-END
}
// ADD-BY-LEETEN 09/01/2011-END
viTimeStamps.push_back(iTimeStamp);
}
int main(int argc, char **argv) {
int i, j;
int status;
int ncid1, ncid2;
int ndims, nvars, ngatts, unlimdimid;
char name[NC_MAX_NAME];
nc_type type, vartypes[NC_MAX_VARS];
MPI_Offset attlen;
MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS];
void *valuep;
int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS];
int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */
int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS];
params opts;
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing independent read ... ");
parse_read_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
/* Read a netCDF file and write it out to another file */
/**
* Open the input dataset - ncid1:
* File name: "../data/test_int.nc"
* Dataset API: Collective
* And create the output dataset - ncid2:
* File name: "testread.nc"
* Dataset API: Collective
*/
status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Inquire the dataset definitions of input dataset AND
* Add dataset definitions for output dataset.
*/
status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid);
if (status != NC_NOERR) handle_error(status);
/* Inquire global attributes, assume CHAR attributes. */
for (i = 0; i < ngatts; i++) {
status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
/* Inquire dimension */
for (i = 0; i < ndims; i++) {
status = ncmpi_inq_dim(ncid1, i, name, &dimlen);
if (status != NC_NOERR) handle_error(status);
if (i == unlimdimid)
dimlen = NC_UNLIMITED;
status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i);
if (status != NC_NOERR) handle_error(status);
}
/* Inquire variables */
for (i = 0; i < nvars; i++) {
status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i);
if (status != NC_NOERR) handle_error(status);
/* var attributes, assume CHAR attributes */
for (j = 0; j < varnatts[i]; j++) {
status = ncmpi_inq_attname(ncid1, varids[i], j, name);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_inq_att (ncid1, varids[i], name, &type, &attlen);
if (status != NC_NOERR) handle_error(status);
switch (type) {
case NC_CHAR:
valuep = (void *)malloc(attlen * sizeof(char));
status = ncmpi_get_att_text(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_SHORT:
valuep = (void *)malloc(attlen * sizeof(short));
status = ncmpi_get_att_short(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(attlen * sizeof(int));
status = ncmpi_get_att_int(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(attlen * sizeof(float));
status = ncmpi_get_att_float(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(attlen * sizeof(double));
status = ncmpi_get_att_double(ncid1, varids[i], name, valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
}
/**
* End Define Mode (switch to data mode) for output dataset
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Read data of variables from input dataset (assume INT variables)
* Write the data out to the corresponding variables in the output dataset
*
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, *), 25*100 from 100*100
* cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*
* Data Mode API: non-collective
*/
status = ncmpi_begin_indep_data(ncid1);
if (status != NC_NOERR) handle_error(status);
status =ncmpi_begin_indep_data(ncid2);
if (status != NC_NOERR) handle_error(status);
for (i = 0; i < NC_MAX_VAR_DIMS; i++)
start[i] = 0;
for (i = 0; i < nvars; i++) {
varsize = 1;
for (j = 0; j < varndims[i]; j++) {
status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j);
if (status != NC_NOERR) handle_error(status);
if (j == 0) {
shape[j] /= nprocs;
start[j] = shape[j] * rank;
}
varsize *= shape[j];
}
switch (vartypes[i]) {
case NC_CHAR:
break;
case NC_SHORT:
valuep = (void *)malloc(varsize * sizeof(short));
status = ncmpi_get_vara_short(ncid1, i, start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_short(ncid2, varids[i],
start, shape, (short *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_INT:
valuep = (void *)malloc(varsize * sizeof(int));
status = ncmpi_get_vara_int(ncid1, i, start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid2, varids[i],
start, shape, (int *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_FLOAT:
valuep = (void *)malloc(varsize * sizeof(float));
status = ncmpi_get_vara_float(ncid1, i, start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_float(ncid2, varids[i],
start, shape, (float *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
case NC_DOUBLE:
valuep = (void *)malloc(varsize * sizeof(double));
status = ncmpi_get_vara_double(ncid1, i, start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid2, varids[i],
start, shape, (double *)valuep);
if (status != NC_NOERR) handle_error(status);
free(valuep);
break;
default:
;
/* handle unexpected types */
}
}
status = ncmpi_end_indep_data(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_end_indep_data(ncid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_sync(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_sync(ncid2);
if (status != NC_NOERR) handle_error(status);
/**
* Close the datasets
* Dataset API: collective
*/
status = ncmpi_close(ncid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_close(ncid2);
if (status != NC_NOERR) handle_error(status);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname);
MPI_Finalize();
return 0;
}
int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0;
int ncid, varid, dimid[2], req, st;
MPI_Offset start[2], count[2], stride[2];
unsigned char buffer[NY][NX];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 1;
}
if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
else strcpy(filename, "testfile.nc");
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for ncmpi_end_indep_data ", basename(argv[0]));
printf("%-66s ------ ",cmd_str);
free(cmd_str);
}
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA,
MPI_INFO_NULL, &ncid);
CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var", NC_UBYTE, NDIMS, dimid, &varid); CHECK_ERR
err = ncmpi_enddef(ncid); CHECK_ERR
for (i=0; i<NY; i++) for (j=0; j<NX; j++) buffer[i][j] = rank+10;
start[0] = 0; start[1] = NX*rank;
count[0] = NY/2; count[1] = NX/2;
stride[0] = 2; stride[1] = 2;
err = ncmpi_buffer_attach(ncid, NY*NX); CHECK_ERR
err = ncmpi_begin_indep_data(ncid); CHECK_ERR
err = ncmpi_bput_vars_uchar(ncid, varid, start, count, stride,
&buffer[0][0], &req);
CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_end_indep_data(ncid); CHECK_ERR
/* calling wait API after exiting independent data mode on purpose */
err = ncmpi_wait_all(ncid, 1, &req, &st); CHECK_ERR
err = st; CHECK_ERR
/* check if write buffer contents have been altered */
for (i=0; i<NY; i++)
for (j=0; j<NX; j++) {
if (buffer[i][j] != rank+10) {
printf("Error at line %d in %s: put buffer[%d][%d]=%hhu altered, should be %d\n",
__LINE__,__FILE__,i,j,buffer[i][j],rank+10);
nerrs++;
}
}
err = ncmpi_buffer_detach(ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0) {
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
ncmpi_inq_malloc_list();
}
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return (nerrs > 0);
}
void
ITLRandomField::_CloseNetCdf
(
)
{
// MOD-BY-LEETEN 09/01/2011-FROM:
// if( iNcId > 0 )
// TO:
if( iNcId >= 0 )
// MOD-BY-LEETEN 09/01/2011-END
{
// write the time stamp
TBuffer<int> piTemp;
piTemp.alloc(this->IGetNrOfTimeStamps());
for(int t = 0; t < (int)piTemp.USize(); t++)
piTemp[t] = this->viTimeStamps[t];
#ifndef WITH_PNETCDF // ADD-BY-LEETEN 08/12/2011
#if 0 // DEL-BY-LEETEN 09/01/2011-BEGIN
// since the time step wil lbe written earlier, this part can be removed
size_t uStart = 0;
size_t uCount = piTemp.USize();
ASSERT_NETCDF(nc_put_vara_int(
iNcId,
iNcTimeVarId,
&uStart,
&uCount,
&piTemp[0]));
#endif // DEL-BY-LEETEN 09/01/2011-END
/* Close the file. */
ASSERT_NETCDF(nc_close(iNcId));
// ADD-BY-LEETEN 08/12/2011-BEGIN
#else // #ifndef WITH_PNETCDF
#if 0 // DEL-BY-LEETEN 09/01/2011-BEGIN
MPI_Offset uStart = 0;
MPI_Offset uCount = piTemp.USize();
ASSERT_NETCDF(ncmpi_begin_indep_data(iNcId));
if( 0 == iRank )
ASSERT_NETCDF(ncmpi_put_vara_int(
iNcId,
iNcTimeVarId,
&uStart,
&uCount,
&piTemp[0]));
ASSERT_NETCDF(ncmpi_end_indep_data(iNcId));
#endif // DEL-BY-LEETEN 09/01/2011-END
/* Close the file. */
ASSERT_NETCDF(ncmpi_close(iNcId));
#endif // #ifndef WITH_PNETCDF
// ADD-BY-LEETEN 08/12/2011-END
// MOD-BY-LEETEN 09/01/2011-FROM:
// iNcId = 0;
// TO:
iNcId = -1;
// MOD-BY-LEETEN 09/01/2011-END
}
};
/*
The test write a NP * NP matrix M, NP is the number of process:
put_vara:
Process N write N copy of it's rank to row N ([N, 0...WIDTH]) using different APIs on different variable
final result should be:
0 0 0 0 ...
1 1 1 1 ...
2 2 2 2 ...
.
.
.
*/
int simpletest(char* fname, int enable_log) {
int buffer[MAXPROCESSES];
MPI_Offset start[2], count[2];
int i, j, ret, errlen;
int NProc, MyRank, NP; // Total process; Rank
int fid; // Data set ID
int did[2]; // IDs of dimension
int vid; // IDs for variables
int dims[2];
char tmp[1024], tmp2[1024];
MPI_Info Info;
MPI_Comm_size(MPI_COMM_WORLD, &NP);
MPI_Comm_rank(MPI_COMM_WORLD, &MyRank);
if (NP == 1) { // Act if there is WIDTH processes for easy debugging. Most debugger supports only single processes.
NProc = SINGLEPROCNP;
MyRank = SINGLEPROCRANK;
}
else{
NProc = NP;
}
if (MyRank < MAXPROCESSES) {
// Ensure each process have a independent buffer directory
MPI_Info_create(&Info);
if (enable_log) {
MPI_Info_set(Info, "pnetcdf_log", "enable");
}
// Create new cdf file
ret = ncmpi_create(MPI_COMM_WORLD, fname, NC_CLOBBER, Info, &fid);
if (ret != NC_NOERR) {
printf("Error create file\n");
goto ERROR;
}
ret = ncmpi_set_fill(fid, NC_FILL, NULL);
if (ret != NC_NOERR) {
printf("Error set fill\n");
goto ERROR;
}
ret = ncmpi_def_dim(fid, "X", NProc, did); // X
if (ret != NC_NOERR) {
printf("Error def dim X\n");
goto ERROR;
}
ret = ncmpi_def_dim(fid, "Y", NProc, did + 1); // Y
if (ret != NC_NOERR) {
printf("Error def dim Y\n");
goto ERROR;
}
ret = ncmpi_def_var(fid, "M", NC_INT, 2, did, vid);
if (ret != NC_NOERR) {
printf("Error def var M\n");
goto ERROR;
}
ret = ncmpi_enddef(fid);
if (ret != NC_NOERR) {
printf("Error enddef\n");
goto ERROR;
}
// Indep mode
ret = ncmpi_begin_indep_data(fid);
if (ret != NC_NOERR) {
printf("Error begin indep\n");
goto ERROR;
}
// We all write rank from now on
for (i = 0; i < NProc; i++) {
buffer[i] = MyRank;
}
// put_vara
count[0] = 1;
count[1] = NProc;
start[0] = MyRank;
start[1] = 0;
ret = ncmpi_put_vara_int(fid, vid, start, count, buffer);
if (ret != NC_NOERR) {
MPI_Error_string(ret, tmp, &errlen);
printf("Error put_varn: %d\n%s\n", errlen, tmp);
goto ERROR;
}
// Collective mode
ncmpi_end_indep_data(fid);
if (ret != NC_NOERR) {
printf("Error end indep");
goto ERROR;
}
ncmpi_close(fid); // Close file
if (ret != NC_NOERR) {
printf("Error close");
goto ERROR;
}
}
ERROR:
return 0;
}
int main(int argc, char **argv) {
int i, j, k;
int status;
int ncid;
int dimid1, dimid2, dimid3, udimid;
int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1];
MPI_Offset square_start[2], cube_start[3] = {0, 0, 0};
MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50};
MPI_Offset xytime_start[3] = {0, 0, 0};
MPI_Offset xytime_count[3] = {100, 50, 50};
MPI_Offset time_start[1], time_count[1] = {25};
int square_id, cube_id, xytime_id, time_id;
static char title[] = "example netCDF dataset";
static char description[] = "2-D integer array";
int data[100][50][50], buffer[100];
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
params opts;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing independent write ... ");
parse_write_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
/**
* Create the dataset
* File name: "testwrite.nc"
* Dataset API: Collective
*/
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Create a global attribute:
* :title = "example netCDF dataset";
*/
status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title",
strlen(title), title);
if (status != NC_NOERR) handle_error(status);
/**
* Add 4 pre-defined dimensions:
* x = 100, y = 100, z = 100, time = NC_UNLIMITED
*/
status = ncmpi_def_dim(ncid, "x", 100L, &dimid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "y", 100L, &dimid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "z", 100L, &dimid3);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid);
if (status != NC_NOERR) handle_error(status);
/**
* Define the dimensionality and then add 4 variables:
* square(x, y), cube(x,y,z), time(time), xytime(time, x, y)
*/
square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1;
square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2;
cube_dim[2] = dimid3;
xytime_dim[0] = udimid;
time_dim[0] = udimid;
status = ncmpi_def_var (ncid, "square", NC_INT, 2, square_dim, &square_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "cube", NC_INT, 3, cube_dim, &cube_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "xytime", NC_INT, 3, xytime_dim, &xytime_id);
if (status != NC_NOERR) handle_error(status);
/**
* Add an attribute for variable:
* square: decsription = "2-D integer array"
*/
/*
status = ncmpi_put_att_text (ncid, square_id, "description",
strlen(description), description);
if (status != NC_NOERR) handle_error(status);
*/
/**
* End Define Mode (switch to data mode)
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, Block), 50*50 from 100*100
* cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*/
square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50;
square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50;
time_start[0] = (rank%4) * 25;
/**
* Packing data in the buffer
*/
/* Data for variable: time */
for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ )
buffer[i - time_start[0]] = i;
/* Data for variable: square, cube and xytime */
for ( i = 0; i < 100; i++ )
for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ )
for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ )
data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k;
/**
* Write data into variables: square, cube, time and xytime
* Access Method: subarray
* Data Mode API: non-collective
*/
status = ncmpi_begin_indep_data(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid, square_id,
square_start, square_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid, cube_id,
cube_start, cube_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid, time_id,
time_start, time_count,
(void *)buffer);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int(ncid, xytime_id,
xytime_start, xytime_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_end_indep_data(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_redef(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_att_text (ncid, square_id, "description",
strlen(description), description);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Close the dataset
* Dataset API: collective
*/
status = ncmpi_close(ncid);
if (status != NC_NOERR) handle_error(status);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname);
MPI_Finalize();
return 0;
}
int
main(int argc, char **argv) { /* create foo.nc */
int stat; /* return status */
int ncid; /* netCDF id */
/* dimension ids */
int lat_dim;
int lon_dim;
int time_dim;
/* dimension lengths */
size_t lat_len = 10;
size_t lon_len = 5;
size_t time_len = NC_UNLIMITED;
/* variable ids */
int lat_id;
int lon_id;
int time_id;
int z_id;
int t_id;
int p_id;
int rh_id;
/* rank (number of dimensions) for each variable */
# define RANK_lat 1
# define RANK_lon 1
# define RANK_time 1
# define RANK_z 3
# define RANK_t 3
# define RANK_p 3
# define RANK_rh 3
/* variable shapes */
int lat_dims[RANK_lat];
int lon_dims[RANK_lon];
int time_dims[RANK_time];
int z_dims[RANK_z];
int t_dims[RANK_t];
int p_dims[RANK_p];
int rh_dims[RANK_rh];
/* attribute vectors */
double z_valid_range[2];
double p__FillValue[1];
int rh__FillValue[1];
int stat=0;
MPI_Init(&argc, &argv);
/* enter define mode */
stat = ncmpi_create(MPI_COMM_WORLD, "foo.nc", NC_CLOBBER, MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
/* define dimensions */
stat = ncmpi_def_dim(ncid, "lat", lat_len, &lat_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "lon", lon_len, &lon_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "time", time_len, &time_dim);
check_err(stat,__LINE__,__FILE__);
/* define variables */
lat_dims[0] = lat_dim;
stat = ncmpi_def_var(ncid, "lat", NC_INT, RANK_lat, lat_dims, &lat_id);
check_err(stat,__LINE__,__FILE__);
lon_dims[0] = lon_dim;
stat = ncmpi_def_var(ncid, "lon", NC_INT, RANK_lon, lon_dims, &lon_id);
check_err(stat,__LINE__,__FILE__);
time_dims[0] = time_dim;
stat = ncmpi_def_var(ncid, "time", NC_INT, RANK_time, time_dims, &time_id);
check_err(stat,__LINE__,__FILE__);
z_dims[0] = time_dim;
z_dims[1] = lat_dim;
z_dims[2] = lon_dim;
stat = ncmpi_def_var(ncid, "z", NC_FLOAT, RANK_z, z_dims, &z_id);
check_err(stat,__LINE__,__FILE__);
t_dims[0] = time_dim;
t_dims[1] = lat_dim;
t_dims[2] = lon_dim;
stat = ncmpi_def_var(ncid, "t", NC_FLOAT, RANK_t, t_dims, &t_id);
check_err(stat,__LINE__,__FILE__);
p_dims[0] = time_dim;
p_dims[1] = lat_dim;
p_dims[2] = lon_dim;
stat = ncmpi_def_var(ncid, "p", NC_DOUBLE, RANK_p, p_dims, &p_id);
check_err(stat,__LINE__,__FILE__);
rh_dims[0] = time_dim;
rh_dims[1] = lat_dim;
rh_dims[2] = lon_dim;
stat = ncmpi_def_var(ncid, "rh", NC_INT, RANK_rh, rh_dims, &rh_id);
check_err(stat,__LINE__,__FILE__);
/* assign attributes */
stat = ncmpi_put_att_text(ncid, lat_id, "units", 13, "degrees_north");
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_put_att_text(ncid, lon_id, "units", 12, "degrees_east");
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_put_att_text(ncid, time_id, "units", 7, "seconds");
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_put_att_text(ncid, z_id, "units", 6, "meters");
check_err(stat,__LINE__,__FILE__);
z_valid_range[0] = 0;
z_valid_range[1] = 5000;
stat = ncmpi_put_att_double(ncid, z_id, "valid_range", NC_DOUBLE, 2, z_valid_range);
check_err(stat,__LINE__,__FILE__);
p__FillValue[0] = -9999;
stat = ncmpi_put_att_double(ncid, p_id, "_FillValue", NC_DOUBLE, 1, p__FillValue);
check_err(stat,__LINE__,__FILE__);
rh__FillValue[0] = -1;
stat = ncmpi_put_att_int(ncid, rh_id, "_FillValue", NC_INT, 1, rh__FillValue);
check_err(stat,__LINE__,__FILE__);
/* leave define mode */
stat = ncmpi_enddef (ncid);
check_err(stat,__LINE__,__FILE__);
{ /* store lat */
static int lat[] = {0, 10, 20, 30, 40, 50, 60, 70, 80, 90};
ncmpi_begin_indep_data(ncid);
stat = ncmpi_put_var_int(ncid, lat_id, lat);
ncmpi_end_indep_data(ncid);
check_err(stat,__LINE__,__FILE__);
}
{ /* store lon */
static int lon[] = {-140, -118, -96, -84, -52};
ncmpi_begin_indep_data(ncid);
stat = ncmpi_put_var_int(ncid, lon_id, lon);
ncmpi_end_indep_data(ncid);
check_err(stat,__LINE__,__FILE__);
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
MPI_Finalize();
return 0;
}
/*----< main() >------------------------------------------------------------*/
int main(int argc, char **argv) {
char filename[256];
int i, j, err, ncid, varid0, varid1, varid2, dimids[2], nerrs=0;
int rank, nprocs, debug=0, blocklengths[2], **buf, *bufptr;
int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2];
MPI_Offset start[2], count[2];
MPI_Aint a0, a1, disps[2];
MPI_Datatype buftype, ghost_buftype, rec_filetype, fix_filetype;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (argc > 2) {
if (!rank) printf("Usage: %s [filename]\n",argv[0]);
MPI_Finalize();
return 0;
}
strcpy(filename, "testfile.nc");
if (argc == 2) strcpy(filename, argv[1]);
MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
if (rank == 0) {
char cmd_str[256];
sprintf(cmd_str, "*** TESTING C %s for flexible put and get ", argv[0]);
printf("%-66s ------ ", cmd_str); fflush(stdout);
}
buf = (int**)malloc(NY * sizeof(int*));
buf[0] = (int*)malloc(NY * NX * sizeof(int));
for (i=1; i<NY; i++) buf[i] = buf[i-1] + NX;
/* construct various MPI derived data types */
/* construct an MPI derived data type for swapping 1st row with 2nd row */
blocklengths[0] = blocklengths[1] = NX;
MPI_Get_address(buf[1], &a0);
MPI_Get_address(buf[0], &a1);
disps[0] = 0;
disps[1] = a1 - a0;
bufptr = buf[1];
err = MPI_Type_create_hindexed(2, blocklengths, disps, MPI_INT, &buftype);
if (err != MPI_SUCCESS) printf("MPI error MPI_Type_create_hindexed\n");
MPI_Type_commit(&buftype);
start[0] = 0; start[1] = NX*rank;
count[0] = 2; count[1] = NX;
if (debug) printf("put start=%lld %lld count=%lld %lld\n",start[0],start[1],count[0],count[1]);
/* create a file type for the fixed-size variable */
array_of_sizes[0] = 2;
array_of_sizes[1] = NX*nprocs;
array_of_subsizes[0] = count[0];
array_of_subsizes[1] = count[1];
array_of_starts[0] = start[0];
array_of_starts[1] = start[1];
MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes,
array_of_starts, MPI_ORDER_C,
MPI_INT, &fix_filetype);
MPI_Type_commit(&fix_filetype);
/* create a buftype with ghost cells on each side */
array_of_sizes[0] = count[0]+4;
array_of_sizes[1] = count[1]+4;
array_of_subsizes[0] = count[0];
array_of_subsizes[1] = count[1];
array_of_starts[0] = 2;
array_of_starts[1] = 2;
MPI_Type_create_subarray(2, array_of_sizes, array_of_subsizes,
array_of_starts, MPI_ORDER_C,
MPI_INT, &ghost_buftype);
MPI_Type_commit(&ghost_buftype);
/* create a new file for write */
err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL,
&ncid); ERR
/* define a 2D array */
err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimids[0]); ERR
err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimids[1]); ERR
err = ncmpi_def_var(ncid, "rec_var", NC_INT, 2, dimids, &varid0); ERR
err = ncmpi_def_var(ncid, "dummy_rec", NC_INT, 2, dimids, &varid2); ERR
err = ncmpi_def_dim(ncid, "FIX_DIM", 2, &dimids[0]); ERR
err = ncmpi_def_var(ncid, "fix_var", NC_INT, 2, dimids, &varid1); ERR
err = ncmpi_enddef(ncid); ERR
/* create a file type for the record variable */
int *array_of_blocklengths=(int*) malloc(count[0]*sizeof(int));
MPI_Aint *array_of_displacements=(MPI_Aint*) malloc(count[0]*sizeof(MPI_Aint));
MPI_Offset recsize;
err = ncmpi_inq_recsize(ncid, &recsize);
for (i=0; i<count[0]; i++) {
array_of_blocklengths[i] = count[1];
array_of_displacements[i] = start[1]*sizeof(int) + recsize * i;
}
MPI_Type_create_hindexed(2, array_of_blocklengths, array_of_displacements,
MPI_INT, &rec_filetype);
MPI_Type_commit(&rec_filetype);
free(array_of_blocklengths);
free(array_of_displacements);
/* initialize the contents of the array */
for (j=0; j<NY; j++) for (i=0; i<NX; i++) buf[j][i] = rank*100 + j*10 + i;
/* write the record variable */
err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR
/* check if the contents of buf are altered */
CHECK_VALUE
/* check if root process can write to file header in data mode */
err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR
/* write the fixed-size variable */
err = ncmpi_put_vard_all(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR
/* check if the contents of buf are altered */
CHECK_VALUE
/* check if root process can write to file header in data mode */
err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR
/* test the same routines in independent data mode */
err = ncmpi_begin_indep_data(ncid); ERR
err = ncmpi_put_vard(ncid, varid0, rec_filetype, bufptr, 1, buftype); ERR
CHECK_VALUE
err = ncmpi_rename_var(ncid, varid0, "rec_VAR"); ERR
err = ncmpi_put_vard(ncid, varid1, fix_filetype, bufptr, 1, buftype); ERR
CHECK_VALUE
err = ncmpi_rename_var(ncid, varid0, "rec_var"); ERR
err = ncmpi_end_indep_data(ncid); ERR
err = ncmpi_close(ncid); ERR
/* open the same file and read back for validate */
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL,
&ncid); ERR
err = ncmpi_inq_varid(ncid, "rec_var", &varid0); ERR
err = ncmpi_inq_varid(ncid, "fix_var", &varid1); ERR
nerrs += get_var_and_verify(ncid, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype);
nerrs += get_var_and_verify(ncid, varid1, start, count, buf, buftype, ghost_buftype, fix_filetype);
err = ncmpi_close(ncid); ERR
MPI_Type_free(&rec_filetype);
MPI_Type_free(&fix_filetype);
MPI_Type_free(&buftype);
MPI_Type_free(&ghost_buftype);
free(buf[0]); free(buf);
/* check if PnetCDF freed all internal malloc */
MPI_Offset malloc_size, sum_size;
err = ncmpi_inq_malloc_size(&malloc_size);
if (err == NC_NOERR) {
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && sum_size > 0)
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
sum_size);
}
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (nerrs) printf(FAIL_STR,nerrs);
else printf(PASS_STR);
}
MPI_Finalize();
return 0;
}
int
main(int argc, char ** argv)
{
/* IDs for the netCDF file, dimensions, and variables. */
int nprocs, rank;
int ncid;
int lon_dimid, lat_dimid, lvl_dimid, rec_dimid;
int lat_varid, lon_varid, pres_varid, temp_varid;
int dimids[NDIMS];
/* The start and count arrays will tell the netCDF library where to
write our data. */
MPI_Offset start[NDIMS], count[NDIMS];
/* Program variables to hold the data we will write out. We will only
need enough space to hold one timestep of data; one record. */
float pres_out[NLVL][NLAT][NLON];
float temp_out[NLVL][NLAT][NLON];
/* These program variables hold the latitudes and longitudes. */
float lats[NLAT], lons[NLON];
/* Loop indexes. */
int lvl, lat, lon, rec, i = 0;
/* Error handling. */
int retval;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* Create some pretend data. If this wasn't an example program, we
* would have some real data to write, for example, model
* output. */
for (lat = 0; lat < NLAT; lat++)
lats[lat] = START_LAT + 5.*lat;
for (lon = 0; lon < NLON; lon++)
lons[lon] = START_LON + 5.*lon;
for (lvl = 0; lvl < NLVL; lvl++)
for (lat = 0; lat < NLAT; lat++)
for (lon = 0; lon < NLON; lon++)
{
pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i;
temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++;
}
/* Create the file. */
if ((retval = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER, MPI_INFO_NULL, &ncid)))
check_err(retval,__LINE__,__FILE__);
/* Define the dimensions. The record dimension is defined to have
* unlimited length - it can grow as needed. In this example it is
* the time dimension.*/
if ((retval = ncmpi_def_dim(ncid, LVL_NAME, NLVL, &lvl_dimid)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_def_dim(ncid, LON_NAME, NLON, &lon_dimid)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_def_dim(ncid, REC_NAME, NC_UNLIMITED, &rec_dimid)))
check_err(retval,__LINE__,__FILE__);
/* Define the coordinate variables. We will only define coordinate
variables for lat and lon. Ordinarily we would need to provide
an array of dimension IDs for each variable's dimensions, but
since coordinate variables only have one dimension, we can
simply provide the address of that dimension ID (&lat_dimid) and
similarly for (&lon_dimid). */
if ((retval = ncmpi_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid,
&lat_varid)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid,
&lon_varid)))
check_err(retval,__LINE__,__FILE__);
/* Assign units attributes to coordinate variables. */
if ((retval = ncmpi_put_att_text(ncid, lat_varid, UNITS,
strlen(DEGREES_NORTH), DEGREES_NORTH)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_put_att_text(ncid, lon_varid, UNITS,
strlen(DEGREES_EAST), DEGREES_EAST)))
check_err(retval,__LINE__,__FILE__);
/* The dimids array is used to pass the dimids of the dimensions of
the netCDF variables. Both of the netCDF variables we are
creating share the same four dimensions. In C, the
unlimited dimension must come first on the list of dimids. */
dimids[0] = rec_dimid;
dimids[1] = lvl_dimid;
dimids[2] = lat_dimid;
dimids[3] = lon_dimid;
/* Define the netCDF variables for the pressure and temperature
* data. */
if ((retval = ncmpi_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS,
dimids, &pres_varid)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS,
dimids, &temp_varid)))
check_err(retval,__LINE__,__FILE__);
/* Assign units attributes to the netCDF variables. */
if ((retval = ncmpi_put_att_text(ncid, pres_varid, UNITS,
strlen(PRES_UNITS), PRES_UNITS)))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_put_att_text(ncid, temp_varid, UNITS,
strlen(TEMP_UNITS), TEMP_UNITS)))
check_err(retval,__LINE__,__FILE__);
/* End define mode. */
if ((retval = ncmpi_enddef(ncid)))
check_err(retval,__LINE__,__FILE__);
retval = ncmpi_begin_indep_data(ncid);
/* Write the coordinate variable data. This will put the latitudes
and longitudes of our data grid into the netCDF file. */
if ((retval = ncmpi_put_var_float(ncid, lat_varid, &lats[0]))){
check_err(retval,__LINE__,__FILE__);
printf("------------------------\n");
}
if ((retval = ncmpi_put_var_float(ncid, lon_varid, &lons[0])))
check_err(retval,__LINE__,__FILE__);
retval = ncmpi_end_indep_data(ncid);
/* These settings tell netcdf to write one timestep of data. (The
setting of start[0] inside the loop below tells netCDF which
&data[0][0][0]);
timestep to write.) */
count[0] = 1;
count[1] = NLVL;
count[2] = NLAT;
count[3] = NLON;
start[1] = 0;
start[2] = 0;
start[3] = 0;
/* Write the pretend data. This will write our surface pressure and
surface temperature data. The arrays only hold one timestep worth
of data. We will just rewrite the same data for each timestep. In
a real application, the data would change between timesteps. */
for (rec = 0; rec < NREC; rec++)
{
start[0] = rec;
if ((retval = ncmpi_put_vara_float_all(ncid, pres_varid, start, count, &pres_out[0][0][0])))
check_err(retval,__LINE__,__FILE__);
if ((retval = ncmpi_put_vara_float_all(ncid, temp_varid, start, count, &temp_out[0][0][0])))
check_err(retval,__LINE__,__FILE__);
}
/* Close the file. */
if ((retval = ncmpi_close(ncid)))
check_err(retval,__LINE__,__FILE__);
printf("*** SUCCESS writing example file %s!\n", FILE_NAME);
MPI_Finalize();
return 0;
}
int main(int argc, char** argv) {
int i;
double power_M;
int *array_of_sizes, *array_of_subsizes, *array_of_starts;
int ncid, *dimids, varid_1, varid_2;
MPI_Offset *local_starts, *local_edges;
char dimname[20];
nc_type nc_etype;
MPI_Datatype mpi_etype, mpi_subarray;
TEST_NATIVE_ETYPE *buf1, *buf2, *tmpbuf;
void *packbuf;
int packsz;
int packpos;
int total_sz, local_sz;
int nprocs, rank;
int status;
int success, successall;
int malloc_failure, malloc_failure_any;
int request;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* test initializations: nc_file, nc_variables, dimensions, arrays */
parse_args(argc, argv, rank);
TEST_SET_NCMPI_ETYPE(nc_etype, mpi_etype);
#ifdef TEST_NCTYPE
nc_etype = TEST_NCTYPE;
#endif
if (rank == 0) {
printf("testing memory subarray layout ...\n");
}
status = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER,
MPI_INFO_NULL, &ncid);
TEST_HANDLE_ERR(status);
array_of_sizes = (int *)
malloc(sizeof(int)*ndims*4 + sizeof(MPI_Offset)*ndims*4);
array_of_subsizes = array_of_sizes + ndims;
array_of_starts = array_of_subsizes + ndims;
dimids = array_of_starts + ndims;
local_starts = (MPI_Offset *)(dimids + ndims);
local_edges = local_starts + ndims;
total_sz = 1;
power_M = 1;
for (i=0; i<ndims; i++, power_M*=test_m) {
array_of_sizes[i] = (int)(test_n*power_M);
if (array_of_sizes[i] < 1) {
/* lower bound check */
array_of_sizes[i] = 1;
} else if ( (double)total_sz*array_of_sizes[i] > (double)TEST_MAX_INT ){
/* upper bound check */
if (rank == 0) {
fprintf(stderr, "Total size of array is too big to be represented\n");
fprintf(stderr, "Current size = %f, Max size = %d\n",
(double)total_sz*array_of_sizes[i], TEST_MAX_INT);
}
TEST_EXIT(-1);
}
total_sz *= array_of_sizes[i];
sprintf(dimname, "dim_%d", i);
status = ncmpi_def_dim(ncid, dimname,
(MPI_Offset)array_of_sizes[i], dimids+i);
TEST_HANDLE_ERR(status);
}
if (order == MPI_ORDER_FORTRAN) {
/* reverse the filearray dimension, since NC always use C ORDER */
TEST_REVERSE(dimids, ndims, int);
}
status = ncmpi_def_var(ncid, "var_1", nc_etype, ndims, dimids, &varid_1);
TEST_HANDLE_ERR(status);
TEST_REVERSE(dimids, ndims, int);
status = ncmpi_def_var(ncid, "var_2", nc_etype, ndims, dimids, &varid_2);
TEST_HANDLE_ERR(status);
status = ncmpi_enddef(ncid);
TEST_HANDLE_ERR(status);
if (rank == 0) {
printf("\t Filesize = %2.3fMB, MAX_Memory_needed = %2.3fMB\n\n",
2*total_sz*TEST_NCTYPE_LEN(nc_etype)/1024.0/1024.0,
( (2*total_sz + 4*total_sz/nprocs)*sizeof(TEST_NATIVE_ETYPE)
+ total_sz*TEST_NCTYPE_LEN(nc_etype) )/1024.0/1024.0);
}
buf1 = (TEST_NATIVE_ETYPE *)malloc(total_sz*sizeof(TEST_NATIVE_ETYPE)*2);
malloc_failure = (buf1 == NULL ||
(float)total_sz*sizeof(TEST_NATIVE_ETYPE)*2 > TEST_MAX_INT);
MPI_Allreduce(&malloc_failure, &malloc_failure_any, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if (malloc_failure_any) {
if (rank == 0) {
fprintf(stderr, "malloc(%2.3fMB) failed!\n",
(float)total_sz*sizeof(TEST_NATIVE_ETYPE)*2/1024/1024);
fprintf(stderr, "The whole array may be too big for malloc to handle!\n");
fprintf(stderr, "Please choose smaller array size.\n");
}
TEST_EXIT(-1);
}
buf2 = buf1 + total_sz;
for (i=0; i<total_sz; i++)
/* just make sure any type can represent the number */
/* and make it irregular (cycle != power of 2) for random test */
buf1[i] = buf2[i] = (TEST_NATIVE_ETYPE)(i%127);
/* PARTITION and calculate the local target region */
partition_array(ndims,
array_of_sizes, array_of_subsizes, array_of_starts,
nprocs, rank);
local_sz = 1;
for (i=0; i<ndims; i++) {
local_sz *= array_of_subsizes[i];
local_edges[i] = (MPI_Offset)array_of_subsizes[i];
local_starts[i] = (MPI_Offset)array_of_starts[i];
}
if (order == MPI_ORDER_FORTRAN) {
/* reverse the filearray dimension, since NC always use C ORDER */
TEST_REVERSE(local_edges, ndims, MPI_Offset);
TEST_REVERSE(local_starts, ndims, MPI_Offset);
}
/* CREATE local subarray memory view */
if (local_sz == 0)
MPI_Type_contiguous(0, mpi_etype, &mpi_subarray);
else
MPI_Type_create_subarray(ndims,
array_of_sizes,
array_of_subsizes,
array_of_starts,
order,
mpi_etype,
&mpi_subarray);
MPI_Type_commit(&mpi_subarray);
/* PRINT stats */
if (rank == 0) {
printf("Initialization: NDIMS = %d, NATIVE_ETYPE = %s, NC_TYPE = %s\n\n",
ndims, TEST_NATIVE_ETYPE_STR, TEST_GET_NCTYPE_STR(nc_etype));
printf("\t NC Var_1 Shape:\t [");
if (order == MPI_ORDER_C) {
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
} else {
TEST_PRINT_LIST(array_of_sizes, ndims-1, 0, -1);
}
printf("] Always ORDER_C\n");
printf("\t NC Var_2 Shape:\t [");
if (order == MPI_ORDER_C) {
TEST_PRINT_LIST(array_of_sizes, ndims-1, 0, -1);
} else {
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
}
printf("] Always ORDER_C\n");
printf("\t Memory Array Shape:\t [");
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
printf("] %s\n", ((order==MPI_ORDER_C)?"MPI_ORDER_C":"MPI_ORDER_FORTRAN"));
printf("\t Memory Array Copys: buf1 for write, buf2 for read back (and compare)\n");
printf("\n");
printf("Logical Array Partition:\t BLOCK partition along all dimensions\n\n");
printf("Access Pattern (subarray): NPROCS = %d\n\n", nprocs);
}
fflush(stdout);
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<nprocs; i++) {
if (rank == i) {
printf("\t Proc %2d of %2d: starts = [", rank, nprocs);
TEST_PRINT_LIST(local_starts, 0, ndims-1, 1);
printf("], ");
printf("counts = [");
TEST_PRINT_LIST(local_edges, 0, ndims-1, 1);
printf("] \n");
}
fflush(stdout);
/* Synchronizer: processes should print out their stuffs in turn :) */
MPI_Barrier(MPI_COMM_WORLD);
}
if (rank == 0) {
printf("\n");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
/* RESET the target region of buf2 */
MPI_Pack_size(local_sz, mpi_etype, MPI_COMM_SELF, &packsz);
tmpbuf = (TEST_NATIVE_ETYPE *)
malloc(local_sz*sizeof(TEST_NATIVE_ETYPE) + packsz);
packbuf = (void *)(tmpbuf + local_sz);
for (i=0; i<local_sz; i++)
tmpbuf[i] = (TEST_NATIVE_ETYPE)(-1);
packpos = 0;
MPI_Pack((void *)tmpbuf, local_sz, mpi_etype,
packbuf, packsz, &packpos, MPI_COMM_SELF);
packpos = 0;
MPI_Unpack(packbuf, packsz, &packpos, buf2, 1, mpi_subarray, MPI_COMM_SELF);
/* Begin of TEST1: test local write-n-readback */
fflush(stdout);
if (rank == 0) {
printf("TEST1: \n");
}
/* WRITE target region from buf1 */
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t [nonblocking] all procs writing their subarrays into Var_1 ...\n");
}
status = ncmpi_begin_indep_data(ncid);
TEST_HANDLE_ERR(status);
status = ncmpi_iput_vara(ncid, varid_1, local_starts, local_edges,
(const void *)buf1, 1, mpi_subarray, &request);
TEST_HANDLE_ERR(status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O returns ...\n");
fflush(stdout);
}
ncmpi_wait(ncid, 1, &request, &status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O finishes ...\n");
}
status = ncmpi_end_indep_data(ncid);
TEST_HANDLE_ERR(status);
/* READ target region back into buf2 */
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t [nonblocking] all procs reading their subarrays from Var_1 ...\n");
}
status = ncmpi_begin_indep_data(ncid);
TEST_HANDLE_ERR(status);
status = ncmpi_iget_vara(ncid, varid_1, local_starts, local_edges,
(void *)buf2, 1, mpi_subarray, &request);
TEST_HANDLE_ERR(status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O returns ...\n");
fflush(stdout);
}
ncmpi_wait(ncid, 1, &request, &status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O finishes ...\n");
}
status = ncmpi_end_indep_data(ncid);
TEST_HANDLE_ERR(status);
/* COMPARE buf1 and buf2 for equality */
if (memcmp((void *)buf1, (void *)buf2, total_sz*sizeof(TEST_NATIVE_ETYPE)))
success = 0;
else
success = 1;
MPI_Allreduce(&success, &successall, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD);
if (rank == 0) {
if (successall)
printf("\t PASS: memory subarray layout passes test1! \n\n");
else
printf("\t ERROR: memory subarray layout fails test1! \n\n");
}
/* End of TEST1 */
/* test finalization */
ncmpi_close(ncid);
MPI_Type_free(&mpi_subarray);
free(tmpbuf);
free(buf1);
free(array_of_sizes);
MPI_Finalize();
return !successall;
}
