void sdatio_create_file(struct sdatio_file * sfile ) {
/*printf("called\n");*/
int retval;
/*if (0){}*/
/*else {*/
/*char * args;*/
retval = 0;
if (sfile->is_open){
printf("ERROR: The supplied sdatio_file struct corresponds to an already open file.\n");
abort();
}
/*MPI_Init(&retval, &args);*/
if (sfile->is_parallel) {
#ifdef PARALLEL
if ((retval = nc_create_par(sfile->name, sfile->mode, *(sfile->communicator), MPI_INFO_NULL, &(sfile->nc_file_id)))) ERR(retval);
#else
printf("sdatio was built without --enable-parallel, sdatio_create_file will not work for parallel files\n");
abort();
#endif
}
else {
if ((retval = nc_create(sfile->name, sfile->mode, &(sfile->nc_file_id)))) ERR(retval);
}
sfile->is_open = 1;
sfile->n_dimensions = 0;
sfile->n_variables = 0;
sfile->data_written = 0;
/*}*/
sdatio_end_definitions(sfile);
}
/* test extending variables */
int test_pio_extend(int flag){
int rank, procs;
int ncFile;
int ncDimPart;
int ncDimVrtx;
int ncVarVrtx;
int dimsVrtx[2];
size_t start[2];
size_t count[2];
int vertices[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &procs);
/* Create netcdf file */
if (nc_create_par("test.nc", NC_NETCDF4 | NC_MPIIO, MPI_COMM_WORLD, MPI_INFO_NULL, &ncFile)) ERR;
/* Create netcdf dimensions */
if (nc_def_dim(ncFile, "partitions", procs, &ncDimPart)) ERR;
if (nc_def_dim(ncFile, "vertices", NC_UNLIMITED, &ncDimVrtx)) ERR;
/* Create netcdf variables */
dimsVrtx[0] = ncDimPart;
dimsVrtx[1] = ncDimVrtx;
if (nc_def_var(ncFile, "vertex", NC_INT, 2, dimsVrtx, &ncVarVrtx)) ERR;
/* Start writing data */
if (nc_enddef(ncFile)) ERR;
/* Set access mode */
if (nc_var_par_access(ncFile, ncVarVrtx, flag)) ERR;
/* Write vertices */
start[0] = rank;
start[1] = 0;
count[0] = 1;
count[1] = rank;
if (nc_put_vara_int(ncFile, ncVarVrtx, start, count, vertices)) ERR;
/* Close netcdf file */
if (nc_close(ncFile)) ERR;
return 0;
}
/* when calling from fortran: convert MPI_Comm and MPI_Info to C */
int
nc_create_par_fortran(const char *path, int cmode, int comm,
int info, int *ncidp)
{
#ifndef USE_PARALLEL
return NC_ENOPAR;
#else
MPI_Comm comm_c;
MPI_Info info_c;
/* Convert fortran comm and info to C comm and info, if there is a
* function to do so. Otherwise just pass them. */
#ifdef HAVE_MPI_COMM_F2C
comm_c = MPI_Comm_f2c(comm);
info_c = MPI_Info_f2c(info);
#else
comm_c = (MPI_Comm)comm;
info_c = (MPI_Info)info;
#endif
return nc_create_par(path, cmode, comm_c, info_c, ncidp);
#endif
}
/* Both read and write will be tested */
int test_pio(int flag)
{
/* MPI stuff. */
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Netcdf-4 stuff. */
int ncid;
int nvid,uvid;
int rvid;
unsigned m,k,j,i;
/* two dimensional integer data test */
int dimids[NDIMS1];
size_t start[NDIMS1];
size_t count[NDIMS1];
int *data;
int *tempdata;
int *rdata;
int *temprdata;
/* four dimensional integer data test,
time dimension is unlimited.*/
int dimuids[NDIMS2];
size_t ustart[NDIMS2];
size_t ucount[NDIMS2];
int *udata;
int *tempudata;
int *rudata;
int *temprudata;
/* Initialize MPI. */
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
/* Create a parallel netcdf-4 file. */
if (nc_create_par(file_name, facc_type, comm, info, &ncid)) ERR;
/* The first case is two dimensional variables, no unlimited dimension */
/* Create two dimensions. */
if (nc_def_dim(ncid, "d1", DIMSIZE2, dimids)) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
/* Create one var. */
if (nc_def_var(ncid, "v1", NC_INT, NDIMS1, dimids, &nvid)) ERR;
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
start[0] = 0;
start[1] = mpi_rank * DIMSIZE/mpi_size;
count[0] = DIMSIZE2;
count[1] = DIMSIZE/mpi_size;
/* start parallel netcdf4 */
if (nc_var_par_access(ncid, nvid, flag)) ERR;
if (!(data = malloc(sizeof(int)*count[1]*count[0]))) ERR;
tempdata = data;
for (j = 0; j < count[0]; j++){
for (i = 0; i < count[1]; i++)
{
*tempdata = mpi_rank * (j + 1);
tempdata++;
}
}
/* Write two dimensional integer data */
if (nc_put_vara_int(ncid, nvid, start, count, data)) ERR;
free(data);
/* Case 2: create four dimensional integer data,
one dimension is unlimited. */
/* Create four dimensions. */
if (nc_def_dim(ncid, "ud1", NC_UNLIMITED, dimuids)) ERR;
if (nc_def_dim(ncid, "ud2", DIMSIZE3, &dimuids[1])) ERR;
if (nc_def_dim(ncid, "ud3", DIMSIZE2, &dimuids[2])) ERR;
if (nc_def_dim(ncid, "ud4", DIMSIZE, &dimuids[3])) ERR;
/* Create one var. */
if (nc_def_var(ncid, "uv1", NC_INT, NDIMS2, dimuids, &uvid)) ERR;
if (nc_enddef(ncid)) ERR;
/* Set up selection parameters */
ustart[0] = 0;
ustart[1] = 0;
ustart[2] = 0;
ustart[3] = DIMSIZE*mpi_rank/mpi_size;
ucount[0] = TIMELEN;
ucount[1] = DIMSIZE3;
ucount[2] = DIMSIZE2;
ucount[3] = DIMSIZE/mpi_size;
/* Access parallel */
if (nc_var_par_access(ncid, uvid, flag)) ERR;
/* Create phony data. */
if (!(udata = malloc(ucount[0]*ucount[1]*ucount[2]*ucount[3]*sizeof(int)))) ERR;
tempudata = udata;
for( m=0; m<ucount[0];m++)
for( k=0; k<ucount[1];k++)
for (j=0; j<ucount[2];j++)
for (i=0; i<ucount[3]; i++)
{
*tempudata = (1+mpi_rank)*2*(j+1)*(k+1)*(m+1);
tempudata++;
}
/* Write slabs of phoney data. */
if (NC_INDEPENDENT == flag) {
int res;
res = nc_put_vara_int(ncid, uvid, ustart, ucount, udata);
if(res != NC_ECANTEXTEND) ERR;
}
else {
if (nc_put_vara_int(ncid, uvid, ustart, ucount, udata)) ERR;
}
free(udata);
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
if (nc_open_par(file_name, facc_type_open, comm, info, &ncid)) ERR;
/* Case 1: read two-dimensional variables, no unlimited dimension */
/* Set up slab for this process. */
start[0] = 0;
start[1] = mpi_rank * DIMSIZE/mpi_size;
count[0] = DIMSIZE2;
count[1] = DIMSIZE/mpi_size;
if (nc_inq_varid(ncid, "v1", &rvid)) ERR;
if (nc_var_par_access(ncid, rvid, flag)) ERR;
if (!(rdata = malloc(sizeof(int)*count[1]*count[0]))) ERR;
if (nc_get_vara_int(ncid, rvid, start, count, rdata)) ERR;
temprdata = rdata;
for (j=0; j<count[0];j++){
for (i=0; i<count[1]; i++){
if(*temprdata != mpi_rank*(j+1))
{
ERR_RET;
break;
}
temprdata++;
}
}
free(rdata);
/* Case 2: read four dimensional data, one dimension is unlimited. */
/* set up selection parameters */
ustart[0] = 0;
ustart[1] = 0;
ustart[2] = 0;
ustart[3] = DIMSIZE*mpi_rank/mpi_size;
ucount[0] = TIMELEN;
ucount[1] = DIMSIZE3;
ucount[2] = DIMSIZE2;
ucount[3] = DIMSIZE/mpi_size;
/* Inquiry the data */
/* (NOTE: This variable isn't written out, when access is independent) */
if (NC_INDEPENDENT != flag) {
if (nc_inq_varid(ncid, "uv1", &rvid)) ERR;
/* Access the parallel */
if (nc_var_par_access(ncid, rvid, flag)) ERR;
if (!(rudata = malloc(ucount[0]*ucount[1]*ucount[2]*ucount[3]*sizeof(int)))) ERR;
temprudata = rudata;
/* Read data */
if (nc_get_vara_int(ncid, rvid, ustart, ucount, rudata)) ERR;
for(m = 0; m < ucount[0]; m++)
for(k = 0; k < ucount[1]; k++)
for(j = 0; j < ucount[2]; j++)
for(i = 0; i < ucount[3]; i++)
{
if(*temprudata != (1+mpi_rank)*2*(j+1)*(k+1)*(m+1))
ERR_RET;
temprudata++;
}
free(rudata);
}
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
return 0;
}
/* This function creates a file with 10 2D variables, no unlimited
* dimension. */
int test_pio_2d(size_t cache_size, int facc_type, int access_flag, MPI_Comm comm,
MPI_Info info, int mpi_size, int mpi_rank,
size_t *chunk_size)
{
double starttime, endtime, write_time = 0, bandwidth = 0;
int ncid;
int dimids[NDIMS1];
size_t start[NDIMS1], count[NDIMS1];
float *data;
char file_name[NC_MAX_NAME + 1];
char var_name1[NUMVARS][NC_MAX_NAME + 1] = {"GWa", "JAd", "TJe", "JMa", "JMo",
"JQA", "AJa", "MVB", "WHH", "JTy"};
int varid1[NUMVARS];
size_t nelems_in;
float preemption_in;
int j, i, t;
/* Create some data. */
if (!(data = malloc(sizeof(float) * DIMSIZE2 * DIMSIZE1 / mpi_size)))
return -2;
for (j = 0; j < DIMSIZE2; j++)
for (i = 0; i < DIMSIZE1 / mpi_size; i++)
data[j * DIMSIZE1 / mpi_size + i] = (float)mpi_rank * (j + 1);
/* Get the file name. */
sprintf(file_name, "%s/%s", TEMP_LARGE, FILENAME);
/* Set the cache size. */
if (nc_get_chunk_cache(NULL, &nelems_in, &preemption_in)) ERR;
if (nc_set_chunk_cache(cache_size, nelems_in, preemption_in)) ERR;
for (t = 0; t < NUM_TRIES; t++)
{
/* Create a netcdf-4 file, opened for parallel I/O. */
if (nc_create_par(file_name, facc_type|NC_NETCDF4, comm,
info, &ncid)) ERR;
/* Create two dimensions. */
if (nc_def_dim(ncid, "d1", DIMSIZE2, &dimids[0])) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE1, &dimids[1])) ERR;
/* Create our variables. */
for (i = 0; i < NUMVARS; i++)
{
if (nc_def_var(ncid, var_name1[i], NC_INT, NDIMS1,
dimids, &varid1[i])) ERR;
if (chunk_size[0])
if (nc_def_var_chunking(ncid, varid1[i], 0, chunk_size)) ERR;
}
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
start[0] = 0;
start[1] = mpi_rank * DIMSIZE1/mpi_size;
count[0] = DIMSIZE2;
count[1] = DIMSIZE1 / mpi_size;
/* start parallel netcdf4 */
for (i = 0; i < NUMVARS; i++)
if (nc_var_par_access(ncid, varid1[i], access_flag)) ERR;
starttime = MPI_Wtime();
/* Write two dimensional float data */
for (i = 0; i < NUMVARS; i++)
if (nc_put_vara_float(ncid, varid1[i], start, count, data)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
endtime = MPI_Wtime();
if (!mpi_rank)
{
bandwidth += ((sizeof(float) * DIMSIZE1 * DIMSIZE2 * NUMVARS) /
((endtime - starttime) * 1024 * 1024)) / NUM_TRIES;
write_time += (endtime - starttime) / NUM_TRIES;
}
}
free(data);
if (!mpi_rank)
{
char chunk_string[NC_MAX_NAME + 1] = "";
/* What was our chunking? */
if (chunk_size[0])
sprintf(chunk_string, "%dx%d ", (int)chunk_size[0], (int)chunk_size[1]);
else
strcat(chunk_string, "contiguous");
/* Print the results. */
printf("%d\t\t%s\t%s\t%d\t\t%dx%d\t\t%s\t%f\t\t%f\t\t\t%d\n", mpi_size,
(facc_type == NC_MPIIO ? "MPI-IO " : "MPI-POSIX"),
(access_flag == NC_INDEPENDENT ? "independent" : "collective"),
(int)cache_size/MEGABYTE, DIMSIZE1, DIMSIZE2, chunk_string,
write_time, bandwidth, NUM_TRIES);
}
/* Delete this file. */
remove(file_name);
return 0;
}
/* Case 2: create four dimensional integer data,
one dimension is unlimited. */
int test_pio_4d(size_t cache_size, int facc_type, int access_flag, MPI_Comm comm,
MPI_Info info, int mpi_size, int mpi_rank, size_t *chunk_size)
{
int ncid, dimuids[NDIMS2], varid2[NUMVARS];
size_t ustart[NDIMS2], ucount[NDIMS2];
float *udata, *tempudata;
char file_name[NC_MAX_NAME + 1];
char var_name2[NUMVARS][NC_MAX_NAME + 1] = {"JKP", "ZTa", "MFi", "FPi", "JBu",
"ALi", "AJo", "USG", "RBH", "JAG"};
double starttime, endtime, write_time = 0, bandwidth = 0;
size_t nelems_in;
float preemption_in;
int k, j, i, t;
udata = malloc(DIMSIZE3 * DIMSIZE2 * DIMSIZE1 / mpi_size * sizeof(int));
/* Create phony data. */
tempudata = udata;
for(k = 0; k < DIMSIZE3; k++)
for(j = 0; j < DIMSIZE2; j++)
for(i = 0; i < DIMSIZE1 / mpi_size; i++)
{
*tempudata = (float)(1 + mpi_rank) * 2 * (j + 1) * (k + 1);
tempudata++;
}
/* Get the file name. */
sprintf(file_name, "%s/%s", TEMP_LARGE, FILENAME);
/* Set the cache size. */
if (nc_get_chunk_cache(NULL, &nelems_in, &preemption_in)) ERR;
if (nc_set_chunk_cache(cache_size, nelems_in, preemption_in)) ERR;
for (t = 0; t < NUM_TRIES; t++)
{
/* Create a netcdf-4 file. */
if (nc_create_par(file_name, facc_type|NC_NETCDF4, comm, info,
&ncid)) ERR;
/* Create four dimensions. */
if (nc_def_dim(ncid, "ud1", TIMELEN, dimuids)) ERR;
if (nc_def_dim(ncid, "ud2", DIMSIZE3, &dimuids[1])) ERR;
if (nc_def_dim(ncid, "ud3", DIMSIZE2, &dimuids[2])) ERR;
if (nc_def_dim(ncid, "ud4", DIMSIZE1, &dimuids[3])) ERR;
/* Create 10 variables. */
for (i = 0; i < NUMVARS; i++)
if (nc_def_var(ncid, var_name2[i], NC_INT, NDIMS2,
dimuids, &varid2[i])) ERR;
if (nc_enddef(ncid)) ERR;
/* Set up selection parameters */
ustart[0] = 0;
ustart[1] = 0;
ustart[2] = 0;
ustart[3] = DIMSIZE1 * mpi_rank / mpi_size;
ucount[0] = 1;
ucount[1] = DIMSIZE3;
ucount[2] = DIMSIZE2;
ucount[3] = DIMSIZE1 / mpi_size;
/* Access parallel */
for (i = 0; i < NUMVARS; i++)
if (nc_var_par_access(ncid, varid2[i], access_flag)) ERR;
starttime = MPI_Wtime();
/* Write slabs of phony data. */
for(ustart[0] = 0; ustart[0] < TIMELEN; ustart[0]++)
for (i = 0; i < NUMVARS; i++)
if (nc_put_vara_float(ncid, varid2[i], ustart, ucount, udata)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
endtime = MPI_Wtime();
if (!mpi_rank)
{
write_time += (endtime - starttime) / NUM_TRIES;
bandwidth += (sizeof(float) * TIMELEN * DIMSIZE1 * DIMSIZE2 * DIMSIZE3 * NUMVARS) /
((endtime - starttime) * 1024 * 1024 * NUM_TRIES);
}
}
free(udata);
if (!mpi_rank)
{
char chunk_string[NC_MAX_NAME + 1] = "";
/* What was our chunking? */
if (chunk_size[0])
sprintf(chunk_string, "%dx%dx%dx%d", (int)chunk_size[0], (int)chunk_size[1],
(int)chunk_size[2], (int)chunk_size[3]);
else
strcat(chunk_string, "contiguous");
/* Print our results. */
printf("%d\t\t%s\t%s\t%d\t\t%dx%dx%dx%d\t%s\t%f\t\t%f\t\t\t%d\n", mpi_size,
(facc_type == NC_MPIIO ? "MPI-IO " : "MPI-POSIX"),
(access_flag == NC_INDEPENDENT ? "independent" : "collective"),
(int)cache_size / MEGABYTE, TIMELEN, DIMSIZE3, DIMSIZE2, DIMSIZE1, chunk_string, write_time,
bandwidth, NUM_TRIES);
}
/* Delete this file. */
remove(file_name);
return 0;
}
int main(int argc, char* argv[])
{
int err = 0;
int ecode = 0;
int ncid;
int cmode, format;
int nprocs, rank;
MPI_Comm comm=MPI_COMM_SELF;
MPI_Info info=MPI_INFO_NULL;
printf("\n*** Testing nc_inq_format_extended for pnetcdf...");
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (nprocs > 1 && rank == 0)
printf("This test program is intended to run on ONE process\n");
if (rank > 0) goto fn_exit;
/* first, use PnetCDF to create a file with default header/variable alignment */
#ifdef DISABLE_PNETCDF_ALIGNMENT
MPI_Info_create(&info);
MPI_Info_set(info, "nc_header_align_size", "1");
MPI_Info_set(info, "nc_var_align_size", "1");
#endif
/* test CDF-1 file format */
cmode = NC_PNETCDF | NC_CLOBBER;
if (nc_create_par(FILENAME, cmode, comm, info, &ncid)) ERR_RET;
if (nc_enddef(ncid)) ERR;
if(nc_inq_format_extended(ncid,&format,&cmode)) ERR;
if((cmode & NC_PNETCDF) != NC_PNETCDF) {
printf("***FAIL at line %d: mode was %08x ; expected %08x\n",__LINE__,cmode,NC_PNETCDF);
ecode = 1;
ERR;
}
if(format != NC_FORMATX_PNETCDF) {
printf("***FAIL at line %d: format was %d ; expected %d\n",__LINE__,format,NC_FORMATX_PNETCDF);
ecode = 1;
ERR;
}
/* test CDF-2 file format */
cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_OFFSET;
if (nc_create_par(FILENAME, cmode, comm, info, &ncid)) ERR_RET;
if (nc_enddef(ncid)) ERR;
if(nc_inq_format_extended(ncid,&format,&cmode)) ERR;
if((cmode & NC_64BIT_OFFSET) != NC_64BIT_OFFSET) {
printf("***FAIL at line %d: mode was %08x ; expected %08x\n",__LINE__,cmode,NC_64BIT_OFFSET);
ecode = 1;
ERR;
}
if(format != NC_FORMATX_PNETCDF) {
printf("***FAIL at line %d: format was %d ; expected %d\n",__LINE__,format,NC_FORMATX_PNETCDF);
ecode = 1;
ERR;
}
/* test CDF-5 file format */
cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_DATA;
if (nc_create_par(FILENAME, cmode, comm, info, &ncid)) ERR_RET;
if (nc_enddef(ncid)) ERR;
if(nc_inq_format_extended(ncid,&format,&cmode)) ERR;
if((cmode & NC_64BIT_DATA) != NC_64BIT_DATA) {
printf("***FAIL at line %d: mode was %08x ; expected %08x\n",__LINE__,cmode,NC_64BIT_DATA);
ecode = 1;
ERR;
}
if(format != NC_FORMATX_PNETCDF) {
printf("***FAIL at line %d: format was %d ; expected %d\n",__LINE__,format,NC_FORMATX_PNETCDF);
ecode = 1;
ERR;
}
if (nc_abort(ncid)) ERR;
fn_exit:
MPI_Finalize();
SUMMARIZE_ERR;
FINAL_RESULTS;
return ecode;
}
int
main(int argc, char **argv)
{
/* MPI stuff. */
int mpi_namelen;
char mpi_name[MPI_MAX_PROCESSOR_NAME];
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
double start_time = 0, total_time;
/* Netcdf-4 stuff. */
int ncid, varid, dimids[NDIMS];
size_t start[NDIMS] = {0, 0, 0};
size_t count[NDIMS] = {1, DIMSIZE, DIMSIZE};
int data[DIMSIZE * DIMSIZE], data_in[DIMSIZE * DIMSIZE];
int j, i;
char file_name[NC_MAX_NAME + 1];
int ndims_in, nvars_in, natts_in, unlimdimid_in;
#ifdef USE_MPE
int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close;
#endif /* USE_MPE */
/* Initialize MPI. */
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Get_processor_name(mpi_name, &mpi_namelen);
/*printf("mpi_name: %s size: %d rank: %d\n", mpi_name, mpi_size, mpi_rank);*/
/* Must be able to evenly divide my slabs between processors. */
if (NUM_SLABS % mpi_size != 0)
{
if (!mpi_rank) printf("NUM_SLABS (%d) is not evenly divisible by mpi_size(%d)\n",
NUM_SLABS, mpi_size);
ERR;
}
#ifdef USE_MPE
MPE_Init_log();
s_init = MPE_Log_get_event_number();
e_init = MPE_Log_get_event_number();
s_define = MPE_Log_get_event_number();
e_define = MPE_Log_get_event_number();
s_write = MPE_Log_get_event_number();
e_write = MPE_Log_get_event_number();
s_close = MPE_Log_get_event_number();
e_close = MPE_Log_get_event_number();
s_open = MPE_Log_get_event_number();
e_open = MPE_Log_get_event_number();
MPE_Describe_state(s_init, e_init, "Init", "red");
MPE_Describe_state(s_define, e_define, "Define", "yellow");
MPE_Describe_state(s_write, e_write, "Write", "green");
MPE_Describe_state(s_close, e_close, "Close", "purple");
MPE_Describe_state(s_open, e_open, "Open", "blue");
MPE_Start_log();
MPE_Log_event(s_init, 0, "start init");
#endif /* USE_MPE */
/* if (!mpi_rank) */
/* { */
/* printf("\n*** Testing parallel I/O some more.\n"); */
/* printf("*** writing a %d x %d x %d file from %d processors...\n", */
/* NUM_SLABS, DIMSIZE, DIMSIZE, mpi_size); */
/* } */
/* We will write the same slab over and over. */
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
data[i] = mpi_rank;
#ifdef USE_MPE
MPE_Log_event(e_init, 0, "end init");
MPE_Log_event(s_define, 0, "start define file");
#endif /* USE_MPE */
/* Create a parallel netcdf-4 file. */
sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
if (nc_create_par(file_name, NC_PNETCDF, comm, info, &ncid)) ERR;
/* A global attribute holds the number of processors that created
* the file. */
if (nc_put_att_int(ncid, NC_GLOBAL, "num_processors", NC_INT, 1, &mpi_size)) ERR;
/* Create three dimensions. */
if (nc_def_dim(ncid, DIM1_NAME, NUM_SLABS, dimids)) ERR;
if (nc_def_dim(ncid, DIM2_NAME, DIMSIZE, &dimids[1])) ERR;
if (nc_def_dim(ncid, DIM3_NAME, DIMSIZE, &dimids[2])) ERR;
/* Create one var. */
if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIMS, dimids, &varid)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
#ifdef USE_MPE
MPE_Log_event(e_define, 0, "end define file");
if (mpi_rank)
sleep(mpi_rank);
#endif /* USE_MPE */
/* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/
if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
if (!mpi_rank)
start_time = MPI_Wtime();
/* Write all the slabs this process is responsible for. */
for (i = 0; i < NUM_SLABS / mpi_size; i++)
{
start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
#ifdef USE_MPE
MPE_Log_event(s_write, 0, "start write slab");
#endif /* USE_MPE */
/* Write one slab of data. */
if (nc_put_vara_int(ncid, varid, start, count, data)) ERR;
#ifdef USE_MPE
MPE_Log_event(e_write, 0, "end write file");
#endif /* USE_MPE */
}
if (!mpi_rank)
{
total_time = MPI_Wtime() - start_time;
/* printf("num_proc\ttime(s)\n");*/
printf("%d\t%g\t%g\n", mpi_size, total_time, DIMSIZE * DIMSIZE * NUM_SLABS * sizeof(int) / total_time);
}
#ifdef USE_MPE
MPE_Log_event(s_close, 0, "start close file");
#endif /* USE_MPE */
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
#ifdef USE_MPE
MPE_Log_event(e_close, 0, "end close file");
#endif /* USE_MPE */
/* Reopen the file and check it. */
if (nc_open_par(file_name, NC_NOWRITE, comm, info, &ncid)) ERR;
if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR;
if (ndims_in != NDIMS || nvars_in != 1 || natts_in != 1 ||
unlimdimid_in != -1) ERR;
/* Read all the slabs this process is responsible for. */
for (i = 0; i < NUM_SLABS / mpi_size; i++)
{
start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
#ifdef USE_MPE
MPE_Log_event(s_read, 0, "start read slab");
#endif /* USE_MPE */
/* Read one slab of data. */
if (nc_get_vara_int(ncid, varid, start, count, data_in)) ERR;
/* Check data. */
for (j = 0; j < DIMSIZE * DIMSIZE; j++)
if (data_in[j] != mpi_rank)
{
ERR;
break;
}
#ifdef USE_MPE
MPE_Log_event(e_read, 0, "end read file");
#endif /* USE_MPE */
}
#ifdef USE_MPE
MPE_Log_event(s_close, 0, "start close file");
#endif /* USE_MPE */
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
#ifdef USE_MPE
MPE_Log_event(e_close, 0, "end close file");
#endif /* USE_MPE */
/* Delete this large file. */
remove(file_name);
/* Shut down MPI. */
MPI_Finalize();
/* if (!mpi_rank) */
/* { */
/* SUMMARIZE_ERR; */
/* FINAL_RESULTS; */
/* } */
return total_err;
}
void seissol::LoopStatistics::writeSamples() {
std::string loopStatFile = utils::Env::get<std::string>("SEISSOL_LOOP_STAT_PREFIX", "");
if (!loopStatFile.empty()) {
#if defined(USE_NETCDF) && defined(USE_MPI)
unsigned nRegions = m_times.size();
for (unsigned region = 0; region < nRegions; ++region) {
std::ofstream file;
std::stringstream ss;
ss << loopStatFile << m_regions[region] << ".nc";
std::string fileName = ss.str();
int nSamples = m_times[region].size();
int sampleOffset;
MPI_Scan(&nSamples, &sampleOffset, 1, MPI_INT, MPI_SUM, seissol::MPI::mpi.comm());
int ncid, stat;
stat = nc_create_par(fileName.c_str(), NC_MPIIO | NC_CLOBBER | NC_NETCDF4, seissol::MPI::mpi.comm(), MPI_INFO_NULL, &ncid); check_err(stat,__LINE__,__FILE__);
int sampledim, rankdim, sampletyp, offsetid, sampleid;
stat = nc_def_dim(ncid, "rank", 1+seissol::MPI::mpi.size(), &rankdim); check_err(stat,__LINE__,__FILE__);
stat = nc_def_dim(ncid, "sample", NC_UNLIMITED, &sampledim); check_err(stat,__LINE__,__FILE__);
stat = nc_def_compound(ncid, sizeof(Sample), "Sample", &sampletyp); check_err(stat,__LINE__,__FILE__);
{
stat = nc_insert_compound(ncid, sampletyp, "time", NC_COMPOUND_OFFSET(Sample,time), NC_DOUBLE); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(ncid, sampletyp, "loopLength", NC_COMPOUND_OFFSET(Sample,numIters), NC_UINT); check_err(stat,__LINE__,__FILE__);
}
stat = nc_def_var(ncid, "offset", NC_INT, 1, &rankdim, &offsetid); check_err(stat,__LINE__,__FILE__);
stat = nc_def_var(ncid, "sample", sampletyp, 1, &sampledim, &sampleid); check_err(stat,__LINE__,__FILE__);
stat = nc_enddef(ncid); check_err(stat,__LINE__,__FILE__);
stat = nc_var_par_access(ncid, offsetid, NC_COLLECTIVE); check_err(stat,__LINE__,__FILE__);
stat = nc_var_par_access(ncid, sampleid, NC_COLLECTIVE); check_err(stat,__LINE__,__FILE__);
size_t start, count;
int offsetData[2];
if (seissol::MPI::mpi.rank() == 0) {
start = 0;
count = 2;
offsetData[0] = 0;
} else {
start = 1+seissol::MPI::mpi.rank();
count = 1;
}
offsetData[count-1] = sampleOffset;
stat = nc_put_vara_int(ncid, offsetid, &start, &count, offsetData); check_err(stat,__LINE__,__FILE__);
start = sampleOffset-nSamples;
count = nSamples;
stat = nc_put_vara(ncid, sampleid, &start, &count, m_times[region].data()); check_err(stat,__LINE__,__FILE__);
stat = nc_close(ncid); check_err(stat,__LINE__,__FILE__);
}
#else
logWarning(seissol::MPI::mpi.rank()) << "Writing loop statistics requires NetCDF and MPI.";
#endif
}
}
int ex_create_par_int(const char *path, int cmode, int *comp_ws, int *io_ws, MPI_Comm comm,
MPI_Info info, int run_version)
{
int exoid;
int status;
int dimid;
int old_fill;
int lio_ws;
int filesiz;
float vers;
char errmsg[MAX_ERR_LENGTH];
char *mode_name;
int nc_mode = 0;
int int64_status;
const char *routine = "ex_create_par";
int pariomode = 0;
int is_mpiio = 0;
int is_pnetcdf = 0;
unsigned int my_mode = cmode;
/* Contains a 1 in all bits corresponding to file modes */
static unsigned int all_modes = EX_NORMAL_MODEL | EX_64BIT_OFFSET | EX_64BIT_DATA | EX_NETCDF4;
exerrval = 0; /* clear error code */
#if !NC_HAS_PARALLEL
/* Library does NOT support parallel output via netcdf-4 or pnetcdf */
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: Parallel output requires the netcdf-4 and/or "
"pnetcdf library format, but this netcdf library does not "
"support either.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
#endif
if (run_version != EX_API_VERS_NODOT && warning_output == 0) {
int run_version_major = run_version / 100;
int run_version_minor = run_version % 100;
int lib_version_major = EX_API_VERS_NODOT / 100;
int lib_version_minor = EX_API_VERS_NODOT % 100;
fprintf(stderr, "EXODUS: Warning: This code was compiled with exodusII "
"version %d.%02d,\n but was linked with exodusII "
"library version %d.%02d\n This is probably an "
"error in the build process of this code.\n",
run_version_major, run_version_minor, lib_version_major, lib_version_minor);
warning_output = 1;
}
/*
* See if specified mode is supported in the version of netcdf we
* are using
*/
#if !NC_HAS_HDF5
if (my_mode & EX_NETCDF4) {
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: File format specified as netcdf-4, but the "
"NetCDF library being used was not configured to enable "
"this format\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
#endif
#if !defined(NC_64BIT_DATA)
if (my_mode & EX_64BIT_DATA) {
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: File format specified as 64bit_data, but "
"the NetCDF library being used does not support this "
"format\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
#endif
/* Check that one and only one format mode is specified... */
{
unsigned int set_modes = all_modes & my_mode;
if (set_modes == 0) {
my_mode |= EX_64BIT_OFFSET; /* Default if nothing specified */
}
else {
/* Checks that only a single bit is set */
set_modes = set_modes && !(set_modes & (set_modes - 1));
if (!set_modes) {
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: More than 1 file format "
"(EX_NORMAL_MODEL, EX_LARGE_MODEL, EX_64BIT_OFFSET, "
"EX_64BIT_DATA, or EX_NETCDF4)\nwas specified in the "
"mode argument of the ex_create call. Only a single "
"format can be specified.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
}
}
/*
* See if any integer data is to be stored as int64 (long long). If
* so, then need to set NC_NETCDF4 and unset NC_CLASSIC_MODEL (or
* set EX_NOCLASSIC. Output meaningful error message if the library
* is not NetCDF-4 enabled...
*
* As of netcdf-4.4.0, can also use NC_64BIT_DATA (CDF5) mode for this...
*/
int64_status = my_mode & (EX_ALL_INT64_DB | EX_ALL_INT64_API);
if ((int64_status & EX_ALL_INT64_DB) != 0) {
#if NC_HAS_HDF5 || defined(NC_64BIT_DATA)
/* Library DOES support netcdf4 and/or cdf5 ... See if user
* specified either of these and use that one; if not, pick
* netcdf4, non-classic as default.
*/
if (my_mode & EX_NETCDF4) {
my_mode |= EX_NOCLASSIC;
}
#if defined(NC_64BIT_DATA)
else if (my_mode & EX_64BIT_DATA) {
; /* Do nothing, already set */
}
#endif
else {
/* Unset the current mode so we don't have multiples specified */
/* ~all_modes sets to 1 all bits not associated with file format */
my_mode &= ~all_modes;
#if NC_HAS_HDF5
/* Pick netcdf4 as default mode for 64-bit integers */
my_mode |= EX_NOCLASSIC;
my_mode |= EX_NETCDF4;
#else
/* Pick 64bit_data as default mode for 64-bit integers */
my_mode |= EX_64BIT_DATA;
#endif
}
#else
/* Library does NOT support netcdf4 or cdf5 */
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: 64-bit integer storage requested, but the "
"netcdf library does not support the required netcdf-4 or "
"64BIT_DATA extensions.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
#endif
}
/* Check parallel io mode. Valid is NC_MPIPOSIX or NC_MPIIO or NC_PNETCDF
* Exodus uses different flag values; map to netcdf values
*/
{
int tmp_mode = 0;
if (my_mode & EX_MPIPOSIX) {
pariomode = NC_MPIPOSIX;
tmp_mode = EX_NETCDF4;
#if !NC_HAS_HDF5
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: EX_MPIPOSIX parallel output requested "
"which requires NetCDF-4 support, but the library does "
"not have that option enabled.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
#endif
}
else if (my_mode & EX_MPIIO) {
pariomode = NC_MPIIO;
is_mpiio = 1;
tmp_mode = EX_NETCDF4;
#if !NC_HAS_HDF5
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: EX_MPIIO parallel output requested which "
"requires NetCDF-4 support, but the library does not "
"have that option enabled.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
#endif
}
else if (my_mode & EX_PNETCDF) {
pariomode = NC_PNETCDF;
is_pnetcdf = 1;
/* See if client specified 64-bit or not... */
if ((int64_status & EX_ALL_INT64_DB) != 0) {
tmp_mode = EX_64BIT_DATA;
}
else {
tmp_mode = EX_64BIT_OFFSET;
}
#if !NC_HAS_PNETCDF
exerrval = EX_BADPARAM;
snprintf(errmsg, MAX_ERR_LENGTH, "EXODUS: ERROR: EX_PNETCDF parallel output requested "
"which requires PNetCDF support, but the library does "
"not have that option enabled.\n");
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
#endif
}
/* If tmp_mode was set here, then need to clear any other mode that
was potentially already set in my_mode... */
my_mode &= ~all_modes;
my_mode |= tmp_mode;
}
if (my_mode & EX_NETCDF4) {
nc_mode |= NC_NETCDF4;
}
if (!(my_mode & EX_NOCLASSIC)) {
nc_mode |= NC_CLASSIC_MODEL;
}
/*
* See if "large file" mode was specified in a ex_create my_mode. If
* so, then pass the NC_64BIT_OFFSET flag down to netcdf.
* If netcdf4 mode specified, don't use NC_64BIT_OFFSET mode.
*/
if (my_mode & EX_NORMAL_MODEL) {
filesiz = 0;
#if NC_HAS_HDF5
}
else if (nc_mode & NC_NETCDF4) {
filesiz = 1;
#endif
#if defined(NC_64BIT_DATA)
}
else if (nc_mode & NC_64BIT_DATA) {
filesiz = 1;
#endif
}
else {
filesiz = (int)((my_mode & EX_64BIT_OFFSET) || (ex_large_model(-1) == 1));
}
if (
#if NC_HAS_HDF5
!(nc_mode & NC_NETCDF4) &&
#endif
#if defined(NC_64BIT_DATA)
!(nc_mode & NC_64BIT_DATA) &&
#endif
filesiz == 1) {
nc_mode |= NC_64BIT_OFFSET;
}
if (my_mode & EX_SHARE) {
nc_mode |= NC_SHARE;
}
/*
* set error handling mode to no messages, non-fatal errors
*/
ex_opts(exoptval); /* call required to set ncopts first time through */
if (my_mode & EX_CLOBBER) {
nc_mode |= NC_CLOBBER;
mode_name = "CLOBBER";
}
else {
nc_mode |= NC_NOCLOBBER;
mode_name = "NOCLOBBER";
}
#if defined NC_IGNORE_MAX_DIMS
nc_mode |= NC_IGNORE_MAX_DIMS;
#endif
#if defined NC_IGNORE_MAX_VARS
nc_mode |= NC_IGNORE_MAX_VARS;
#endif
if ((status = nc_create_par(path, nc_mode | pariomode, comm, info, &exoid)) != NC_NOERR) {
exerrval = status;
#if NC_HAS_HDF5
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: file create failed for %s, mode: %s", path, mode_name);
#else
if (my_mode & EX_NETCDF4) {
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: file create failed for %s in NETCDF4 and %s "
"mode.\n\tThis library does not support netcdf-4 files.",
path, mode_name);
}
else {
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: file create failed for %s, mode: %s", path,
mode_name);
}
#endif
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* turn off automatic filling of netCDF variables */
if ((status = nc_set_fill(exoid, NC_NOFILL, &old_fill)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to set nofill mode in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* Verify that there is not an existing file_item struct for this
exoid This could happen (and has) when application calls
ex_open(), but then closes file using nc_close() and then reopens
file. NetCDF will possibly reuse the exoid which results in
internal corruption in exodus data structures since exodus does
not know that file was closed and possibly new file opened for
this exoid
*/
if (ex_find_file_item(exoid) != NULL) {
char errmsg[MAX_ERR_LENGTH];
exerrval = EX_BADFILEID;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: There is an existing file already using the file "
"id %d which was also assigned to file %s.\n\tWas "
"nc_close() called instead of ex_close() on an open Exodus "
"file?\n",
exoid, path);
ex_err(routine, errmsg, exerrval);
nc_close(exoid);
return (EX_FATAL);
}
/* initialize floating point size conversion. since creating new file,
* i/o wordsize attribute from file is zero.
*/
if (ex_conv_ini(exoid, comp_ws, io_ws, 0, int64_status, 1, is_mpiio, is_pnetcdf) != EX_NOERR) {
exerrval = EX_FATAL;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to init conversion routines in file id %d",
exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* put the EXODUS version number, and i/o floating point word size as
* netcdf global attributes
*/
/* store Exodus API version # as an attribute */
vers = EX_API_VERS;
if ((status = nc_put_att_float(exoid, NC_GLOBAL, ATT_API_VERSION, NC_FLOAT, 1, &vers)) !=
NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to store Exodus II API version attribute in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file version # as an attribute */
vers = EX_VERS;
if ((status = nc_put_att_float(exoid, NC_GLOBAL, ATT_VERSION, NC_FLOAT, 1, &vers)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to store Exodus II file version attribute in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file float word size as an attribute */
lio_ws = (int)(*io_ws);
if ((status = nc_put_att_int(exoid, NC_GLOBAL, ATT_FLT_WORDSIZE, NC_INT, 1, &lio_ws)) !=
NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to store Exodus II file float word size "
"attribute in file id %d",
exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file size (1=large, 0=normal) as an attribute */
if ((status = nc_put_att_int(exoid, NC_GLOBAL, ATT_FILESIZE, NC_INT, 1, &filesiz)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to store Exodus II file size attribute in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
{
int max_so_far = 32;
if ((status = nc_put_att_int(exoid, NC_GLOBAL, ATT_MAX_NAME_LENGTH, NC_INT, 1, &max_so_far)) !=
NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to add maximum_name_length attribute in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
}
/* define some dimensions and variables */
/* create string length dimension */
if ((status = nc_def_dim(exoid, DIM_STR, (MAX_STR_LENGTH + 1), &dimid)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to define string length in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* The name string length dimension is delayed until the ex_put_init function
*/
/* create line length dimension */
if ((status = nc_def_dim(exoid, DIM_LIN, (MAX_LINE_LENGTH + 1), &dimid)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to define line length in file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
/* create number "4" dimension; must be of type long */
if ((status = nc_def_dim(exoid, DIM_N4, 4L, &dimid)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to define number \"4\" dimension in file id %d",
exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
{
int int64_db_status = int64_status & EX_ALL_INT64_DB;
if ((status = nc_put_att_int(exoid, NC_GLOBAL, ATT_INT64_STATUS, NC_INT, 1,
&int64_db_status)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to add int64_status attribute in file id %d",
exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
}
if ((status = nc_enddef(exoid)) != NC_NOERR) {
exerrval = status;
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to complete definition for file id %d", exoid);
ex_err(routine, errmsg, exerrval);
return (EX_FATAL);
}
return (exoid);
}
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
int msg;
int mpierr;
size_t len;
iosystem_desc_t *ios;
file_desc_t *file;
ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);
file = (file_desc_t *) malloc(sizeof(file_desc_t));
file->next = NULL;
file->iosystem = ios;
file->iotype = *iotype;
file->buffer.validvars=0;
file->buffer.data=NULL;
file->buffer.next=NULL;
file->buffer.vid=NULL;
file->buffer.ioid=-1;
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;
for(int i=0; i<PIO_MAX_VARS;i++){
file->varlist[i].record = -1;
file->varlist[i].ndims = -1;
#ifdef _PNETCDF
file->varlist[i].request = NULL;
file->varlist[i].nreqs=0;
#endif
file->varlist[i].fillbuf = NULL;
file->varlist[i].iobuf = NULL;
}
msg = PIO_MSG_CREATE_FILE;
file->mode = mode;
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
// The 64 bit options are not compatable with hdf5 format files
// printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO);
file->mode = file->mode | NC_MPIIO | NC_NETCDF4;
//printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4);
ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh));
break;
case PIO_IOTYPE_NETCDF4C:
file->mode = file->mode | NC_NETCDF4;
#endif
case PIO_IOTYPE_NETCDF:
if(ios->io_rank==0){
ierr = nc_create(filename, file->mode, &(file->fh));
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh));
if(ierr == PIO_NOERR){
if(ios->io_rank==0){
printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT);
}
int oldfill;
ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT );
// ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill);
}
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
}
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
pio_add_to_file_list(file);
*ncidp = file->fh;
}
if(ios->io_rank==0){
printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr);
// if(file->fh==5) print_trace(stdout);
}
return ierr;
}
int ex_create_par_int (const char *path,
int cmode,
int *comp_ws,
int *io_ws,
MPI_Comm comm,
MPI_Info info,
int run_version)
{
int exoid, dims[1];
int status;
int dimid, time_dim;
int old_fill;
int lio_ws;
int filesiz;
float vers;
char errmsg[MAX_ERR_LENGTH];
char *mode_name;
int mode = 0;
int int64_status;
int pariomode = 0;
unsigned int my_mode = cmode;
assert(my_mode == cmode);
exerrval = 0; /* clear error code */
#if !defined(NC_NETCDF4)
/* Library does NOT support netcdf4 */
exerrval = EX_BADPARAM;
sprintf(errmsg,
"EXODUS: Error: Parallel output requires the netcdf-4 library format, but this netcdf library does not support that.\n");
ex_err("ex_create_par",errmsg,exerrval);
return (EX_FATAL);
#endif
if (run_version != EX_API_VERS_NODOT && warning_output == 0) {
int run_version_major = run_version / 100;
int run_version_minor = run_version % 100;
int lib_version_major = EX_API_VERS_NODOT / 100;
int lib_version_minor = EX_API_VERS_NODOT % 100;
fprintf(stderr, "EXODUS: Warning: This code was compiled with exodusII version %d.%02d,\n but was linked with exodusII library version %d.%02d\n This is probably an error in the build process of this code.\n",
run_version_major, run_version_minor, lib_version_major, lib_version_minor);
warning_output = 1;
}
/*
* See if any integer data is to be stored as int64 (long long). If
* so, then need to set NC_NETCDF4 and unset NC_CLASSIC_MODEL (or
* set EX_NOCLASSIC. Output meaningful error message if the library
* is not NetCDF-4 enabled...
*/
int64_status = my_mode & (EX_ALL_INT64_DB | EX_ALL_INT64_API);
if ((int64_status & EX_ALL_INT64_DB) != 0) {
/* Library DOES support netcdf4... Set modes required to use
* netcdf-4 in non-classic mode
*/
my_mode |= EX_NOCLASSIC;
my_mode |= EX_NETCDF4;
}
/* Check parallel io mode. Valid is NC_MPIPOSIX or NC_MPIIO or NC_PNETCDF
* Exodus uses different flag values; map to netcdf values
*/
if (my_mode & EX_MPIPOSIX) {
pariomode = NC_MPIPOSIX;
my_mode |= EX_NETCDF4;
}
else if (my_mode & EX_MPIIO) {
pariomode = NC_MPIIO;
my_mode |= EX_NETCDF4;
}
else if (my_mode & EX_PNETCDF) {
pariomode = NC_PNETCDF;
mode |= NC_64BIT_OFFSET;
}
if (my_mode & EX_NETCDF4) {
mode |= NC_NETCDF4;
}
if (! (my_mode & EX_NOCLASSIC)) {
mode |= NC_CLASSIC_MODEL;
}
/*
* See if "large file" mode was specified in a ex_create my_mode. If
* so, then pass the NC_64BIT_OFFSET flag down to netcdf.
* If netcdf4 mode specified, don't use NC_64BIT_OFFSET mode.
*/
if ( (my_mode & EX_LARGE_MODEL) && (my_mode & EX_NORMAL_MODEL)) {
exerrval = EX_BADPARAM;
sprintf(errmsg,
"Warning: conflicting mode specification for file %s, mode %d. Using normal",
path, (int)my_mode);
ex_err("ex_create",errmsg,exerrval);
}
filesiz = 1;
if (my_mode & EX_SHARE) {
mode |= NC_SHARE;
}
/*
* set error handling mode to no messages, non-fatal errors
*/
ex_opts(exoptval); /* call required to set ncopts first time through */
if (my_mode & EX_CLOBBER) {
mode |= NC_CLOBBER;
mode_name = "CLOBBER";
} else {
mode |= NC_NOCLOBBER;
mode_name = "NOCLOBBER";
}
if ((status = nc_create_par (path, mode|pariomode, comm, info, &exoid)) != NC_NOERR) {
exerrval = status;
if (my_mode & EX_NETCDF4) {
sprintf(errmsg,
"Error: file create failed for %s in NETCDF4 and %s mode.\n\tThis library probably does not support netcdf-4 files.",
path, mode_name);
} else {
sprintf(errmsg,
"Error: file create failed for %s, mode: %s",
path, mode_name);
}
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
/* turn off automatic filling of netCDF variables */
if ((status = nc_set_fill (exoid, NC_NOFILL, &old_fill)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to set nofill mode in file id %d",
exoid);
ex_err("ex_create", errmsg, exerrval);
return (EX_FATAL);
}
/* initialize floating point size conversion. since creating new file,
* i/o wordsize attribute from file is zero.
*/
if (ex_conv_ini( exoid, comp_ws, io_ws, 0, int64_status ) != EX_NOERR) {
exerrval = EX_FATAL;
sprintf(errmsg,
"Error: failed to init conversion routines in file id %d",
exoid);
ex_err("ex_create", errmsg, exerrval);
return (EX_FATAL);
}
/* put the EXODUS version number, and i/o floating point word size as
* netcdf global attributes
*/
/* store Exodus API version # as an attribute */
vers = EX_API_VERS;
if ((status=nc_put_att_float(exoid, NC_GLOBAL, ATT_API_VERSION,
NC_FLOAT, 1, &vers)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to store Exodus II API version attribute in file id %d",
exoid);
ex_err("ex_create",errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file version # as an attribute */
vers = EX_VERS;
if ((status=nc_put_att_float(exoid, NC_GLOBAL, ATT_VERSION, NC_FLOAT, 1, &vers)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to store Exodus II file version attribute in file id %d",
exoid);
ex_err("ex_create",errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file float word size as an attribute */
lio_ws = (int)(*io_ws);
if ((status=nc_put_att_int (exoid, NC_GLOBAL, ATT_FLT_WORDSIZE, NC_INT, 1, &lio_ws)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to store Exodus II file float word size attribute in file id %d",
exoid);
ex_err("ex_create",errmsg, exerrval);
return (EX_FATAL);
}
/* store Exodus file size (1=large, 0=normal) as an attribute */
if ((status = nc_put_att_int (exoid, NC_GLOBAL, ATT_FILESIZE, NC_INT, 1, &filesiz)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to store Exodus II file size attribute in file id %d",
exoid);
ex_err("ex_create",errmsg, exerrval);
return (EX_FATAL);
}
/* define some dimensions and variables
*/
/* create string length dimension */
if ((status=nc_def_dim (exoid, DIM_STR, (MAX_STR_LENGTH+1), &dimid)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to define string length in file id %d",exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
/* The name string length dimension is delayed until the ex_put_init function */
/* create line length dimension */
if ((status = nc_def_dim(exoid, DIM_LIN, (MAX_LINE_LENGTH+1), &dimid)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to define line length in file id %d",exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
/* create number "4" dimension; must be of type long */
if ((status = nc_def_dim(exoid, DIM_N4, 4L, &dimid)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to define number \"4\" dimension in file id %d",exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
if ((status = nc_def_dim(exoid, DIM_TIME, NC_UNLIMITED, &time_dim)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to define time dimension in file id %d", exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
dims[0] = time_dim;
if ((status = nc_def_var(exoid, VAR_WHOLE_TIME, nc_flt_code(exoid), 1, dims, &dimid)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to define whole time step variable in file id %d",
exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
ex_compress_variable(exoid, dimid, 2);
{
int int64_db_status = int64_status & EX_ALL_INT64_DB;
if ((status=nc_put_att_int(exoid, NC_GLOBAL, ATT_INT64_STATUS, NC_INT, 1, &int64_db_status)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to add int64_status attribute in file id %d",exoid);
ex_err("ex_put_init_ext",errmsg,exerrval);
return (EX_FATAL);
}
}
if ((status = nc_enddef (exoid)) != NC_NOERR) {
exerrval = status;
sprintf(errmsg,
"Error: failed to complete definition for file id %d", exoid);
ex_err("ex_create",errmsg,exerrval);
return (EX_FATAL);
}
return (exoid);
}
int main(int argc, char* argv[])
{
int i, j, rank, nprocs, ncid, cmode, varid[NVARS], dimid[2], *buf;
int err = 0;
char str[32];
size_t start[2], count[2];
MPI_Comm comm=MPI_COMM_SELF;
MPI_Info info=MPI_INFO_NULL;
printf("\n*** Testing bug fix with changing pnetcdf variable offsets...");
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (nprocs > 1 && rank == 0)
printf("This test program is intended to run on ONE process\n");
if (rank > 0) goto fn_exit;
/* first, use PnetCDF to create a file with default header/variable alignment */
#ifdef DISABLE_PNETCDF_ALIGNMENT
MPI_Info_create(&info);
MPI_Info_set(info, "nc_header_align_size", "1");
MPI_Info_set(info, "nc_var_align_size", "1");
#endif
cmode = NC_PNETCDF | NC_CLOBBER;
if (nc_create_par(FILENAME, cmode, comm, info, &ncid)) ERR_RET;
/* define dimension */
if (nc_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0])) ERR;
if (nc_def_dim(ncid, "X", NX, &dimid[1])) ERR;
/* Odd numbers are fixed variables, even numbers are record variables */
for (i=0; i<NVARS; i++) {
if (i%2) {
sprintf(str,"fixed_var_%d",i);
if (nc_def_var(ncid, str, NC_INT, 1, dimid+1, &varid[i])) ERR;
}
else {
sprintf(str,"record_var_%d",i);
if (nc_def_var(ncid, str, NC_INT, 2, dimid, &varid[i])) ERR;
}
}
if (nc_enddef(ncid)) ERR;
for (i=0; i<NVARS; i++) {
/* Note NC_INDEPENDENT is the default */
if (nc_var_par_access(ncid, varid[i], NC_INDEPENDENT)) ERR;
}
/* write all variables */
buf = (int*) malloc(NX * sizeof(int));
for (i=0; i<NVARS; i++) {
for (j=0; j<NX; j++) buf[j] = i*10 + j;
if (i%2) {
start[0] = 0; count[0] = NX;
if (nc_put_vara_int(ncid, varid[i], start, count, buf)) ERR;
}
else {
start[0] = 0; start[1] = 0;
count[0] = 1; count[1] = NX;
if (nc_put_vara_int(ncid, varid[i], start, count, buf)) ERR;
}
}
if (nc_close(ncid)) ERR;
if (info != MPI_INFO_NULL) MPI_Info_free(&info);
/* re-open the file with netCDF (parallel) and enter define mode */
if (nc_open_par(FILENAME, NC_WRITE|NC_PNETCDF, comm, info, &ncid)) ERR_RET;
if (nc_redef(ncid)) ERR;
/* add attributes to make header grow */
for (i=0; i<NVARS; i++) {
sprintf(str, "annotation_for_var_%d",i);
if (nc_put_att_text(ncid, varid[i], "text_attr", strlen(str), str)) ERR;
}
if (nc_enddef(ncid)) ERR;
/* read variables and check their contents */
for (i=0; i<NVARS; i++) {
for (j=0; j<NX; j++) buf[j] = -1;
if (i%2) {
start[0] = 0; count[0] = NX;
if (nc_get_var_int(ncid, varid[i], buf)) ERR;
for (j=0; j<NX; j++)
if (buf[j] != i*10 + j)
printf("unexpected read value var i=%d buf[j=%d]=%d should be %d\n",i,j,buf[j],i*10+j);
}
else {
start[0] = 0; start[1] = 0;
count[0] = 1; count[1] = NX;
if (nc_get_vara_int(ncid, varid[i], start, count, buf)) ERR;
for (j=0; j<NX; j++)
if (buf[j] != i*10+j)
printf("unexpected read value var i=%d buf[j=%d]=%d should be %d\n",i,j,buf[j],i*10+j);
}
}
if (nc_close(ncid)) ERR;
fn_exit:
MPI_Finalize();
SUMMARIZE_ERR;
FINAL_RESULTS;
return 0;
}
int test_pio_attr(int flag)
{
/* MPI stuff. */
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Netcdf-4 stuff. */
int ncid;
int nvid;
int j, i;
double rh_range[2];
static char title[] = "parallel attr to netCDF";
nc_type st_type,vr_type;
size_t vr_len,st_len;
size_t orivr_len;
double *vr_val;
char *st_val;
/* two dimensional integer data*/
int dimids[NDIMS1];
size_t start[NDIMS1];
size_t count[NDIMS1];
int *data;
int *tempdata;
/* Initialize MPI. */
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
/* Create a parallel netcdf-4 file. */
/* nc_set_log_level(NC_TURN_OFF_LOGGING); */
/* nc_set_log_level(3);*/
if (nc_create_par(file_name, facc_type, comm, info, &ncid)) ERR;
/* Create a 2-D variable so that an attribute can be added. */
if (nc_def_dim(ncid, "d1", DIMSIZE2, dimids)) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
/* Create one var. */
if (nc_def_var(ncid, "v1", NC_INT, NDIMS1, dimids, &nvid)) ERR;
orivr_len = 2;
rh_range[0] = 1.0;
rh_range[1] = 1000.0;
/* Write attributes of a variable */
if (nc_put_att_double (ncid, nvid, "valid_range", NC_DOUBLE,
orivr_len, rh_range)) ERR;
if (nc_put_att_text (ncid, nvid, "title", strlen(title),
title)) ERR;
/* Write global attributes */
if (nc_put_att_double (ncid, NC_GLOBAL, "g_valid_range", NC_DOUBLE,
orivr_len, rh_range)) ERR;
if (nc_put_att_text (ncid, NC_GLOBAL, "g_title", strlen(title), title)) ERR;
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
start[0] = 0;
start[1] = mpi_rank * DIMSIZE/mpi_size;
count[0] = DIMSIZE2;
count[1] = DIMSIZE/mpi_size;
/* Access parallel */
if (nc_var_par_access(ncid, nvid, flag)) ERR;
/* Allocating data */
data = malloc(sizeof(int)*count[1]*count[0]);
tempdata = data;
for(j = 0; j < count[0]; j++)
for (i = 0; i < count[1]; i++)
{
*tempdata = mpi_rank * (j + 1);
tempdata++;
}
if (nc_put_vara_int(ncid, nvid, start, count, data)) ERR;
free(data);
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
/* Read attributes */
if (nc_open_par(file_name, facc_type_open, comm, info, &ncid)) ERR;
/* Set up slab for this process. */
start[0] = 0;
start[1] = mpi_rank * DIMSIZE/mpi_size;
count[0] = DIMSIZE2;
count[1] = DIMSIZE/mpi_size;
/* Inquiry variable */
if (nc_inq_varid(ncid, "v1", &nvid)) ERR;
/* Access parallel */
if (nc_var_par_access(ncid, nvid, flag)) ERR;
/* Inquiry attribute */
if (nc_inq_att (ncid, nvid, "valid_range", &vr_type, &vr_len)) ERR;
/* check stuff */
if(vr_type != NC_DOUBLE || vr_len != orivr_len) ERR;
vr_val = (double *) malloc(vr_len * sizeof(double));
/* Get variable attribute values */
if (nc_get_att_double(ncid, nvid, "valid_range", vr_val)) ERR;
/* Check variable attribute value */
for(i = 0; i < vr_len; i++)
if (vr_val[i] != rh_range[i])
ERR_RET;
free(vr_val);
/* Inquiry global attribute */
if (nc_inq_att (ncid, NC_GLOBAL, "g_valid_range", &vr_type, &vr_len)) ERR;
/* Check stuff. */
if(vr_type != NC_DOUBLE || vr_len != orivr_len) ERR;
/* Obtain global attribute value */
vr_val = (double *) malloc(vr_len * sizeof(double));
if (nc_get_att_double(ncid, NC_GLOBAL, "g_valid_range", vr_val)) ERR;
/* Check global attribute value */
for(i = 0; i < vr_len; i++)
if (vr_val[i] != rh_range[i]) ERR_RET;
free(vr_val);
/* Inquiry string attribute of a variable */
if (nc_inq_att (ncid, nvid, "title", &st_type, &st_len)) ERR;
/* check string attribute length */
if(st_len != strlen(title)) ERR_RET;
/* Check string attribute type */
if(st_type != NC_CHAR) ERR_RET;
/* Allocate meory for string attribute */
st_val = (char *) malloc(st_len * (sizeof(char)));
/* Obtain variable string attribute value */
if (nc_get_att_text(ncid, nvid,"title", st_val)) ERR;
/*check string value */
if(strncmp(st_val,title,st_len)) {
free(st_val);
ERR_RET;
}
free(st_val);
/*Inquiry global attribute */
if (nc_inq_att (ncid, NC_GLOBAL, "g_title", &st_type, &st_len)) ERR;
/* check attribute length*/
if(st_len != strlen(title)) ERR_RET;
/*check attribute type*/
if(st_type != NC_CHAR) ERR_RET;
/* obtain global string attribute value */
st_val = (char*)malloc(st_len*sizeof(char));
if (nc_get_att_text(ncid, NC_GLOBAL,"g_title", st_val)) ERR;
/* check attribute value */
if(strncmp(st_val,title,st_len)){
free(st_val);
ERR_RET;
}
free(st_val);
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
return 0;
}
/*ARGSUSED*/
int
main(int argc, char *argv[])
{
int cmode=NC_CLOBBER, omode, ret;
int id;
char buf[256];
#ifdef SYNCDEBUG
char *str = "one";
#endif
int ii;
size_t ui;
const struct tcdfvar *tvp = testvars;
union getret got;
const size_t initialsz = 8192;
size_t chunksz = 8192;
size_t align = 8192/32;
MPI_Init(&argc, &argv);
/* cmode |= NC_PNETCDF |NC_64BIT_OFFSET; */
cmode != NC_PNETCDF |NC_64BIT_DATA;
ret = nc_create_par(fname,cmode, MPI_COMM_WORLD, MPI_INFO_NULL, &id);
if(ret != NC_NOERR) {
fprintf(stderr,"Error %s in file %s at line %d\n",nc_strerror(ret),__FILE__,__LINE__);
exit(ret);
}
assert( nc_put_att_text(id, NC_GLOBAL,
"TITLE", 12, "another name") == NC_NOERR);
assert( nc_get_att_text(id, NC_GLOBAL,
"TITLE", buf) == NC_NOERR);
/* (void) printf("title 1 \"%s\"\n", buf); */
assert( nc_put_att_text(id, NC_GLOBAL,
"TITLE", strlen(fname), fname) == NC_NOERR);
assert( nc_get_att_text(id, NC_GLOBAL,
"TITLE", buf) == NC_NOERR);
buf[strlen(fname)] = 0;
/* (void) printf("title 2 \"%s\"\n", buf); */
assert( strcmp(fname, buf) == 0);
createtestdims(id, NUM_DIMS, sizes, dim_names);
testdims(id, NUM_DIMS, sizes, dim_names);
createtestvars(id, testvars, NUM_TESTVARS);
{
int ifill = -1; double dfill = -9999;
assert( nc_put_att_int(id, Long_id,
_FillValue, NC_INT, 1, &ifill) == NC_NOERR);
assert( nc_put_att_double(id, Double_id,
_FillValue, NC_DOUBLE, 1, &dfill) == NC_NOERR);
}
#ifdef REDEF
assert( nc__enddef(id, 0, align, 0, 2*align) == NC_NOERR );
assert( nc_put_var1_int(id, Long_id, indices[3], &birthday)
== NC_NOERR );
fill_seq(id);
assert( nc_redef(id) == NC_NOERR );
/* assert( nc_rename_dim(id,2, "a long dim name") == NC_NOERR); */
#endif
assert( nc_rename_dim(id,1, "IXX") == NC_NOERR);
assert( nc_inq_dim(id, 1, buf, &ui) == NC_NOERR);
/* (void) printf("dimrename: %s\n", buf); */
assert( nc_rename_dim(id,1, dim_names[1]) == NC_NOERR);
#ifdef ATTRX
assert( nc_rename_att(id, 1, "UNITS", "units") == NC_NOERR);
assert( nc_del_att(id, 4, "FIELDNAM")== NC_NOERR);
assert( nc_del_att(id, 2, "SCALEMIN")== NC_NOERR);
assert( nc_del_att(id, 2, "SCALEMAX")== NC_NOERR);
#endif /* ATTRX */
assert( nc__enddef(id, 0, align, 0, 2*align) == NC_NOERR );
#ifndef REDEF
fill_seq(id);
assert( nc_put_var1_int(id, Long_id, indices[3], &birthday)== NC_NOERR );
#endif
assert( nc_put_vara_schar(id, Byte_id, s_start, s_edges,
(signed char *)sentence)
== NC_NOERR);
assert( nc_put_var1_schar(id, Byte_id, indices[6], (signed char *)(chs+1))
== NC_NOERR);
assert( nc_put_var1_schar(id, Byte_id, indices[5], (signed char *)chs)
== NC_NOERR);
assert( nc_put_vara_text(id, Char_id, s_start, s_edges, sentence)
== NC_NOERR);
assert( nc_put_var1_text(id, Char_id, indices[6], (chs+1))
== NC_NOERR) ;
assert( nc_put_var1_text(id, Char_id, indices[5], chs)
== NC_NOERR);
assert( nc_put_var1_short(id, Short_id, indices[4], shs)
== NC_NOERR);
assert( nc_put_var1_float(id, Float_id, indices[2], &e)
== NC_NOERR);
assert( nc_put_var1_double(id, Double_id, indices[1], &zed)
== NC_NOERR);
assert( nc_put_var1_double(id, Double_id, indices[0], &pinot)
== NC_NOERR);
#ifdef SYNCDEBUG
(void) printf("Hit Return to sync\n");
gets(str);
nc_sync(id,0);
(void) printf("Sync done. Hit Return to continue\n");
gets(str);
#endif /* SYNCDEBUG */
ret = nc_close(id);
/* (void) printf("nc_close ret = %d\n\n", ret); */
/*
* read it
*/
omode = NC_NOWRITE;
omode = NC_NOWRITE | NC_PNETCDF;
if(ret != NC_NOERR)
{
(void) printf("Could not open %s: %s\n", fname,
nc_strerror(ret));
exit(1);
}
/* (void) printf("reopen id = %d for filename %s\n", */
/* id, fname); */
/* NC */
/* (void) printf("NC "); */
assert( nc_inq(id, &(cdesc->num_dims), &(cdesc->num_vars),
&(cdesc->num_attrs), &(cdesc->xtendim) ) == NC_NOERR);
assert((size_t) cdesc->num_dims == num_dims);
assert(cdesc->num_attrs == 1);
assert(cdesc->num_vars == NUM_TESTVARS);
/* (void) printf("done\n"); */
/* GATTR */
/* (void) printf("GATTR "); */
assert( nc_inq_attname(id, NC_GLOBAL, 0, adesc->mnem) == 0);
assert(strcmp("TITLE",adesc->mnem) == 0);
assert( nc_inq_att(id, NC_GLOBAL, adesc->mnem, &(adesc->type), &(adesc->len))== NC_NOERR);
assert( adesc->type == NC_CHAR );
assert( adesc->len == strlen(fname) );
assert( nc_get_att_text(id, NC_GLOBAL, "TITLE", buf)== NC_NOERR);
buf[adesc->len] = 0;
assert( strcmp(fname, buf) == 0);
/* VAR */
/* (void) printf("VAR "); */
assert( cdesc->num_vars == NUM_TESTVARS );
for(ii = 0; ii < cdesc->num_vars; ii++, tvp++ )
{
int jj;
assert( nc_inq_var(id, ii,
vdesc->mnem,
&(vdesc->type),
&(vdesc->ndims),
vdesc->dims,
&(vdesc->num_attrs)) == NC_NOERR);
if(strcmp(tvp->mnem , vdesc->mnem) != 0)
{
(void) printf("attr %d mnem mismatch %s, %s\n",
ii, tvp->mnem, vdesc->mnem);
continue;
}
if(tvp->type != vdesc->type)
{
(void) printf("attr %d type mismatch %d, %d\n",
ii, (int)tvp->type, (int)vdesc->type);
continue;
}
for(jj = 0; jj < vdesc->ndims; jj++ )
{
if(tvp->dims[jj] != vdesc->dims[jj] )
{
(void) printf(
"inconsistent dim[%d] for variable %d: %d != %d\n",
jj, ii, tvp->dims[jj], vdesc->dims[jj] );
continue;
}
}
/* VATTR */
/* (void) printf("VATTR\n"); */
for(jj=0; jj<vdesc->num_attrs; jj++ )
{
assert( nc_inq_attname(id, ii, jj, adesc->mnem) == NC_NOERR);
if( strcmp(adesc->mnem, reqattr[jj]) != 0 )
{
(void) printf("var %d attr %d mismatch %s != %s\n",
ii, jj, adesc->mnem, reqattr[jj] );
break;
}
}
if( nc_inq_att(id, ii, reqattr[0], &(adesc->type), &(adesc->len))
!= -1) {
assert( adesc->type == NC_CHAR );
assert( adesc->len == strlen(tvp->units) );
assert( nc_get_att_text(id,ii,reqattr[0],buf)== NC_NOERR);
buf[adesc->len] = 0;
assert( strcmp(tvp->units, buf) == 0);
}
if(
nc_inq_att(id, ii, reqattr[1], &(adesc->type), &(adesc->len))
!= -1)
{
assert( adesc->type == NC_DOUBLE );
assert( adesc->len == 1 );
assert( nc_get_att_double(id, ii, reqattr[1], &got.dbl)== NC_NOERR);
chkgot(adesc->type, got, tvp->validmin);
}
if(
nc_inq_att(id, ii, reqattr[2], &(adesc->type), &(adesc->len))
!= -1)
{
assert( adesc->type == NC_DOUBLE );
assert( adesc->len == 1 );
assert( nc_get_att_double(id, ii, reqattr[2], &got.dbl)== NC_NOERR);
chkgot(adesc->type, got, tvp->validmax);
}
if(
nc_inq_att(id, ii, reqattr[3], &(adesc->type), &(adesc->len))
!= -1)
{
assert( adesc->type == NC_DOUBLE );
assert( adesc->len ==1 );
assert( nc_get_att_double(id, ii, reqattr[3], &got.dbl)== NC_NOERR);
chkgot(adesc->type, got, tvp->scalemin);
}
if(
nc_inq_att(id, ii, reqattr[4], &(adesc->type), &(adesc->len))
!= -1)
{
assert( adesc->type == NC_DOUBLE );
assert( adesc->len == 1 );
assert( nc_get_att_double(id, ii, reqattr[4], &got.dbl)== NC_NOERR);
chkgot(adesc->type, got, tvp->scalemax);
}
if( nc_inq_att(id, ii, reqattr[5], &(adesc->type), &(adesc->len))== NC_NOERR)
{
assert( adesc->type == NC_CHAR );
assert( adesc->len == strlen(tvp->fieldnam) );
assert( nc_get_att_text(id,ii,reqattr[5],buf)== NC_NOERR);
buf[adesc->len] = 0;
assert( strcmp(tvp->fieldnam, buf) == 0);
}
}
/* (void) printf("fill_seq "); */
check_fill_seq(id);
/* (void) printf("Done\n"); */
assert( nc_get_var1_double(id, Double_id, indices[0], &got.dbl)== NC_NOERR);
/* (void) printf("got val = %f\n", got.dbl ); */
assert( nc_get_var1_double(id, Double_id, indices[1], &got.dbl)== NC_NOERR);
/* (void) printf("got val = %f\n", got.dbl ); */
assert( nc_get_var1_float(id, Float_id, indices[2], &got.fl[0])== NC_NOERR);
/* (void) printf("got val = %f\n", got.fl[0] ); */
assert( nc_get_var1_int(id, Long_id, indices[3], &got.in[0])== NC_NOERR);
/* (void) printf("got val = %d\n", got.in[0] ); */
assert( nc_get_var1_short(id, Short_id, indices[4], &got.sh[0])== NC_NOERR);
/* (void) printf("got val = %d\n", got.sh[0] ); */
assert( nc_get_var1_text(id, Char_id, indices[5], &got.by[0]) == NC_NOERR);
/* (void) printf("got NC_CHAR val = %c (0x%02x) \n", */
/* got.by[0] , got.by[0]); */
assert( nc_get_var1_text(id, Char_id, indices[6], &got.by[0]) == NC_NOERR);
/* (void) printf("got NC_CHAR val = %c (0x%02x) \n", */
/* got.by[0], got.by[0] ); */
(void) memset(buf,0,sizeof(buf));
assert( nc_get_vara_text(id, Char_id, s_start, s_edges, buf) == NC_NOERR);
/* (void) printf("got NC_CHAR val = \"%s\"\n", buf); */
assert( nc_get_var1_schar(id, Byte_id, indices[5],
(signed char *)&got.by[0])== NC_NOERR);
/* (void) printf("got val = %c (0x%02x) \n", got.by[0] , got.by[0]); */
assert( nc_get_var1_schar(id, Byte_id, indices[6],
(signed char *)&got.by[0])== NC_NOERR);
/* (void) printf("got val = %c (0x%02x) \n", got.by[0], got.by[0] ); */
(void) memset(buf,0,sizeof(buf));
assert( nc_get_vara_schar(id, Byte_id, s_start, s_edges,
(signed char *)buf)== NC_NOERR );
/* (void) printf("got val = \"%s\"\n", buf); */
{
double dbuf[NUM_RECS * SIZE_1 * SIZE_2];
assert(nc_get_var_double(id, Float_id, dbuf) == NC_NOERR);
/* (void) printf("got vals = %f ... %f\n", dbuf[0], */
/* dbuf[NUM_RECS * SIZE_1 * SIZE_2 -1] ); */
}
ret = nc_close(id);
/* (void) printf("re nc_close ret = %d\n", ret); */
MPI_Finalize();
return 0;
}
/* test different hyperslab settings */
int test_pio_hyper(int flag){
/* MPI stuff. */
int mpi_size, mpi_rank;
int res = NC_NOERR;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Netcdf-4 stuff. */
int ncid;
int nvid;
int rvid;
int j, i;
/* two dimensional integer data test */
int dimids[NDIMS1];
size_t start[NDIMS1], count[NDIMS1];
int *data;
int *tempdata;
int *rdata;
int *temprdata;
int count_atom;
/* Initialize MPI. */
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
if(mpi_size == 1) return 0;
/* Create a parallel netcdf-4 file. */
/* nc_set_log_level(NC_TURN_OFF_LOGGING); */
/* nc_set_log_level(4);*/
if (nc_create_par(file_name, facc_type, comm, info, &ncid)) ERR;
/* The case is two dimensional variables, no unlimited dimension */
/* Create two dimensions. */
if (nc_def_dim(ncid, "d1", DIMSIZE2, dimids)) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
/* Create one var. */
if (nc_def_var(ncid, "v1", NC_INT, NDIMS1, dimids, &nvid)) ERR;
if (nc_enddef(ncid)) ERR;
/* hyperslab illustration for 3-processor case
--------
|aaaacccc|
|aaaacccc|
|bbbb |
|bbbb |
--------
*/
/* odd number of processors should be treated differently */
if(mpi_size%2 != 0) {
count_atom = DIMSIZE*2/(mpi_size+1);
if(mpi_rank <= mpi_size/2) {
start[0] = 0;
start[1] = mpi_rank*count_atom;
count[0] = DIMSIZE2/2;
count[1] = count_atom;
}
else {
start[0] = DIMSIZE2/2;
start[1] = (mpi_rank-mpi_size/2-1)*count_atom;
count[0] = DIMSIZE2/2;
count[1] = count_atom;
}
}
else {
count_atom = DIMSIZE*2/mpi_size;
if(mpi_rank < mpi_size/2) {
start[0] = 0;
start[1] = mpi_rank*count_atom;
count[0] = DIMSIZE2/2;
count[1] = count_atom;
}
else {
start[0] = DIMSIZE2/2;
start[1] = (mpi_rank-mpi_size/2)*count_atom;
count[0] = DIMSIZE2/2;
count[1] = count_atom;
}
}
if (nc_var_par_access(ncid, nvid, flag)) ERR;
data = malloc(sizeof(int)*count[1]*count[0]);
tempdata = data;
for (j=0; j<count[0];j++){
for (i=0; i<count[1]; i++){
*tempdata = mpi_rank*(j+1);
tempdata ++;
}
}
if (nc_put_vara_int(ncid, nvid, start, count, data)) ERR;
free(data);
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
if (nc_open_par(file_name, facc_type_open, comm, info, &ncid)) ERR;
/* Inquiry the variable */
if (nc_inq_varid(ncid, "v1", &rvid)) ERR;
if (nc_var_par_access(ncid, rvid, flag)) ERR;
rdata = malloc(sizeof(int)*count[1]*count[0]);
/* Read the data with the same slab settings */
if (nc_get_vara_int(ncid, rvid, start, count, rdata)) ERR;
temprdata = rdata;
for (j=0; j<count[0];j++){
for (i=0; i<count[1]; i++){
if(*temprdata != mpi_rank*(j+1))
{
res = -1;
break;
}
temprdata++;
}
}
free(rdata);
if(res == -1) ERR_RET;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
return 0;
}
int main(int argc, char* argv[])
{
//init MPI
MPI_Init( &argc, &argv);
int rank, size;
MPI_Comm_rank( MPI_COMM_WORLD, &rank);
MPI_Comm_size( MPI_COMM_WORLD, &size);
//create a grid and some data
if( size != 4){ std::cerr << "Please run with 4 threads!\n"; return -1;}
double Tmax=2.*M_PI;
double NT = 10;
double dt = Tmax/NT;
dg::Grid1d<double> gx( 0, 2.*M_PI, 3, 10);
dg::Grid1d<double> gy( 0, 2.*M_PI, 3, 10);
dg::Grid1d<double> gz( 0, 2.*M_PI, 1, 20);
dg::Grid3d<double> g( gx, gy, gz);
std::string hello = "Hello world\n";
thrust::host_vector<double> data = dg::evaluate( function, g);
//create NetCDF File
int ncid;
file::NC_Error_Handle err;
MPI_Info info = MPI_INFO_NULL;
err = nc_create_par( "testmpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, MPI_COMM_WORLD, info, &ncid);
err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
int dimids[4], tvarID;
err = file::define_dimensions( ncid, dimids, &tvarID, g);
int dataID;
err = nc_def_var( ncid, "data", NC_DOUBLE, 4, dimids, &dataID);
/* Write metadata to file. */
err = nc_enddef(ncid);
//err = nc_enddef( ncid);
size_t start[4] = {0, rank*g.Nz()/size, 0, 0};
size_t count[4] = {1, g.Nz()/size, g.Ny()*g.n(), g.Nx()*g.n()};
if( rank==0) std::cout<< "Write from "<< start[0]<< " "<<start[1]<<" "<<start[2]<<" "<<start[3]<<std::endl;
if( rank==0) std::cout<< "Number of elements "<<count[0]<< " "<<count[1]<<" "<<count[2]<<" "<<count[3]<<std::endl;
err = nc_var_par_access(ncid, dataID , NC_COLLECTIVE);
err = nc_var_par_access(ncid, tvarID , NC_COLLECTIVE);
size_t Tcount=1, Tstart=0;
double time = 0;
//err = nc_close(ncid);
for(unsigned i=0; i<=NT; i++)
{
if(rank==0)std::cout<<"Write timestep "<<i<<"\n";
//err = nc_open_par( "testmpi.nc", NC_WRITE|NC_MPIIO, MPI_COMM_WORLD, info, &ncid); //doesn't work I don't know why
time = i*dt;
Tstart = i;
data = dg::evaluate( function, g);
dg::blas1::scal( data, cos( time));
start[0] = i;
//write dataset (one timeslice)
err = nc_put_vara_double( ncid, dataID, start, count, data.data() + start[1]*count[2]*count[3]);
err = nc_put_vara_double( ncid, tvarID, &Tstart, &Tcount, &time);
//err = nc_close(ncid);
}
err = nc_close(ncid);
MPI_Finalize();
return 0;
}
int
main(int argc, char **argv)
{
int mpi_namelen;
char mpi_name[MPI_MAX_PROCESSOR_NAME];
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
double start_time = 0, total_time;
int mpi_size_in;
#define NUM_TEST_TYPES 11
nc_type test_type[NUM_TEST_TYPES] = {NC_BYTE, NC_CHAR, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
int tt, fv;
int j, i, k, ret;
/* Initialize MPI. */
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Get_processor_name(mpi_name, &mpi_namelen);
/* Must be able to evenly divide my slabs between processors. */
if (NUM_SLABS % mpi_size)
{
if (!mpi_rank)
printf("NUM_SLABS (%d) is not evenly divisible by mpi_size(%d)\n",
NUM_SLABS, mpi_size);
ERR;
}
if (!mpi_rank)
printf("\n*** Testing parallel I/O some more.\n");
/* Test for different fill value settings. */
for (fv = 0; fv < NUM_FILL_TEST_RUNS; fv++)
{
/* Test for different netCDF types. */
for (tt = 0; tt < NUM_TEST_TYPES; tt++)
{
char file_name[NC_MAX_NAME + 1];
int fill_mode_in;
void *data, *data_in;
void *fill_value, *fill_value_in;
size_t type_size;
size_t write_start[NDIMS] = {0, 0, 1};
size_t write_count[NDIMS] = {1, DIMSIZE, DIMSIZE - 1};
size_t read_start[NDIMS] = {0, 0, 0};
size_t read_count[NDIMS] = {1, DIMSIZE, DIMSIZE};
int ncid, varid, dimids[NDIMS];
int ndims_in, nvars_in, natts_in, unlimdimid_in;
/* Fill values to be expected. */
signed char byte_expected_fill_value;
unsigned char char_expected_fill_value;
short short_expected_fill_value;
int int_expected_fill_value;
float float_expected_fill_value;
double double_expected_fill_value;
unsigned char ubyte_expected_fill_value;
unsigned short ushort_expected_fill_value;
unsigned int uint_expected_fill_value;
long long int int64_expected_fill_value;
unsigned long long int uint64_expected_fill_value;
/* Fill values used when writing. */
signed char byte_fill_value = -TEST_VAL_42;
unsigned char char_fill_value = 'x';
short short_fill_value = TEST_VAL_42 * 100;
int int_fill_value = TEST_VAL_42 * 1000;
float float_fill_value = TEST_VAL_42 * 1000;
double double_fill_value = TEST_VAL_42 * 1000;
unsigned char ubyte_fill_value = TEST_VAL_42;
unsigned short ushort_fill_value = TEST_VAL_42 * 100;
unsigned int uint_fill_value = TEST_VAL_42 * 1000;
long long int int64_fill_value = TEST_VAL_42 * 1000;
unsigned long long int uint64_fill_value = TEST_VAL_42 * 1000;
/* Fill values read in. */
signed char byte_fill_value_in;
unsigned char char_fill_value_in;
short short_fill_value_in;
int int_fill_value_in;
float float_fill_value_in;
double double_fill_value_in;
unsigned char ubyte_fill_value_in;
unsigned short ushort_fill_value_in;
unsigned int uint_fill_value_in;
long long int int64_fill_value_in;
unsigned long long int uint64_fill_value_in;
/* Data to write and read. */
signed char byte_data[DIMSIZE * DIMSIZE], byte_data_in[DIMSIZE * DIMSIZE];
unsigned char char_data[DIMSIZE * DIMSIZE], char_data_in[DIMSIZE * DIMSIZE];
short short_data[DIMSIZE * DIMSIZE], short_data_in[DIMSIZE * DIMSIZE];
int int_data[DIMSIZE * DIMSIZE], int_data_in[DIMSIZE * DIMSIZE];
float float_data[DIMSIZE * DIMSIZE], float_data_in[DIMSIZE * DIMSIZE];
double double_data[DIMSIZE * DIMSIZE], double_data_in[DIMSIZE * DIMSIZE];
unsigned char ubyte_data[DIMSIZE * DIMSIZE], ubyte_data_in[DIMSIZE * DIMSIZE];
unsigned short ushort_data[DIMSIZE * DIMSIZE], ushort_data_in[DIMSIZE * DIMSIZE];
unsigned int uint_data[DIMSIZE * DIMSIZE], uint_data_in[DIMSIZE * DIMSIZE];
long long int int64_data[DIMSIZE * DIMSIZE], int64_data_in[DIMSIZE * DIMSIZE];
unsigned long long int uint64_data[DIMSIZE * DIMSIZE], uint64_data_in[DIMSIZE * DIMSIZE];
if (!mpi_rank)
printf("*** writing a %d x %d x %d file from %d processors for fill value test %d type %d...\n",
NUM_SLABS, DIMSIZE, DIMSIZE, mpi_size, fv, test_type[tt]);
/* Initialize test data. */
switch(test_type[tt])
{
case NC_BYTE:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
byte_data[i] = mpi_rank;
data = byte_data;
data_in = byte_data_in;
byte_expected_fill_value = fv ? byte_fill_value : NC_FILL_BYTE;
fill_value = &byte_expected_fill_value;
fill_value_in = &byte_fill_value_in;
break;
case NC_CHAR:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
char_data[i] = mpi_rank;
data = char_data;
data_in = char_data_in;
char_expected_fill_value = fv ? char_fill_value : NC_FILL_CHAR;
fill_value = &char_expected_fill_value;
fill_value_in = &char_fill_value_in;
break;
case NC_SHORT:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
short_data[i] = mpi_rank;
data = short_data;
data_in = short_data_in;
short_expected_fill_value = fv ? short_fill_value : NC_FILL_SHORT;
fill_value = &short_expected_fill_value;
fill_value_in = &short_fill_value_in;
break;
case NC_INT:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
int_data[i] = mpi_rank;
data = int_data;
data_in = int_data_in;
int_expected_fill_value = fv ? int_fill_value : NC_FILL_INT;
fill_value = &int_expected_fill_value;
fill_value_in = &int_fill_value_in;
break;
case NC_FLOAT:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
float_data[i] = mpi_rank;
data = float_data;
data_in = float_data_in;
float_expected_fill_value = fv ? float_fill_value : NC_FILL_FLOAT;
fill_value = &float_expected_fill_value;
fill_value_in = &float_fill_value_in;
break;
case NC_DOUBLE:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
double_data[i] = mpi_rank;
data = double_data;
data_in = double_data_in;
double_expected_fill_value = fv ? double_fill_value : NC_FILL_DOUBLE;
fill_value = &double_expected_fill_value;
fill_value_in = &double_fill_value_in;
break;
case NC_UBYTE:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
ubyte_data[i] = mpi_rank;
data = ubyte_data;
data_in = ubyte_data_in;
ubyte_expected_fill_value = fv ? ubyte_fill_value : NC_FILL_UBYTE;
fill_value = &ubyte_expected_fill_value;
fill_value_in = &ubyte_fill_value_in;
break;
case NC_USHORT:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
ushort_data[i] = mpi_rank;
data = ushort_data;
data_in = ushort_data_in;
ushort_expected_fill_value = fv ? ushort_fill_value : NC_FILL_USHORT;
fill_value = &ushort_expected_fill_value;
fill_value_in = &ushort_fill_value_in;
break;
case NC_UINT:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
uint_data[i] = mpi_rank;
data = uint_data;
data_in = uint_data_in;
uint_expected_fill_value = fv ? uint_fill_value : NC_FILL_UINT;
fill_value = &uint_expected_fill_value;
fill_value_in = &uint_fill_value_in;
break;
case NC_INT64:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
int64_data[i] = mpi_rank;
data = int64_data;
data_in = int64_data_in;
int64_expected_fill_value = fv ? int64_fill_value : NC_FILL_INT64;
fill_value = &int64_expected_fill_value;
fill_value_in = &int64_fill_value_in;
break;
case NC_UINT64:
for (i = 0; i < DIMSIZE * DIMSIZE; i++)
uint64_data[i] = mpi_rank;
data = uint64_data;
data_in = uint64_data_in;
uint64_expected_fill_value = fv ? uint64_fill_value : NC_FILL_UINT64;
fill_value = &uint64_expected_fill_value;
fill_value_in = &uint64_fill_value_in;
break;
}
/* Create a file name. */
sprintf(file_name, "%s_type_%d_fv_%d.nc", TEST_NAME, test_type[tt], fv);
/* Create a parallel netcdf-4 file. */
if (nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid)) ERR;
/* Get the type len. */
if (nc_inq_type(ncid, test_type[tt], NULL, &type_size)) ERR;
/* A global attribute holds the number of processors that created
* the file. */
if (nc_put_att_int(ncid, NC_GLOBAL, "num_processors", NC_INT, 1, &mpi_size)) ERR;
/* Create three dimensions. */
if (nc_def_dim(ncid, DIM1_NAME, NUM_SLABS, dimids)) ERR;
if (nc_def_dim(ncid, DIM2_NAME, DIMSIZE, &dimids[1])) ERR;
if (nc_def_dim(ncid, DIM3_NAME, DIMSIZE, &dimids[2])) ERR;
/* Create one var. */
if (nc_def_var(ncid, VAR_NAME, test_type[tt], NDIMS, dimids, &varid)) ERR;
if (nc_put_att_int(ncid, varid, "var_num_processors", NC_INT, 1, &mpi_size)) ERR;
if (fv == 1)
{
if (nc_def_var_fill(ncid, varid, NC_FILL, fill_value)) ERR;
if (nc_inq_var_fill(ncid, varid, &fill_mode_in, fill_value_in)) ERR;
if (fill_mode_in != NC_FILL) ERR;
if (memcmp(fill_value_in, fill_value, type_size)) ERR;
}
else if (fv == 2)
{
if (nc_def_var_fill(ncid, varid, NC_NOFILL, NULL)) ERR;
if (nc_inq_var_fill(ncid, varid, &fill_mode_in, NULL)) ERR;
if (!fill_mode_in) ERR; /* nofill will be true */
}
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Change access mode to collective, then back to independent. */
if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;
if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
if (!mpi_rank)
start_time = MPI_Wtime();
/* Write all the slabs this process is responsible for. */
for (i = 0; i < NUM_SLABS / mpi_size; i++)
{
write_start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
/* Write one slab of data. Due to start/count settings,
* every 16th value will be a fill value. */
if (nc_put_vara(ncid, varid, write_start, write_count, data)) ERR;
}
/* On rank 0, keep track of time. */
if (!mpi_rank)
{
total_time = MPI_Wtime() - start_time;
printf("%d\t%g\t%g\n", mpi_size, total_time, DIMSIZE * DIMSIZE * NUM_SLABS *
sizeof(int) / total_time);
}
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
/* Reopen the file and check it. */
if ((ret = nc_open_par(file_name, NC_NOWRITE, comm, info, &ncid))) ERR;
if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR;
if (ndims_in != NDIMS || nvars_in != 1 || natts_in != 1 ||
unlimdimid_in != -1) ERR;
/* Check the attributes. */
if (nc_get_att_int(ncid, NC_GLOBAL, "num_processors", &mpi_size_in)) ERR;
if (mpi_size_in != mpi_size) ERR;
if (nc_get_att_int(ncid, 0, "var_num_processors", &mpi_size_in)) ERR;
if (mpi_size_in != mpi_size) ERR;
if (fv == 1)
{
if (nc_inq_var_fill(ncid, varid, &fill_mode_in, fill_value_in)) ERR;
if (fill_mode_in != NC_FILL) ERR;
if (memcmp(fill_value_in, fill_value, type_size)) ERR;
}
/* Read all the slabs this process is responsible for. */
for (i = 0; i < NUM_SLABS / mpi_size; i++)
{
read_start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
/* printf("mpi_rank %d i %d read_start[0] %ld\n", mpi_rank, i, read_start[0]); */
/* Read one slab of data. */
if (nc_get_vara(ncid, varid, read_start, read_count, data_in)) ERR;
/* Check data. For the third fill value test, fill is
* turned off. So don't bother testing the values where k
* is zero. */
/* printf("mpi_rank %d fv %d i %d j %d k %d int_data_in[j * k] %d int_expected_fill_value %d " */
/* "expected_value %d\n", mpi_rank, fv, i, j, k, int_data_in[j * k], */
/* int_expected_fill_value, expected_value); */
switch (test_type[tt])
{
case NC_BYTE:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (byte_data_in[j * DIMSIZE + k] != (signed char)(k ? mpi_rank : byte_expected_fill_value)) ERR;
break;
case NC_SHORT:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (short_data_in[j * DIMSIZE + k] != (short)(k ? mpi_rank : short_expected_fill_value)) ERR;
break;
case NC_INT:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (int_data_in[j * DIMSIZE + k] != (int)(k ? mpi_rank : int_expected_fill_value)) ERR;
break;
case NC_FLOAT:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (float_data_in[j * DIMSIZE + k] != (float)(k ? mpi_rank : float_expected_fill_value)) ERR;
break;
case NC_DOUBLE:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (double_data_in[j * DIMSIZE + k] != (double)(k ? mpi_rank : double_expected_fill_value)) ERR;
break;
case NC_UBYTE:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (ubyte_data_in[j * DIMSIZE + k] != (unsigned char)(k ? mpi_rank : ubyte_expected_fill_value)) ERR;
break;
case NC_USHORT:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (ushort_data_in[j * DIMSIZE + k] != (unsigned short)(k ? mpi_rank : ushort_expected_fill_value)) ERR;
break;
case NC_UINT:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (uint_data_in[j * DIMSIZE + k] != (unsigned int)(k ? mpi_rank : uint_expected_fill_value)) ERR;
break;
case NC_INT64:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (int64_data_in[j * DIMSIZE + k] != (long long int)(k ? mpi_rank : int64_expected_fill_value)) ERR;
break;
case NC_UINT64:
for (j = 0; j < DIMSIZE; j++)
for (k = 0; k < DIMSIZE; k++)
if (fv < 2 || k)
if (uint64_data_in[j * DIMSIZE + k] != (unsigned long long int)(k ? mpi_rank : uint64_expected_fill_value)) ERR;
break;
}
} /* next slab */
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
if (!mpi_rank)
SUMMARIZE_ERR;
} /* next test type */
} /* next fill value test run */
/* Shut down MPI. */
MPI_Finalize();
if (!mpi_rank)
FINAL_RESULTS;
return 0;
}
int
main(int argc, char **argv)
{
/* MPI stuff. */
int mpi_namelen;
char mpi_name[MPI_MAX_PROCESSOR_NAME];
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Netcdf-4 stuff. */
int ncid, v1id, dimids[NDIMS];
size_t start[NDIMS], count[NDIMS];
int data[DIMSIZE * DIMSIZE], i, res;
int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
char file_name[NC_MAX_NAME + 1];
#ifdef USE_MPE
int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close;
#endif /* USE_MPE */
/* Initialize MPI. */
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Get_processor_name(mpi_name, &mpi_namelen);
/*printf("mpi_name: %s size: %d rank: %d\n", mpi_name,
mpi_size, mpi_rank);*/
#ifdef USE_MPE
MPE_Init_log();
s_init = MPE_Log_get_event_number();
e_init = MPE_Log_get_event_number();
s_define = MPE_Log_get_event_number();
e_define = MPE_Log_get_event_number();
s_write = MPE_Log_get_event_number();
e_write = MPE_Log_get_event_number();
s_close = MPE_Log_get_event_number();
e_close = MPE_Log_get_event_number();
MPE_Describe_state(s_init, e_init, "Init", "red");
MPE_Describe_state(s_define, e_define, "Define", "yellow");
MPE_Describe_state(s_write, e_write, "Write", "green");
MPE_Describe_state(s_close, e_close, "Close", "purple");
MPE_Start_log();
MPE_Log_event(s_init, 0, "start init");
#endif /* USE_MPE */
if (mpi_rank == 1)
{
printf("\n*** tst_parallel testing very basic parallel access.\n");
printf("*** tst_parallel testing whether we can create file for parallel access and write to it...");
}
/* Create phony data. We're going to write a 24x24 array of ints,
in 4 sets of 144. */
/*printf("mpi_rank*QTR_DATA=%d (mpi_rank+1)*QTR_DATA-1=%d\n",
mpi_rank*QTR_DATA, (mpi_rank+1)*QTR_DATA);*/
for (i = mpi_rank * QTR_DATA; i < (mpi_rank + 1) * QTR_DATA; i++)
data[i] = mpi_rank;
for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
slab_data[i] = mpi_rank;
#ifdef USE_MPE
MPE_Log_event(e_init, 0, "end init");
MPE_Log_event(s_define, 0, "start define file");
#endif /* USE_MPE */
/* Create a parallel netcdf-4 file. */
/*nc_set_log_level(3);*/
sprintf(file_name, "%s/%s", TEMP_LARGE, FILE);
if ((res = nc_create_par(file_name, NC_NETCDF4|NC_MPIIO, comm,
info, &ncid))) ERR;
/* Create three dimensions. */
if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
/* Create one var. */
if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
/* Write metadata to file. */
if ((res = nc_enddef(ncid))) ERR;
#ifdef USE_MPE
MPE_Log_event(e_define, 0, "end define file");
if (mpi_rank)
sleep(mpi_rank);
#endif /* USE_MPE */
/* Set up slab for this process. */
start[0] = mpi_rank * DIMSIZE/mpi_size;
start[1] = 0;
count[0] = DIMSIZE/mpi_size;
count[1] = DIMSIZE;
count[2] = 1;
/*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
mpi_rank, start[0], start[1], count[0], count[1]);*/
if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
/* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
{
#ifdef USE_MPE
MPE_Log_event(s_write, 0, "start write slab");
#endif /* USE_MPE */
/* Write slabs of phoney data. */
if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
#ifdef USE_MPE
MPE_Log_event(e_write, 0, "end write file");
#endif /* USE_MPE */
}
#ifdef USE_MPE
MPE_Log_event(s_close, 0, "start close file");
#endif /* USE_MPE */
/* Close the netcdf file. */
if ((res = nc_close(ncid))) ERR;
#ifdef USE_MPE
MPE_Log_event(e_close, 0, "end close file");
#endif /* USE_MPE */
/* Delete this large file. */
remove(file_name);
/* Shut down MPI. */
MPI_Finalize();
if (mpi_rank == 1)
{
SUMMARIZE_ERR;
FINAL_RESULTS;
}
return 0;
}