/* Run tests for darray functions. */
int main(int argc, char **argv)
{
int my_rank;
int ntasks;
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
MPI_Comm test_comm; /* A communicator for this test. */
int ret; /* Return code. */
/* Initialize test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
MIN_NTASKS, -1, &test_comm)))
ERR(ERR_INIT);
if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
return ret;
/* Only do something on max_ntasks tasks. */
if (my_rank < TARGET_NTASKS)
{
int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
int iosysid; /* The ID for the parallel I/O system. */
int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */
int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */
int ret; /* Return code. */
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
{
/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
ioproc_start, rearranger[r], &iosysid)))
return ret;
/* Run tests. */
if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank,
rearranger[r], test_comm)))
return ret;
/* Finalize PIO system. */
if ((ret = PIOc_finalize(iosysid)))
return ret;
} /* next rearranger */
} /* endif my_rank < TARGET_NTASKS */
/* Finalize the MPI library. */
if ((ret = pio_test_finalize(&test_comm)))
return ret;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/* Run Tests for Init_Intercomm. */
int main(int argc, char **argv)
{
/* Zero-based rank of processor. */
int my_rank;
/* Number of processors involved in current execution. */
int ntasks;
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
/* Names for the output files. */
char filename[NUM_FLAVORS][NC_MAX_NAME + 1];
/* The ID for the parallel I/O system. */
int iosysid[COMPONENT_COUNT];
/* Return code. */
int ret;
MPI_Comm test_comm;
char too_long_name[PIO_MAX_NAME * 5 + 1];
/* Create a name that is too long. */
memset(too_long_name, 74, PIO_MAX_NAME * 5);
too_long_name[PIO_MAX_NAME * 5] = 0;
/* Set up test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS,
-1, &test_comm)))
ERR(ERR_INIT);
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
if (my_rank < TARGET_NTASKS)
{
/* How many processors will be used for our IO and 2 computation components. */
int num_procs[COMPONENT_COUNT] = {2};
/* Is the current process a computation task? */
int comp_task = my_rank < 2 ? 0 : 1;
/* Index of computation task in iosysid array. Varies by rank and
* does not apply to IO component processes. */
int my_comp_idx = comp_task ? 0 : -1;
/* Initialize the IO system. */
if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid)))
ERR(ERR_AWFUL);
/* All the netCDF calls are only executed on the computation
* tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
* and when the do, they should go straight to finalize. */
if (comp_task)
{
for (int fmt = 0; fmt < num_flavors; fmt++)
{
int ncid, varid, dimid;
PIO_Offset start[NDIM], count[NDIM] = {0};
int data[DIM_LEN];
/* Create the filename for this flavor. */
sprintf(filename[fmt], "test_intercomm2_%d.nc", flavor[fmt]);
/* Create a netCDF file with one dimension and one variable. */
if ((ret = PIOc_createfile(iosysid[my_comp_idx], &ncid, &flavor[fmt], filename[fmt],
NC_CLOBBER)))
ERR(ret);
/* End define mode, then re-enter it. */
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
if ((ret = PIOc_redef(ncid)))
ERR(ret);
/* Test the inq_format function. */
int myformat;
if (PIOc_inq_format(ncid + TEST_VAL_42, &myformat) != PIO_EBADID)
ERR(ERR_WRONG);
if ((ret = PIOc_inq_format(ncid, &myformat)))
ERR(ret);
if ((flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF) &&
myformat != 1)
ERR(ERR_AWFUL);
else if ((flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) &&
myformat != 3)
ERR(ERR_AWFUL);
/* Test the inq_type function for atomic types. */
char type_name[NC_MAX_NAME + 1];
PIO_Offset type_size;
nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8};
int max_type = flavor[fmt] == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64;
/* This should not work. */
if (PIOc_inq_type(ncid + TEST_VAL_42, xtype[0], type_name, &type_size) != PIO_EBADID)
ERR(ERR_WRONG);
/* These should work. */
for (int i = 0; i < max_type; i++)
{
if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size)))
ERR(ret);
if (type_size != type_len[i])
ERR(ERR_AWFUL);
}
/* Define a dimension. */
char dimname2[NC_MAX_NAME + 1];
if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME, DIM_LEN, &dimid)))
ERR(ret);
if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
ERR(ret);
if (strcmp(dimname2, FIRST_DIM_NAME))
ERR(ERR_WRONG);
if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME)))
ERR(ret);
/* These should not work. */
if (PIOc_rename_dim(ncid + TEST_VAL_42, 0, DIM_NAME) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_rename_dim(ncid, 0, NULL) != PIO_EINVAL)
ERR(ERR_WRONG);
if (PIOc_rename_dim(ncid, 0, too_long_name) != PIO_EINVAL)
ERR(ERR_WRONG);
/* Define a 1-D variable. */
char varname2[NC_MAX_NAME + 1];
if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME, NC_INT, NDIM, &dimid, &varid)))
ERR(ret);
if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
ERR(ret);
if (strcmp(varname2, FIRST_VAR_NAME))
ERR(ERR_WRONG);
if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME)))
ERR(ret);
/* These should not work. */
if (PIOc_rename_var(ncid + TEST_VAL_42, 0, VAR_NAME) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_rename_var(ncid, 0, NULL) != PIO_EINVAL)
ERR(ERR_WRONG);
if (PIOc_rename_var(ncid, 0, too_long_name) != PIO_EINVAL)
ERR(ERR_WRONG);
/* Add a global attribute. */
int att_data = ATT_VALUE;
short short_att_data = ATT_VALUE;
float float_att_data = ATT_VALUE;
double double_att_data = ATT_VALUE;
char attname2[NC_MAX_NAME + 1];
/* Write an att and rename it. */
if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
ERR(ret);
if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2)))
ERR(ret);
if (strcmp(attname2, FIRST_ATT_NAME))
ERR(ERR_WRONG);
if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME)))
ERR(ret);
/* These should not work. */
if (PIOc_inq_attname(ncid + TEST_VAL_42, NC_GLOBAL, 0, attname2) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_rename_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL) != PIO_EINVAL)
ERR(ERR_WRONG);
if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, too_long_name) != PIO_EINVAL)
ERR(ERR_WRONG);
if (PIOc_del_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_del_att(ncid, NC_GLOBAL, NULL) != PIO_EINVAL)
ERR(ERR_WRONG);
if (PIOc_del_att(ncid, NC_GLOBAL, too_long_name) != PIO_EINVAL)
ERR(ERR_WRONG);
/* Write an att and delete it. */
if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
ERR(ret);
if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME)))
ERR(ret);
/* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL, NULL)) != PIO_ENOTATT) */
/* { */
/* printf("ret = %d\n", ret); */
/* ERR(ERR_AWFUL); */
/* } */
/* Write some atts of different types. */
if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data)))
ERR(ret);
if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data)))
ERR(ret);
if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, NC_DOUBLE, 1, &double_att_data)))
ERR(ret);
/* Check some att types. */
nc_type myatttype;
if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, SHORT_ATT_NAME, &myatttype)))
ERR(ret);
if (myatttype != NC_SHORT)
ERR(ERR_WRONG);
if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &myatttype)))
ERR(ret);
if (myatttype != NC_FLOAT)
ERR(ERR_WRONG);
if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &myatttype)))
ERR(ret);
if (myatttype != NC_DOUBLE)
ERR(ERR_WRONG);
/* End define mode. */
if ((ret = PIOc_enddef(ncid)))
ERR(ret);
/* Write some data. For the PIOc_put/get functions, all
* data must be on compmaster before the function is
* called. Only compmaster's arguments are passed to the
* async msg handler. All other computation tasks are
* ignored. */
for (int i = 0; i < DIM_LEN; i++)
data[i] = i;
start[0] = 0;
count[0] = DIM_LEN;
if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
ERR(ret);
/* Close the file. */
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
/* Check the file for correctness. */
if ((ret = check_file(iosysid[my_comp_idx], flavor[fmt], filename[fmt], my_rank)))
ERR(ret);
/* Now delete the file. */
/* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */
/* ERR(ret); */
/* if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename[fmt], */
/* NC_NOWRITE)) != PIO_ENFILE) */
/* ERR(ERR_AWFUL); */
} /* next netcdf flavor */
/* Finalize the IO system. Only call this from the computation tasks. */
if ((ret = PIOc_finalize(iosysid[my_comp_idx])))
ERR(ret);
}
} /* my_rank < TARGET_NTASKS */
/* Finalize test. */
if ((ret = pio_test_finalize(&test_comm)))
return ERR_AWFUL;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/** Run async tests. */
int
main(int argc, char **argv)
{
int my_rank; /* Zero-based rank of processor. */
int ntasks; /* Number of processors involved in current execution. */
int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
int flv; /* Index for loop of PIO netcdf flavors. */
int ret; /* Return code. */
int flavor[NUM_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF,
PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P};
/* Num procs for IO and computation. */
int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{3, 1}, {2, 2}, {1, 3}};
/* Number of processors that will do IO. */
int num_io_procs[NUM_COMBOS] = {3, 2, 1};
/* Initialize test. */
if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS)))
ERR(ERR_INIT);
for (int combo = 0; combo < NUM_COMBOS; combo++)
{
/* Is the current process a computation task? */
int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
/* Initialize the IO system. */
if ((ret = PIOc_Init_Async(MPI_COMM_WORLD, num_io_procs[combo], NULL, COMPONENT_COUNT,
num_procs[combo], NULL, iosysid)))
ERR(ERR_INIT);
for (int c = 0; c < COMPONENT_COUNT; c++)
printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
/* All the netCDF calls are only executed on the computation
* tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
* and when the do, they should go straight to finalize. */
if (comp_task)
{
for (int flv = 0; flv < NUM_FLAVORS; flv++)
{
char filename[NC_MAX_NAME + 1]; /* Test filename. */
int my_comp_idx = 0; /* Index in iosysid array. */
for (int sample = 0; sample < NUM_SAMPLES; sample++)
{
/* Create a filename. */
sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, flavor_name(flv), sample, my_comp_idx);
/* Create sample file. */
printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank)))
ERR(ret);
/* Check the file for correctness. */
if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank)))
ERR(ret);
}
} /* next netcdf flavor */
/* Finalize the IO system. Only call this from the computation tasks. */
printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
for (int c = 0; c < COMPONENT_COUNT; c++)
{
if ((ret = PIOc_finalize(iosysid[c])))
ERR(ret);
printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
iosysid[c]);
}
} /* endif comp_task */
/* Wait for everyone to catch up. */
printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
MPI_Barrier(MPI_COMM_WORLD);
} /* next combo */
/* Finalize test. */
printf("%d %s finalizing...\n", my_rank, TEST_NAME);
if ((ret = pio_test_finalize()))
ERR(ERR_AWFUL);
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/* Run async tests. */
int main(int argc, char **argv)
{
int my_rank; /* Zero-based rank of processor. */
int ntasks; /* Number of processors involved in current execution. */
int iosysid_world; /* The ID for the parallel I/O system. */
int even_iosysid; /* The ID for iosystem of even_comm. */
int overlap_iosysid; /* The ID for iosystem of even_comm. */
MPI_Group world_group; /* An MPI group of world. */
MPI_Group even_group; /* An MPI group of 0 and 2. */
MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */
MPI_Comm even_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 2 */
MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
int even_rank = -1, overlap_rank = -1; /* Tasks rank in communicator. */
int even_size = 0, overlap_size = 0; /* Size of communicator. */
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
MPI_Comm test_comm;
int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
int ret; /* Return code. */
/* Initialize test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS,
-1, &test_comm)))
ERR(ERR_INIT);
/* Test code runs on TARGET_NTASKS tasks. The left over tasks do
* nothing. */
if (my_rank < TARGET_NTASKS)
{
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
/* Test with both rearrangers. */
for (int r = 0; r < NUM_REARRANGERS; r++)
{
/* Initialize PIO system on world. */
if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, rearranger[r],
&iosysid_world)))
ERR(ret);
/* Set the error handler. */
if ((ret = PIOc_set_iosystem_error_handling(iosysid_world, PIO_BCAST_ERROR, NULL)))
ERR(ret);
/* Get MPI_Group of world comm. */
if ((ret = MPI_Comm_group(test_comm, &world_group)))
ERR(ret);
/* Create a group with tasks 0 and 2. */
int even_ranges[EVEN_NUM_RANGES][3] = {{0, 2, 2}};
if ((ret = MPI_Group_range_incl(world_group, EVEN_NUM_RANGES, even_ranges,
&even_group)))
ERR(ret);
/* Create a communicator from the even_group. */
if ((ret = MPI_Comm_create(test_comm, even_group, &even_comm)))
ERR(ret);
/* Learn my rank and the total number of processors in even group. */
if (even_comm != MPI_COMM_NULL)
{
if ((ret = MPI_Comm_rank(even_comm, &even_rank)))
MPIERR(ret);
if ((ret = MPI_Comm_size(even_comm, &even_size)))
MPIERR(ret);
}
/* Create a group with tasks 0, 1, and 3. */
int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}};
if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES, overlap_ranges,
&overlap_group)))
ERR(ret);
/* Create a communicator from the overlap_group. */
if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm)))
ERR(ret);
/* Learn my rank and the total number of processors in overlap
* group. */
if (overlap_comm != MPI_COMM_NULL)
{
if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank)))
MPIERR(ret);
if ((ret = MPI_Comm_size(overlap_comm, &overlap_size)))
MPIERR(ret);
}
/* Initialize PIO system for even. */
if (even_comm != MPI_COMM_NULL)
{
if ((ret = PIOc_Init_Intracomm(even_comm, NUM_IO1, STRIDE1, BASE1, rearranger[r],
&even_iosysid)))
ERR(ret);
/* These should not work. */
if (PIOc_set_hint(even_iosysid + TEST_VAL_42, NULL, NULL) != PIO_EBADID)
ERR(ERR_WRONG);
if (PIOc_set_hint(even_iosysid, NULL, NULL) != PIO_EINVAL)
ERR(ERR_WRONG);
/* Set the hint (which will be ignored). */
if ((ret = PIOc_set_hint(even_iosysid, "hint", "hint_value")))
ERR(ret);
/* Set the error handler. */
/*PIOc_Set_IOSystem_Error_Handling(even_iosysid, PIO_BCAST_ERROR);*/
if ((ret = PIOc_set_iosystem_error_handling(even_iosysid, PIO_BCAST_ERROR, NULL)))
ERR(ret);
}
/* Initialize PIO system for overlap comm. */
if (overlap_comm != MPI_COMM_NULL)
{
if ((ret = PIOc_Init_Intracomm(overlap_comm, NUM_IO2, STRIDE1, BASE1, rearranger[r],
&overlap_iosysid)))
ERR(ret);
/* Set the error handler. */
PIOc_Set_IOSystem_Error_Handling(overlap_iosysid, PIO_BCAST_ERROR);
}
for (int i = 0; i < num_flavors; i++)
{
char fname0[PIO_MAX_NAME + 1];
char fname1[PIO_MAX_NAME + 1];
char fname2[PIO_MAX_NAME + 1];
sprintf(fname0, "%s_file_0_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]);
if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname0, ATTNAME,
DIMNAME, my_rank)))
ERR(ret);
sprintf(fname1, "%s_file_1_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]);
if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname1, ATTNAME,
DIMNAME, my_rank)))
ERR(ret);
sprintf(fname2, "%s_file_2_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[i], rearranger[r]);
if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname2, ATTNAME,
DIMNAME, my_rank)))
ERR(ret);
/* Now check the first file from WORLD communicator. */
int ncid;
if ((ret = open_and_check_file(test_comm, iosysid_world, flavor[i], &ncid, fname0,
ATTNAME, DIMNAME, 1, my_rank)))
ERR(ret);
/* Now have the even communicators check the files. */
int ncid2;
if (even_comm != MPI_COMM_NULL)
{
if ((ret = open_and_check_file(even_comm, even_iosysid, flavor[i], &ncid2,
fname2, ATTNAME, DIMNAME, 1, my_rank)))
ERR(ret);
if ((ret = check_file(even_comm, even_iosysid, flavor[i], ncid2, fname2,
ATTNAME, DIMNAME, my_rank)))
ERR(ret);
}
/* Now have the overlap communicators check the files. */
int ncid3;
if (overlap_comm != MPI_COMM_NULL)
{
if ((ret = open_and_check_file(overlap_comm, overlap_iosysid, flavor[i],
&ncid3, fname1, ATTNAME, DIMNAME, 1, my_rank)))
ERR(ret);
if ((ret = check_file(overlap_comm, overlap_iosysid, flavor[i], ncid3, fname1,
ATTNAME, DIMNAME, my_rank)))
ERR(ret);
}
/* Close the still-open files. */
if (even_comm != MPI_COMM_NULL)
if ((ret = PIOc_closefile(ncid2)))
ERR(ret);
if (overlap_comm != MPI_COMM_NULL)
if ((ret = PIOc_closefile(ncid3)))
ERR(ret);
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
} /* next iotype */
/* Finalize PIO systems. */
if (even_comm != MPI_COMM_NULL)
if ((ret = PIOc_finalize(even_iosysid)))
ERR(ret);
if (overlap_comm != MPI_COMM_NULL)
{
if ((ret = PIOc_finalize(overlap_iosysid)))
ERR(ret);
}
if ((ret = PIOc_finalize(iosysid_world)))
ERR(ret);
/* Free MPI resources used by test. */
if ((ret = MPI_Group_free(&overlap_group)))
ERR(ret);
if ((ret = MPI_Group_free(&even_group)))
ERR(ret);
if ((ret = MPI_Group_free(&world_group)))
ERR(ret);
if (overlap_comm != MPI_COMM_NULL)
if ((ret = MPI_Comm_free(&overlap_comm)))
ERR(ret);
if (even_comm != MPI_COMM_NULL)
if ((ret = MPI_Comm_free(&even_comm)))
ERR(ret);
} /* next rearranger */
} /* my_rank < TARGET_NTASKS */
/* Finalize test. */
if ((ret = pio_test_finalize(&test_comm)))
return ERR_AWFUL;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/* Run tests for darray functions. */
int main(int argc, char **argv)
{
int my_rank;
int ntasks;
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
MPI_Comm test_comm; /* A communicator for this test. */
int ret; /* Return code. */
/* Initialize test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
MIN_NTASKS, -1, &test_comm)))
ERR(ERR_INIT);
if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
return ret;
/* Only do something on max_ntasks tasks. */
if (my_rank < TARGET_NTASKS)
{
int iosysid; /* The ID for the parallel I/O system. */
int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */
int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */
int wioid, rioid;
int maplen = MAPLEN;
MPI_Offset wcompmap[MAPLEN];
MPI_Offset rcompmap[MAPLEN];
int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
/* Data we will write for each type. */
signed char byte_data[MAPLEN];
char char_data[MAPLEN];
short short_data[MAPLEN];
int int_data[MAPLEN];
float float_data[MAPLEN];
double double_data[MAPLEN];
#ifdef _NETCDF4
unsigned char ubyte_data[MAPLEN];
unsigned short ushort_data[MAPLEN];
unsigned int uint_data[MAPLEN];
long long int64_data[MAPLEN];
unsigned long long uint64_data[MAPLEN];
#endif /* _NETCDF4 */
/* Expected results for each type. */
signed char byte_expected[MAPLEN];
char char_expected[MAPLEN];
short short_expected[MAPLEN];
int int_expected[MAPLEN];
float float_expected[MAPLEN];
double double_expected[MAPLEN];
#ifdef _NETCDF4
unsigned char ubyte_expected[MAPLEN];
unsigned short ushort_expected[MAPLEN];
unsigned int uint_expected[MAPLEN];
long long int64_expected[MAPLEN];
unsigned long long uint64_expected[MAPLEN];
#endif /* _NETCDF4 */
/* Custom fill value for each type. */
signed char byte_fill = -2;
char char_fill = 2;
short short_fill = -2;
int int_fill = -2;
float float_fill = -2;
double double_fill = -2;
#ifdef _NETCDF4
unsigned char ubyte_fill = 2;
unsigned short ushort_fill = 2;
unsigned int uint_fill = 2;
long long int64_fill = 2;
unsigned long long uint64_fill = 2;
#endif /* _NETCDF4 */
/* Default fill value for each type. */
signed char byte_default_fill = NC_FILL_BYTE;
char char_default_fill = NC_FILL_CHAR;
short short_default_fill = NC_FILL_SHORT;
int int_default_fill = NC_FILL_INT;
float float_default_fill = NC_FILL_FLOAT;
double double_default_fill = NC_FILL_DOUBLE;
#ifdef _NETCDF4
unsigned char ubyte_default_fill = NC_FILL_UBYTE;
unsigned short ushort_default_fill = NC_FILL_USHORT;
unsigned int uint_default_fill = NC_FILL_UINT;
long long int64_default_fill = NC_FILL_INT64;
unsigned long long uint64_default_fill = NC_FILL_UINT64;
#endif /* _NETCDF4 */
int ret; /* Return code. */
/* Set up the compmaps. Don't forget these are 1-based
* numbers, like in Fortran! */
for (int i = 0; i < MAPLEN; i++)
{
wcompmap[i] = i % 2 ? my_rank * MAPLEN + i + 1 : 0; /* Even values missing. */
rcompmap[i] = my_rank * MAPLEN + i + 1;
}
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
/* Test for each rearranger. */
for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
{
/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO_PROCS, ioproc_stride, ioproc_start,
rearranger[r], &iosysid)))
return ret;
/* Test with and without custom fill values. */
for (int fv = 0; fv < NUM_TEST_CASES_FILLVALUE; fv++)
{
#ifndef _NETCDF4
#define NUM_TYPES 6
int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
#else
#define NUM_TYPES 11
int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
#endif /* _NETCDF4 */
/* Determine what data to write. Put value of 42 into
* array elements that will not get written. Due to
* the decomposition, these will be replaced by fill
* values. */
for (int i = 0; i < MAPLEN; i++)
{
byte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
char_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
short_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
int_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
float_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
double_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
#ifdef _NETCDF4
ubyte_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
ushort_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
uint_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
int64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
uint64_data[i] = i % 2 ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
#endif /* _NETCDF4 */
}
/* Determine what data to expect from the test. For
* even values of i, the fill value will be used, and
* it may be custom or default fill value. */
for (int i = 0; i < MAPLEN; i++)
{
byte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? byte_default_fill : byte_fill);
char_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? char_default_fill : char_fill);
short_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? short_default_fill : short_fill);
int_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int_default_fill : int_fill);
float_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? float_default_fill : float_fill);
double_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? double_default_fill : double_fill);
#ifdef _NETCDF4
ubyte_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ubyte_default_fill : ubyte_fill);
ushort_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? ushort_default_fill : ushort_fill);
uint_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint_default_fill : uint_fill);
int64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? int64_default_fill : int64_fill);
uint64_expected[i] = i % 2 ? my_rank * MAPLEN + i + 1 : (fv ? uint64_default_fill : uint64_fill);
#endif /* _NETCDF4 */
}
/* Test for each available type. */
for (int t = 0; t < NUM_TYPES; t++)
{
void *expected;
void *fill;
void *data;
int ncid, dimid, varid;
char filename[NC_MAX_NAME + 1];
switch (test_type[t])
{
case PIO_BYTE:
expected = byte_expected;
fill = fv ? &byte_default_fill : &byte_fill;
data = byte_data;
break;
case PIO_CHAR:
expected = char_expected;
fill = fv ? &char_default_fill : &char_fill;
data = char_data;
break;
case PIO_SHORT:
expected = short_expected;
fill = fv ? &short_default_fill : &short_fill;
data = short_data;
break;
case PIO_INT:
expected = int_expected;
fill = fv ? &int_default_fill : &int_fill;
data = int_data;
break;
case PIO_FLOAT:
expected = float_expected;
fill = fv ? &float_default_fill : &float_fill;
data = float_data;
break;
case PIO_DOUBLE:
expected = double_expected;
fill = fv ? &double_default_fill : &double_fill;
data = double_data;
break;
#ifdef _NETCDF4
case PIO_UBYTE:
expected = ubyte_expected;
fill = fv ? &ubyte_default_fill : &ubyte_fill;
data = ubyte_data;
break;
case PIO_USHORT:
expected = ushort_expected;
fill = fv ? &ushort_default_fill : &ushort_fill;
data = ushort_data;
break;
case PIO_UINT:
expected = uint_expected;
fill = fv ? &uint_default_fill : &uint_fill;
data = uint_data;
break;
case PIO_INT64:
expected = int64_expected;
fill = fv ? &int64_default_fill : &int64_fill;
data = int64_data;
break;
case PIO_UINT64:
expected = uint64_expected;
fill = fv ? &uint64_default_fill : &uint64_fill;
data = uint64_data;
break;
#endif /* _NETCDF4 */
default:
return ERR_AWFUL;
}
/* Initialize decompositions. */
if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap,
&wioid, &rearranger[r], NULL, NULL)))
return ret;
if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap,
&rioid, &rearranger[r], NULL, NULL)))
return ret;
/* Create the test file in each of the available iotypes. */
for (int fmt = 0; fmt < num_flavors; fmt++)
{
PIO_Offset type_size;
void *data_in;
/* Byte type doesn't work with pnetcdf. */
if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (test_type[t] == PIO_BYTE || test_type[t] == PIO_CHAR))
continue;
/* NetCDF-4 types only work with netCDF-4 formats. */
if (test_type[t] > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
flavor[fmt] != PIO_IOTYPE_NETCDF4P)
continue;
/* Put together filename. */
sprintf(filename, "%s_iotype_%d_rearr_%d_type_%d.nc", TEST_NAME, flavor[fmt],
rearranger[r], test_type[t]);
/* Create file. */
if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER)))
return ret;
/* Define metadata. */
if ((ret = PIOc_def_dim(ncid, DIM_NAME, dim_len[0], &dimid)))
return ret;
if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid)))
return ret;
if ((ret = PIOc_put_att(ncid, varid, FILL_VALUE_NAME, test_type[t],
1, fill)))
return ret;
if ((ret = PIOc_enddef(ncid)))
return ret;
/* Write some data. */
if ((ret = PIOc_write_darray(ncid, varid, wioid, MAPLEN, data, fill)))
return ret;
if ((ret = PIOc_sync(ncid)))
return ret;
/* What is size of type? */
if ((ret = PIOc_inq_type(ncid, test_type[t], NULL, &type_size)))
return ret;
/* Allocate space to read data into. */
if (!(data_in = malloc(type_size * MAPLEN)))
return PIO_ENOMEM;
/* Read the data. */
if ((ret = PIOc_read_darray(ncid, varid, rioid, MAPLEN, data_in)))
return ret;
/* Check results. */
if (memcmp(data_in, expected, type_size * MAPLEN))
return ERR_AWFUL;
/* Release storage. */
free(data_in);
/* Close file. */
if ((ret = PIOc_closefile(ncid)))
return ret;
} /* next iotype */
/* Free decompositions. */
if ((ret = PIOc_freedecomp(iosysid, wioid)))
return ret;
if ((ret = PIOc_freedecomp(iosysid, rioid)))
return ret;
} /* next type */
} /* next fill value test case */
} /* next rearranger */
/* Finalize PIO system. */
if ((ret = PIOc_finalize(iosysid)))
return ret;
} /* endif my_rank < TARGET_NTASKS */
/* Finalize the MPI library. */
if ((ret = pio_test_finalize(&test_comm)))
return ret;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
/* Run async tests. */
int main(int argc, char **argv)
{
int my_rank; /* Zero-based rank of processor. */
int ntasks; /* Number of processors involved in current execution. */
int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
int ret; /* Return code. */
MPI_Comm test_comm;
/* Num procs for computation. */
int num_procs2[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}, {3}};
/* Number of processors that will do IO. */
int num_io_procs[NUM_COMBOS] = {3, 2, 1};
/* Initialize test. */
if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS,
-1, &test_comm)))
ERR(ERR_INIT);
/* Test code runs on TARGET_NTASKS tasks. The left over tasks do
* nothing. */
if (my_rank < TARGET_NTASKS)
{
/* Figure out iotypes. */
if ((ret = get_iotypes(&num_flavors, flavor)))
ERR(ret);
for (int combo = 0; combo < NUM_COMBOS; combo++)
{
/* Is the current process a computation task? */
int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
/* Initialize the IO system. */
if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT,
num_procs2[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid)))
ERR(ERR_INIT);
/* All the netCDF calls are only executed on the computation
* tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
* and when the do, they should go straight to finalize. */
if (comp_task)
{
for (int flv = 0; flv < num_flavors; flv++)
{
char filename[NC_MAX_NAME + 1]; /* Test filename. */
int my_comp_idx = 0; /* Index in iosysid array. */
for (int sample = 0; sample < NUM_SAMPLES; sample++)
{
char iotype_name[NC_MAX_NAME + 1];
/* Create a filename. */
if ((ret = get_iotype_name(flavor[flv], iotype_name)))
return ret;
sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
/* Create sample file. */
if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
ERR(ret);
/* Check the file for correctness. */
if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
ERR(ret);
}
} /* next netcdf flavor */
/* Finalize the IO system. Only call this from the computation tasks. */
for (int c = 0; c < COMPONENT_COUNT; c++)
if ((ret = PIOc_finalize(iosysid[c])))
ERR(ret);
} /* endif comp_task */
/* Wait for everyone to catch up. */
MPI_Barrier(test_comm);
} /* next combo */
}/* my_rank < TARGET_NTASKS */
/* Finalize test. */
if ((ret = pio_test_finalize(&test_comm)))
return ERR_AWFUL;
printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
return 0;
}
