int main(int argc, char **argv) {
int i, j, rank, nprocs, ret;
int ncfile, ndims, nvars, ngatts, unlimited;
int var_ndims, var_natts;;
MPI_Offset *dim_sizes, var_size;
MPI_Offset *start, *count;
int *requests, *statuses;
char varname[NC_MAX_NAME+1];
int dimids[NC_MAX_VAR_DIMS];
nc_type type;
int **data;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc != 2) {
if (rank == 0) printf("Usage: %s filename\n", argv[0]);
MPI_Finalize();
exit(-1);
}
ret = ncmpi_open(MPI_COMM_WORLD, argv[1], NC_NOWRITE, MPI_INFO_NULL,
&ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* reader knows nothing about dataset, but we can interrogate with query
* routines: ncmpi_inq tells us how many of each kind of "thing"
* (dimension, variable, attribute) we will find in the file */
/* no commnunication needed after ncmpi_open: all processors have a cached
* veiw of the metadata once ncmpi_open returns */
ret = ncmpi_inq(ncfile, &ndims, &nvars, &ngatts, &unlimited);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* we do not really need the name of the dimension or the variable for
* reading in this example. we could, in a different example, take the
* name of a variable on the command line and read just that one */
dim_sizes = calloc(ndims, sizeof(MPI_Offset));
/* netcdf dimension identifiers are allocated sequentially starting
* at zero; same for variable identifiers */
for(i=0; i<ndims; i++) {
ret = ncmpi_inq_dimlen(ncfile, i, &(dim_sizes[i]) );
if (ret != NC_NOERR) handle_error(ret, __LINE__);
}
requests = calloc(nvars, sizeof(int));
statuses = calloc(nvars, sizeof(int));
data = malloc(nvars*sizeof(int*));
for(i=0; i<nvars; i++) {
/* much less coordination in this case compared to rank 0 doing all
* the i/o: everyone already has the necessary information */
ret = ncmpi_inq_var(ncfile, i, varname, &type, &var_ndims, dimids,
&var_natts);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
start = calloc(var_ndims, sizeof(MPI_Offset));
count = calloc(var_ndims, sizeof(MPI_Offset));
/* we will simply decompose along one dimension. Generally the
* application has some algorithim for domain decomposistion. Note
* that data decomposistion can have an impact on i/o performance.
* Often it's best just to do what is natural for the application,
* but something to consider if performance is not what was
* expected/desired */
start[0] = (dim_sizes[dimids[0]]/nprocs)*rank;
count[0] = (dim_sizes[dimids[0]]/nprocs);
var_size = count[0];
for (j=1; j<var_ndims; j++) {
start[j] = 0;
count[j] = dim_sizes[dimids[j]];
var_size *= count[j];
}
switch(type) {
case NC_INT:
data[i] = calloc(var_size, sizeof(int));
/* as with the writes, this call is independent: we
* will do any coordination (if desired) in a
* subsequent ncmpi_wait_all() call */
ret = ncmpi_iget_vara(ncfile, i, start, count, data[i],
var_size, MPI_INT, &(requests[i]));
if (ret != NC_NOERR) handle_error(ret, __LINE__);
break;
default:
/* we can do this for all the known netcdf types but this
* example is already getting too long */
fprintf(stderr, "unsupported NetCDF type \n");
}
free(start);
free(count);
}
ret = ncmpi_wait_all(ncfile, nvars, requests, statuses);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
/* now that the ncmpi_wait_all has returned, the caller can do stuff with
* the buffers passed in to the non-blocking operations. The buffer resue
* rules are similar to MPI non-blocking messages */
for (i=0; i<nvars; i++) {
if (data[i] != NULL) free(data[i]);
}
free(data);
free(dim_sizes);
free(requests);
free(statuses);
ret = ncmpi_close(ncfile);
if (ret != NC_NOERR) handle_error(ret, __LINE__);
MPI_Finalize();
return 0;
}
int main(int argc, char** argv) {
int i;
double power_M;
int *array_of_sizes, *array_of_subsizes, *array_of_starts;
int ncid, *dimids, varid_1, varid_2;
MPI_Offset *local_starts, *local_edges;
char dimname[20];
nc_type nc_etype;
MPI_Datatype mpi_etype, mpi_subarray;
TEST_NATIVE_ETYPE *buf1, *buf2, *tmpbuf;
void *packbuf;
int packsz;
int packpos;
int total_sz, local_sz;
int nprocs, rank;
int status;
int success, successall;
int malloc_failure, malloc_failure_any;
int request;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* test initializations: nc_file, nc_variables, dimensions, arrays */
parse_args(argc, argv, rank);
TEST_SET_NCMPI_ETYPE(nc_etype, mpi_etype);
#ifdef TEST_NCTYPE
nc_etype = TEST_NCTYPE;
#endif
if (rank == 0) {
printf("testing memory subarray layout ...\n");
}
status = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER,
MPI_INFO_NULL, &ncid);
TEST_HANDLE_ERR(status);
array_of_sizes = (int *)
malloc(sizeof(int)*ndims*4 + sizeof(MPI_Offset)*ndims*4);
array_of_subsizes = array_of_sizes + ndims;
array_of_starts = array_of_subsizes + ndims;
dimids = array_of_starts + ndims;
local_starts = (MPI_Offset *)(dimids + ndims);
local_edges = local_starts + ndims;
total_sz = 1;
power_M = 1;
for (i=0; i<ndims; i++, power_M*=test_m) {
array_of_sizes[i] = (int)(test_n*power_M);
if (array_of_sizes[i] < 1) {
/* lower bound check */
array_of_sizes[i] = 1;
} else if ( (double)total_sz*array_of_sizes[i] > (double)TEST_MAX_INT ){
/* upper bound check */
if (rank == 0) {
fprintf(stderr, "Total size of array is too big to be represented\n");
fprintf(stderr, "Current size = %f, Max size = %d\n",
(double)total_sz*array_of_sizes[i], TEST_MAX_INT);
}
TEST_EXIT(-1);
}
total_sz *= array_of_sizes[i];
sprintf(dimname, "dim_%d", i);
status = ncmpi_def_dim(ncid, dimname,
(MPI_Offset)array_of_sizes[i], dimids+i);
TEST_HANDLE_ERR(status);
}
if (order == MPI_ORDER_FORTRAN) {
/* reverse the filearray dimension, since NC always use C ORDER */
TEST_REVERSE(dimids, ndims, int);
}
status = ncmpi_def_var(ncid, "var_1", nc_etype, ndims, dimids, &varid_1);
TEST_HANDLE_ERR(status);
TEST_REVERSE(dimids, ndims, int);
status = ncmpi_def_var(ncid, "var_2", nc_etype, ndims, dimids, &varid_2);
TEST_HANDLE_ERR(status);
status = ncmpi_enddef(ncid);
TEST_HANDLE_ERR(status);
if (rank == 0) {
printf("\t Filesize = %2.3fMB, MAX_Memory_needed = %2.3fMB\n\n",
2*total_sz*TEST_NCTYPE_LEN(nc_etype)/1024.0/1024.0,
( (2*total_sz + 4*total_sz/nprocs)*sizeof(TEST_NATIVE_ETYPE)
+ total_sz*TEST_NCTYPE_LEN(nc_etype) )/1024.0/1024.0);
}
buf1 = (TEST_NATIVE_ETYPE *)malloc(total_sz*sizeof(TEST_NATIVE_ETYPE)*2);
malloc_failure = (buf1 == NULL ||
(float)total_sz*sizeof(TEST_NATIVE_ETYPE)*2 > TEST_MAX_INT);
MPI_Allreduce(&malloc_failure, &malloc_failure_any, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if (malloc_failure_any) {
if (rank == 0) {
fprintf(stderr, "malloc(%2.3fMB) failed!\n",
(float)total_sz*sizeof(TEST_NATIVE_ETYPE)*2/1024/1024);
fprintf(stderr, "The whole array may be too big for malloc to handle!\n");
fprintf(stderr, "Please choose smaller array size.\n");
}
TEST_EXIT(-1);
}
buf2 = buf1 + total_sz;
for (i=0; i<total_sz; i++)
/* just make sure any type can represent the number */
/* and make it irregular (cycle != power of 2) for random test */
buf1[i] = buf2[i] = (TEST_NATIVE_ETYPE)(i%127);
/* PARTITION and calculate the local target region */
partition_array(ndims,
array_of_sizes, array_of_subsizes, array_of_starts,
nprocs, rank);
local_sz = 1;
for (i=0; i<ndims; i++) {
local_sz *= array_of_subsizes[i];
local_edges[i] = (MPI_Offset)array_of_subsizes[i];
local_starts[i] = (MPI_Offset)array_of_starts[i];
}
if (order == MPI_ORDER_FORTRAN) {
/* reverse the filearray dimension, since NC always use C ORDER */
TEST_REVERSE(local_edges, ndims, MPI_Offset);
TEST_REVERSE(local_starts, ndims, MPI_Offset);
}
/* CREATE local subarray memory view */
if (local_sz == 0)
MPI_Type_contiguous(0, mpi_etype, &mpi_subarray);
else
MPI_Type_create_subarray(ndims,
array_of_sizes,
array_of_subsizes,
array_of_starts,
order,
mpi_etype,
&mpi_subarray);
MPI_Type_commit(&mpi_subarray);
/* PRINT stats */
if (rank == 0) {
printf("Initialization: NDIMS = %d, NATIVE_ETYPE = %s, NC_TYPE = %s\n\n",
ndims, TEST_NATIVE_ETYPE_STR, TEST_GET_NCTYPE_STR(nc_etype));
printf("\t NC Var_1 Shape:\t [");
if (order == MPI_ORDER_C) {
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
} else {
TEST_PRINT_LIST(array_of_sizes, ndims-1, 0, -1);
}
printf("] Always ORDER_C\n");
printf("\t NC Var_2 Shape:\t [");
if (order == MPI_ORDER_C) {
TEST_PRINT_LIST(array_of_sizes, ndims-1, 0, -1);
} else {
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
}
printf("] Always ORDER_C\n");
printf("\t Memory Array Shape:\t [");
TEST_PRINT_LIST(array_of_sizes, 0, ndims-1, 1);
printf("] %s\n", ((order==MPI_ORDER_C)?"MPI_ORDER_C":"MPI_ORDER_FORTRAN"));
printf("\t Memory Array Copys: buf1 for write, buf2 for read back (and compare)\n");
printf("\n");
printf("Logical Array Partition:\t BLOCK partition along all dimensions\n\n");
printf("Access Pattern (subarray): NPROCS = %d\n\n", nprocs);
}
fflush(stdout);
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<nprocs; i++) {
if (rank == i) {
printf("\t Proc %2d of %2d: starts = [", rank, nprocs);
TEST_PRINT_LIST(local_starts, 0, ndims-1, 1);
printf("], ");
printf("counts = [");
TEST_PRINT_LIST(local_edges, 0, ndims-1, 1);
printf("] \n");
}
fflush(stdout);
/* Synchronizer: processes should print out their stuffs in turn :) */
MPI_Barrier(MPI_COMM_WORLD);
}
if (rank == 0) {
printf("\n");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
/* RESET the target region of buf2 */
MPI_Pack_size(local_sz, mpi_etype, MPI_COMM_SELF, &packsz);
tmpbuf = (TEST_NATIVE_ETYPE *)
malloc(local_sz*sizeof(TEST_NATIVE_ETYPE) + packsz);
packbuf = (void *)(tmpbuf + local_sz);
for (i=0; i<local_sz; i++)
tmpbuf[i] = (TEST_NATIVE_ETYPE)(-1);
packpos = 0;
MPI_Pack((void *)tmpbuf, local_sz, mpi_etype,
packbuf, packsz, &packpos, MPI_COMM_SELF);
packpos = 0;
MPI_Unpack(packbuf, packsz, &packpos, buf2, 1, mpi_subarray, MPI_COMM_SELF);
/* Begin of TEST1: test local write-n-readback */
fflush(stdout);
if (rank == 0) {
printf("TEST1: \n");
}
/* WRITE target region from buf1 */
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t [nonblocking] all procs writing their subarrays into Var_1 ...\n");
}
status = ncmpi_begin_indep_data(ncid);
TEST_HANDLE_ERR(status);
status = ncmpi_iput_vara(ncid, varid_1, local_starts, local_edges,
(const void *)buf1, 1, mpi_subarray, &request);
TEST_HANDLE_ERR(status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O returns ...\n");
fflush(stdout);
}
ncmpi_wait(ncid, 1, &request, &status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O finishes ...\n");
}
status = ncmpi_end_indep_data(ncid);
TEST_HANDLE_ERR(status);
/* READ target region back into buf2 */
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t [nonblocking] all procs reading their subarrays from Var_1 ...\n");
}
status = ncmpi_begin_indep_data(ncid);
TEST_HANDLE_ERR(status);
status = ncmpi_iget_vara(ncid, varid_1, local_starts, local_edges,
(void *)buf2, 1, mpi_subarray, &request);
TEST_HANDLE_ERR(status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O returns ...\n");
fflush(stdout);
}
ncmpi_wait(ncid, 1, &request, &status);
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("\t nonblocking I/O finishes ...\n");
}
status = ncmpi_end_indep_data(ncid);
TEST_HANDLE_ERR(status);
/* COMPARE buf1 and buf2 for equality */
if (memcmp((void *)buf1, (void *)buf2, total_sz*sizeof(TEST_NATIVE_ETYPE)))
success = 0;
else
success = 1;
MPI_Allreduce(&success, &successall, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD);
if (rank == 0) {
if (successall)
printf("\t PASS: memory subarray layout passes test1! \n\n");
else
printf("\t ERROR: memory subarray layout fails test1! \n\n");
}
/* End of TEST1 */
/* test finalization */
ncmpi_close(ncid);
MPI_Type_free(&mpi_subarray);
free(tmpbuf);
free(buf1);
free(array_of_sizes);
MPI_Finalize();
return !successall;
}
