int
main(int argc, char **argv) {
int stat; /* return status */
int ncid; /* netCDF id */
int rec, i, j, k;
signed char x[] = {42, 21};
/* dimension ids */
int rec_dim;
int i_dim;
int j_dim;
int k_dim;
int n_dim;
#define NUMRECS 1
#define I_LEN 4104
#define J_LEN 1023
#define K_LEN 1023
#define N_LEN 2
/* dimension lengths */
MPI_Offset rec_len = NC_UNLIMITED;
MPI_Offset i_len = I_LEN;
MPI_Offset j_len = J_LEN;
MPI_Offset k_len = K_LEN;
MPI_Offset n_len = N_LEN;
/* variable ids */
int var1_id;
int x_id;
/* rank (number of dimensions) for each variable */
# define RANK_var1 4
# define RANK_x 2
/* variable shapes */
int var1_dims[RANK_var1];
int x_dims[RANK_x];
printf("\n*** Testing large files, slowly.\n");
printf("*** Creating large file %s...", FILE_NAME);
MPI_Init(&argc, &argv);
/* enter define mode */
stat = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER|NC_64BIT_OFFSET,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
/* define dimensions */
stat = ncmpi_def_dim(ncid, "rec", rec_len, &rec_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "i", i_len, &i_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "j", j_len, &j_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "k", k_len, &k_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "n", n_len, &n_dim);
check_err(stat,__LINE__,__FILE__);
/* define variables */
var1_dims[0] = rec_dim;
var1_dims[1] = i_dim;
var1_dims[2] = j_dim;
var1_dims[3] = k_dim;
stat = ncmpi_def_var(ncid, "var1", NC_BYTE, RANK_var1, var1_dims, &var1_id);
check_err(stat,__LINE__,__FILE__);
x_dims[0] = rec_dim;
x_dims[1] = n_dim;
stat = ncmpi_def_var(ncid, "x", NC_BYTE, RANK_x, x_dims, &x_id);
check_err(stat,__LINE__,__FILE__);
/* don't initialize variables with fill values */
stat = ncmpi_set_fill(ncid, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
/* leave define mode */
stat = ncmpi_enddef (ncid);
check_err(stat,__LINE__,__FILE__);
{ /* store var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
var1[j][k] = n++;
}
}
var1_start[1] = i;
stat = ncmpi_put_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
}
}
stat = ncmpi_put_vara_schar_all(ncid, x_id, x_start, x_count, x);
check_err(stat,__LINE__,__FILE__);
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Reading large file %s...", FILE_NAME);
stat = ncmpi_open(MPI_COMM_WORLD, FILE_NAME, NC_NOWRITE,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
{ /* read var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
var1_start[1] = i;
stat = ncmpi_get_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
if (var1[j][k] != (signed char) n) {
printf("Error on read, var1[%d, %d, %d, %d] = %d wrong, "
"should be %d !\n", rec, i, j, k, var1[j][k], (signed char) n);
return 1;
}
n++;
}
}
}
ncmpi_get_vara_schar_all(ncid, x_id, x_start, x_count, x);
if(x[0] != 42 || x[1] != 21) {
printf("Error on read, x[] = %d, %d\n", x[0], x[1]);
return 1;
}
}
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Tests successful!\n");
/* Delete the file. */
(void) remove(FILE_NAME);
MPI_Finalize();
return 0;
}
/* write out variable's data from in-memory structure */
void
load_netcdf(void *rec_start)
{
int i, idim;
int stat = NC_NOERR;
MPI_Offset *start, *count;
char *charvalp = NULL;
short *shortvalp = NULL;
int *intvalp = NULL;
float *floatvalp = NULL;
double *doublevalp = NULL;
unsigned char *ubytevalp = NULL;
unsigned short *ushortvalp = NULL;
unsigned int *uintvalp = NULL;
long long *int64valp = NULL;
unsigned long long *uint64valp = NULL;
MPI_Offset total_size;
/* load values into variable */
switch (vars[varnum].type) {
case NC_CHAR:
case NC_BYTE:
charvalp = (char *) rec_start;
break;
case NC_SHORT:
shortvalp = (short *) rec_start;
break;
case NC_INT:
intvalp = (int *) rec_start;
break;
case NC_FLOAT:
floatvalp = (float *) rec_start;
break;
case NC_DOUBLE:
doublevalp = (double *) rec_start;
break;
case NC_UBYTE:
ubytevalp = (unsigned char *) rec_start;
break;
case NC_USHORT:
ushortvalp = (unsigned short *) rec_start;
break;
case NC_UINT:
uintvalp = (unsigned int *) rec_start;
break;
case NC_INT64:
int64valp = (long long *) rec_start;
break;
case NC_UINT64:
uint64valp = (unsigned long long *) rec_start;
break;
default:
derror("Unhandled type %d\n", vars[varnum].type);
break;
}
start = (MPI_Offset*) malloc(vars[varnum].ndims * 2 * sizeof(MPI_Offset));
count = start + vars[varnum].ndims;
if (vars[varnum].ndims > 0) {
/* initialize start to upper left corner (0,0,0,...) */
start[0] = 0;
if (vars[varnum].dims[0] == rec_dim) {
count[0] = vars[varnum].nrecs;
}
else {
count[0] = dims[vars[varnum].dims[0]].size;
}
}
for (idim = 1; idim < vars[varnum].ndims; idim++) {
start[idim] = 0;
count[idim] = dims[vars[varnum].dims[idim]].size;
}
total_size = nctypesize(vars[varnum].type);
for (idim=0; idim<vars[varnum].ndims; idim++)
total_size *= count[idim];
/* If the total put size is more than 2GB, then put one subarray at a time.
* Here the subarray is from 1, 2, ... ndims, except 0.
* This is not a perfect solution. To be improved.
*/
if (total_size > INT_MAX) {
MPI_Offset nchunks=count[0];
MPI_Offset subarray_nelems=1;
for (idim=1; idim<vars[varnum].ndims; idim++)
subarray_nelems *= count[idim];
count[0] = 1;
switch (vars[varnum].type) {
case NC_BYTE:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_schar_all(ncid, varnum, start, count, (signed char *)charvalp);
check_err(stat, "ncmpi_put_vara_schar_all", __func__, __LINE__, __FILE__);
charvalp += subarray_nelems;
}
break;
case NC_CHAR:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_text_all(ncid, varnum, start, count, charvalp);
check_err(stat, "ncmpi_put_vara_text_all", __func__, __LINE__, __FILE__);
charvalp += subarray_nelems;
}
break;
case NC_SHORT:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_short_all(ncid, varnum, start, count, shortvalp);
check_err(stat, "ncmpi_put_vara_short_all", __func__, __LINE__, __FILE__);
shortvalp += subarray_nelems;
}
break;
case NC_INT:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_int_all(ncid, varnum, start, count, intvalp);
check_err(stat, "ncmpi_put_vara_int_all", __func__, __LINE__, __FILE__);
intvalp += subarray_nelems;
}
break;
case NC_FLOAT:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_float_all(ncid, varnum, start, count, floatvalp);
check_err(stat, "ncmpi_put_vara_float_all", __func__, __LINE__, __FILE__);
floatvalp += subarray_nelems;
}
break;
case NC_DOUBLE:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_double_all(ncid, varnum, start, count, doublevalp);
check_err(stat, "ncmpi_put_vara_double_all", __func__, __LINE__, __FILE__);
doublevalp += subarray_nelems;
}
break;
case NC_UBYTE:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_uchar_all(ncid, varnum, start, count, ubytevalp);
check_err(stat, "ncmpi_put_vara_uchar_all", __func__, __LINE__, __FILE__);
ubytevalp += subarray_nelems;
}
break;
case NC_USHORT:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_ushort_all(ncid, varnum, start, count, ushortvalp);
check_err(stat, "ncmpi_put_vara_ushort_all", __func__, __LINE__, __FILE__);
ushortvalp += subarray_nelems;
}
break;
case NC_UINT:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_uint_all(ncid, varnum, start, count, uintvalp);
check_err(stat, "ncmpi_put_vara_uint_all", __func__, __LINE__, __FILE__);
uintvalp += subarray_nelems;
}
break;
case NC_INT64:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_longlong_all(ncid, varnum, start, count, int64valp);
check_err(stat, "ncmpi_put_vara_longlong_all", __func__, __LINE__, __FILE__);
int64valp += subarray_nelems;
}
break;
case NC_UINT64:
for (i=0; i<nchunks; i++) {
start[0] = i;
stat = ncmpi_put_vara_ulonglong_all(ncid, varnum, start, count, uint64valp);
check_err(stat, "ncmpi_put_vara_ulonglong_all", __func__, __LINE__, __FILE__);
uint64valp += subarray_nelems;
}
break;
default:
derror("Unhandled type %d\n", vars[varnum].type);
break;
}
}
else {
switch (vars[varnum].type) {
case NC_BYTE:
stat = ncmpi_put_vara_schar_all(ncid, varnum, start, count, (signed char *)charvalp);
check_err(stat, "ncmpi_put_vara_schar_all", __func__, __LINE__, __FILE__);
break;
case NC_CHAR:
stat = ncmpi_put_vara_text_all(ncid, varnum, start, count, charvalp);
check_err(stat, "ncmpi_put_vara_text_all", __func__, __LINE__, __FILE__);
break;
case NC_SHORT:
stat = ncmpi_put_vara_short_all(ncid, varnum, start, count, shortvalp);
check_err(stat, "ncmpi_put_vara_short_all", __func__, __LINE__, __FILE__);
break;
case NC_INT:
stat = ncmpi_put_vara_int_all(ncid, varnum, start, count, intvalp);
check_err(stat, "ncmpi_put_vara_int_all", __func__, __LINE__, __FILE__);
break;
case NC_FLOAT:
stat = ncmpi_put_vara_float_all(ncid, varnum, start, count, floatvalp);
check_err(stat, "ncmpi_put_vara_float_all", __func__, __LINE__, __FILE__);
break;
case NC_DOUBLE:
stat = ncmpi_put_vara_double_all(ncid, varnum, start, count, doublevalp);
check_err(stat, "ncmpi_put_vara_double_all", __func__, __LINE__, __FILE__);
break;
case NC_UBYTE:
stat = ncmpi_put_vara_uchar_all(ncid, varnum, start, count, ubytevalp);
check_err(stat, "ncmpi_put_vara_uchar_all", __func__, __LINE__, __FILE__);
break;
case NC_USHORT:
stat = ncmpi_put_vara_ushort_all(ncid, varnum, start, count, ushortvalp);
check_err(stat, "ncmpi_put_vara_ushort_all", __func__, __LINE__, __FILE__);
break;
case NC_UINT:
stat = ncmpi_put_vara_uint_all(ncid, varnum, start, count, uintvalp);
check_err(stat, "ncmpi_put_vara_uint_all", __func__, __LINE__, __FILE__);
break;
case NC_INT64:
stat = ncmpi_put_vara_longlong_all(ncid, varnum, start, count, int64valp);
check_err(stat, "ncmpi_put_vara_longlong_all", __func__, __LINE__, __FILE__);
break;
case NC_UINT64:
stat = ncmpi_put_vara_ulonglong_all(ncid, varnum, start, count, uint64valp);
check_err(stat, "ncmpi_put_vara_ulonglong_all", __func__, __LINE__, __FILE__);
break;
default:
derror("Unhandled type %d\n", vars[varnum].type);
break;
}
}
free(start);
}
/*
* Write or read access to file using the NCMPI interface.
*/
static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer,
IOR_offset_t length, IOR_param_t * param)
{
signed char *bufferPtr = (signed char *)buffer;
static int firstReadCheck = FALSE, startDataSet;
int var_id, dim_id[NUM_DIMS];
MPI_Offset bufSize[NUM_DIMS], offset[NUM_DIMS];
IOR_offset_t segmentPosition;
int segmentNum, transferNum;
/* Wei-keng Liao: In ior.c line 1979 says "block size must be a multiple
of transfer size." Hence, length should always == param->transferSize
below. I leave it here to double check.
*/
if (length != param->transferSize) {
char errMsg[256];
sprintf(errMsg, "length(%lld) != param->transferSize(%lld)\n",
length, param->transferSize);
NCMPI_CHECK(-1, errMsg);
}
/* determine by offset if need to start data set */
if (param->filePerProc == TRUE) {
segmentPosition = (IOR_offset_t) 0;
} else {
segmentPosition =
(IOR_offset_t) ((rank + rankOffset) % param->numTasks)
* param->blockSize;
}
if ((int)(param->offset - segmentPosition) == 0) {
startDataSet = TRUE;
/*
* this toggle is for the read check operation, which passes through
* this function twice; note that this function will open a data set
* only on the first read check and close only on the second
*/
if (access == READCHECK) {
if (firstReadCheck == TRUE) {
firstReadCheck = FALSE;
} else {
firstReadCheck = TRUE;
}
}
}
if (startDataSet == TRUE &&
(access != READCHECK || firstReadCheck == TRUE)) {
if (access == WRITE) {
int numTransfers =
param->blockSize / param->transferSize;
/* Wei-keng Liao: change 1D array to 3D array of dimensions:
[segmentCount*numTasksWorld][numTransfers][transferSize]
Requirement: none of these dimensions should be > 4G,
*/
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "segments_times_np",
NC_UNLIMITED, &dim_id[0]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "number_of_transfers",
numTransfers, &dim_id[1]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "transfer_size",
param->transferSize, &dim_id[2]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_var
(*(int *)fd, "data_var", NC_BYTE, NUM_DIMS,
dim_id, &var_id),
"cannot define data set variables");
NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
"cannot close data set define mode");
} else {
NCMPI_CHECK(ncmpi_inq_varid
(*(int *)fd, "data_var", &var_id),
"cannot retrieve data set variable");
}
if (param->collective == FALSE) {
NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
"cannot enable independent data mode");
}
param->var_id = var_id;
startDataSet = FALSE;
}
var_id = param->var_id;
/* Wei-keng Liao: calculate the segment number */
segmentNum = param->offset / (param->numTasks * param->blockSize);
/* Wei-keng Liao: calculate the transfer number in each block */
transferNum = param->offset % param->blockSize / param->transferSize;
/* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of
amount param->transferSize */
bufSize[0] = 1;
bufSize[1] = 1;
bufSize[2] = param->transferSize;
offset[0] = segmentNum * numTasksWorld + rank;
offset[1] = transferNum;
offset[2] = 0;
/* access the file */
if (access == WRITE) { /* WRITE */
if (param->collective) {
NCMPI_CHECK(ncmpi_put_vara_schar_all
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot write to data set");
} else {
NCMPI_CHECK(ncmpi_put_vara_schar
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot write to data set");
}
} else { /* READ or CHECK */
if (param->collective == TRUE) {
NCMPI_CHECK(ncmpi_get_vara_schar_all
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot read from data set");
} else {
NCMPI_CHECK(ncmpi_get_vara_schar
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot read from data set");
}
}
return (length);
}
