int
main(int argc, char **argv) {
int stat; /* return status */
int ncid; /* netCDF id */
int rec, i, j, k;
signed char x[] = {42, 21};
/* dimension ids */
int rec_dim;
int i_dim;
int j_dim;
int k_dim;
int n_dim;
#define NUMRECS 1
#define I_LEN 4104
#define J_LEN 1023
#define K_LEN 1023
#define N_LEN 2
/* dimension lengths */
MPI_Offset rec_len = NC_UNLIMITED;
MPI_Offset i_len = I_LEN;
MPI_Offset j_len = J_LEN;
MPI_Offset k_len = K_LEN;
MPI_Offset n_len = N_LEN;
/* variable ids */
int var1_id;
int x_id;
/* rank (number of dimensions) for each variable */
# define RANK_var1 4
# define RANK_x 2
/* variable shapes */
int var1_dims[RANK_var1];
int x_dims[RANK_x];
printf("\n*** Testing large files, slowly.\n");
printf("*** Creating large file %s...", FILE_NAME);
MPI_Init(&argc, &argv);
/* enter define mode */
stat = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER|NC_64BIT_OFFSET,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
/* define dimensions */
stat = ncmpi_def_dim(ncid, "rec", rec_len, &rec_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "i", i_len, &i_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "j", j_len, &j_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "k", k_len, &k_dim);
check_err(stat,__LINE__,__FILE__);
stat = ncmpi_def_dim(ncid, "n", n_len, &n_dim);
check_err(stat,__LINE__,__FILE__);
/* define variables */
var1_dims[0] = rec_dim;
var1_dims[1] = i_dim;
var1_dims[2] = j_dim;
var1_dims[3] = k_dim;
stat = ncmpi_def_var(ncid, "var1", NC_BYTE, RANK_var1, var1_dims, &var1_id);
check_err(stat,__LINE__,__FILE__);
x_dims[0] = rec_dim;
x_dims[1] = n_dim;
stat = ncmpi_def_var(ncid, "x", NC_BYTE, RANK_x, x_dims, &x_id);
check_err(stat,__LINE__,__FILE__);
/* don't initialize variables with fill values */
stat = ncmpi_set_fill(ncid, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
/* leave define mode */
stat = ncmpi_enddef (ncid);
check_err(stat,__LINE__,__FILE__);
{ /* store var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
var1[j][k] = n++;
}
}
var1_start[1] = i;
stat = ncmpi_put_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
}
}
stat = ncmpi_put_vara_schar_all(ncid, x_id, x_start, x_count, x);
check_err(stat,__LINE__,__FILE__);
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Reading large file %s...", FILE_NAME);
stat = ncmpi_open(MPI_COMM_WORLD, FILE_NAME, NC_NOWRITE,
MPI_INFO_NULL, &ncid);
check_err(stat,__LINE__,__FILE__);
{ /* read var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static MPI_Offset var1_start[RANK_var1] = {0, 0, 0, 0};
static MPI_Offset var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static MPI_Offset x_start[RANK_x] = {0, 0};
static MPI_Offset x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
var1_start[1] = i;
stat = ncmpi_get_vara_schar_all(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
if (var1[j][k] != (signed char) n) {
printf("Error on read, var1[%d, %d, %d, %d] = %d wrong, "
"should be %d !\n", rec, i, j, k, var1[j][k], (signed char) n);
return 1;
}
n++;
}
}
}
ncmpi_get_vara_schar_all(ncid, x_id, x_start, x_count, x);
if(x[0] != 42 || x[1] != 21) {
printf("Error on read, x[] = %d, %d\n", x[0], x[1]);
return 1;
}
}
}
stat = ncmpi_close(ncid);
check_err(stat,__LINE__,__FILE__);
printf("ok\n");
printf("*** Tests successful!\n");
/* Delete the file. */
(void) remove(FILE_NAME);
MPI_Finalize();
return 0;
}
/*
* Write or read access to file using the NCMPI interface.
*/
static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer,
IOR_offset_t length, IOR_param_t * param)
{
signed char *bufferPtr = (signed char *)buffer;
static int firstReadCheck = FALSE, startDataSet;
int var_id, dim_id[NUM_DIMS];
MPI_Offset bufSize[NUM_DIMS], offset[NUM_DIMS];
IOR_offset_t segmentPosition;
int segmentNum, transferNum;
/* Wei-keng Liao: In ior.c line 1979 says "block size must be a multiple
of transfer size." Hence, length should always == param->transferSize
below. I leave it here to double check.
*/
if (length != param->transferSize) {
char errMsg[256];
sprintf(errMsg, "length(%lld) != param->transferSize(%lld)\n",
length, param->transferSize);
NCMPI_CHECK(-1, errMsg);
}
/* determine by offset if need to start data set */
if (param->filePerProc == TRUE) {
segmentPosition = (IOR_offset_t) 0;
} else {
segmentPosition =
(IOR_offset_t) ((rank + rankOffset) % param->numTasks)
* param->blockSize;
}
if ((int)(param->offset - segmentPosition) == 0) {
startDataSet = TRUE;
/*
* this toggle is for the read check operation, which passes through
* this function twice; note that this function will open a data set
* only on the first read check and close only on the second
*/
if (access == READCHECK) {
if (firstReadCheck == TRUE) {
firstReadCheck = FALSE;
} else {
firstReadCheck = TRUE;
}
}
}
if (startDataSet == TRUE &&
(access != READCHECK || firstReadCheck == TRUE)) {
if (access == WRITE) {
int numTransfers =
param->blockSize / param->transferSize;
/* Wei-keng Liao: change 1D array to 3D array of dimensions:
[segmentCount*numTasksWorld][numTransfers][transferSize]
Requirement: none of these dimensions should be > 4G,
*/
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "segments_times_np",
NC_UNLIMITED, &dim_id[0]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "number_of_transfers",
numTransfers, &dim_id[1]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim
(*(int *)fd, "transfer_size",
param->transferSize, &dim_id[2]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_var
(*(int *)fd, "data_var", NC_BYTE, NUM_DIMS,
dim_id, &var_id),
"cannot define data set variables");
NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
"cannot close data set define mode");
} else {
NCMPI_CHECK(ncmpi_inq_varid
(*(int *)fd, "data_var", &var_id),
"cannot retrieve data set variable");
}
if (param->collective == FALSE) {
NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
"cannot enable independent data mode");
}
param->var_id = var_id;
startDataSet = FALSE;
}
var_id = param->var_id;
/* Wei-keng Liao: calculate the segment number */
segmentNum = param->offset / (param->numTasks * param->blockSize);
/* Wei-keng Liao: calculate the transfer number in each block */
transferNum = param->offset % param->blockSize / param->transferSize;
/* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of
amount param->transferSize */
bufSize[0] = 1;
bufSize[1] = 1;
bufSize[2] = param->transferSize;
offset[0] = segmentNum * numTasksWorld + rank;
offset[1] = transferNum;
offset[2] = 0;
/* access the file */
if (access == WRITE) { /* WRITE */
if (param->collective) {
NCMPI_CHECK(ncmpi_put_vara_schar_all
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot write to data set");
} else {
NCMPI_CHECK(ncmpi_put_vara_schar
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot write to data set");
}
} else { /* READ or CHECK */
if (param->collective == TRUE) {
NCMPI_CHECK(ncmpi_get_vara_schar_all
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot read from data set");
} else {
NCMPI_CHECK(ncmpi_get_vara_schar
(*(int *)fd, var_id, offset, bufSize,
bufferPtr),
"cannot read from data set");
}
}
return (length);
}
