void BIL_Pio_read_nc_blocks(MPI_Comm all_readers_comm, MPI_Comm io_comm,
int num_blocks, BIL_Block* blocks) {
int i;
for (i = 0; i < num_blocks; i++) {
int fp;
BIL_Timing_fopen_start(all_readers_comm);
assert(ncmpi_open(io_comm, blocks[i].file_name, NC_NOWRITE,
BIL->io_hints, &fp) == NC_NOERR);
BIL_Timing_fopen_stop(all_readers_comm);
ncmpi_begin_indep_data(fp);
// Find the id, type, and size of the variable.
int var_id;
assert(ncmpi_inq_varid(fp, blocks[i].var_name, &var_id) == NC_NOERR);
nc_type var_type;
assert(ncmpi_inq_vartype(fp, var_id, &var_type) == NC_NOERR);
// Create extra variables specifically for the netCDF API.
MPI_Offset nc_dim_starts[BIL_MAX_NUM_DIMS];
MPI_Offset nc_dim_sizes[BIL_MAX_NUM_DIMS];
int j;
for (j = 0; j < blocks[i].num_dims; j++) {
nc_dim_starts[j] = blocks[i].starts[j];
nc_dim_sizes[j] = blocks[i].sizes[j];
}
MPI_Datatype nc_var_type;
BIL_Pio_nc_to_mpi_type(var_type, &nc_var_type, &(blocks[i].var_size));
// Allocate room for data and read it independently.
blocks[i].data = BIL_Misc_malloc(blocks[i].total_size * blocks[i].var_size);
BIL_Timing_io_start(all_readers_comm);
assert(ncmpi_get_vara(fp, var_id, nc_dim_starts, nc_dim_sizes,
blocks[i].data, blocks[i].total_size,
nc_var_type) == NC_NOERR);
BIL_Timing_io_stop(all_readers_comm,
blocks[i].total_size * blocks[i].var_size);
// Clean up.
ncmpi_end_indep_data(fp);
ncmpi_close(fp);
}
}
FORTRAN_API int FORT_CALL nfmpi_get_vara_ ( int *v1, int *v2, MPI_Offset v3[], MPI_Offset v4[], void*v5, MPI_Offset *v6, MPI_Fint *v7 ){
int ierr;
int l2 = *v2 - 1;
MPI_Offset *l3 = 0;
MPI_Offset *l4 = 0;
{ int ln = ncmpixVardim(*v1,*v2-1);
if (ln > 0) {
int li;
l3 = (MPI_Offset *)malloc( ln * sizeof(MPI_Offset) );
for (li=0; li<ln; li++)
l3[li] = v3[ln-1-li] - 1;
}
else if (ln < 0) {
/* Error return */
ierr = ln;
return ierr;
}
}
{ int ln = ncmpixVardim(*v1,*v2-1);
if (ln > 0) {
int li;
l4 = (MPI_Offset *)malloc( ln * sizeof(MPI_Offset) );
for (li=0; li<ln; li++)
l4[li] = v4[ln-1-li];
}
else if (ln < 0) {
/* Error return */
ierr = ln;
return ierr;
}
}
ierr = ncmpi_get_vara( *v1, l2, l3, l4, v5, *v6, MPI_Type_f2c(*v7) );
if (l3) { free(l3); }
if (l4) { free(l4); }
return ierr;
}
IOR_offset_t
IOR_Xfer_NCMPI(int access,
void * fd,
IOR_size_t * buffer,
IOR_offset_t length,
IOR_param_t * param)
{
char * bufferPtr = (char *)buffer;
static int firstReadCheck = FALSE,
startDataSet;
int var_id,
dim_id[NUM_DIMS];
MPI_Offset bufSize[NUM_DIMS],
offset[NUM_DIMS];
IOR_offset_t segmentPosition;
int segmentNum,
transferNum;
/* Wei-keng Liao: In IOR.c line 1979 says "block size must be a multiple
of transfer size." Hence, length should always == param->transferSize
below. I leave it here to double check.
*/
if (length != param->transferSize) {
char errMsg[256];
sprintf(errMsg,"length(%lld) != param->transferSize(%lld)\n",
length, param->transferSize);
NCMPI_CHECK(-1, errMsg);
}
/* determine by offset if need to start data set */
if (param->filePerProc == TRUE) {
segmentPosition = (IOR_offset_t)0;
} else {
segmentPosition = (IOR_offset_t)((rank + rankOffset) % param->numTasks)
* param->blockSize;
}
if ((int)(param->offset - segmentPosition) == 0) {
startDataSet = TRUE;
/*
* this toggle is for the read check operation, which passes through
* this function twice; note that this function will open a data set
* only on the first read check and close only on the second
*/
if (access == READCHECK) {
if (firstReadCheck == TRUE) {
firstReadCheck = FALSE;
} else {
firstReadCheck = TRUE;
}
}
}
if (startDataSet == TRUE &&
(access != READCHECK || firstReadCheck == TRUE)) {
if (access == WRITE) {
int numTransfers = param->blockSize / param->transferSize;
/* Wei-keng Liao: change 1D array to 3D array of dimensions:
[segmentCount*numTasksWorld][numTransfers][transferSize]
Requirement: none of these dimensions should be > 4G,
*/
NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "segments_times_np",
NC_UNLIMITED, &dim_id[0]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "number_of_transfers",
numTransfers, &dim_id[1]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "transfer_size",
param->transferSize, &dim_id[2]),
"cannot define data set dimensions");
NCMPI_CHECK(ncmpi_def_var(*(int *)fd, "data_var", NC_BYTE,
NUM_DIMS, dim_id, &var_id),
"cannot define data set variables");
NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
"cannot close data set define mode");
} else {
NCMPI_CHECK(ncmpi_inq_varid(*(int *)fd, "data_var", &var_id),
"cannot retrieve data set variable");
}
if (param->collective == FALSE) {
NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
"cannot enable independent data mode");
}
param->var_id = var_id;
startDataSet = FALSE;
}
var_id = param->var_id;
/* Wei-keng Liao: calculate the segment number */
segmentNum = param->offset / (param->numTasks * param->blockSize);
/* Wei-keng Liao: calculate the transfer number in each block */
transferNum = param->offset % param->blockSize / param->transferSize;
/* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of
amount param->transferSize */
bufSize[0] = 1;
bufSize[1] = 1;
bufSize[2] = param->transferSize;
offset[0] = segmentNum * numTasksWorld + rank;
offset[1] = transferNum;
offset[2] = 0;
/* access the file */
if (access == WRITE) { /* WRITE */
if (param->collective) {
NCMPI_CHECK(ncmpi_put_vara_all(*(int *)fd, var_id, offset, bufSize,
bufferPtr, length, MPI_BYTE),
"cannot write to data set");
} else {
NCMPI_CHECK(ncmpi_put_vara(*(int *)fd, var_id, offset, bufSize,
bufferPtr, length, MPI_BYTE),
"cannot write to data set");
}
} else { /* READ or CHECK */
if (param->collective == TRUE) {
NCMPI_CHECK(ncmpi_get_vara_all(*(int *)fd, var_id, offset, bufSize,
bufferPtr, length, MPI_BYTE),
"cannot read from data set");
} else {
NCMPI_CHECK(ncmpi_get_vara(*(int *)fd, var_id, offset, bufSize,
bufferPtr, length, MPI_BYTE),
"cannot read from data set");
}
}
return(length);
} /* IOR_Xfer_NCMPI() */
int main(int argc, char ** argv)
{
int ncid, dimid, varid;
MPI_Init(&argc, &argv);
MPI_Datatype vtype, rtype, usertype;
MPI_Aint lb, extent;
int userbufsz, *userbuf, *cmpbuf, i, errs=0;
int count = 25;
double pi = 3.14159;
MPI_Offset start, acount;
ncmpi_create(MPI_COMM_WORLD, "vectors.nc", NC_CLOBBER, MPI_INFO_NULL,
&ncid);
ncmpi_def_dim(ncid, "50k", 1024*50, &dimid);
ncmpi_def_var(ncid, "vector", NC_DOUBLE, 1, &dimid, &varid);
ncmpi_enddef(ncid);
MPI_Type_vector(VECCOUNT, BLOCKLEN, STRIDE, MPI_INT, &vtype);
MPI_Type_create_resized(vtype, 0, STRIDE*VECCOUNT*sizeof(int), &rtype);
MPI_Type_contiguous(count, rtype, &usertype);
MPI_Type_commit(&usertype);
MPI_Type_free(&vtype);
MPI_Type_free(&rtype);
MPI_Type_get_extent(usertype, &lb, &extent);
userbufsz = extent;
userbuf = malloc(userbufsz);
cmpbuf = calloc(userbufsz, 1);
for (i=0; i< userbufsz/sizeof(int); i++) {
userbuf[i] = pi*i;
}
start = 10; acount = count*12;
ncmpi_begin_indep_data(ncid);
ncmpi_put_vara(ncid, varid, &start, &acount,
userbuf, 1, usertype);
ncmpi_close(ncid);
NC_CHECK(ncmpi_open(MPI_COMM_WORLD, "vectors.nc", NC_NOWRITE,
MPI_INFO_NULL, &ncid));
ncmpi_begin_indep_data(ncid);
NC_CHECK(ncmpi_inq_varid(ncid, "vector", &varid));
NC_CHECK(ncmpi_get_vara(ncid, varid, &start, &acount,
cmpbuf, 1, usertype));
ncmpi_close(ncid);
for (i=0; errs < 10 && i < acount; i++) {
/* vector of 4,3,5, so skip 4th and 5th items of every block */
if (i%STRIDE >= BLOCKLEN) continue;
if (userbuf[i] != cmpbuf[i]) {
errs++;
fprintf(stderr, "%d: expected 0x%x got 0x%x\n",
i, userbuf[i], cmpbuf[i]);
}
}
free(userbuf);
free(cmpbuf);
MPI_Type_free(&usertype);
MPI_Finalize();
return 0;
}
