void BIL_Pio_read_nc_blocks(MPI_Comm all_readers_comm, MPI_Comm io_comm, int num_blocks, BIL_Block* blocks) { int i; for (i = 0; i < num_blocks; i++) { int fp; BIL_Timing_fopen_start(all_readers_comm); assert(ncmpi_open(io_comm, blocks[i].file_name, NC_NOWRITE, BIL->io_hints, &fp) == NC_NOERR); BIL_Timing_fopen_stop(all_readers_comm); ncmpi_begin_indep_data(fp); // Find the id, type, and size of the variable. int var_id; assert(ncmpi_inq_varid(fp, blocks[i].var_name, &var_id) == NC_NOERR); nc_type var_type; assert(ncmpi_inq_vartype(fp, var_id, &var_type) == NC_NOERR); // Create extra variables specifically for the netCDF API. MPI_Offset nc_dim_starts[BIL_MAX_NUM_DIMS]; MPI_Offset nc_dim_sizes[BIL_MAX_NUM_DIMS]; int j; for (j = 0; j < blocks[i].num_dims; j++) { nc_dim_starts[j] = blocks[i].starts[j]; nc_dim_sizes[j] = blocks[i].sizes[j]; } MPI_Datatype nc_var_type; BIL_Pio_nc_to_mpi_type(var_type, &nc_var_type, &(blocks[i].var_size)); // Allocate room for data and read it independently. blocks[i].data = BIL_Misc_malloc(blocks[i].total_size * blocks[i].var_size); BIL_Timing_io_start(all_readers_comm); assert(ncmpi_get_vara(fp, var_id, nc_dim_starts, nc_dim_sizes, blocks[i].data, blocks[i].total_size, nc_var_type) == NC_NOERR); BIL_Timing_io_stop(all_readers_comm, blocks[i].total_size * blocks[i].var_size); // Clean up. ncmpi_end_indep_data(fp); ncmpi_close(fp); } }
FORTRAN_API int FORT_CALL nfmpi_get_vara_ ( int *v1, int *v2, MPI_Offset v3[], MPI_Offset v4[], void*v5, MPI_Offset *v6, MPI_Fint *v7 ){ int ierr; int l2 = *v2 - 1; MPI_Offset *l3 = 0; MPI_Offset *l4 = 0; { int ln = ncmpixVardim(*v1,*v2-1); if (ln > 0) { int li; l3 = (MPI_Offset *)malloc( ln * sizeof(MPI_Offset) ); for (li=0; li<ln; li++) l3[li] = v3[ln-1-li] - 1; } else if (ln < 0) { /* Error return */ ierr = ln; return ierr; } } { int ln = ncmpixVardim(*v1,*v2-1); if (ln > 0) { int li; l4 = (MPI_Offset *)malloc( ln * sizeof(MPI_Offset) ); for (li=0; li<ln; li++) l4[li] = v4[ln-1-li]; } else if (ln < 0) { /* Error return */ ierr = ln; return ierr; } } ierr = ncmpi_get_vara( *v1, l2, l3, l4, v5, *v6, MPI_Type_f2c(*v7) ); if (l3) { free(l3); } if (l4) { free(l4); } return ierr; }
IOR_offset_t IOR_Xfer_NCMPI(int access, void * fd, IOR_size_t * buffer, IOR_offset_t length, IOR_param_t * param) { char * bufferPtr = (char *)buffer; static int firstReadCheck = FALSE, startDataSet; int var_id, dim_id[NUM_DIMS]; MPI_Offset bufSize[NUM_DIMS], offset[NUM_DIMS]; IOR_offset_t segmentPosition; int segmentNum, transferNum; /* Wei-keng Liao: In IOR.c line 1979 says "block size must be a multiple of transfer size." Hence, length should always == param->transferSize below. I leave it here to double check. */ if (length != param->transferSize) { char errMsg[256]; sprintf(errMsg,"length(%lld) != param->transferSize(%lld)\n", length, param->transferSize); NCMPI_CHECK(-1, errMsg); } /* determine by offset if need to start data set */ if (param->filePerProc == TRUE) { segmentPosition = (IOR_offset_t)0; } else { segmentPosition = (IOR_offset_t)((rank + rankOffset) % param->numTasks) * param->blockSize; } if ((int)(param->offset - segmentPosition) == 0) { startDataSet = TRUE; /* * this toggle is for the read check operation, which passes through * this function twice; note that this function will open a data set * only on the first read check and close only on the second */ if (access == READCHECK) { if (firstReadCheck == TRUE) { firstReadCheck = FALSE; } else { firstReadCheck = TRUE; } } } if (startDataSet == TRUE && (access != READCHECK || firstReadCheck == TRUE)) { if (access == WRITE) { int numTransfers = param->blockSize / param->transferSize; /* Wei-keng Liao: change 1D array to 3D array of dimensions: [segmentCount*numTasksWorld][numTransfers][transferSize] Requirement: none of these dimensions should be > 4G, */ NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "segments_times_np", NC_UNLIMITED, &dim_id[0]), "cannot define data set dimensions"); NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "number_of_transfers", numTransfers, &dim_id[1]), "cannot define data set dimensions"); NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "transfer_size", param->transferSize, &dim_id[2]), "cannot define data set dimensions"); NCMPI_CHECK(ncmpi_def_var(*(int *)fd, "data_var", NC_BYTE, NUM_DIMS, dim_id, &var_id), "cannot define data set variables"); NCMPI_CHECK(ncmpi_enddef(*(int *)fd), "cannot close data set define mode"); } else { NCMPI_CHECK(ncmpi_inq_varid(*(int *)fd, "data_var", &var_id), "cannot retrieve data set variable"); } if (param->collective == FALSE) { NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd), "cannot enable independent data mode"); } param->var_id = var_id; startDataSet = FALSE; } var_id = param->var_id; /* Wei-keng Liao: calculate the segment number */ segmentNum = param->offset / (param->numTasks * param->blockSize); /* Wei-keng Liao: calculate the transfer number in each block */ transferNum = param->offset % param->blockSize / param->transferSize; /* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of amount param->transferSize */ bufSize[0] = 1; bufSize[1] = 1; bufSize[2] = param->transferSize; offset[0] = segmentNum * numTasksWorld + rank; offset[1] = transferNum; offset[2] = 0; /* access the file */ if (access == WRITE) { /* WRITE */ if (param->collective) { NCMPI_CHECK(ncmpi_put_vara_all(*(int *)fd, var_id, offset, bufSize, bufferPtr, length, MPI_BYTE), "cannot write to data set"); } else { NCMPI_CHECK(ncmpi_put_vara(*(int *)fd, var_id, offset, bufSize, bufferPtr, length, MPI_BYTE), "cannot write to data set"); } } else { /* READ or CHECK */ if (param->collective == TRUE) { NCMPI_CHECK(ncmpi_get_vara_all(*(int *)fd, var_id, offset, bufSize, bufferPtr, length, MPI_BYTE), "cannot read from data set"); } else { NCMPI_CHECK(ncmpi_get_vara(*(int *)fd, var_id, offset, bufSize, bufferPtr, length, MPI_BYTE), "cannot read from data set"); } } return(length); } /* IOR_Xfer_NCMPI() */
int main(int argc, char ** argv) { int ncid, dimid, varid; MPI_Init(&argc, &argv); MPI_Datatype vtype, rtype, usertype; MPI_Aint lb, extent; int userbufsz, *userbuf, *cmpbuf, i, errs=0; int count = 25; double pi = 3.14159; MPI_Offset start, acount; ncmpi_create(MPI_COMM_WORLD, "vectors.nc", NC_CLOBBER, MPI_INFO_NULL, &ncid); ncmpi_def_dim(ncid, "50k", 1024*50, &dimid); ncmpi_def_var(ncid, "vector", NC_DOUBLE, 1, &dimid, &varid); ncmpi_enddef(ncid); MPI_Type_vector(VECCOUNT, BLOCKLEN, STRIDE, MPI_INT, &vtype); MPI_Type_create_resized(vtype, 0, STRIDE*VECCOUNT*sizeof(int), &rtype); MPI_Type_contiguous(count, rtype, &usertype); MPI_Type_commit(&usertype); MPI_Type_free(&vtype); MPI_Type_free(&rtype); MPI_Type_get_extent(usertype, &lb, &extent); userbufsz = extent; userbuf = malloc(userbufsz); cmpbuf = calloc(userbufsz, 1); for (i=0; i< userbufsz/sizeof(int); i++) { userbuf[i] = pi*i; } start = 10; acount = count*12; ncmpi_begin_indep_data(ncid); ncmpi_put_vara(ncid, varid, &start, &acount, userbuf, 1, usertype); ncmpi_close(ncid); NC_CHECK(ncmpi_open(MPI_COMM_WORLD, "vectors.nc", NC_NOWRITE, MPI_INFO_NULL, &ncid)); ncmpi_begin_indep_data(ncid); NC_CHECK(ncmpi_inq_varid(ncid, "vector", &varid)); NC_CHECK(ncmpi_get_vara(ncid, varid, &start, &acount, cmpbuf, 1, usertype)); ncmpi_close(ncid); for (i=0; errs < 10 && i < acount; i++) { /* vector of 4,3,5, so skip 4th and 5th items of every block */ if (i%STRIDE >= BLOCKLEN) continue; if (userbuf[i] != cmpbuf[i]) { errs++; fprintf(stderr, "%d: expected 0x%x got 0x%x\n", i, userbuf[i], cmpbuf[i]); } } free(userbuf); free(cmpbuf); MPI_Type_free(&usertype); MPI_Finalize(); return 0; }