void mpi_file_delete_(_fcd filename_fcd, MPI_Fint *info, MPI_Fint *ierr)
{
char *filename = _fcdtocp(filename_fcd);
int str_len = _fcdlen(filename_fcd);
#else
FORTRAN_API void FORT_CALL mpi_file_delete_(char *filename FORT_MIXED_LEN(str_len), MPI_Fint *info, MPI_Fint *ierr FORT_END_LEN(str_len))
{
#endif
char *newfname;
int real_len, i;
MPI_Info info_c;
info_c = MPI_Info_f2c(*info);
/* strip trailing blanks */
if (filename <= (char *) 0) {
FPRINTF(stderr, "MPI_File_delete: filename is an invalid address\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (i=str_len-1; i>=0; i--) if (filename[i] != ' ') break;
if (i < 0) {
FPRINTF(stderr, "MPI_File_delete: filename is a blank string\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
real_len = i + 1;
newfname = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
strncpy(newfname, filename, real_len);
newfname[real_len] = '\0';
*ierr = MPI_File_delete(newfname, info_c);
ADIOI_Free(newfname);
}
void cache_flush_ind_all_remove_files(int myid,
int numprocs,
char *filename)
{
/* Calculate how much each processor must write */
char *ind_filename = NULL;
int i, ind_filename_size = 0;
MPI_Barrier(MPI_COMM_WORLD);
if (myid == 0)
{
/* We will assume that we are using less than 1,000,000 processors
* therefore add 1 for NULL char and 6 for each individual processor
* for 7 total */
ind_filename_size += strlen(filename) + 7;
if ((ind_filename = (char *) malloc(ind_filename_size)) == NULL)
{
fprintf(stderr, "cache_flush_ind_all: malloc ind_filename of size"
"%d failed\n", ind_filename_size);
}
for (i = 0; i < numprocs; i++)
{
sprintf(ind_filename, "%s%d", filename, i);
MPI_File_delete(ind_filename, MPI_INFO_NULL);
}
free(ind_filename);
}
MPI_Barrier(MPI_COMM_WORLD);
}
int main(int argc, char **argv)
{
MPI_File fh;
int rank, len, err, i;
char *filename;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: excl -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
if (!rank) MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
/* this open should succeed */
err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL , &fh);
if (err != MPI_SUCCESS)
printf("Process %d: open failed when it should have succeeded\n", rank);
else MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
/* this open should fail */
err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL , &fh);
if (err == MPI_SUCCESS)
printf("Process %d: open succeeded when it should have failed\n", rank);
if (!rank) printf("Done\n");
free(filename);
MPI_Finalize();
return 0;
}
void cache_flush_ind(int myid,
int numprocs,
int size,
char *filename)
{
char *buf;
MPI_File fh;
double time;
int64_t comp = 0;
assert(size != 0);
if ((buf = (char *) malloc(MAX_BUFFER_SIZE * sizeof(char))) == NULL)
{
fprintf(stderr, "cache_flush_all: malloc buf of size %d failed\n",
MAX_BUFFER_SIZE);
}
MPI_File_open(MPI_COMM_SELF, filename,
MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
MPI_File_seek(fh, 0, MPI_SEEK_SET);
time = MPI_Wtime();
while (comp != size)
{
if (size - comp > MAX_BUFFER_SIZE)
{
comp += MAX_BUFFER_SIZE;
MPI_File_write(fh, buf, MAX_BUFFER_SIZE,
MPI_BYTE, MPI_STATUS_IGNORE);
}
else
{
int tmp_bytes = size - comp;
comp += size - comp;
MPI_File_write(fh, buf, tmp_bytes,
MPI_BYTE, MPI_STATUS_IGNORE);
}
}
free(buf);
MPI_File_sync(fh);
time = MPI_Wtime() - time;
MPI_File_close(&fh);
MPI_File_delete(filename, MPI_INFO_NULL);
fprintf(stderr,
"proc %d:cache_flush_ind: File %s written/deleted of "
"size %.1f MBytes\n"
"Time: %f secs Bandwidth: %f MBytes / sec\n\n",
myid,
filename, comp*numprocs/1024.0/1024.0,
time, comp*numprocs/1024.0/1024.0 / time);
}
JNIEXPORT void JNICALL Java_mpi_File_delete(
JNIEnv *env, jclass clazz, jstring jfilename, jlong info)
{
const char* filename = (*env)->GetStringUTFChars(env, jfilename, NULL);
int rc = MPI_File_delete((char*)filename, (MPI_Info)info);
ompi_java_exceptionCheck(env, rc);
(*env)->ReleaseStringUTFChars(env, jfilename, filename);
}
/*
* cleanup test files created
*/
void
cleanup(void)
{
int i, n;
n = sizeof(testfiles)/sizeof(testfiles[0]);
for (i=0; i<n; i++){
MPI_File_delete(testfiles[i], MPI_INFO_NULL);
}
}
static int verify_type(char *filename, MPI_Datatype type,
int64_t expected_extent, int do_coll)
{
int rank, canary;
MPI_Count tsize;
int compare=-1;
int errs=0, toterrs=0;
MPI_Status status;
MPI_File fh;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
CHECK( MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
CHECK( MPI_File_set_view(fh, rank*sizeof(int),
MPI_BYTE, type, "native", MPI_INFO_NULL));
MPI_Type_size_x(type, &tsize);
canary=rank+1000000;
/* skip over first instance of type */
if (do_coll) {
CHECK( MPI_File_write_at_all(fh, tsize, &canary, 1, MPI_INT, &status));
} else {
CHECK( MPI_File_write_at(fh, tsize, &canary, 1, MPI_INT, &status));
}
CHECK( MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native",
MPI_INFO_NULL));
if (do_coll) {
CHECK( MPI_File_read_at_all(fh, expected_extent/sizeof(int)+rank,
&compare, 1, MPI_INT, &status));
} else {
CHECK( MPI_File_read_at(fh, expected_extent/sizeof(int)+rank,
&compare, 1, MPI_INT, &status));
}
if (compare != canary)
errs=1;
MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_File_close(&fh);
if (toterrs) {
printf("%d: got %d expected %d\n", rank, compare, canary);
/* keep file if there's an error */
} else {
if (rank == 0) MPI_File_delete(filename, MPI_INFO_NULL);
}
return (toterrs);
}
/* This test checks if datareps given are little- or big-endian */
int main( int argc, char* argv[] ) {
int sample_i = 123456789, i, j;
char sample_i_le[4] = {0x15,0xcd,0x5b,0x07}, c[4];
const char* datarep[3] = { "native", "external32", "internal" };
MPI_File fileh;
int rank;
FILE* fileh_std;
if( sizeof(int) != 4 ) { printf( "non-supported sizeof(int)=%ld\n", sizeof(int) ); return (-1); }
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
/* For each datarep */
for( i = 0; i < 3; i++ ) {
/* Open file */
CHECK(MPI_File_open( MPI_COMM_WORLD, TEST_FILENAME,
MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fileh ) );
/* Set view */
CHECK(MPI_File_set_view( fileh, 0, MPI_INT, MPI_INT, datarep[i], MPI_INFO_NULL ));
/* Write into file */
CHECK(MPI_File_write_at( fileh, (MPI_Offset)rank, (void*)&sample_i, 1,
MPI_INT, MPI_STATUS_IGNORE ));
/* Close file */
CHECK(MPI_File_close( &fileh ));
/* Check if your datarep is little or big endian */
MPI_Barrier( MPI_COMM_WORLD );
if( rank == 0 ) {
fileh_std = fopen( TEST_FILENAME, "r" );
for( j = 0; j < 4; j++ ) {
if( feof( fileh_std ) ) { printf( "unexpected eof, aborted\n" ); return (-1); }
fscanf( fileh_std, "%c", &c[j] );
}
is_little_or_big_endian( datarep[i], c, sample_i_le, 4 );
fclose( fileh_std );
}
/* Delete file */
if( rank == 0 ) {
CHECK(MPI_File_delete( TEST_FILENAME, MPI_INFO_NULL ));
}
}
MPI_Finalize();
return 0;
}
/*
* test file access by communicator besides COMM_WORLD.
* Split COMM_WORLD into two, one (even_comm) contains the original
* processes of even ranks. The other (odd_comm) contains the original
* processes of odd ranks. Processes in even_comm creates a file, then
* cloose it, using even_comm. Processes in old_comm just do a barrier
* using odd_comm. Then they all do a barrier using COMM_WORLD.
* If the file creation and cloose does not do correct collective action
* according to the communicator argument, the processes will freeze up
* sooner or later due to barrier mixed up.
*/
void
test_split_comm_access(char filenames[][PATH_MAX])
{
MPI_Comm comm;
MPI_Info info = MPI_INFO_NULL;
int color, mrc;
int newrank, newprocs;
hid_t fid; /* file IDs */
hid_t acc_tpl; /* File access properties */
herr_t ret; /* generic return value */
if (verbose)
printf("Independent write test on file %s %s\n",
filenames[0], filenames[1]);
color = mpi_rank%2;
mrc = MPI_Comm_split (MPI_COMM_WORLD, color, mpi_rank, &comm);
assert(mrc==MPI_SUCCESS);
MPI_Comm_size(comm,&newprocs);
MPI_Comm_rank(comm,&newrank);
if (color){
/* odd-rank processes */
mrc = MPI_Barrier(comm);
assert(mrc==MPI_SUCCESS);
}else{
/* even-rank processes */
/* setup file access template */
acc_tpl = H5Pcreate (H5P_FILE_ACCESS);
assert(acc_tpl != FAIL);
/* set Parallel access with communicator */
ret = H5Pset_fapl_mpio(acc_tpl, comm, info);
assert(ret != FAIL);
/* create the file collectively */
fid=H5Fcreate(filenames[color],H5F_ACC_TRUNC,H5P_DEFAULT,acc_tpl);
assert(fid != FAIL);
MESG("H5Fcreate succeed");
/* Release file-access template */
ret=H5Pclose(acc_tpl);
assert(ret != FAIL);
ret=H5Fclose(fid);
assert(ret != FAIL);
}
if (mpi_rank == 0){
mrc = MPI_File_delete(filenames[color], info);
assert(mrc==MPI_SUCCESS);
}
}
/*
* parallel_open_file_write()
* Use MPI to open a coordinate file for writing, delete if present.
* Return MPI file handle.
*/
int parallel_open_file_write(coordinateInfo *C, char *filename) {
#ifdef MPI
int err,errtotal;
MPI_File *mfp;
C->mfp=NULL;
// Remove file if present
MPI_File_delete(filename,MPI_INFO_NULL);
if (prnlev>0) fprintf(stdout,"[%i] parallel_open_file_write: Opening output file %s\n",worldrank,filename);
mfp = (MPI_File*) malloc(sizeof(MPI_File));
err=MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, mfp);
if (err!=MPI_SUCCESS) printMPIerr(err,"parallel_open_file_write()");
// Check that everyone opened the file.
if (parallel_check_error(err)!=0) {
free(mfp);
return 1;
} else {
C->mfp=mfp;
return 0;
}
#endif
return 1;
}
int main(int argc, char **argv)
{
int i, len, nkeys, flag, mynod, default_striping_factor=0, nprocs, errs = 0;
MPI_File fh;
MPI_Info info, info_used;
char *filename, key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
if (!strcmp("-v", *argv)) verbose = 1;
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: file_info [-v] -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
/* open the file with MPI_INFO_NULL */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
/* check the default values set by ROMIO */
MPI_File_get_info(fh, &info_used);
MPI_Info_get_nkeys(info_used, &nkeys);
for (i=0; i<nkeys; i++) {
MPI_Info_get_nthkey(info_used, i, key);
MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
#ifdef INFO_DEBUG
if (!mynod)
fprintf(stderr, "Process %d, Default: key = %s, value = %s\n", mynod,
key, value);
#endif
if (!strcmp("striping_factor", key)) {
default_striping_factor = atoi(value);
/* no check */
}
else if (!strcmp("cb_buffer_size", key)) {
if (atoi(value) != DFLT_CB_BUFFER_SIZE) {
errs++;
if (verbose) fprintf(stderr, "cb_buffer_size is %d; should be %d\n",
atoi(value), DFLT_CB_BUFFER_SIZE);
}
}
else if (!strcmp("romio_cb_read", key)) {
if (strcmp(DFLT_ROMIO_CB_READ, value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_read is set to %s; should be %s\n",
value, DFLT_ROMIO_CB_READ);
}
}
else if (!strcmp("romio_cb_write", key)) {
if (strcmp(DFLT_ROMIO_CB_WRITE, value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_write is set to %s; should be %s\n",
value, DFLT_ROMIO_CB_WRITE);
}
}
else if (!strcmp("cb_nodes", key)) {
/* unreliable test -- just ignore value */
}
else if (!strcmp("romio_no_indep_rw", key)) {
if (strcmp("false", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_no_indep_rw is set to %s; should be %s\n",
value, "false");
}
}
else if (!strcmp("ind_rd_buffer_size", key)) {
if (atoi(value) != DFLT_IND_RD_BUFFER_SIZE) {
errs++;
if (verbose) fprintf(stderr, "ind_rd_buffer_size is %d; should be %d\n",
atoi(value), DFLT_IND_RD_BUFFER_SIZE);
}
}
else if (!strcmp("ind_wr_buffer_size", key)) {
if (atoi(value) != DFLT_IND_WR_BUFFER_SIZE) {
errs++;
if (verbose) fprintf(stderr, "ind_wr_buffer_size is %d; should be %d\n",
atoi(value), DFLT_IND_WR_BUFFER_SIZE);
}
}
else if (!strcmp("romio_ds_read", key)) {
if (strcmp("automatic", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_ds_read is set to %s; should be %s\n",
value, "automatic");
}
}
else if (!strcmp("romio_ds_write", key)) {
/* Unreliable test -- value is file system dependent. Ignore. */
}
else if (!strcmp("cb_config_list", key)) {
#ifndef SKIP_CB_CONFIG_LIST_TEST
if (strcmp("*:1", value)) {
errs++;
if (verbose) fprintf(stderr, "cb_config_list is set to %s; should be %s\n",
value, "*:1");
}
#endif
}
/* don't care about the defaults for these keys */
else if (!strcmp("romio_cb_pfr", key)) {
}
else if (!strcmp("romio_cb_fr_types", key)) {
}
else if (!strcmp("romio_cb_fr_alignment", key)) {
}
else if (!strcmp("romio_cb_ds_threshold", key)) {
}
else if (!strcmp("romio_cb_alltoall", key)) {
}
else {
if (verbose) fprintf(stderr, "unexpected key %s (not counted as an error)\n", key);
}
}
MPI_Info_free(&info_used);
MPI_File_close(&fh);
/* delete the file */
if (!mynod) MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
/* set new info values. */
MPI_Info_create(&info);
/* The following four hints are accepted on all machines. They can
be specified at file-open time or later (any number of times). */
/* buffer size for collective I/O */
MPI_Info_set(info, "cb_buffer_size", "8388608");
/* number of processes that actually perform I/O in collective I/O */
sprintf(value, "%d", nprocs/2);
MPI_Info_set(info, "cb_nodes", value);
/* buffer size for data sieving in independent reads */
MPI_Info_set(info, "ind_rd_buffer_size", "2097152");
/* buffer size for data sieving in independent writes */
MPI_Info_set(info, "ind_wr_buffer_size", "1048576");
/* The following three hints related to file striping are accepted only
on Intel PFS and IBM PIOFS file systems and are ignored elsewhere.
They can be specified only at file-creation time; if specified later
they will be ignored. */
/* number of I/O devices across which the file will be striped.
accepted only if 0 < value < default_striping_factor;
ignored otherwise */
if (default_striping_factor - 1 > 0) {
sprintf(value, "%d", default_striping_factor-1);
MPI_Info_set(info, "striping_factor", value);
}
else {
sprintf(value, "%d", default_striping_factor);
MPI_Info_set(info, "striping_factor", value);
}
/* the striping unit in bytes */
MPI_Info_set(info, "striping_unit", "131072");
#ifndef SKIP_CB_CONFIG_LIST_TEST
/* set the cb_config_list so we'll get deterministic cb_nodes output */
MPI_Info_set(info, "cb_config_list", "*:*");
#endif
/* the I/O device number from which to start striping the file.
accepted only if 0 <= value < default_striping_factor;
ignored otherwise */
sprintf(value, "%d", default_striping_factor-2);
MPI_Info_set(info, "start_iodevice", value);
/* The following hint about PFS server buffering is accepted only on
Intel PFS. It can be specified anytime. */
MPI_Info_set(info, "pfs_svr_buf", "true");
/* open the file and set new info */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* check the values set */
MPI_File_get_info(fh, &info_used);
MPI_Info_get_nkeys(info_used, &nkeys);
for (i=0; i<nkeys; i++) {
MPI_Info_get_nthkey(info_used, i, key);
MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
#ifdef INFO_DEBUG
if (!mynod) fprintf(stderr, "Process %d, key = %s, value = %s\n", mynod,
key, value);
#endif
if (!strcmp("striping_factor", key)) {
if ((default_striping_factor - 1 > 0) && (atoi(value) != default_striping_factor-1)) {
errs++;
if (verbose) fprintf(stderr, "striping_factor is %d; should be %d\n",
atoi(value), default_striping_factor-1);
}
else if (atoi(value) != default_striping_factor) {
errs++;
if (verbose) fprintf(stderr, "striping_factor is %d; should be %d\n",
atoi(value), default_striping_factor);
}
}
else if (!strcmp("cb_buffer_size", key)) {
if (atoi(value) != 8388608) {
errs++;
if (verbose) fprintf(stderr, "cb_buffer_size is %d; should be %d\n",
atoi(value), 8388608);
}
}
else if (!strcmp("romio_cb_read", key)) {
if (strcmp(DFLT_ROMIO_CB_READ, value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_read is set to %s; should be %s\n",
value, DFLT_ROMIO_CB_READ);
}
}
else if (!strcmp("romio_cb_write", key)) {
if (strcmp(DFLT_ROMIO_CB_WRITE, value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_write is set to %s; should be %s\n",
value, DFLT_ROMIO_CB_WRITE);
}
}
else if (!strcmp("cb_nodes", key)) {
if (atoi(value) != (nprocs/2)) {
errs++;
if (verbose) fprintf(stderr, "cb_nodes is %d; should be %d\n", atoi(value),
nprocs/2);
}
}
else if (!strcmp("romio_no_indep_rw", key)) {
if (strcmp("false", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_no_indep_rw is set to %s; should be %s\n",
value, "false");
}
}
else if (!strcmp("ind_rd_buffer_size", key)) {
if (atoi(value) != 2097152) {
errs++;
if (verbose) fprintf(stderr, "ind_rd_buffer_size is %d; should be %d\n",
atoi(value), 2097152);
}
}
else if (!strcmp("ind_wr_buffer_size", key)) {
if (atoi(value) != 1048576) {
errs++;
if (verbose) fprintf(stderr, "ind_wr_buffer_size is %d; should be %d\n",
atoi(value), 1048576);
}
}
else if (!strcmp("romio_ds_read", key)) {
if (strcmp("automatic", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_ds_read is set to %s; should be %s\n",
value, "automatic");
}
}
else if (!strcmp("romio_ds_write", key)) {
/* Unreliable test -- value is file system dependent. Ignore. */
}
else if (!strcmp("cb_config_list", key)) {
#ifndef SKIP_CB_CONFIG_LIST_TEST
if (strcmp("*:*", value)) {
errs++;
if (verbose) fprintf(stderr, "cb_config_list is set to %s; should be %s\n",
value, "*:*");
}
#endif
}
else if (!strcmp("romio_cb_pfr", key)) {
if(strcmp("disable", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_pfr is set to %s; should be %s\n",
value, "automatic");
}
}
else if (!strcmp("romio_cb_fr_types", key)) {
if(strcmp("aar", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_fr_types is set to %s; should be %s\n",
value, "aar");
}
}
else if (!strcmp("romio_cb_fr_alignment", key)) {
if(strcmp("1", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_fr_alignment is set to %s; should be %s\n",
value, "1");
}
}
else if (!strcmp("romio_cb_ds_threshold", key)) {
if(strcmp("0", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_ds_threshold is set to %s; should be %s\n",
value, "0");
}
}
else if (!strcmp("romio_cb_alltoall", key)) {
if(strcmp("automatic", value)) {
errs++;
if (verbose) fprintf(stderr, "romio_cb_alltoall is set to %s; should be %s\n",
value, "automatic");
}
}
else {
if (verbose) fprintf(stderr, "unexpected key %s (not counted as an error)\n", key);
}
}
/* Q: SHOULD WE BOTHER LOOKING AT THE OTHER PROCESSES? */
if (!mynod) {
if (errs) fprintf(stderr, "Found %d errors.\n", errs);
else printf(" No Errors\n");
}
MPI_File_close(&fh);
free(filename);
MPI_Info_free(&info_used);
MPI_Info_free(&info);
MPI_Finalize();
return 0;
}
int main(int argc, char ** argv) {
int nprocs, mynod, errcode;
options my_options = {NULL, 0, 0};
MPI_File fh;
MPI_Status status;
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
parse_args(argc, argv, mynod, &my_options);
if (my_options.do_aggregation) {
MPI_Info_create(&info);
MPI_Info_set(info, "romio_no_indep_rw", "true");
MPI_Info_set(info, "cb_config_list", "leela.mcs.anl.gov:1");
} else {
info = MPI_INFO_NULL;
}
/* create the file w/o EXCL: this must not fail */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_close");
}
/* now try to open w/ CREAT|EXCL: this must fail */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
if (errcode == MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open: expected an error: got");
}
/* ignore the error: File_delete is not aggregator-aware */
MPI_File_delete(my_options.fname, info);
/* this must succeed: the file no longer exists */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_close");
}
if (mynod == 0) {
printf(" No Errors\n");
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int i, len, nkeys, flag, mynod, default_striping_factor, nprocs;
MPI_File fh;
MPI_Info info, info_used;
char *filename, key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: file_info -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
/* open the file with MPI_INFO_NULL */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
/* check the default values set by ROMIO */
MPI_File_get_info(fh, &info_used);
MPI_Info_get_nkeys(info_used, &nkeys);
for (i=0; i<nkeys; i++) {
MPI_Info_get_nthkey(info_used, i, key);
MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
if (!mynod)
printf("Process %d, Default: key = %s, value = %s\n", mynod,
key, value);
if (!strcmp("striping_factor", key))
default_striping_factor = atoi(value);
}
MPI_File_close(&fh);
/* delete the file */
if (!mynod) MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
/* set new info values. */
MPI_Info_create(&info);
/* The following four hints are accepted on all machines. They can
be specified at file-open time or later (any number of times). */
/* buffer size for collective I/O */
MPI_Info_set(info, "cb_buffer_size", "8388608");
/* number of processes that actually perform I/O in collective I/O */
sprintf(value, "%d", nprocs/2);
MPI_Info_set(info, "cb_nodes", value);
/* buffer size for data sieving in independent reads */
MPI_Info_set(info, "ind_rd_buffer_size", "2097152");
/* buffer size for data sieving in independent writes */
MPI_Info_set(info, "ind_wr_buffer_size", "1048576");
/* The following three hints related to file striping are accepted only
on Intel PFS and IBM PIOFS file systems and are ignored elsewhere.
They can be specified only at file-creation time; if specified later
they will be ignored. */
/* number of I/O devices across which the file will be striped.
accepted only if 0 < value < default_striping_factor;
ignored otherwise */
sprintf(value, "%d", default_striping_factor-1);
MPI_Info_set(info, "striping_factor", value);
/* the striping unit in bytes */
MPI_Info_set(info, "striping_unit", "131072");
/* the I/O device number from which to start striping the file.
accepted only if 0 <= value < default_striping_factor;
ignored otherwise */
sprintf(value, "%d", default_striping_factor-2);
MPI_Info_set(info, "start_iodevice", value);
/* The following hint about PFS server buffering is accepted only on
Intel PFS. It can be specified anytime. */
MPI_Info_set(info, "pfs_svr_buf", "true");
/* open the file and set new info */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* check the values set */
MPI_File_get_info(fh, &info_used);
MPI_Info_get_nkeys(info_used, &nkeys);
if (!mynod) printf("\n New values\n\n");
for (i=0; i<nkeys; i++) {
MPI_Info_get_nthkey(info_used, i, key);
MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
if (!mynod) printf("Process %d, key = %s, value = %s\n", mynod,
key, value);
}
MPI_File_close(&fh);
free(filename);
MPI_Info_free(&info_used);
MPI_Info_free(&info);
MPI_Finalize();
return 0;
}
/**
* Main function which selects the process to be a master or a worker
* based on MPI myid.
*
* @param argc Number of arguments.
* @param argv Pointer to the argument pointers.
* @return 0 on success.
*/
int main(int argc, char **argv)
{
int myid, numprocs, i;
int err = -1;
struct test_params_s test_params;
struct mpe_events_s mpe_events;
struct frag_preresult_s **query_frag_preresult_matrix = NULL;
memset(&test_params, 0, sizeof(struct test_params_s));
MPI_Init(&argc, &argv);
#ifdef HAVE_MPE
MPI_Pcontrol(0);
#endif
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
init_mpe_events(&mpe_events);
#ifdef HAVE_MPE
if (myid == MASTER_NODE)
{
init_mpe_describe_state(&mpe_events);
}
#endif
if (myid == MASTER_NODE)
{
err = parse_args(argc, argv, &test_params);
if (err >= 0)
print_settings(&test_params, numprocs);
if (test_params.output_file != NULL)
MPI_File_delete(test_params.output_file, MPI_INFO_NULL);
if (numprocs < 2)
{
fprintf(stderr, "Must use at least 2 processes.\n");
err = -1;
}
}
MPI_Bcast(&err, 1, MPI_INT, MASTER_NODE, MPI_COMM_WORLD);
/* Quit if the parse_args failed */
if (err != 0)
{
MPI_Finalize();
return 0;
}
/* Master precalculates all the results for the queries and
* reads in the database histogram parameters */
if (myid == MASTER_NODE)
{
if ((query_frag_preresult_matrix = (struct frag_preresult_s **)
malloc(test_params.query_count *
sizeof(struct frag_preresult_s *))) == NULL)
{
custom_debug(
MASTER_ERR,
"M:malloc query_frag_preresult_matrix of size %d failed\n",
test_params.query_count * sizeof(struct frag_preresult_s *));
return -1;
}
for (i = 0; i < test_params.query_count; i++)
{
if ((query_frag_preresult_matrix[i] = (struct frag_preresult_s *)
malloc(test_params.total_frags *
sizeof(struct frag_preresult_s))) == NULL)
{
custom_debug(
MASTER_ERR,
"M:malloc query_frag_preresult_matrix[%d] "
"of size %d failed\n", i,
test_params.total_frags *
sizeof(struct frag_preresult_s));
return -1;
}
memset(query_frag_preresult_matrix[i], 0,
test_params.total_frags * sizeof(struct frag_preresult_s));
}
precalculate_results(&test_params, query_frag_preresult_matrix);
test_params.query_frag_preresult_matrix = query_frag_preresult_matrix;
if (test_params.query_params_file != NULL)
{
read_hist_params(&test_params, QUERY);
#if 0
print_hist_params(&test_params);
#endif
}
if (test_params.db_params_file != NULL)
{
read_hist_params(&test_params, DATABASE);
#if 0
print_hist_params(&test_params);
#endif
}
}
MPI_Barrier(MPI_COMM_WORLD);
#ifdef HAVE_MPE
MPI_Pcontrol(1);
#endif
/* Divide up into either a Master or Worker */
mpe_events.total_time = MPI_Wtime();
if (myid == 0)
{
err = master(myid,
numprocs,
&mpe_events,
&test_params);
if (err != 0)
custom_debug(MASTER_ERR, "master failed\n");
else
custom_debug(MASTER, "master (proc %d) reached last barrier\n",
myid);
}
else
{
err = worker(myid,
numprocs,
&mpe_events,
&test_params);
if (err != 0)
custom_debug(WORKER_ERR, "worker failed\n");
else
custom_debug(WORKER, "worker (proc %d) reached last barrier\n",
myid);
}
custom_MPE_Log_event(mpe_events.sync_start,
0, NULL, &mpe_events);
MPI_Barrier(MPI_COMM_WORLD);
custom_MPE_Log_event(mpe_events.sync_end,
0, NULL, &mpe_events);
MPI_Pcontrol(0);
mpe_events.total_time = MPI_Wtime() - mpe_events.total_time;
#if 0
print_timing(myid, &mpe_events);
#endif
MPI_Barrier(MPI_COMM_WORLD);
timing_reduce(myid, numprocs, &mpe_events);
/* Clean up precomputed results and file */
if (myid == MASTER_NODE)
{
if (test_params.query_params_file != NULL)
{
free(test_params.query_params_file);
free(test_params.query_hist_list);
}
if (test_params.db_params_file != NULL)
{
free(test_params.db_params_file);
free(test_params.db_hist_list);
}
MPI_File_delete(test_params.output_file, MPI_INFO_NULL);
for (i = 0; i < test_params.query_count; i++)
{
free(query_frag_preresult_matrix[i]);
}
free(query_frag_preresult_matrix);
}
MPI_Info_free(test_params.info_p);
free(test_params.info_p);
free(test_params.output_file);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int *buf, i, mynod, nprocs, len, b[3];
int errs=0, toterrs;
MPI_Aint d[3];
MPI_File fh;
MPI_Status status;
char *filename;
MPI_Datatype typevec, newtype, t[3];
MPIO_Request req;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
if (nprocs != 2) {
fprintf(stderr, "Run this program on two processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: i_noncontig -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE*sizeof(int));
MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod*sizeof(int);
d[2] = SIZE*sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using nonblocking I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_iwrite(fh, buf, 1, newtype, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_iread_at(fh, 0, buf, 1, newtype, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using nonblocking I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_iwrite_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_iread_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using nonblocking I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_iwrite(fh, buf, SIZE, MPI_INT, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_iread_at(fh, 0, buf, SIZE, MPI_INT, &req);
#ifdef MPIO_USES_MPI_REQUEST
MPI_Wait(&req, &status);
#else
MPIO_Wait(&req, &status);
#endif
for (i=0; i<SIZE; i++) {
if (!mynod) {
if (buf[i] != i) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if (buf[i] != i + mynod*SIZE) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
}
}
MPI_File_close(&fh);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (mynod == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
MPI_Type_free(&newtype);
free(buf);
free(filename);
MPI_Finalize();
return 0;
}
/*
* Verify that MPI_Offset exceeding 2**31 can be computed correctly.
* Print any failure as information only, not as an error so that this
* won't abort the remaining test or other separated tests.
*
* Test if MPIO can write file from under 2GB to over 2GB and then
* from under 4GB to over 4GB.
* Each process writes 1MB in round robin fashion.
* Then reads the file back in by reverse order, that is process 0
* reads the data of process n-1 and vice versa.
*/
static int
test_mpio_gb_file(char *filename)
{
int mpi_size, mpi_rank;
MPI_Info info = MPI_INFO_NULL;
int mrc;
MPI_File fh;
int i, j, n;
int vrfyerrs;
int writerrs; /* write errors */
int nerrs;
int ntimes; /* how many times */
char *buf = NULL;
char expected;
MPI_Offset size;
MPI_Offset mpi_off;
MPI_Offset mpi_off_old;
MPI_Status mpi_stat;
h5_stat_t stat_buf;
int is_signed, sizeof_mpi_offset;
nerrs = 0;
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
if (VERBOSE_MED)
printf("MPI_Offset range test\n");
/* figure out the signness and sizeof MPI_Offset */
mpi_off = 0;
is_signed = ((MPI_Offset)(mpi_off - 1)) < 0;
sizeof_mpi_offset = (int)(sizeof(MPI_Offset));
/*
* Verify the sizeof MPI_Offset and correctness of handling multiple GB
* sizes.
*/
if (MAINPROCESS){ /* only process 0 needs to check it*/
printf("MPI_Offset is %s %d bytes integeral type\n",
is_signed ? "signed" : "unsigned", (int)sizeof(MPI_Offset));
if (sizeof_mpi_offset <= 4 && is_signed){
printf("Skipped 2GB range test "
"because MPI_Offset cannot support it\n");
}else {
/* verify correctness of assigning 2GB sizes */
mpi_off = 2 * 1024 * (MPI_Offset)MB;
INFO((mpi_off>0), "2GB OFFSET assignment no overflow");
INFO((mpi_off-1)==TWO_GB_LESS1, "2GB OFFSET assignment succeed");
/* verify correctness of increasing from below 2 GB to above 2GB */
mpi_off = TWO_GB_LESS1;
for (i=0; i < 3; i++){
mpi_off_old = mpi_off;
mpi_off = mpi_off + 1;
/* no overflow */
INFO((mpi_off>0), "2GB OFFSET increment no overflow");
/* correct inc. */
INFO((mpi_off-1)==mpi_off_old, "2GB OFFSET increment succeed");
}
}
if (sizeof_mpi_offset <= 4){
printf("Skipped 4GB range test "
"because MPI_Offset cannot support it\n");
}else {
/* verify correctness of assigning 4GB sizes */
mpi_off = 4 * 1024 * (MPI_Offset)MB;
INFO((mpi_off>0), "4GB OFFSET assignment no overflow");
INFO((mpi_off-1)==FOUR_GB_LESS1, "4GB OFFSET assignment succeed");
/* verify correctness of increasing from below 4 GB to above 4 GB */
mpi_off = FOUR_GB_LESS1;
for (i=0; i < 3; i++){
mpi_off_old = mpi_off;
mpi_off = mpi_off + 1;
/* no overflow */
INFO((mpi_off>0), "4GB OFFSET increment no overflow");
/* correct inc. */
INFO((mpi_off-1)==mpi_off_old, "4GB OFFSET increment succeed");
}
}
}
/*
* Verify if we can write to a file of multiple GB sizes.
*/
if (VERBOSE_MED)
printf("MPIO GB file test %s\n", filename);
if (sizeof_mpi_offset <= 4){
printf("Skipped GB file range test "
"because MPI_Offset cannot support it\n");
}else{
buf = malloc(MB);
VRFY((buf!=NULL), "malloc succeed");
/* open a new file. Remove it first in case it exists. */
/* Must delete because MPI_File_open does not have a Truncate mode. */
/* Don't care if it has error. */
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE|MPI_MODE_RDWR,
info, &fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_OPEN");
printf("MPIO GB file write test %s\n", filename);
/* instead of writing every bytes of the file, we will just write
* some data around the 2 and 4 GB boundaries. That should cover
* potential integer overflow and filesystem size limits.
*/
writerrs = 0;
for (n=2; n <= 4; n+=2){
ntimes = GB/MB*n/mpi_size + 1;
for (i=ntimes-2; i <= ntimes; i++){
mpi_off = (i*mpi_size + mpi_rank)*(MPI_Offset)MB;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: write to mpi_off=%016llx, %lld\n",
mpi_rank, mpi_off, mpi_off);
/* set data to some trivial pattern for easy verification */
for (j=0; j<MB; j++)
*(buf+j) = i*mpi_size + mpi_rank;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: writing %d bytes at offset %lld\n",
mpi_rank, MB, mpi_off);
mrc = MPI_File_write_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat);
INFO((mrc==MPI_SUCCESS), "GB size file write");
if (mrc!=MPI_SUCCESS)
writerrs++;
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync after writes");
/*
* Verify if we can read the multiple GB file just created.
*/
/* open it again to verify the data written */
/* but only if there was no write errors */
printf("MPIO GB file read test %s\n", filename);
if (errors_sum(writerrs)>0){
printf("proc %d: Skip read test due to previous write errors\n",
mpi_rank);
goto finish;
}
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
/* Only read back parts of the file that have been written. */
for (n=2; n <= 4; n+=2){
ntimes = GB/MB*n/mpi_size + 1;
for (i=ntimes-2; i <= ntimes; i++){
mpi_off = (i*mpi_size + (mpi_size - mpi_rank - 1))*(MPI_Offset)MB;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: read from mpi_off=%016llx, %lld\n",
mpi_rank, mpi_off, mpi_off);
mrc = MPI_File_read_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat);
INFO((mrc==MPI_SUCCESS), "GB size file read");
expected = i*mpi_size + (mpi_size - mpi_rank - 1);
vrfyerrs=0;
for (j=0; j<MB; j++){
if ((*(buf+j) != expected) &&
(vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)){
printf("proc %d: found data error at [%ld+%d], expect %d, got %d\n",
mpi_rank, (long)mpi_off, j, expected, *(buf+j));
}
}
if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED)
printf("proc %d: [more errors ...]\n", mpi_rank);
nerrs += vrfyerrs;
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
/*
* one more sync to ensure all processes have done reading
* before ending this test.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync before leaving test");
/*
* Check if MPI_File_get_size works correctly. Some systems (only SGI Altix
* Propack 4 so far) return wrong file size. It can be avoided by reconfiguring
* with "--disable-mpi-size".
*/
#ifdef H5_HAVE_MPI_GET_SIZE
printf("Test if MPI_File_get_size works correctly with %s\n", filename);
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
if (MAINPROCESS){ /* only process 0 needs to check it*/
mrc = MPI_File_get_size(fh, &size);
VRFY((mrc==MPI_SUCCESS), "");
mrc=HDstat(filename, &stat_buf);
VRFY((mrc==0), "");
/* Hopefully this casting is safe */
if(size != (MPI_Offset)(stat_buf.st_size)) {
printf("Warning: MPI_File_get_size doesn't return correct file size. To avoid using it in the library, reconfigure and rebuild the library with --disable-mpi-size.\n");
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
/*
* one more sync to ensure all processes have done reading
* before ending this test.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync before leaving test");
#else
printf("Skipped testing MPI_File_get_size because it's disabled\n");
#endif
}
finish:
if (buf)
HDfree(buf);
return (nerrs);
}
int main(int argc, char **argv)
{
MPI_Datatype newtype;
int i, ndims, array_of_gsizes[3], array_of_distribs[3];
int order, nprocs, len, flag, err;
int array_of_dargs[3], array_of_psizes[3];
int *readbuf, *writebuf, bufcount, mynod;
char filename[1024];
MPI_File fh;
MPI_Status status;
MPI_Aint size_with_aint;
MPI_Offset size_with_offset;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: large_array -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
printf("This program creates a 4 Gbyte file. Don't run it if you don't have that much disk space!\n");
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
/* create the distributed array filetype */
ndims = 3;
order = MPI_ORDER_C;
array_of_gsizes[0] = 1024;
array_of_gsizes[1] = 1024;
array_of_gsizes[2] = 4*1024/sizeof(int);
array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
MPI_Dims_create(nprocs, ndims, array_of_psizes);
/* check if MPI_Aint is large enough for size of global array.
if not, complain. */
size_with_aint = sizeof(int);
for (i=0; i<ndims; i++) size_with_aint *= array_of_gsizes[i];
size_with_offset = sizeof(int);
for (i=0; i<ndims; i++) size_with_offset *= array_of_gsizes[i];
if (size_with_aint != size_with_offset) {
printf("Can't use an array of this size unless the MPI implementation defines a 64-bit MPI_Aint\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes,
array_of_distribs, array_of_dargs,
array_of_psizes, order, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
/* initialize writebuf */
MPI_Type_size(newtype, &bufcount);
bufcount = bufcount/sizeof(int);
writebuf = (int *) malloc(bufcount * sizeof(int));
if (!writebuf) printf("Process %d, not enough memory for writebuf\n", mynod);
for (i=0; i<bufcount; i++) writebuf[i] = mynod*1024 + i;
/* write the array to the file */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, writebuf, bufcount, MPI_INT, &status);
MPI_File_close(&fh);
free(writebuf);
/* now read it back */
readbuf = (int *) calloc(bufcount, sizeof(int));
if (!readbuf) printf("Process %d, not enough memory for readbuf\n", mynod);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
MPI_File_read_all(fh, readbuf, bufcount, MPI_INT, &status);
MPI_File_close(&fh);
/* check the data read */
flag = 0;
for (i=0; i<bufcount; i++)
if (readbuf[i] != mynod*1024 + i) {
printf("Process %d, readbuf=%d, writebuf=%d\n", mynod, readbuf[i], mynod*1024 + i);
flag = 1;
}
if (!flag) printf("Process %d: data read back is correct\n", mynod);
MPI_Type_free(&newtype);
free(readbuf);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
err = MPI_File_delete(filename, MPI_INFO_NULL);
if (err == MPI_SUCCESS) printf("file deleted\n");
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int *buf, i, mynod, nprocs, len, b[3];
int errs=0, toterrs;
MPI_Aint d[3];
MPI_File fh;
MPI_Status status;
char *filename;
MPI_Datatype typevec, newtype, t[3];
MPI_Info info;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
if (nprocs != 2) {
fprintf(stderr, "Run this program on two processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes (length first, then string) */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: noncontig -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE*sizeof(int));
MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
/* create a struct type with explicitly set LB and UB; displacements
* of typevec are such that the types for the two processes won't
* overlap.
*/
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod*sizeof(int);
d[2] = SIZE*sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
/* keep the struct, ditch the vector */
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* set the file view for each process -- now writes go into the non-
* overlapping but interleaved region defined by the struct type up above
*/
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
/* fill our buffer with a pattern and write, using our type again */
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write(fh, buf, 1, newtype, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* fill the entire buffer with -1's. read back with type.
* note that the result of this read should be that every other value
* in the buffer is still -1, as defined by our type.
*/
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, 0, buf, 1, newtype, &status);
/* check that all the values read are correct and also that we didn't
* overwrite any of the -1 values that we shouldn't have.
*/
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* in this case we write to either the first half or the second half
* of the file space, so the regions are not interleaved. this is done
* by leaving the file view at its default.
*/
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* same as before; fill buffer with -1's and then read; every other
* value should still be -1 after the read
*/
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
/* verify that the buffer looks like it should */
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* set the file view so that we have interleaved access again */
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
/* this time write a contiguous buffer */
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write(fh, buf, SIZE, MPI_INT, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* fill buffer with -1's; this time they will all be overwritten */
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, 0, buf, SIZE, MPI_INT, &status);
for (i=0; i<SIZE; i++) {
if (!mynod) {
if (buf[i] != i) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if (buf[i] != i + mynod*SIZE) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
}
}
MPI_File_close(&fh);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (mynod == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
MPI_Type_free(&newtype);
MPI_Info_free(&info);
free(buf);
free(filename);
MPI_Finalize();
return 0;
}
static int
test_mpio_1wMr(char *filename, int special_request)
{
char hostname[128];
int mpi_size, mpi_rank;
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
unsigned char writedata[DIMSIZE], readdata[DIMSIZE];
unsigned char expect_val;
int i, irank;
int nerrs = 0; /* number of errors */
int atomicity;
MPI_Offset mpi_off;
MPI_Status mpi_stat;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
if (MAINPROCESS && VERBOSE_MED){
printf("Testing one process writes, all processes read.\n");
printf("Using %d processes accessing file %s\n", mpi_size, filename);
printf(" (Filename can be specified via program argument)\n");
}
/* show the hostname so that we can tell where the processes are running */
if (VERBOSE_DEF){
if (gethostname(hostname, 128) < 0){
PRINTID;
printf("gethostname failed\n");
return 1;
}
PRINTID;
printf("hostname=%s\n", hostname);
}
/* Delete any old file in order to start anew. */
/* Must delete because MPI_File_open does not have a Truncate mode. */
/* Don't care if it has error. */
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE ,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
if (special_request & USEATOM){
/* ==================================================
* Set atomcity to true (1). A POSIX compliant filesystem
* should not need this.
* ==================================================*/
if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str);
}
if (VERBOSE_HI)
printf("Initial atomicity = %d\n", atomicity);
if ((mpi_err = MPI_File_set_atomicity(fh, 1)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_set_atomicity failed (%s)\n", mpi_err_str);
}
if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str);
}
if (VERBOSE_HI)
printf("After set_atomicity atomicity = %d\n", atomicity);
}
/* This barrier is not necessary but do it anyway. */
MPI_Barrier(MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("between MPI_Barrier and MPI_File_write_at\n");
}
/* ==================================================
* Each process calculates what to write but
* only process irank(0) writes.
* ==================================================*/
irank=0;
for (i=0; i < DIMSIZE; i++)
writedata[i] = irank*DIMSIZE + i;
mpi_off = irank*DIMSIZE;
/* Only one process writes */
if (mpi_rank==irank){
if (VERBOSE_HI){
PRINTID; printf("wrote %d bytes at %ld\n", DIMSIZE, (long)mpi_off);
}
if ((mpi_err = MPI_File_write_at(fh, mpi_off, writedata, DIMSIZE,
MPI_BYTE, &mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, DIMSIZE, mpi_err_str);
return 1;
};
};
/* Bcast the return code and */
/* make sure all writing are done before reading. */
MPI_Bcast(&mpi_err, 1, MPI_INT, irank, MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("MPI_Bcast: mpi_err = %d\n", mpi_err);
}
if (special_request & USEFSYNC){
/* ==================================================
* Do a file sync. A POSIX compliant filesystem
* should not need this.
* ==================================================*/
if (VERBOSE_HI)
printf("Apply MPI_File_sync\n");
/* call file_sync to force the write out */
if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_sync failed (%s)\n", mpi_err_str);
}
MPI_Barrier(MPI_COMM_WORLD);
/* call file_sync to force the write out */
if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_sync failed (%s)\n", mpi_err_str);
}
}
/* This barrier is not necessary because the Bcase or File_sync above */
/* should take care of it. Do it anyway. */
MPI_Barrier(MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("after MPI_Barrier\n");
}
/* ==================================================
* Each process reads what process 0 wrote and verify.
* ==================================================*/
irank=0;
mpi_off = irank*DIMSIZE;
if ((mpi_err = MPI_File_read_at(fh, mpi_off, readdata, DIMSIZE, MPI_BYTE,
&mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_read_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, DIMSIZE, mpi_err_str);
return 1;
};
for (i=0; i < DIMSIZE; i++){
expect_val = irank*DIMSIZE + i;
if (readdata[i] != expect_val){
PRINTID;
printf("read data[%d:%d] got %02x, expect %02x\n", irank, i,
readdata[i], expect_val);
nerrs++;
}
}
MPI_File_close(&fh);
if (VERBOSE_HI){
PRINTID;
printf("%d data errors detected\n", nerrs);
}
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
return nerrs;
}
int main(int argc, char **argv)
{
int *writebuf, *readbuf, i, mynod, nprocs, len, err;
char *filename;
MPI_Datatype newtype;
MPI_File fh;
MPI_Status status;
MPI_Info info;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: atmoicity <mpiparameter> -- -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
writebuf = (int *) malloc(BUFSIZE*sizeof(int));
readbuf = (int *) malloc(BUFSIZE*sizeof(int));
/* test atomicity of contiguous accesses */
/* initialize file to all zeros */
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
printf("\ntesting contiguous accesses\n");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
/* set atomicity to true */
err = MPI_File_set_atomicity(fh, 1);
if (err != MPI_SUCCESS) {
printf("Atomic mode not supported on this file system.\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Barrier(MPI_COMM_WORLD);
/* process 0 writes and others concurrently read. In atomic mode,
the data read must be either all old values or all new values; nothing
in between. */
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) { /* the rest must also be 0 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
printf("Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) { /* the rest must also be 10 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
printf("Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else printf("Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
/* repeat the same test with a noncontiguous filetype */
MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
printf("\ntesting noncontiguous accesses\n");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
MPI_File_set_atomicity(fh, 1);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
printf("Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
printf("Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else printf("Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Type_free(&newtype);
MPI_Info_free(&info);
free(writebuf);
free(readbuf);
free(filename);
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int buf[1024], amode, flag, mynod, len, i;
MPI_File fh;
MPI_Status status;
MPI_Datatype newtype;
MPI_Offset disp, offset;
MPI_Group group;
MPI_Datatype etype, filetype;
char datarep[25], *filename;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: misc <mpiparameter> -- -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_write(fh, buf, 1024, MPI_INT, &status);
MPI_File_sync(fh);
MPI_File_get_amode(fh, &amode);
if (!mynod) printf("testing MPI_File_get_amode\n");
if (amode != (MPI_MODE_CREATE | MPI_MODE_RDWR))
printf("amode is %d, should be %d\n\n", amode, MPI_MODE_CREATE |
MPI_MODE_RDWR);
MPI_File_get_atomicity(fh, &flag);
if (flag) printf("atomicity is %d, should be 0\n", flag);
if (!mynod) printf("setting atomic mode\n");
MPI_File_set_atomicity(fh, 1);
MPI_File_get_atomicity(fh, &flag);
if (!flag) printf("atomicity is %d, should be 1\n", flag);
MPI_File_set_atomicity(fh, 0);
if (!mynod) printf("reverting back to nonatomic mode\n");
MPI_Type_vector(10, 10, 20, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
MPI_File_set_view(fh, 1000, MPI_INT, newtype, "native", MPI_INFO_NULL);
if (!mynod) printf("testing MPI_File_get_view\n");
MPI_File_get_view(fh, &disp, &etype, &filetype, datarep);
if ((disp != 1000) || strcmp(datarep, "native"))
printf("disp = %I64, datarep = %s, should be 1000, native\n\n", disp, datarep);
if (!mynod) printf("testing MPI_File_get_byte_offset\n");
MPI_File_get_byte_offset(fh, 10, &disp);
if (disp != (1000+20*sizeof(int))) printf("byte offset = %I64, should be %d\n\n", disp, (int) (1000+20*sizeof(int)));
MPI_File_get_group(fh, &group);
if (!mynod) printf("testing MPI_File_set_size\n");
MPI_File_set_size(fh, 1000+15*sizeof(int));
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_sync(fh);
MPI_File_get_size(fh, &disp);
if (disp != 1000+15*sizeof(int)) printf("file size = %I64, should be %d\n\n", disp, (int) (1000+15*sizeof(int)));
if (!mynod) printf("seeking to eof and testing MPI_File_get_position\n");
MPI_File_seek(fh, 0, MPI_SEEK_END);
MPI_File_get_position(fh, &disp);
if (disp != 10) printf("file pointer posn = %I64, should be 10\n\n", disp);
if (!mynod) printf("testing MPI_File_get_byte_offset\n");
MPI_File_get_byte_offset(fh, disp, &offset);
if (offset != (1000+20*sizeof(int))) printf("byte offset = %I64, should be %d\n\n", offset, (int) (1000+20*sizeof(int)));
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) printf("testing MPI_File_seek with MPI_SEEK_CUR\n");
MPI_File_seek(fh, -10, MPI_SEEK_CUR);
MPI_File_get_position(fh, &disp);
MPI_File_get_byte_offset(fh, disp, &offset);
if (offset != 1000)
printf("file pointer posn in bytes = %I64, should be 1000\n\n", offset);
if (!mynod) printf("preallocating disk space up to 8192 bytes\n");
MPI_File_preallocate(fh, 8192);
if (!mynod) printf("closing the file and deleting it\n");
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Type_free(&newtype);
MPI_Type_free(&filetype);
MPI_Group_free(&group);
free(filename);
MPI_Finalize();
return 0;
}
void cache_flush_ind_all(int myid,
int numprocs,
int size,
char *filename)
{
char *buf;
MPI_File fh;
double time;
/* Calculate how much each processor must write */
int64_t ind_size = ceil(size / numprocs);
int64_t comp = 0;
char *ind_filename = NULL;
int ind_filename_size = 0;
/* We will assume that we are using less than 1,000,000 processors
* therefore add 1 for NULL char and 6 for each individual processor
* for 7 total */
ind_filename_size += strlen(filename) + 7;
if ((ind_filename = (char *) malloc(ind_filename_size)) == NULL)
{
fprintf(stderr, "cache_flush_ind_all: malloc ind_filename of size"
"%d failed\n", ind_filename_size);
}
sprintf(ind_filename, "%s%d", filename, myid);
ind_size = ind_size * 1024 * 1024; /* ind_size converted to MBytes */
assert(ind_size != 0);
if ((buf = (char *) malloc(MAX_BUFFER_SIZE * sizeof(char))) == NULL)
{
fprintf(stderr, "cache_flush_all: malloc buf of size %d failed\n",
MAX_BUFFER_SIZE);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_SELF, ind_filename,
MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
MPI_File_seek(fh, 0, MPI_SEEK_SET);
time = MPI_Wtime();
while (comp != ind_size)
{
if (ind_size - comp > MAX_BUFFER_SIZE)
{
comp += MAX_BUFFER_SIZE;
MPI_File_write(fh, buf, MAX_BUFFER_SIZE,
MPI_BYTE, MPI_STATUS_IGNORE);
}
else
{
int tmp_bytes = ind_size - comp;
comp += ind_size - comp;
MPI_File_write(fh, buf, tmp_bytes,
MPI_BYTE, MPI_STATUS_IGNORE);
}
}
MPI_File_sync(fh);
time = MPI_Wtime() - time;
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
#if 0
MPI_File_delete(ind_filename, MPI_INFO_NULL);
#endif
if (myid == 0)
{
fprintf(stderr,
"cache_flush_ind_all: File(s) written of "
"size %.1f MBytes\n"
"Time: %f secs Bandwidth: %f MBytes / sec\n\n",
comp*numprocs/1024.0/1024.0,
time, comp*numprocs/1024.0/1024.0/time);
}
MPI_Barrier(MPI_COMM_WORLD);
free(ind_filename);
free(buf);
}
/*
* access style is explicitly described as modifiable. values include
* read_once, read_mostly, write_once, write_mostlye, random
*
*
*/
int main(int argc, char *argv[])
{
int errs = 0, err;
int buf[10];
int rank;
MPI_Comm comm;
MPI_Status status;
MPI_File fh;
MPI_Info infoin, infoout;
char value[1024];
int flag, count;
MTest_Init(&argc, &argv);
comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm, &rank);
MPI_Info_create(&infoin);
MPI_Info_set(infoin, (char *) "access_style", (char *) "write_once,random");
MPI_File_open(comm, (char *) "testfile", MPI_MODE_RDWR | MPI_MODE_CREATE, infoin, &fh);
buf[0] = rank;
err = MPI_File_write_ordered(fh, buf, 1, MPI_INT, &status);
if (err) {
errs++;
MTestPrintError(err);
}
MPI_Info_set(infoin, (char *) "access_style", (char *) "read_once");
err = MPI_File_seek_shared(fh, 0, MPI_SEEK_SET);
if (err) {
errs++;
MTestPrintError(err);
}
err = MPI_File_set_info(fh, infoin);
if (err) {
errs++;
MTestPrintError(err);
}
MPI_Info_free(&infoin);
buf[0] = -1;
err = MPI_File_read_ordered(fh, buf, 1, MPI_INT, &status);
if (err) {
errs++;
MTestPrintError(err);
}
MPI_Get_count(&status, MPI_INT, &count);
if (count != 1) {
errs++;
printf("Expected to read one int, read %d\n", count);
}
if (buf[0] != rank) {
errs++;
printf("Did not read expected value (%d)\n", buf[0]);
}
err = MPI_File_get_info(fh, &infoout);
if (err) {
errs++;
MTestPrintError(err);
}
MPI_Info_get(infoout, (char *) "access_style", 1024, value, &flag);
/* Note that an implementation is allowed to ignore the set_info,
* so we'll accept either the original or the updated version */
if (!flag) {
;
/*
* errs++;
* printf("Access style hint not saved\n");
*/
} else {
if (strcmp(value, "read_once") != 0 && strcmp(value, "write_once,random") != 0) {
errs++;
printf("value for access_style unexpected; is %s\n", value);
}
}
MPI_Info_free(&infoout);
err = MPI_File_close(&fh);
if (err) {
errs++;
MTestPrintError(err);
}
MPI_Barrier(comm);
MPI_Comm_rank(comm, &rank);
if (rank == 0) {
err = MPI_File_delete((char *) "testfile", MPI_INFO_NULL);
if (err) {
errs++;
MTestPrintError(err);
}
}
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
void cache_flush_all(int myid,
int numprocs,
int size,
char *filename)
{
char *buf;
MPI_File fh;
double time;
/* Calculate how much each processor must write */
int64_t ind_size = ceil(size / numprocs);
int64_t comp = 0;
ind_size = ind_size * 1024 * 1024; /* ind_size converted to MBytes */
assert(ind_size != 0);
if ((buf = (char *) malloc(MAX_BUFFER_SIZE * sizeof(char))) == NULL)
{
fprintf(stderr, "cache_flush_all: malloc buf of size %d failed\n",
MAX_BUFFER_SIZE);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, ind_size * myid, MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
MPI_File_seek(fh, 0, MPI_SEEK_SET);
time = MPI_Wtime();
while (comp != ind_size)
{
if (ind_size - comp > MAX_BUFFER_SIZE)
{
comp += MAX_BUFFER_SIZE;
MPI_File_write(fh, buf, MAX_BUFFER_SIZE,
MPI_BYTE, MPI_STATUS_IGNORE);
}
else
{
int tmp_bytes = ind_size - comp;
comp += ind_size - comp;
MPI_File_write(fh, buf, tmp_bytes,
MPI_BYTE, MPI_STATUS_IGNORE);
}
}
free(buf);
MPI_File_sync(fh);
time = MPI_Wtime() - time;
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
#if 0
if (myid == 0)
{
MPI_File_delete(filename, MPI_INFO_NULL);
fprintf(stderr,
"cache_flush_all: File %s written/deleted of "
"size %.1f MBytes\n"
"Time: %f secs Bandwidth: %f MBytes / sec\n\n",
filename, comp*numprocs/1024.0/1024.0,
time, comp*numprocs/1024.0/1024.0/time);
}
MPI_Barrier(MPI_COMM_WORLD);
#endif
}
int main(int argc, char *argv[])
{
MPI_File fh;
char emsg[MPI_MAX_ERROR_STRING];
int emsglen, err, ec, errs = 0;
int amode, rank;
char *name = 0;
MPI_Status st;
int outbuf[BUFLEN], inbuf[BUFLEN];
MTest_Init(&argc, &argv);
name = "not-a-file-to-use";
/* Try to open a file that does/should not exist */
/* Note that no error message should be printed by MPI_File_open,
* even when there is an error */
err = MPI_File_open(MPI_COMM_WORLD, name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (err == MPI_SUCCESS) {
errs++;
printf("Did not return error when opening a file that does not exist\n");
MPI_File_close(&fh);
MPI_File_delete(name, MPI_INFO_NULL);
} else {
MPI_Error_class(err, &ec);
MPI_Error_string(err, emsg, &emsglen);
MTestPrintfMsg(2, "Error msg from open: %s\n", emsg);
if (ec != MPI_ERR_NO_SUCH_FILE && ec != MPI_ERR_IO) {
errs++;
printf("Did not return class ERR_NO_SUCH_FILE or ERR_IO\n");
printf("Returned class %d, message %s\n", ec, emsg);
}
}
/* Now, create a file, write data into it; close, then reopen as
* read only and try to write to it */
amode = MPI_MODE_CREATE | MPI_MODE_WRONLY;
name = "mpio-test-openerrs";
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to open file for writing", err);
} else {
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
memset(outbuf, 'A' + rank, BUFLEN);
err = MPI_File_write_at(fh, rank * BUFLEN, outbuf, BUFLEN, MPI_BYTE, &st);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to write file", err);
}
MPI_File_close(&fh);
}
/* Now, open for read only, and delete on close */
amode = MPI_MODE_RDONLY | MPI_MODE_DELETE_ON_CLOSE;
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to reopen file for reading", err);
} else {
/* Try to read it */
/* Clear buffer before reading into it */
memset(inbuf, 0, BUFLEN);
err = MPI_File_read_at(fh, rank * BUFLEN, inbuf, BUFLEN, MPI_BYTE, &st);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to read file", err);
}
/* Try to write it (should fail) */
err = MPI_File_write_at(fh, rank * BUFLEN, outbuf, BUFLEN, MPI_BYTE, &st);
if (err == MPI_SUCCESS) {
errs++;
printf("Write operation succeeded to read-only file\n");
} else {
/* Look at error class */
MPI_Error_class(err, &ec);
if (ec != MPI_ERR_READ_ONLY && ec != MPI_ERR_ACCESS) {
errs++;
printf("Unexpected error class %d when writing to read-only file\n", ec);
MTestPrintErrorMsg("Error msg is: ", err);
}
}
err = MPI_File_close(&fh);
if (err) {
errs++;
MTestPrintErrorMsg("Failed to close", err);
}
/* No MPI_Barrier is required here */
/*
* Test open without CREATE to see if DELETE_ON_CLOSE worked.
* This should fail if file was deleted correctly.
*/
amode = MPI_MODE_RDONLY;
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err == MPI_SUCCESS) {
errs++;
printf("File was not deleted!\n");
MPI_File_close(&fh);
} else {
MPI_Error_class(err, &ec);
if (ec != MPI_ERR_NO_SUCH_FILE && ec != MPI_ERR_IO) {
errs++;
printf("Did not return class ERR_NO_SUCH_FILE or ERR_IO\n");
printf("Returned class %d, message %s\n", ec, emsg);
}
}
}
/* */
/* Find out how many errors we saw */
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
// this is currently slower than it could be because we write in chunks (data.size()..) with barriers in between - instead we could take care of correct format of matrix on the read-in side and write everything at once, or assume same format of data and only do one barrier
void Storage::SaveDataFloatMPIBin(char* filename, vector<vector<float> > data, int mpiRank, int mpiSize, MPI_Comm comm)
{
MPI_File fh;
// assumes specific data distribution for processes
MPI_File_delete(filename,MPI_INFO_NULL);
MPI_File_open(comm,filename,MPI_MODE_RDWR|MPI_MODE_CREATE,MPI_INFO_NULL,&fh);
int globalOffset = 0;
// (!) Currently assumes same size of all items in data
int size = 0;
for(int i=0;i<data.size();i++) // assumes same nr items each process
{
size+= data[i].size();
}
vector<float> tempData(size);
int index=0;
for(int i=0;i<data.size();i++) // assumes same nr items each process
{
for(int j=0;j<data[i].size();j++)
{
tempData[index] = data[i][j];
index++;
}
}
vector<int> allSizes(mpiSize);
MPI_Allgather(&size,1,MPI_INT,&allSizes[0],1,MPI_INT,comm);
int startPos = 0;
for(int j=0;j<mpiRank;j++)
startPos+=allSizes[j];
//for(int i=0;i<data.size();i++) // assumes same nr items each process
//{
// /*int size = data[i].size(); // each item can be of different size
// vector<int> allSizes(mpiSize);
// MPI_Allgather(&size,1,MPI_INT,&allSizes[0],1,MPI_INT,comm);
// int startPos = 0;
// for(int j=0;j<mpiRank;j++)
// startPos+=allSizes[j];*/
// MPI_Status status;
// MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&data[i][0],data[i].size(),MPI_FLOAT,&status);
// //if(i==0)
// //MPI_File_write_at(fh,(startPos+globalStartPos),&data[i][0],data[i].size(),MPI_FLOAT,&status);
// for(int j=0;j<mpiSize;j++)
// globalOffset+=allSizes[j];
// //globalOffset = 0;
// //for(int j=mpiRank;j<mpiSize;j++)
// // globalOffset+=allSizes[j];
//}
MPI_Status status;
if(size>0)
MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&tempData[0],size,MPI_FLOAT,&status);//&data[0][0],size,MPI_FLOAT,&status);
//MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&data[0][0],size,MPI_FLOAT,&status);
MPI_File_close(&fh);
}
int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, const char *msg, int verbose)
{
MPI_Datatype typevec, newtype, t[3];
int *buf, i, b[3], errcode, errors=0;
MPI_File fh;
MPI_Aint d[3];
MPI_Status status;
int SIZE = (STARTING_SIZE/nprocs)*nprocs;
MPI_Info info;
if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg);
buf = (int *) malloc(SIZE*sizeof(int));
if (buf == NULL) {
perror("test_file");
MPI_Abort(MPI_COMM_WORLD, -1);
}
if (cb_hosts != NULL ) {
MPI_Info_create(&info);
MPI_Info_set(info, "cb_config_list", cb_hosts);
} else {
info = MPI_INFO_NULL;
}
MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec);
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod*sizeof(int);
d[2] = SIZE*sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
errcode = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
errcode = MPI_File_write_all(fh, buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "nc mem - nc file: MPI_File_write_all");
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all");
}
/* the verification for N compute nodes is tricky. Say we have 3
* processors.
* process 0 sees: 0 -1 -1 3 -1 -1 ...
* process 1 sees: -1 34 -1 -1 37 -1 ...
* process 2 sees: -1 -1 68 -1 -1 71 ... */
/* verify those leading -1s exist if they should */
for (i=0; i<mynod; i++ ) {
if ( buf[i] != -1 ) {
if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
errors++;
}
}
/* now the modulo games are hairy. processor 0 sees real data in the 0th,
* 3rd, 6th... elements of the buffer (assuming nprocs==3 ). proc 1 sees
* the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */
for(/* 'i' set in above loop */; i<SIZE; i++) {
if ( ((i-mynod)%nprocs) && buf[i] != -1) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
errors++;
}
if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], SEEDER(mynod,i,SIZE));
errors++;
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int),
buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "nc mem - c file: MPI_File_write_at_all");
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int),
buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "nc mem - c file: MPI_File_read_at_all");
/* just like as above */
for (i=0; i<mynod; i++ ) {
if ( buf[i] != -1 ) {
if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
errors++;
}
}
for(/* i set in above loop */; i<SIZE; i++) {
if ( ((i-mynod)%nprocs) && buf[i] != -1) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
errors++;
}
if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], SEEDER(mynod,i,SIZE) );
errors++;
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE);
errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "c mem - nc file: MPI_File_write_all");
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "c mem - nc file: MPI_File_read_at_all");
/* same crazy checking */
for (i=0; i<SIZE; i++) {
if (buf[i] != SEEDER(mynod, i, SIZE)) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE));
errors++;
}
}
MPI_File_close(&fh);
MPI_Type_free(&newtype);
free(buf);
if (info != MPI_INFO_NULL) MPI_Info_free(&info);
return errors;
}
void
IOR_Delete_MPIIO(char * testFileName, IOR_param_t * param)
{
MPI_CHECK(MPI_File_delete(testFileName, (MPI_Info)MPI_INFO_NULL),
"cannot delete file");
} /* IOR_Delete_MPIIO() */
int
main(int argc, char* argv[])
{
int n, my_rank;
int array_of_subsizes[NDIMS], array_of_starts[NDIMS], array_of_sizes[NDIMS];
int size = 4;
int sqrtn;
int ln;
MPI_Datatype filetype, memtype;
MPI_File fh;
char hdr[128];
int header_bytes;
unsigned char *cur;
char name[128];
int resultlen;
int ret;
int i, j;
/* Initialize MPI. */
MPI_Init(&argc, &argv);
MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
/* Learn my rank and the total number of processors. */
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
MPI_Comm_size(MPI_COMM_WORLD, &n);
/* Speak! */
MPI_Get_processor_name(name, &resultlen);
printf("process %d running on %s\n", my_rank, name);
/* Set up our values. */
sqrtn = (int)sqrt(n);
ln = size/sqrtn;
printf("n = %d, sqrtn = %d, ln = %d storage = %d\n", n, sqrtn, ln, (ln + 2) * (ln + 2));
/* Allocation storage. */
if (!(cur = calloc((ln + 2) * (ln + 2), 1)))
return ERR;
/* Initialize data. */
for (i = 1; i < ln + 1; i++)
for (j = 1; j < ln + 1; j++)
cur[i * (ln + 2) + j] = my_rank;
/* Create a subarray type for the file. */
array_of_sizes[0] = array_of_sizes[1] = size;
array_of_subsizes[0] = array_of_subsizes[1] = ln;
array_of_starts[0] = my_rank/sqrtn * ln;
array_of_starts[1] = (my_rank % sqrtn) * ln;
if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &filetype)))
MPIERR(ret);
if ((ret = MPI_Type_commit(&filetype)))
MPIERR(ret);
/* Create a subarray type for memory. */
array_of_sizes[0] = array_of_sizes[1] = ln + 2;
array_of_subsizes[0] = array_of_subsizes[1] = ln;
array_of_starts[0] = array_of_starts[1] = 1;
if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &memtype)))
MPIERR(ret);
if ((ret = MPI_Type_commit(&memtype)))
MPIERR(ret);
MPI_File_delete(FILE_NAME, MPI_INFO_NULL);
if ((ret = MPI_File_open(MPI_COMM_WORLD, FILE_NAME, MPI_MODE_CREATE|MPI_MODE_RDWR,
MPI_INFO_NULL, &fh)))
MPIERR(ret);
/* Create header info, and have process 0 write it to the file. */
sprintf(hdr, "P5\n%d %d\n255\n", size, size);
header_bytes = strlen(hdr);
if ((ret = MPI_File_write_all(fh, hdr, header_bytes, MPI_BYTE, MPI_STATUS_IGNORE)))
MPIERR(ret);
/* Set the file view to translate our memory data into the file's data layout. */
MPI_File_set_view(fh, header_bytes, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
/* Write the output. */
MPI_File_write(fh, cur, 1, memtype, MPI_STATUS_IGNORE);
if ((ret = MPI_File_close(&fh)))
MPIERR(ret);
MPI_Finalize();
return 0;
}
/**
* \brief Finalization function of measure function
* measure_MPI_IO_write_large_file_once().
*
* Only one process is active. It writes once to a file.
*
* Since SKaMPI measurement functions are not allowed to use MPI_Offset
* parameters, it is impossible to tell an init_-routine to create a file
* which is larger than \f$2^{\mbox{\texttt{sizeof(int)}}-1}-1\f$ directly. As
* a preliminary solution we use a parameter (<tt>power</tt>) which commits the
* power to 2 as an indicator for the file size.
*
* Remark concerning the <em>HP XC6000</em>:<br>
* Measurements showed that there is no significant difference between MPI-API
* and POSIX-API I/O accesses, if files are larger than 1MB. Thus there is no
* choice between these two modes like in measure_MPI_IO_read_file_once(),
* which makes type compatibilty problems much easier. Only MPI-API is
* supported.
*
* \param[in] power size of memory buffer; 2 to the power of `power' <tt>MPI_BYTE</tt>s
* \param[in] create_flag write into existing file (FALSE) or create it (TRUE)
*
* \return void
*/
void finalize_MPI_IO_write_large_file_once (int power, int create_flag){
MPI_File_delete (io_filename, MPI_INFO_NULL);
mpi_free (io_filename);
}
