static MPI_Offset get_filesize(const char *filename) { int mpierr; MPI_File fd; MPI_Offset filesize; #ifndef H5_HAVE_MPI_GET_SIZE h5_stat_t stat_buf; #endif #ifdef H5_HAVE_MPI_GET_SIZE mpierr = MPI_File_open(MPI_COMM_SELF, (char*)filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fd); VRFY((mpierr == MPI_SUCCESS), ""); mpierr = MPI_File_get_size(fd, &filesize); VRFY((mpierr == MPI_SUCCESS), ""); mpierr = MPI_File_close(&fd); VRFY((mpierr == MPI_SUCCESS), ""); #else /* Some systems (only SGI Altix Propack 4 so far) doesn't return correct * file size for MPI_File_get_size. Use stat instead. */ if((mpierr=HDstat(filename, &stat_buf))<0) VRFY((mpierr == MPI_SUCCESS), ""); /* Hopefully this casting is safe */ filesize = (MPI_Offset)(stat_buf.st_size); #endif return(filesize); }
/*------------------------------------------------------------------------- * Function: is_sparse * * Purpose: Determines if the file system of the current working * directory supports holes. * * Return: Success: Non-zero if holes are supported; zero * otherwise. * * Failure: zero * * Programmer: Robb Matzke * Wednesday, July 15, 1998 * *------------------------------------------------------------------------- */ static int is_sparse(void) { int fd; h5_stat_t sb; if ((fd=HDopen("x.h5", O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) return 0; if (HDlseek(fd, (off_t)(1024*1024), SEEK_SET)!=1024*1024) return 0; if (5!=HDwrite(fd, "hello", (size_t)5)) return 0; if (HDclose(fd) < 0) return 0; if (HDstat("x.h5", &sb) < 0) return 0; if (HDremove("x.h5") < 0) return 0; #ifdef H5_HAVE_STAT_ST_BLOCKS return ((unsigned long)sb.st_blocks*512 < (unsigned long)sb.st_size); #else return (0); #endif }
static void fill_with_random_data(Bytef *src, uLongf src_len) { register unsigned u; h5_stat_t stat_buf; if (HDstat("/dev/urandom", &stat_buf) == 0) { uLongf len = src_len; Bytef *buf = src; int fd = HDopen("/dev/urandom", O_RDONLY, 0); HDfprintf(stdout, "Using /dev/urandom for random data\n"); if (fd < 0) error(HDstrerror(errno)); for (;;) { ssize_t rc = HDread(fd, buf, src_len); if (rc == -1) error(HDstrerror(errno)); if (rc == (ssize_t)len) break; buf += rc; len -= rc; } } else { HDfprintf(stdout, "Using random() for random data\n"); for (u = 0; u < src_len; ++u) src[u] = (Bytef)(0xff & HDrandom()); } if (compress_percent) { unsigned long s = src_len * compress_percent / 100; HDmemset(src, '\0', s); } }
/*------------------------------------------------------------------------- * Function: h5_get_file_size * * Purpose: Get the current size of a file (in bytes) * * Return: Success: Size of file in bytes * Failure: -1 * * Programmer: Quincey Koziol * Saturday, March 22, 2003 * *------------------------------------------------------------------------- */ h5_stat_size_t h5_get_file_size(const char *filename, hid_t fapl) { char temp[2048]; /* Temporary buffer for file names */ h5_stat_t sb; /* Structure for querying file info */ int j = 0; if(fapl == H5P_DEFAULT) { /* Get the file's statistics */ if(0 == HDstat(filename, &sb)) return((h5_stat_size_t)sb.st_size); } /* end if */ else { hid_t driver; /* VFD used for file */ /* Get the driver used when creating the file */ if((driver = H5Pget_driver(fapl)) < 0) return(-1); /* Check for simple cases */ if(driver == H5FD_SEC2 || driver == H5FD_STDIO || driver == H5FD_CORE || #ifdef H5_HAVE_PARALLEL driver == H5FD_MPIO || #endif /* H5_HAVE_PARALLEL */ #ifdef H5_HAVE_WINDOWS driver == H5FD_WINDOWS || #endif /* H5_HAVE_WINDOWS */ #ifdef H5_HAVE_DIRECT driver == H5FD_DIRECT || #endif /* H5_HAVE_DIRECT */ driver == H5FD_LOG) { /* Get the file's statistics */ if(0 == HDstat(filename, &sb)) return((h5_stat_size_t)sb.st_size); } /* end if */ else if(driver == H5FD_MULTI) { H5FD_mem_t mt; h5_stat_size_t tot_size = 0; HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { /* Create the filename to query */ HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); /* Check for existence of file */ if(0 == HDaccess(temp, F_OK)) { /* Get the file's statistics */ if(0 != HDstat(temp, &sb)) return(-1); /* Add to total size */ tot_size += (h5_stat_size_t)sb.st_size; } /* end if */ } /* end for */ /* Return total size */ return(tot_size); } /* end if */ else if(driver == H5FD_FAMILY) { h5_stat_size_t tot_size = 0; /* Try all filenames possible, until we find one that's missing */ for(j = 0; /*void*/; j++) { /* Create the filename to query */ HDsnprintf(temp, sizeof temp, filename, j); /* Check for existence of file */ if(HDaccess(temp, F_OK) < 0) break; /* Get the file's statistics */ if(0 != HDstat(temp, &sb)) return(-1); /* Add to total size */ tot_size += (h5_stat_size_t)sb.st_size; } /* end for */ /* Return total size */ return(tot_size); } /* end if */ else { HDassert(0 && "Unknown VFD!"); } /* end else */ } /* end else */ return(-1); } /* end get_file_size() */
/*------------------------------------------------------------------------- * Function: h5_fixname * * Purpose: Create a file name from a file base name like `test' and * return it through the FULLNAME (at most SIZE characters * counting the null terminator). The full name is created by * prepending the contents of HDF5_PREFIX (separated from the * base name by a slash) and appending a file extension based on * the driver supplied, resulting in something like * `ufs:/u/matzke/test.h5'. * * Return: Success: The FULLNAME pointer. * * Failure: NULL if BASENAME or FULLNAME is the null * pointer or if FULLNAME isn't large enough for * the result. * * Programmer: Robb Matzke * Thursday, November 19, 1998 * *------------------------------------------------------------------------- */ char * h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size) { const char *prefix = NULL; const char *suffix = ".h5"; /* suffix has default */ char *ptr, last = '\0'; size_t i, j; hid_t driver = -1; int isppdriver = 0; /* if the driver is MPI parallel */ if (!base_name || !fullname || size < 1) return NULL; HDmemset(fullname, 0, size); /* figure out the suffix */ if(H5P_DEFAULT != fapl) { if((driver = H5Pget_driver(fapl)) < 0) return NULL; if(H5FD_FAMILY == driver) suffix = "%05d.h5"; else if (H5FD_MULTI == driver) suffix = NULL; } /* Must first check fapl is not H5P_DEFAULT (-1) because H5FD_XXX * could be of value -1 if it is not defined. */ isppdriver = H5P_DEFAULT != fapl && (H5FD_MPIO==driver); /* Check HDF5_NOCLEANUP environment setting. * (The #ifdef is needed to prevent compile failure in case MPI is not * configured.) */ if(isppdriver) { #ifdef H5_HAVE_PARALLEL if(getenv_all(MPI_COMM_WORLD, 0, "HDF5_NOCLEANUP")) SetTestNoCleanup(); #endif /* H5_HAVE_PARALLEL */ } else { if(HDgetenv("HDF5_NOCLEANUP")) SetTestNoCleanup(); } /* Check what prefix to use for test files. Process HDF5_PARAPREFIX and * HDF5_PREFIX. * Use different ones depending on parallel or serial driver used. * (The #ifdef is needed to prevent compile failure in case MPI is not * configured.) */ if(isppdriver) { #ifdef H5_HAVE_PARALLEL /* * For parallel: * First use command line option, then the environment * variable, then try the constant */ static int explained = 0; prefix = (paraprefix ? paraprefix : getenv_all(MPI_COMM_WORLD, 0, "HDF5_PARAPREFIX")); if (!prefix && !explained) { /* print hint by process 0 once. */ int mpi_rank; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); if (mpi_rank == 0) printf("*** Hint ***\n" "You can use environment variable HDF5_PARAPREFIX to " "run parallel test files in a\n" "different directory or to add file type prefix. E.g.,\n" " HDF5_PARAPREFIX=pfs:/PFS/user/me\n" " export HDF5_PARAPREFIX\n" "*** End of Hint ***\n"); explained = TRUE; #ifdef HDF5_PARAPREFIX prefix = HDF5_PARAPREFIX; #endif /* HDF5_PARAPREFIX */ } #endif /* H5_HAVE_PARALLEL */ } else { /* * For serial: * First use the environment variable, then try the constant */ prefix = HDgetenv("HDF5_PREFIX"); #ifdef HDF5_PREFIX if (!prefix) prefix = HDF5_PREFIX; #endif /* HDF5_PREFIX */ } /* Prepend the prefix value to the base name */ if (prefix && *prefix) { if (isppdriver) { /* This is a parallel system */ char *subdir; if (!HDstrcmp(prefix, HDF5_PARAPREFIX)) { /* * If the prefix specifies the HDF5_PARAPREFIX directory, then * default to using the "/tmp/$USER" or "/tmp/$LOGIN" * directory instead. */ char *user, *login; user = HDgetenv("USER"); login = HDgetenv("LOGIN"); subdir = (user ? user : login); if (subdir) { for (i = 0; i < size && prefix[i]; i++) fullname[i] = prefix[i]; fullname[i++] = '/'; for (j = 0; i < size && subdir[j]; ++i, ++j) fullname[i] = subdir[j]; } } if (!fullname[0]) { /* We didn't append the prefix yet */ HDstrncpy(fullname, prefix, size); fullname[size -1] = '\0'; } if (HDstrlen(fullname) + HDstrlen(base_name) + 1 < size) { /* * Append the base_name with a slash first. Multiple * slashes are handled below. */ h5_stat_t buf; if (HDstat(fullname, &buf) < 0) /* The directory doesn't exist just yet */ if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) /* * We couldn't make the "/tmp/${USER,LOGIN}" * subdirectory. Default to PREFIX's original * prefix value. */ HDstrcpy(fullname, prefix); HDstrcat(fullname, "/"); HDstrcat(fullname, base_name); } else { /* Buffer is too small */ return NULL; } } else { if (HDsnprintf(fullname, size, "%s/%s", prefix, base_name) == (int)size) /* Buffer is too small */ return NULL; } } else if (HDstrlen(base_name) >= size) { /* Buffer is too small */ return NULL; } else { HDstrcpy(fullname, base_name); } /* Append a suffix */ if (suffix) { if (HDstrlen(fullname) + HDstrlen(suffix) >= size) return NULL; HDstrcat(fullname, suffix); } /* Remove any double slashes in the filename */ for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { if (*ptr != '/' || last != '/') fullname[j++] = *ptr; last = *ptr; } return fullname; }
/* * Function: sio_create_filename * Purpose: Create a new filename to write to. Determine the correct * suffix to append to the filename by the type of I/O we're * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if * USER or LOGIN are specified in the environment. * Return: Pointer to filename or NULL * Programmer: Bill Wendling, 21. November 2001 * Modifications: Support for file drivers. Christian Chilan, April, 2008 */ static char * sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, parameters *param) { const char *prefix, *suffix=""; char *ptr, last = '\0'; size_t i, j; vfdtype vfd; vfd = param->vfd; if (!base_name || !fullname || size < 1) return NULL; memset(fullname, 0, size); switch (iot) { case POSIXIO: suffix = ".posix"; break; case HDF5: suffix = ".h5"; if (vfd == family) suffix = "%05d.h5"; else if (vfd == multi) suffix = NULL; break; default: /* unknown request */ HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); HDassert(0 && "Unknown IO type"); break; } /* First use the environment variable and then try the constant */ prefix = HDgetenv("HDF5_PREFIX"); #ifdef HDF5_PREFIX if (!prefix) prefix = HDF5_PREFIX; #endif /* HDF5_PREFIX */ /* Prepend the prefix value to the base name */ if (prefix && *prefix) { /* If the prefix specifies the HDF5_PREFIX directory, then * default to using the "/tmp/$USER" or "/tmp/$LOGIN" * directory instead. */ register char *user, *login, *subdir; user = HDgetenv("USER"); login = HDgetenv("LOGIN"); subdir = (user ? user : login); if (subdir) { for (i = 0; i < size-1 && prefix[i]; i++) fullname[i] = prefix[i]; fullname[i++] = '/'; for (j = 0; i < size && subdir[j]; i++, j++) fullname[i] = subdir[j]; } else { /* We didn't append the prefix yet */ HDstrncpy(fullname, prefix, size); fullname[size - 1] = '\0'; } if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { /* Append the base_name with a slash first. Multiple slashes are * handled below. */ h5_stat_t buf; if (HDstat(fullname, &buf) < 0) /* The directory doesn't exist just yet */ if (HDmkdir(fullname, 0755) < 0 && errno != EEXIST) { /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. * Default to PREFIX's original prefix value. */ HDstrcpy(fullname, prefix); } HDstrcat(fullname, "/"); HDstrcat(fullname, base_name); } else { /* Buffer is too small */ return NULL; } } else if (strlen(base_name) >= size) { /* Buffer is too small */ return NULL; } else { HDstrcpy(fullname, base_name); } /* Append a suffix */ if (suffix) { if (HDstrlen(fullname) + HDstrlen(suffix) >= size) return NULL; HDstrcat(fullname, suffix); } /* Remove any double slashes in the filename */ for (ptr = fullname, i = j = 0; ptr && (i < size); i++, ptr++) { if (*ptr != '/' || last != '/') fullname[j++] = *ptr; last = *ptr; } return fullname; }
/* * Verify that MPI_Offset exceeding 2**31 can be computed correctly. * Print any failure as information only, not as an error so that this * won't abort the remaining test or other separated tests. * * Test if MPIO can write file from under 2GB to over 2GB and then * from under 4GB to over 4GB. * Each process writes 1MB in round robin fashion. * Then reads the file back in by reverse order, that is process 0 * reads the data of process n-1 and vice versa. */ static int test_mpio_gb_file(char *filename) { int mpi_size, mpi_rank; MPI_Info info = MPI_INFO_NULL; int mrc; MPI_File fh; int i, j, n; int vrfyerrs; int writerrs; /* write errors */ int nerrs; int ntimes; /* how many times */ char *buf = NULL; char expected; MPI_Offset size; MPI_Offset mpi_off; MPI_Offset mpi_off_old; MPI_Status mpi_stat; h5_stat_t stat_buf; int is_signed, sizeof_mpi_offset; nerrs = 0; /* set up MPI parameters */ MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); if (VERBOSE_MED) printf("MPI_Offset range test\n"); /* figure out the signness and sizeof MPI_Offset */ mpi_off = 0; is_signed = ((MPI_Offset)(mpi_off - 1)) < 0; sizeof_mpi_offset = (int)(sizeof(MPI_Offset)); /* * Verify the sizeof MPI_Offset and correctness of handling multiple GB * sizes. */ if (MAINPROCESS){ /* only process 0 needs to check it*/ printf("MPI_Offset is %s %d bytes integeral type\n", is_signed ? "signed" : "unsigned", (int)sizeof(MPI_Offset)); if (sizeof_mpi_offset <= 4 && is_signed){ printf("Skipped 2GB range test " "because MPI_Offset cannot support it\n"); }else { /* verify correctness of assigning 2GB sizes */ mpi_off = 2 * 1024 * (MPI_Offset)MB; INFO((mpi_off>0), "2GB OFFSET assignment no overflow"); INFO((mpi_off-1)==TWO_GB_LESS1, "2GB OFFSET assignment succeed"); /* verify correctness of increasing from below 2 GB to above 2GB */ mpi_off = TWO_GB_LESS1; for (i=0; i < 3; i++){ mpi_off_old = mpi_off; mpi_off = mpi_off + 1; /* no overflow */ INFO((mpi_off>0), "2GB OFFSET increment no overflow"); /* correct inc. */ INFO((mpi_off-1)==mpi_off_old, "2GB OFFSET increment succeed"); } } if (sizeof_mpi_offset <= 4){ printf("Skipped 4GB range test " "because MPI_Offset cannot support it\n"); }else { /* verify correctness of assigning 4GB sizes */ mpi_off = 4 * 1024 * (MPI_Offset)MB; INFO((mpi_off>0), "4GB OFFSET assignment no overflow"); INFO((mpi_off-1)==FOUR_GB_LESS1, "4GB OFFSET assignment succeed"); /* verify correctness of increasing from below 4 GB to above 4 GB */ mpi_off = FOUR_GB_LESS1; for (i=0; i < 3; i++){ mpi_off_old = mpi_off; mpi_off = mpi_off + 1; /* no overflow */ INFO((mpi_off>0), "4GB OFFSET increment no overflow"); /* correct inc. */ INFO((mpi_off-1)==mpi_off_old, "4GB OFFSET increment succeed"); } } } /* * Verify if we can write to a file of multiple GB sizes. */ if (VERBOSE_MED) printf("MPIO GB file test %s\n", filename); if (sizeof_mpi_offset <= 4){ printf("Skipped GB file range test " "because MPI_Offset cannot support it\n"); }else{ buf = malloc(MB); VRFY((buf!=NULL), "malloc succeed"); /* open a new file. Remove it first in case it exists. */ /* Must delete because MPI_File_open does not have a Truncate mode. */ /* Don't care if it has error. */ MPI_File_delete(filename, MPI_INFO_NULL); MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */ mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_OPEN"); printf("MPIO GB file write test %s\n", filename); /* instead of writing every bytes of the file, we will just write * some data around the 2 and 4 GB boundaries. That should cover * potential integer overflow and filesystem size limits. */ writerrs = 0; for (n=2; n <= 4; n+=2){ ntimes = GB/MB*n/mpi_size + 1; for (i=ntimes-2; i <= ntimes; i++){ mpi_off = (i*mpi_size + mpi_rank)*(MPI_Offset)MB; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: write to mpi_off=%016llx, %lld\n", mpi_rank, mpi_off, mpi_off); /* set data to some trivial pattern for easy verification */ for (j=0; j<MB; j++) *(buf+j) = i*mpi_size + mpi_rank; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: writing %d bytes at offset %lld\n", mpi_rank, MB, mpi_off); mrc = MPI_File_write_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat); INFO((mrc==MPI_SUCCESS), "GB size file write"); if (mrc!=MPI_SUCCESS) writerrs++; } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync after writes"); /* * Verify if we can read the multiple GB file just created. */ /* open it again to verify the data written */ /* but only if there was no write errors */ printf("MPIO GB file read test %s\n", filename); if (errors_sum(writerrs)>0){ printf("proc %d: Skip read test due to previous write errors\n", mpi_rank); goto finish; } mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); /* Only read back parts of the file that have been written. */ for (n=2; n <= 4; n+=2){ ntimes = GB/MB*n/mpi_size + 1; for (i=ntimes-2; i <= ntimes; i++){ mpi_off = (i*mpi_size + (mpi_size - mpi_rank - 1))*(MPI_Offset)MB; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: read from mpi_off=%016llx, %lld\n", mpi_rank, mpi_off, mpi_off); mrc = MPI_File_read_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat); INFO((mrc==MPI_SUCCESS), "GB size file read"); expected = i*mpi_size + (mpi_size - mpi_rank - 1); vrfyerrs=0; for (j=0; j<MB; j++){ if ((*(buf+j) != expected) && (vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)){ printf("proc %d: found data error at [%ld+%d], expect %d, got %d\n", mpi_rank, (long)mpi_off, j, expected, *(buf+j)); } } if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) printf("proc %d: [more errors ...]\n", mpi_rank); nerrs += vrfyerrs; } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); /* * one more sync to ensure all processes have done reading * before ending this test. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync before leaving test"); /* * Check if MPI_File_get_size works correctly. Some systems (only SGI Altix * Propack 4 so far) return wrong file size. It can be avoided by reconfiguring * with "--disable-mpi-size". */ #ifdef H5_HAVE_MPI_GET_SIZE printf("Test if MPI_File_get_size works correctly with %s\n", filename); mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); if (MAINPROCESS){ /* only process 0 needs to check it*/ mrc = MPI_File_get_size(fh, &size); VRFY((mrc==MPI_SUCCESS), ""); mrc=HDstat(filename, &stat_buf); VRFY((mrc==0), ""); /* Hopefully this casting is safe */ if(size != (MPI_Offset)(stat_buf.st_size)) { printf("Warning: MPI_File_get_size doesn't return correct file size. To avoid using it in the library, reconfigure and rebuild the library with --disable-mpi-size.\n"); } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); /* * one more sync to ensure all processes have done reading * before ending this test. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync before leaving test"); #else printf("Skipped testing MPI_File_get_size because it's disabled\n"); #endif } finish: if (buf) HDfree(buf); return (nerrs); }