/* given 'info', incorporate any hints in 'sysinfo' that are not already set
* into 'new_info'. Caller must free 'new_info' later. */
void ADIOI_incorporate_system_hints(MPI_Info info,
MPI_Info sysinfo,
MPI_Info *new_info)
{
int i, nkeys_sysinfo, flag;
char val[MPI_MAX_INFO_VAL], key[MPI_MAX_INFO_KEY];
if (sysinfo == MPI_INFO_NULL)
nkeys_sysinfo = 0;
else
MPI_Info_get_nkeys(sysinfo, &nkeys_sysinfo);
/* short-circuit: return immediately if no hints to process */
if (info == MPI_INFO_NULL && nkeys_sysinfo == 0) {
*new_info = MPI_INFO_NULL;
return;
}
if (info == MPI_INFO_NULL)
MPI_Info_create(new_info);
else
MPI_Info_dup(info, new_info);
for (i=0; i<nkeys_sysinfo; i++) {
MPI_Info_get_nthkey(sysinfo, i, key);
/* don't care about the value, just want to know if hint set already*/
if (info != MPI_INFO_NULL) ADIOI_Info_get(info, key, 1, val, &flag);
if (flag == 1) continue; /* skip any hints already set by user */
ADIOI_Info_get(sysinfo, key, MPI_MAX_INFO_VAL-1, val, &flag);
ADIOI_Info_set(*new_info, key, val);
flag = 0;
}
return;
}
void mpi_info_dup_(MPI_Fint * info, MPI_Fint * newinfo, int *ierr)
{
MPI_Info info_c, newinfo_c;
info_c = MPI_Info_f2c(*info);
*ierr = MPI_Info_dup(info_c, &newinfo_c);
*newinfo = MPI_Info_c2f(newinfo_c);
}
void ompi_info_dup_f(MPI_Fint *info, MPI_Fint *newinfo, MPI_Fint *ierr)
{
int c_ierr;
MPI_Info c_info, c_new_info;
c_info = MPI_Info_f2c(*info);
c_ierr = MPI_Info_dup(c_info, &c_new_info);
if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
if (MPI_SUCCESS == c_ierr) {
*newinfo = MPI_Info_c2f(c_new_info);
}
}
void ADIOI_GRIDFTP_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
{
if (!(fd->info))
{
if ( users_info==MPI_INFO_NULL )
{
/* This must be part of the open call. */
MPI_Info_create(&(fd->info));
}
else
{
MPI_Info_dup(users_info,&(fd->info));
}
}
else
{
int i,nkeys,valuelen,flag;
char key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
if ( users_info!=MPI_INFO_NULL )
{
MPI_Info_get_nkeys(users_info,&nkeys);
for (i=0;i<nkeys;i++)
{
MPI_Info_get_nthkey(users_info,i,key);
MPI_Info_get_valuelen(users_info,key,&valuelen,&flag);
if (flag)
{
ADIOI_Info_get(users_info,key,valuelen,value,&flag);
if (flag) ADIOI_Info_set(fd->info,key,value);
}
}
}
}
/* let the generic ROMIO and MPI-I/O stuff happen... */
ADIOI_GEN_SetInfo(fd, users_info, error_code);
}
/*@
MPI_File_get_info - Returns the hints for a file that are actually being used by MPI
Input Parameters:
. fh - file handle (handle)
Output Parameters:
. info_used - info object (handle)
.N fortran
@*/
int MPI_File_get_info(MPI_File fh, MPI_Info *info_used)
{
int error_code;
ADIO_File adio_fh;
static char myname[] = "MPI_FILE_GET_INFO";
ROMIO_THREAD_CS_ENTER();
adio_fh = MPIO_File_resolve(fh);
/* --BEGIN ERROR HANDLING-- */
MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
/* --END ERROR HANDLING-- */
error_code = MPI_Info_dup(adio_fh->info, info_used);
/* --BEGIN ERROR HANDLING-- */
if (error_code != MPI_SUCCESS)
error_code = MPIO_Err_return_file(adio_fh, error_code);
/* --END ERROR HANDLING-- */
fn_exit:
ROMIO_THREAD_CS_EXIT();
return error_code;
}
int main(int argc, char *argv[])
{
int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size;
int nproc[2], periods[2], icoord[2];
int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err;
MPI_Datatype filetype;
MPI_File fh;
MPI_Comm cartcomm;
MPI_Info info0, info3;
double t, topen, twrite, tclose, wrate;
double *local_array;
char nstripesStr[12], stripeUnitStr[12];
int nstripes = -1;
int stripeUnit = -1;
MPI_Offset headerSize = 0;
MPI_Init(0,0);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
/* Get global array size */
m = n = 128; /* Set default size */
/* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */
if (argc > 0) {
if (argc > 1) m = atoi(argv[1]);
if (argc > 2) n = atoi(argv[2]);
if (argc > 3) nstripes = atoi(argv[3]);
if (argc > 4) stripeUnit = atoi(argv[4]);
if (argc > 5) headerSize = atoi(argv[5]);
if (argc > 6) {
if (wrank == 0)
fprintf(stderr,"Unrecognized argument %s\n", argv[6]);
MPI_Abort(MPI_COMM_WORLD,1);
}
}
if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR );
/* The default number of stripes = totalsize/1M */
if (nstripes < 0) {
nstripes = n * m * sizeof(double) / (1024*1024);
if (nstripes < 1) nstripes = 1;
}
if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n",
nstripes, stripeUnit, (int)headerSize);
/* Use topology routines to get decomposition and coordinates */
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
nproc[0] = 0; nproc[1] = 0;
ndims = 2;
MPI_Dims_create(wsize, ndims, nproc);
periods[0] = 0; periods[1] = 0;
MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm);
MPI_Comm_rank(cartcomm, &crank);
MPI_Cart_coords(cartcomm, crank, ndims, icoord);
iarrayOfSizes[0] = m;
iarrayOfSizes[1] = n;
iarrayOfSubsizes[0] = m/nproc[0];
iarrayOfSubsizes[1] = n/nproc[1];
iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0];
iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1];
/* Initialize my block of the data */
ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1];
lrows = iarrayOfSubsizes[0];
lcols = iarrayOfSubsizes[1];
local_array = (double *)malloc(lrows*lcols*sizeof(double));
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (i=0; i<lrows; i++) {
for (j=0; j<lcols; j++) {
local_array[j*lrows+i] = (grow+i) + (gcol+j)*m;
}
}
/* Fortran order simply means the data is stored by columns */
MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes,
iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE,
&filetype);
MPI_Type_commit(&filetype);
info0 = MPI_INFO_NULL;
info3 = MPI_INFO_NULL;
if (nstripes > 0 || stripeUnit > 0) {
MPI_Info_create(&info0);
if (nstripes > 0) {
snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes);
MPI_Info_set(info0, "striping_factor", nstripesStr);
MPI_Info_set(info0, "cb_nodes", nstripesStr);
}
if (stripeUnit > 0) {
snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit);
MPI_Info_set(info0, "striping_unit", stripeUnitStr);
}
MPI_Info_dup(info0, &info3);
MPI_Info_set(info3, "romio_no_indep_rw", "true");
/* Other hints to consider:
direct_io=true
The default cb_buffer_size is 16777216 , but is overridden by the
striping unit, which is smaller by default.
*/
}
/* level - 3 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE,
MPI_STATUS_IGNORE);
twrite = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "collective write");
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
/* level - 0 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (j=0; j<lcols; j++) {
MPI_Offset offset = headerSize +
((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double);
err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE,
MPI_STATUS_IGNORE);
if (err != MPI_SUCCESS) myAbort(err, "write at");
}
twrite = MPI_Wtime() - t;
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-0");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
if (info0 != MPI_INFO_NULL) {
MPI_Info_free(&info0);
MPI_Info_free(&info3);
}
free(local_array);
MPI_Finalize();
return 0;
}
int main( int argc, char *argv[] )
{
int errs = 0;
MPI_Info info1, infodup;
int nkeys, nkeysdup, i, vallen, flag, flagdup;
char key[MPI_MAX_INFO_KEY], keydup[MPI_MAX_INFO_KEY];
char value[MPI_MAX_INFO_VAL], valdup[MPI_MAX_INFO_VAL];
MTest_Init( &argc, &argv );
MPI_Info_create( &info1 );
/* Use only named keys incase the info implementation only supports
the predefined keys (e.g., IBM) */
MPI_Info_set( info1, (char*)"host", (char*)"myhost.myorg.org" );
MPI_Info_set( info1, (char*)"file", (char*)"runfile.txt" );
MPI_Info_set( info1, (char*)"soft", (char*)"2:1000:4,3:1000:7" );
MPI_Info_dup( info1, &infodup );
MPI_Info_get_nkeys( infodup, &nkeysdup );
MPI_Info_get_nkeys( info1, &nkeys );
if (nkeys != nkeysdup) {
errs++;
printf( "Dup'ed info has a different number of keys; is %d should be %d\n",
nkeysdup, nkeys );
}
vallen = MPI_MAX_INFO_VAL;
for (i=0; i<nkeys; i++) {
/* MPI requires that the keys are in the same order after the dup */
MPI_Info_get_nthkey( info1, i, key );
MPI_Info_get_nthkey( infodup, i, keydup );
if (strcmp(key, keydup)) {
errs++;
printf( "keys do not match: %s should be %s\n", keydup, key );
}
vallen = MPI_MAX_INFO_VAL;
MPI_Info_get( info1, key, vallen, value, &flag );
MPI_Info_get( infodup, keydup, vallen, valdup, &flagdup );
if (!flag || !flagdup) {
errs++;
printf( "Info get failed for key %s\n", key );
}
else if (strcmp( value, valdup )) {
errs++;
printf( "Info values for key %s not the same after dup\n", key );
}
}
/* Change info and check that infodup does NOT have the new value
(ensure that lazy dups are still duped) */
MPI_Info_set( info1, (char*)"path", (char*)"/a:/b:/c/d" );
MPI_Info_get( infodup, (char*)"path", vallen, value, &flag );
if (flag) {
errs++;
printf( "inserting path into info changed infodup\n" );
}
MPI_Info_free( &info1 );
MPI_Info_free( &infodup );
MTest_Finalize( errs );
MPI_Finalize();
return 0;
}
void mpi_info_dup(int* info, int* newinfo, int* ierr){
*ierr = MPI_Info_dup(*info, newinfo);
}